% Created 2024-11-12 Tue 13:19
% Intended LaTeX compiler: pdflatex
\documentclass[dvipsnames,colorlinks,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{color}
\usepackage{listings}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{siunitx}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\setbeamerfont{alerted text}{series=\bfseries}
\setbeamercolor{alerted text}{fg=blue}
\usecolortheme{default}
\usefonttheme{serif}
\usefonttheme{professionalfonts}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{Ideas for a new 3D LAr Pixel Simulation}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={Ideas for a new 3D LAr Pixel Simulation},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 28.2 (Org mode 9.7-pre)}, 
 pdflang={English}}
\begin{document}

\maketitle
\begin{frame}[label={sec:org06195a2}]{Overview}
\begin{itemize}
\item Present the basic problem, data domains and calculation overview.
\item Required computation resource estimation and optimization.
\item Various issues
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org95b6632}]{Major components and data}
\begin{columns}
\begin{column}{0.6\columnwidth}
Basic operations:
\begin{itemize}
\item particle \alert{tracking} \(\to\) \alert{original steps}
\item \alert{drift} \(\to\) \alert{drifted steps}
\item \alert{raster} steps \(\to\) \alert{signal voxels}
\item \alert{partition} into per-impact \alert{array slices}
\item \alert{response} to signal \(\to\) \alert{pixel impact currents}
\item \alert{sum} over impact slices \(\to\) \alert{pixel currents}
\item \alert{electronics} \(\to\) \alert{readouts}
\end{itemize}
\end{column}
\begin{column}{0.4\columnwidth}
\vspace{-1cm}

\begin{center}
\includegraphics[height=\textheight]{stages.pdf}
\end{center}
\end{column}
\end{columns}
\end{frame}
\begin{frame}[label={sec:org3817d4d}]{The core calculation}
\[M = R * S + N\]


\begin{description}
\item[{\(M\)}] the ``measurement'' represents a family of \alert{induced current waveforms} input to \alert{pixel readout electronics}.
\item[{\(S\)}] the ``signal'' represents the distribution of \alert{ionized electrons} after \alert{drifting to the response plane}.
\item[{\(R\)}] the ``response'' operator \alert{transforms} \(S\) to the domain of \(M\).  With pixels, it is ``simply'' the \alert{field response}.  With wires, ``electronics response'' is also included.  In both cases, the operation is \alert{convolution and sum}.
\item[{\(N\)}] some generator of \alert{noise} in the measurement domain (will ignore this today).
\end{description}
\end{frame}
\begin{frame}[label={sec:orge458244}]{Data forms}
\[M = R * S + N\]


Define \alert{multi-dimensional array} representation with values \alert{binned} on \alert{uniform grids}.
\begin{itemize}
\item A 3D ``voxel'' defined as (2D transverse spatial bin) \(\times\) (1D longitudinal / drift / time bin).
\item \(S\) gives ionization charge per voxel,
\item \(M\) gives induced charge per voxel,
\item \(R\) transforms from the space of \(S\)-voxels to the space of \(M\)-voxels.
\end{itemize}


Choose use common sizes for \(S\) and \(M\) 3D voxels.  For example:

\begin{itemize}
\item \(V_y \times V_z = (\qty{0.5}{mm})^2 = (\qty{5}{mm} / 10)^2\) spanning a \alert{sub-pixel} area.
\item \(V_x \equiv T = \qty{100}{ns} \equiv D/v = \frac{\qty{0.16}{mm}}{\qty{1.6}{mm/\mu{}s}}\) longitudinal / drift / time spanning an FR \alert{step}.
\end{itemize}
\(S\) and \(M\) spaces need not be so coincident.  We may need to break this symmetry in the name of optimizing.
\end{frame}
\begin{frame}[label={sec:org1b893f7}]{Response voxelization}
Field response calculations give us
\begin{itemize}
\item \alert{instantaneous current} at \alert{discrete steps} along a \alert{single path} starting at a \alert{point}.
\end{itemize}

\alert{Voxelization} requires \alert{bins} and we must:
\begin{itemize}
\item Form an average/integral of the response values over the sub-pixel area.
\item Integrate over time step and go from induced \alert{current} to induced \alert{charge}.
\begin{itemize}
\item Practically, this requires simply multiplying: \(Q=I\cdot T\).
\end{itemize}
\end{itemize}

We now assume \(R\) is voxelized in some manner.
\begin{itemize}
\item Doing this efficiently is a core problem.
\end{itemize}

\begin{center}
A drawing would be nice here\ldots{}
\end{center}
\end{frame}
\begin{frame}[label={sec:org7e50a49}]{Interlaced partitioning}
Up to now we think of \(S\) and \(R\) defined on a large, fine-grained, sub-pixel grid spanning pixels.
\begin{itemize}
\item But, \(M\) must be defined on a full-pixel grid and each pixel in fact \alert{sums} over their area.
\end{itemize}

The domain for \(S\) and \(R\) is actually  an \alert{interlacing} of a \alert{partitioning} of the sub-pixel grid.
\begin{itemize}
\item We partition (slice) signal arrays: \(S \to S_p\) for each partition \(p\) and \emph{etc} for \(R\).
\end{itemize}

Each partition selects a 2D array of a family of 1D waveforms corresponding to common relative sub-pixel locations.

\begin{center}
A drawing would be nice here\ldots{}
\end{center}
\end{frame}
\begin{frame}[label={sec:org37db4ee}]{Response operator}
\[M = R * S + N\]


\begin{itemize}
\item So, \(R*\) operator becomes a sum of 3D convolutions, each over the one impact partition.
\begin{itemize}
\item \(R*S \to \{(R_p*S_p)\}\)

\item \(Q = \sum_p (R_p*S_p) \Rightarrow\) 1D waveforms \(\times\) 2D pixel grid.
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org822ce06}]{Electronics readout}
This requires a straight-forward model for the pixel electronics.

\[E(\{Q(t)\}) \to \{(q_i, t_i), ...\}\]

Given an induced charge waveform input at one pixel, return a list of q/t readout pairs.
\end{frame}
\begin{frame}[label={sec:org8f29581},shrink=10]{Approximate array sizes}
\begin{block}{Response domain}
\begin{itemize}
\item 2D grid of \alert{response pixels} spanning all response path starting locations.
\begin{itemize}
\item \(N_{res,y} \times N_{res,z} = 9 \times 9\).
\end{itemize}
\item 1D grid of \alert{response ticks} spanning response path steps.
\begin{itemize}
\item \(N_{res,t} = 1000\).
\end{itemize}
\end{itemize}
\end{block}
\begin{block}{Signal and measurement domains}
\begin{itemize}
\item 2D grid of \alert{measurement pixels} in one detector ``unit''.
\begin{itemize}
\item \(N_{pix,y} \times N_{pix,z} = 100 \times 100\).
\end{itemize}
\item 1D array of \alert{measurement ticks} spanning the max drift time.
\begin{itemize}
\item \(N_{tick} = 10000\).
\end{itemize}
\end{itemize}
\end{block}
\begin{block}{Impact partitioning}
\begin{itemize}
\item 2D grid of \alert{sub-pixels per pixel} matching response path \alert{binning} (not points).
\begin{itemize}
\item \(N_{imp,y} \times N_{imp,z} = 10 \times 10\).
\end{itemize}
\item A \alert{universal clock} of drift period \(T=\qty{100}{ns} \equiv D/v = \frac{\qty{0.16}{mm}}{\qty{1.6}{mm/\mu{}s}}\)
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:orga7c0b01}]{Array sizes}
\begin{itemize}
\item \(N_R = (N_{imp,y} \times N_{imp,z}) (N_{res,t} \times N_{res,y} \times N_{res,z}) = (10 \times 10)(1000 \times 10 \times 10) = 10^7\)
\item \(N_S = (N_{imp,y} \times N_{imp,z})(N_{tick} \times N_{pix,y} \times N_{pix,z}) = (10 \times 10)(10000 \times 100 \times 100) = 10^{10}\)
\item \(N_M = (N_{tick} \times N_{pix,y} \times N_{pix,z}) = (10000 \times 100 \times 100) = 10^{8}\)
\end{itemize}

Each 3D convolution:

\begin{itemize}
\item Size: \((1000 \times 10 \times 10) * (10000 \times 100 \times 100) \to (10000 \times 100 \times 100)\)

\item Must sum each 400 MB result over \((10\times10)\) convolutions.
\begin{itemize}
\item If \((10 \times 10)\) convolutions are serial, can accumulate in fixed 400 MB.
\item If convolutions are in parallel we must store 40 GB before the sum.
\begin{itemize}
\item Or perhaps can make store the 400 MB arrays sparsely.
\end{itemize}
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org0274a46},fragile]{Some timing estimates}
 Using one signal partition \(S_p = (10000 \times 100 \times 100) = (2\cdot 5)^8\) (small primes)


\begin{center}
\begin{tabular}{lrrr}
operation & DFT & mult & add\\
\hline
CPU-ms & 548 & 31.1 & 32.2\\
\hline
GPU-ms & 8.36 & 0.85 & 0.84\\
\hline
\end{tabular}
\end{center}


\begin{itemize}
\item Time in millisecond, excludes any CPU/GPU transfer
\begin{itemize}
\item Round trip CPU-GPU-CPU requires 53 ms
\end{itemize}
\item PyTorch and \texttt{ipython}'s \texttt{\%timeit} and \texttt{torch.set\_num\_threads(1)}
\begin{itemize}
\item Care taken to \texttt{detatch()} from autograd.
\end{itemize}
\item Both CPU and GPU results utilize 1 CPU core.  GPU sees 100\% util.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org7c4f4b4}]{Some memory estimates}
Voxelize steps into full \(S\) then partition \(S \to \{S_p\}\) 

\begin{itemize}
\item Easiest but requires 40 GB array
\end{itemize}

Voxelize steps into per-step arrays, partition and accumulate into \(S_p\)

\begin{itemize}
\item Assume raster each step to in \((\qty{5}{mm})^3=(10\times10\times10) + (1\times1\times1)\) location.
\item With \(\mathcal{O}(10^6)\) steps, 1 GB memory needed.  Could easily remain in GPU RAM.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org8ffd40c}]{Voxelization time}
Voxelization is a many (steps) to many (voxels) accumulation problem.
\begin{itemize}
\item No simple SIMD type algorithm exists.
\item HEP-CCE/PPS discovered this when accelerating WCT
\begin{itemize}
\item A divide-and-conquer method was developed.
\end{itemize}
\item Some ``real'' work needed to estimate how much step costs here.
\begin{itemize}
\item External choice can impact this, eg \alert{chunking}.
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org141febb}]{Chunking}
Break up the problem domain, solve problem on smaller domains, connect the results.


For example,
\begin{itemize}
\item \((10,10,10) = 1000\) chunk \(c\) problem signal \(S^{c} = (10,10)\times(1000,10,10))\)
\item Size: \(10^{10}\to10^{7}\) or 40 GB \(\to\) 40 MB.
\end{itemize}

Plus extra memory and compute to assure no cyclic artifacts
\begin{itemize}
\item Enlarge convolution to be size of \(S^c\) + size of \(R\).
\item Apply the overlap-add or overlap-save alg to add this enlargement to proper neighbors.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org81bbb10}]{More voxelization issues}
\begin{block}{The ``Gaussian hotdog''}
\begin{itemize}
\item A 3D Gaussian \((\sigma_L, \sigma_T, \sigma_L)\) with \(\sigma_{x}(\Delta t)\) is convolved with a 3D step line.
\item Consider binning \(\Delta t\) coarsely and pre-tabulating Fourier representation of each Gaussian.
\item The ``hotdog'' is then formed by multiplication with Fourier representation of a step line.
\end{itemize}
\end{block}
\begin{block}{Reduce steps to the usual ``depos''}
\begin{itemize}
\item The min step size in G4 is \(\mathcal{O}(\qty{1}{mm})\) and nominal voxel is \((\qty{0.5}{mm})^3\)
\begin{itemize}
\item Maybe step line shape is not important?
\end{itemize}
\item Or, perhaps bin possible step directions to make lookup of Fourier representation.
\begin{itemize}
\item This meshes well with the hotdog optimization above.
\end{itemize}
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:org1be09f5}]{Time bases}
We chose a universal \(T = \qty{100}{ns}\) which already drives problem size.  But,
\begin{itemize}
\item Characteristic diffusion time is \(\mathcal{O}(\qty{1}{\mu{}s})\).
\item FRs need \(T_{FR} = \qty{50}{ns}\) at least to avoid being undersampled.
\item Real electronics clock is \qty{100}{ns}.
\begin{itemize}
\item But, is there an effective low-pass filter cutoff at lower frequency?
\end{itemize}
\end{itemize}

Consequences / options:
\begin{itemize}
\item Double the problem size with a universal \(T = \qty{50}{ns}\)
\item Keep \(T_{FR} = \qty{50}{ns}\) but do proper resampling to \(T\) as part of FR convolution.
\begin{itemize}
\item Ie, we think of diffusion as a low-pass filter.
\end{itemize}
\item Choose \(T > \qty{100}{ns}\) if downstream electronics has an effective low-pass filter.
\end{itemize}
\end{frame}
\end{document}
