% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]
\usepackage{tabularx}

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{L0+CC}
\renewcommand{\bvtitle}{\LARGE L0 Trigger Primitives: \\Functionality and Performance on Commodity CPU\\(an initial study)}
\renewcommand{\bvevent}{{\small 7 Feb 2018}}
\renewcommand{\bvdate}{7 Feb 2018}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}

\def\ar39{$^{39}Ar$\xspace}

% supports conductor/channel map tables
% \newcolumntype{C}{>{\centering\arraybackslash}p{0.75mm}}
\newcolumntype{C}{p{0.75mm}}


\begin{document}
\input{beamer/title.tex}
%\input{beamer/toc.tex}

\section{Overview}
\begin{frame}
  \frametitle{Outline}
  \begin{center}
      \tableofcontents          
  \end{center}

\end{frame}

\begin{frame}
  \frametitle{Overview}

  \vspace{-3mm}

  \begin{center}
    \includegraphics[width=\textwidth]{blocks.pdf}    

    \scriptsize
    DUNE FD DAQ Option: FELIX + Commodity Computing.
  \end{center}


  Focused here on the \textcolor{red}{\textbf{L0 pipeline}}:
  \begin{enumerate}
  \item Produced Wire-Cell Toolkit simulation: noise + \ar39 signal.
  \item Used real WIB channel addressing.
  \item Produce tick-vectors of 480 collection channels (\SI{5}{\milli\second}).
  \item Calculate baseline and noise levels per channel.
  \item \textbf{L0 primitives}: per channel, regions of time-over-threshold.
  \item Measure CPU usage.
  \end{enumerate}

  \footnotesize
  Note: for now focus only on \textbf{finding} \ar39 signal. 
  They serve as a proxy for any near-threshold activity.  
  Will enlarge signal sample later.

\end{frame}

\section{Simulation}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Wire-Cell Toolkit Simulation Feature Overview}
  \footnotesize
  \begin{itemize}
  \item Noise: Rayleigh sampled amplitude spectrum, uniformly sampled
    phases, apply inverse FFT to get noise waveforms, use MicroBooNE's
    post-filtered spectra.
  \item Deposition:  \ar39 energy spectrum and decay rate (\SI{1}{\becquerel/\kilo\gram}, adjustable)
  \item Drift: recombination, longitudinal and transverse diffusion.
  \item Induction: integrate over and interpolate between long-range ($\pm$10 wires) and
    fine-grain ($\frac{1}{10}^{th}$ pitch) induction drift paths.
  \item Shaping: detailed Cold-FE electronic response function and a simple ADC model.
  \item Output: write Numpy arrays (other formats available).
  \end{itemize}

  $\to$ Stand-alone simulation usable now.  Integration with LArSoft waiting on LS data model developments.
\end{frame}

\begin{frame}[fragile]
  \frametitle{Generate Wires}
  \vspace{-1cm}
  \flushright
\tiny
\hspace{5cm}\verb|$ wirecell-util make-wires -d apa dune-wires.json.bz2|

\verb|$ wirecell-util plot-wires dune-wires.json.bz2 dune-wires.pdf|

\begin{center}
  \includegraphics[width=4cm,page=1,clip,trim=4cm 0.3cm 4cm 1.3cm]{dune-wires.pdf}%
  \includegraphics[width=4cm,page=2,clip,trim=4cm 0.3cm 4cm 1.3cm]{dune-wires.pdf}%
  \includegraphics[width=4cm,page=3,clip,trim=4cm 0.3cm 4cm 1.3cm]{dune-wires.pdf}%
\end{center}

\vspace{-5mm}

\footnotesize
\begin{itemize}
\item Real DUNE/protoDUNE wire length, angles, pitches and multiplicities.
\item An APA-level channel identifier number built from WIB addresses:
\end{itemize}
\verb|[WIB connector(1-4)][WIB slot(1-5)][chip(1-8)][channel(01-16)]|

\end{frame}

\begin{frame}
  \frametitle{FEMB-level Channel/Conductor map}

  % copy-paste from channel-map.tex
  \footnotesize
  \begin{center}
\begin{tabular}{r|rrrrrrrr}
\hline
ASIC:&1&2&3&4&5&6&7&8\\
\hline
ch00 & \textcolor{red}{u19} & \textcolor{red}{u09} & \textcolor{black}{w14} & \textcolor{black}{w02} & \textcolor{red}{u29} & \textcolor{red}{u39} & \textcolor{black}{w26} & \textcolor{black}{w38}\\
ch01 & \textcolor{red}{u17} & \textcolor{red}{u07} & \textcolor{black}{w16} & \textcolor{black}{w04} & \textcolor{red}{u27} & \textcolor{red}{u37} & \textcolor{black}{w28} & \textcolor{black}{w40}\\
ch02 & \textcolor{red}{u15} & \textcolor{red}{u05} & \textcolor{black}{w18} & \textcolor{black}{w06} & \textcolor{red}{u25} & \textcolor{red}{u35} & \textcolor{black}{w30} & \textcolor{black}{w42}\\
ch03 & \textcolor{red}{u13} & \textcolor{red}{u03} & \textcolor{black}{w20} & \textcolor{black}{w08} & \textcolor{red}{u23} & \textcolor{red}{u33} & \textcolor{black}{w32} & \textcolor{black}{w44}\\
ch04 & \textcolor{red}{u11} & \textcolor{red}{u01} & \textcolor{black}{w22} & \textcolor{black}{w10} & \textcolor{red}{u21} & \textcolor{red}{u31} & \textcolor{black}{w34} & \textcolor{black}{w46}\\
ch05 & \textcolor{blue}{v19} & \textcolor{blue}{v09} & \textcolor{black}{w24} & \textcolor{black}{w12} & \textcolor{blue}{v29} & \textcolor{blue}{v39} & \textcolor{black}{w36} & \textcolor{black}{w48}\\
ch06 & \textcolor{blue}{v17} & \textcolor{blue}{v07} & \textcolor{blue}{v12} & \textcolor{blue}{v02} & \textcolor{blue}{v27} & \textcolor{blue}{v37} & \textcolor{blue}{v22} & \textcolor{blue}{v32}\\
ch07 & \textcolor{blue}{v15} & \textcolor{blue}{v05} & \textcolor{blue}{v14} & \textcolor{blue}{v04} & \textcolor{blue}{v25} & \textcolor{blue}{v35} & \textcolor{blue}{v24} & \textcolor{blue}{v34}\\
ch08 & \textcolor{blue}{v13} & \textcolor{blue}{v03} & \textcolor{blue}{v16} & \textcolor{blue}{v06} & \textcolor{blue}{v23} & \textcolor{blue}{v33} & \textcolor{blue}{v26} & \textcolor{blue}{v36}\\
ch09 & \textcolor{blue}{v11} & \textcolor{blue}{v01} & \textcolor{blue}{v18} & \textcolor{blue}{v08} & \textcolor{blue}{v21} & \textcolor{blue}{v31} & \textcolor{blue}{v28} & \textcolor{blue}{v38}\\
ch10 & \textcolor{black}{w23} & \textcolor{black}{w11} & \textcolor{blue}{v20} & \textcolor{blue}{v10} & \textcolor{black}{w35} & \textcolor{black}{w47} & \textcolor{blue}{v30} & \textcolor{blue}{v40}\\
ch11 & \textcolor{black}{w21} & \textcolor{black}{w09} & \textcolor{red}{u12} & \textcolor{red}{u02} & \textcolor{black}{w33} & \textcolor{black}{w45} & \textcolor{red}{u22} & \textcolor{red}{u32}\\
ch12 & \textcolor{black}{w19} & \textcolor{black}{w07} & \textcolor{red}{u14} & \textcolor{red}{u04} & \textcolor{black}{w31} & \textcolor{black}{w43} & \textcolor{red}{u24} & \textcolor{red}{u34}\\
ch13 & \textcolor{black}{w17} & \textcolor{black}{w05} & \textcolor{red}{u16} & \textcolor{red}{u06} & \textcolor{black}{w29} & \textcolor{black}{w41} & \textcolor{red}{u26} & \textcolor{red}{u36}\\
ch14 & \textcolor{black}{w15} & \textcolor{black}{w03} & \textcolor{red}{u18} & \textcolor{red}{u08} & \textcolor{black}{w27} & \textcolor{black}{w39} & \textcolor{red}{u28} & \textcolor{red}{u38}\\
ch15 & \textcolor{black}{w13} & \textcolor{black}{w01} & \textcolor{red}{u20} & \textcolor{red}{u10} & \textcolor{black}{w25} & \textcolor{black}{w37} & \textcolor{red}{u30} & \textcolor{red}{u40}\\
\hline
\end{tabular}

  See DUNE \href{http://docs.dunescience.org/cgi-bin/ShowDocument?docid=4064}{DocDB 4064} for details.
  \end{center}
\end{frame}

\begin{frame}[fragile]
  \frametitle{FEMB-level Conductor/Channel Map (inverse)}
  % copy-paste from channel-map.tex
  \tiny
\begin{center}
\begin{tabular}{|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|}
\multicolumn{20}{c}{U layer, first half: conductor / chip / chan} \\
\hline
 1& 2& 3& 4& 5& 6& 7& 8& 9&10&11&12&13&14&15&16&17&18&19&20\\
\hline
 2& 4& 2& 4& 2& 4& 2& 4& 2& 4& 1& 3& 1& 3& 1& 3& 1& 3& 1& 3\\
 4&11& 3&12& 2&13& 1&14& 0&15& 4&11& 3&12& 2&13& 1&14& 0&15\\
\hline
\multicolumn{20}{c}{U layer, second half: conductor / chip / chan} \\
\hline
21&22&23&24&25&26&27&28&29&30&31&32&33&34&35&36&37&38&39&40\\
\hline
 5& 7& 5& 7& 5& 7& 5& 7& 5& 7& 6& 8& 6& 8& 6& 8& 6& 8& 6& 8\\
 4&11& 3&12& 2&13& 1&14& 0&15& 4&11& 3&12& 2&13& 1&14& 0&15\\
\hline
\end{tabular}
\end{center}

\begin{center}
\begin{tabular}{|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|}
\multicolumn{20}{c}{V layer, first half: conductor / chip / chan} \\
\hline
 1& 2& 3& 4& 5& 6& 7& 8& 9&10&11&12&13&14&15&16&17&18&19&20\\
\hline
 2& 4& 2& 4& 2& 4& 2& 4& 2& 4& 1& 3& 1& 3& 1& 3& 1& 3& 1& 3\\
 9& 6& 8& 7& 7& 8& 6& 9& 5&10& 9& 6& 8& 7& 7& 8& 6& 9& 5&10\\
\hline
\multicolumn{20}{c}{V layer, second half: conductor / chip / chan} \\
\hline
21&22&23&24&25&26&27&28&29&30&31&32&33&34&35&36&37&38&39&40\\
\hline
 5& 7& 5& 7& 5& 7& 5& 7& 5& 7& 6& 8& 6& 8& 6& 8& 6& 8& 6& 8\\
 9& 6& 8& 7& 7& 8& 6& 9& 5&10& 9& 6& 8& 7& 7& 8& 6& 9& 5&10\\
\hline
\end{tabular}
\end{center}

\begin{center}
\begin{tabular}{|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|C|}
\multicolumn{24}{c}{W layer, first half: conductor / chip / chan} \\
\hline
 1& 2& 3& 4& 5& 6& 7& 8& 9&10&11&12&13&14&15&16&17&18&19&20&21&22&23&24\\
\hline
 2& 4& 2& 4& 2& 4& 2& 4& 2& 4& 2& 4& 1& 3& 1& 3& 1& 3& 1& 3& 1& 3& 1& 3\\
15& 0&14& 1&13& 2&12& 3&11& 4&10& 5&15& 0&14& 1&13& 2&12& 3&11& 4&10& 5\\
\hline
\multicolumn{24}{c}{W layer, second half: conductor / chip / chan} \\
\hline
25&26&27&28&29&30&31&32&33&34&35&36&37&38&39&40&41&42&43&44&45&46&47&48\\
\hline
 5& 7& 5& 7& 5& 7& 5& 7& 5& 7& 5& 7& 6& 8& 6& 8& 6& 8& 6& 8& 6& 8& 6& 8\\
15& 0&14& 1&13& 2&12& 3&11& 4&10& 5&15& 0&14& 1&13& 2&12& 3&11& 4&10& 5\\
\hline
\end{tabular}
\end{center}
\end{frame}


\begin{frame}
  \frametitle{WCT Simulation Raw Frame Data}
  \vspace{-5mm}
  \flushright
\tiny
\hspace{5cm}
  \verb|$ wire-cell -c wctcfg/noise-ar39-sim.jsonnet|

  \verb|$ blips plot-frames -o plots.pdf dune-wctsim-fullrate-adc-noise.npz|

  \begin{center}
    \includegraphics[width=0.4\textwidth,page=1]{plot-primitives.pdf}%
    \includegraphics[width=0.4\textwidth,page=2]{plot-primitives.pdf}
  \end{center}

  \footnotesize

  \begin{itemize}
  \item Simulation provides 2D \textbf{frame array} ADC sample in
    channel vs tick.
    \begin{itemize}\scriptsize
    \item \SI{5}{\milli\second} frames used here, but size is configurable.
    \end{itemize}
  \item 1D \textbf{channel array} associates frame array rows
    with \textbf{channel identifier numbers}.
  \item Current limitation (bug): no ``back'' face collection plane.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Group Channels By Plane}

  \begin{center}
    \includegraphics[width=0.6\textwidth,page=3]{plot-primitives.pdf}
  \end{center}

  \scriptsize
  \begin{itemize}
  \item Use channel/conductor map to sort out data into U/V/W planes.
  \item Focus next only on \textbf{just the 480 collection channels}.
  \end{itemize}
\end{frame}

\begin{frame}
  \begin{center}
    \includegraphics[height=\textheight,page=4]{plot-primitives.pdf}
  \end{center}
\end{frame}
\begin{frame}
  \begin{center}
    \includegraphics[height=\textheight,page=6]{plot-primitives.pdf}
  \end{center}
\end{frame}
\begin{frame}
  \begin{center}
    \includegraphics[height=\textheight,page=5]{plot-primitives.pdf}
  \end{center}
\end{frame}

\section{Baseline and Thresholds}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Baseline and Thresholds}

  \begin{center}
    \includegraphics[width=0.6\textwidth,page=7,clip,trim=0 0 0 7mm]{plot-primitives.pdf}    
  \end{center}

  \footnotesize

  \vspace{-5mm}
  \begin{itemize}
  \item Fill  \textbf{ADC code} histogram and its \textbf{cumulative} histograms.
    \begin{itemize}\tiny
    \item[$\to$] Histogram bins: 4096 ADC $\times$ 480 channels,
    \end{itemize}
  \item Find $\mbox{ADC}_{low}$ as 0.1\%-percentile and $\mbox{ADC}_{bl}$ as median.
  \item Set threshold: $\mbox{ADC}_{thresh} = \mbox{ADC}_{bl} + N_\sigma \left(\mbox{ADC}_{bl} - \mbox{ADC}_{low}\right), N_\sigma=2$
    \begin{itemize}\tiny
    \item[$\to$] These are tuning parameters.
    \end{itemize}
  \item Per-channel L0 trigger primitives are regions of consecutive
    ticks with above-threshold samples.
  \end{itemize}
\end{frame}

\section{L0 Primitives}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \begin{center}
    \includegraphics<1>[height=\textheight,page=4]{plot-primitives.pdf}        
    \includegraphics<2>[height=\textheight,page=8]{plot-primitives.pdf}        
  \end{center}
\end{frame}
\begin{frame}
  \begin{center}
    \includegraphics<1>[height=\textheight,page=6]{plot-primitives.pdf}        
    \includegraphics<2>[height=\textheight,page=10]{plot-primitives.pdf}        
  \end{center}
\end{frame}
\begin{frame}
  \begin{center}
    \includegraphics<1>[height=\textheight,page=5]{plot-primitives.pdf}        
    \includegraphics<2>[height=\textheight,page=9]{plot-primitives.pdf}        
  \end{center}
\end{frame}


\section{CPU Time}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}[fragile]
  \frametitle{Performance on CPU}
  \scriptsize

  \vspace{-4mm}

  \begin{columns}
    \begin{column}{0.1\textwidth}
      
    \end{column}
    \begin{column}{0.8\textwidth}
      

\begin{verbatim}
$ ./build/test_primitives channel-map.npz \
       dune-wctsim-fullrate-adc-noise.npz  \
          primitives.npz > primitives.log
...
copy 480 collection channels x 10000 ticks in 0.0331903
find baseline/thresholds in 0.0203175
median: 5000, threshold: 10 (0.001)
find 10888 primitives in 0.0186963
dump intermdiates to numpy in 0.227635
\end{verbatim}

    \end{column}
  \end{columns}

  \footnotesize

\begin{itemize}
\item Compiled with \texttt{g++ -O3} (\texttt{clang++} gives similar)
\item Xeon E5-2630 v4 @ 2.20GHz (20 threads, 640k/2.5M/25M cache)
  \begin{itemize}\tiny
  \item[$\to$] this test is single threaded
  \end{itemize}
\item Rounding up, need 50 core-\si{\milli\second} for \SI{5}{\milli\second} $\times$ 480 channels.
\item \textbf{If} linear scaling holds, this Xeon CPU keeps up with one APA!
\end{itemize}

Still a big ``if'', \textbf{big fat caveats:}
\begin{itemize}
\item not streaming data, single thread, scaling not yet proven.
\item 20MB collection frame, 32bit int samples, cache may be fooling me.
\end{itemize}

\end{frame}

\section{Next steps}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Next steps}
  \begin{itemize}
  \item Increase simulation data set to cover more time (few seconds) and event variety
    \begin{itemize}\scriptsize
    \item[$\to$] toss in a few cosmic-$\mu$'s
    \end{itemize}
  \item Implement feeding ``raw data'' to multi-threaded ring buffer.
  \item Parallelize the current L0 pipeline to test MT scalability.
  \item Develop MT'ed mechanisms to access ``live'' ring buffer.
    \begin{itemize}\scriptsize
    \item[$\to$] feed MT L0 pipelines.
    \item[$\to$] demonstrate execution of trigger commands.
    \item[$\to$] readout final ``events''.
    \end{itemize}
  \item New ideas to try (details next) 
    \begin{itemize}\scriptsize
    \item[$\to$] Develop streamed/chunked baseline/threshold calculation.
    \item[$\to$] Reduce data input to L0 pipeline.
    \end{itemize}
  \end{itemize}

  
\end{frame}

\begin{frame}[fragile]
  \frametitle{Streamed/Chunked Baselines/Thresholds}

  \begin{columns}
    \begin{column}{0.05\textwidth}
    \end{column}
    \begin{column}{0.55\textwidth}
      {\footnotesize Calculate baselines/thresholds with data from \textbf{recent past},
         avoid \textbf{slow recalculations} and \textbf{allow for drifts}.}
    \end{column}
    \begin{column}{0.4\textwidth}
      \includegraphics[width=0.6\textwidth,page=7,clip,trim=0 0 0 7mm]{plot-primitives.pdf}
    \end{column}
  \end{columns}

  \begin{itemize}\footnotesize
  \item Calculate  $N$ \textbf{interval cumulative ADC
      distribution} histograms $\{H(i)\}$,\\ each spanning a subsequent time interval $\Delta t$.

  \item Store individual $\{H_i\}$ in ordered \texttt{deque}.
  \item Sum: $C_N = \sum_i^N H_i$, \textbf{total cumulative ADC distribution} at $t_N$
  \item For each new $t_{i+N+1} = t_{i+N} + \Delta t$ do:
    \begin{itemize}\scriptsize
    \item Form fresh interval cumulative: $H_{N+i+1}$
    \item Update total cumulative: $C_{i+N+1} = C_{i+N} + H_{i+N+1} - H_{i}$
    \item \texttt{deque.push(}$H_{i+N+1}$\texttt{)} and \texttt{deque.pop(}$H_{i}$\texttt{)}.
    \item Recalculate channel \textbf{baselines/thresholds} from $C_N$.
    \end{itemize}

  \item[$\to$] $n$ and $\Delta t$ are tuning parameters ($n \approx 10$ and $\Delta t \approx \SI{1}{\milli\second}$?)
  \item[$\to$] Also, place per-channel guards on max/min ADC.
    \begin{itemize}\scriptsize
    \item Elongated tracks or big showers leading to signal in $>$50\%
      samples will bias median away from true baseline. 
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Reduce Input Data Rate to L0 Pipeline}
  \begin{center}
    \tiny \ar39 ``blip'' in raw ADC

    \includegraphics[height=2cm,page=5,clip,trim=2.2cm 8.5cm 12cm 3.2cm]{plot-primitives.pdf}            

  \end{center}

  Idea from Giovanna:
  \begin{itemize}
  \item[$\Rightarrow$] Look at \textbf{resampling} or \textbf{rebinning} by $\times2 - 4$ ticks.
  \item 2 MHz is actually \textbf{oversampling} our waveforms.
    \begin{itemize}\footnotesize
    \item 2-3\si{\micro\second} \textbf{anti-aliasing filter} in electronics.
    \item \SI{5}{\milli\meter}/\SI{1.6}{\milli\meter/\micro\second} $\approx$ \SI{3}{\micro\second} characteristic time for induction rise.
    \item $\approx$\SI{1}{\micro\second} spread due to electron diffusion in LAr.
    \end{itemize}
  \item APA makes 76.8 Gbps, $\frac{3}{8}$ from collection channels.
    \begin{itemize}
    \item[$\to$] resampling/rebinning: 14.4 $\to$ 7.2 $\to$ 3.6 Gbps
    \end{itemize}
  \end{itemize}


\end{frame}

\end{document}

