% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{DUNE DAQ}
\renewcommand{\bvtitle}{\LARGE Some ideas for DUNE DAQ\\Architecture, Triggering, Reduction\\(focus on single-phase)}
\renewcommand{\bvevent}{{\small DUNE FD DAQ Workshop\\Columbia, 30-31 OCT 2017}}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}

% outline
% Architecture
% Triggering
% data reduction



\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Some Numbers}

\begin{frame}
  \frametitle{Some Relevant Numbers}

  \begin{description}
  \item[$^{39}$Ar] 1 Bq/kg $\times$ 10 kt / 200 TPC\footnote{Aka ``drift cells'' aka ``LAr volumes''} = 50 kHz/TPC = 50/ms/TPC
  \item[SNB] $\mathcal{O}(0)$ event/ms/TPC, $\sim$1000 events/burst/10kt
  \item[Beam] $\sim$1Hz rate, trigger latency = MINOS: 0.6-5s, NOvA: 20s
  \item[APA] 10 GByte/sec (full-stream @ 2 Byte samples)
  \item[10 kt] 1.5 TByte/sec = 12 Tbps
  \item[PCIe] 15 GByte/sec (v3x16 today, v4 will be $\times$ faster)
  \item[RAM] 25 GByte/sec (DDR4-3200)
  \item[NIC] 10 Gbps ``common'', 40 or 100 Gbps available
  \item[FFT] 5ms / GPU card, 500ms / CPU-FPU core \\(2D, round trip, per plane, single-prec. FP)
  \item[NF+SP] Wire-Cell noise filter + sig.proc. 2 min/event ($\mu$B).
  \item[reco] protoDUNE LArSoft reco full chain 30-45 min/event.
  \end{description}
  
\end{frame}

\section{Conceptual Design}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Concept In a Nutshell}

  \begin{itemize}
  \item \textbf{Flow} all data through \textbf{RAM in commodity computers}.
  \item \textbf{Process} data quickly while still in RAM.
  \item \textbf{Trigger} locally (per-TPC), process triggers globally,
    return trigger commands.
  \item \textbf{Interpret} trigger commands locally (per-APA) to
    dispatch requested subset of data.
  \item \textbf{Scale} commodity computers \textbf{up and out} as needed to accommodate above.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Minimal Concept Requirements: ADC to RAM}
  \begin{itemize}
  \item Get data across cold $\to$ warm boundary
    \begin{itemize}\footnotesize
    \item Introduce \textbf{no excess noise} in the process!
    \end{itemize}
  \item Aggregate data through whatever needed stages (ADC, FEMB, WIB,
    FELIX) finally to RAM via \textbf{just a few, fast links}.
  \item Reliable, constant data transfer of 82~Gbps/APA.
  \item Perform \textbf{minimal processing} on the way:
    \begin{itemize}\footnotesize
    \item \textbf{NO compression/decompression} needed/wanted.
    \item \textbf{Reformat} data from hardware packing to ``software-friendly''
      format for a \textbf{RAM ring buffer} (channel $\times$ tick block and 12bit $\to$ 2Byte)
    \end{itemize}
  \item A single interface board per host computer preferred for
    simpler firmware and software interface to RAM buffer.
  \end{itemize}
  WIB+FELIX provides a working example.
\end{frame}

\begin{frame}
  \frametitle{High Level, Minimal Concept}

  \vspace{-1cm}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{overview.pdf}
  \end{center}
  \vspace{-1cm}

  \begin{itemize}\footnotesize
  \item Full-stream, \textbf{constant data flow} from ADCs host system RAM.
  \item Buffer $N_{ch}$ TPC waveforms in $N_{tick}$-deep \textbf{shared-memory ring buffer}.
  \item Operate directly on data with ``\textbf{reader}'' processes running on \texttt{localhost}.
  \end{itemize}

  Call this a ``Tier 1 Host''.

\end{frame}

\begin{frame}
  \frametitle{Generalize and Scale}
  Likely one host is not enough to process all data from one APA.

  \vfill

  Build a  \textbf{tiered hierarchy of hosts}.  \\Each generic host node has:
  \begin{itemize}\scriptsize
  \item[1] Data stream input source (FELIX, NIC).
  \item[2] RAM ring buffer.
  \item[3] Local processing.
  \item[4] Result stream output sink (NIC, disk).
  \end{itemize}

  \vfill

  Data flows generally ``outward'' toward higher tiers, but some
  upstream messaging needed (in particular global triggering).

  \vfill

  Tier may specialize: RAM-heavy, CPU-heavy, have GPUs,
  perform meta processing (eg, trigger, DQM), etc.
  
\end{frame}


\begin{frame}
  \frametitle{Example: Scalable Ring Buffer}

  \vspace{-0.9cm}
  \begin{center}
    \includegraphics[width=0.7\textwidth]{deeper.pdf}    
  \end{center}
  \vspace{-1cm}

  \footnotesize

  \begin{itemize}
  \item Can make 
    \textbf{deeper buffer} by flowing data through $2^{nd}$ (or $3^{rd}$, etc) tier.
  \item Tier 1 ``reader'' processes would flow data between tiers at fixed rate.
  \item Tier 2 ``writer'' process replaces the FELIX spot in Tier 1 
  \item A switch can allow dynamic routing, but the \textbf{12 Tbps/10
      kt} total throughput may require some segmentation.
  \end{itemize}
\end{frame}



\begin{frame}
  \frametitle{Note on RAM Buffer Size and Cost}
  How big must it be?
  \begin{itemize}\footnotesize
  \item Beam trigger packet latency: MINOS: 0.6-5s, NOvA: 20s.
  \item[?] Latency of software trigger processors needs study.
  \item[?] Latency for SNB trigger formation? (10s?)
    \begin{itemize}\scriptsize
    \item[?] once raised, can we \textbf{stream} SNB data to sink?
    \item[?] or do we need dedicated \textbf{long-term buffer} for whole SNB period?
    \end{itemize}    
  \end{itemize}

  Costs?
  \begin{itemize}\footnotesize
  \item Let's not speculate on Moore's law?\footnote{Okay, I know you are all dividing by factors of 2 in your head!}
  \item \textbf{Today:} RAM costs $\sim$ \$10/GB (but it's currently \textbf{rising!})
    \begin{itemize}\scriptsize
    \item[$\to$] \$100/APA $\cdot$ sec, {\bf \$6k/APA $\cdot$ min}
    \end{itemize}
  \item[$\to$] 1 minute RAM buffer is not so crazy. 
    \begin{itemize}\scriptsize
    \item[\checkmark] per-APA, costs less than RCE and comparable to FELIX.
    \item[\checkmark] can avoid bespoke concentrated high-RAM systems by scaling-out, as above.
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Readers}
  \begin{description}
  \item[Q:] What do these ``readers'' do?
  \item[A:] Whatever you want! (if you have enough CPU)
  \end{description}

  \vfill

  Some likely readers:
  \begin{itemize}
  \item Form and emit \textbf{local trigger primitives}
  \item Accept and interpret \textbf{global trigger commands}
  \item Perform data reduction or selection processing.
  \item Transfer data to $2^{nd}$ tier hosts for deeper buffer or more distributed CPU.
  \item Data quality monitoring, ``express lane'' processes.
  \item Save data to file.
  \end{itemize}
\end{frame}


\section{Collection Plane Triggering}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Induction vs Collection Waveforms}
  \begin{itemize}
  \item Raw \textbf{induction} waveforms
    \begin{itemize}\footnotesize
    \item Biploar, no direct charge measure,  often in the noise.
    \item Any threshold will be inefficient or noise-dominated (or both). 
    \item Need relatively expensive signal processing to use properly.
    \item Each channel is sensitive to activity on both sides of APA.
    \end{itemize}
  \item Raw \textbf{collection} waveforms
    \begin{itemize}\footnotesize
    \item Good measure of ionization energy as 2D profile (drift $\times$ transverse)
    \item 1 view, so can only reduce data by time slices, no spatial reduction.
    \item Immediately useful signals, no expensive processing required.
    \item Activity from either side of APA can be distinguished.
    \end{itemize}
  \end{itemize}

  $\Rightarrow$ Try to use \textbf{collection planes} to form basic \textbf{trigger
  primitives} independently for each \textbf{TPC} on either side of an APA.
\end{frame}

\begin{frame}
  \frametitle{Collection Waveform Segment Categories}

  Consider some \textbf{mutually exclusive categories} for collection waveform segments:

  \begin{description}
  \item[noise] contiguous waveform chunks \textbf{consistent with
      noise}, eg contain no samples above some $n\sigma$ RMS level.
  \item[blips] a set of waveform samples that are ``\textbf{connected}'',
    ``\textbf{compact}'' and ``\textbf{isolated}'' (by some
    metrics)
    \begin{itemize}\footnotesize
    \item Ie, ``small, compact islands'' in channel vs tick space
      surrounded by ``enough'' noise samples.  
    \item Intention is to efficiently select a rich sample $^{39}$Ar
      decays, SNB interactions and similar low-energy activity.
    \end{itemize}
  \item[signals] ``everything else''
    \begin{itemize}\footnotesize
    \item Waveform chunks \textbf{not} consistent with \textbf{noise} nor \textbf{blips}.
    \item With more thought, additional categories may present themselves.
    \end{itemize}
  \end{description}

\end{frame}

\begin{frame}
  \frametitle{Local Collection Plane Trigger Primitives}

  Raise a local collection plane trigger primitive whenever any
  \textbf{non-noise waveform} is present.

  \vfill

  Local trigger packet holds:

  \begin{description}
  \item[type] follows waveform category (``blip'' or ``signal'').
  \item[ident] TPC number (ie, APA+face)
  \item[extent] rectangular extent in time and channel.
  \item[charge] baseline subtracted ADC sum over extent.
  \item[stats] any other stats which can be quickly calculated.
  \end{description}

\end{frame}

\begin{frame}
  \frametitle{Example Global Trigger Logic}

  \begin{itemize}
  \item \textbf{``SNB'' trigger} accept stream of local ``\textbf{blip}'' triggers
    \begin{itemize}\footnotesize
    \item Select blips above some charge threshold, 
    \item ignore blips inside ``signal'' trigger, veto known ``extra blippy'' TPCs.
    \item[$\to$] trigger when remaining blip rate is above some threshold.

    \end{itemize}
  \item \textbf{``High Energy'' trigger} watch local ``\textbf{signal}'' trigger.
    \begin{itemize}\footnotesize
    \item Merge overlapping local extents.
    \item[$\to$] trigger on all.
    \end{itemize}
  \item \textbf{Trigger command} sent back to \textbf{multiple, specific} APA host
    computers with readout instructions.  Eg:
    \begin{itemize}\footnotesize
    \item[cmd:] Save select time region in all ``signal'' triggered APAs and
      their nearest neighbors.
    \item[cmd:] Save all data in all APAs for next N-seconds as a SNB is
      happening.
    \end{itemize}
  \end{itemize}
  
  \footnotesize

  Depending on trigger type, triggered data may then undergo
  \textbf{further processing} (per-trigger data reduction, compression, etc).
\end{frame}

\begin{frame}
  \frametitle{One Possible Triggering and Data Flow}
  \begin{columns}
    \begin{column}{0.5\textwidth}
      \begin{center}
        \includegraphics[width=1.1\textwidth]{trigger.pdf}

        \tiny
        Just two APAs shown for simplicity.
      \end{center}
    \end{column}
    \begin{column}{0.5\textwidth}
      \begin{itemize}\footnotesize
      \item One collection trigger processor per APA face (\texttt{coltrig})
      \item Global trigger command interpreter (\texttt{selector})
      \item Farm selected data some worker (\texttt{reducer})
      \item Save reduced data to concentrated sink.
      \item[?] Lots of possible options.
        \begin{itemize}\scriptsize
        \item Trigger reduction is enough, no ``reducers'' needed?
        \item Single ``reducer'' stage is NOT enough?
        \item Add ``stream branches'' (DQM, ``express lane'')?
        \item Sink I/O is still too high for just one node, add more?
        \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
  
\end{frame}



\section{To Do}

\begin{frame}
  \frametitle{To Do}
  \begin{description}\footnotesize
  \item[triggering] much more thought is needed about local trigger
    categories, algorithms as well as global trigger logic and commands.
  \item[simulation] use \textbf{Wire-Cell signal and noise sim} for
    accurate event ``size'' determination and estimating trigger alg
    CPU (GPU, etc) requirements.
  \item[scenarios] develop 2 or 3 conceptual designs (eg, the one
    shown here, many another exploiting GPU accel, etc).
  \item[scaling] Use sim to estimate the requirements for number and
    sizes of Tiers.  Estimate throughput between tiers and final data
    rate to disk/tape.  If ``too much'', iterate scenario definition.
  \item[costing] Use commodity costs from today.  If ``too
    expensive'', consider conservative Moore scaling and/or iterate
    the scenario(s).
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Simulation Samples}
  \begin{description}
  \item[noise + $^{39}$Ar] understand signal/noise energy threshold
    and \\ the rate of local ``blip'' triggers.
    \begin{itemize}\footnotesize
    \item MicroBooNE (M.Mooney+co) now looking at $^{39}$Ar in data.
    \item Can join forces.
    \end{itemize}
  \item[noise + SNB] understand SNB trigger logic (ie, how to separate $^{39}$Ar and SNB ``blip'' triggers).
  \item[high energy] $\nu$ and cosmic-$\mu$ sim to understand local trigger alg's CPU requirements.
    \begin{itemize}\footnotesize
    \item For trigger-based reduction, their data size can be
      estimated with just flux, cross-section and dE/dX.
    \item More sophistication, eg
      \href{https://docs.dunescience.org/cgi-bin/private/ShowDocument?docid=2089}{signal-processing based reduction},
      needs understanding of processing time variances.
    \end{itemize}
  \end{description}
\end{frame}

\section{}

\begin{frame}
  \begin{center}
    Extras
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Infrastructure Implementation}
  \begin{columns}
    \begin{column}{0.5\textwidth}
      \begin{center}
        \includegraphics[width=1.1\textwidth]{trigger.pdf}
      \end{center}
    \end{column}
    \begin{column}{0.5\textwidth}
      Try to reuse as much as possible
      \begin{itemize}\footnotesize
      \item Ring-buffer implies async, distributed, real-time, multi-processing.
      \item Does/can this model map to artDAQ? (hope ``yes''!)
      \item Many software roles:
        \begin{itemize}\scriptsize
        \item RAM filled by FELIX Linux driver(?)
        \item long-lived steam processors (\texttt{coltrig, selector})
        \item triggering is client/server model
        \item chunked transfer / processing clearly artDAQ'ish\\ (\texttt{selector} $\to$ \texttt{reducer} $\to$ \texttt{saver})
        \end{itemize}
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}
  \frametitle{What Might Go Where?}
  \begin{columns}
    \begin{column}{0.5\textwidth}
      \begin{center}
        \includegraphics[width=1.1\textwidth]{trigger.pdf}
      \end{center}
    \end{column}
    \begin{column}{0.5\textwidth}
      Possible locations:
      \begin{description}
      \item[under] \textbf{Tier 1} and \textbf{Global Trigger Service}
      \item[above] \textbf{Tier 2+}
      \end{description}
      \begin{itemize}\footnotesize
      \item Gives some underground autonomy if network fibers cut.
      \item Need bandwidth to surface for just triggered readout and \\not full 12 Tbps.
      \item Allows scaling up (possible majority) of nodes (Tier 2+) in the cheaper, above
        ground environment.
      \end{itemize}
    \end{column}
  \end{columns}
\end{frame}




\end{document}


%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:
