% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usepackage{onimage}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{DAQ Backend}
\renewcommand{\bvtitle}{\LARGE DAQ Backend\\Trigger, Data Selection and Egress}
\renewcommand{\bvevent}{{\small DAQ Data Selection WS -- 13 Aug 2018}}
\renewcommand{\bvdate}{13 Aug 2018}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}

\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{DAQ Concept Overview}
% \begin{frame}
%   \tableofcontents[currentsection,hideothersubsections]
% \end{frame}

\begin{frame}
  \frametitle{Conceptual DAQ Subsystem Relationships}
  \vspace{-6.6mm}
  \begin{center}
    \includegraphics[height=\textheight]{top.pdf}    
  \end{center}
\end{frame}

\begin{frame}

  \begin{center}
    \textcolor{red!40!white}{Front-end Computers}
    \textcolor{blue!40!white}{Back-end Computers}

      \begin{tikzonimage}[height=\textheight]{top.pdf}%[tsx/show help lines]
        \draw [fill=red!40!white, draw=black, fill opacity=0.2] plot [smooth cycle]
        coordinates {(0.2,0.4) (0.2, 0.55) (0.4, 0.85) (.5,.86) (.5,.70) (.4,0.5) (.45,.25) (.4,.1) };
        \draw [fill=blue!40!white, draw=black, fill opacity=0.2] plot [smooth cycle]
        coordinates {(.55,.86) (0.50,.70) (0.55,0.55) (0.55,0.40) (0.45,0.28) (0.45,0.1) (0.55,0.1) (0.75,0.4) (0.75,0.55)};
      \end{tikzonimage}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Details: One FEC + Backend Computers}
  \begin{center}


    \includegraphics[width=\textwidth]{fec-be.pdf}
    
    \tiny Here shows FELIX+CPU design, FELIX+DPM moves buffer + TP to DPM
  \end{center}
  \begin{itemize}
  \item Each Front-end Computer (FEC) services two APAs.
    \begin{itemize}\footnotesize
    \item[o] \textbf{Receive} $2\times 8$ GB/s from WIB via fiber optic into FELIX boards
    \item[o] 10 seconds RAM for normal and SNB pre-trigger \textbf{buffering}
      \begin{itemize}\scriptsize
      \item[:] a few design variations on how to handle SNB dumping
      \end{itemize}
    \item[o] Source of \textbf{trigger primitives}.
    \item[o] Interface for \textbf{data selection} output.
    \end{itemize}
  \item Back-end computers for Trigger and Egress subystems.
  \end{itemize}

\end{frame}

\section{DAQ Backend}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}


\begin{frame}
  \frametitle{DUNE SP Trigger and Egress}
  \vfill
  \includegraphics[width=\textwidth]{bow-tie-1.pdf}
  \vfill
  \footnotesize
  \begin{center}
      \begin{description}     \scriptsize
    \item[150] APAs
    \item[75] trigger-primitive sources (TPS), 1/FEC,
    \item[5] trigger candidate sources (TCS), exact \# tbd,
    \item[1] Module Trigger Logic (MTL) unit,
    \item[1] External Trigger Logic (ETL) unit,
    \item[1] Egress System Manager (ESM),
    \item[$\mathcal{N}$] Event Builders (EB) at any given time,
    \item[75] Data Selectors (DS), 1/FEC.
    \end{description}

  \end{center}
\end{frame}

\begin{frame}
  \frametitle{$\approx\frac{1}{15}^{th}$ DUNE SP Trigger and Egress (zoom)}
  \vspace{-5mm}
  \begin{center}
    \includegraphics[height=4cm, clip, trim=155cm 0 140cm 0]{bow-tie-1.pdf}

    \begin{enumerate}     \scriptsize
    \item Full stream data flows from 2 APAs into one FEC.
    \item $\frac{3}{8}^{th}$ collection channels input to Trigger Subsystem.
    \item Trigger primitive sources (TPS) compute and emit per-channel trigger info.
    \item Trigger candidate sources (TCS) compute emit per-APA-face trigger info.
    \item The Module Trigger Logic unit (MTL) forms and issues a trigger command.
    \item The Egress System Manager executes command by spawning Event Builder (EB)
    \item Each EB queries each front end Data Selector (DS) by channel \textit{vs} tick range.
    \item EB saves replies + trigger command to file (not depicted)
    \end{enumerate}

  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Trigger Primitives}

  Trigger primitives localize any above-threshold activity to a single collection channel and a time region.    

  \vfill

  Straw data structure for DUNE SP:
  \begin{description}\footnotesize
  \item[32 bit] channel address (need $>$ 19 bits for 150$\times$2560).
  \item[64 bit] time stamp ($\approx$300 millennia of 2MHz tick, 32 bits $\approx$ 0.6 hour).
  \item[16 bit] ticks over threshold (14 bit spans 5.4ms).
  \item[32 bit] ADC sum (25 bits covers all possible in 5.4ms).
  \item[16 bit] error flag, setting it may override meaning of above.
  \item[20 Byte] total.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Calculating Trigger Primitives}

  \begin{itemize}
  \item Algorithm computational performance is mostly independent of information content.
    \begin{itemize}\footnotesize
    \item[o] It may be possible to ``bog down''  at pipeline output.
    \end{itemize}
  \item Initial demonstration of feasibility of forming TP on CPU!
    \begin{itemize}\footnotesize
    \item[$\to$] See Phil's talk (spoiler: 4 threads / CPU / APA)
    \item[o] Data reordering demonstrated on CPU in protoDUNE 
      \begin{itemize}\scriptsize
      \item[:] initial benchmark: keep-up with 7 threads per DUNE APA
      \item[$\to$] see \href{https://indico.cern.ch/event/747649/}{Thijs Miedema}'s presentation
      \end{itemize}
    \end{itemize}
  \item Expect some/all processing can be on-loaded to FPGA.
    \begin{itemize}\footnotesize
    \item[o] The trade-offs need understanding/demonstration.
      \begin{itemize}\scriptsize
      \item[$\to$] some desiderata: costs/effort/electrical power/flexibility.
      \end{itemize}
    \item[o] Choice effects other parts of the design (buffer, data selection).
    \item[o] Ongoing study by DPM and FELIX teams.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Trigger Primitives to Trigger Candidate Source}

    \includegraphics[height=1cm, clip, trim=155cm 12cm 155cm 4cm]{bow-tie-1.pdf}

  Where is the danger zone?
  \begin{itemize}\footnotesize
  \item At $E_{true} <1$ MeV: 
    \begin{itemize}\scriptsize
    \item[o] $\approx$3 TP/Ar39 decay: 6-12 MB/s and 300-600 kHz TPs per FEC. 
    \item[o] Would dominate: 19 PB/year/SP compared to 10 PB/year for cosmic-$\mu$
    \item[o] Still unclear if TP is enough and how to do the processing.
      \begin{itemize}\tiny
      \item[$\to$] M.Mooney \& co. investigating.
      \end{itemize}
    \item[o] At FEC, data rate low, but rate too high for individual network packets.
    \item[$\Rightarrow$] May want to consume Ar39 TPs at the FEC.
    \end{itemize}
  \item In $1 < E_{true} <5$ MeV: 
    \begin{itemize}\scriptsize
    \item[o] $\approx$5 TP/Ar42 decay: 260kB/s and 13 kHz TPs per FEC, 
    \item[o] Will aggregate by $\times\mathcal{N}\approx15$ at TCS, packet rate okay for $>$GbE.
    \item[o] Many Ar42 TPs will be indistinguishable from MIP TPs.
    \item[$\Rightarrow$] Can TC processor handle the rate?  
      May want to filter out at FEC level?
    \end{itemize}
  \item Above $E_{true} > 5$ MeV: what problems for this stage?
    \begin{itemize}\scriptsize
    \item[o] Big showers local to small number of TCSs, lead to temporary rate spike.
    \item[o] Institute TP combining, signal via TP error flag?
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Trigger Candidate Source to MTL}

    \includegraphics[height=1cm, clip, trim=155cm 10cm 155cm 5cm]{bow-tie-1.pdf}

  Accept TPs from many sources, correlates across channels to produce ``trigger candidate''.
  \begin{itemize}\footnotesize
  \item A TC datum needs more thought but is a summary of all its TPs
    \begin{itemize}\scriptsize
    \item[o] Is a ``rectangle'' in tick vs channel, probably not spanning APAs
    \end{itemize}
  \item Past discussion about stream vs. chunked algorithms.
    \begin{itemize}\scriptsize
    \item[o] Let's stream. Chunking introduces magic/tuned numbers.
    \item[o] Each TP source required to emit in time order and can send heartbeat
    \item[o] TC always know what TPs are safe to use and which must be held back.
    \end{itemize}
  \item TCs from all $\mathcal{N}$ TCSs aggregate into MTL
  \end{itemize}
  
\end{frame}

\begin{frame}
  \frametitle{Module Trigger Logic unit to Egress Subsystem Manager}
  \begin{center}
    \includegraphics[height=2cm, clip, trim=180cm 6cm 180cm 8cm]{bow-tie-1.pdf}    
  \end{center}
  \begin{itemize}
  \item Issues \textbf{trigger command} based on input candidates.
    \begin{itemize}\footnotesize
    \item Candidates may also come External Trigger Logic unit.
    \item Command sent to ETL to be forwarded as candidate to other MTls
    \end{itemize}
  \item Trigger command:
    \begin{itemize}\footnotesize
    \item Summary of all TCs that formed it.
    \item Data selection addressing in form of FEC/channel/tick ranges.
    \item \textbf{MTL guarantees not to issue overlapping commands}
    \end{itemize}
  \item Command is issued to ESM
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Egress Subsys Manager to Event Builders}
    \includegraphics[height=2cm, clip, trim=165cm 1cm 155cm 13cm]{bow-tie-1.pdf}    

  \begin{itemize}\footnotesize
  \item ESM spawns Event Builder (EB) to service each trigger command.
    \begin{itemize}\scriptsize
    \item[o] Multiple triggers may be ``in flight'' for servicing at any time.
    \end{itemize}
  \item EB executes trigger command, querying Data Selectors (DSs).
    \begin{itemize}\scriptsize
    \item[o] At most 75 parallel requests per trigger command
    \item[o] Possible not all DSs are implicated in any given trigger
      command.
    \end{itemize}
  \item EB waits for response (data or error) and aggregates and saves to file.
    \begin{itemize}\scriptsize
    \item[o] Long wait possible, in particular for SNB dumps.
    \item[o] 5.4ms$\times$SP = 8 GB.  Multiple files per event?  Certainly for SNB dumps.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Data Selectors}
  \begin{center}
    \tiny (FELIX+CPU design)

    \includegraphics[height=3cm, clip, trim=6cm 0 7cm 1.5cm]{fec-be.pdf}    

  \end{center}
  \vspace{-5mm}
  Back to the front-end.
  \begin{itemize}\footnotesize
  \item DS is intermediate between buffer implementation and network.
    \begin{itemize}\tiny
    \item[o] FELIX+CPU: PC RAM + PCIe NVMe/SSD
    \item[o] FELIX+DPM: RAM + NVMe/SSD onboard DPM and accessed via FELIX via PCIe
    \end{itemize}
  \item Services network queries (from EB) for range of channel and tick.
  \item Queries may come out-of-order and may come too late.
  \end{itemize}
  
\end{frame}

\section{Comments}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{DAQ Partitions}
  \begin{center}
    \includegraphics[height=2cm, clip, trim=170cm 5cm 170cm 6cm]{bow-tie-1.pdf}
  \end{center}

  \begin{itemize}\footnotesize
  \item Each detector module must operate effectively autonomously.
    \begin{itemize}\scriptsize
    \item[o] Loose coupling via MTL $\leftrightarrow$ ETL communication.
    \end{itemize}
  \item Must be able to run majority of module \textbf{and} some fraction for calib/debug.
  \end{itemize}
  $\Rightarrow$ need another layer in the hierarchy: \textbf{DAQ partition}.
  \begin{description}\footnotesize
  \item[A?] Multiple MTLs per detector module
  \item[B?] Explicit new layer above or below MTL.
  \end{description}
  \footnotesize
  In principle, ESM can stay singleton.
  
\end{frame}

\begin{frame}
  \frametitle{Implementation}
  \begin{itemize}\footnotesize
  \item DAQ will have a \textbf{large number of partly autonomous agents/processes}.
    \begin{itemize}\scriptsize
    \item Need robust network communication, error handling.  
    \item Let's be adversarial in our testing. Break stuff now, not underground.
    \end{itemize}
  \item \textbf{Fast and flexible} process initialization and mutual discovery.
    \begin{itemize}\scriptsize
    \item Processes should find others on the network, minimize centralization.
    \end{itemize}
  \item Strive for \textbf{zero latency / zero data loss} for reconfiguration.
    \begin{itemize}\scriptsize
    \item[o] Change partition makeup, triggering policy, etc, ``on the fly''.
    \item[o] Any electronics reconfiguration latency notwithstanding.
    \end{itemize}
  \item Pervasive \textbf{configuration, logging and alert} system.
    \begin{itemize}\scriptsize
    \item[o] Embrace pub/sub pattern.
    \item[o] Select implementation with multiple language bindings.
    \end{itemize}
  \item \textbf{Integrated software infrastructure} from the start
    \begin{itemize}\scriptsize
    \item[o] Cover trigger, egress and run control subsystem process communication.
    \item[o] Shared system for defining packets, protocols, socket access patterns.
    \item[o] Include thread$\to$process and process$\to$node migration.
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \begin{center}
    \Huge $\mathcal{FIN}$
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Edits and Points Raised}
  This slide was added after the presentation.
  \begin{itemize}\footnotesize
  \item The number of TCSs (``5'') on slide 7 is a guess based on
    benchmarks of clustering for SNB studies.
    Some studies make it as many as 25 for higher energy clusters. 
    A few mitigations:
    \begin{itemize}\scriptsize
    \item[o] Only need to cluster ``enough'' to know activity of interest.
    \item[o] Could apply a local dead time to skip ahead after a TC
      was emitted that was ``sufficiently large'' so as to guarantee
      it leads to a trigger command.
    \end{itemize}
  \item Fixed typos on slide 11 ``TC'' $\to$ ``TP''.
  \item Slide 12: PDS trigger candidates need representation. 
    They could be accommodated by one or more TCS computers feeding
    TCs to MTL.
  \item Strict time ordering of trigger commands requires the MTL to
    wait ``sufficiently long'' to get an SNB trigger candidate from
    another MTL via the ETL.
    \begin{itemize}\scriptsize
    \item[o] SNB physics ``rise time'' is ~0.4s, but not well understood.
    \item[o] We may want to relax the strict time order requirement for
      SNB.
    \item[o] This will then lead to some data read out more than once.
    \end{itemize}
  \item Slide 18: ``discoverability'' doesn't make sense when we talk
    of inter-thread communication.
  \end{itemize}
\end{frame}

\end{document}
