% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{regulating memory usage}
\renewcommand{\bvtitle}{\LARGE Regulating Memory Usage}
\renewcommand{\bvevent}{{\small DUNE Computing Model Discussion -- 2019-05-21}}
\renewcommand{\bvdate}{21 May 2019}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}

\begin{document}
\input{beamer/title.tex}
% \input{beamer/toc.tex}

% \section{The Problem}
% \begin{frame}
%   \tableofcontents[currentsection,hideothersubsections]
% \end{frame}

\begin{frame}
  \frametitle{Drivers of Memory Usage}
  \begin{center}
    
    One APA readout, just TPC data, in-memory in Bytes:
    
    $(800+800+960) \times \SI{2}{\mega\hertz} \times 2\mathrm{B} \times T_{ro}$

    \vfill

    \begin{tabular}[h]{c|r|r}
      \hline
      PDSP & per-APA & 6 APA \\
      \hline
      \SI{3.0}{\milli\second} & \SI{30.7}{MB} & \SI{184}{MB} \\
      \SI{7.5}{\milli\second} & \SI{76.8}{MB} & \SI{461}{MB} \\
      \hline
      \hline
      FDSP & per-APA & 150 APA \\
      \hline
      \SI{5.4}{\milli\second} & \SI{54.3}{MB} & \SI{8.3}{GB} \\
      \SI{100}{\second} & \SI{1.0}{TB} & \SI{154}{TB} \\
      \hline
      
    \end{tabular}


    \vfill

    \footnotesize
    (note: nominal FDSP ``event'' is $45\times$ nominal PDSP event.)

  \end{center}


\end{frame}

\begin{frame}
  \frametitle{First Stage Processing}
  \begin{center}
    \includegraphics[width=\textwidth]{first-stage.pdf}
  \end{center}
  \begin{itemize}
  \item \texttt{vector<raw::RawDigit>} currently whole-detector loaded
    \begin{itemize}\scriptsize
    \item[$\circ$] (\SI{184}{MB} or \SI{461}{MB} for PDSP)
    \end{itemize}
  \item NF+SP require \texttt{float}, naively causes $3\times$ inflation 
    \begin{itemize}\scriptsize
    \item[$\circ$] (\SI{552}{MB} or \SI{1383}{MB} for PDSP)
    \item[$\circ$] plus working memory but it is per-\{channel,plane,APA\}
    \end{itemize}
  \item \texttt{vector<recob::Wire>} is \textbf{sparse}
    \begin{itemize}\scriptsize
    \item[$\circ$] Example PDSP data sees $40\times$ reduction.
    \item[$\circ$] \href{http://docs.dunescience.org/cgi-bin/ShowDocument?docid=2089}{DocDB2089} predicts $\sim 100 \times$ reduction, need to understand discrepancy.
    \item[$\to$] \textbf{``Signal'' data is substantially smaller than raw}.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Recomendations for Processing}
  First per-APA stage:
  \begin{itemize}\footnotesize
  \item Hold only one APA data in memory at any time (for single-thread job)
  \item Run only decode+NF+SP
  \item Do not write \texttt{vector<raw::RawDigit>} to output file.
  \end{itemize}
  Second per-APA stage:
  \begin{itemize}\footnotesize
  \item Input \texttt{vector<recob::Wire>} files and raw light data.
  \item Do imaging, clustering and charge/light ``flash matching''.
  \item Write out results (3D space-charge points and their associations)
  \item Drop inputs
  \end{itemize}
  Third+ whole-detector stage:
  \begin{itemize}\footnotesize
  \item Read in all $\mathcal{N}$ per-APA files from stage 2.
  \item Do cross-APA stitching of clusters, build tracks/showers
  \item Pattern recognition, kinematic reconstruction, etc
  \item Output analysis \texttt{TTree}.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Wire-Cell/LArSoft Improvements for PDSP}

  \begin{center}
    \includegraphics[width=0.45\textwidth]{dnfsp-monolith-6apa.png}%
    \includegraphics[width=0.45\textwidth]{dnfsp-lazy-6apa.png}
  \end{center}

  \begin{itemize}\footnotesize
  \item Current decoder loads 6-APA \texttt{raw::RawDigit} to \texttt{art::Event}
  \item Conversion to 32 bit \texttt{float} was monolithic (all 15k channels)
  \item New \texttt{LazyFrame\{Source,Frame,Trace\}} in \texttt{larwirecell} does conversion ``just in time''.
  \item New Wire-Cell graph components maintain lazy pattern.
  \end{itemize}

  PDSP decode + WCT NF + WCT SP jobs now use \SI{1.4}{GB} RSS for 3ms and about \SI{2}{GB} RSS for 7.5ms.

\end{frame}

\begin{frame}
  \frametitle{Toward DUNE FD (offline)}
  \begin{itemize}
  \item Wire-Cell/LArSoft ``lazy conversion'' should scale to DUNE FD.
  \item But, loading in full raw ``event'' will not.
  \item Work has started (Tom Junk) to factor decoding into a Tool to support per-APA loading.
    \begin{itemize}\footnotesize
    \item[$\to$] Even better would be to have the decoding logic provided in a generic stand-alone shared library.
    \end{itemize}
  \item Wire-Cell/LArSoft will use this refactored code to load per-APA ADC-level data as transient input to WCT NF+SP.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Toward DUNE FD (DAQ)}
  To facilitate Offline raw data loading and optimize DAQ design, DAQ is considering to:
  \begin{itemize}
  \item Select more optimal file format (than ROOT) for raw data.
  \item Write one ``event'' as collection of per-APA ``files''\footnote{Exact technology being studied}.
    \begin{itemize}\footnotesize
    \item[$\circ$] Nominal \SI{5.4}{\milli\second} ``event'' has one file per APA.
    \item[$\circ$] SNB has many files per-APA, covering common time ranges substantially longer than nominal \SI{5.4}{\milli\second} but less than full \SI{100}{\second}.

    \end{itemize}

  \item A data transform stage at DAQ$\to$Offline hand-off is possible and an optimal solution for both DAQ and Offline is sought.
  \end{itemize}
  
\end{frame}

\end{document}

