% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{data reduction}
\renewcommand{\bvtitle}{\LARGE Work Needed for the \\protoDUNE Data Reduction Process}
\renewcommand{\bvevent}{DRA\\2017 Mar 16}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}

\begin{frame}
  \frametitle{Data Reduction Overview} 

  \begin{itemize}
  \item Run the required pre-reco algorithms.
    \begin{itemize}\footnotesize
    \item Only processes TPC waveform data (nominally).
    \item Algs: ADC corrections, noise filtering, signal processing.
    \item Consists of $\sim$20\% of full-chain CPU time.
    \end{itemize}
  \item Save a much reduced output file.
    \begin{itemize}\footnotesize
    \item But, don't delete full, raw input files!
    \item Remove oversampling and employ data packing.
    \item Essentially, information-lossless for signal.
    \item 2.5 PB $\to$ 6.5 TB (just TPC part).
    \end{itemize}
  \item Keep up with protoDUNE/SP data taking
    \begin{itemize} \footnotesize
    \item Requires $\sim$200 nodes / $\sim$4000 cores during running.
    \end{itemize}
  \end{itemize}

  \begin{center}
    \href{http://docs.dunescience.org/cgi-bin/ShowDocument?docid=2089}{DocDB 2089 for details}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Data Reduction Process Steps}
  \begin{enumerate}
  \item \textbf{input unpacking:} with DAQ's raw data access library
    (DAL) 
    \begin{itemize}\scriptsize
    \item[$\to$] undo the $\sim 4\times$ compression, 12 $\to$ 32 bit:
      $\sim 10\times$ \textbf{inflation}
    \end{itemize}

  \item \textbf{ADC mitigation:} stuck codes and non-linearity
    corrections
    \begin{itemize}\scriptsize
    \item[$\to$] may improve compression at $\sim$10\% level
    \end{itemize}

  \item \textbf{noise filter:} reduce excess noise (if any)
    \begin{itemize}\scriptsize
    \item[$\to$] if pD/SP is \textbf{not} noisy, raw compression goes:
      $4\times \to 6\times$
    \end{itemize}

  \item \textbf{signal processing:} waveform deconvolution + signal-ROI 
    \begin{itemize}\scriptsize
    \item[$\to$] 150$\times$ total reduction factor if stop here
    \end{itemize}

  \item \textbf{rebinning:} exploit oversampling but respect Nyquist
    \begin{itemize}\scriptsize
    \item[$\to$] $400\times$ total reduction (for ``rebin-3'')
    \end{itemize}

  \item \textbf{output packing:} save in raw, fragment format 
    \begin{itemize}\scriptsize
    \item[$\to$] relies on ROOT compression
    \end{itemize}
  \end{enumerate}
  
\end{frame}

\begin{frame}
  \frametitle{Work needed and ongoing}

  For each data reduction process step, I try to summarize:

  \begin{itemize}
  \item What exists or is needed.
  \item[+] Who is currently working on parts.
  \item[$\to$] Possible areas for new people to get involved.
  \end{itemize}

\end{frame}


\begin{frame}
  \frametitle{Unpacking}

  \begin{itemize}
  \item Need a way to read and unpack raw data.
  \item Outer layer handled by \texttt{artdaq::Fragment}
  \item Need raw ``Data Access Library'' (DAL) 
    \begin{itemize}\footnotesize
    \item unpacks the ``fragment payload'' array 
    \item each type of fragment unique (TPC, PDS, TI, CR, maybe BI).
    \item must be developed by DAQ ``fragment experts''.
    \item data reduction nominally only unpacks TPC fragments,
    \item non-TPC fragments ``shunted'' as-is to output.
    \item RCE and FELIX fragments nominally to be identically packed.
    \item needs to handle evolution of fragment schema.
    \end{itemize}
  \item[+] \textbf{List of requirements has been developed.}
    \begin{itemize}\scriptsize
    \item No code development yet.
    \end{itemize}
  \item[$\to$] A volunteer could start the library/repo
    \begin{itemize}\footnotesize
    \item Needs close coordination with \textit{art}, DAQ and offline experts.
    \item Could start with ``toy'' packing as \texttt{v0} schemas.
    \end{itemize}
  \end{itemize}

  \footnotesize

  Comment: the DAL is DAQ responsibility, but as prime ``clients'' offline/DRA
  really should participate and directly help out (where appropriate).

\end{frame}

\begin{frame}
  \frametitle{ADC Mitigation and Calibration}
  
  \begin{itemize}
  \item The ADCs will need some level of \textbf{stuck-code mitigation} and \textbf{non-linear calibrations}
  \item[+] \textbf{David Adams}' ongoing studies of initial cold
    electronics test data taken at BNL.
  \item[+] All pD/SP electronics will undergo \textbf{production CE testing}.
    \begin{itemize}\footnotesize
    \item Effort already well underway on production test lab.
    \item Matt Worcester, Brian Kirby, Justin Hugon, others.
    \end{itemize}
  \item[$\to$] Effort needed toward an ADC mitigation/calibration.
    \begin{itemize}\footnotesize
    \item Get involved in the CE production testing.
    \item Develop some form of mit/calib parameter database
      \begin{itemize}\scriptsize
      \item Could be substantial (all ADCs, maybe 4096 numbers per!).
      \item If time variance is an issue, may need ongoing updating.
      \end{itemize}
    \item Develop code to apply these values to raw data.
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Excess Noise Filtering}

  \begin{itemize}
  \item No ``planned'' excess noise for protoDUNE.
  \item Strategy: assume the worse and prepare software filters for
    all known types of excess noise in protoDUNE.
  \item[$\to$] Need dedicated noise hunters for very first days of data.
  \item[$\to$] Effort for using existing software filters:
    \begin{itemize}\footnotesize
    \item[+] MB's noise filter in Wire Cell Toolkit, used via \texttt{larwirecell}
    \item[+/$\to$] Needs improvement to make modular (all in one \textit{art} Module). 
    \item[+] 35t's in \texttt{dunetpc} (David Adams)
    \item[$\to$] Needs ``porting'' to protoDUNE?
    \item[$\to$] Comprehensive/modular job configuration.
    \end{itemize}
  \item[$\to$] Expect need for fast deployment of code improvements
    \begin{itemize}\footnotesize
    \item[$\to$] Need s/w infrastructure expert to assure that noise
      filter experts can quickly (in hours) update ``keep up'' data
      reduction process with improved new noise filter code and
      configuration.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Signal Processing}
  \begin{itemize}
  \item[+] 2D signal deconvolution pioneered in MicroBooNE exists.
    \begin{itemize}\footnotesize
    \item 2D needed to remove long range induction effects, although
      protoDUNE's grid wires reduce the size of this problem.
    \end{itemize}
  \item[$\to$] In WC prototype, port to Toolkit (Xin Qian, BV)
    \begin{itemize}\footnotesize
    \item Required now'ish in any case for MB sig. proc. paper.
    \end{itemize}
  \item[$\to$] Validation with \textit{realistic detector response simulation}
    \begin{itemize}\footnotesize
    \item[+/$\to$] Truth metric based validation (Brook Russell, BV)
    \item (details on simulation next)
    \end{itemize}
  \item[$\to$] Integrate signal processing into \texttt{larwirecell}
    \begin{itemize}\footnotesize
    \item volunteers welcome, o.w. probably falls to me.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Rebinning and Packing}
  \begin{itemize}
  \item Rebinning alg is trivial.
  \item[+] Validated with simple Gaussian.
  \item[$\to$] More validation as a side effect when validating signal
    processing with \textit{realistic detector response simulation}.
    \begin{itemize}\footnotesize
      \item Brook, BV, others welcome.
    \end{itemize}
  \item[$\to$] Develop ``reduced DAL'' (BV, or ??)
    \begin{itemize}\footnotesize
    \item Should closely mimic ``raw DAL'' as non-TPC data in the
      reduced file will still be raw fragments.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Improved Detector Response Simulation}
  \footnotesize
  \begin{itemize}
  \item[+] Main improvements over current simulation:
    \begin{itemize}\scriptsize
    \item Long-range induction effects.
    \item Inter-pitch field variations.
    \item Drift model and its statistics.
    \end{itemize}
  \item Required for correct signal shapes, especially important for
    evaluating signal processing on induction plane channels.
  \item Needs completion for upcoming MB sig. proc. paper.
  \item[+] Core functionality exists, application layer a WIP (BV).
  \item[+/$\to$] ``Truth metrics'' need defining and implementing (Brook/BV).
  \item[+/$\to$] Drift model partly implemented (BV or ??).
  \item LArSoft integration not needed here, but eventually will happen.
  \end{itemize}

  \footnotesize Comment: could provide ``simulated real'' raw data
  files if we can get a proper fragment payload \textbf{packing} code
  from DAQ group (ie, the inverse of the ``raw DAL'' unpackers).

\end{frame}

\begin{frame}[fragile]
  \frametitle{LArSoft Integration}
  \begin{itemize}\footnotesize
  \item Data reduction algorithms are needed in many contexts.
  \item Some implementations are in Wire Cell Toolkit, potentially other libraries.
  \item Integrate WCT implementations into LArSoft via \texttt{larwirecell}
    \begin{itemize}\scriptsize
    \item only part of LS that depends on WCT.
    \item WCT does not depend on LS
    \end{itemize}
  \item Use a layered approach for code (and data):
  \end{itemize}

  \begin{center}\footnotesize
    \begin{tabular}[h]{|c|}
      \hline
      Wire Cell Toolkit and other implementations\\
      {\scriptsize (implementation-specific data structures)}\\
      \hline
      \textit{art} \textbf{Service} or \textbf{Tool}\\
      {\scriptsize (generic data structures)}\\
      \hline
      \textit{art} \textbf{Module}\\
      {\scriptsize (LArSoft data structures)}\\
      \hline
      \texttt{lar} executable \\
      {\scriptsize (FCL configuration)} \\
      \hline
    \end{tabular}
  \end{center}
  
\end{frame}

\end{document}

