% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{data reduction}
\renewcommand{\bvtitle}{\LARGE Data Reduction Effort at BNL}
\renewcommand{\bvevent}{Local EDG\\2017 Feb 6}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}

\begin{frame}
  \frametitle{Data Reduction Job and Efficacy} 

  Job will consist of these major parts 
  \begin{itemize}\scriptsize
  \item[$\to$] gives reduction factor relative to canonical 12 bit
    packing and $4\times$ compression if saved after given step
  \end{itemize}

  \begin{enumerate}
  \item \textbf{input unpacking:} with DAQ's raw data access library
    (DAL) 
    \begin{itemize}\scriptsize
    \item[$\to$] undo $\sim 4\times$ compression, 12 $\to$ 32 bit:
      $\sim 10\times$ \textbf{inflation}
    \end{itemize}

  \item \textbf{ADC mitigation:} stuck codes and non-linearity
    corrections
    \begin{itemize}\scriptsize
    \item[$\to$] may improve compression at few 10\% level
      (\textbf{total guess})
    \end{itemize}

  \item \textbf{noise filter:} reduce excess noise
    \begin{itemize}\scriptsize
    \item[$\to$] if pD/SP is noisy 
      $4\times \to \sim 6\times$ compression, o.w. no data reduction here.
    \end{itemize}

  \item \textbf{signal processing:} waveform deconvolution + signal-ROI 
    \begin{itemize}\scriptsize
    \item[$\to$] 150$\times$ (drop noise waveforms, reduce entropy)
    \end{itemize}

  \item \textbf{rebinning:} exploit oversampling but respect Nyquist
    \begin{itemize}\scriptsize
    \item[$\to$] rebin-3 $400\times$ ($3\times$ and some entropy reduction)
    \end{itemize}
  \item \textbf{output packing:} save in raw-like format 
    \begin{itemize}\scriptsize
    \item needs a ``reduced DAL'' similar to ``raw DAL''
    \item relies on ROOT compression
    \item this step is assumed for above reduction estimates
    \end{itemize}
  \end{enumerate}
  \begin{center}
    \href{http://docs.dunescience.org/cgi-bin/ShowDocument?docid=2089}{DocDB 2089 for details}
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Many Contexts for Data Reduction Code} 

  Need steps 2 -- 4 (ADC, noise filter, signal processing) for:
  \begin{description}
  \item[OM] online monitoring inside artDAQ
    \begin{itemize}\footnotesize
    \item This is a ``maybe''
    \item I think all OM should stick with showing raw quantities.
    \end{itemize}
  \item[DQM] aka prompt processing, aka p3s 
    \begin{itemize}\footnotesize
    \item at least any reco job will need the ``cooked'' data
    \end{itemize}
  \item[Offline] production processing, reconstruction
    \begin{itemize}\footnotesize
    \item Either offline uses $\sim$10 TB reduced data (preferred) or
    \item Offline has to run steps 2 -- 4 directly on 2.5 PB
    \item[$\to$] if so, $\sim$10 TB reduced data is (hopefully) a side-product
    \end{itemize}
  \item[Sim] simulation connection to the reduction code:
    \begin{itemize}\footnotesize
    \item model ADC features and expected excess and inherent noise to
      generate data to \textbf{evaluate reduction efficacy}.
    \item[$\to$] requires to pack simulation output in raw DAL format
    \item run signal processing on simulation output in any case (even
      if simulating noise-free and ``perfect'' ADC).
    \end{itemize}
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{LArSoft and Wire Cell Toolkit}
  \textbf{Right now:}
  \begin{itemize}
  \item 35t ADC mit. and noise filt. \textbf{dev} in LArSoft (David)
  \item MicroBooNE noise filt \textbf{done}(?) in Wire Cell Toolkit (Xin)
    \begin{itemize}\footnotesize
    \item[$\to$] noise filt. is integrated to LArSoft (Brian \& Brett)
    \item[$\to$] not following Service pattern due to lack of LS expertise

    \end{itemize}
  \item Signal processing \textbf{dev} in WC prototype (Xin)
  \item Drift simulation \textbf{dev} in Wire Cell Toolkit (Brett)
  \end{itemize}
  \textbf{Want/expect for protoDUNE:}
  \begin{itemize}
  \item ADC mit. \textbf{dev} based on CE testing results
    \begin{itemize}\footnotesize
    \item Likely in LS(?)  Not strictly needed in WCT.
    \end{itemize}
  \item Generalize MicroBooNE noise. filt. in WCT
    \begin{itemize}\footnotesize
    \item Improve/generalize current WCT/LS integration.
    \item This may end up being a no-op, but we have to be ready
    \end{itemize}
  \item Implement sig. proc. in WCT and integrate into LS
  \item Finish drift sim in WCT and integrate into LS
  \end{itemize}
  \textbf{Constraint:} WCT will not depend on LS, integration involves writing
  LS job \textbf{modules} or \textbf{services} which call out to WCT.
\end{frame}

\begin{frame}
  \frametitle{What I can do and what I need help with}
  \textbf{My software plate:}
  \begin{itemize}
  \item Provide drift simulation and integrate uBoone and Milind's noise models.
  \item Do more data reduction studies using uBoone data?
  \item Sig.proc. prototype $\to$ WCT (either assist or do the work)
  \item Raw and reduced ``DAL''.
  \item Make LS integration succeed.
  \end{itemize}
  \textbf{Where I would need help:} (basically, LS integration)
  \begin{itemize}
  \item Working in LS, build, developer environment, configuration,
    release issues.
    \begin{itemize}\footnotesize
    \item[$\to$] this may have just gotten a little easier with the
      announcement of official Ubuntu 16.04 support by FNAL.
    \end{itemize}
  \item Defining Service Interfaces for integration
    \begin{itemize}\footnotesize
    \item need's coordination with David and greater LS community.
    \item if we even go this route, module-level integration is an option
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Other concerns}

  \begin{itemize}
  \item Data reduction ``keep up'' needs $\sim$3000 cores 
    \begin{itemize}\footnotesize
    \item need $\sim 6\times$ this for full processing
    \end{itemize}
  \item Data reduction ``keep up'' needs job management
    \begin{itemize}\footnotesize
    \item a second instance of \textbf{p3s} would work.
    \item need production processing or some group to include it in
      their scope.
    \end{itemize}
  \item Data access library (DAL)
    \begin{itemize}\footnotesize
    \item Must work closely with DAQ group
    \item Want both packing (for sim) and unpacking code for all raw fragments
    \item Want a ``DAL'' for reduced data (or include in raw DAL)
    \item Must release new raw DAL \textbf{before} production DAQ
      writes new blocks.  (ideally!)
    \item New DAL code must still pack/unpack old data.
    \end{itemize}
  \end{itemize}

\end{frame}

\begin{frame}
  \begin{center}
    backups
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{LS Integration: Module vs. Service}
  \begin{itemize}\footnotesize
  \item An art/LS job is a string of Modules communicating by data products.
  \item LS Services allow to share code across Modules or even outside
    of a Module.
  \item Defining a Service requires defining its method interface and
    the data types the methods take and return.
  \item The interface should be general enough but capture the
    abstraction.  It's hard to get this right.  Needs community
    involvement.
  \item Implementing an interface is somewhat more work than dumping
    code into a job module.
  \item Only one instance of an Art Service implementation can be
    created.  Would like to extend Art to include ``Tools'' which
    overcome this limitation.
  \end{itemize}

  N.B.: Wire Cell Toolkit is heavily based on interfaces of which most
  are ``Tools'' rather than singleton ``Services''.
\end{frame}

\end{document}

