% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren\\{\small (for WCT team)}}
\renewcommand{\bvtit}{WCT}
\renewcommand{\bvtitle}{\LARGE Wire Cell Toolkit Status\\{\small With Focus On LArSoft Integration}}
\renewcommand{\bvevent}{{\small FD SimReco\\2017 Aug 17}}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Toolkit Overview \& Recent Improvements}

\begin{frame}
  \frametitle{Overview of Wire Cell Toolkit}
  The toolkit consists of \textit{software components} covering:
  \begin{itemize}
  \item Full \textbf{signal simulation}  using
    \textbf{long-range and fine-grained detector response} 
    and accurate \textbf{electronics response} functions.
  \item Full \textbf{noise simulation} using realistic, empirical and
    first-principles models.
  \item \textbf{Noise filter} to reduce ($\mu$Boone's) excess noise.
  \item \textbf{Signal processing} to deconvolve \textbf{detector
      response} and select \textbf{signal-ROI}.
    \begin{itemize}\footnotesize
    \item Charge matching on all 3 planes, confirmed with $\mu$BooNE data.
    \item Evaluated against full simulation to remove biases.
    \end{itemize}
  \end{itemize}

  \footnotesize
  On the the way from Wire Cell \textbf{Prototype}:
  \begin{itemize}
  \item 3D ionization charge image reconstruct, pattern recognition, optical, ...
  \end{itemize}

  
\end{frame}


\begin{frame}
  \frametitle{Recent Toolkit Improvements}
  Since last meeting, many important improvements to WCT:
  \begin{itemize}
  \item Prototype \textbf{signal processing} ported into toolkit (Xin, bv)
  \item Improved and validated signal processing to greatly
    \textbf{reduce charge bias} as a function of track angle (Hanyu)
    \begin{itemize}\footnotesize
    \item[$\to$] Now feel WCT sigproc gives solid foundation for 3D imaging.
    \end{itemize}
  \item Refinement of sim's \textbf{signal induction} model (Hanyu)
  \item Corrections to \textbf{electronics response} model (Hanyu)
  \item Initial plan and steps for \textbf{simulation truth} recording (Brooke)
  \item Modeling and validating \textbf{2D field calculations} of
    detector pathology regions (Yichen, Mike).
  \item Start of a \textbf{validation suite} (bv)
  \item Improved \textbf{LArSoft integration} (bv) ... focus of this talk
  \end{itemize}
\end{frame}


\section{Wire Cell / LArSoft Integration Design}

\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Initial Noise Filter Integration}
  \begin{itemize}
  \item Dedicated MicroBooNE \textbf{noise filter} (Brian).
    \begin{itemize}\footnotesize
    \item Got something working quickly, minimizing up-front effort.
    \item It is kept as-is in \texttt{larwirecell/LarInterfaces}
    \end{itemize}
  \item This helped to identify some issues:
    \begin{itemize}\footnotesize
    \item Code implements a \textbf{tight coupling} between LS and WCT
      sides.
    \item Requires detailed development updates as LS or WCT sides
      change.
    \item Extending this pattern into the future will require a lot of
      coding, is error prone and tends toward copy-paste programming.
    \item Lack of clear boundary leads to difficulty capturing full
      code, data and configuration state from LS-side for WCT-side
      debugging.
    \end{itemize}
  \end{itemize}
  $\to$ The new integration design addresses these issues.
\end{frame}

% \begin{frame}
%   \frametitle{New WCT/LS Integration Design}

%   \begin{center}
%     \includegraphics[height=8cm,rotate=270,clip,trim=2cm 0 0 6cm]{consortium-example-org2.pdf}    
%   \end{center}
% \end{frame}

\begin{frame}
  
  \frametitle{New WCT/LS Integration Design}
  \begin{center}
    \includegraphics[width=0.8\textwidth]{nfsp-integ.pdf}
  \end{center}
  \footnotesize

  \textcolor{blue}{WCLS integration code} in \href{https://cdcvs.fnal.gov/redmine/projects/larwirecell}{\texttt{larwirecell}} branch \href{https://cdcvs.fnal.gov/redmine/projects/larwirecell/repository/show/larwirecell?rev=feature\%2Fintegration}{\texttt{feature/integration}}.

  Single \textit{art} Tool for any/all entries to WCT.  Reference
  \textit{art} Module provided.

  Dual-faced \textit{converter components} to bridge LS/WCT data boundary.
\end{frame}


\begin{frame}
  \frametitle{Details: Integration Package \texttt{larwirecell}}
  \begin{itemize}
  \item \texttt{Interfaces} directory holds any new \textbf{WCT Interface} classes
    that depend on \textit{art}/LS data products
  \item \texttt{WireCellToolkit} is a reference \textit{art}
    \textbf{Module} for running WCT. In principle, only one that is needed.
  \item \texttt{WCLS} \textit{art} \textbf{Class Tool} providing main WCT entry point (equivalent to \texttt{wire-cell} CLI).
  \item WCT/LS \textbf{converter component} classes, eg:
    \begin{itemize}
    \item \texttt{RawFrameSource} (\texttt{raw::RawDigit} $\to$ \texttt{WireCell::IFrame})
    \item \texttt{CookedFrameSink} (\texttt{WireCell::IFrame} $\to$ \texttt{recob::Wire})
    \item \texttt{ChannelNoiseDB} access to time-varying info for noise filtering.
    \end{itemize}
  \end{itemize}
  \footnotesize
  
  \begin{itemize}
  \item[$\to$] Support for new WCT functionality only requires new
    \textit{converter components} to be written, and only if new data
    products are introduced.
    
  \item[$\to$] WCT code run in LS can be rerun, largely unchanged, in
    a pure-WCT environment which help experts to debug problems.

  \end{itemize}
\end{frame}

\section{Configuration}

\begin{frame}[fragile]
  \frametitle{Configuration: \textit{art}-side FHiCL}
  \scriptsize
\begin{verbatim}
source :{
   module_type: RootInput
   inputCommands: ["drop *", "keep raw::RawDigits_*_*_*"]
}
physics :{ producers: { nfsp : {
  module_type : WireCellToolkit
  wcls_main: {
    tool_type: WCLS
    apps: ["Omnibus"]
    plugins: ["WireCellGen", "WireCellSigProc", "WireCellLarsoft"]
    configs: ["uboone-nf-sp.jsonnet"]
    inputers: ["wclsRawFrameSource"]
    outputers: ["wclsCookedFrameSink"]
    params: { detector: "uboone" }
   }
   p1 : [ nfsp ]
   trigger_paths : [ p1 ]
}
\end{verbatim}

  \footnotesize The \texttt{wcls\_main} Tool accepts FHiCL parameter
  set which is essentially identical to the set of arguments accepted
  by the WCT CLI (``\texttt{wire-cell}'').


\end{frame}

\begin{frame}[fragile]
  \frametitle{Aside: \textit{art} Configuration Support}

  \textit{art} has some great support for defining configuration
  parameter set schema but \textbf{very few} LArSoft Tools/Modules/Services
  use it!
  \begin{itemize}
  \item Dev adds a little code to define each parameter in terms of
    \textbf{type, name and docstring}.
  \item Allows \textit{art} to perform \textbf{input validation} on the FHiCL.
  \item Allows \textit{art} to provide \textbf{command line help} messages.
  \end{itemize}
\begin{verbatim}
$ art --print-description WCLS
...
[Description of all FHiCL parameters this tool takes]
\end{verbatim}

  \footnotesize

  It takes a little extra effort to add this support code, but I
  think it pays off for the user.  Let's all use it!

  Many thanks to \textbf{Kyle Knoepfel} for help on the LS mailing
  list to make use of these features!
\end{frame}


\section{Challenges \& Issues}

\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Integration Challenges}

  \begin{itemize}\footnotesize
  \item Developers must build both WCT and \texttt{larwirecell} from Git.
  \item Building \texttt{larwirecell} requires WCT as \texttt{wirecell} UPS product.
  \item Dev's edit-build-test cycle requires injecting UPS product building!
  \item Done naively this would be complicated and painfully slow.
  \end{itemize}
  Solution: {\tiny (still rather complicated!)}
  \begin{itemize}\scriptsize
  \item Run FNAL's \texttt{buildFW} scripts used to produce UPS
    ecosystem (binary downloads but could potentially build some
    packages from source).
  \item Hack these scripts as needed, and esp. to define a new,
    temporary/bogus \texttt{wirecell} ``release'' version string.
  \item Clone, build and install WCT against UPS ecosystem and into
    \texttt{wirecell} UPS product area created above.
  \item Use \texttt{mrb} to build \texttt{larwirecell} against all the above.
  \item Hop back and forth between these last two shells, rebuilding/reinstalling
  \end{itemize}


  Works, but takes much effort to figure out and to replicate.

  Many thanks to \textbf{Lynn Garren} for the help to make this work!
\end{frame}

\begin{frame}
  \frametitle{Memory Issue (not a leak!)}
  \footnotesize
  Observe memory footprint (VmSize) issue
  \begin{center}
    \begin{tabular}[h]{ccc}
      1.5 GB & pure WCT job & no output file \\
      2.0 GB & pure WCT job & simple TH2 output\\
      3.0 GB & WCT + LS & no output file \\
      4.5 GB & WCT + LS & \textit{art} output file 
    \end{tabular}
  \end{center}
  \begin{itemize}
  \item Memory footprint ``should'' be no more than 1 GB
    \begin{itemize}\scriptsize
    \item naively: 2 $\times$ 300MB $\mu$B frames + $\sim$200MB config+lib data
    \end{itemize}
  \item See evidence pointing to memory \textbf{heap fragmentation/inflation}:
    \begin{itemize}\scriptsize
    \item[1] Allocate temporary working area
    \item[2] Allocate output, free working area
    \item[3] Job retains high water mark, with free'd ``holes''
    \end{itemize}
  \end{itemize}

  What do?
  \begin{itemize}
  \item More checking needed.  Some heap layout viewer would help.
  \item Some mitigation ideas for WCT but seem much is beyond my
    control.
  \item For now, I leave it but I wanted people to be aware.
  \end{itemize}
\end{frame}

\section{To Do \& Summary}

\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]  
\end{frame}


\begin{frame}
  \frametitle{To Do list}
  Near term:
  \begin{itemize}
  \item NF+SP: add writing of ``bad regions'' into \texttt{art::Event} (bv)
    \begin{itemize}\footnotesize
    \item Probably as a \texttt{recob::Wire} collection.
    \item Allow decoupling of NF \& SP and expose intermediate data products to
      LArSoft
    \end{itemize}
  \item Implement \texttt{SimEDep} $\to$ \texttt{IDepo} converter (Brooke, bv)
    \begin{itemize}\footnotesize
    \item Relies on some LS development to make \texttt{SimEDep} available.
    \end{itemize}
  \item Implement sim truth in WCT itself and saving to LS data
    product (Brooke, bv)
  \end{itemize}
  Longer term:
  \begin{itemize}\footnotesize
  \item Use WCT Sim+SP to investigate \textbf{protoDUNE data
      reduction}, implement corresponding LArSoft job config.
  \item Port \textbf{3D imaging} from WCP to WCT and develop its LS
    integration support (will need a converter for 3D image data).
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{WCT Release Roadmap}

  \begin{description}
  \item[0.5.2] Support for MB NF module (past release).
  \item[0.6.x] Support for NF+SP with new LS integration design.
    \begin{itemize}\footnotesize
    \item Expect to start on this after the collab meeting.
    \item Will also merge in \texttt{larwirecell}'s
      \texttt{feature/integration} branch for a LS release.
    \end{itemize}
  \item[0.7.x] Support for saving sim truth and multi-field response
    (eg, for ``touching wires'' in MB).
    \begin{itemize}\footnotesize
    \item Followed by \texttt{larwirecell} update with needed
      converters.
    \end{itemize}
  \item[0.8.x] 3D imaging in WCT.
  \end{description}

  \footnotesize
  \begin{center}
    Caveat: exact version strings may differ and priorities may change.    
  \end{center}

\end{frame}


\begin{frame}
  \frametitle{Summary}
  \begin{itemize}
  \item Many improvements and refinements in WCT.
    \begin{itemize}
    \item[$\to$] \textbf{``Ready for 3D imaging!''}
    \end{itemize}
  \item \textbf{WCT/LS integration is a solved problem}
    \begin{itemize}\footnotesize
    \item All issues are at least understood.
    \item NF+SP integration provides examples for future work.
    \item New WCT functionality now easy to integrate and maintain.
    \item New data converters only new integration coding needed.
    \end{itemize}
  \item NF+SP release RSN. Then, sim followed by 3D imaging.
  \item Memory inflation issue may be a (broader?) concern.
  \end{itemize}

\end{frame}



\section{}

\begin{frame}
  \begin{center}
    fin
  \end{center}
\end{frame}




\begin{frame}
  \frametitle{WCT Release Validation System}
  New \href{https://github.com/wirecell/wire-cell-validate}{\texttt{wire-cell-validate}} package.
  \begin{itemize}
  \item Define small data set (MB, pD, WCT Sim).
  \item Run a number of \textit{art} and \texttt{wire-cell} jobs.
  \item Also various plotting and comparison jobs.
  \item Generate JSON summaries and HTML reports.
  \item Package as Waf project for automation, parallel
    processing and idempotency.
  \item Rerun periodically and before releases to understand changes.
  \end{itemize}

\end{frame}



\end{document}


%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:
