% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}

\usepackage[outline]{contour}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{\small Brett Viren and Maxim Potekhin}
\renewcommand{\bvtit}{PD Comp. Model}
\renewcommand{\bvtitle}{\LARGE ProtoDUNE Computing Items}
\renewcommand{\bvevent}{DUNE Collaboration\\ \small 21 May 2016}
\renewcommand{\bvbeamerbackground}{}

\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Documents}


\begin{frame}
  \frametitle{Documents}

  \begin{description}
  \item[\href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=914}{DocDB 914}] \textbf{DUNE Computing Model}, Recently updated.
    \begin{itemize}
    \item[$\rightarrow$] Needs more input, particularly from DP.
    \item \href{https://github.com/DUNE/dune-computing-model}{GitHub project page}
    \end{itemize}
  \item[\href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1209}{DocDB 1209}] \textbf{Requirements for Data Management System}.
  \item[\href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1212}{DocDB 1212}] \textbf{ProtoDUNE Data Management System}, final conceptual / early technical design.
  \item[\href{https://github.com/DUNE/protodune-tdr}{protoDUNE TDR git}] \textbf{ProtoDUNE TDR}.  In preparation, Computing and Software section (15 pages).
  \end{description}

\end{frame}

\section{Data Rates}
\begin{frame}
  \frametitle{Outline}
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}
  \frametitle{Initial SP Data Scenarios}
  As in \href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1086&version=4}{DocDB 1086v4}, lots of caveats:
  \begin{itemize}
  \item Three scenarios, ``low'', ``high'' and ``Goldilocks''.
  \item 5M triggers with 100\% efficiency and purity.
  \item No accounting for dedicated cosmic-$\mu$ or other non-beam running.
  \item No zero-suppression (ZS) with 35t-like compression ($2\times$)
  \item low/Goldi: short readout time (2ms) and not all APAs.
  \item[$\rightarrow$] \textbf{total data: 75 TB - 1.2 PB}
  \end{itemize}
  Some takeaways.
  \begin{itemize}
  \item Even ``low'' requires parallel I/O or fast disks.  
    \begin{itemize}
    \item \textbf{DAQ must be designed to handle parallel writes!}
    \end{itemize}
  \item ``High'' finishes 5M events in 2 days, (no accounting for beam reconfiguration time included here).
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Improved SP Scenarios}
  As in \href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1086&version=5}{DocDB 1086v5}, the changes:
  \begin{itemize}
  \item Assume MicroBooNE noise levels or better: $5\times$ compression.
  \item Take one cosmic-$\mu$ trigger per beam trigger (out of spill).
  \item 5ms readout time for all scenarios.
  \item[$\rightarrow$] total data: 230 TB - 345 TB.
  \end{itemize}
  Still uncertain:
  \begin{itemize}
  \item What efficiency and purity can we expect from beam veto?
  \item How much laser data?  
  \item Improve on 1-cosmic-$\mu$ trigger per 1-beam trigger estimate?
  \item Is 10Hz trigger right?  If faster, still only 5M triggers?
  \end{itemize}
  Better engagement with and input from \textbf{Measurement and Calibration
  groups} needed.
\end{frame}

\begin{frame}
  \frametitle{Contingency for SP Scenarios}
  \textbf{What if}\footnote{Numbers are non-exhaustive, wild-ass guesses.}:
  \begin{enumerate}
  \item make up for delays by raising 10Hz$\to$50Hz trigger rate?
    \begin{itemize}\scriptsize
    \item But still 5M total triggers: parallel-I/O and buffer size limits.
    \end{itemize}
  \item veto is 50\% pure so run longer to get 10M triggers?
    \begin{itemize}\scriptsize
    \item Problem pushed downstream to CERN EOS/CASTOR and FNAL ENSTORE.
    \end{itemize}
  \item we see 35t noise levels instead of $\mu$B's?
    \begin{itemize}\scriptsize
    \item Compression goes from $5\times \to 2\times$, need extra disk I/O and buffer size.
    \end{itemize}
  \end{enumerate}
  Overall worse cases:
  \begin{description}
  \item[volume] 2\&3 $\Rightarrow 10\times \to 3.3$ PB total raw data.
  \item[parallel I/O] 1\&3 $\Rightarrow 25\times \to 65$ parallel writes, sustained
  \item[buffer] 1\&3 $\Rightarrow 25\times \to 850$ TB 3-day disk buffer.
  \end{description}
  
\end{frame}

\begin{frame}
  \frametitle{DP Data Estimates}
  Numbers from \href{https://indico.fnal.gov/materialDisplay.py?contribId=12&sessionId=6&materialId=slides&confId=10612}{Dario Autiero}:
  \begin{itemize}
  \item $\mathcal{O}$(100M) beam triggers + cosmic ray + tests.
  \item $10\times$ compression via Huffman coding, no ZS.
  \item 15 MByte/trigger
  \item[$\rightarrow$] 2.4 PB total 
    \begin{itemize}
    \item[=] 3.8 Gbps sustained for two months.
    \end{itemize}
  \end{itemize}

\end{frame}

\section{SP \& DP Differences and Symmetries}

\begin{frame}
  \frametitle{Outline}
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}
  \frametitle{SP \& DP strategies: differences}

  \begin{center}
    \includegraphics[width=0.7\textwidth]{ftsinstances.pdf}
  \end{center}

  \vspace{-8mm}


  SP:
  \begin{itemize}  \footnotesize
  \item \textbf{Transfer all raw data} to FNAL for processing.
    \begin{itemize} \scriptsize
    \item CERN$\to$FNAL bandwidth $>10\times$ SP$\to$CERN bandwidth.
    \item Nominal throughput requires relatively modest disk buffer.
    \end{itemize}
  \item Promptly process small fraction of data at CERN for QA/QC.
  \item Use \textbf{F-FTS} for buffer$\to$EOS transfer.
  \end{itemize}
  DP:
  \begin{itemize} \footnotesize
  \item \textbf{Promptly reconstruct all raw data} right next to detector.
    \begin{itemize} \scriptsize
    \item Requires substantial storage and CPU farm.
    \end{itemize}
  \item The DP buffer$\to$EOS transfer method is as yet \textbf{needs clarity}.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{SP \& DP strategies: symmetries}
  \begin{center}
    \includegraphics[width=0.7\textwidth]{ftsinstances.pdf}
  \end{center}


  Both detectors will:
  \begin{itemize}
  \item \textbf{transfer all raw data} to CERN and FNAL tape.
  \item use \textbf{F-FTS} for CERN$\to$FNAL transfers.
  \item use same \textbf{file catalog} (F-FTS $\Rightarrow$ SAM)
  \end{itemize}
  Both detectors might share a common:
  \begin{itemize}
  \item disk buffer technology (if not hardware).
  \item production workflow management system. 
  \end{itemize}

\end{frame}

\section{Technology Choices}

\begin{frame}
  \frametitle{Outline}
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}[fragile]
  \frametitle{Fermilab File Transfer System}
  Selected F-FTS for Data Management System
  \begin{itemize}
  \item \href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1209}{Requirements document}
  \item \href{https://groups.google.com/forum/#!topic/hep-sf-tech-forum/wIlfx31r8UM}{HSF technical forum query.}
  \item Responses from LHCb/DIRAC and Fermilab/F-FTS.
  \item Reviewed documents and engaged experts in discussion.
  \item F-FTS pursued due to existing use in neutrinos @ FNAL,
    existing and further planned integration with CERN computing.
  \item Detailed technical questioning over email $\rightarrow$ all positive responses.
  \item In-person + phone meeting at FNAL with
    experts\footnote{Many thanks to Andrew Norman, Robert Illingworth,
      Oliver Gutsche} and nailed down initial design and document.
  \item \href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1212}{Final conceptual design}.
  \end{itemize}
  To do: tests between FNAL, CERN, BNL/RACF, LBNL/NERSC

\end{frame}

\begin{frame}
  \frametitle{Detector Storage Buffers}

  \begin{itemize}
  \item CERN requires 3 day buffer living close to detector.
  \item DP/SP require different scales, both require parallel I/O.
  \item Jacques Marteau's R\&D results on BeeGFS and XRootD, note expected soon.
  \item LBNL interested in setting up early server at CERN which can
    serve as initial test bed for transferring from buffer via F-FTS.
  \item BNL will evaluate XRootD to receive data for for SP buffer.
  \end{itemize}
  $\rightarrow$ \textbf{But wait, some breaking new ideas coming up.}
\end{frame}

\begin{frame}
  \frametitle{Other Technological Decisions / Tasks}
  \begin{itemize}
  \item Monitoring for data transfer and data QA/QC:
    \begin{itemize}
    \item ``Offline Data Monitor'' (LBNL/Daya Bay)
    \end{itemize}
  \item Workflow management system for production processing,
    exploitation of CPU outside of FNAL+CERN (EU Grid, OSG)
  \item Data federation for fast and efficient access to data by all
    collaborating institutions.
  \item Various offline related items on \href{https://listserv.fnal.gov/scripts/wa.exe?A2=ind16&L=dune-fd-daq&F=&S=&P=16539}{Giles Barr's recent list to \texttt{dune-fd-daq@fnal.gov}}.
  \end{itemize}
\end{frame}

\section{New Ideas / Discussion}

\begin{frame}
  \frametitle{Outline}
  \tableofcontents[currentsection]
\end{frame}

\begin{frame}
  \frametitle{Let's Unify DAQ End Nodes with Buffer Nodes}
  Hallway discussion with Giles:
  \begin{itemize}
  \item XRootD provides a nice conceptual separation between
    distributed artDAQ and buffer farm.
  \item \textbf{But} it makes the system more complicated
    \begin{itemize}
    \item DAQ has to learn about XRootD
    \item DAQ is already distributed, then why XRootD for dist. I/O?
      \begin{itemize}
      \item (well, the ``interleave problem'' but see next)
      \end{itemize}
    \end{itemize}
  \item So, let's put all this on each buffer node:
    \begin{itemize}
    \item artDAQ endpoint output module
    \item F-FTS ingest dropbox
    \item Some JBOD disks (not RAID)
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{The ``Interleave Problem''}
  \begin{itemize}
  \item Users want sequential triggers for convenience.
  \item Distributed DAQ builds events asynchronously.
  \item A DAQ event builder node does not receive sequential triggers.
  \item For DAQ to provide sequential triggers requires it to
    concentrate all stream into one node with 
    $\sim$1 file's worth of data $\times$ N output streams.
  \item This limits scalability, increases cost.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{The ``Interleave Opportunity''}

  Let's manage our raw data like this:

  \begin{enumerate}
  \item Let DAQ write interleaved events.
  \item Write to local disk buffer which  also has F-FTS inbox.
  \item Save this to permanent archive.
  \item Preprocess this data to:
    \begin{enumerate}
    \item Read N interleaved streams.
    \item Deinterlace.
    \item Apply software noise removal.
    \item Apply detector response deconvolution.
    \item Apply thresholding
    \item Write 1 stream
    \end{enumerate}
  \item Production reco processing.
  \end{enumerate}

  The point is to ``fix'' interleaving and also produce much smaller
  input files to the production processing.
\end{frame}

\end{document}



%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:

