% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{Buffer }
\renewcommand{\bvtitle}{\LARGE NP04 Online Buffer System\\Requirements and Options}
\renewcommand{\bvevent}{DUNE Collab\\Sep 2016}
\renewcommand{\bvbeamerbackground}{}

\newcommand{\microboone}{MicroBooNE\xspace}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Requirements}

\begin{frame}
  \frametitle{NP04 Online Buffer System Context}
  \begin{center}
    \includegraphics[height=4cm]{context.pdf}

    \textbf{The concept:} place sufficient storage ``close'' to DAQ to accept
    all data files in order to isolate from potential CERN network
    and/or EOS outage.
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{High-level Performance Requirements}



  \begin{description}
  \item[throughput] accept in-spill ``high rate'' data scenario throughput
    \begin{itemize}
    \item[$\rightarrow$] \textbf{3 GByte/s}\footnote{Detailed scenarios and estimations in \href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1086}{DocDB 1086}.}
    \end{itemize}
  \item[capacity] cover expected outage duration
    \begin{itemize}
    \item[$\rightarrow$] \textbf{1-3 days}\footnote{Rumor, need to nail down!}
    \end{itemize}
  \item[latency] small compared to prompt-processing
    \begin{itemize}
    \item[$\rightarrow$] \textbf{$\lesssim$1 minute}\footnote{A guess, want 10 minute turn-around for any prompt-processing.}
    \end{itemize}

  \end{description}

\end{frame}

\begin{frame}
  \frametitle{Component Hardware Options}
  Some simple assumptions:
  \begin{description}
  \item[Storage] SATA:50 MB/s, SAS:200 MB/s, SSD:500MB/s, \\all half-duplex.
  \item[NICs] 1Gbps or 10Gbps, full-duplex, theoretical max.
  \item[Switch] match NICs but $\sim\infty$ fabric bandwidth.
  \item[CPU] various benchmarks (eg, artDAQ, \texttt{md5sum}).
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Aside: DAQ Minimum Multiplicity}

  \textbf{Minimum}\footnote{Details in \href{http://docs.dunescience.org:8080/cgi-bin/ShowDocument?docid=1656}{DocDB 1656}.} to keep up with \textbf{steady-state} 3 GB/s:
  \begin{itemize}
  \item 4 EB computers with 10 Gbps NICs.
  \item 2 EB artDAQ processes/comp each pushing 375 MB/s 
    \begin{itemize}
    \item[$\rightarrow$] artDAQ benchmark demonstrates 440 MB/s !!
    \end{itemize}
  \item Need $2\times1.5$ core for artDAQ + $2$ for \textbf{external} checksum
    \begin{itemize}
    \item Do we need this?  We expect internal checksums exist as well \\
      (at DAQ-level and/or ROOT-level).
    \end{itemize}
  \item[$\Rightarrow$] 5 GB file every $\sim 13$ seconds from each EB proc.
  \end{itemize}

  Two competing consequences of assumptions:
  \begin{itemize}
  \item[+] Absorb real-world fluctuations require more ($\sim2\times$?).
  \item[-] See full 3~GB/s only during the spill ($\sim$20\% of the time).
  \end{itemize}



\end{frame}


\begin{frame}
  \frametitle{Network Edges of Online Buffer System}
  \begin{center}
    \includegraphics[height=4cm]{interface.pdf}
  \end{center}
  Assume 20 Gbps network bandwidth \textbf{from DAQ} and \textbf{to EOS}.
  \begin{itemize}
  \item[$\rightarrow$] 3 GB/s = 24Gbps, so we are \textbf{rounding down} here!
  \item[$\rightarrow$] As-is, \textbf{the network is the bottleneck}.
  \end{itemize}
  \vfill
  \footnotesize
  Note: in ``concentrated'' design, DAQ$ \to $Buffer connection is more intimate.
\end{frame}

\section{Designs}


\begin{frame}
  \frametitle{Transport and Bookkeeping}
  High level tech decisions: \textbf{XRootD} and \textbf{F-FTS}/\textbf{SAM}:
  \begin{enumerate}
  \item \textbf{DAQ} transfer data into buffer via \textbf{XRootD} or \textbf{local FS}
    \begin{itemize} \footnotesize
    \item ``\textbf{distributed}'' buffer: XRootD ingest
    \item ``\textbf{concentrated}'' buffer: local FS writes
    \item Nominally DAQ also produces \& transfers ``metadata'' file.
    \end{itemize}
  \item \textbf{F-FTS} learns of newly buffered files:
    \begin{itemize}\footnotesize
    \item via \textbf{push-notify} or \textbf{delay-poll} 
    \end{itemize}
  \item \textbf{F-FTS} initiates an \textbf{XRootD} transfer to EOS.
    \begin{itemize} \footnotesize
    \item speak EOS's native tongue: XRootD
    \end{itemize}
  \item \textbf{F-FTS} registers file location and other metadata with \textbf{SAM}'s file catalog.
  \item \textbf{F-FTS} initiates file purge from buffer.
    \begin{itemize}
    \item purge cooperates with any other users of buffered files
    \end{itemize}
  \end{enumerate}
  \textbf{See Stu's presentation Tomorrow morning in
    Software/Computing session for more about F-FTS and Onward.}
\end{frame}

\begin{frame}
  \frametitle{Hardware Spectrum}
  \begin{center}
    \begin{tabular}{r|c|c}
      & \textbf{Distributed} & \textbf{Concentrated} \\
      \hline
      NIC & 1 Gbps & 10 Gbps \\
      HDD & 120 SATA & 90-360 SSD / 45 SAS \\
      \#nodes & 60 & $\sim$2 FE + 4 storage \\
      tech & commodity & leading edge \\
      h/w & generic/existing & specialized \\
      cost & US\$ 25K (+repurpose) & US\$ 100K+ \\
    \end{tabular}
  \end{center}

  ``\textbf{Distributed}'' - flat scaling
  \begin{itemize}
  \item[$\rightarrow$] \textbf{pros}: simpler, cheaper/existing systems, fine-grained redundancy, more
    available idle CPU.
  \item[$\rightarrow$] \textbf{cons}: larger rack footprint, power,
    cooling.
  \end{itemize}
  ``\textbf{Concentrated}'' - hierarchical scaling
  \begin{itemize}
  \item[$\rightarrow$] \textbf{pros}: fewer boxes to deal with, higher
    performance per unit.
  \item[$\rightarrow$] \textbf{cons}: cost, larger unit of redundancy.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Distributed Option: the \texttt{neut-buffer} Cluster}

  The idea:
  \begin{itemize}\footnotesize
  \item Buy 120 SATA disks to upgrade 60 nodes of existing ``\texttt{neut}'' cluster ($\sim$ \$25k).
  \item Buy a switch ($60\times 1 \mathsf{Gbps} + 4\times 10\mathsf{Gbps}$) ($\sim$ \$5k), or 2 smaller switches.
  \item Test in-situ, extract nodes and move to NP04 hall for production running.
  \item $60 \times 7.5$ cores potentially available (eg, for checksumming).
  \item Loss of one node represents $\sim 2\%$ throughput degradation.
  \item Option for DAQ to stream data into buffer or to copy from temp DAQ FS.
  \end{itemize}
  Status:
  \begin{itemize}\footnotesize
  \item Nectarios Benekos has large fraction of \texttt{neut} online.
    \begin{itemize}\footnotesize
    \item[$\rightarrow$] Ready for purchasing HDD upgrades.
    \end{itemize}
  \item Maxim Potekhin has installed and configured XRootD on \texttt{neut}.
    \begin{itemize}\footnotesize
    \item[$\rightarrow$] Ready to start some initial throughput testing.
    \end{itemize}
  \end{itemize}

  Independent of buffer decision: do we have a need to ``fill up''
  \texttt{neut} with some disk?

\end{frame}

\begin{frame}
  \frametitle{Open questions for the \texttt{neut-buffer} Cluster}
  \begin{itemize}
  \item Can we get agreement to use $\sim$20\% of the \texttt{neut}\footnote{\href{https://twiki.cern.ch/twiki/bin/view/CENF/NeutrinoClusterCERN}{neut twiki} for more info} cluster hardware for protoDUNE/SP buffer?
  \item Will we have rack space, power, cooling at NP04 hall for 60~``U'' of rack space?
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Concentrated option}
  \Put(200,-10){\includegraphics[height=5cm,page=1,angle=270,clip,trim=0 0 0 0]{atlas.pdf}}

  The idea:\footnote{Giovanna Lehmann Miotto}
  \begin{itemize}\footnotesize
  \item Buy new, high-end storage hardware.
  \item Each front end mounts its storage units\\
    as ``local'' FS.
  \item Used successfully in ATLAS, existing experience.
  \item Option: use same hardware to host EB (and maybe also BR) artDAQ
    processes as well as an F-FTS instance.
    \begin{itemize}\footnotesize
    \item[$\rightarrow$] No distinct buffer ``system'' per se.
    \end{itemize}
  \end{itemize}
  Status:
  \begin{itemize}\footnotesize
  \item Candidate hardware identified.
  \item Initial understanding of multiplicity and costs.
  \item Discussions underway to understand the idea more.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Open questions for Concentrated Option}
  
  \begin{itemize}
  \item What testing needs to be done before acceptance of a design?
  \item How many spares and of which parts are needed to reduce risk
    of hardware failure or to allow quick fixing?
  \item Is the DELL proprietary storage management software
    acceptable?
  \item Is it acceptable to mix the different DAQ+Buffer layers
    (BR+EB) on the same hosts?
  \end{itemize}
\end{frame}

\section{Discussion}

\begin{frame}
  \frametitle{Important protoDUNE-SP Milestones for Buffer}
  \begin{description}
  \item[May 2017] APA 1 integration tests
    \begin{itemize}
    \item[$\rightarrow$] need \textbf{some} buffer storage in place prior to this.
    \end{itemize}
  \item[Aug 2017] APA 2 integration tests
  \item[May 2018] Filling and commissioning
    \begin{itemize}
    \item[$\rightarrow$] full buffer system in place, ready to use before commissioning starts!
    \end{itemize}
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Open Questions for Discussion}
  \begin{itemize}
  \item How many days must we \textbf{really} buffer?  3?  1?
  \item What end of the hardware spectrum do we take?
    \begin{itemize}
    \item concentrated vs. distributed
    \end{itemize}
  \item Who will take charge of hardware procurement and installation?
  \item Who will provide sysadmin duties?
    \begin{itemize}\footnotesize
    \item See also Geoff Savage's questions from morning session.
    \end{itemize}
  \item All estimation based on steady-state throughput, need to think
    more about fluctuations and back-pressure?
  \item ...what else?
  \end{itemize}
  Let's try to decide \textbf{concentrated} vs. \textbf{distributed}
  quickly so we can move on to testing and start thinking about
  operation.
\end{frame}


\end{document}

