% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{Data Access}
\renewcommand{\bvtitle}{\LARGE protoDUNE/SP Raw Data Access}
\renewcommand{\bvevent}{DAQ Collab Week Prep\\19 Jan 2017}
\renewcommand{\bvbeamerbackground}{}


\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Data Access Requirements}

\begin{frame}
  \frametitle{Data Access Requirements}
  \begin{center}
    This is mostly a summary of recent email threads.
  \end{center}

  
\end{frame}

\begin{frame}
  \begin{description}
  \item[file names] universally unique, human-oriented semantic labels 
    \begin{itemize}\footnotesize
    \item eg: \texttt{pdsp-raw-<runtype>-<runnum>-<subrun>.root}
    \item bonus if lexical sorting gives time-ordering!
    \end{itemize}
  \item[file size] $\sim$5 GB
  \item[file format] ROOT, likely uncompressed
  \item[metadata file] JSON formatted file holding summary values
    \begin{itemize}\footnotesize
    \item holds run, time/trigger range, detector config name, etc.  details to be determined.
    \item one JSON produced along side each ROOT file
    \item used for registering data file into file catalog (SAM)
    \end{itemize}
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{High-level File Schema}
  \begin{description}
  \item[fragment] one array for each from each board reader
  \item[grouping] subdetector fragments grouped together
  \item[fragment array] composed of artDAQ \textbf{header} + \textbf{payload}
  \item[header] common artDAQ packing schema across fragments
  \item[payload] BR-specific packing (more on this)
  \item[event] collection of fragments from a common trigger
  \item[file] collect N events to reach $\sim$5 GB/file
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Event Definition and Identifiers}
  \begin{description}
  \item[ident] each event uniquely defined by run and an event
    identifier
  \item[contiguous] events span non-overlapping times
  \item[span] event covers $\sim$5 ms around trigger
  \item[contents] TPC waveforms, PDS (self trigger + few us waveform),
    trigger and timing, possibly muon tagger data and some beam info.
    \begin{itemize}\footnotesize
    \item BI is still out-of-band?
    \end{itemize}
  \end{description}  
\end{frame}

\begin{frame}
  \frametitle{Raw Data Access Library - ``raw DAL''}
  \footnotesize
  \begin{description}
  \item[dependencies] access data with ``minimal'' set of software
    dependencies \\
    (eg, only ROOT?)
  \item[payload] DAQ-provided, shared code to pack and unpack\\
    \textbf{fragment payload} arrays.
  \item[header] artDAQ packs/unpacks \textbf{header arrays}, would like to have this
    code available also outside of artDAQ.
  \item[versioning] artDAQ header and payload arrays place a
    \textbf{packing schema version identifier} in a ``well known'' location in
    their arrays and update it for any change in the packing schema.
  \item[release] new version of raw DAL is tested, released and distributed
    before any new packing schema is deployed in production DAQ.
  \item[evolution] new versions of raw DAL must read old versions of data.
  \item[TPC] common interface to access RCE and FELIX APA data
  \item[granularity] unpack one APA w/out need to read/unpack others, \\
    unpack SSP w/out need to unpack APAs, etc.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{Numbering Conventions}

  \begin{itemize}
  \item Want to document existing numbering conventions in use right now by
    different groups.
  \item Try to define an \textbf{common convention} that is least
    disruptive to existing use.
  \item Provide \textbf{mappings} between domain-specific and common
    convention in both documentation and code.
  \item Ideally, DAL uses common convention o.w. provide mapping from
    DAQ-domain to common.
  \item I promised to organize this but it's not done,  Mea culpa!
  \end{itemize}
  
\end{frame}

\begin{frame}
  \frametitle{Some Work Needed Toward Data Access}

  \begin{enumerate}
  \item Numbering convention organization
  \item Initial packing schema and raw DAL implementation
  \item Test code to write example files with raw DAL
  \item Test reading into various clients
  \item ...
  \end{enumerate}
  
\end{frame}

\section{Raw Data Clients}

\begin{frame}
  \frametitle{Raw Data Clients}
  Before files, in artDAQ (both packing and unpacking)
  \begin{itemize}
  \item artDAQ DataLogger
  \item FELIX APA compressor
  \item Online Monitor
  \end{itemize}
  After files (unpacking or file-level)
  \begin{itemize}
  \item Fermilab-File Transfer System
  \item Detector Debugging Experts
  \item Prompt Processing
  \item Data Reduction
  \item Production Processing 
  \end{itemize}
  
  Some thoughts on each client follow if there is time....

\end{frame}

\begin{frame}

  \frametitle{artDAQ DataLogger}
  \begin{itemize}
  \item sees full events ``instantly''
  \item writes events to $\sim5$ GB ROOT file on online buffer disk
  \item artDAQ header packing, treats payload as opaque (?)
  \item contract with F-FTS:
    \begin{itemize}
    \item write SAM metadata JSON file, 
    \item provide ``file-ready'' notification (eg, via a \texttt{mv} to a watched directory).
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{FELIX Compressor}
  \begin{itemize}
  \item Uncompressed FELIX APA data $\to \sim2\times$ more raw data
  \item FELIX compression in artDAQ 
    \begin{itemize}
    \item Yes, please!
    \end{itemize}
  \item \textbf{needs raw DAL} for unpacking and packing
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Online Monitor}

  \begin{itemize}
  \item sees either fragments or fully built events ``instantly''
  \item inputs artDAQ stream, no data files
  \item will write summary/visualizing files
  \item \textbf{needs raw DAL} for unpacking
  \item potentially sees uncompressed FELIX APA data
  \end{itemize}
  
\end{frame}

\begin{frame}
  \frametitle{Fermilab File Transfer System}
  \begin{itemize}
  \item \textbf{most important client} of data files
  \item reads all artDAQ files from online buffer storage
  \item treats data file contents as opaque
  \item watches for ``file-ready'' notification
  \item ingests SAM metadata JSON file
  \item transfer to EOS, CASTOR and beyond
  \item \textbf{deletes} its input files (configurable conditions)
  \end{itemize}
\end{frame}


\begin{frame}
  \frametitle{Detector Debugger Experts}
  \begin{itemize}
  \item Needs \textbf{immediate access} ($\sim$minutes) to small
    number of particular raw files.
  \item Needs fast and lightweight access to data
  \item \textbf{needs raw DAL} for unpacking
  \item may use LArSoft, plain art or potentially stand-alone code
    (plain ROOT, dump out to allow Python or Mathematica)
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Prompt Processing}
  \begin{itemize}
  \item Inputs 1-10\% of files  from EOS or online buffer via XRootd
    (buffer access needs discussion/assurances)
  \item Wants low latency ($<$1min) file access and robust against
    20Gbps linke outage.
  \item Jobs must unpack artDAQ headers and payload
  \item \textbf{needs raw DAL} for unpacking
  \item may be LArSoft, plain art or potentially stand-alone code
  \item writes summary and visualization files for later display by
    p3s web server
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Data Reduction}
  \begin{itemize}
  \item Reduce data by 400x while keeping all useful
    signal after noise subtraction and signal processing (2.5 PB $\to$ 6 TB)
  \item Run it to keep up with data taking: 45s/trigger/APA.
  \item Still keep full raw data, can rerun reduction later.
  \item \textbf{needs raw DAL} for unpacking \textbf{reduced DAL} for packing
  \item may be LArSoft, plain art or potentially stand-alone code
  \item writes reduced data, probably in same format as artDAQ but
    different packing.  Uses F-FTS to archive/distribute.
  \item nominally ``shunts'' non wire data fragments as-is, to output
    but could do something more useful.
  \item would be a good place to merge in Beam Info.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Production Processing}
  \begin{itemize}
  \item Runs on reduced data, if we can run in ``keep up'' mode
  \item LArSoft based code with input module
  \item needs \textbf{raw DAL} or \textbf{reduced DAL} for unpacking
  \item Likely uses SAM to select input file sets.
  \end{itemize}
\end{frame}

\end{document}
