% Created 2021-06-11 Fri 12:25
% Intended LaTeX compiler: pdflatex
\documentclass[presentation]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{minted}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\hypersetup{colorlinks}
\setbeamerfont{alerted text}{series=\bfseries}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{Testing environment, interface to data storage and databases}
\subtitle{Based on experience with PDSP CE test setup at BNL}
\institute[]{\includegraphics[width=3cm, clip, trim=0 8cm 0 8cm]{../../../../common/BNL_Logo_Horizontal_RGB.pdf}}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={Testing environment, interface to data storage and databases},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 27.2 (Org mode 9.4.4)}, 
 pdflang={English}}
\begin{document}

\maketitle

\begin{frame}[label={sec:org5d30ea6}]{Overview}
Some opinionated suggestions for the architecture of DUNE ASIC/FEMB
test stands.

\begin{itemize}
\item Basic idea is to scale out what was done for BNL PDSP CE testing lab

\item And to seek to do better in places.

\item See also 2018-07-18 BNL workshop slides.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org8af487b}]{}
\includegraphics[width=\textwidth, page=3]{cetesc.pdf}
\end{frame}

\begin{frame}[label={sec:orgd296b6c}]{The good and the bad with the PDSP CE Testing}
\footnotesize
Good:

\begin{itemize}
\item Ample CPU, storage, RAM, computing and network hardware.
\item Strongly controlled system deployment (Ansible) but allow ``get `er
done'' responses to emergencies.
\item Common file storage patterns, summary maker, web pages.
\item Shared software development, good code repo hygiene, build, release
management, etc.
\item Close coordination between test operators, managers and computer
systems stuff.
\end{itemize}

Not-so-good:

\begin{itemize}
\item More attention was needed to software quality and design (effort/time problem)

\begin{itemize}
\item \scriptsize Unnecessary output file format/schema variation and bugs requiring
special post-processing to fix.

\item Code complexity (python calling python calling ROOT/C++ app)
\end{itemize}

\item The rigorous data provenance system (Sumatra) was nice/cool but we
never really used it and so its complexity was not payed back.

\item The web page summaries were useful but their automatic generator was
a bit of a hack which had scaling issues as more data was taken.

\item Validation of human data input could have been better.

\begin{itemize}
\item \scriptsize Rare, but annoying, typos in IDs could have been caught
\end{itemize}
\end{itemize}
\end{frame}




\begin{frame}[label={sec:org0da610f}]{}
\includegraphics[width=\textwidth, page=4]{cetesc.pdf}
\end{frame}


\begin{frame}[label={sec:org83fec12},fragile]{System Configuration}
 We want autonomous but ``functionally similar'' testing sites.

Rigorous role definitions:
\begin{itemize}
\item Map role to Unix user, enforce login / file permissions.
\item \emph{eg.} \texttt{oper} runs tests, \texttt{inst} installs releases, \texttt{arch} archives data
\item Only \texttt{oper} logs into console for data taking, others via SSH
\item ``Normal work'' not allowed in these roles,
\begin{itemize}
\item Use personal account for development.
\end{itemize}
\end{itemize}

Use Ansible or similar:

\begin{itemize}
\item Maintain central repo for Ansible recipes and testing software.

\item OS config, role/user account setup and software build/deploy.

\item Each site admin updates their lab in coordination with local
operators and global mgt.
\end{itemize}
\end{frame}



\begin{frame}[label={sec:org4962104}]{Remaining problem layers: ``The Five S's''}
\begin{block}{Sampling}
Software to perform the basic test data acquisition.
\end{block}

\begin{block}{Store}
How we store and organize the acquired data.
\end{block}

\begin{block}{Summarize}
Extract important values from all samples into unified representation.
\end{block}

\begin{block}{Sync}
Copy samples and summaries to a central location.
\end{block}

\begin{block}{Serve}
Distribute and display summaries, statistics, inform hardware selections.
\end{block}
\end{frame}


\begin{frame}[label={sec:org71c8356},fragile]{Sampling - the data acquisition software units}
 Develop single Python package with multiple modules.
\begin{itemize}
\item One module per each test, functions take/return Python objects
\begin{itemize}
\item no file I/O in test modules
\end{itemize}
\item Avoid compiled code, no ROOT macros
\item Define limited number of supported output formats
\begin{itemize}
\item Numpy NPZ for arrays + JSON for structure
\item or consider HDF5 for both.
\item Avoid ROOT format, but if needed, use \texttt{uproot} over PyROOT.
\end{itemize}
\end{itemize}

Single ``main'' program via \texttt{click} module or similar
\begin{itemize}
\item Provides per-test sub-commands
\item Enforces object interface to test module functions.
\item Enforces \emph{storage} patterns (next)
\end{itemize}

Experts call directly, operators have ``scripted'' layer on top.
\end{frame}

\begin{frame}[label={sec:org7f6d6d8},fragile]{Store - organize test data}
 Large disks, \alert{local} to each DAQ computer but all with a common file
system tree \alert{pattern}.

Some bike shedding on the pattern is useful but like:

\begin{verbatim}
/data/samples/{hostname}/{testtype}/{uniqueID}/
\end{verbatim}

Each run of the single \texttt{main} CLI:
\begin{itemize}
\item generates path, assures it is unused, creates the directory.
\item sets path as \texttt{\$CWD} calls test module
\item writes output from test module function to \texttt{\$CWD}
\end{itemize}

The \texttt{uniqueID} may be a seconds-level timestamp or a monotonic counter.
\end{frame}

\begin{frame}[label={sec:orgfb12e23},fragile]{Summarize - distill important result info}
 After a ``sample'' is taken, run a separate post-processing analysis.
\begin{itemize}
\item Per-test code + common code
\item Generate ``standardized'' summary file + test-specific.
\item Develop this code in same package as for ``sample''
\item Same software patterns: per-test summary module, unified CLI
\end{itemize}
Same file system pattern but with
\begin{itemize}
\item \texttt{/data/summaries/} root
\item A \texttt{/\{version\}/} leaf to allow for future re-running
\end{itemize}

This can run synchronous after ``sampling'' or periodically eg via \texttt{cron}.
\end{frame}

\begin{frame}[label={sec:org721109f},fragile]{Sync - collect get data from sites}
 Periodically sync \texttt{samples/} and \texttt{summaries/} from each DAQ computer at
each site to a central location.

\begin{itemize}
\item ``Central'' likely should mean FNAL.

\item Archive \texttt{samples/} to tape (read-back unlikely to ever happen)

\item Keep \texttt{summaries/} on central file system for collaboration access and
as input to \emph{serving} (next)
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org0612a30},fragile]{Serve - browse, query, view summaries via WWW}
 Deploy to \alert{local sites} and \alert{centrally}

\begin{itemize}
\item The \texttt{summaries/} area provides read-only ``database''

\item Enable local site and global decision making
\end{itemize}

Summary views include at least:

\begin{itemize}
\item Meta-summaries, historical trends, cross-site slices of given test types

\item Ability to drill down to full summary of individual test runs

\item Plan to develop novel views all though operation!
\end{itemize}

Leave the detailed DB and web tech choices to the developers.

\begin{itemize}
\item Flask is an excellent choice for a web app framework.

\item If RDBMS is needed, consider SQLite3.  Server-based RDBMS is not
warranted and will require additional effort with no payback.

\item No reason to disallow multiple independent ``services'', but
centralize their repos and deployment.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org0d8ac8c}]{Some main take-aways}
\begin{itemize}
\item The BNL CE testing model scaled out to multi sites should work.
\item Each site to look like the BNL CE test lab did but with:
\begin{itemize}
\item Centrally stored but independently applied configuration mgt.
\item Rigorous role/user access and file permissions.
\item Independent data egress from site to central location (FNAL)
\end{itemize}
\item Identify separate stages of data process
\begin{itemize}
\item Implement in a coherent software development process.
\end{itemize}
\item Put in effort NOW to provide initial Python scaffolding/toolkit.
\begin{itemize}
\item Or, better, go back to 2018 and start then!.
\end{itemize}
\end{itemize}
\end{frame}
\end{document}
