% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{title}
\renewcommand{\bvtitle}{\LARGE Online Software\\Application Configuration and Deployment}
\renewcommand{\bvevent}{{\small EVENT -- DATE}}
\renewcommand{\bvdate}{DD Mmm YYYY}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Introduction}
\begin{frame}
  \begin{center}
    Caveat: I'm not sure what I'm supposed to say!

    So, I'll just give some ideas and discussion starters.
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Current DAQ Subsystem Concepts}
  \begin{center}
    \includegraphics[height=0.6\textheight]{everybody.pdf}    
  \end{center}

\end{frame}

\begin{frame}
  \frametitle{Incomplete List Of DAQ Apps}
  \begin{description}
  \item[CCM] run number service, various configuration, partition and other manager services, RC itself, expert systems and human UIs, log aggregators, metric monitors, configuration sequencing, detector electronics config service(?).
  \item[FE] TP source, buffer interface service.
  \item[DS] TC processors, MLTs, ELTs.
  \item[BE] Data flow orchestrators, event builders.
  \item[DQM] Suite of ``live'' data quality processing and visualizations.
  \item[ad-hoc] Detector debugging, data stream taps, one-off monitoring.
  \item[post-hoc] Nearline processing / data selection.
  \end{description}
\end{frame}


\section{Configuration}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Categories of Configuration}

  \begin{description}
  \item[initial] initial parameters given at execution time to \textbf{DAQ applications}, not tied to run changes.
  \item[dynamic] (re)configuration of \textbf{DAQ applications} at subrun\footnote{Data Model definition} boundaries.
  \item[services] ``\texttt{/etc}'' files for any supporting services (SSHd, PTP, Supervisord, Ganglia, CVMFS, Apache, RDBMS, ....).
  \item[OS] definition of base set of packages and their versions, Linux boot/tuning parameters.
  \end{description}
\end{frame}


\begin{frame}
  \frametitle{Some questions we must answer}
  \begin{itemize}
  \item How do we manage the parameter set(s), deploy them and schedule, maintain history and rollback those deployments?

  \item How do we handle interdependencies? eg a subrun change may require new DAQ applications to execute.

  \item What is \textbf{definitive source} of configuration information and what are \textbf{derivatives} and how to go from source to target?

  \end{itemize}
  
\end{frame}

\begin{frame}
  \frametitle{Jsonnet Based Configuration Management}
  \begin{itemize}\footnotesize
  \item Jsonnet is a sane JSON-like functional+OO, data structure language.
  \item Established language, used to configure Kubernetes and Wire-Cell Toolkit
    \begin{itemize}\scriptsize
    \item[o] WCT can provide Jsonnet code for complex graph construction.
    \item[o] Jsonnet syntax is simple and somewhat FHiCL'esque.
    \end{itemize}
  \item Directly supports compiling to JSON, INI, YAML
    \begin{itemize}\scriptsize
    \item[o] easily generate arbitrary formats with JSON+Jinja2 templates\\
    eg as used in \texttt{dune-tdr}/\texttt{dune-params}.
    \end{itemize}
  \item A single, central Jsonnet file set could build out to forms taken by all configuration consumers.
  \item Encourages Don't-Repeat-Yourself patterns, consistency as well as factoring of configuration effort/duties.
  \item Could maintain Jsonnet source in Git for history, branching, rollback.
  \item The approach decouples the development and the consuming of configuration information, helps when debugging and developing apps.
  \end{itemize}
  \footnotesize

  The model: the git repo represents all past and possible future configurations.

  The point in that commit history actually applied at any time is determined and recorded by CCM.
\end{frame}

\section{Deployment}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Application Deployment Scope}

  \begin{itemize}
  \item Assume Computing provides/manages OS and ``some'' services.
    DAQ+Computing must formalize the division.
  \item DAQ application software binaries must be available on DAQ host computers.
  \item Expect to need multiple versions available for execution and to promptly add new ones.
  \item Must determine mechanisms to execute, monitor and reap executing apps and to define, construct, debug, monitor and archive their execution environments.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Some Deployment Technologies to Evaluate}
  \begin{description}
  \item[Ansible] central configuration files pushed to hosts over SSH, services restarted, can reboot, act across host categories or ad-hoc/targeted.
  \item[Puppet] like Ansible, but hosts run daemon that polls master.  
  \item[Supervisord] daemon to run other applications, can respond to events and be monitored via XML-RPC, batteries mostly included but also extensible via Python.
  \item[Containers] bake OS or OS+apps into image, run multiple containers on a host.
  \item[Kubernetes] large scale container orchestration.
  \end{description}
\end{frame}


\begin{frame}
  \frametitle{Launching and Reaping DAQ Applications}

  Normal starts, ``self healing'' and ``zero-downtime'' require:

  \begin{itemize}
  \item Launching a large and varied set of DAQ applications.
  \item Specifying their operating environment.
  \item Matching apps to hosts given \textbf{constraints}.
  \item Monitoring their execution and detect crash (IPC presence).
  \item Potentially reaping errant instances.
  \item Minimizing delay between launch request and application execution.
  \end{itemize}

  It occurs to me this closely resembles \href{http://news.pandawms.org/panda.html}{ATLAS's PanDA} or other ``pilot based'' batch system.

\end{frame}

\begin{frame}
  \frametitle{Fast Whole-DAQ Bootup}
  \begin{itemize}
  \item Want to recover fast and easy from full power outage.
  \item Remote power on for all computers?
  \item Minimize reliance on waiting for HDD fsck'ing.
    \begin{itemize}
    \item Diskless servers, PXE boot, NFS root are possible.
    \item Or, take care in OS install, read-only root FS on SSD.
    \end{itemize}
  \end{itemize}
\end{frame}


\section{Interactions}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{No tight coupling}
  \begin{itemize}
  \item Don't have a major DAQ application (eg, RC) talk directly to a database.
  \item Have RC \textbf{PUB}lish information and another app \textbf{SUB}scribe in order to fill a database.
  \item Have RC query a configuration application, which may front a DB or other source of configuration (eg, Jsonnet).
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Example: Access FE Buffer}

\begin{verbatim}
  10s RAM buffer <--> BIS <--> BR
                 <-->debugging (``oscilloscope'')
                 <-->Ar39 special "patch" readout
\end{verbatim}  
\end{frame}

\section{DAQ Host Usage Policy}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Who can access what and how?}
  \begin{itemize}
  \item Disallow any human to log in to a production account.
    \begin{itemize}\scriptsize
    \item[o] Eg: take model how Apache, PostgreSQL, etc is run on most OSes.
    \end{itemize}
  \item Limited number of expert accounts on DAQ hosts.
    \begin{itemize}\scriptsize
    \item[o] SSH key access control.
    \item[o] Used for testing of new versions, debugging, ad-hoc application execution.
    \end{itemize}
  \end{itemize}
\end{frame}

\section{}

\begin{frame}
  \begin{center}
    \Huge $\mathcal{FIN}$    
  \end{center}
\end{frame}

\begin{frame}
  \frametitle{Reminder: Zero-downtime Reconfiguration Mechanism}

  \begin{enumerate}\footnotesize
  \item RC decides to change a subrun some time in the near future.
  \item Determine target \textbf{data time} for it to apply.
  \item Any newly required DAQ apps are started via RC command.
  \item RC forms reconfiguration commands with \textbf{data time} stamp.
  \item Send reconfig commands to all DAQ applications (old and new).
  \item Both old/new apps immediately form any new connections, keeping existing ones nominal.
  \item New apps accept input to monitor for \textbf{data time} but otherwise idle.
  \item Old apps operate nominally.
  \item When \textbf{data time} is reached, apps finalize execution of reconfiguration command, flush buffers, drop any unwanted connections and terminate if instructed..
  \end{enumerate}
\end{frame}

\end{document}

