% Created 2021-02-12 Fri 18:08
% Intended LaTeX compiler: pdflatex
\documentclass[presentation]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{minted}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\hypersetup{colorlinks}
\setbeamerfont{alerted text}{series=\bfseries}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{From 1 to N DAQ Jobs}
\subtitle{(for N not too big)}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={From 1 to N DAQ Jobs},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 27.1.90 (Org mode 9.4.4)}, 
 pdflang={English}}
\begin{document}

\maketitle

\begin{frame}[label={sec:orgd7d32f3},fragile]{Aside: Taxonomy}
 We need a way to speak precisely about some very similar things.  

\begin{description}
\item[{application}] an \alert{executable file} on host disk
\begin{itemize}
\item \footnotesize (eg \texttt{daq\_application})
\end{itemize}
\item[{job}] an \alert{application command line}
\begin{itemize}
\item \footnotesize (eg \texttt{daq\_application -{}-arg...})
\end{itemize}
\item[{process}] a \alert{running job}
\begin{itemize}
\item \footnotesize (eg seen in \texttt{ps -ef|grep daq\_application})
\end{itemize}
\item[{node}] a \alert{process with an identity}
\begin{itemize}
\item \footnotesize (has been given \emph{init} command)
\end{itemize}
\end{description}

We may bike shed labels but at least these four are distinct.
\end{frame}


\begin{frame}[label={sec:org14616db},fragile]{Where we are}
 \footnotesize
\begin{itemize}
\item Single node with multiple modules.
\begin{itemize}
\item \scriptsize JSON objects generated from \texttt{moo.otypes} classes in Python
\item Python classes themselves generated from moo schema in Jsonnet.
\end{itemize}
\item Start single job by hand to make a process.
\item Various means to ``manually'' send \alert{command} to process.
\item Each command has a \alert{type} (\texttt{.id}) and a payload (\texttt{.data}) of type \alert{any}
\begin{itemize}
\item \scriptsize  types: \emph{init}, \emph{conf}, \emph{start}, \emph{stop}, \emph{scrap}, \ldots{}
\item structure of \texttt{.data} depends on value of \texttt{.id} and determined by schema
\end{itemize}
\item The \emph{init} command is \alert{fixed structure} for all jobs
\begin{itemize}
\item \scriptsize \emph{init} ``constructs'' (process \(\to\) node) in terms of \alert{modules} and \alert{queues} and soon \alert{sockets}
\item \scriptsize Mostly consumed by \texttt{DAQModuleManager} but modules may also receive notice.
\end{itemize}
\item The commands supports \alert{command dispatch protocol}
\begin{itemize}
\item \scriptsize Generally maps regex match on module \alert{name} to module-level sub-object \alert{payload}
\item \scriptsize Most commands use \texttt{".*"} match and common or no payload
\item The \emph{conf} command carries very module-specific payloads.
\end{itemize}
\item \texttt{appfwk} command handling:
\begin{itemize}
\item \scriptsize  ``command facility'' (\texttt{stdin}, \texttt{file} and \texttt{rest}) accepts commands
\item \texttt{DAQModuleManager} command \alert{payload} dispatched to modules
\item \alert{no support for any kind of reply to a command}
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org3b0ec39}]{Next, scale to multiple processes}
My take-away of the essentials of Giovanna's proposal:

\begin{itemize}
\item Generate \alert{per-job} \emph{init} and \emph{conf} command objects \alert{as we do now}.

\item \alert{Collate} these across \alert{N jobs} by their command type (\emph{init,} etc)
\begin{itemize}
\item Must retain the association of command \(\to\) job
\end{itemize}

\item Augment this collection with \emph{boot} info (to enact job \(\to\) process)
\begin{itemize}
\item Then play \emph{init} (process \(\to\) node) and \emph{conf}, \emph{start}, etc
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[label={sec:orgda5a65a},fragile]{Prototypical CLI operations:}
 Run existing scripts to produce \emph{init} and \emph{conf} commands for each job.
\begin{itemize}
\item \scriptsize maybe also \emph{start}, \emph{stop}, etc, but these are simple enough to be produced later.
\end{itemize}

\begin{verbatim}
$ my_gen_proc1 > proc1.json 
$ my_gen_proc2 > proc2.json 
\end{verbatim}

Collate into run-level configuration, details how coming up.
\begin{verbatim}
$ gen_run -r 42 proc[12].json > run-42.json
\end{verbatim}

Apply run using a temporary/mock RC (exact nature still t.b.d.)
\begin{verbatim}
$ run-control run-42.json
\end{verbatim}
\end{frame}


\begin{frame}[label={sec:org3cbc01f},fragile]{How \alert{NOT} to collate?}
 \begin{center}
Invent some way to express \texttt{\#include} in JSON
\end{center}
\end{frame}

\begin{frame}[label={sec:org8e5132a},fragile]{How \alert{TO} collate?}
 \begin{center}
Use Jsonnet \texttt{import}.
\end{center}

\begin{itemize}
\item Jsonnet may \texttt{import} JSON files statically (no computed filenames)

\item \texttt{moo} has more flexible ways to get data of many formats in to Jsonnet
via top-level arguments (``TLAs'') to a Jsonnet function.

\item All info needed for collation is available in the per-job commands
so we may write a Jsonnet program to perform any collation.

\item Still need to supply \emph{boot} info, run number, map from \alert{job name} to its
commands.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orge13a69d}]{Example construction}
I'll walk though one way to construct run config.

This exposes guts.  We'd easily script it.

Consider it \alert{just a prototype} for something more formal, likely
implemented Python.

\begin{enumerate}
\item make \alert{mock} per-job commands
\item collate commands and keeping \alert{job name correlation}
\item build \alert{run config} object
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:org6a6a6ef},fragile]{Example construction - mock per-job commands}
 \footnotesize
\alert{FAKE, just for illustration} replace full blow per-job command
generator with:

\tiny
\begin{verbatim}
local dummy(name) = [
    {id:"init",data:name+" init"},
    {id:"conf", data:name+" conf"}
];
function(procs) {
    [n+".json"]:dummy(n) for n in procs
}
\end{verbatim}

\footnotesize
Run like:

\begin{verbatim}
$ moo -A procs='["proc1","proc2"]' \
      compile -m . per-proc.jsonnet 
$ cat proc1.json
\end{verbatim}

\tiny
\begin{verbatim}
[
    {
        "data": "proc1 init",
        "id": "init"
    },
    {
        "data": "proc1 conf",
        "id": "conf"
    }
]
\end{verbatim}

\normalsize
Of course \alert{real commands} have some mongo object hanging on \texttt{.data}!
\end{frame}

\begin{frame}[label={sec:org0d7ec82},fragile]{Example construction - correlate commands and job name}
 Proposal calls for \alert{collating by command id} (\emph{init} etc).

We must first \alert{correlate *job name} to each command object.

Let's take job name from file name with some \texttt{jq} hackery:

\footnotesize

\begin{verbatim}
$ ls proc?.json 
proc1.json  proc2.json

$ jq -n \
 'reduce inputs as $s (.; .[input_filename|split(".")[0]] += $s)' \
  proc?.json  > run-procs.json
\end{verbatim}

The \texttt{run-procs.json} now has all commands of all jobs and each command
holds a \texttt{.job} attribute keeping the job name.
\end{frame}

\begin{frame}[label={sec:orgb552eb3},fragile]{Example construction - collate and build \emph{boot}}
 Use \texttt{moo} to provide data a \texttt{run.jsonnet} program to produce a final \alert{run
configuration object} with collated commands and \emph{boot} part:

\footnotesize
\begin{verbatim}
$ moo -A run=42 -A procs=run-procs.json run.jsonnet \
       > run-42.json
\end{verbatim}

The \texttt{run-42.json} now contains:

\begin{description}
\item[{\texttt{.boot}}] minimal example of \emph{boot} command
\item[{\texttt{.inits}}] array of per-job \emph{init} command
\item[{\texttt{.confs}}] array of per-job \emph{conf} command
\end{description}

Embellishments certainly still needed:

\begin{itemize}
\item exhaustively include also the \texttt{.starts}, \texttt{.stops}, etc commands.
\item expand \emph{boot} info beyond just example \texttt{.jobs} (names) and \texttt{.run}
\item generate \texttt{rest://} URLs and \texttt{daq\_application} command lines
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org1810f87},fragile]{\texttt{run.jsonnet} - just for reference}
 \footnotesize
\begin{verbatim}
local boot(run, jobs) = {
    id: "boot", run: run, jobs: jobs
};

local select(cmdid, job, cmds) =
    [ c{job:job} for c in cmds if c.id == cmdid ];

local procs_to_cmds(procs, cmdid) =
    std.flattenArrays([select(cmdid, job, procs[job])
                       for job in std.objectFields(procs)]);

function(run, procs) {
    inits:procs_to_cmds(procs, "init"),
    confs:procs_to_cmds(procs, "conf"),
    boot:boot(run, std.objectFields(procs)),
}
\end{verbatim}
\end{frame}

\begin{frame}[label={sec:orgb846468},fragile]{Steps to make this useful}
 Step 1: Package the hackery into a short script to run as show above:

\begin{verbatim}
$ gen_run -r 42 proc[12].json > run-42.json
\end{verbatim}

Step 2: there no step 2.

That's it except for adding needed embellishments mentioned above!
\end{frame}

\begin{frame}[label={sec:orgc804e20},fragile]{Summary}
 \begin{itemize}
\item We have more or less solid single-job DAQ config now.
\begin{itemize}
\item True, users are still getting up to speed!
\end{itemize}
\item Giovanna's proposal looks good for next-step scale-up!
\begin{itemize}
\item But, let's use Jsonnet to aggregate instead of inventing some kind
of JSON \texttt{\#include} requiring some invented interpreter.
\end{itemize}
\item A simple \texttt{jq + moo} tool-based construction demonstrated and seems
sufficient for now.
\item Let's gain experience with this, 
\begin{itemize}
\item understand the nature of the eventual minidaq jobs
\item extend it with \alert{real} jobs as fodder
\item likely outgrow the \texttt{jq + moo} hackery and re-implement in Python
\item simultaneously lets us be flexible as we understand how the
coming new CCM components (RC, PM, AC) will look.
\end{itemize}
\end{itemize}
\end{frame}
\end{document}
