% Created 2022-03-21 Mon 15:31
% Intended LaTeX compiler: pdflatex
\documentclass[presentation]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{color}
\usepackage{listings}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\hypersetup{colorlinks}
\setbeamerfont{alerted text}{series=\bfseries}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{Wire-Cell Toolkit Configuration}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={Wire-Cell Toolkit Configuration},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 27.2 (Org mode 9.4.6)}, 
 pdflang={English}}
\begin{document}

\maketitle
\colorlet{punct}{red!60!black}
\definecolor{background}{HTML}{EEEEEE}
\definecolor{delim}{RGB}{20,105,176}
\definecolor{keyword}{RGB}{20,105,176}
\definecolor{comment}{RGB}{20,200,10}
\colorlet{numb}{magenta!60!black}

\lstdefinelanguage{bash}{
    basicstyle=\normalfont\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=8pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    backgroundcolor=\color{background},
    keywords = [1]{import, local},
    keywordstyle=[2]\color{keyword},
    comment=[l]{//},
    commentstyle=\color{comment}\ttfamily,
    stringstyle=\color{blue}\ttfamily,
    morestring=[b]',
    morestring=[b]",
}
\lstdefinelanguage{jsonnet}{
    basicstyle=\normalfont\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=8pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    backgroundcolor=\color{background},
    keywords = {$},
    keywords = [2]{self, import, local, true, false, null, function},
    keywordstyle=[2]\color{keyword},
    comment=[l]{//},
    commentstyle=\color{comment}\ttfamily,
    stringstyle=\color{blue}\ttfamily,
    morestring=[b]',
    morestring=[b]",
    otherkeywords={:, =, ==}
}
\lstdefinelanguage{json}{
    basicstyle=\normalfont\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=8pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    backgroundcolor=\color{background},
    keywords = [1]{import, local},
    keywordstyle=[2]\color{keyword},
    comment=[l]{//},
    commentstyle=\color{comment}\ttfamily,
    stringstyle=\color{blue}\ttfamily,
    morestring=[b]',
    morestring=[b]",
}
\lstdefinelanguage{cpp}{
    basicstyle=\normalfont\ttfamily,
    numbers=left,
    numberstyle=\scriptsize,
    stepnumber=1,
    numbersep=8pt,
    showstringspaces=false,
    breaklines=true,
    frame=lines,
    backgroundcolor=\color{background},
    keywords = [2]{include, if, for, const, typedef, using, public, private, virtual, class, return, void},
    keywordstyle=[2]\color{keyword},
    comment=[l]{//},
    commentstyle=\color{comment}\ttfamily,
    stringstyle=\color{blue}\ttfamily,
    morestring=[b]',
    morestring=[b]",
    otherkeywords={:, =, ==, ::}
}

\begin{frame}[label={sec:orgdcd6ee0}]{Outline}
We will cover these aspects of Wire-Cell Toolkit (WCT) \alert{configuration}:

\begin{itemize}
\item Layers
\begin{itemize}
\item code
\item object
\item files
\end{itemize}
\item Types of files
\item Jsonnet primer
\item Current configuration structure
\item Developer tips
\item Future plans
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgc07eed3}]{Layered Configuration Model}
\begin{center}
\begin{tabular}{c}
\hline
configurable component (code)\\
\hline
[configuration dispatch]\\
\hline
transient representation (object)\\
\hline
[representation conversion]\\
\hline
persistent representation (files)\\
\hline
[configuration sources]\\
\hline
\end{tabular}
\end{center}


\footnotesize

\begin{description}
\item[{code}] C++ code expecting config objects of a certain form.

\item[{objects}] a ``dynamic'' data object representing the config info.

\item[{files}] source config info convertable to config objects.
\end{description}
\end{frame}

\begin{frame}[label={sec:orgef234ea}]{}
\begin{center}
\Large C++ code layer of WCT config

\normalsize
in top-down order
\end{center}
\end{frame}


\begin{frame}[label={sec:org6d779fb},fragile]{WCT Main Configuration Sequence}
 \texttt{WireCell::Main} class expects an \alert{array of objects} like:

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
[    // This is in Jsonnet language
    {type:"A", name:"a", data: {...}},
    {type:"B", name:"b", data: {...}},
    // ...
    {type:"Z", name:"z", data: {...}},
]
\end{lstlisting}

\footnotesize

\begin{itemize}
\item The \texttt{Main} class is the entry point to WCT for \texttt{wire-cell} CLI and the \emph{art} tool.
\begin{itemize}
\item \tiny (though other applications are free to enter at other WCT layers)
\end{itemize}
\item Main array must be ordered by ``usage'' dependency.
\begin{itemize}
\item \scriptsize WCT provides a function to assure this.
\end{itemize}
\item Each config object is associated with an instance of \texttt{IConfigurable}
\begin{itemize}
\item \scriptsize C++ instance located using the (\alert{type}, \alert{name}) pair.
\item The \alert{data} must follow a schema expected by the given \alert{type}.
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgbd14f58},fragile]{Brief detour: flavor of a configuration object}
 \footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
  type: "Drifter",  // Match C++'s WIRECELL_FACTORY()
  name: "drifter",  // name is optional, default is ""
  data: {           // some are optional if set in C++
      DL:          7.2 * wc.cm2/wc.s,
      DT:         12.0 * wc.cm2/wc.s,
      lifetime:      8 * wc.ms,
      drift_speed: 1.6 * wc.mm/wc.us,
      fluctuate:  true,
  }
}
\end{lstlisting}

\footnotesize
This is some basic Jsonnet.
\begin{itemize}
\item \scriptsize Do not worry if it does not make sense yet.
\item We will come to the syntax and how to know what to provide.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org8fc414f}]{}
\begin{center}
Continuing with C++ layer\ldots{}.
\end{center}
\end{frame}


\begin{frame}[label={sec:orgb8e8ba0},fragile]{\texttt{WireCell::Persist} namespace}
 \lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
#include "WireCellUtil/Persist.h"
\end{lstlisting}


Low-level functions for:
\begin{itemize}
\item Locating configuration files.
\begin{itemize}
\item \scriptsize Uses WCT's only env var: \texttt{WIRECELL\_PATH}
\end{itemize}
\item Loading file or string to product an object.
\begin{itemize}
\item \scriptsize and vice-versa.
\end{itemize}
\item Provides a C++ interface to C or GO Jsonnet libraries.
\begin{itemize}
\item \scriptsize May pass Jsonnet ``top level arguments'' and ``external variables''.
\end{itemize}
\item Component developers usually need not encounter this level.
\begin{itemize}
\item \scriptsize A new type of ``data'' file will need a new component that does.
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org72b9af7},fragile]{WCT \texttt{IConfigurable}}
 \scriptsize
\lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
class MyFilter : public virtual IConfigurable {
  public:
    // IConfigurable
    virtual void configure(const WireCell::Configuration& cfg);
    virtual WireCell::Configuration default_configuration() const;
  private:
    int m_a{42};                // a "sane" default
    float m_b{0};               // always initialize
}
\end{lstlisting}

WCT will:

\begin{enumerate}
\item Call \texttt{default\_configuration()} to get an initial object, treating it
as opaque other than it is a JSON ``object'' (not scalar nor array).
\item Merge in to the default the contents of \texttt{data:\{...\}}.
\item Provide merged result back via \texttt{configure()}
\end{enumerate}
\end{frame}

\begin{frame}[label={sec:org7cc2aab},fragile]{Implementing \texttt{IConfigurable}}
 Best practice (now) is to ``filter'' class data members through configuration.

\footnotesize
\lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
WireCell::Configuration MyFilter::default_configuration() {
    WireCell::Configuration cfg;
    cfg["a"] = m_a;
    cfg["b"] = m_b;
    return cfg;
}
\end{lstlisting}

\lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
void configure(const WireCell::Configuration& cfg) {
    m_a = get<int>(cfg, "a", m_a);
    m_b = get<float>(cfg, "b", m_b);
}
\end{lstlisting}

\begin{itemize}
\item \scriptsize Can do error checking and throw exceptions here, early before any processing occurs.
\item Currently, we must read this kind of code to discover
what to provide in \texttt{data:\{...\}}.
\item Future plans will make all this boilerplate go away and give us configuration documentation!
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org5efd381},fragile]{}
 \begin{center}
\Large \texttt{WireCell::Configuration} object
\end{center}
\end{frame}


\begin{frame}[label={sec:org3ad7113},fragile]{\texttt{WireCell::Configuration} namespace}
 \lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
#include "WireCellUtil/Configuration.h"
\end{lstlisting}

Mainly provides a thin wrapper over:

\lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
typedef Json::Value Configuration;
\end{lstlisting}

All \texttt{IConfigurable}'s use this object.
\end{frame}

\begin{frame}[label={sec:orgcf0e1b8},fragile]{Example usage of \texttt{Json::Value}}
 \href{https://open-source-parsers.github.io/jsoncpp-docs/doxygen/index.html}{JsonCPP} is a ``venerable'' JSON library and provides \href{https://open-source-parsers.github.io/jsoncpp-docs/doxygen/class\_json\_1\_1\_value.html}{Json::Value}.

\lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
Configuration cfg;    // aka Json::Value
cfg["key"] = "value";
cfg["n"] = 42;
cfg["arr"] = {1,2,3};
int n = cfg["n"].asInt();
int oops = cfg["key"].asFloat();
\end{lstlisting}

\vfill
\end{frame}

\begin{frame}[label={sec:orga92ee91},fragile]{Prefer helper functions over direct access}
 \scriptsize

\lstset{language=cpp,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// Convert to C++ type
T convert<T>(const Configuration& cfg, const T& def=T());

// Get value via its attribute.dot.path into object
T get<T>(const Configuration& cfg, const std::string& dotpath,
         const T& def=T());
\end{lstlisting}

\footnotesize

They add to \texttt{Json::Value} methods by:

\begin{itemize}
\item Encapsulating some error checking.

\item Adding more flexible indexing.

\item Adding a return-default-on-error idiom.

\item Providing a more ``modern'' C++ typed interface.

\item Somewhat shielding WCT component code from exact choice of
\texttt{Configuration} type implementation.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgaa935e6}]{}
\begin{center}
\Large Configuration files
\end{center}
\end{frame}

\begin{frame}[label={sec:orgfd5d061},fragile]{Two ``same but different'' types of config files}
 \begin{description}
\item[{cfg}] goes through \texttt{IConfigurable} and typically in hand-written
\texttt{.jsonnet} files with official versions in \texttt{wire-cell-toolkit/cfg} repo.

\item[{data}] read more directly by ``service'' type components and
typically are generated \texttt{.json.bz2} file produced by programs in
\texttt{wire-cell-python} and with official copies in \texttt{wire-cell-data} repo.
\end{description}


These distinctions are not mandatory.

\begin{itemize}
\item \footnotesize We can pre-compile \texttt{.jsonnet} and compress to provide \texttt{.json.bz2} \alert{cfg} type.
\item Could, in principle, generate wire geometry \alert{data} file as a Jsonnet program.
\end{itemize}

\footnotesize

For the rest of the slides, we focus on \alert{cfg} type written in Jsonnet.
\end{frame}


\begin{frame}[label={sec:org9ba4aea}]{}
\begin{center}
Jsonnet Primer
\end{center}
\end{frame}

\begin{frame}[label={sec:orgbc8d778},fragile]{Jsonnet \emph{data templating} language.}
 \begin{columns}
\begin{column}{0.55\columnwidth}
\footnotesize


Pure functional programming language

\begin{itemize}
\item \scriptsize No side effects, lazy evaluation

\item Full featured standard library

\item ``Lisp/Scheme with curly braces''
\end{itemize}

Compiles to JSON text by default

\begin{itemize}
\item \scriptsize or YAML, INI, Python (\texttt{dict}), XML(svg), TOML

\item Novel formats can implemented in Jsonnet code.
\end{itemize}
\end{column}



\begin{column}{0.45\columnwidth}
\begin{center}
\includegraphics[width=.9\linewidth]{jsonnet-venn.png}
\end{center}

\begin{center}
\tiny 
Plus friendlier syntax: 
comments, optional trailing commas, optional quote-free object keys, single and double quotes.

Diagram from \url{https://jsonnet.org/}

See web site for excellent documentation.
\end{center}
\end{column}
\end{columns}
\end{frame}


\begin{frame}[label={sec:org7d062af},fragile]{Hello Jsonnet command line}
 Compile expressions as strings:
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet -e '{hello:"world",} // quick CLI test'
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "hello": "world"
}
\end{lstlisting}

Or, provide Jsonnet in source files:
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
echo '2 + 2' > sum.jsonnet
jsonnet sum.jsonnet
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
4
\end{lstlisting}

\footnotesize
WCT also provides \texttt{wcsonnet} which is \texttt{jsonnet} + WCT Jsonnet handling layers.
\end{frame}

\begin{frame}[label={sec:orgc539a23},fragile]{Jsonnet Types}
 \begin{description}
\item[{Null}] only one value: \texttt{null}, equal only to itself.
\item[{Boolean}] two values: \texttt{true} and \texttt{false}.
\item[{Number}] 64 bit IEEE754 FP representation.
\begin{itemize}
\item \scriptsize exact integers \(\in [-2^{53},2^{53}]\)
\end{itemize}
\item[{String}] sequence of Unicode codepoints.
\begin{itemize}
\item \scriptsize may use \texttt{'single quotes'}, or \texttt{"double quotes"}.
\item may index like \texttt{s[2]} but strings are not arrays.
\end{itemize}
\item[{Array}] Finite sequence, possibly heterogeneous in element types.
\begin{itemize}
\item \scriptsize \texttt{[1,2]}, \texttt{[true, "you", \{can:, "nest"\}]}
\end{itemize}
\item[{Object}] Collection of pairs: key of type string, value of any type.
\begin{itemize}
\item \scriptsize \texttt{\{a:1, b:2, c:\{greet: "hi!"\}\}}
\item ordered by key \texttt{\{b:2, a:1\}} \(\to\) \texttt{\{"a":1, "b":2\}}
\end{itemize}
\item[{Function}] A code context as a callable object.
\begin{itemize}
\item \scriptsize \texttt{\{a::1, f::function(x=0) x+self.a, b:self.f()\}} \\
\(\to\) \texttt{\{"b":1\}}
\end{itemize}
\end{description}
\end{frame}

\begin{frame}[label={sec:org085b392},fragile]{Some string operations}
 \footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
["%0.5f"%(1/19), "cat" < "dog", "cat"=="CAT", "catdog"[:3],
 std.asciiUpper("meow")]
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
[
   "0.05263",
   true,
   false,
   "cat",
   "MEOW"
]
\end{lstlisting}

\footnotesize
\begin{itemize}
\item Case-sensitive values, Unicode support.
\item Usual set of string functions in \texttt{std.} name space
\begin{itemize}
\item \scriptsize \texttt{split() startsWith() substr() strReplace() format(), ...}
\end{itemize}
\item Strings and arrays are distinct types.
\begin{itemize}
\item \scriptsize A string is \alert{not} an array of characters.  There is no ``char'' type in Jsonnet.
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[label={sec:orgd7a605a},fragile]{Some array operations}
 \vspace{-1cm}

\begin{columns}
\begin{column}{0.5\columnwidth}
\footnotesize
Length, indexing, slicing, appending, concatenating.

\begin{itemize}
\item \scriptsize All similar to what one expects from Python.
\end{itemize}
Many more array functions under \texttt{std}.
\begin{itemize}
\item \tiny \texttt{join() find() map() filter() foldl() range() reverse() sort() uniq()}
\end{itemize}
\end{column}

\begin{column}{0.5\columnwidth}
\tiny
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local a = [1,2,3]; local b = [40,50,60];
{
    len: std.length(a),
    index: b[0],
    append: a + [4],
    concat: a + b,
    slice: self.concat[3:5],
}
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "append": [
      1,
      2,
      3,
      4
   ],
   "concat": [
      1,
      2,
      3,
      40,
      50,
      60
   ],
   "index": 40,
   "len": 3,
   "slice": [
      40,
      50
   ]
}
\end{lstlisting}
\end{column}
\end{columns}
\end{frame}


\begin{frame}[label={sec:org5c02556},fragile]{Objects - a set of key/value attributes}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{ "b b":2, c_c:1, a:0 }
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "a": 0,
   "b b": 2,
   "c_c": 1
}
\end{lstlisting}

\begin{itemize}
\item Compiles to a JSON object, ordered by key.
\item Attribute keys must be of type string.
\begin{itemize}
\item \scriptsize Need quotes only if key does not look like a variable name.
\end{itemize}
\item Attribute values can be of any Jsonnet type.
\item Also, object defines a scope for \emph{referencing} (more on this coming).
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgd8ed6c3},fragile]{Object - \texttt{:} vs \texttt{::}}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
    see_me : "yes",
    not_me :: "no",
}
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "see_me": "yes"
}
\end{lstlisting}
\footnotesize
Jsonnet will use but not emit any attribute defined with \texttt{::}
\begin{itemize}
\item Mostly useful to assure we do not try to render a \texttt{function()} object
to JSON.
\item Can also be useful for ``temporary'' values that need to be shared
outside of file scope but should not compile to final JSON.
\begin{itemize}
\item \scriptsize Inside file scope, use of a \texttt{local} is cleaner.
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org55e694a}]{}
\begin{center}
Aside to understand \emph{referencing} of values.
\end{center}
\end{frame}

\begin{frame}[label={sec:org26f5d22},fragile]{\texttt{local} values}
 Defines \emph{file-local} value which may then be referenced directly by name.

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local x = 42;         // may now refer to 42 as x
local y = x + 1;      // use semicolon at top level 
{                     // inside object scope
    local z = y - x,  // end with comma
    result: z         // define an attribute
}
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "result": 1
}
\end{lstlisting}

Use \texttt{=} to define \texttt{local} values and \texttt{:} (or \texttt{::}) for object attributes.
\end{frame}

\begin{frame}[label={sec:org3f41185},fragile]{Relative and self referencing}
 \scriptsize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{   a: $.sub.c,    // reference from file-level top
    b: self.sub.c, // reference from current context
    sub: { 
        c: self.d,
        d: "all the same",
        e: $.sub.d, f: $.a, } }
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "a": "all the same",
   "b": "all the same",
   "sub": {
      "c": "all the same",
      "d": "all the same",
      "e": "all the same",
      "f": "all the same"
   }
}
\end{lstlisting}
\footnotesize
Lazy evaluation means reference can precede definition.
\begin{itemize}
\item \texttt{\$} represents the outer-most \alert{object} (if top is another type, can't use \texttt{\$})
\item \texttt{self} represents current lexical scoped object.
\end{itemize}
\scriptsize Infinite reference loops cause compile-time error (stack smash).
\end{frame}

\begin{frame}[label={sec:org815b435},fragile]{File referencing}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{ // a.jsonnet (tutorial prefers a.libjsonnet)
    x = 1
}
\end{lstlisting}

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// b.jsonnet
local myvar = import "a.jsonnet";
{
    y = myvar.x
}
\end{lstlisting}
\footnotesize
\begin{itemize}
\item The \texttt{b.jsonnet} file compiles to \texttt{\{"y":1\}}
\item The file name to \texttt{import} must be a literal string, not a computed value.
\begin{itemize}
\item \scriptsize In part, this assures results are stable.
\end{itemize}
\item Files for \texttt{import} are located via a search path given to the application.
\begin{itemize}
\item \scriptsize \texttt{jsonnet -J /path/to/my/libjsonnet ...}
\item \texttt{wire-cell} and \texttt{wcsonnet} use \texttt{\textasciitilde{}-P} and honors \texttt{WIRECELL\_PATH}
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org4a86bad},fragile]{Lazy evaluation and no side effects}
 Lazy evaluation:
\begin{itemize}
\item Jsonnet will not evaluate any code that is not used for the final result.
\item In previous example, the \texttt{a.jsonnet} may be a gigantic file but
\texttt{b.jsonnet} used only \texttt{myvar.x} and so only \texttt{a.x} is evaluated.
\end{itemize}

No side effects:
\begin{itemize}
\item There is absolutely no way to modify any ``variable''
\begin{itemize}
\item (perhaps better to call Jsonnet ``variables'' as ``named values'')
\end{itemize}
\item But we can make new values based on existing ones.
\end{itemize}
\end{frame}




\begin{frame}[label={sec:org3fa984a}]{}
\begin{center}
Back to objects
\end{center}
\end{frame}

\begin{frame}[label={sec:org5a34e85},fragile]{Object inheritance}
 The \texttt{+} operator on Object types is inheritance, or really \alert{shallow merge}.

\footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{a:1, b:2, c:{x:"hi", y:"there"}} + {b:20, c:30, d:40}
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "a": 1,
   "b": 20,
   "c": 30,
   "d": 40
}
\end{lstlisting}

\footnotesize
\begin{itemize}
\item When attributes collide, those of the latter object ``win''.
\item There is also \href{https://datatracker.ietf.org/doc/html/rfc7396}{std.mergePatch()} which operates deeply.
\end{itemize}

\scriptsize
(Of course, \texttt{+} is also concatenate for string and array and usual meaning for numbers.)
\end{frame}


\begin{frame}[label={sec:orgbee8c87},fragile]{For merge, the \texttt{+} is optional and usually omitted}
 Look for two or more objects bumping against each other.
\footnotesize

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local one = {a:1, b:2, c:{x:"hi", y:"there"}};
// three object merge/inheritance
local three = one {b:20, c:30, d:40} {d:400};
three
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "a": 1,
   "b": 20,
   "c": 30,
   "d": 400
}
\end{lstlisting}
\end{frame}

\begin{frame}[label={sec:org62a0dfa},fragile]{Array and Object Comprehensions}
 Similar to Python \texttt{list} and \texttt{dict} comprehensions:

\footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local squares = [x*x for x in std.range(1,3)];
local sqrts = {["%d"%xx]:std.sqrt(xx) for xx in squares};
sqrts
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "1": 1,
   "4": 2,
   "9": 3
}
\end{lstlisting}

\footnotesize
\begin{itemize}
\item Need to put the computed object attribute key inside \texttt{[]}'s.
\item Keys need to be strings so we use string interpolation to build from number.
\item \alert{Notice} \texttt{std.range(Nmin, Nmax)} returns \alert{inclusive range}!!
\begin{itemize}
\item \scriptsize \texttt{std.range(1,3)} \(\to\) \texttt{[1,2,3]}
\item This is \alert{unlike} Python \texttt{range()} or C++ \texttt{std::iota()}.
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org0d2ab80},fragile]{\texttt{if/then/else} switch}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local x = true;
local have = if x
             then "Yes"
             else "No";
have
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
"Yes"
\end{lstlisting}

\footnotesize
\begin{itemize}
\item White space / indentation does not matter.

\item The \texttt{else} is optional, \texttt{if/then/else if/...} chains allowed.

\item Use sparingly as it tends to obscure structure/value of the results.
\begin{itemize}
\item \scriptsize Prefer object ``inheritance'' to override a pre-built object.
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[label={sec:orgb350f09},fragile]{Functions - file scope definitions}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// Explicit creation of function type
local bye = function(name) "bye " + name;

// simpler, implicit syntax
local hi(name) = "hi " + name;

[hi("me"), bye("you")]
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
[
   "hi me",
   "bye you"
]
\end{lstlisting}
\end{frame}



\begin{frame}[label={sec:orgf490e97},fragile]{Functions - object scope definitions}
 \footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{   // object-local scope
    local greeting(g) = g,

    // attr func calling local func
    greet:: function(who)  
        greeting("bye") + " " + who,

    me: self.greet("me"),
    you: self.greet("you"),
}
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "me": "bye me",
   "you": "bye you"
}
\end{lstlisting}

\footnotesize
\begin{itemize}
\item \texttt{.greet()} may be called on this object (eg, object used elsewhere).
\begin{itemize}
\item \scriptsize The \texttt{::} keeps \texttt{greet()} from every rendering to JSON if object is in final result.
\end{itemize}
\item \texttt{greeting()} can not be seen outside the object.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org1a60683},fragile]{Function returning internal values}
 \normalsize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local f(x) = {
    local a = x*x,
    res: a + 2,
}.res;
f(42)
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
1766
\end{lstlisting}

\footnotesize
\begin{itemize}
\item When you need a ``working space'' object to hold intermediate values and wish to return some portion.
\item Simply dereference one object attribute immediately.
\end{itemize}
\end{frame}


\begin{frame}[label={sec:orgca82961},fragile]{Function arguments}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local f(x, y=2, z=0) = {
    local a = x*x,
    res: a + y + z,
}.res;
[f(42), f(42, 12), f(42, z=1, y=-42*42)]
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
[
   1766,
   1776,
   1
]
\end{lstlisting}
\footnotesize
Required positional args and optional keyword args with defaults (like Python)

\begin{itemize}
\item Keyword arguments given in any order, but must follow positional.
\item No Pythonic ``exploding dict'', can not call like:  \texttt{f(**d)}.
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org0ad4582},fragile]{Function environment}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local a=1;
{
    f:: function(x) a + self.b + x,
    b:: " + ",
    zz: self.f("2 = 3")
}.zz
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
"1 + 2 = 3"
\end{lstlisting}

\footnotesize
\begin{itemize}
\item The function \texttt{f(x)} ``captures'' (in C++ terms) the \texttt{local} variables and the object context in which it was defined.
\end{itemize}
\end{frame}


\begin{frame}[label={sec:orgd1b6337},fragile]{Functions returning functions}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local a=1;
local f(x) = function(b) a+b+x;
local five = f(5);
five(2)

\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
8
\end{lstlisting}
\footnotesize
\begin{itemize}
\item Functions are \emph{first class} instances.
\item One can form \emph{function closures} over values.
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org8d9a9db},fragile]{Top-level functions}
 A file can evaluate to a function which we call ``top-level function''.

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// f.jsonnet
function(x, y=1) { a:x+y }
\end{lstlisting}

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// g.jsonnet
local f = import "f.jsonnet";
f(2) // -> { "a":3 }
\end{lstlisting}

\footnotesize
As such, this is just a consequence of Jsonnet language.  
\end{frame}


\begin{frame}[label={sec:org41043bc},fragile]{TLAs: Top-level (function) Arguments}
 \scriptsize
Application can inject external values through TLAs

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// tla.jsonnet
function(x, y=2) {x:x, y:y}
\end{lstlisting}

\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet -A x=1 -A y="foo" tla.jsonnet
jsonnet --tla-code 'x=22' tla.jsonnet      
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "x": "1",
   "y": "foo"
}
{
   "x": 22,
   "y": 2
}
\end{lstlisting}
\scriptsize
\begin{itemize}
\item CLI processing makes values given by \texttt{-A/-{}-tla-str} into strings.
\item TLAs with default values need not be specified.
\item The \texttt{-{}-tla-code} can set a TLA to arbitrary Jsonnet code, (take care with shell quoting).
\item There is also \texttt{-{}-tla-code-file} to put this code in a secondary file.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org8b20c2a},fragile]{The power of TLAs}
 Break large structures into many files.
\begin{itemize}
\item Compose larger structure through \texttt{import} and function calls.
\item Test all structure scales via command line and TLAs.
\item End-user can give highest-level settings on CLI with sane defaults.
\item Bake user's favorite settings in yet higher-level Jsonnet.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org9e35868},fragile]{TLAs in today's applications}
 WCT's CLI args are similar to what \texttt{jsonnet} takes:
\footnotesize
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
$ wire-cell -A x=1, --tla-code y='{a:2}' [...] -c foo.jsonnet
$ wcsonnet  -A x=1, --tla-code y='{a:2}' [...] foo.jsonnet
\end{lstlisting}


\footnotesize
\begin{itemize}
\item No \texttt{-c} with \texttt{wcsonnet} and not \texttt{-tla-code-file} for either.

\item But, some \alert{big caveats}:
\begin{itemize}
\item Most of WCT config was structured before TLAs.  FIXME!
\item \texttt{larwirecell} lacks TLA support.  FIXME!
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgecb9b24},fragile]{The old bad way to inject: \texttt{std.extVar()}}
 \footnotesize

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// extvar.jsonnet
{ x: std.extVar("x") }
\end{lstlisting}

\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet -V x=2 extvar.jsonnet      
jsonnet --ext-code 'x=2' extvar.jsonnet      
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
{
   "x": "2"
}
{
   "x": 2
}
\end{lstlisting}

\begin{itemize}
\item Same CLI args for \texttt{wire-cell} and this \alert{is} supported by \texttt{larwirecell}
\item But, there are major problems with \texttt{std.extVar()}
\begin{itemize}
\item \scriptsize No way to provide a default value, the caller \alert{must} provide \alert{all} \texttt{extVar}'s
\item \texttt{std.extVar()} calls tend to be sprinkled everywhere, hard to spot in code.
\item Can not \alert{set} them from Jsonnet, so full composition is not possible.
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org6c8894d},fragile]{stdlib - full feature but ``small'' library of functions}
 \begin{center}
\url{https://jsonnet.org/ref/stdlib.html}
\end{center}

\begin{itemize}
\item enumerate or test for object keys/values
\item map, filter, reduce, sort, unique, set operations on arrays
\item expected math, string and array functions
\item type reflection: \texttt{std.type()} and \texttt{std.parseXXX()}, \texttt{std.manifestXXX()}
\end{itemize}

\footnotesize There is enough support that it was fairly easily to
develop elaborate functions for, eg, vector arithmetic, graph
construction, rewriting and topological sort!
\end{frame}


\begin{frame}[label={sec:orga3f5b5a}]{}
\begin{center}
\(\mathcal{INTERMISSION}\)
\end{center}
\end{frame}

\begin{frame}[label={sec:orgd114b53}]{}
\begin{center}
WCT configuration structure
\end{center}
\end{frame}

\begin{frame}[label={sec:orgadd4fbe},fragile]{File layout}
 Official configuration file set is under \href{https://github.com/WireCell/wire-cell-toolkit/tree/master/cfg}{wire-cell-toolkit/cfg/}.

\begin{description}
\item[{\texttt{wirecell.jsonnet}}] WCT system of units and basic helper functions
\item[{\texttt{vector.jsonnet}}] simple vector arithmetic
\item[{\texttt{pgraph.jsonnet}}] support for describing WCT data flow graphs
\item[{\texttt{pgrapher/}}] directory structure holding experiment configs
\begin{itemize}
\item \footnotesize \texttt{common/} holds generic ``base'' data structures
\item \texttt{experiment/<name>/} holds per-experiment derived structures
\item despite the name, valid for both \texttt{Pgrapher} and \texttt{TbbFlow}
\end{itemize}
\end{description}
Current experiments:
\begin{itemize}
\item \footnotesize \texttt{dune-vd}, \texttt{dune10kt-1x2x6}, \texttt{icarus}, \texttt{iceberg}, \texttt{pdps}, \texttt{sbnd}, \texttt{uboone}.
\end{itemize}

\scriptsize Caveat: the current \texttt{cfg/} area really needs a big refactor and cleanup!
\end{frame}

\begin{frame}[label={sec:orgc3b221e},fragile]{Main steps to defining a WCT config:}
 \begin{enumerate}
\item Define individual component configuration objects ``\texttt{cfg}''
\begin{itemize}
\item \emph{ie} \texttt{\{type:"...", name:"...", data:\{...\}\}}
\end{itemize}
\item If component is also a DFP node, embed it \texttt{cfg} in a \texttt{pnode} object.
\item Form the graph from \texttt{pnodes} culminating in a single aggregate \texttt{pnode}.
\item Define \texttt{Pgrapher} or \texttt{TbbFlow} component holding graph \texttt{edges}.
\item Optionally, define special \texttt{wire-cell} component to avoid having to specify a bunch of CLI args.
\item Emit the WCT ``main sequence'' array of \texttt{cfg} objects.
\end{enumerate}

We now take each in trun.
\end{frame}

\begin{frame}[label={sec:orga4f0dd5},fragile]{1. Component \texttt{cfg} object - high level parameters}
 \scriptsize

We may write each \texttt{cfg} object \emph{by hand} like we saw with the \texttt{Drifter} example but
that will be error prone as we must share the same values across many.
Eg, the parameter\ldots{}

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
drift_speed: 1.6 * wc.mm/wc.us,
\end{lstlisting}

\ldots{}is needed in a few places.  So, we set it once in a shared \texttt{params} data structure\ldots{}

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local wc = import "wirecell.jsonnet";
local params = import "pgrapher/common/params.jsonnet";
params.lar.drift_speed/(wc.mm/wc.us)
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
1.6000000000000001
\end{lstlisting}

\ldots{}and we override that value per each experiment:

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local wc = import "wirecell.jsonnet";
local params = import "pgrapher/experiment/uboone/params.jsonnet";
params.lar.drift_speed/(wc.mm/wc.us)
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
1.0980000000000001
\end{lstlisting}
\end{frame}

\begin{frame}[label={sec:orgffe2d39},fragile]{1. Component \texttt{cfg} object - construction tools}
 \footnotesize To apply \texttt{params} itself consistently we make set of
helper \texttt{tools} and from them make a set of \emph{makers}:

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local wc = import "wirecell.jsonnet";
local tool_maker = import "pgrapher/common/tools.jsonnet";
local params = import "pgrapher/experiment/uboone/params.jsonnet";
local tools = tool_maker(params); // uses TLAs!
local sim_maker = import "pgrapher/common/sim/nodes.jsonnet";
local sim = sim_maker(params, tools);
sim.drifter.uses[1].data.drift_speed/(wc.mm/wc.us)
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
1.0980000000000001
\end{lstlisting}

\scriptsize
\begin{itemize}
\item Do not worry about the form of this last line, we normally need not
dig into what \texttt{sim.drifter} holds as it is already a \texttt{pnode}.
\item Construction is nicely automated but using it requires some more
complicated boilerplate than it really should.
\item When diving into the guts of the ``tools'' and ``makers'' can be hard to
understand what and how to change something.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgfd53535},fragile]{Aside: ported graph model.}
 A WCT data flow graph is made of \emph{nodes} with \emph{ports}.
\begin{itemize}
\item A node has \texttt{nin} \emph{input} and \texttt{nout} \emph{output} ports, each zero or more.
\item An \emph{edge} is from an output port to an input port.
\item A graph is \emph{complete} when every port has exactly one edge.
\item A \emph{subgraph} of nodes may form an \emph{aggregate} node.
\item The aggregate node exposes any as-yet unconnected ports.
\end{itemize}

\begin{center}


\includegraphics[height=1cm,clip,trim=0 1.3cm 0 1.3cm]{pgraph1.pdf}

\includegraphics[height=1.4cm,clip,trim=0 1.5cm 0 1.5cm]{pgraph2.pdf}

\includegraphics[height=1cm,clip,trim=0 1.3cm 0 1.3cm]{pgraph3.pdf}
\end{center}
\end{frame}

\begin{frame}[label={sec:orgbdf2420},fragile]{2. Form graph nodes - primitive nodes}
 \footnotesize

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local pg = import "pgraph.jsonnet";
local drifter = pg.pnode({type:"Drifter", name:"drifter"},
                         nin=1, nout=1, uses=[tools.random]);
\end{lstlisting}

\begin{itemize}
\item First arg to \texttt{pnode()} is a component \texttt{cfg} object.
\item Must give \texttt{nin} and \texttt{nout} which is the input/output node \emph{port cardinality}.
\begin{itemize}
\item \scriptsize Used to catch graph construction errors.
\end{itemize}
\item The \texttt{uses} argument names a dependency list to assure these components
are represented in the final configuration sequence.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orga60986a}]{3. Form aggregates}
Assume these nodes:

\begin{center}
\includegraphics[width=5cm]{threenodes.pdf}
\end{center}
\end{frame}


\begin{frame}[label={sec:org3dbd190},fragile]{3. Form aggregate - \texttt{intern()}}
 \lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local e1 = pg.edge(node1, node2);
local e2 = pg.edge(node2, node3);
local pl1 = pg.intern(
    innodes     = [node1],
    centernodes = [node2],
    outnodes    = [node3],
    edges       = [e1, e2]);
\end{lstlisting}

\footnotesize

\begin{itemize}
\item \texttt{edge(n1, n2, p1=0, p2=0)} selects ports from nodes to connect.
\item \texttt{intern()} is a heavy lifter: can connect up arbitray subgraph.
\item Result aggregate is pipeline: \texttt{node1 -> node2 -> node3}
\item Exposing remaining ports:
\begin{description}
\item[{input}] \texttt{node1} input port 0
\item[{output}] \texttt{node1} output port 1 and \texttt{node3} output port 0
\end{description}
\end{itemize}
\end{frame}



\begin{frame}[label={sec:orgc70657d},fragile]{3. Form aggregate - \texttt{pipeline()}}
 Same result but written more simply:

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local pl2 = pg.pipeline([node1, node2, node3]);
\end{lstlisting}

\footnotesize

However, \texttt{pipeline()} will only hook up port 0's.
\end{frame}


\begin{frame}[label={sec:org90968c4},fragile]{3. Form aggregate - etc}
 The \texttt{pgraph.jsonnet} also provides:

\begin{description}
\item[{``fans''}] 1-N \texttt{fanin()} and N-1 \texttt{fanout()} pattern
\item[{``taps''}] a 1-2 fanout with port 1 ``capped'' with a sink
\item[{``insert''}] may use \texttt{insert\_one()/insert\_node()} to ``break'' an edge to insert a new node
\end{description}
\end{frame}


\begin{frame}[label={sec:org38e8dac},fragile]{4. Define main ``app''}
 \footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
  local graph = make_graph(); // top aggregate node 
  local app = {
  type: 'Pgrapher',  // or "TbbFlow" for multithread
  data: {
    edges: g.edges(graph),
  },
};  
\end{lstlisting}

\footnotesize
\begin{itemize}
\item The edges are simply lists of pairs of node ``type:name'' and port
numbers, all calculated in Jsonnet from the top aggregate node here
called \texttt{graph}.

\item The WCT \texttt{Pgrapher} or \texttt{TbbFlow} ``app'' will use \texttt{type:name} to lookup the
C++ \texttt{INode} components and perform edge connections given their port
numbers.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org0c4837f},fragile]{5. Special \texttt{wire-cell} component}
 \footnotesize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local cmdline = {
    type: "wire-cell",
    data: {
        plugins: ["WireCellGen", "WireCellPgraph",
                  "WireCellSio", "WireCellSigProc",
                  "WireCellRoot"],
        apps: ["Pgrapher"]      // or "TbbFlow"
    }
};
\end{lstlisting}

\begin{itemize}
\item This special ``component'' configuration is purely to avoid giving CLI options.
\begin{itemize}
\item \scriptsize It is used by \texttt{WireCell::Main} so useful for both \texttt{wire-cell} and \texttt{larwirecell}.
\end{itemize}
\item Here or on CLI, one must give all plugins providing any referenced
component ``type''.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgdd31052},fragile]{6. The last line: configuration sequence}
 Finally, the actual Jsonnet ``program'' is almost trivial!

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
[cmdline] + g.uses(graph) + [app]
\end{lstlisting}

\footnotesize
The \texttt{uses(graph)} does heavy lifting
\begin{itemize}
\item \scriptsize all \texttt{cfg} objects either held directly via \texttt{pnode} or via \texttt{pnode} attribute \texttt{uses}
\item sorted by order of the ``uses'' dependency graph
\item prepend \texttt{cmdline} and append \texttt{app} by hand.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org6d876ab}]{}
\begin{center}
Development tips
\end{center}
\end{frame}

\begin{frame}[label={sec:org302054b},fragile]{Editors}
 \begin{description}
\item[{Emacs}] Use this \texttt{jsonnet-mode} \url{https://github.com/mgyucht/jsonnet-mode}
\item[{VI}] Run \texttt{:JSONNETFMT} which I guess runs the \texttt{jsonnetfmt} program (sorry, I'm not a VI'er)
\item[{Others}] Likely provide some contributed support.
\end{description}

Generally, formatting support in an editor is not required to develop
Jsonnet but it does help to provide hints that some syntax is wrong.
\end{frame}

\begin{frame}[label={sec:org37e8228},fragile]{Workflow guidelines}
 \begin{itemize}
\item Start with small files.
\item Test compile as you go.
\item Keep files small, factor into more files, use \texttt{import} liberally.
\item Use top level functions, provide TLAs with sane defaults.
\item Avoid \texttt{if/then/else}.
\item Avoid copy-paste.  Refactor and then \texttt{import}.
\item Avoid passing megaliths to functions in favor of just what is
needed.
\end{itemize}

\footnotesize
Current WCT config does not always provide the best role model!
\end{frame}

\begin{frame}[label={sec:orgc886eb8},fragile]{Dump default configurations}
 \footnotesize
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
$ wire-cell --help
$ wire-cell -a ConfigDumper -p WireCellApps -p WireCellImg 
\end{lstlisting}

\lstset{language=json,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
[
  {
    "data": {
      "spans": 1
    },
    "name": "",
    "type": "BlobClustering"
  },
...]
\end{lstlisting}

Tip: pipe through \texttt{jq -C | less -R}
\begin{itemize}
\item replace \texttt{WireCellImg} with any other WCT plugin to see its components
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org95d959a},fragile]{Understanding Jsonnet errors}
 \tiny
\begin{verbatim}
RUNTIME ERROR: array bounds error: 0 not within [0, 0)
	/home/bv/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg/pgraph.jsonnet:19:15-30	object <b>
	std.jsonnet:1338:50-54	thunk <b>
	std.jsonnet:1338:42-54	function <anonymous>
	std.jsonnet:1338:42-54	function <anonymous>
	std.jsonnet:1342:11-23	function <anonymous>
	std.jsonnet:231:30-65	
	std.jsonnet:231:19-66	function <anonymous>
	/home/bv/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg/wirecell.jsonnet:325:30-44	thunk <a>
	/home/bv/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg/wirecell.jsonnet:325:30-49	function <anonymous>
	/home/bv/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg/wirecell.jsonnet:325:30-49	function <func>
	std.jsonnet:789:24-47	thunk <running>
	std.jsonnet:789:9-57	function <aux>
	std.jsonnet:789:9-57	function <aux>
	std.jsonnet:790:5-28	function <anonymous>
	/home/bv/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg/wirecell.jsonnet:326:22-55	function <anonymous>
	/home/bv/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg/pgraph.jsonnet:177:21-48	function <anonymous>
	cfg/wcls-sim-adc.jsonnet:137:12-27	object <anonymous>
	cfg/wcls-sim-adc.jsonnet:(136:9)-(138:4)	object <app>
	cfg/wcls-sim-adc.jsonnet:141:19-22	thunk <array_element>
	During manifestation	
\end{verbatim}

\begin{itemize}
\item A little cryptic but one gets used to reading these stack traces.
\item Error message is first line, then stack trace.
\item First line of stack trace is the direct error site.
\item Last line is the main-file entry point.
\item A \texttt{thunk} is simply a non-function line of code.
\item Numbers: \texttt{line:col1-col2}, useful to check source code.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org409f883},fragile]{Debugging - \texttt{std.trace(msg, res)}}
 \texttt{msg} string printed to \texttt{stderr} and \texttt{res} is returned

\scriptsize
\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// extrace1.jsonnet
local x = true;
{ a: if x then std.trace('x is true', 42)
     else std.trace('x is false', -1) }  
\end{lstlisting}
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet extrace1.jsonnet 2>&1 1>/dev/null
\end{lstlisting}

\begin{verbatim}
TRACE: extrace1.jsonnet:3 x is true
\end{verbatim}

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// extrace2.jsonnet
local o = {a:1, b:"Bee"};
std.trace(std.toString(o), o)
\end{lstlisting}
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet extrace2.jsonnet 2>&1 1>/dev/null 
\end{lstlisting}

\begin{verbatim}
TRACE: extrace2.jsonnet:3 {"a": 1, "b": "Bee"}
\end{verbatim}
\end{frame}


\begin{frame}[label={sec:org2d058f5},fragile]{Debugging - temporarily produce intermediate value}
 We want to build \texttt{three} but get an error, comment out and temporarily
build intermediates:

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local one = [1];
local two = one[1];
local three = two*2;
// three // hmm, leads to stack trace.
// two   // okay, try this.  Nope also crashes.
one      // works and, ah ha, problem obious now !
\end{lstlisting}

We see we should have used instead

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
local two = one[0];
\end{lstlisting}
\end{frame}

\begin{frame}[label={sec:orgb3141f8},fragile]{Assert truth}
 One may assert the truth of something which ``should'' always be so.

\lstset{language=jsonnet,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
// exassert.jsonnet
local f(x) = 
    assert x >= 0 : 'x must be positive';
    std.sqrt(x);
f(-1)  
\end{lstlisting}
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet exassert.jsonnet     2>&1 || true
\end{lstlisting}

\begin{verbatim}
RUNTIME ERROR: x must be positive
	exassert.jsonnet:(3:5)-(4:16)	function <f>
	exassert.jsonnet:5:1-6	
\end{verbatim}

\scriptsize There is no way to ``catch'' an \texttt{assert}, this is not a
exception mechanism.
\end{frame}

\begin{frame}[label={sec:org65756bb},fragile]{Raise error}
 \footnotesize
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet -e 'local a = [error "no zero element", 1, 2]; a[1]'
\end{lstlisting}

\begin{verbatim}
1
\end{verbatim}

\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
jsonnet -e 'local a = [error "no zero element", 1, 2]; a[0]' 2>&1 || true
\end{lstlisting}

\begin{verbatim}
RUNTIME ERROR: no zero element
	<cmdline>:1:12-35	thunk <array_element>
	<cmdline>:1:44-48	
\end{verbatim}

\scriptsize  Like C's \texttt{abort()}.  Note, effect of lazy evaluation.  Again, not an exception to be caught.
\end{frame}



\begin{frame}[label={sec:org9cb0247},fragile]{Visualize the configured graph}
 \href{https://github.com/wirecell/wire-cell-python}{Install \texttt{wire-cell-python}} and:

\footnotesize
\begin{verbatim}
$ wirecell-pgraph dotify --no-params --jpath=-1 \
      mycfg.jsonnet mycfg.dot
$ dot -Tpdf -o mycfg.pdf mycfg.dot
\end{verbatim}

\begin{center}
\includegraphics[width=.9\linewidth]{mycfg.pdf}
\end{center}

\scriptsize
\begin{itemize}
\item Finds connection errors immediately and helps understand the overall structure.
\item Remove \texttt{-{}-no-params} to see the \texttt{cfg} object attributes.
\end{itemize}

\tiny The \texttt{-{}-jpath=-1} tells the program how to find the graph data structure
in the overall \texttt{.jsonnet/.json} file.
\end{frame}

\begin{frame}[label={sec:org27fbaf2},fragile]{\texttt{jq} for ``querying'' the configuration}
 \scriptsize
\lstset{language=bash,label= ,caption= ,captionpos=b,numbers=none}
\begin{lstlisting}
$ cd wire-cell-toolkit/cfg/

$ jsonnet -J . pgrapher/experiment/pdsp/wct-sim-check.jsonnet |
    wc -l
162837

$ jsonnet -J . pgrapher/experiment/pdsp/wct-sim-check.jsonnet |
    jq '.type' | wc -l
138

$ jsonnet -J . pgrapher/experiment/pdsp/wct-sim-check.jsonnet |
    jq '.[0]'
# -> prints first cfg object

$ jsonnet -J . pgrapher/experiment/pdsp/wct-sim-check.jsonnet |
   jq '.[] | select(.type=="Pgrapher").data.edges[0]'
 # -> prints first edge object of Pgrapher cfg

\end{lstlisting}


\footnotesize
\begin{itemize}
\item \texttt{jq} is like \texttt{grep/sed/awk} for JSON.
\item Very powerful, has a learning curve but worth learning a few basics.
\end{itemize}
\end{frame}


\begin{frame}[label={sec:orgd3a6844}]{}
\begin{center}
\Large Plans for WCT configuration improvement
\end{center}
\end{frame}

\begin{frame}[label={sec:org80bcc70},fragile]{The main problems}
 \begin{itemize}
\item Every C++ component author repeats config related boilerplate.
\item Each forced to invent novel config object setting/getting patterns.
\item Generally not done safely and not with uniform exception patterns.
\item An implicit object schema lurks in each component, diffused throughout the C++ source.
\item Even with \texttt{ConfigDumper} can only discovery part of the schema where defaults were given.
\item No comprehensive reference documentation of configuration objects.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org4f9731e},fragile]{Strategy}
 \begin{itemize}
\item Define \emph{schema} describing expectation of type of configurable object.
\item Generate C++ code from \emph{schema} to provide
\begin{itemize}
\item C++ \texttt{struct} reflecting the schema.
\item Bidirectional serialization between C++ \texttt{struct} and JSON object/file.
\end{itemize}
\item Lift the \texttt{configure()} and \texttt{default\_configure()} methods of all
\texttt{IConfigurable} implementations into an \texttt{Aux::Configurable} base.
\begin{itemize}
\item It is templated on \texttt{<struct>}, holds as \texttt{protected} data member.
\item \texttt{IConfigurable} implementation then becomes automatic.
\item Still provide a ``hook'' method for code wanting to run at configure time.
\end{itemize}
\item C++ component then gets ready-to-use and safe access to config in
explicit, static \texttt{struct} type.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orga8a06e0},fragile]{Implementing schema-based configuration}
 For DUNE DAQ I made \href{https://github.com/brettviren/moo}{moo} which provides all we need (and more).

\begin{itemize}
\item In the \href{https://github.com/brettviren/wire-cell-toolkit/tree/cfgschema}{cfgschema} branch of my fork of wire-cell-toolkit has the start.

\item Work largely consists of, for each component, define a schema, rip
out existing usage, replace with \texttt{struct}
\end{itemize}

Goal: keep external configuration expectations 100\% unchanged.
\end{frame}

\begin{frame}[label={sec:orgc5843dd},fragile]{A wrinkle: JsonCPP \(\to\) \texttt{nlohmann::json}}
 \begin{itemize}
\item \texttt{moo} really wants to use ``JSON for Modern C++'' (\href{https://github.com/nlohmann/json/}{nlohmann::json}).

\item WCT uses JsonCPP and its API is not friendly to \texttt{moo}
\begin{itemize}
\item \scriptsize I've tried/failed to make it work already.
\end{itemize}

\item JsonCPP as become not so great for other reasons
\begin{itemize}
\item \scriptsize old fashioned C++ idioms, neglected development, hard to find docs (at times).
\end{itemize}

\item Replacement would remove entire JsonCPP shared library dependency in
exchange for just a single C++ header file.

\item This swap will touch a lot of code.  Easy, but very tedious work.
Still, I want to do this.
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org8dbfaf8},fragile]{Simplifying/refactoring \texttt{wire-cell-toolkit/cfg/}}
 \begin{itemize}
\item Leverage \emph{schema} to generate Jsonnet code.
\begin{itemize}
\item Provide \texttt{cfg} object ``constructor'' functions.
\item Use them instead of directly making \texttt{cfg} objects.
\item Jsonnet then forces valid schema.
\item Much of this is done already in \texttt{moo}.
\end{itemize}
\item Develop ``bottom-up'' functions to build larger structures from these
``constructor functions''.
\begin{itemize}
\item Eg, one general function to create ``a sim pipeline''
\item We have that now but not well factored.
\end{itemize}
\item Develop ``top-down'' functions to meet end-user patterns.
\begin{itemize}
\item Eg, one general function to create ``a sim job''.
\item Any top-level file should be only a few unique lines long, not
100s of copy-pasted lines typical now!
\end{itemize}
\item Strongly exploit composition via TLAs.
\begin{itemize}
\item Need to add TLA support to \texttt{larwirecell}.
\begin{itemize}
\item Can have a \texttt{std.ExtVar()} to TLA layer in Jsonnet.
\end{itemize}
\end{itemize}
\end{itemize}
\end{frame}


\begin{frame}[label={sec:org01dae40}]{}
\begin{center}
\Huge

\(\mathcal{FIN}\)
\end{center}
\end{frame}
\end{document}