% TEXINPUTS=.:$HOME/git/bvtex: latexmk  -pdf <main>.tex
\documentclass[xcolor=dvipsnames]{beamer}

\input{defaults}
\input{beamer/preamble}

\setbeamertemplate{navigation symbols}{}
% \setbeamertemplate{background}[grid][step=1cm]

\usepackage{siunitx}
\usepackage{xmpmulti}
\usepackage[export]{adjustbox}
\usepackage{ulem}
\usepackage[outline]{contour}
\usepackage{pdfpages}
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows}
\usetikzlibrary{positioning}

\definecolor{bvtitlecolor}{rgb}{0.98, 0.92, 0.84}
\definecolor{bvoutline}{rgb}{0.1, 0.1, 0.1}

\renewcommand{\bvtitleauthor}{Brett Viren}
\renewcommand{\bvtit}{IPC}
\renewcommand{\bvtitle}{\LARGE Online Software\\Inter-process Communication}
\renewcommand{\bvevent}{{\small DUNE DAQ WS -- 4 Feb 2019}}
\renewcommand{\bvdate}{4 Feb 2019}
\renewcommand{\bvbeamerbackground}{}

\def\Put(#1,#2)#3{\leavevmode\makebox(0,0){\put(#1,#2){#3}}}


\begin{document}
\input{beamer/title.tex}
\input{beamer/toc.tex}

\section{Introduction and Scope}


\begin{frame}
  \frametitle{Current DAQ Subsystem Concepts}
  \begin{center}
    \includegraphics[height=0.6\textheight]{everybody.pdf}    
  \end{center}

    \vfill
    \begin{itemize}    \footnotesize
    \item Many ``boxes'' (applications) needed, but also many ``arrows'' (IPC).
    \item Implementing arrows takes comparable effort boxes.
    \item Many shared requirements for the arrows.
    \item We should avoid creating a ``babel'' of IPC implementations.
    \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Inter-Process Communication (IPC)}
  \begin{center}
    \includegraphics[height=0.2\textheight]{appcommapp.pdf}
  \end{center}
  \begin{description}\footnotesize
  \item[processes] tightly coupled functionality bound into an application, operating ``locally'' on input message, producing output messages.
  \item[communication] asynchronous exchange of units of data obeying well defined message schema and protocol contracts.
  \end{description}

  \begin{itemize}\footnotesize
  \item Bulk of a process is the \textbf{application logic}, but a sliver of each must deal with the communication.
  \item Two independent applications must share a common communication, they can't each invent something.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Established and Missing IPC}

  \begin{itemize}
  \item[+] artDAQ has a well developed IPC
  \item[+] whatever RDBMS we use (eg, PostgreSQL) has its IPC
  \item[+] ``OS-level'' Apache/Ganglia/Superivord/Puppet/etc
  \item[?] Internal comms between Run Control / CCM applications?
  \item[?] RC to basically everything else?
  \item[?] Trigger / Data Selection internals?
  \item[?] Inter-module trigger exchange?
  \item[?] Trigger command hand-off to Back-End?
  \item[?] Front-end buffer data query/response?
  \item[?] Monitoring, logging from essentially everything.
  \item[?] extra-DAQ interfacing (eg CISC, det h/w)
  \item[?] what do I miss?
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Proposal}

  \begin{center}
    Develop a single, common IPC covering the missing IPC.
  \end{center}

  Require that DAQ applications:
  \begin{itemize}
  \item Use it for all \textbf{inter}-subsystem IPC.
  \item Use it for any \textbf{intra}-subystem IPC \textbf{where it's needed}.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Some Requirements on common IPC system}
  
  \begin{itemize}\footnotesize
  \item Satisfy latency and throughput needs (inter-- and intra--subsys).
  \item Support formally defined message schema and protocols with versioning.
  \item Provide mechanisms for redundancy, discovery and presence.
  \item API for application development.
  \item Portable and with minimal and stable dependencies.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Scope and implementation of the Common IPC}
  \begin{itemize}\footnotesize
  \item[?] artDAQ is one player in this space and we must leverage it, not replace it.  Does it make sense for artDAQ to become the basis for all subsys?
  \item[?] NetIO is used at protoDUNE/FELIX for transfer of full data from FE to the two BE computers.  DUNE FD won't do this but should NetIO also be considered as a basis?
    \vfill
  \item I don't know these answers with 100\% certainty, but I feel the common IPC should be built from a more general base.
  \item I'll describe what I think is right for implementing a common IPC system but think of it as a \textbf{proposal of work}.
  \item If more or less accepted, I'd like to lead the effort to develop it and I'll need some help.

    \vfill
  \item[!] Discussion and buy-in by all subsystems are needed to determine a proper scope and design for a common IPC.
    
  \end{itemize}

\end{frame}

\section{Common IPC System}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{Common IPC System Scope}

  \begin{itemize}
  \item Methods to formally define message schema and code generation for functions to operate messages
    \begin{itemize}\scriptsize
    \item[o] eg, to form/parse/\texttt{send()}/\texttt{recv()}
    \end{itemize}
  \item A set of C++ shared libraries providing IPC functionality for DAQ application development.
  \item A C++ API which supports layered entry points to match the diversity of application development
    \begin{itemize}\scriptsize
    \item[o] eg layers: function primitives $\to$ toolkit $\to$ application framework
    \end{itemize}
  \item Proper factorization of concerns
    \begin{itemize}\scriptsize
    \item[o] general purpose core libraries.
    \item[o] DUNE DAQ semantics as extension libraries.
    \end{itemize}
  \item Python API bindings.
  \end{itemize}
\end{frame}

\begin{frame}
  Next I give an outline of the Common IPC System based on prototype software I've been developing at 

  \begin{center}
      \url{https://github.com/brettviren/digrex}
  \end{center}

  The most recent and promising line is in \href{https://github.com/brettviren/digrex/tree/master/dexnet}{\texttt{dexnet}/}

  \vfill

  I take \texttt{dexnet} as working title for now.

\end{frame}

\begin{frame}
  \frametitle{\texttt{dexnet} Foundational Dependencies}
  \begin{description}\footnotesize
  \item[ZeroMQ] provides high-performance networking abstractions.
  \item[Protobuf] candidate support for structured message data. 
  \item[nlohmann/json] modern C++ JSON support, another candidate for handling structured message data.
  \item[Jsonnet] powerful, simple human-oriented configuration language.
  \item[Boost] for various C++ supports, including Boost.SML state machine.
  \item[Waf] for portable build system.
  \end{description}
\end{frame}

\begin{frame}
  \frametitle{ZeroMQ}
  \begin{description}
  \item[ZeroMQ]  high performance, high level \texttt{socket(2)} abstractions, variety of communication patterns, reliable, robust against connectivity loss and congestion,  support multiple types of transport.
  \item[CZMQ] High-level C bindings to ZeroMQ, behavior patterns: \textbf{actor}, poller, looper, auth/auth.
  \item[Zyre] A CZMQ actor providing network \textbf{discovery} and \textbf{presence}.
  \end{description}

  Comments:\scriptsize
  \begin{itemize}
  \item All are C libraries and usable in C++, minimal dependencies, very portable,
  \item Free Software (LGPL), long lived, strong and helpful development community.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{ZeroMQ Transports}
  Three types message transports supported:
  \begin{enumerate}\footnotesize
  \item \texttt{inproc://thread-label} between \textbf{threads} in the same process
  \item \texttt{ipc:///path/to/pipe} between \textbf{processes} on the same computer
  \item \texttt{tcp://hostname.dune.daq:1234} over the \textbf{network}
  \end{enumerate}

  \vfill
  Application is \textbf{independent of the transport}.
  \begin{itemize}  \footnotesize
  \item Transport is \textbf{run-time configuration} with strings like the above.
  \item We can develop DAQ applications which \textbf{scale} from threads to processes to computers with no code changes.  Clear benefit for DUNE DAQ.
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Sample of ZeroMQ socket patterns}

  \begin{description}\footnotesize
  \item[PUB/SUB] subscribe to topic, receive published messages, topic filtering on PUB side (don't send unwanted messages).  After subscribe, async, one-way streaming.  (drops)
  \item[PUSH/PULL] unidirectional, ``FIFO''. (blocks)
  \item[REQ/REP] classic client/server, synchronous. (blocks)
  \item[ROUTER/DEALER] advanced extension to REP/REQ. (blocks)
  \item[PAIR] Exclusive peer connection, two-way async ``pipe''. (blocks)
  \end{description}

  
  \begin{itemize}\scriptsize
  \item Except for PAIR, M-to-N connectivity pattern may be used as good redundancy mechanism.
  \item The ``(blocks)/(drops)'' indicates what happens if the socket maintained internal buffer reaches its adjustable high-water mark.
  \item There are additional socket types available, these are most common.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{CZMQ Actor}

  \begin{center}
    A \textbf{function} running in a \textbf{thread} communicating \\
    via a thread-safe, bidirectional \textbf{pipe} (PAIR socket).
  \end{center}

  \vfill
  \texttt{dexnet} uses the \textit{actor} pattern as a basic building block for its mid-level application API.  More on this in a bit.

\end{frame}

\begin{frame}
  \frametitle{Zyre Actor}
  \begin{center}
    An actor which implements network \textbf{discovery} and \textbf{presenece} \\ {\scriptsize (as well as a simple form of many-to-many ``group chat'' like communication)}.
  \end{center}

  Discovery:
  \begin{itemize}
  \item Send UDP broadcast \textbf{beacon} containing:
    \begin{itemize}\footnotesize
    \item[$\to$] \textbf{identifier} (a name), an \textbf{endpoint} string and set of \textbf{key/value} pairs.
    \end{itemize}
  \item Listen on endpoint, respond to any peer beacons by sending above info to peer's endpoint.
  \item The time scale for discovery is 1-2 seconds.
  \end{itemize}
  Presence:
  \begin{itemize}
  \item Peer heartbeat messages at configurable interval.
    \begin{itemize}\footnotesize
    \item[o] heartbeats use both UDP and TCP (efficient vs reliable)
    \end{itemize}
  \item Timeout interpreted as a loss of presence, Zyre actor notifies the rest of the application so that it may react.
  \end{itemize}
\end{frame}


\section{Online Software Application Design}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}


\begin{frame}
  \frametitle{API Layers of \texttt{dexnet}}

  An application may be developed leveraging as little or as much of 
  \texttt{dexnet} as desired.  In order of increasing abstraction:

  \begin{enumerate}\footnotesize
  \item No \texttt{dexnet},  application uses raw \texttt{socket(2)} and fully handles responsible for proper message forming/parsing.
  \item Replace raw \texttt{socket(2)} with ZeroMQ sockets.
  \item Use \texttt{dexnet} message schema codegen to provide form/parse code.
  \item Use \texttt{dexnet} graph abstractions (described next).
  \item Bundles those abstractions in form of \textit{actor} but manages its lifetime.
  \item Provides one or more \textit{actors} in a \texttt{dexnet} plugin library and use \texttt{dexnet-agent} command line application for  fully configuration-driven application aggregation.
  \end{enumerate}
\end{frame}

\begin{frame}
  \frametitle{\texttt{dexnet} Ported Graph Abstraction}

  \begin{description}
  \item[node] a \textbf{procedure} operating asynchronously from others.
  \item[port] a \textbf{labeled point} of a node through which data flows.
  \item[edge] a \textbf{connection between two ports} which transmits \textbf{messages} following one or more \textbf{protocols}
  \item[message] a typed \textbf{unit of data} transmitted along an edge
  \item[protocol] a set of \textbf{message types} and \textbf{expected behavior} of their transmission.
  \end{description}

  \footnotesize

  Note: any connected subgraph of a ported graph may be recast into a single ported node.   This has strong implications on complexity management.

  \vfill

  A \textit{node} may be a fundamental \textit{actor}, an \textit{actor} which manages other \textit{actors} (an ``agent''), a logical subgraph, 
or an application that merely provides or emulates \textit{ports}.

\end{frame}



\begin{frame}
  \frametitle{Actor Example}
  Four actors work together to process data
  \begin{columns}
    \begin{column}{0.5\textwidth}
      \begin{itemize}\scriptsize
      \item The ``actor pipes'' (PAIR sockets) speak ``control'' protocol sending RC commands to the actors.
      \item Actors create additional sockets and \texttt{bind}/\texttt{connect} to endpoint addresses based on ``control'' protocol messages and initial configuration.
      \item The ``ribbon'' protocol is the exchange of working data.
      \end{itemize}
    \end{column}
    \begin{column}{0.5\textwidth}
      \includegraphics[width=\textwidth]{actor.pdf}
    \end{column}
  \end{columns}

  \begin{itemize}\scriptsize
  \item The ``control'' protocol connects to RC directly, or via intervening application code. 
  \item Actors may be in individual applications communicating across network or aggregated into one and communicating across local thread-safe queues.
  \end{itemize}
\end{frame}

\section{Development and Demonstration}
\begin{frame}
  \tableofcontents[currentsection,hideothersubsections]
\end{frame}

\begin{frame}
  \frametitle{\texttt{dexnet} Development Effort}

  \begin{itemize}\footnotesize
  \item A lot of prototyping exists already so many problems worked out.  
  \item My intention (if the consortium is behind it) is:
    \begin{itemize}\scriptsize
    \item[o] Make \texttt{dexnet} a higher priority (it's mostly been a ``weekend project'' so far).
    \item[o] Complete remaining prototype issues.
    \item[o] Move the code into production quality code base.
    \item[o] Along the way, respond to input from DAQ application developers.
    \end{itemize}
  \item I think I need a minimum of \textbf{two or three part-time individuals} to closely collaborate on \textbf{design, development and testing}.
  \item It would be best if these individuals were using their other part-time to develop the ``business logic'' of actual DAQ applications.
  \end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Timeline}
  \begin{itemize}
  \item I am a bad judge of time.
  \item But, I feel the effort for \texttt{dexnet} + applications is comparable to what was needed for the Wire-Cell Toolkit
    \begin{itemize}\footnotesize
    \item That is now in its $\approx$3rd year.
    \item WCT had off-again/on-again effort, so ``3 years'' should be in quotes.
    \item Much of the core development of WCT has been in place a while and it's mostly been new ``application'' (``algorithm'') work lately.
    \end{itemize}
  \item So, I think, given the help, the \texttt{dexnet} core might take 6 months to a year to be very usable.  
  \item Full ``business logic'' applications need their own timeline estimation but it should progress in parallel with \texttt{dexnet} development.

  \end{itemize}

\end{frame}


\begin{frame}
  \frametitle{ProtoDUNE Demonstration}
  Paraphrasing Dave Newbold:
  \begin{quote}
    Splice in a software trigger into current protoDUNE timing system based trigger to test stuff.
  \end{quote}

  \begin{columns}
    \begin{column}{0.8\textwidth}
      \begin{center}
        \includegraphics[width=0.9\textwidth]{protodune-demo.pdf}
      \end{center}
    \end{column}
    \begin{column}{0.2\textwidth}
      \scriptsize 

      dash: hard wired

      dots: network

      solid: full data
    \end{column}
  \end{columns}


\end{frame}

\begin{frame}
  \frametitle{ProtoDUNE Demonstration Caveats}
  \begin{itemize}
  \item We can (and will) test \texttt{dexnet} without protoDUNE.
  \item Writing protoDUNE ``adapters'' will take real work.
  \item We need to understand what collective/emergent features/problems such a demonstration would test in exchange for the effort.
  \end{itemize}
\end{frame}


\section{}

\begin{frame}
  \begin{center}
    \Huge $\mathcal{FIN}$    
  \end{center}
\end{frame}


\end{document}

