% Created 2026-05-20 Wed 14:42
% Intended LaTeX compiler: pdflatex
\documentclass[usenames,dvipsnames,colorlinks,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{color}
\usepackage{listings}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{siunitx}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\setbeamerfont{alerted text}{series=\bfseries}
\setbeamercolor{alerted text}{fg=blue}
\usecolortheme{default}
\usefonttheme{serif}
\usefonttheme{professionalfonts}
\lstloadlanguages{[ISO]C++}
\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.50,0,0.82}
\definecolor{codeorange}{rgb}{0.95,0.3,0.1}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\lstdefinestyle{mystyle}{
commentstyle=\color{codegreen},
keywordstyle=\color{codeorange},
numberstyle=\tiny\color{codegray},
stringstyle=\color{codepurple},
basicstyle=\footnotesize,
breakatwhitespace=false,
breaklines=true,
captionpos=b,
keepspaces=true,
numbers=left,
numbersep=5pt,
showspaces=false,
showstringspaces=false,
showtabs=false,
tabsize=2
}
\lstset{style=mystyle}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{DUNE Data Model in the Phlex Era}
\subtitle{Toward a \emph{lingua franca} for our algorithms}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={DUNE Data Model in the Phlex Era},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 30.1 (Org mode 9.8-pre)},
 pdflang={English}}

% Setup for code blocks [1/2]

\usepackage{fvextra}

\fvset{%
  commandchars=\\\{\},
  highlightcolor=white!95!black!80!blue,
  breaklines=true,
  breaksymbol=\color{white!60!black}\tiny\ensuremath{\hookrightarrow}}

% Make line numbers smaller and grey.
\renewcommand\theFancyVerbLine{\footnotesize\color{black!40!white}\arabic{FancyVerbLine}}

\usepackage{xcolor}

% In case engrave-faces-latex-gen-preamble has not been run.
\providecolor{EfD}{HTML}{f7f7f7}
\providecolor{EFD}{HTML}{28292e}

% Define a Code environment to prettily wrap the fontified code.
\usepackage[breakable,xparse]{tcolorbox}
\DeclareTColorBox[]{Code}{o}%
{colback=EfD!98!EFD, colframe=EfD!95!EFD,
  fontupper=\footnotesize\setlength{\fboxsep}{0pt},
  colupper=EFD,
  IfNoValueTF={#1}%
  {boxsep=2pt, arc=2.5pt, outer arc=2.5pt,
    boxrule=0.5pt, left=2pt}%
  {boxsep=2.5pt, arc=0pt, outer arc=0pt,
    boxrule=0pt, leftrule=1.5pt, left=0.5pt},
  right=2pt, top=1pt, bottom=0.5pt,
  breakable}

% Support listings with captions
\usepackage{float}
\floatstyle{plain}
\newfloat{listing}{htbp}{lst}
\newcommand{\listingsname}{Listing}
\floatname{listing}{\listingsname}
\newcommand{\listoflistingsname}{List of Listings}
\providecommand{\listoflistings}{\listof{listing}{\listoflistingsname}}


% Setup for code blocks [2/2]: syntax highlighting colors

\newcommand\efstrut{\vrule height 2.1ex depth 0.8ex width 0pt}
\definecolor{EFD}{HTML}{000000}
\definecolor{EfD}{HTML}{ffffff}
\newcommand{\EFD}[1]{\textcolor{EFD}{#1}} % default
\definecolor{EFvp}{HTML}{000000}
\newcommand{\EFvp}[1]{\textcolor{EFvp}{#1}} % variable-pitch
\definecolor{EFh}{HTML}{7f7f7f}
\newcommand{\EFh}[1]{\textcolor{EFh}{#1}} % shadow
\definecolor{EFsc}{HTML}{228b22}
\newcommand{\EFsc}[1]{\textcolor{EFsc}{\textbf{#1}}} % success
\definecolor{EFw}{HTML}{ff8e00}
\newcommand{\EFw}[1]{\textcolor{EFw}{\textbf{#1}}} % warning
\definecolor{EFe}{HTML}{ff0000}
\newcommand{\EFe}[1]{\textcolor{EFe}{\textbf{#1}}} % error
\definecolor{EFl}{HTML}{ff0000}
\newcommand{\EFl}[1]{\textcolor{EFl}{#1}} % link
\definecolor{EFlv}{HTML}{ff0000}
\newcommand{\EFlv}[1]{\textcolor{EFlv}{#1}} % link-visited
\definecolor{EFhi}{HTML}{ff0000}
\newcommand{\EFhi}[1]{\textcolor{EFhi}{#1}} % highlight
\definecolor{EFc}{HTML}{b22222}
\newcommand{\EFc}[1]{\textcolor{EFc}{#1}} % font-lock-comment-face
\definecolor{EFcd}{HTML}{b22222}
\newcommand{\EFcd}[1]{\textcolor{EFcd}{#1}} % font-lock-comment-delimiter-face
\definecolor{EFs}{HTML}{8b2252}
\newcommand{\EFs}[1]{\textcolor{EFs}{#1}} % font-lock-string-face
\definecolor{EFd}{HTML}{8b2252}
\newcommand{\EFd}[1]{\textcolor{EFd}{#1}} % font-lock-doc-face
\definecolor{EFm}{HTML}{008b8b}
\newcommand{\EFm}[1]{\textcolor{EFm}{#1}} % font-lock-doc-markup-face
\definecolor{EFk}{HTML}{9370db}
\newcommand{\EFk}[1]{\textcolor{EFk}{#1}} % font-lock-keyword-face
\definecolor{EFb}{HTML}{483d8b}
\newcommand{\EFb}[1]{\textcolor{EFb}{#1}} % font-lock-builtin-face
\definecolor{EFf}{HTML}{0000ff}
\newcommand{\EFf}[1]{\textcolor{EFf}{#1}} % font-lock-function-name-face
\definecolor{EFv}{HTML}{a0522d}
\newcommand{\EFv}[1]{\textcolor{EFv}{#1}} % font-lock-variable-name-face
\definecolor{EFt}{HTML}{228b22}
\newcommand{\EFt}[1]{\textcolor{EFt}{#1}} % font-lock-type-face
\definecolor{EFo}{HTML}{008b8b}
\newcommand{\EFo}[1]{\textcolor{EFo}{#1}} % font-lock-constant-face
\definecolor{EFwr}{HTML}{ff0000}
\newcommand{\EFwr}[1]{\textcolor{EFwr}{\textbf{#1}}} % font-lock-warning-face
\newcommand{\EFnc}[1]{#1} % font-lock-negation-char-face
\definecolor{EFpp}{HTML}{483d8b}
\newcommand{\EFpp}[1]{\textcolor{EFpp}{#1}} % font-lock-preprocessor-face
\newcommand{\EFrc}[1]{\textbf{#1}} % font-lock-regexp-grouping-construct
\newcommand{\EFrb}[1]{\textbf{#1}} % font-lock-regexp-grouping-backslash
\newcommand{\EFob}[1]{#1} % org-block
\newcommand{\EFobb}[1]{#1} % org-block-begin-line
\newcommand{\EFobe}[1]{#1} % org-block-end-line
\definecolor{EFOa}{HTML}{0000ff}
\newcommand{\EFOa}[1]{\textcolor{EFOa}{#1}} % outline-1
\definecolor{EFOb}{HTML}{a0522d}
\newcommand{\EFOb}[1]{\textcolor{EFOb}{#1}} % outline-2
\definecolor{EFOc}{HTML}{a020f0}
\newcommand{\EFOc}[1]{\textcolor{EFOc}{#1}} % outline-3
\definecolor{EFOd}{HTML}{b22222}
\newcommand{\EFOd}[1]{\textcolor{EFOd}{#1}} % outline-4
\definecolor{EFOe}{HTML}{228b22}
\newcommand{\EFOe}[1]{\textcolor{EFOe}{#1}} % outline-5
\definecolor{EFOf}{HTML}{008b8b}
\newcommand{\EFOf}[1]{\textcolor{EFOf}{#1}} % outline-6
\definecolor{EFOg}{HTML}{483d8b}
\newcommand{\EFOg}[1]{\textcolor{EFOg}{#1}} % outline-7
\definecolor{EFOh}{HTML}{8b2252}
\newcommand{\EFOh}[1]{\textcolor{EFOh}{#1}} % outline-8
\definecolor{EFhn}{HTML}{008b8b}
\newcommand{\EFhn}[1]{\textcolor{EFhn}{#1}} % highlight-numbers-number
\definecolor{EFhq}{HTML}{9370db}
\newcommand{\EFhq}[1]{\textcolor{EFhq}{#1}} % highlight-quoted-quote
\definecolor{EFhs}{HTML}{008b8b}
\newcommand{\EFhs}[1]{\textcolor{EFhs}{#1}} % highlight-quoted-symbol
\definecolor{EFrda}{HTML}{707183}
\newcommand{\EFrda}[1]{\textcolor{EFrda}{#1}} % rainbow-delimiters-depth-1-face
\definecolor{EFrdb}{HTML}{7388d6}
\newcommand{\EFrdb}[1]{\textcolor{EFrdb}{#1}} % rainbow-delimiters-depth-2-face
\definecolor{EFrdc}{HTML}{909183}
\newcommand{\EFrdc}[1]{\textcolor{EFrdc}{#1}} % rainbow-delimiters-depth-3-face
\definecolor{EFrdd}{HTML}{709870}
\newcommand{\EFrdd}[1]{\textcolor{EFrdd}{#1}} % rainbow-delimiters-depth-4-face
\definecolor{EFrde}{HTML}{907373}
\newcommand{\EFrde}[1]{\textcolor{EFrde}{#1}} % rainbow-delimiters-depth-5-face
\definecolor{EFrdf}{HTML}{6276ba}
\newcommand{\EFrdf}[1]{\textcolor{EFrdf}{#1}} % rainbow-delimiters-depth-6-face
\definecolor{EFrdg}{HTML}{858580}
\newcommand{\EFrdg}[1]{\textcolor{EFrdg}{#1}} % rainbow-delimiters-depth-7-face
\definecolor{EFrdh}{HTML}{80a880}
\newcommand{\EFrdh}[1]{\textcolor{EFrdh}{#1}} % rainbow-delimiters-depth-8-face
\definecolor{EFrdi}{HTML}{887070}
\newcommand{\EFrdi}[1]{\textcolor{EFrdi}{#1}} % rainbow-delimiters-depth-9-face
\definecolor{EFany}{HTML}{CDCD00}
\newcommand{\EFany}[1]{\textcolor{EFany}{#1}} % ansi-color-yellow
\definecolor{EFanr}{HTML}{CD0000}
\newcommand{\EFanr}[1]{\textcolor{EFanr}{#1}} % ansi-color-red
\definecolor{EFanb}{HTML}{000000}
\newcommand{\EFanb}[1]{\textcolor{EFanb}{#1}} % ansi-color-black
\definecolor{EFang}{HTML}{00CD00}
\newcommand{\EFang}[1]{\textcolor{EFang}{#1}} % ansi-color-green
\definecolor{EFanB}{HTML}{0000EE}
\newcommand{\EFanB}[1]{\textcolor{EFanB}{#1}} % ansi-color-blue
\definecolor{EFanc}{HTML}{00CDCD}
\newcommand{\EFanc}[1]{\textcolor{EFanc}{#1}} % ansi-color-cyan
\definecolor{EFanw}{HTML}{E5E5E5}
\newcommand{\EFanw}[1]{\textcolor{EFanw}{#1}} % ansi-color-white
\definecolor{EFanm}{HTML}{CD00CD}
\newcommand{\EFanm}[1]{\textcolor{EFanm}{#1}} % ansi-color-magenta
\definecolor{EFANy}{HTML}{EEEE00}
\newcommand{\EFANy}[1]{\textcolor{EFANy}{#1}} % ansi-color-bright-yellow
\definecolor{EFANr}{HTML}{EE0000}
\newcommand{\EFANr}[1]{\textcolor{EFANr}{#1}} % ansi-color-bright-red
\newcommand{\EFANb}[1]{#1} % ansi-color-bright-black
\definecolor{EFANg}{HTML}{00EE00}
\newcommand{\EFANg}[1]{\textcolor{EFANg}{#1}} % ansi-color-bright-green
\definecolor{EFANB}{HTML}{0000FF}
\newcommand{\EFANB}[1]{\textcolor{EFANB}{#1}} % ansi-color-bright-blue
\definecolor{EFANc}{HTML}{00EEEE}
\newcommand{\EFANc}[1]{\textcolor{EFANc}{#1}} % ansi-color-bright-cyan
\newcommand{\EFANw}[1]{#1} % ansi-color-bright-white
\newcommand{\EFANm}[1]{#1} % ansi-color-bright-magenta
\begin{document}

\maketitle
\begin{frame}[label={sec:org3b36c69}]{Topics}
\begin{itemize}
\item Why DUNE needs a shared data model in Phlex
\item Lessons from the past: nativist vs. generalist strategies
\item The problem space: what ``data model'' actually means
\item A concrete but partial proposal: decide \alert{format} now, defer \alert{schema} choices for later
\item What we may decide ``today''; what work remains
\end{itemize}
\end{frame}
\begin{frame}[label={sec:orgc57a9f1}]{The coordination problem}
Phlex executes algorithms from many independent packages:

\begin{description}
\item[{Wire-Cell}] drift, det sim, signal processing, imaging, clustering
\item[{Pandora}] multi-algorithm pattern recognition
\item[{SPINE}] deep-learning reconstruction
\item[{\ldots{}}] and many individuals' efforts
\end{description}

Each package has its own data model, some directly use LArSoft's.

Without a shared model, every type of producer-consumer pairing needs a converter:
\[
N_{\mathrm{producers}} \times M_{\mathrm{consumers}} \;\text{ converters}
\]
\end{frame}
\begin{frame}[label={sec:orga5c83dd}]{Data model lessons from art/LArSoft}
\begin{block}{What worked}
\begin{itemize}
\item Expressive, common ``vocabulary'' of ``nouns'' (data products) and ``verbs'' (art modules).
\item Made data products that are independent from art (I think generally true?).
\item Reduced but did not eliminate need for data product converters.
\end{itemize}
\end{block}
\begin{block}{What did not work}
\begin{itemize}
\item Data products do not provide the right ``shape'' for all needs
\begin{itemize}
\item WCT's 3D imaging and clustering have no match in LArSoft.
\end{itemize}
\item C++ structures with no direct file I/O support
\end{itemize}
\end{block}
\begin{block}{Key lessons}
\begin{itemize}
\item Keep data model independent from any particular framework.
\item Do not lock-in too many specifics data features from the start.
\item Extensibility as first-class feature and have an eye open for file I/O.
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:org1a206f0},fragile]{Two approaches to interoperability}
 \begin{block}{\alert{Nativist}: wrap native types at the exchange boundary}
\begin{Code}
\begin{Verbatim}
\color{EFD}\EFk{struct} \EFt{Frame} \{ \EFo{WireCell}::\EFo{IFrame}::\EFt{pointer} \EFv{ptr}; \};  \EFcd{//} \EFc{thin wrapper}
\end{Verbatim}
\end{Code}
\begin{description}
\item[{Good}] perfect fidelity and efficiency (locally), easy to add new algorithm.
\item[{Bad}] consumers must depend on the producer's library or \(N\times M\) converter problem.
\end{description}

\normalsize
\end{block}
\begin{block}{\alert{Generalist}: define shared types, any converts are an algorithm's problem}
\begin{description}
\item[{Good}] common vocabulary, decouples inter-package dependencies
\item[{Bad}] wrong choices marginalize packages; still need some converters.
\end{description}
\end{block}
\end{frame}
\begin{frame}[label={sec:org9af77ce}]{For best results we need both and the key is where to draw the lines}
Major packages will not adopt a new internal data model.
\begin{itemize}
\item Some nativism is inevitable and some converters are always needed.
\end{itemize}

A \textbf{shared data model} collapses the combinatoric:
\[
N \times M \;\longrightarrow\; N + M \;\text{ converters}
\]

The shared model must be \textbf{expressive enough} not to marginalize any package.
\begin{itemize}
\item But, still lock down enough details to provide a useful \emph{lingua franca}.
\end{itemize}

\vfill
\textit{The choice of shared model is therefore a high-stakes design decision.}
\end{frame}
\begin{frame}[label={sec:orgd53f132}]{The full problem is large. Let's break it into two layers.}
\begin{block}{\alert{Format}}
The low-level \textbf{in-memory representation} --- how bits are arranged in memory
Concrete, technology-bounded, implementable today.
\end{block}
\begin{block}{\alert{Schema}}
The \textbf{semantic layer} --- field names, units, relationships, conventions.  Best to gain consensus between developers of producer/consumer algorithm pairs with eye toward future novel work.
\end{block}
\end{frame}
\begin{frame}[label={sec:org60f0a1f}]{The full problem space is rather larger}
\begin{center}
\begin{tabular}{l>{\raggedright\arraybackslash}p{11cm}}
Concern & Key questions\\
\hline
\textbf{Format} & What in-memory representations are \alert{sufficient}?\\
\textbf{Schema} & What fields, units, relationships describe our data?\\
File I/O & HDF5? ROOT? Zarr? Custom scripts?\\
CPU/GPU & Can the same object live on device and host memory?\\
C++/Python & Zero-copy interop or a serialization boundary?\\
Phlex layers & How does data fit Phlex's \textbf{DFP graph} and  \textbf{data-layer hierarchy}?\\
Versioning & How do schema evolve without breaking existing readers?\\
Provenance & What metadata must travel with every data product?\\
\end{tabular}
\end{center}

\vfill
\alert{This talk addresses Format.} We try to keep in mind the larger problems but defer them.
\end{frame}
\begin{frame}[label={sec:org6fcdd2a}]{The key move: decouple format from schema}
Most data model designs conflate the two.

\begin{itemize}
\item \textbf{Schema} changes often: new detector, new algorithm, new physics.
\item \textbf{Format} can be stable: it is a small vocabulary of representations.
\end{itemize}

Decoupling means:
\begin{itemize}
\item Schema translation becomes \textit{reinterpretation of the same memory} --- not data copying.
\item Adding a new schema does not require a new format library.
\item Some problems can be solved at the simpler format level (eg, file I/O).
\end{itemize}

\vfill
\textit{Proposal: standardize on a small set of general formats now; let schema emerge from algorithm community discussions.}
\end{frame}
\begin{frame}[label={sec:orgda78b4a},fragile]{Format primitive 1: \alert{structured objects}}
 JSON-like nested key-value objects with typed scalars and arrays.

\begin{itemize}
\item Naturally expresses heterogeneous, hierarchical data.
\item \textbf{Use for:} metadata, geometry, configuration, provenance records.
\item C++ candidates: \texttt{nlohmann/json}, \texttt{boost::json}; Python: native \texttt{dict/json}.
\item Easy transfer between C++ and Python via serialization.  Some candidates allow direct sharing of object memory.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org3f0101d},fragile]{Format primitive 2: \alert{rectangular arrays}}
 Typed, shaped, strided dense arrays --- the Numpy array model.

\begin{itemize}
\item \textbf{Use for:} ADC waveforms, deposition clouds, images, feature tensors, point clouds.
\item Zero-copy C++/Python transfer via the buffer protocol and DLPack standard.
\begin{itemize}
\item \texttt{Eigen::Map}, \texttt{torch::from\_blob} map existing memory without copying.
\end{itemize}
\item Candidate solving other problems: \texttt{libtorch} (PyTorch C++ API).
\begin{itemize}
\item CPU/GPU transparent: same type, same code, different device.
\item Already required for ML inference and GPU-accelerated Wire-Cell (SPNG).
\item True, it is a large dependency --- but one we carry regardless.
\end{itemize}
\item More discussion, research and testing before a final decision.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org930d405},fragile]{Format primitive 3: \alert{sparse and irregular arrays}}
 Variable-length, nested, irregular structure --- the ``awkward'' array model.

\begin{itemize}
\item \textbf{Use for:} hit lists, cluster contents, track segments, signal-processed wires.
\begin{itemize}
\item WCT signal processing output is inherently sparse.
\item Graphs fit naturally as nested irregular arrays.
\end{itemize}
\item \textbf{Leading candidate:} the Scikit-HEP \texttt{awkward} library.
\begin{itemize}
\item Python-native with \texttt{awkward-cpp} for zero-copy C++ interop.
\item Columnar layout: efficient for bulk operations over collections.
\item Commonly used in the HEP Python ecosystem, less so on C++ side.
\end{itemize}
\end{itemize}
\begin{block}{Note}
Arguably, awkward array types are a combination of structured object and
rectangular array types.  It could be emitted though it provides representations
that are closer to working memory and may avoid the need for some conversions.
The cost/benefit needs discussion.
\end{block}
\end{frame}
\begin{frame}[label={sec:org976944b}]{Every DUNE data type fits these three primitives}
\begin{center}
\begin{tabular}{l l l}
Data type & Format & Notes\\
\hline
ADC waveforms, images & Rectangular & (channels \(\times\) ticks)\\
Simulated depositions & Rectangular & (N deposits \(\times\) 7+4 fields)\\
Signal-processed signals & Sparse/awkward & zero-suppressed waveform fragments\\
Hit/cluster lists & Sparse/awkward & variable hits per cluster\\
Track segments & Sparse/awkward & variable points per track\\
ML feature tensors & Rectangular & CPU or GPU, same type\\
Run/event/geometry metadata & Structured object & JSON-like, hierarchical\\
Provenance records & Structured object & attached to any data product\\
\end{tabular}
\end{center}

Again, sparse/awkward types can reduce to rectangular array and/or structured objects.
\end{frame}
\begin{frame}[label={sec:org8efd26c}]{Proposed package architecture}
A layered factoring keeps format independent of framework and application.
\footnotesize
\begin{enumerate}
\item \textbf{Format libraries} (no Phlex, no application dependency)
  \begin{itemize}
    \item \texttt{dune-data-model-cpp} --- C++ \textbf{format types} and utilities
    \item \texttt{dune-data-model-python} --- Python \textbf{format} equivalent, zero-copy interop (goal)
    \item \texttt{dune-data-model-transfer} --- cross-language transfer (depends on both)
  \end{itemize}
\item \textbf{Phlex integration layer} \texttt{phlex-dune-data}
  \begin{itemize}
    \item Thin wrappers to expose \textbf{format types} to the Phlex graph.
    \item Depends on format libs + Phlex. \textbf{Does not} depend on any application lib.
  \end{itemize}
\item \textbf{Application converters} (one per major package)
  \begin{itemize}
    \item \texttt{\{wire-cell,pandora,spine\}-dune-data-model}
    \item Each depends only on its native library + one format lib. No cross-dependencies.
  \end{itemize}
\item \textbf{Schema libraries} (future) overlay specific structure types on format types.
  \begin{itemize}
    \item Some \textit{lingua franca}, some package-specific.
  \end{itemize}
\end{enumerate}
\normalsize
\end{frame}
\begin{frame}[label={sec:org5d2d3c1},fragile]{What remains open}
 Specifying \textit{format} is necessary but not sufficient. Three large open problems:
\begin{block}{Schema}
What fields, units, and relationships constitute, e.g., a ``hit'' or a ``cluster''?
Needs a working group with representatives from WCT, Pandora, SPINE, and the ND.
Wrong early commitments here replicate the LArSoft mistake.
\end{block}
\begin{block}{File I/O and serialization}
HDF5 (\texttt{HighFive})? ROOT \texttt{RNTuple}? Zarr? The format choice must be I/O-friendly.
``Official'' files and ``custom'' analysis scripts need different strategies.
\end{block}
\begin{block}{Versioning and provenance}
How do schema evolve without breaking existing analysis code?
What provenance metadata must every data product carry?
These policies must be defined before the first schema is frozen.
\end{block}
\end{frame}
\begin{frame}[label={sec:org112fa11},fragile]{Summary}
 \begin{block}{We propose to decide ``now'' at least in principle}
\begin{itemize}
\item Use a \textbf{three-primitive format vocabulary}: structured objects, rectangular arrays, sparse/awkward arrays.
\item Adopt a \textbf{layered package factoring} that isolates format from framework and application.
\item Straw-person choice of format-level technology candidates: \texttt{libtorch} / \texttt{awkward} / JSON library.
\end{itemize}
\end{block}
\begin{block}{We defer, pending community input}
\begin{itemize}
\item \textbf{Schema} definitions for all data types (needs a working group).
\item \textbf{File I/O} strategy: format choice, ROOT vs.$\backslash$ HDF5 vs.$\backslash$ Zarr.
\item \textbf{Versioning policy} and provenance metadata requirements.
\item \textbf{Phlex data-layer} specifics (frames, products, graph hierarchy).
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:org6d0b4b9}]{}
\Huge
\[\mathcal{FIN}\]
\end{frame}
\begin{frame}[label={sec:org993a836}]{Backups}
\end{frame}


\begin{frame}[label={sec:org383329c},fragile]{libtorch as the rectangular array candidate}
 \begin{block}{Pros}
\begin{itemize}
\item CPU and GPU tensors share the same type: \texttt{torch::Tensor}.
\item Mature, well-maintained, large community (PyTorch).
\item Already needed for ML inference and Wire-Cell SPNG.
\item DLPack interop with NumPy, JAX, CuPy.
\end{itemize}
\end{block}
\begin{block}{Cons}
\begin{itemize}
\item Large (but required) dependency.
\item C++ API is less ergonomic than Python; documentation lags (but LLMs know it).
\end{itemize}

Alternative: decouple GPU from the format layer --- use \texttt{xtensor} / \texttt{Eigen} on CPU and defer GPU to algorithm-internal tensors. Simpler but loses CPU/GPU transparency at the exchange boundary.
\end{block}
\end{frame}
\begin{frame}[label={sec:org197350b}]{Graph and point cloud representations}
Graphs and point clouds are often treated as special cases, but both reduce to arrays.
\begin{block}{Point cloud}
A rectangular array of shape \((N, d)\) where \(N\) is the number of points and \(d\) is the number of spatial (or feature) dimensions. No special format needed.
\end{block}
\begin{block}{Graph}
Two representations:
\begin{itemize}
\item \textbf{Adjacency matrix}: rectangular array, shape \((N, N)\). Dense; impractical for sparse graphs.
\item \textbf{Edge list}: rectangular array of shape \((2, E)\) for \(E\) edges (COO format). Sparse; standard in PyTorch Geometric and DGL.
\end{itemize}

Node and edge \textbf{features} are additional rectangular arrays aligned by index.
Variable-length neighbor lists fit naturally as awkward arrays.
\end{block}
\end{frame}
\end{document}
