% Created 2026-05-14 Thu 10:06
% Intended LaTeX compiler: pdflatex
\documentclass[usenames,dvipsnames,colorlinks,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{color}
\usepackage{listings}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{siunitx}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\setbeamerfont{alerted text}{series=\bfseries}
\setbeamercolor{alerted text}{fg=blue}
\usecolortheme{default}
\usefonttheme{serif}
\usefonttheme{professionalfonts}
\lstloadlanguages{[ISO]C++}
\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.50,0,0.82}
\definecolor{codeorange}{rgb}{0.95,0.3,0.1}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\lstdefinestyle{mystyle}{
commentstyle=\color{codegreen},
keywordstyle=\color{codeorange},
numberstyle=\tiny\color{codegray},
stringstyle=\color{codepurple},
basicstyle=\footnotesize,
breakatwhitespace=false,
breaklines=true,
captionpos=b,
keepspaces=true,
numbers=left,
numbersep=5pt,
showspaces=false,
showstringspaces=false,
showtabs=false,
tabsize=2
}
\lstset{style=mystyle}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{LLM-assisted WCT testing cleanup}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={LLM-assisted WCT testing cleanup},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 30.1 (Org mode 9.8-pre)},
 pdflang={English}}
\begin{document}

\maketitle
\begin{frame}[label={sec:org1324c57},fragile]{Overview}
 \href{https://github.com/WireCell/wire-cell-toolkit/issues/474}{Issue 474} describes problems with WCT \texttt{<pkg>/test/test*.cxx} style unit tests:

\begin{itemize}
\item We have about 100, each has a `main()` and needs a link leading to long build times.
\item Each is a isolated monolith, no way to run ``suites'' nor individual test cases.
\item Thoughtless \texttt{cout/cerr} spray, no logging.
\item Some \alert{do not even test} (no asserts) and are more demos or rely on human examination of output.
\item Some require command line args or input files.
\end{itemize}

We also have about 250 Doctest test cases which avoid most of these problems.
\end{frame}
\begin{frame}[label={sec:orgba51188}]{Want a magic converter}
\begin{center}
\LARGE test \(\to\) doctest
\end{center}

\vfill

Doing this by hand is tedious.

\begin{itemize}
\item LLMs excel at tedium
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org6e16390},fragile]{Just do it}
 \begin{verbatim}
<user> Hey claude, convert all this crap to gold.
<claude> Uh, okay, uh,... uh... ouch, my context just exploded.
\end{verbatim}
\end{frame}
\begin{frame}[label={sec:org0a5d2e1}]{Construct a partitioning of the problem - diagram}
\begin{center}
\includegraphics[width=.9\linewidth]{./migrate-construct.pdf}
\end{center}
\end{frame}
\begin{frame}[label={sec:org5786704},fragile]{Construct a partitioning of the problem - description}
 \begin{enumerate}
\item LLM surveys/categorizes each test: ``migrate to doctest'' vs ``convert to a check''
\begin{itemize}
\item This is quick and can be done in the main agent with a simple ad-hoc prompt.
\item Target \texttt{util/test/}, LLM makes: \texttt{util-tests.txt} listing ``tests that actually test''.
\begin{itemize}
\item And \texttt{util-checks.txt} for tests that are better as WCT ``check'' programs.
\end{itemize}
\end{itemize}
\item LLM iterates \texttt{util-tests.txt}, makes a Beads issue for each test.
\begin{itemize}
\item It writes a deterministic Bash loop calling \texttt{bd create} for each.
\end{itemize}
\item LLM writes a ``claude command'' Markdown file with arguments: bead issue + test file name.
\begin{itemize}
\item It's content described next.
\end{itemize}
\item LLM writes a shell script \texttt{run-migration.sh}
\begin{itemize}
\item Calls `claude -p` on one Beads issue or can iterate on all remaining.
\item Allows a fresh LLM context for each issue and a human-friendly CLI.
\item Can load new issues and rerun script for new \texttt{test*.cxx} to migrate.
\end{itemize}
\item Human reviews the ``command'' Markdown file and the script.
\begin{itemize}
\item Smoke test a single migration before letting the script loop over the remaining.
\end{itemize}
\end{enumerate}
\end{frame}
\begin{frame}[label={sec:orgb5cc46a},fragile]{The migration ``command'' \texttt{migrate-test.md}}
 The ``command''\footnote{A Claude ``command'' is run explicitly while a ``skill'' is chosen by the LLM to run.} is just a markdown file with \texttt{\$ARGUMENTS} and LLM prompt content:
\begin{itemize}
\item Parse arguments, form Doctest file and suite name based on old \texttt{test*.cxx} file name.
\item Read the \texttt{test*.cxx} source, write new \texttt{doctest-*.cxx} source:
\begin{itemize}
\item Replace \texttt{cout/cerr} with \texttt{spdlog::debug()}, remove \texttt{main()}.
\item Identify test-like functions, make each into a Doctest \texttt{TEST\_CASE()}.
\item Map existing types of \texttt{assert} to equivalent in Doctest.
\item Add \texttt{CHECK()} wherever it looks good but include a \texttt{CHECK(true)} to mark unchecked code.
\end{itemize}
\item Post-migration actions:
\begin{itemize}
\item New doctest must build, give build instruction.
\item New doctest must pass, fix test until it does.
\item Remove old \texttt{test*.cxx} file.
\item Update Beads issue with summary and close.
\item Make new Beads issue if human follow up is warranted.
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org8c8173a},fragile]{Exercising the workflow}
 \begin{center}
\includegraphics[width=.9\linewidth]{./migrate-arch.pdf}
\end{center}

\begin{itemize}
\item Most of the ``smarts'' is in \texttt{migrate-test.md} ``command'' to generate the prompt.
\item Beads tracks progress and allows state to survive loss of an LLM session.
\item The \texttt{run-migrate.sh} gives a simple high-level CLI for the human to type.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:orga08053b},fragile]{Doing the migration}
 \begin{block}{Single target smoke test}
\begin{verbatim}
./run-migrate wct-0jr.36   # <-- that's a beads issue ID 
\end{verbatim}

Make sure all the machinery works with a single target.
\begin{itemize}
\item Minor outer shell script bugs found/fixed.
\end{itemize}
\end{block}
\begin{block}{Do full loop}
\begin{verbatim}
./run-migrate              # no arg means loop on issues
\end{verbatim}

Run `claude -p` with the ``command'' serially on each Bead issue.
\begin{itemize}
\item Each conversion takes about 1-5 minutes.
\item Can watch progress on terminal or go get a coffee.
\begin{itemize}
\item No stopping, uses \texttt{-{}-{}allowedTools} to pre-authorize only tools that are needed.
\end{itemize}
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:org35092f8},fragile]{Summary of resulting DocTests}
 40 \texttt{test*.cxx} become 91 \texttt{TEST\_CASE}'s in about 1 hour of processing time.

\begin{verbatim}
[doctest] test cases:   244 |   244 passed | 0 failed | 0 skipped
[doctest] assertions: 13685 | 13685 passed | 0 failed |
[doctest] Status: SUCCESS!
\end{verbatim}
\end{frame}
\begin{frame}[label={sec:orgfecac09}]{New issues generated}
The LLM decided 3 from 40 migrations need human follow up.
\begin{itemize}
\item No assertions in original and none could be determined by the LLM.
\item No assertions and non-trivial to invent some without deeper checking of code under test.
\item Original assertion commented, human needs to define intention.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org9eb969b}]{The other half of the problem}
45/85 are categorized as ``migrate to a check''

Roughly equal sized sub-categories (human sorting)
\begin{itemize}
\item Demo code just to see how some code works.
\item Any real ``testing'' requires interpreting stdout or generated figures.
\item Requires file input needing fodder and/or does file output needing tempfile support.
\end{itemize}
We can iterate this method to handle these with novel ``command'' files.
\end{frame}
\begin{frame}[label={sec:org15dee5d},fragile]{Summary}
 Translating about 40 annoying tests to 91 better ones was automated with LLM.
\begin{itemize}
\item Claude Sonnet 4.6.
\item Took about 1 hour of setup (half day if you count constant interruption of daily stuff).
\begin{itemize}
\item Would take weeks of painful hand-coding.
\end{itemize}
\item Took about 1 hour of actual processing time.
\end{itemize}
Migration process had structural design driven by LLM limitations.
\begin{itemize}
\item Beads issues for persistent task information.
\item A ``command'' to for consistent, parameterized instruction prompt.
\item Fresh context (\texttt{claude -p}) for each file conserved token usage.
\item An overall script loop for serial automation.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:orgb6e53ce},fragile]{Caveats and comments on generalizing method}
 We do a really \alert{bad job} of regularly running \texttt{test*.cxx} tests so even with \alert{low-fidelity} results, the migration is \alert{low risk}.
\begin{itemize}
\item We also do a bad job of running Doctests but that's another story.
\item Higher risk code would need more human attention to the migration process and validation.
\end{itemize}

Gauged the time required by doing a few single-file migrations.
\begin{itemize}
\item Fast enough to let things run serially.
\item Larger migrations may push to adopt some parallelism.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org92b0138}]{}
\Huge
\[\mathcal{FIN}\]
\end{frame}
\begin{frame}[label={sec:org4ccdb62}]{Backups}
\end{frame}
\end{document}
