% Created 2026-03-06 Fri 08:49
% Intended LaTeX compiler: pdflatex
\documentclass[usenames,dvipsnames,colorlinks,aspectratio=169]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{color}
\usepackage{listings}
\usepackage{pdfpages}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{siunitx}
\usepackage{libertine}
\usepackage{fontawesome}
\mode<beamer>{\usetheme{Boadilla}}
\setbeamertemplate{navigation symbols}{}
\setbeamerfont{alerted text}{series=\bfseries}
\setbeamercolor{alerted text}{fg=blue}
\usecolortheme{default}
\usefonttheme{serif}
\usefonttheme{professionalfonts}
\lstloadlanguages{[ISO]C++}
\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.50,0,0.82}
\definecolor{codeorange}{rgb}{0.95,0.3,0.1}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}
\lstdefinestyle{mystyle}{
commentstyle=\color{codegreen},
keywordstyle=\color{codeorange},
numberstyle=\tiny\color{codegray},
stringstyle=\color{codepurple},
basicstyle=\footnotesize,
breakatwhitespace=false,
breaklines=true,
captionpos=b,
keepspaces=true,
numbers=left,
numbersep=5pt,
showspaces=false,
showstringspaces=false,
showtabs=false,
tabsize=2
}
\lstset{style=mystyle}
\usetheme{default}
\author{Brett Viren}
\date{\today}
\title{GPU on WCWC and SDCC}
\hypersetup{
 pdfauthor={Brett Viren},
 pdftitle={GPU on WCWC and SDCC},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 30.1 (Org mode 9.8-pre)},
 pdflang={English}}
\begin{document}

\maketitle
\begin{frame}[label={sec:org33a8177},fragile]{WCWC - Wire-Cell Workstation Cluster}
 Two main ``workstations'' with GPUs: \texttt{wcgpu0} and \texttt{wcgpu1}, each has:
\begin{itemize}
\item GPUs: \(2\times\) RTX 4090, 24 GB VRAM
\item CPU: AMD Ryzen Threadripper 7970X 32-Cores
\item 256 GB CPU RAM, \(\mathcal{O}(1)\) TB fast nVME SSD storage.
\end{itemize}
Caveats: \alert{NO BACKUPS}, SSDs are node-local, \texttt{\$HOME} is shared but on slow NFS.
\end{frame}
\begin{frame}[label={sec:org40e8e5e},fragile]{WCWC - usage policy}
 Guiding philosophy: \alert{make it easy to try new things}.

\begin{itemize}
\item \alert{Not intended for production processing nor operational use.}

\item Rough separation of usage:

\begin{description}
\item[{\texttt{wcgpu0}}] is for Prompt-Processing work

\item[{\texttt{wcgpu1}}] is for ``everything else''.
\end{description}

\item We manage the shared resources through ad-hoc coordination.
\begin{itemize}
\item Talk to each other to work out resource conflicts.
\item Use \texttt{nvidia-smi} to determine if and which GPUs are already busy.
\item Check \texttt{top} to see if you have orphan VSCode/node or claude.
\begin{itemize}
\item These use a LOT of RAM.  Sometimes 100\% CPU zombies.
\end{itemize}
\end{itemize}
\end{itemize}

\alert{Avoid long running GPU usage if/when it impacts other users.}
\end{frame}
\begin{frame}[label={sec:org0090f3d},fragile]{WCWC - some future plans}
 \begin{block}{Hardware additions}
I am pursuing some hand-me-down EOL hardware from SDCC:

\begin{itemize}
\item \(\mathcal{O}(1)\) PB HDD storage server.
\item 24 GB SSD storage server.
\item May need to purchase/find 10+ Gbps switch to fully benefit.
\end{itemize}
\end{block}
\begin{block}{OS upgrades}
\begin{itemize}
\item Need to upgrade from Debian 12 to 13 at some point.
\item Will fix bad partitioning choices (fix tiny \texttt{/tmp} problem).
\item May re-do the authentication to separate from Phy dept servers.
\item Will try to combine hardware and OS upgrades at same time.
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:org81ccd47}]{}
\Huge

\begin{center}
WCWC \(\to\) SDCC
\end{center}
\end{frame}
\begin{frame}[label={sec:org08ed0f3},fragile]{SDCC interactive and batch access}
 Need SDCC account in the ``dune'' group to allow:

\scriptsize
\begin{verbatim}
$ ssh -A user@ssh.sdcc.bnl.gov  # SSH keys
$ ssh    user@dunesub01         # SSH keys or Kerberos password
$ condor_submit myjob.sub
\end{verbatim}
\normalsize
\end{frame}
\begin{frame}[label={sec:org105ce69},fragile]{Aside: an SDCC login trick}
 Jump right to internal node
\scriptsize
\begin{verbatim}
bv@external$ ssh dunesub01
bvlbne@dunesub01:~>
\end{verbatim}
\normalsize
With \texttt{\textasciitilde{}/.ssh/config} bits like:
\scriptsize
\begin{verbatim}
Host dunesub01 dunesub02
     Hostname %h.sdcc.bnl.gov
Match host *.sdcc.bnl.gov
      User USERNAME           # <-- replace with your SDCC user
Match host ssh.sdcc.bnl.gov 
      ProxyJump none
Match host ssh*.sdcc.bnl.gov 
      ProxyJump none
Match host *.sdcc.bnl.gov 
      ProxyJump ssh.sdcc.bnl.gov
      PubkeyAcceptedKeyTypes +ssh-rsa
      PubkeyAcceptedAlgorithms +ssh-rsa
\end{verbatim}

\normalsize

Can do similar for campus hosts using Physics gateway or SDCC's cssh.
\end{frame}
\begin{frame}[label={sec:org30a106f},fragile]{SDCC GPUs}
 \scriptsize
\begin{verbatim}
$ bvlbne@dunesub01:~> condor_status -constraint 'TotalGpus > 0' \
  -af Machine TotalGpus GPUs_DeviceName GPUs_Capability GPUs_GlobalMemoryMb

sgpu0001.usatlas.bnl.gov 4 undefined undefined undefined
sgpu0002.usatlas.bnl.gov 4 NVIDIA A100 80GB PCIe MIG 2g.20gb undefined 19894
sgpu0003.sdcc.bnl.gov 8 NVIDIA L40S 8.9 45458
sgpu0004.sdcc.bnl.gov 8 NVIDIA L40S 8.9 45458
\end{verbatim}

\normalsize

\begin{itemize}
\item Prompt-processing R\&D proposal payed for the \texttt{sgpu0004} node.
\item \(1\times\) \alert{L40S} GPU \(\approx\) \(2\times\) RTX 4090 (by VRAM and CUDA core count).
\item Only 8 of 10 GPUs shown, 2 held for JupyterHub access.  This allocation may change.
\item Not sure if the USATLAS \alert{A100} node(s) are accessible to ``dune''.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:orgac0a436}]{SDCC GPU node access}
Prompt Processing group's policy (for now):

\begin{quote}
Any ``dune'' user can land jobs on the GPU node.
\end{quote}

If/when this puts a crunch on Prompt Processing R\&D we may put up some technical
guardrails.

\vfill

Not yet clear if there is already a ``special list'' for GPU node access.
\begin{itemize}
\item PP group members have confirmed their access.
\item When you try to submit to the GPU nodes, let me know how it goes.
\item Also not yet clear if we can land jobs on the other visible GPU nodes.
\end{itemize}
\end{frame}
\begin{frame}[label={sec:org59e9ebc},fragile]{SDCC batch GPU jobs - minimal example HTCondor file}
 \scriptsize
\begin{verbatim}
$ cat myjob.sub
num_gpus = 1

executable     = /usr/bin/nvidia-smi  # A quick test
transfer_executable = False  # False if executable not on the submit host

universe       = vanilla
output         = nvidia_smi.$(Cluster).$(Process).out
error          = nvidia_smi.$(Cluster).$(Process).err
log            = nvidia_smi.log

request_gpus   = $(num_gpus)
Requirements = (GPUs_DeviceName == "NVIDIA L40S") && (GPUs_Capability == 8.9)

should_transfer_files = YES
when_to_transfer_output = ON_EXIT
queue 1

$ condor_submit myjob.sub
\end{verbatim}
\end{frame}
\begin{frame}[label={sec:org6ede6c8},fragile]{SDCC batch GPU - allocating GPUs}
 Set number of GPUs you need in HTCondor job script:
\begin{verbatim}
request_gpus = 2
\end{verbatim}

In the job's environment:
\begin{verbatim}
echo $CUDA_VISIBLE_DEVICES
4,7
\end{verbatim}

If your job does not honor \texttt{CUDA\_VISIBLE\_DEVICES} you must translate that into
some way to set the GPU IDs to use.

\alert{Never manually set \texttt{CUDA\_VISIBLE\_DEVICES} in your code/script.}
\end{frame}
\begin{frame}[label={sec:orgee55f90},fragile]{SDCC storage caveats}
 \begin{block}{DUNE storage is gifted to us by SDCC}
Until we can pay real money, we must beg and thank you for anything we get.
\end{block}
\begin{block}{Storage for DUNE on SDCC}
\begin{itemize}
\item Shared 100 GB global for \texttt{\$HOME}.
\item Shared 10 TB global at \texttt{/gpfs/mnt/gpfs01/lbne}.
\item Node-local SSD tmp during job life (potentially up to 10 TB).
\item \alert{No long term storage, NO BACKUPS}
\end{itemize}
\end{block}
\begin{block}{Suggestions / comments}
\begin{itemize}
\item FR calculations are easy for GPU batch, minimal data files.
\item FM training with sub-TB of data can put files into GPFS.
\begin{itemize}
\item GPFS is network so not ``SSD-fast'', GPFS \(\to\) local disk caching may be useful.
\end{itemize}
\end{itemize}
\end{block}
\end{frame}
\begin{frame}[label={sec:orgf286051}]{JupyterHub}
Web UI for Jupyter Notebooks and a web-based SSH terminal.

\begin{center}
\url{https://dune-jupyter.sdcc.bnl.gov/}
\end{center}

\begin{itemize}
\item Access CPU-only and CPU+GPU nodes.
\begin{itemize}
\item Select how many CPU cores, CPU RAM and number of GPU devices.
\item Your session will only ``see'' what was requested.
\end{itemize}
\item Can use/supply containers to provide the environment.
\item This service is still in development so fasten seat belts.
\begin{itemize}
\item Doug Benjamin can provide help.
\item Moving to federated login and we will need to enroll existing SDCC account in ``COmanage''
\item Requires ``SDCC MFA'' TOTP code + SDCC Kerberos password.
\end{itemize}
\end{itemize}

\scriptsize

\begin{center}
\url{https://www.sdcc.bnl.gov/information/comanage-setup-new-accounts}
\end{center}
\end{frame}
\begin{frame}[label={sec:orgce7c2d1}]{}
\Huge
\[\mathcal{FIN}\]
\end{frame}
\end{document}
