#!/bin/bash

#SBATCH -C gpu
#SBATCH --nodes=1
#SBATCH --gres=gpu:1
#SBATCH --exclusive
#SBATCH -t 4:00:00
#SBATCH --mail-type=begin,end,fail
#SBATCH --mail-user=matti@fnal.gov

NGPU=1

SRUN="srun -n 1 -c 80"
NUMACTL="numactl --cpunodebind=0"

SCAN_OPTS="--append --tryAgain 4"
OPTS=
PREFIX=result_cuda_pr129_cuda11
#PREFIX=result_cudauvm_pr53
#PREFIX=result_cudauvm_pr53_noadvise
#PREFIX=result_cudauvm_pr53_noprefetch

#OPTS=--transfer
#PREFIX=result_cuda_transfer

if [ "x${NGPU}" == "x1" ]; then
    SCAN_OPTS+=" --maxStreamsToAddEvents 5"
    NUMTHREADS=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
    #NUMTHREADS=1,2,3,4,5,6,7,8,10,11,12,14,16,18
    #NUMTHREADS=1,2,3,4,5,6,7,8,10,12,14,16
    #NUMTHREADS=1,2,3,4,5,6,7,8,9,10,12,14,16,17,18,19,20
elif [ "x${NGPU}" == "x2" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 8"
    NUMTHREADS=2,4,6,8,10,12,14,16,18,20
elif [ "x${NGPU}" == "x3" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 9"
    NUMTHREADS=3,6,9,12,15,18,21
elif [ "x${NGPU}" == "x4" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 8"
    NUMTHREADS=4,8,12,16,20
elif [ "x${NGPU}" == "x5" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 5"
    NUMTHREADS=5,10,15,20
elif [ "x${NGPU}" == "x6" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 6"
    NUMTHREADS=6,12,18
elif [ "x${NGPU}" == "x7" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 7"
    NUMTHREADS=7,14,21
elif [ "x${NGPU}" == "x8" ]; then 
    SCAN_OPTS+=" --maxStreamsToAddEvents 8"
    NUMTHREADS=8,16
fi

#PREFIX=result_cuda_pr129_cuda11_nstr9
#NUMTHREADS=2,3,4,5,6,7,8,9
#SCAN_OPTS+=" --numStreams 9"

#echo $SCAN_OPTS
#exit

source $SCRATCH/cce/pixeltrack-standalone/env.sh
$SRUN nvidia-smi
$SRUN $NUMACTL $SCRATCH/cce/pixeltrack-standalone/run-scan.py -o ${PREFIX}_ngpu${NGPU} --numThreads ${NUMTHREADS} ${SCAN_OPTS} $SCRATCH/cce/pixeltrack-standalone/cuda $OPTS
#$SRUN $NUMACTL $SCRATCH/cce/pixeltrack-standalone/run-scan.py -o ${PREFIX}_ngpu${NGPU} --numThreads ${NUMTHREADS} ${SCAN_OPTS} $SCRATCH/cce/pixeltrack-standalone/cudauvm $OPTS
