{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess as sp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from io import StringIO\n",
    "import os\n",
    "import re\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#global parameters\n",
    "cudadir = \"/usr/common/software/cuda/10.2.89\"\n",
    "datadir = \"/global/cfs/cdirs/m2845/roofline/patatrack\"\n",
    "outputdir = \"/global/cfs/cdirs/m2845/roofline/patatrack\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transpose_frame(metricdf):\n",
    "    selectkeys = [\"Kernel Name\"]\n",
    "    \n",
    "    tc_peak_perf_flops = 125*10**12\n",
    "    \n",
    "\n",
    "    ####### Get timing information\n",
    "    \n",
    "    # get cycles and rates\n",
    "    cyclesdf = metricdf.loc[(metricdf[\"Metric Name\"]==\"smsp__cycles_elapsed\") & (metricdf[\"Metric Type\"]==\"total\"),\n",
    "                            selectkeys+[\"Metric Unit\", \"Metric Value\"]].copy().reset_index(drop=True).rename(columns={\"Metric Value\": \"CUDA Cycles\"})\n",
    "    ratesdf = metricdf.loc[(metricdf[\"Metric Name\"]==\"smsp__cycles_elapsed\") & (metricdf[\"Metric Type\"]==\"rate\"),\n",
    "                           selectkeys+[\"Metric Unit\", \"Metric Value\"]].copy().reset_index(drop=True).rename(columns={\"Metric Value\": \"CUDA Rates\"})\n",
    "    \n",
    "    # adjust metric unit\n",
    "    ratesdf.loc[ratesdf[\"Metric Unit\"].str.contains(\"cycle/nsecond\"), [\"CUDA Rates\"]] *= 1e9\n",
    "\n",
    "    # combine & cleanup\n",
    "    profiledf = pd.concat([cyclesdf, ratesdf], axis=1)\n",
    "    del profiledf[\"Metric Unit\"]\n",
    "    profiledf = profiledf.loc[:,~profiledf.columns.duplicated()]\n",
    "    \n",
    "    # compute CUDA time\n",
    "    profiledf[\"CUDA Time\"] = profiledf[\"CUDA Cycles\"] / profiledf[\"CUDA Rates\"]\n",
    "    \n",
    "    # cleanups\n",
    "    del profiledf[\"CUDA Cycles\"]\n",
    "    del profiledf[\"CUDA Rates\"]\n",
    "    profiledf = profiledf.groupby('Kernel Name').sum().reset_index()\n",
    "    #display(profiledf)\n",
    "        \n",
    "    ####### Get number of FLOPs\n",
    "    \n",
    "    ### FMA FLOPs = number of FMA instructions x 2\n",
    "    metricdf.loc[metricdf[\"Metric Name\"].str.contains(\"fma_pred_on\"), [\"Metric Value\"]] *= 2\n",
    "    \n",
    "\n",
    "    ### FP64 FLOPs\n",
    "    metrics = ['smsp__sass_thread_inst_executed_op_dadd_pred_on',\n",
    "               'smsp__sass_thread_inst_executed_op_dfma_pred_on',\n",
    "               'smsp__sass_thread_inst_executed_op_dmul_pred_on']\n",
    "    tmpdf = metricdf.loc[ metricdf[\"Metric Name\"].isin(metrics), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    tmpdf = tmpdf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"FP64 FLOPs\"})\n",
    "\n",
    "    profiledf = profiledf.merge(tmpdf, on=selectkeys, how=\"inner\")\n",
    "    \n",
    "    ### FP32 FLOPs\n",
    "    metrics = ['smsp__sass_thread_inst_executed_op_fadd_pred_on',\n",
    "               'smsp__sass_thread_inst_executed_op_ffma_pred_on',\n",
    "               'smsp__sass_thread_inst_executed_op_fmul_pred_on']\n",
    "    tmpdf = metricdf.loc[ metricdf[\"Metric Name\"].isin(metrics), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    tmpdf = tmpdf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"FP32 FLOPs\"})\n",
    "    \n",
    "    profiledf = profiledf.merge(tmpdf, on=selectkeys, how=\"inner\")\n",
    "    \n",
    "    ### Total FLOPs\n",
    "    profiledf[\"FLOPs\"] = profiledf[\"FP32 FLOPs\"] + profiledf[\"FP64 FLOPs\"]\n",
    "\n",
    "    \n",
    "    ####### Get number of bytes\n",
    "    \n",
    "    ### Shared transactions\n",
    "    # project out\n",
    "    shareddf = metricdf.loc[metricdf[\"Metric Name\"].str.contains(\"l1tex__data_pipe_lsu_wavefronts_mem_shared_op\"), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    shareddf = shareddf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"Shared Transactions\"})\n",
    "    # merge with output\n",
    "    profiledf = profiledf.merge(shareddf, on=selectkeys, how=\"inner\")\n",
    "    \n",
    "    ### L1 atomic transactions\n",
    "    # project out\n",
    "    metrics = ['l1tex__t_set_accesses_pipe_lsu_mem_global_op_atom',\n",
    "               'l1tex__t_set_accesses_pipe_lsu_mem_global_op_red',\n",
    "               'l1tex__t_set_accesses_pipe_tex_mem_surface_op_atom',\n",
    "               'l1tex__t_set_accesses_pipe_tex_mem_surface_op_red']\n",
    "    atomicdf = metricdf.loc[ metricdf[\"Metric Name\"].isin(metrics), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    atomicdf = atomicdf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"L1 Atomic Transactions\"})\n",
    "    # merge with output\n",
    "    profiledf = profiledf.merge(atomicdf, on=selectkeys, how=\"inner\")\n",
    "\n",
    "    ### Local transactions \n",
    "    # project out\n",
    "    localdf = metricdf.loc[metricdf[\"Metric Name\"].str.contains(\"l1tex__t_sectors_pipe_lsu_mem_local_op\"), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    localdf = localdf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"Local Transactions\"})\n",
    "    # merge with output\n",
    "    profiledf = profiledf.merge(localdf, on=selectkeys, how=\"inner\")\n",
    " \n",
    "    ### Global transactions \n",
    "    # project out\n",
    "    globaldf = metricdf.loc[metricdf[\"Metric Name\"].str.contains(\"l1tex__t_sectors_pipe_lsu_mem_global_op\"), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    globaldf = globaldf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"Global Transactions\"})\n",
    "    # merge with output\n",
    "    profiledf = profiledf.merge(globaldf, on=selectkeys, how=\"inner\")  \n",
    "    \n",
    "    ### L1 Bytes\n",
    "    profiledf[\"L1 Transactions\"] = (profiledf[\"Shared Transactions\"] + profiledf[\"L1 Atomic Transactions\"]\n",
    "                            + profiledf[\"Local Transactions\"] + profiledf[\"Global Transactions\"])\n",
    "    profiledf[\"L1 Bytes\"] = profiledf[\"L1 Transactions\"] * 32\n",
    "\n",
    "    # clean up\n",
    "    del profiledf[\"Shared Transactions\"]\n",
    "    del profiledf[\"L1 Atomic Transactions\"]\n",
    "    del profiledf[\"Local Transactions\"]\n",
    "    del profiledf[\"Global Transactions\"]\n",
    "        \n",
    "    \n",
    "    ### L2 atomic & reduction\n",
    "    metricdf.loc[(metricdf[\"Metric Name\"].str.contains(\"lts__t_sectors_op\")) & (metricdf[\"Metric Type\"]==\"total\"), [\"Metric Value\"]] *= 2\n",
    "    \n",
    "    ### L2 transactions\n",
    "    # project out\n",
    "    l2df = metricdf.loc[metricdf[\"Metric Name\"].str.contains(\"lts__t_sectors_op\"), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    l2df = l2df.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"L2 Transactions\"})\n",
    "    l2df[\"L2 Bytes\"] = l2df[\"L2 Transactions\"] * 32\n",
    "    # merge with output\n",
    "    profiledf = profiledf.merge(l2df, on=selectkeys, how=\"inner\")\n",
    "\n",
    "    \n",
    "    ### DRAM Bytes\n",
    "    # project out\n",
    "    dramdf = metricdf.loc[metricdf[\"Metric Name\"].str.contains(\"dram__sectors\"), selectkeys+[\"Metric Value\"] ].copy()\n",
    "    dramdf = dramdf.groupby(selectkeys).sum().reset_index().rename(columns={\"Metric Value\": \"DRAM Transactions\"})\n",
    "    dramdf[\"DRAM Bytes\"] = dramdf[\"DRAM Transactions\"] * 32\n",
    "    # merge with output\n",
    "    profiledf = profiledf.merge(dramdf, on=selectkeys, how=\"inner\")\n",
    "    \n",
    "\n",
    "    ### Get performance\n",
    "    profiledf[\"Performance GFlop/s\"]      = profiledf[\"FLOPs\"]      / (profiledf[\"CUDA Time\"]*10**9)\n",
    "    profiledf[\"FP64 Performance GFlop/s\"] = profiledf[\"FP64 FLOPs\"] / (profiledf[\"CUDA Time\"]*10**9)\n",
    "    profiledf[\"FP32 Performance GFlop/s\"] = profiledf[\"FP32 FLOPs\"] / (profiledf[\"CUDA Time\"]*10**9)\n",
    "\n",
    "    \n",
    "    ### Get AI\n",
    "    # L1\n",
    "    profiledf[\"L1 AI\"]        = profiledf[\"FLOPs\"]      / profiledf[\"L1 Bytes\"]\n",
    "    profiledf[\"FP64 L1 AI\"]   = profiledf[\"FP64 FLOPs\"] / profiledf[\"L1 Bytes\"]\n",
    "    profiledf[\"FP32 L1 AI\"]   = profiledf[\"FP32 FLOPs\"] / profiledf[\"L1 Bytes\"]\n",
    "    # L2\n",
    "    profiledf[\"L2 AI\"]        = profiledf[\"FLOPs\"]      / profiledf[\"L2 Bytes\"]\n",
    "    profiledf[\"FP64 L2 AI\"]   = profiledf[\"FP64 FLOPs\"] / profiledf[\"L2 Bytes\"]\n",
    "    profiledf[\"FP32 L2 AI\"]   = profiledf[\"FP32 FLOPs\"] / profiledf[\"L2 Bytes\"]\n",
    "    # DRAM\n",
    "    profiledf[\"DRAM AI\"]      = profiledf[\"FLOPs\"]      / profiledf[\"DRAM Bytes\"]\n",
    "    profiledf[\"FP64 DRAM AI\"] = profiledf[\"FP64 FLOPs\"] / profiledf[\"DRAM Bytes\"]\n",
    "    profiledf[\"FP32 DRAM AI\"] = profiledf[\"FP32 FLOPs\"] / profiledf[\"DRAM Bytes\"]\n",
    "\n",
    "    return profiledf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Kernel Name</th>\n",
       "      <th>Metric Name</th>\n",
       "      <th>Metric Type</th>\n",
       "      <th>Metric Unit</th>\n",
       "      <th>Metric Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...</td>\n",
       "      <td>dram__sectors</td>\n",
       "      <td>read</td>\n",
       "      <td>sector</td>\n",
       "      <td>22932.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...</td>\n",
       "      <td>dram__sectors</td>\n",
       "      <td>write</td>\n",
       "      <td>sector</td>\n",
       "      <td>50046.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...</td>\n",
       "      <td>l1tex__data_pipe_lsu_wavefronts_mem_shared_op</td>\n",
       "      <td>read</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...</td>\n",
       "      <td>l1tex__data_pipe_lsu_wavefronts_mem_shared_op</td>\n",
       "      <td>write</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...</td>\n",
       "      <td>l1tex__t_sectors_pipe_lsu_mem_global_op</td>\n",
       "      <td>read</td>\n",
       "      <td>sector</td>\n",
       "      <td>30624.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95995</td>\n",
       "      <td>gpuVertexFinder::vertexFinderOneKernel(ZVertex...</td>\n",
       "      <td>smsp__sass_thread_inst_executed_op_dfma_pred_on</td>\n",
       "      <td>total</td>\n",
       "      <td>inst</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95996</td>\n",
       "      <td>gpuVertexFinder::vertexFinderOneKernel(ZVertex...</td>\n",
       "      <td>smsp__sass_thread_inst_executed_op_dmul_pred_on</td>\n",
       "      <td>total</td>\n",
       "      <td>inst</td>\n",
       "      <td>580.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95997</td>\n",
       "      <td>gpuVertexFinder::vertexFinderOneKernel(ZVertex...</td>\n",
       "      <td>smsp__sass_thread_inst_executed_op_fadd_pred_on</td>\n",
       "      <td>total</td>\n",
       "      <td>inst</td>\n",
       "      <td>46053.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95998</td>\n",
       "      <td>gpuVertexFinder::vertexFinderOneKernel(ZVertex...</td>\n",
       "      <td>smsp__sass_thread_inst_executed_op_ffma_pred_on</td>\n",
       "      <td>total</td>\n",
       "      <td>inst</td>\n",
       "      <td>10261.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>95999</td>\n",
       "      <td>gpuVertexFinder::vertexFinderOneKernel(ZVertex...</td>\n",
       "      <td>smsp__sass_thread_inst_executed_op_fmul_pred_on</td>\n",
       "      <td>total</td>\n",
       "      <td>inst</td>\n",
       "      <td>49382.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>96000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Kernel Name  \\\n",
       "0      pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...   \n",
       "1      pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...   \n",
       "2      pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...   \n",
       "3      pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...   \n",
       "4      pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...   \n",
       "...                                                  ...   \n",
       "95995  gpuVertexFinder::vertexFinderOneKernel(ZVertex...   \n",
       "95996  gpuVertexFinder::vertexFinderOneKernel(ZVertex...   \n",
       "95997  gpuVertexFinder::vertexFinderOneKernel(ZVertex...   \n",
       "95998  gpuVertexFinder::vertexFinderOneKernel(ZVertex...   \n",
       "95999  gpuVertexFinder::vertexFinderOneKernel(ZVertex...   \n",
       "\n",
       "                                           Metric Name Metric Type  \\\n",
       "0                                        dram__sectors        read   \n",
       "1                                        dram__sectors       write   \n",
       "2        l1tex__data_pipe_lsu_wavefronts_mem_shared_op        read   \n",
       "3        l1tex__data_pipe_lsu_wavefronts_mem_shared_op       write   \n",
       "4              l1tex__t_sectors_pipe_lsu_mem_global_op        read   \n",
       "...                                                ...         ...   \n",
       "95995  smsp__sass_thread_inst_executed_op_dfma_pred_on       total   \n",
       "95996  smsp__sass_thread_inst_executed_op_dmul_pred_on       total   \n",
       "95997  smsp__sass_thread_inst_executed_op_fadd_pred_on       total   \n",
       "95998  smsp__sass_thread_inst_executed_op_ffma_pred_on       total   \n",
       "95999  smsp__sass_thread_inst_executed_op_fmul_pred_on       total   \n",
       "\n",
       "      Metric Unit  Metric Value  \n",
       "0          sector       22932.0  \n",
       "1          sector       50046.0  \n",
       "2             NaN           0.0  \n",
       "3             NaN           0.0  \n",
       "4          sector       30624.0  \n",
       "...           ...           ...  \n",
       "95995        inst           0.0  \n",
       "95996        inst         580.0  \n",
       "95997        inst       46053.0  \n",
       "95998        inst       10261.0  \n",
       "95999        inst       49382.0  \n",
       "\n",
       "[96000 rows x 5 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get raw data file\n",
    "filename = \"patatrack_roofline.csv\"\n",
    "metricdf = pd.read_csv(os.path.join(datadir,filename))\n",
    "\n",
    "#fuse read/write metrics together:\n",
    "unique_metrics = metricdf[\"Metric Name\"].unique()\n",
    "unique_metrics = set([x.replace(\".sum\",\"\").replace(\"_write\",\"\").replace(\"_read\",\"\").replace(\"_ld\",\"\").replace(\"_st\",\"\") for x in unique_metrics])\n",
    "#add the metric type\n",
    "metricdf[\"Metric Type\"] = \"total\"\n",
    "#read\n",
    "metricdf.loc[ metricdf[ \"Metric Name\" ].str.contains(\"_read\"), \"Metric Type\" ] = \"read\"\n",
    "metricdf.loc[ metricdf[ \"Metric Name\" ].str.contains(\"_ld\"), \"Metric Type\" ] = \"read\"\n",
    "#write\n",
    "metricdf.loc[ metricdf[ \"Metric Name\" ].str.contains(\"_write\"), \"Metric Type\" ] = \"write\"\n",
    "metricdf.loc[ metricdf[ \"Metric Name\" ].str.contains(\"_st\"), \"Metric Type\" ] = \"write\"\n",
    "#rate\n",
    "metricdf.loc[ metricdf[ \"Metric Name\" ].str.contains(\".per_second\"), \"Metric Type\" ] = \"rate\"\n",
    "#rename metrics\n",
    "for metric in unique_metrics:\n",
    "    metricdf.loc[ metricdf[ \"Metric Name\"].str.startswith(metric), \"Metric Name\" ] = metric\n",
    "\n",
    "metricdf = metricdf[[\"Kernel Name\", \"Metric Name\", \"Metric Type\", \"Metric Unit\", \"Metric Value\"]]\n",
    "    \n",
    "display(metricdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute AI Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Kernel Name</th>\n",
       "      <th>CUDA Time</th>\n",
       "      <th>FP64 FLOPs</th>\n",
       "      <th>FP32 FLOPs</th>\n",
       "      <th>FLOPs</th>\n",
       "      <th>L1 Transactions</th>\n",
       "      <th>L1 Bytes</th>\n",
       "      <th>L2 Transactions</th>\n",
       "      <th>L2 Bytes</th>\n",
       "      <th>DRAM Transactions</th>\n",
       "      <th>...</th>\n",
       "      <th>FP32 Performance GFlop/s</th>\n",
       "      <th>L1 AI</th>\n",
       "      <th>FP64 L1 AI</th>\n",
       "      <th>FP32 L1 AI</th>\n",
       "      <th>L2 AI</th>\n",
       "      <th>FP64 L2 AI</th>\n",
       "      <th>FP32 L2 AI</th>\n",
       "      <th>DRAM AI</th>\n",
       "      <th>FP64 DRAM AI</th>\n",
       "      <th>FP32 DRAM AI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>kernel_find_ntuplets(TrackingRecHit2DSOAView c...</td>\n",
       "      <td>0.018547</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>221543156.0</td>\n",
       "      <td>7.089381e+09</td>\n",
       "      <td>225398161.0</td>\n",
       "      <td>7.212741e+09</td>\n",
       "      <td>278432471.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>void kernelBLFit&lt;int=4&gt;(HistoContainer&lt;unsigne...</td>\n",
       "      <td>0.014658</td>\n",
       "      <td>265470717.0</td>\n",
       "      <td>2.071517e+07</td>\n",
       "      <td>2.861859e+08</td>\n",
       "      <td>9403323.0</td>\n",
       "      <td>3.009063e+08</td>\n",
       "      <td>7501232.0</td>\n",
       "      <td>2.400394e+08</td>\n",
       "      <td>5333291.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.413275</td>\n",
       "      <td>0.951080</td>\n",
       "      <td>0.882237</td>\n",
       "      <td>0.068843</td>\n",
       "      <td>1.192245</td>\n",
       "      <td>1.105946</td>\n",
       "      <td>0.086299</td>\n",
       "      <td>1.676884</td>\n",
       "      <td>1.555505</td>\n",
       "      <td>0.121379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>gpuVertexFinder::vertexFinderOneKernel(ZVertex...</td>\n",
       "      <td>0.012052</td>\n",
       "      <td>56235.0</td>\n",
       "      <td>1.143011e+07</td>\n",
       "      <td>1.148635e+07</td>\n",
       "      <td>8849541.0</td>\n",
       "      <td>2.831853e+08</td>\n",
       "      <td>1176192.0</td>\n",
       "      <td>3.763814e+07</td>\n",
       "      <td>211899.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.948378</td>\n",
       "      <td>0.040561</td>\n",
       "      <td>0.000199</td>\n",
       "      <td>0.040363</td>\n",
       "      <td>0.305178</td>\n",
       "      <td>0.001494</td>\n",
       "      <td>0.303684</td>\n",
       "      <td>1.693960</td>\n",
       "      <td>0.008293</td>\n",
       "      <td>1.685667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>gpuPixelDoublets::getDoubletsFromHisto(GPUCACe...</td>\n",
       "      <td>0.011098</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.459555e+08</td>\n",
       "      <td>5.459555e+08</td>\n",
       "      <td>339960809.0</td>\n",
       "      <td>1.087875e+10</td>\n",
       "      <td>168911671.0</td>\n",
       "      <td>5.405173e+09</td>\n",
       "      <td>67157969.0</td>\n",
       "      <td>...</td>\n",
       "      <td>49.192655</td>\n",
       "      <td>0.050186</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.050186</td>\n",
       "      <td>0.101006</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.101006</td>\n",
       "      <td>0.254044</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.254044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>gpuClustering::findClus(unsigned short const *...</td>\n",
       "      <td>0.009689</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>150617334.0</td>\n",
       "      <td>4.819755e+09</td>\n",
       "      <td>71288368.0</td>\n",
       "      <td>2.281228e+09</td>\n",
       "      <td>19548934.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>kernel_connect(AtomicPairCounter*,AtomicPairCo...</td>\n",
       "      <td>0.009273</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.756497e+09</td>\n",
       "      <td>1.756497e+09</td>\n",
       "      <td>199460307.0</td>\n",
       "      <td>6.382730e+09</td>\n",
       "      <td>92061427.0</td>\n",
       "      <td>2.945966e+09</td>\n",
       "      <td>83274917.0</td>\n",
       "      <td>...</td>\n",
       "      <td>189.415546</td>\n",
       "      <td>0.275195</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.275195</td>\n",
       "      <td>0.596238</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.596238</td>\n",
       "      <td>0.659149</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.659149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>void kernelBLFit&lt;int=3&gt;(HistoContainer&lt;unsigne...</td>\n",
       "      <td>0.006044</td>\n",
       "      <td>862708818.0</td>\n",
       "      <td>6.009670e+07</td>\n",
       "      <td>9.228055e+08</td>\n",
       "      <td>29479274.0</td>\n",
       "      <td>9.433368e+08</td>\n",
       "      <td>25623242.0</td>\n",
       "      <td>8.199437e+08</td>\n",
       "      <td>12158427.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.942758</td>\n",
       "      <td>0.978236</td>\n",
       "      <td>0.914529</td>\n",
       "      <td>0.063707</td>\n",
       "      <td>1.125450</td>\n",
       "      <td>1.052156</td>\n",
       "      <td>0.073294</td>\n",
       "      <td>2.371826</td>\n",
       "      <td>2.217363</td>\n",
       "      <td>0.154463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>gpuPixelDoublets::fishbone(TrackingRecHit2DSOA...</td>\n",
       "      <td>0.005508</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.291534e+08</td>\n",
       "      <td>1.291534e+08</td>\n",
       "      <td>55955823.0</td>\n",
       "      <td>1.790586e+09</td>\n",
       "      <td>32633287.0</td>\n",
       "      <td>1.044265e+09</td>\n",
       "      <td>52896500.0</td>\n",
       "      <td>...</td>\n",
       "      <td>23.446294</td>\n",
       "      <td>0.072129</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.072129</td>\n",
       "      <td>0.123679</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.123679</td>\n",
       "      <td>0.076301</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.076301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>gpuPixelRecHits::getHits(pixelCPEforGPU::Param...</td>\n",
       "      <td>0.005237</td>\n",
       "      <td>48629316.0</td>\n",
       "      <td>1.757141e+08</td>\n",
       "      <td>2.243434e+08</td>\n",
       "      <td>74233078.0</td>\n",
       "      <td>2.375458e+09</td>\n",
       "      <td>13837022.0</td>\n",
       "      <td>4.427847e+08</td>\n",
       "      <td>8975533.0</td>\n",
       "      <td>...</td>\n",
       "      <td>33.551715</td>\n",
       "      <td>0.094442</td>\n",
       "      <td>0.020472</td>\n",
       "      <td>0.073971</td>\n",
       "      <td>0.506665</td>\n",
       "      <td>0.109826</td>\n",
       "      <td>0.396839</td>\n",
       "      <td>0.781094</td>\n",
       "      <td>0.169312</td>\n",
       "      <td>0.611781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>void kernelBLFastFit&lt;int=4&gt;(HistoContainer&lt;uns...</td>\n",
       "      <td>0.002464</td>\n",
       "      <td>10586982.0</td>\n",
       "      <td>1.102198e+07</td>\n",
       "      <td>2.160896e+07</td>\n",
       "      <td>4044162.0</td>\n",
       "      <td>1.294132e+08</td>\n",
       "      <td>2676045.0</td>\n",
       "      <td>8.563344e+07</td>\n",
       "      <td>4435744.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.473264</td>\n",
       "      <td>0.166976</td>\n",
       "      <td>0.081808</td>\n",
       "      <td>0.085169</td>\n",
       "      <td>0.252343</td>\n",
       "      <td>0.123631</td>\n",
       "      <td>0.128711</td>\n",
       "      <td>0.152236</td>\n",
       "      <td>0.074586</td>\n",
       "      <td>0.077650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>kernel_fastDuplicateRemover(GPUCACell const *,...</td>\n",
       "      <td>0.002291</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.255730e+05</td>\n",
       "      <td>2.255730e+05</td>\n",
       "      <td>15968332.0</td>\n",
       "      <td>5.109866e+08</td>\n",
       "      <td>13325212.0</td>\n",
       "      <td>4.264068e+08</td>\n",
       "      <td>33382056.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098464</td>\n",
       "      <td>0.000441</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000441</td>\n",
       "      <td>0.000529</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000529</td>\n",
       "      <td>0.000211</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>kernel_earlyDuplicateRemover(GPUCACell const *...</td>\n",
       "      <td>0.001883</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>16798424.0</td>\n",
       "      <td>5.375496e+08</td>\n",
       "      <td>13118927.0</td>\n",
       "      <td>4.198057e+08</td>\n",
       "      <td>33448644.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>gpuClustering::clusterChargeCut(unsigned short...</td>\n",
       "      <td>0.001838</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>19344017.0</td>\n",
       "      <td>6.190085e+08</td>\n",
       "      <td>5170048.0</td>\n",
       "      <td>1.654415e+08</td>\n",
       "      <td>3624740.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>void cub::DeviceScanKernel&lt;cub::DispatchScan&lt;u...</td>\n",
       "      <td>0.001608</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2031529.0</td>\n",
       "      <td>6.500893e+07</td>\n",
       "      <td>1321764.0</td>\n",
       "      <td>4.229645e+07</td>\n",
       "      <td>1646333.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>void kernelBLFastFit&lt;int=3&gt;(HistoContainer&lt;uns...</td>\n",
       "      <td>0.001144</td>\n",
       "      <td>45042830.0</td>\n",
       "      <td>3.578728e+07</td>\n",
       "      <td>8.083011e+07</td>\n",
       "      <td>13025818.0</td>\n",
       "      <td>4.168262e+08</td>\n",
       "      <td>8409971.0</td>\n",
       "      <td>2.691191e+08</td>\n",
       "      <td>4930462.0</td>\n",
       "      <td>...</td>\n",
       "      <td>31.274700</td>\n",
       "      <td>0.193918</td>\n",
       "      <td>0.108061</td>\n",
       "      <td>0.085857</td>\n",
       "      <td>0.300351</td>\n",
       "      <td>0.167371</td>\n",
       "      <td>0.132979</td>\n",
       "      <td>0.512313</td>\n",
       "      <td>0.285488</td>\n",
       "      <td>0.226825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>kernel_tripletCleaner(TrackingRecHit2DSOAView ...</td>\n",
       "      <td>0.001038</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.645110e+05</td>\n",
       "      <td>1.645110e+05</td>\n",
       "      <td>4731904.0</td>\n",
       "      <td>1.514209e+08</td>\n",
       "      <td>1923370.0</td>\n",
       "      <td>6.154784e+07</td>\n",
       "      <td>1977586.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.158471</td>\n",
       "      <td>0.001086</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001086</td>\n",
       "      <td>0.002673</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002673</td>\n",
       "      <td>0.002600</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>pixelgpudetails::fillHitsModuleStart(unsigned ...</td>\n",
       "      <td>0.000836</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>251134.0</td>\n",
       "      <td>8.036288e+06</td>\n",
       "      <td>157193.0</td>\n",
       "      <td>5.030176e+06</td>\n",
       "      <td>131478.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...</td>\n",
       "      <td>0.000811</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>9645895.0</td>\n",
       "      <td>3.086686e+08</td>\n",
       "      <td>8496372.0</td>\n",
       "      <td>2.718839e+08</td>\n",
       "      <td>8122109.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>kernel_fillHitInTracks(HistoContainer&lt;unsigned...</td>\n",
       "      <td>0.000752</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2746634.0</td>\n",
       "      <td>8.789229e+07</td>\n",
       "      <td>2822053.0</td>\n",
       "      <td>9.030570e+07</td>\n",
       "      <td>1825586.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>kernel_classifyTracks(HistoContainer&lt;unsigned ...</td>\n",
       "      <td>0.000748</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.048192e+06</td>\n",
       "      <td>3.048192e+06</td>\n",
       "      <td>4657927.0</td>\n",
       "      <td>1.490537e+08</td>\n",
       "      <td>3624459.0</td>\n",
       "      <td>1.159827e+08</td>\n",
       "      <td>4367582.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.076872</td>\n",
       "      <td>0.020450</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.020450</td>\n",
       "      <td>0.026281</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.026281</td>\n",
       "      <td>0.021810</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>kernel_fillMultiplicity(HistoContainer&lt;unsigne...</td>\n",
       "      <td>0.000747</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1510071.0</td>\n",
       "      <td>4.832227e+07</td>\n",
       "      <td>1405484.0</td>\n",
       "      <td>4.497549e+07</td>\n",
       "      <td>1141576.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>gpuCalibPixel::calibDigis(unsigned short*,unsi...</td>\n",
       "      <td>0.000707</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.431565e+07</td>\n",
       "      <td>5.431565e+07</td>\n",
       "      <td>7282382.0</td>\n",
       "      <td>2.330362e+08</td>\n",
       "      <td>3660331.0</td>\n",
       "      <td>1.171306e+08</td>\n",
       "      <td>6993283.0</td>\n",
       "      <td>...</td>\n",
       "      <td>76.849056</td>\n",
       "      <td>0.233078</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.233078</td>\n",
       "      <td>0.463719</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.463719</td>\n",
       "      <td>0.242713</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.242713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>kernel_countMultiplicity(HistoContainer&lt;unsign...</td>\n",
       "      <td>0.000675</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1218239.0</td>\n",
       "      <td>3.898365e+07</td>\n",
       "      <td>1113844.0</td>\n",
       "      <td>3.564301e+07</td>\n",
       "      <td>1077542.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>void cub::DeviceScanInitKernel&lt;cub::ScanTileSt...</td>\n",
       "      <td>0.000656</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3300.0</td>\n",
       "      <td>1.056000e+05</td>\n",
       "      <td>12600.0</td>\n",
       "      <td>4.032000e+05</td>\n",
       "      <td>17577.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>void cms::cuda::fillFromVector&lt;HistoContainer&lt;...</td>\n",
       "      <td>0.000583</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1144366.0</td>\n",
       "      <td>3.661971e+07</td>\n",
       "      <td>1688932.0</td>\n",
       "      <td>5.404582e+07</td>\n",
       "      <td>565733.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>gpuVertexFinder::loadTracks(TrackSoAT&lt;int=3276...</td>\n",
       "      <td>0.000568</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.623500e+04</td>\n",
       "      <td>5.623500e+04</td>\n",
       "      <td>1207916.0</td>\n",
       "      <td>3.865331e+07</td>\n",
       "      <td>679323.0</td>\n",
       "      <td>2.173834e+07</td>\n",
       "      <td>1380305.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.098944</td>\n",
       "      <td>0.001455</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001455</td>\n",
       "      <td>0.002587</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002587</td>\n",
       "      <td>0.001273</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>void cms::cuda::countFromVector&lt;HistoContainer...</td>\n",
       "      <td>0.000550</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>727864.0</td>\n",
       "      <td>2.329165e+07</td>\n",
       "      <td>1272469.0</td>\n",
       "      <td>4.071901e+07</td>\n",
       "      <td>316439.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>kernel_countHitInTracks(HistoContainer&lt;unsigne...</td>\n",
       "      <td>0.000541</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1789441.0</td>\n",
       "      <td>5.726211e+07</td>\n",
       "      <td>2053051.0</td>\n",
       "      <td>6.569763e+07</td>\n",
       "      <td>1703708.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>gpuPixelDoublets::initDoublets(GPU::VecArray&lt;u...</td>\n",
       "      <td>0.000539</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1525991.0</td>\n",
       "      <td>4.883171e+07</td>\n",
       "      <td>2132565.0</td>\n",
       "      <td>6.824208e+07</td>\n",
       "      <td>4987862.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>gpuClustering::countModules(unsigned short con...</td>\n",
       "      <td>0.000532</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2625136.0</td>\n",
       "      <td>8.400435e+07</td>\n",
       "      <td>1702057.0</td>\n",
       "      <td>5.446582e+07</td>\n",
       "      <td>1665165.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>kernel_fillHitDetIndices(HistoContainer&lt;unsign...</td>\n",
       "      <td>0.000526</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3270678.0</td>\n",
       "      <td>1.046617e+08</td>\n",
       "      <td>2621674.0</td>\n",
       "      <td>8.389357e+07</td>\n",
       "      <td>2133900.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>_GLOBAL__N__59_tmpxft_0000f012_00000000_10_Pix...</td>\n",
       "      <td>0.000351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1618.0</td>\n",
       "      <td>5.177600e+04</td>\n",
       "      <td>4734.0</td>\n",
       "      <td>1.514880e+05</td>\n",
       "      <td>12896.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>void cms::cuda::finalizeBulk&lt;HistoContainer&lt;un...</td>\n",
       "      <td>0.000294</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>606657.0</td>\n",
       "      <td>1.941302e+07</td>\n",
       "      <td>437611.0</td>\n",
       "      <td>1.400355e+07</td>\n",
       "      <td>618679.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>gpuVertexFinder::init(ZVertexSoA*,gpuVertexFin...</td>\n",
       "      <td>0.000249</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>300.0</td>\n",
       "      <td>9.600000e+03</td>\n",
       "      <td>3400.0</td>\n",
       "      <td>1.088000e+05</td>\n",
       "      <td>5819.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>34 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          Kernel Name  CUDA Time   FP64 FLOPs  \\\n",
       "0   kernel_find_ntuplets(TrackingRecHit2DSOAView c...   0.018547          0.0   \n",
       "1   void kernelBLFit<int=4>(HistoContainer<unsigne...   0.014658  265470717.0   \n",
       "2   gpuVertexFinder::vertexFinderOneKernel(ZVertex...   0.012052      56235.0   \n",
       "3   gpuPixelDoublets::getDoubletsFromHisto(GPUCACe...   0.011098          0.0   \n",
       "4   gpuClustering::findClus(unsigned short const *...   0.009689          0.0   \n",
       "5   kernel_connect(AtomicPairCounter*,AtomicPairCo...   0.009273          0.0   \n",
       "6   void kernelBLFit<int=3>(HistoContainer<unsigne...   0.006044  862708818.0   \n",
       "7   gpuPixelDoublets::fishbone(TrackingRecHit2DSOA...   0.005508          0.0   \n",
       "8   gpuPixelRecHits::getHits(pixelCPEforGPU::Param...   0.005237   48629316.0   \n",
       "9   void kernelBLFastFit<int=4>(HistoContainer<uns...   0.002464   10586982.0   \n",
       "10  kernel_fastDuplicateRemover(GPUCACell const *,...   0.002291          0.0   \n",
       "11  kernel_earlyDuplicateRemover(GPUCACell const *...   0.001883          0.0   \n",
       "12  gpuClustering::clusterChargeCut(unsigned short...   0.001838          0.0   \n",
       "13  void cub::DeviceScanKernel<cub::DispatchScan<u...   0.001608          0.0   \n",
       "14  void kernelBLFastFit<int=3>(HistoContainer<uns...   0.001144   45042830.0   \n",
       "15  kernel_tripletCleaner(TrackingRecHit2DSOAView ...   0.001038          0.0   \n",
       "16  pixelgpudetails::fillHitsModuleStart(unsigned ...   0.000836          0.0   \n",
       "17  pixelgpudetails::RawToDigi_kernel(SiPixelFedCa...   0.000811          0.0   \n",
       "18  kernel_fillHitInTracks(HistoContainer<unsigned...   0.000752          0.0   \n",
       "19  kernel_classifyTracks(HistoContainer<unsigned ...   0.000748          0.0   \n",
       "20  kernel_fillMultiplicity(HistoContainer<unsigne...   0.000747          0.0   \n",
       "21  gpuCalibPixel::calibDigis(unsigned short*,unsi...   0.000707          0.0   \n",
       "22  kernel_countMultiplicity(HistoContainer<unsign...   0.000675          0.0   \n",
       "23  void cub::DeviceScanInitKernel<cub::ScanTileSt...   0.000656          0.0   \n",
       "24  void cms::cuda::fillFromVector<HistoContainer<...   0.000583          0.0   \n",
       "25  gpuVertexFinder::loadTracks(TrackSoAT<int=3276...   0.000568          0.0   \n",
       "26  void cms::cuda::countFromVector<HistoContainer...   0.000550          0.0   \n",
       "27  kernel_countHitInTracks(HistoContainer<unsigne...   0.000541          0.0   \n",
       "28  gpuPixelDoublets::initDoublets(GPU::VecArray<u...   0.000539          0.0   \n",
       "29  gpuClustering::countModules(unsigned short con...   0.000532          0.0   \n",
       "30  kernel_fillHitDetIndices(HistoContainer<unsign...   0.000526          0.0   \n",
       "31  _GLOBAL__N__59_tmpxft_0000f012_00000000_10_Pix...   0.000351          0.0   \n",
       "32  void cms::cuda::finalizeBulk<HistoContainer<un...   0.000294          0.0   \n",
       "33  gpuVertexFinder::init(ZVertexSoA*,gpuVertexFin...   0.000249          0.0   \n",
       "\n",
       "      FP32 FLOPs         FLOPs  L1 Transactions      L1 Bytes  \\\n",
       "0   0.000000e+00  0.000000e+00      221543156.0  7.089381e+09   \n",
       "1   2.071517e+07  2.861859e+08        9403323.0  3.009063e+08   \n",
       "2   1.143011e+07  1.148635e+07        8849541.0  2.831853e+08   \n",
       "3   5.459555e+08  5.459555e+08      339960809.0  1.087875e+10   \n",
       "4   0.000000e+00  0.000000e+00      150617334.0  4.819755e+09   \n",
       "5   1.756497e+09  1.756497e+09      199460307.0  6.382730e+09   \n",
       "6   6.009670e+07  9.228055e+08       29479274.0  9.433368e+08   \n",
       "7   1.291534e+08  1.291534e+08       55955823.0  1.790586e+09   \n",
       "8   1.757141e+08  2.243434e+08       74233078.0  2.375458e+09   \n",
       "9   1.102198e+07  2.160896e+07        4044162.0  1.294132e+08   \n",
       "10  2.255730e+05  2.255730e+05       15968332.0  5.109866e+08   \n",
       "11  0.000000e+00  0.000000e+00       16798424.0  5.375496e+08   \n",
       "12  0.000000e+00  0.000000e+00       19344017.0  6.190085e+08   \n",
       "13  0.000000e+00  0.000000e+00        2031529.0  6.500893e+07   \n",
       "14  3.578728e+07  8.083011e+07       13025818.0  4.168262e+08   \n",
       "15  1.645110e+05  1.645110e+05        4731904.0  1.514209e+08   \n",
       "16  0.000000e+00  0.000000e+00         251134.0  8.036288e+06   \n",
       "17  0.000000e+00  0.000000e+00        9645895.0  3.086686e+08   \n",
       "18  0.000000e+00  0.000000e+00        2746634.0  8.789229e+07   \n",
       "19  3.048192e+06  3.048192e+06        4657927.0  1.490537e+08   \n",
       "20  0.000000e+00  0.000000e+00        1510071.0  4.832227e+07   \n",
       "21  5.431565e+07  5.431565e+07        7282382.0  2.330362e+08   \n",
       "22  0.000000e+00  0.000000e+00        1218239.0  3.898365e+07   \n",
       "23  0.000000e+00  0.000000e+00           3300.0  1.056000e+05   \n",
       "24  0.000000e+00  0.000000e+00        1144366.0  3.661971e+07   \n",
       "25  5.623500e+04  5.623500e+04        1207916.0  3.865331e+07   \n",
       "26  0.000000e+00  0.000000e+00         727864.0  2.329165e+07   \n",
       "27  0.000000e+00  0.000000e+00        1789441.0  5.726211e+07   \n",
       "28  0.000000e+00  0.000000e+00        1525991.0  4.883171e+07   \n",
       "29  0.000000e+00  0.000000e+00        2625136.0  8.400435e+07   \n",
       "30  0.000000e+00  0.000000e+00        3270678.0  1.046617e+08   \n",
       "31  0.000000e+00  0.000000e+00           1618.0  5.177600e+04   \n",
       "32  0.000000e+00  0.000000e+00         606657.0  1.941302e+07   \n",
       "33  0.000000e+00  0.000000e+00            300.0  9.600000e+03   \n",
       "\n",
       "    L2 Transactions      L2 Bytes  DRAM Transactions  ...  \\\n",
       "0       225398161.0  7.212741e+09        278432471.0  ...   \n",
       "1         7501232.0  2.400394e+08          5333291.0  ...   \n",
       "2         1176192.0  3.763814e+07           211899.0  ...   \n",
       "3       168911671.0  5.405173e+09         67157969.0  ...   \n",
       "4        71288368.0  2.281228e+09         19548934.0  ...   \n",
       "5        92061427.0  2.945966e+09         83274917.0  ...   \n",
       "6        25623242.0  8.199437e+08         12158427.0  ...   \n",
       "7        32633287.0  1.044265e+09         52896500.0  ...   \n",
       "8        13837022.0  4.427847e+08          8975533.0  ...   \n",
       "9         2676045.0  8.563344e+07          4435744.0  ...   \n",
       "10       13325212.0  4.264068e+08         33382056.0  ...   \n",
       "11       13118927.0  4.198057e+08         33448644.0  ...   \n",
       "12        5170048.0  1.654415e+08          3624740.0  ...   \n",
       "13        1321764.0  4.229645e+07          1646333.0  ...   \n",
       "14        8409971.0  2.691191e+08          4930462.0  ...   \n",
       "15        1923370.0  6.154784e+07          1977586.0  ...   \n",
       "16         157193.0  5.030176e+06           131478.0  ...   \n",
       "17        8496372.0  2.718839e+08          8122109.0  ...   \n",
       "18        2822053.0  9.030570e+07          1825586.0  ...   \n",
       "19        3624459.0  1.159827e+08          4367582.0  ...   \n",
       "20        1405484.0  4.497549e+07          1141576.0  ...   \n",
       "21        3660331.0  1.171306e+08          6993283.0  ...   \n",
       "22        1113844.0  3.564301e+07          1077542.0  ...   \n",
       "23          12600.0  4.032000e+05            17577.0  ...   \n",
       "24        1688932.0  5.404582e+07           565733.0  ...   \n",
       "25         679323.0  2.173834e+07          1380305.0  ...   \n",
       "26        1272469.0  4.071901e+07           316439.0  ...   \n",
       "27        2053051.0  6.569763e+07          1703708.0  ...   \n",
       "28        2132565.0  6.824208e+07          4987862.0  ...   \n",
       "29        1702057.0  5.446582e+07          1665165.0  ...   \n",
       "30        2621674.0  8.389357e+07          2133900.0  ...   \n",
       "31           4734.0  1.514880e+05            12896.0  ...   \n",
       "32         437611.0  1.400355e+07           618679.0  ...   \n",
       "33           3400.0  1.088000e+05             5819.0  ...   \n",
       "\n",
       "    FP32 Performance GFlop/s     L1 AI  FP64 L1 AI  FP32 L1 AI     L2 AI  \\\n",
       "0                   0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "1                   1.413275  0.951080    0.882237    0.068843  1.192245   \n",
       "2                   0.948378  0.040561    0.000199    0.040363  0.305178   \n",
       "3                  49.192655  0.050186    0.000000    0.050186  0.101006   \n",
       "4                   0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "5                 189.415546  0.275195    0.000000    0.275195  0.596238   \n",
       "6                   9.942758  0.978236    0.914529    0.063707  1.125450   \n",
       "7                  23.446294  0.072129    0.000000    0.072129  0.123679   \n",
       "8                  33.551715  0.094442    0.020472    0.073971  0.506665   \n",
       "9                   4.473264  0.166976    0.081808    0.085169  0.252343   \n",
       "10                  0.098464  0.000441    0.000000    0.000441  0.000529   \n",
       "11                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "12                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "13                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "14                 31.274700  0.193918    0.108061    0.085857  0.300351   \n",
       "15                  0.158471  0.001086    0.000000    0.001086  0.002673   \n",
       "16                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "17                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "18                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "19                  4.076872  0.020450    0.000000    0.020450  0.026281   \n",
       "20                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "21                 76.849056  0.233078    0.000000    0.233078  0.463719   \n",
       "22                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "23                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "24                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "25                  0.098944  0.001455    0.000000    0.001455  0.002587   \n",
       "26                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "27                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "28                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "29                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "30                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "31                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "32                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "33                  0.000000  0.000000    0.000000    0.000000  0.000000   \n",
       "\n",
       "    FP64 L2 AI  FP32 L2 AI   DRAM AI  FP64 DRAM AI  FP32 DRAM AI  \n",
       "0     0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "1     1.105946    0.086299  1.676884      1.555505      0.121379  \n",
       "2     0.001494    0.303684  1.693960      0.008293      1.685667  \n",
       "3     0.000000    0.101006  0.254044      0.000000      0.254044  \n",
       "4     0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "5     0.000000    0.596238  0.659149      0.000000      0.659149  \n",
       "6     1.052156    0.073294  2.371826      2.217363      0.154463  \n",
       "7     0.000000    0.123679  0.076301      0.000000      0.076301  \n",
       "8     0.109826    0.396839  0.781094      0.169312      0.611781  \n",
       "9     0.123631    0.128711  0.152236      0.074586      0.077650  \n",
       "10    0.000000    0.000529  0.000211      0.000000      0.000211  \n",
       "11    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "12    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "13    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "14    0.167371    0.132979  0.512313      0.285488      0.226825  \n",
       "15    0.000000    0.002673  0.002600      0.000000      0.002600  \n",
       "16    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "17    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "18    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "19    0.000000    0.026281  0.021810      0.000000      0.021810  \n",
       "20    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "21    0.000000    0.463719  0.242713      0.000000      0.242713  \n",
       "22    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "23    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "24    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "25    0.000000    0.002587  0.001273      0.000000      0.001273  \n",
       "26    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "27    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "28    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "29    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "30    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "31    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "32    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "33    0.000000    0.000000  0.000000      0.000000      0.000000  \n",
       "\n",
       "[34 rows x 23 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "profiledf = transpose_frame(metricdf)\n",
    "profiledf = profiledf.sort_values('CUDA Time', ascending=False).reset_index(drop=True)\n",
    "\n",
    "display(profiledf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "profiledf.to_csv(os.path.join(outputdir,\"full_profile.csv\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
