3
I4W^5P  ã               @   sþ   d Z ddlZddlmZmZmZ dd„ Zdd„ Zdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zedkrúeƒ jƒ \ZZej s–ej s–ej r¢eƒ jƒ  nXejddge ejr¼dgng  ejrÒdej gng  ƒ eee	eedœZeeej eƒ dS )a¸  
Benchmark for common convnets.

Speed on Titan X, with 10 warmup steps and 10 main steps and with different
versions of cudnn, are as follows (time reported below is per-batch time,
forward / forward+backward):

                    CuDNN V3        CuDNN v4
AlexNet         32.5 / 108.0    27.4 /  90.1
OverFeat       113.0 / 342.3    91.7 / 276.5
Inception      134.5 / 485.8   125.7 / 450.6
VGG (batch 64) 200.8 / 650.0   164.1 / 551.7

Speed on Inception with varied batch sizes and CuDNN v4 is as follows:

Batch Size   Speed per batch     Speed per image
 16             22.8 /  72.7         1.43 / 4.54
 32             38.0 / 127.5         1.19 / 3.98
 64             67.2 / 233.6         1.05 / 3.65
128            125.7 / 450.6         0.98 / 3.52

Speed on Tesla M40, which 10 warmup steps and 10 main steps and with cudnn
v4, is as follows:

AlexNet         68.4 / 218.1
OverFeat       210.5 / 630.3
Inception      300.2 / 1122.2
VGG (batch 64) 405.8 / 1327.7

(Note that these numbers involve a "full" backprop, i.e. the gradient
with respect to the input image is also computed.)

To get the numbers, simply run:

for MODEL in AlexNet OverFeat Inception; do
  PYTHONPATH=../gen:$PYTHONPATH python convnet_benchmarks.py     --batch_size 128 --model $MODEL --forward_only True
done
for MODEL in AlexNet OverFeat Inception; do
  PYTHONPATH=../gen:$PYTHONPATH python convnet_benchmarks.py     --batch_size 128 --model $MODEL
done
PYTHONPATH=../gen:$PYTHONPATH python convnet_benchmarks.py   --batch_size 64 --model VGGA --forward_only True
PYTHONPATH=../gen:$PYTHONPATH python convnet_benchmarks.py   --batch_size 64 --model VGGA

for BS in 16 32 64 128; do
  PYTHONPATH=../gen:$PYTHONPATH python convnet_benchmarks.py     --batch_size $BS --model Inception --forward_only True
  PYTHONPATH=../gen:$PYTHONPATH python convnet_benchmarks.py     --batch_size $BS --model Inception
done

Note that VGG needs to be run at batch 64 due to memory limit on the backward
pass.
é    N)Ú	workspaceÚbrewÚmodel_helperc       
         sî   t jdd�}d}d‰ d}xjtˆ ƒD ]^}xXt|ƒD ]L}|dkrHdj||ƒnd}dj|d	 |ƒ}tj|||||d
i fd
i fd� q0W q"W tj|‡ fdd„t|ƒD ƒdgƒ tj|dd|dd
i fd
i fd� |jjddgdƒ}	|jj	|	dƒ ||fS )NÚMLP)Únameé   é   é   r   zfc_{}_{}Údataé   Ú
XavierFill)Zdim_inZdim_outZweight_initZ	bias_initc                s   g | ]}d j ˆ |ƒ‘qS )zfc_{}_{})Úformat)Ú.0Új)Údepth© úA/tmp/pip-build-e66j97b2/torch/caffe2/python/convnet_benchmarks.pyú
<listcomp>U   s    zMLP.<locals>.<listcomp>ÚsumÚlastiè  ÚlabelÚxentÚloss)
r   ÚModelHelperÚranger   r   Úfcr   ÚnetÚLabelCrossEntropyÚAveragedLoss)
ÚorderÚcudnn_wsÚmodelÚdÚwidthÚir   ÚcurrentZnext_r   r   )r   r   r   B   s:    r   c             C   s
  | dddœ}|r||d< t jd|d�}tj|dddd	d
di fdi fddd�
}tj||dƒ}tj||dddd�}tj||dd	dddi fdi fdd�	}tj||dƒ}tj||dddd�}	tj||	dddddi fdi fdd�	}
tj||
dƒ}tj||dddddi fdi fdd�	}tj||dƒ}tj||dddddi fdi fdd�	}tj||dƒ}tj||dddd�}tj||dd*d di fdi fƒ}tj||dƒ}tj||d!d d di fdi fƒ}tj||d!ƒ}tj||d"d d#di fdi fƒ}tj||d$ƒ}|jj	|d%gd&ƒ}|jj
|d'ƒ |d(fS )+NT)r   Ú	use_cudnnÚcudnn_exhaustive_searchÚws_nbytes_limitZalexnet)r   Ú	arg_scoper
   Úconv1r	   é@   é   r   ÚConstantFillé   é   )ÚstrideÚpadÚpool1)Úkernelr0   Úconv2éÀ   é   )r1   Úpool2Úconv3i€  r   Úconv4r   Úconv5Úpool5Úfc6é   i   Úfc7Úfc8iè  Úpredr   r   r   éà   i   i $  )r   r   r   ÚconvÚreluÚmax_poolr   Úsoftmaxr   r   r   )r   r    Úmy_arg_scoper!   r*   Úrelu1r2   r4   Úrelu2r7   r8   Úrelu3r9   Úrelu4r:   Úrelu5r;   r<   Úrelu6r>   Úrelu7r?   r@   r   r   r   r   ÚAlexNete   sž    
rN   c             C   s  | dddœ}|r||d< t jd|d�}tj|dddd	d
di fdi fdd�	}tj||dƒ}tj||dddd�}tj||dd	dddi fdi fƒ}tj||dƒ}tj||dddd�}	tj||	dddddi fdi fdd�	}
tj||
dƒ}tj||dddddi fdi fdd�	}tj||dƒ}tj||dddddi fdi fdd�	}tj||dƒ}tj||dddd�}tj||dd+d di fdi fƒ}tj||dƒ}tj||d!d d"di fdi fƒ}tj||d!ƒ}tj||d#d"d$di fdi fƒ}tj||d%ƒ}|jj	|d&gd'ƒ}|jj
|d(ƒ |d)fS ),NT)r   r&   r'   r(   Zoverfeat)r   r)   r
   r*   r	   é`   r,   r   r-   r.   )r0   r2   r/   )r3   r0   r4   r   r6   r7   r8   i   r   )r1   r9   i   r:   r;   r<   r=   i   r>   i   r?   iè  r@   r   r   r   éç   i   i �  )r   r   r   rB   rC   rD   r   rE   r   r   r   )r   r    rF   r!   r*   rG   r2   r4   rH   r7   r8   rI   r9   rJ   r:   rK   r;   r<   rL   r>   rM   r?   r@   r   r   r   r   ÚOverFeatÂ   s�    

rQ   c              C   sÆ  | dddœ}|r||d< t jd|d�}tj|dddd	dd
i fdi fdd�	}tj||dƒ}tj||dddd�}tj||dd	ddd
i fdi fdd�	}tj||dƒ}tj||dddd�}	tj||	ddddd
i fdi fdd�	}
tj||
dƒ}tj||ddddd
i fdi fdd�	}tj||dƒ}tj||dddd�}tj||ddddd
i fdi fdd�	}tj||dƒ}tj||ddddd
i fdi fdd�	}tj||dƒ}tj||dddd�}tj||ddddd
i fdi fdd�	}tj||dƒ}tj||ddddd
i fdi fdd�	}tj||dƒ}tj||dddd�}tj||dd+d!d
i fdi fƒ}tj||dƒ}tj||d"d!d!d
i fdi fƒ}tj||d"ƒ}tj||d#d!d$d
i fdi fƒ}tj||d%ƒ}|jj	|d&gd'ƒ}|jj
|d(ƒ |d)fS ),NT)r   r&   r'   r(   Zvgga)r   r)   r
   r*   r	   r+   r   r-   r   )r1   r2   r/   )r3   r0   r4   é€   r7   r8   r   r9   Úpool4r:   i   Úconv6Úpool6Úconv7Úconv8Úpool8Úfcixé   i   ÚfcxÚfcxiiè  r@   r   r   r   rP   i   i b  )r   r   r   rB   rC   rD   r   rE   r   r   r   ) r   r    rF   r!   r*   rG   r2   r4   rH   r7   r8   rI   r9   rJ   rS   r:   rK   rT   rL   rU   rV   rM   rW   Zrelu8rX   rY   Zreluixr[   Zreluxr\   r@   r   r   r   r   ÚVGGA  sè    


r]   c             C   s€  t j| ||d ||ddi fdi fƒ}t j| ||ƒ}t j| ||d ||d ddi fdi fƒ}	t j| |	|	ƒ}	t j| |	|d |d |d ddi fdi fdd	�	}
t j| |
|
ƒ}
t j| ||d
 ||d ddi fdi fƒ}t j| ||ƒ}t j| ||d |d |d ddi fdi fdd	�	}t j| ||ƒ}t j| ||d dddd�}t j| ||d ||ddi fdi fƒ}t j| ||ƒ}t j| ||
||g|ƒ}|S )Nz:conv1r   r   r-   z:conv3_reducer   z:conv3r	   )r1   z:conv5_reducez:conv5r6   r/   z:pool)r3   r0   r1   z
:pool_proj)r   rB   rC   rD   Úconcat)r!   Z
input_blobZinput_depthZoutput_nameZconv1_depthZconv3_depthsZconv5_depthsZ
pool_depthr*   Zconv3_reducer8   Zconv5_reducer:   ÚpoolZ	pool_projÚoutputr   r   r   Ú_InceptionModulež  s^    ra   c             C   sˆ  | dddœ}|r||d< t jd|d�}tj|dddd	d
di fdi fddd�
}tj||dƒ}tj||ddddd�}tj||dd	d	ddi fdi fƒ}tj|||ƒ}tj||dd	dddi fdi fdd�	}tj||dƒ}	tj||	ddddd�}
t||
ddd	ddgddgdƒ}t||dddddgddgd	ƒ}tj||ddddd�}t||dd ddd!gdd"gd	ƒ}t||d#d$d%d&d'gd(d	gd	ƒ}t||d#d)dddgd(d	gd	ƒ}t||d#d*d&d+d,gdd	gd	ƒ}t||d-d.dd%d/gddgdƒ}tj||d0dddd�}t||d1d2dd%d/gddgdƒ}t||d1d3d4dd4gd"dgdƒ}tj||d5d
dd6�}tj||d7d8d9di fdi fƒ}tj	||d:ƒ}|j
j|d;gd<ƒ}|j
j|d=ƒ |d'fS )>NT)r   r&   r'   r(   Z	inception)r   r)   r
   r*   r	   r+   rZ   r   r-   r/   )r0   r1   r2   r   )r3   r0   r1   Úconv2ar4   r5   )r1   r7   Úinc3rO   rR   é   é    r   Úinc4r;   ià  Úinc5éÐ   é0   i   Úinc6é    ép   rA   é   Úinc7Úinc8é�   i   i  Úinc9i@  Úpool9i@  Úinc10Úinc11i€  Úpool11)r3   r0   r   i   iè  r@   r   r   r   )r   r   r   rB   rC   rD   ra   Zaverage_poolr   rE   r   r   r   )r   r    rF   r!   r*   rG   r2   rb   r4   rH   r7   rc   rf   r;   rg   rj   rn   ro   rq   rr   rs   rt   ru   r   r@   r   r   r   r   Ú	InceptionÞ  s‚    

rv   c             C   sn   t j| dƒ}| jj|dddddd�}| jjg dd	gd
d�}x.| jD ]$}| j| }| jj||||g|ƒ qBW dS )zC Simple plain SGD update -- not tuned to actually train the models ÚiterÚLRg:Œ0âŽyE>Ústepi'  g+‡ÙÎ÷ï?)Zbase_lrÚpolicyZstepsizeÚgammaÚONEr   g      ð?)ÚshapeÚvalueNg:Œ0âŽyE¾)	r   rw   r   ZLearningRateÚparam_init_netr-   ÚparamsZparam_to_gradZWeightedSum)r!   ZITERrx   r|   ÚparamZ
param_gradr   r   r   ÚAddParameterUpdate6  s    
r‚   c             C   sÖ  | |j |jƒ\}}|j|jƒ _|j|jƒ _|j dkrD|jd||g}n|j||dg}|jdkrf|j|g}|jj	g d|ddd� |jj
g d|jgd	d
d� |jr®tdj|jƒƒ n6tdj|jƒƒ |jdgƒ t|ƒ |j dkrätdƒ |j�s |jjƒ  |jjƒ  |j�r(x|jjƒ jD ]}|j|_�qW |j�rœtdj|j|jƒdƒ�}|jt|jjƒ ƒƒ W d Q R X tdj|j|jƒdƒ�}|jt|jjƒ ƒƒ W d Q R X tj|jƒ tj|jƒ tj|jjƒ j|j|j|j ƒ d S )NÚNCHWr	   r   r
   g        g      ð?)r}   ZmeanZstdr   r   iç  )r}   ÚminÚmaxz{}: running forward only.z{}: running forward-backward.r   ZNHWCzU==WARNING==
NHWC order with CuDNN may not be supported yet, so I might
exit suddenly.z{0}_init_batch_{1}.pbtxtÚwz	{0}.pbtxt)!r   r    Znet_typeZProtoÚtypeZnum_workersÚ
batch_sizer!   r   ZGaussianFillZUniformIntFillZforward_onlyÚprintr   ZAddGradientOperatorsr‚   ÚcpuZRunAllOnGPUr   ZengineÚopZ
dump_modelÚopenÚwriteÚstrr   Z
RunNetOnceZ	CreateNetZBenchmarkNetr   Zwarmup_iterationsZ
iterationsZlayer_wise_benchmark)Z	model_genÚargr!   Z
input_sizeZinput_shaper‹   Zfidr   r   r   Ú	BenchmarkA  s\    





r�   c              C   s  t jdd�} | jdtddd� | jdtdd	� | jd
tddd� | jdtdd	� | jdtddd� | jdtddd� | jdddd� | jdddd� | jdddd� | jdtddd� | jddd d� | jd!td"d#� | jd$td%d#� | jd&d'dd(� | jd)td*� | S )+NzCaffe2 benchmark.)Údescriptionz--batch_sizerR   zThe batch size.)r‡   ÚdefaultÚhelpz--modelzThe model to benchmark.)r‡   r“   z--orderrƒ   zThe order to evaluate.z
--cudnn_wszThe cudnn workspace size.z--iterationsé
   z(Number of iterations to run the network.z--warmup_iterationsz1Number of warm-up iterations before benchmarking.z--forward_onlyÚ
store_truez"If set, only run the forward pass.)Úactionr“   z--layer_wise_benchmarkz.If True, run the layer-wise benchmark as well.z--cpuz+If True, run testing on CPU instead of GPU.z--engineÚ z8If set, blindly prefer the given engine(s) for every op.z--dump_modelz*If True, dump the model prototxts to disk.z
--net_typeZdag)r‡   r’   z--num_workersr/   z
--use-nvtxF)r’   r–   z--htrace_span_log_path)r‡   )ÚargparseÚArgumentParserÚadd_argumentÚintrŽ   )Úparserr   r   r   ÚGetArgumentParserƒ  sh    r�   Ú__main__Zcaffe2z--caffe2_log_level=0z--caffe2_use_nvtxz--caffe2_htrace_span_log_path=)rN   rQ   r]   rv   r   )Ú__doc__r˜   Zcaffe2.pythonr   r   r   r   rN   rQ   r]   ra   rv   r‚   r�   r�   Ú__name__Úparse_known_argsÚargsÚ
extra_argsrˆ   r!   r   Ú
print_helpZ
GlobalInitZuse_nvtxZhtrace_span_log_pathZ	model_mapr   r   r   r   Ú<module>;   s2   #]V @XB@