3
H4W^CP  ã               @   s(  d dl mZ d dl mZ d dl mZ d dl mZ d dlZd dlZd dlmZm	Z	m
Z
 d dljjZdd„ Zd	d
„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zedk�r$eƒ jƒ Zej sìej sìej sìej röeƒ j ƒ  e	j!ddgƒ eeeeed œZ"ee"ej eƒ dS )!é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)ÚcnnÚ	workspaceÚcorec       	   
      sÚ   t jƒ }d}d‰ d}xÀtˆ ƒD ]´}x®t|ƒD ]¢}|dkrDdj||ƒnd}dj|d |ƒ}|j|||||j|jd� |j‡ fd	d
„t|ƒD ƒdgƒ |jdd|d|j|jd� |jddgdƒ}|j|dƒ ||fS W qW d S )Né   é   é   r   zfc_{}_{}Údataé   )Zdim_inZdim_outZweight_initZ	bias_initc                s   g | ]}d j ˆ |ƒ‘qS )zfc_{}_{})Úformat)Ú.0Új)Údepth© úM/tmp/pip-build-e66j97b2/torch/caffe2/experiments/python/convnet_benchmarks.pyú
<listcomp>h   s   zMLP.<locals>.<listcomp>ÚsumÚlastiè  ÚlabelÚxentÚloss)	r   ÚCNNModelHelperÚranger   ÚFCZ
XavierInitZSumÚLabelCrossEntropyÚAveragedLoss)	ÚorderÚmodelÚdÚwidthÚir   ÚcurrentZnext_r   r   )r   r   ÚMLPZ   s,    

r%   c             C   sÌ  t j| dddd�}|jdddddd	i fd
i fddd�	}|j|dƒ}|j|dddd�}|j|ddddd	i fd
i fdd�}|j|dƒ}|j|dddd�}|j|ddddd	i fd
i fdd�}|j|dƒ}	|j|	ddddd	i fd
i fdd�}
|j|
dƒ}|j|ddddd	i fd
i fdd�}|j|dƒ}|j|dddd�}|j|dd(dd	i fd
i fƒ}|j|dƒ}|j|dddd	i fd
i fƒ}|j|dƒ}|j|d dd!d	i fd
i fƒ}|j|d"ƒ}|j|d#gd$ƒ}|j|d%ƒ |d&fS ))NZalexnetT)ÚnameÚ	use_cudnnÚcudnn_exhaustive_searchr   Úconv1r   é@   é   Ú
XavierFillÚConstantFillé   é   )ÚstrideÚpadÚpool1)Úkernelr0   Úconv2éÀ   é   )r1   Úpool2Úconv3i€  r   Úconv4r	   Úconv5Úpool5Úfc6é   i   Úfc7Úfc8iè  Úpredr   r   r   éà   i   i $  )	r   r   ÚConvÚReluÚMaxPoolr   ÚSoftmaxr   r   )r   r    r)   Úrelu1r2   r4   Úrelu2r7   r8   Úrelu3r9   Úrelu4r:   Úrelu5r;   r<   Úrelu6r>   Úrelu7r?   r@   r   r   r   r   ÚAlexNets   sŠ    

rM   c          
   C   sÆ  t j| dddd�}|jdddddd	i fd
i fdd�}|j|dƒ}|j|dddd�}|j|ddddd	i fd
i fƒ}|j|dƒ}|j|dddd�}|j|ddddd	i fd
i fdd�}|j|dƒ}	|j|	ddddd	i fd
i fdd�}
|j|
dƒ}|j|ddddd	i fd
i fdd�}|j|dƒ}|j|dddd�}|j|dd)dd	i fd
i fƒ}|j|dƒ}|j|ddd d	i fd
i fƒ}|j|dƒ}|j|d!d d"d	i fd
i fƒ}|j|d#ƒ}|j|d$gd%ƒ}|j|d&ƒ |d'fS )*NZoverfeatT)r&   r'   r(   r   r)   r   é`   r+   r,   r-   r.   )r0   r2   r/   )r3   r0   r4   r	   r6   r7   r8   i   r   )r1   r9   i   r:   r;   r<   r=   i   r>   i   r?   iè  r@   r   r   r   éç   i   i �  )	r   r   rB   rC   rD   r   rE   r   r   )r   r    r)   rF   r2   r4   rG   r7   r8   rH   r9   rI   r:   rJ   r;   r<   rK   r>   rL   r?   r@   r   r   r   r   ÚOverFeatÄ   sz    

rP   c          
   C   sx  t j| dddd�}|jddddddi fd	i fd
d�}|j|dƒ}|j|dddd�}|j|dddddi fd	i fd
d�}|j|dƒ}|j|dddd�}|j|dddddi fd	i fd
d�}|j|dƒ}	|j|	dddddi fd	i fd
d�}
|j|
dƒ}|j|dddd�}|j|dddddi fd	i fd
d�}|j|dƒ}|j|dddddi fd	i fd
d�}|j|dƒ}|j|dddd�}|j|dddddi fd	i fd
d�}|j|dƒ}|j|dddddi fd	i fd
d�}|j|dƒ}|j|dddd�}|j|dd)ddi fd	i fƒ}|j|dƒ}|j|d dddi fd	i fƒ}|j|d ƒ}|j|d!dd"di fd	i fƒ}|j|d#ƒ}|j|d$gd%ƒ}|j|d&ƒ |d'fS )*Nzvgg-aT)r&   r'   r(   r   r)   r   r*   r,   r-   r   )r1   r2   r/   )r3   r0   r4   é€   r7   r8   r	   r9   Úpool4r:   i   Úconv6Úpool6Úconv7Úconv8Úpool8Úfcixé   i   ÚfcxÚfcxiiè  r@   r   r   r   rO   i   i b  )	r   r   rB   rC   rD   r   rE   r   r   )r   r    r)   rF   r2   r4   rG   r7   r8   rH   r9   rI   rR   r:   rJ   rS   rK   rT   rU   rL   rV   Zrelu8rW   rX   ZreluixrZ   Zreluxr[   r@   r   r   r   r   ÚVGGA  sÈ    

r\   c             C   s&   t dƒ t dƒ t dƒ tj| ƒ}|S )Nz4====================================================z4                 Start Building DAG                 )ÚprintÚSparseTransformerZ
netbuilder)r    Znet_rootr   r   r   Únet_DAG_Builder  s
    
r_   c          
   C   sd  | j ||d ||ddi fdi fƒ}| j||ƒ}| j ||d ||d ddi fdi fƒ}	| j|	|	ƒ}	| j |	|d |d |d ddi fdi fdd	�}
| j|
|
ƒ}
| j ||d
 ||d ddi fdi fƒ}| j||ƒ}| j ||d |d |d ddi fdi fdd	�}| j||ƒ}| j||d dddd�}| j ||d ||ddi fdi fƒ}| j||ƒ}| j||
||g|ƒ}|S )Nz:conv1r   r,   r-   z:conv3_reducer   z:conv3r   )r1   z:conv5_reducez:conv5r6   r/   z:pool)r3   r0   r1   z
:pool_proj)rB   rC   rD   ZConcat)r    Z
input_blobZinput_depthZoutput_nameZconv1_depthZconv3_depthsZconv5_depthsZ
pool_depthr)   Zconv3_reducer8   Zconv5_reducer:   ÚpoolZ	pool_projÚoutputr   r   r   Ú_InceptionModule‡  s\    rb   c             C   sV  t j| dddd�}|jdddddd	i fd
i fddd�	}|j|dƒ}|j|ddddd�}|j|ddddd	i fd
i fƒ}|j||ƒ}|j|ddddd	i fd
i fdd�}|j|dƒ}|j|ddddd�}t||dddddgddgdƒ}	t||	dddddgddgdƒ}
|j|
ddddd�}t||dddddgdd gdƒ}t||d!d"d#d$d%gd&dgdƒ}t||d!d'dddgd&dgdƒ}t||d!d(d$d)d*gddgdƒ}t||d+d,dd#d-gddgdƒ}|j|d.dddd�}t||d/d0dd#d-gddgdƒ}t||d/d1d2dd2gd dgdƒ}|j|d3ddd4�}|j|d5d6d7d	i fd
i fƒ}|j|d8ƒ}|j	|d9gd:ƒ}|j
|d;ƒ |d%fS )<NZ	inceptionT)r&   r'   r(   r   r)   r   r*   rY   r,   r-   r/   )r0   r1   r2   r   )r3   r0   r1   Úconv2ar4   r5   )r1   r7   Úinc3rN   rQ   é   é    r	   Úinc4r;   ià  Úinc5éÐ   é0   i   Úinc6é    ép   rA   é   Úinc7Úinc8é�   i   i  Úinc9i@  Úpool9i@  Úinc10Úinc11i€  Úpool11)r3   r0   Úfci   iè  r@   r   r   r   )r   r   rB   rC   rD   rb   ZAveragePoolr   rE   r   r   )r   r    r)   rF   r2   rc   r4   rG   r7   rd   rg   r;   rh   rk   ro   rp   rr   rs   rt   ru   rv   rw   r@   r   r   r   r   Ú	InceptionÆ  sn    
rx   c             C   sd   | j g ddg|||d�\}}| j|dtjjd�}| j|dƒ}| j||td
ƒd	�}| j||ƒ}||fS )zAdds the data input part.Ú
data_uint8r   )Ú
batch_sizeÚdbÚdb_typeZ	data_nhwc)Ztor   g      ð?r	   )Zscaleg      p?)	ZTensorProtosDBInputZCastr   ZDataTypeÚFLOATZ	NHWC2NCHWZScaleÚfloatZStopGradient)r    rz   r{   r|   ry   r   r   r   r   r   ÚAddInput  s    
r   c             C   sh   | j dƒ}| j|dddddd�}| jjg dd	gd
d�}x,| jD ]"}| j| }| j||||g|ƒ q>W dS )zC Simple plain SGD update -- not tuned to actually train the models ÚiterÚLRg:Œ0âŽyE>Ústepi'  g+‡ÙÎ÷ï?)Zbase_lrÚpolicyZstepsizeÚgammaÚONEr   g      ð?)ÚshapeÚvalueNg:Œ0âŽyE¾)ZIterZLearningRateÚparam_init_netr-   ÚparamsZparam_to_gradZWeightedSum)r    ZITERr�   r…   ÚparamZ
param_gradr   r   r   ÚAddParameterUpdate  s    

r‹   c       	      C   s,  | |j ƒ\}}|j|jƒ _|j|jƒ _|j dkr@|jd||g}n"|j||dg}|jdkrb|j|g}|jjg d|ddd� |jj	g d|jgd	d
d� |j
rªtdj|jƒƒ n6tdj|jƒƒ |jdgƒ t|ƒ |j dkràtdƒ |jsú|jjƒ  |jjƒ  |j�rntdj|j|jƒdƒ�P}|jt|jjƒ ƒƒ tdj|j|jƒdƒ�}|jt|jjƒ ƒƒ W d Q R X W d Q R X tj|jƒ tj|jƒ x&t|jƒD ]}tj|jjƒ jƒ �q’W tjdƒ}|jtj d|j|j!ƒƒ t"j"ƒ }tj#|ƒ tdjt"j"ƒ | |j! ƒƒ |j$�r(tdƒ tj%|jjƒ jd|j!dƒ d S )NÚNCHWr   r%   r   g        g      ð?)r†   ZmeanZstdr   r   iç  )r†   ÚminÚmaxz{}: running forward only.z{}: running forward-backward.r   ZNHWCzU==WARNING==
NHWC order with CuDNN may not be supported yet, so I might
exit suddenly.z{0}_init_batch_{1}.pbtxtÚwz	{0}.pbtxtÚplanÚrunz	Spent: {}zLayer-wise benchmark.r   T)&r   Znet_typeZProtoÚtypeZnum_workersrz   r    rˆ   ZGaussianFillZUniformIntFillZforward_onlyr]   r   ZAddGradientOperatorsr‹   ÚcpuZRunAllOnGPUÚnetZ
dump_modelÚopenÚwriteÚstrr   Z
RunNetOnceZ	CreateNetr   Zwarmup_iterationsZRunNetr&   r   ZPlanZAddStepZExecutionStepZ
iterationsÚtimeZRunPlanZlayer_wise_benchmarkZBenchmarkNet)	Z	model_genÚargr    Z
input_sizeZinput_shapeZfidr#   r�   Ústartr   r   r   Ú	Benchmark)  sf    






(

r›   c              C   sÚ   t jdd�} | jdtddd� | jdtdd	� | jd
tddd� | jdtd$dd� | jdtddd� | jdtddd� | jdddd� | jdddd� | jdddd� | jdddd� | jdtd d!� | jd"td#d!� | S )%NzCaffe2 benchmark.)Údescriptionz--batch_sizerQ   zThe batch size.)r’   ÚdefaultÚhelpz--modelzThe model to benchmark.)r’   rž   z--orderrŒ   zThe order to evaluate.z
--cudnn_wsr   zThe cudnn workspace size.z--iterationsé
   z(Number of iterations to run the network.z--warmup_iterationsz1Number of warm-up iterations before benchmarking.z--forward_onlyÚ
store_truez"If set, only run the forward pass.)Úactionrž   z--layer_wise_benchmarkz.If True, run the layer-wise benchmark as well.z--cpuz+If True, run testing on CPU instead of GPU.z--dump_modelz*If True, dump the model prototxts to disk.z
--net_typeZdag)r’   r�   z--num_workersr/   éÿÿÿÿ)ÚargparseÚArgumentParserÚadd_argumentÚintr—   )Úparserr   r   r   ÚGetArgumentParserr  s\    r¨   Ú__main__Zcaffe2z--caffe2_log_level=0)rM   rP   r\   rx   r%   )#Ú
__future__r   r   r   r   r£   r˜   Zcaffe2.pythonr   r   r   Zcaffe2.python.SparseTransformerÚpythonr^   r%   rM   rP   r\   r_   rb   rx   r   r‹   r›   r¨   Ú__name__Ú
parse_argsÚargsrz   r    r   Zcudnn_wsÚ
print_helpZ
GlobalInitZ	model_mapr   r   r   r   Ú<module>   s<   =QHs?KI:


