3
I4W^�j  ã               @   s‚  d dl mZ d dl mZ d dl mZ d dl mZ d dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZmZmZmZ d dlmZmZ d dlmZmZmZ d d	lmZ d dljjjZd dljjjZd d
lmZmZ d dl jj!j"Z#d dl$jj!j%Z&d dl'm(Z( ej)ƒ  ej*dƒZ+e+j,ej-ƒ ej.dƒ ej.dƒ d dd„Z/dd„ Z0dd„ Z1dd„ Z2dd„ Z3dd„ Z4dd„ Z5e6dk�r~ej7ddgƒ e5ƒ  dS )!é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)ÚcoreÚ	workspaceÚexperiment_utilÚdata_parallel_model)ÚdyndepÚ	optimizer)Útimeout_guardÚmodel_helperÚbrew)Ú
caffe2_pb2)ÚInitializerÚPseudoFP16Initializer)Úpredictor_constantsZImagenet_trainerz2@/caffe2/caffe2/distributed:file_store_handler_opsz3@/caffe2/caffe2/distributed:redis_store_handler_opsc       
      C   sP   t j| |ddg||tj| jƒr"dndd||ddd|d|d�\}}	| j||ƒ}d	S )
zž
    The image input operator loads image and label data from the reader and
    applies transformations to the images (random cropping, mirroring, ...).
    ÚdataÚlabelTFg      `@é   é   )Ú
batch_sizeÚoutput_typeZuse_gpu_transformZuse_caffe_datumÚmean_per_channelÚstd_per_channelZmeanZstdÚscaleZcropZmirrorÚis_testN)r   Zimage_inputr   ZIsGPUDeviceTypeZ_device_typeZStopGradient)
ÚmodelÚreaderr   Úimg_sizeÚdtyper   r   r   r   r   © r!   úH/tmp/pip-build-e66j97b2/torch/caffe2/python/examples/imagenet_trainer.pyÚAddImageInput4   s     r#   c             C   sl   |dkrdnd}| j jg d| g|d||gd� |dkrJ| j jd| dƒ | j jg dg|gdtjjd	� d
S )zê
    The null input function uses a gaussian fill operator to emulate real image
    input. A label blob is hardcoded to a single value. This is useful if you
    want to test compute throughput or don't have a dataset available.
    Úfloat16Z_fp16Ú r   é   )Úshaper   r   )r'   Úvaluer    N)Úparam_init_netZGaussianFillZFloatToHalfZConstantFillr   ÚDataTypeZINT32)r   r   r   r   r    Úsuffixr!   r!   r"   ÚAddNullInputW   s    r,   c             C   sŠ   dj |j|jd ƒ}tj|jjƒ tj|ƒ|d g|d g|d d| j	f|d | j
| j| jfid�}d| j| j|f }tjd|||d	� d S )
Nz[]_{}r   z/dataz/softmaxr   )Zpredict_netÚ
parametersÚinputsÚoutputsZshapesz%s/%s_%d.mdlÚminidb)Údb_typeZdb_destinationÚpredictor_export_metaÚ	use_ideep)ÚformatÚ_device_prefixÚ_devicesÚpred_expZPredictorExportMetaÚnetÚProtor	   ZGetCheckpointParamsÚ
num_labelsÚnum_channelsÚ
image_sizeÚfile_store_pathÚsave_model_nameZ
save_to_db)ÚargsÚtrain_modelÚepochr3   Úprefixr2   Ú
model_pathr!   r!   r"   Ú	SaveModelo   s"    rD   c             C   s´   t jdj| ƒƒ tj| dƒ}tjtj|t	j
ƒƒ}tjtj|t	jƒƒ}|rR|jƒ  n|jƒ  |rh|jƒ  n|jƒ  tj|ƒs~t‚tj|ƒsŒt‚tjdƒ}tjd|tjtjdƒd� dS )z)
    Load pretrained model from file
    zLoading path: {}r0   Zoptimizer_iterationr   )Zdevice_optionN)ÚlogÚinfor4   r7   Zload_from_dbr   ZNetÚ
pred_utilsZGetNetr   ZGLOBAL_INIT_NET_TYPEZPREDICT_INIT_NET_TYPEZRunAllOnIDEEPZRunAllOnGPUr   Ú
RunNetOnceÚAssertionErrorÚ	FetchBlobZFeedBlobZDeviceOptionr   ZCPU)Úpathr   r3   Zmeta_net_defZinit_netZpredict_init_netZitercntr!   r!   r"   Ú	LoadModelŽ   s&    


rL   c             C   sJ  t jdj|| jƒƒ t| j| | ƒ}t| j| | ƒ}	xÌt|ƒD ]À}
|
dkrT| jn| j	}t
j|ƒ�0 tjƒ }tj|jjƒ jƒ tjƒ }|| }W dQ R X d}t j|j|
d |||| ƒƒ dj|j|jd ƒ}tj|d ƒ}tj|d ƒ}d	}t j|j||ƒƒ qBW || | }dj|j|jd ƒ}tj|d ƒ}tj|d ƒ}tjtj|ƒd ƒ}d}d}|dk	�rd}x‚t|	ƒD ]v}tj|jjƒ jƒ x\|jD ]R}|tjtjdj|j|ƒd ƒƒ7 }|tjtjdj|j|ƒd
 ƒƒ7 }|d7 }�q”W �qvW || }|| }nd}d}|j ||
||  ||||||dœd� |dk �sBtdƒ‚|d S )zI
    Run one epoch of the trainer.
    TODO: add checkpointing here.
    zStarting epoch {}/{}r   Nz8Finished iteration {}/{} of epoch {} ({:.2f} images/sec)r   z{}_{}z	/accuracyz/losszTraining loss: {}, accuracy: {}z/accuracy_top5)ÚaccuracyÚlossÚlearning_raterA   Ztop1_test_accuracyZtop5_test_accuracy)Zinput_countZbatch_countZadditional_valuesé(   zExploded gradients :(éÿÿÿÿrQ   )rE   rF   r4   Ú
num_epochsÚintÚ
epoch_sizeZtest_epoch_sizeÚrangeZfirst_iter_timeoutÚtimeoutr   ZCompleteInTimeOrDieÚtimer   ZRunNetr8   r9   Únamer5   r6   rJ   r	   ZGetLearningRateBlobNamesÚnpZasscalarrI   )r?   rA   r@   Ú
test_modelÚtotal_batch_sizeÚ
num_shardsÚexpnameÚexplogÚepoch_itersZtest_epoch_itersÚirV   Út1Út2ÚdtÚfmtrB   rM   rN   Z	train_fmtZ
num_imagesrO   Ztest_accuracyZtest_accuracy_top5ZntestsÚ_Úgr!   r!   r"   ÚRunEpoch®   sh    




rg   c                sð  ˆ j dkrdtˆ jƒ }nˆ j dkr(d}ˆ jd k	rRdd„ ˆ jjdƒD ƒ}t|ƒ}nttˆ jƒƒ}ˆ j}t	j
dj|ƒƒ ˆ j‰ˆ| ‰ˆ| dks˜tdƒ‚ˆ jr¶tˆ jƒˆ jks¶td	ƒ‚ˆ jrÔtˆ jƒˆ jksÔtd
ƒ‚ˆˆ j }tˆ j| ƒ}|dksütdƒ‚|| ˆ _t	j
djˆ jƒƒ ˆ j�r.ddddœ}ndddˆ jd d dœ}tj||d�}ˆ j‰ˆ j}ˆ jjdƒ}	tjdƒd k	�rÆttjddƒƒ‰ttjddƒƒ}ˆdk�rHtd ˆ|dˆ j|	d dd d�}
n‚ˆdk�rDd}ˆ jd k	�rtj t!j"dg |gˆ jˆ j#ˆ j$d�ƒ n tj t!j"dg |gˆ j%ˆ j$d�ƒ t||ˆdˆ j|	d d d�}
nd }
‡ fdd „}‡ fd!d"„}‡ ‡‡fd#d$„}ˆ j&d%k�rŒ‡ ‡fd&d'„}n(|j'd(ˆ j&ˆ j(ˆ|d)�‰‡ ‡‡fd*d'„}d+d,„ }t)j*||ˆ j dk�rÔ|n|||||
dˆ j+ˆ jˆ j+ˆ j+d-� t)j,|i t-ƒ dƒ tj.|j/ƒ tj0|j1ƒ d }ˆ j2d k	�rÔt	j
d.ƒ ˆ j�rNddd/œ}ndddd0œ}tj|d1 |dd2�}|j'd3ˆ j2ˆ j(d4�‰‡ ‡‡fd5d6„}t)j*||ˆ j dk�rª|n||d |ˆ j+d7� tj.|j/ƒ tj0|j1ƒ d}ˆ j3d k	�rFt4ˆ j3|ˆ jƒ t)j5|ƒ ˆ j3jd8ƒd? }|j6d9ƒ�r<t|d d@… ƒ}t	j
d;j|ƒƒ n
t	j7d<ƒ d=|ˆ jˆˆ j8ˆ j9f }t:j;|ˆ ƒ}x€|ˆ j<k �rêt=ˆ |||ˆˆ||ƒ}t>ˆ ||ˆ jƒ d>ˆ j%ˆ j?f }tj@jA|t|d ƒ d9 ƒ�rltjB|t|d ƒ d9 ƒ �qlW d S )ANÚresnextÚ
shufflenetc             S   s   g | ]}t |ƒ‘qS r!   )rS   )Ú.0Úxr!   r!   r"   ú
<listcomp>
  s    zTrain.<locals>.<listcomp>ú,zRunning on GPUs: {}r   z%Number of GPUs must divide batch sizez8The number of channels of image mean doesn't match inputz7The number of channels of image std doesn't match inputz;Epoch size must be larger than batch size times shard countzUsing epoch size: {}Fr   )Ú	use_cudnnÚcudnn_exhaustive_searchZtraining_modeZNCHWTi   )Úorderrn   ro   Zws_nbytes_limit)rX   Ú	arg_scopeZOMPI_COMM_WORLD_SIZEZOMPI_COMM_WORLD_RANKZGLOO)Ú
kv_handlerr\   Úshard_idÚengineÚ	transportÚ	interfaceZmpi_rendezvousÚ	exit_netsÚstore_handlerZRedisStoreHandlerCreate)ÚhostÚportrB   ZFileStoreHandlerCreate)rK   rB   )rr   rs   r\   rt   ru   rv   rw   c                sØ   ˆ j dkrtnt}tjtjtjg||ˆ jˆ jd��, t	j
| dˆ jˆ jˆ jˆ jˆ jddd�	}W d Q R X ˆ j dkr€| jj||d ƒ}| j|dgdd	gƒ\}}| j||d
�}tj| |dgddd� tj| |dgddd� |gS )Nr$   )ÚWeightInitializerÚBiasInitializerÚenable_tensor_coreÚfloat16_computer   T)Únum_input_channelsr:   Ú
num_layersZ
num_groupsZnum_width_per_groupZno_biasÚno_lossÚ_fp32r   ÚsoftmaxrN   )r   rM   r   )Útop_kÚaccuracy_top5é   )r    r   r   r   rq   ÚconvÚfcr}   r~   ÚresnetZcreate_resnextr;   r:   r€   Zresnext_num_groupsZresnext_width_per_groupr8   ÚHalfToFloatÚSoftmaxWithLossÚScalerM   )r   Ú
loss_scaleÚinitializerÚpredrƒ   rN   )r?   r!   r"   Úcreate_resnext_model_ops|  s2    

z'Train.<locals>.create_resnext_model_opsc                sÊ   ˆ j dkrtnt}tjtjtjg||ˆ jˆ jd�� t	j
| dˆ jˆ jdd�}W d Q R X ˆ j dkrr| jj||d ƒ}| j|dgdd	gƒ\}}| j||d
�}tj| |dgddd� tj| |dgddd� |gS )Nr$   )r{   r|   r}   r~   r   T)r   r:   r�   r‚   r   rƒ   rN   )r   rM   r   )r„   r…   r†   )r    r   r   r   rq   r‡   rˆ   r}   r~   ri   Zcreate_shufflenetr;   r:   r8   rŠ   r‹   rŒ   rM   )r   r�   rŽ   r�   rƒ   rN   )r?   r!   r"   Úcreate_shufflenet_model_ops›  s*    

z*Train.<locals>.create_shufflenet_model_opsc          
      sh   t dˆ j ˆ ˆ ƒ}ˆ jr<tj| ˆ jddˆ jd|dd�}n(tj| ˆ jƒ tj| ˆ jddd|dd�}|S )Né   gÍÌÌÌÌÌì?r   Ústepgš™™™™™¹?)ÚmomentumÚnesterovÚweight_decayÚpolicyÚstepsizeÚgamma)r”   r•   r—   r˜   r™   )	rS   rT   r~   r   Zbuild_fp16_sgdÚbase_learning_rater–   Zadd_weight_decayZbuild_multi_precision_sgd)r   ZstepszÚopt)r?   r\   r[   r!   r"   Úadd_optimizer¶  s*    
zTrain.<locals>.add_optimizerÚnullc                s   t | d ˆˆ jˆ jd� d S )N)r   r   r    )r,   r<   r    )r   )r?   Úbatch_per_devicer!   r"   Úadd_image_inputÖ  s    zTrain.<locals>.add_image_inputr   )Údbr1   r\   rs   c          
      s$   t | ˆˆˆ jˆ jdˆ jˆ jd� d S )NF)r   r   r    r   r   r   )r#   r<   r    Úimage_mean_per_channelÚimage_std_per_channel)r   )r?   rž   r   r!   r"   rŸ   ç  s    c             S   s@   x:| j | jƒ ƒD ](}|jdk	r| jj|j|jtjj ƒ qW dS )z-Add ops applied after initial parameter sync.N)	ZGetOptimizationParamInfoZ	GetParamsZ	blob_copyr)   rŠ   Zblobr   r*   ÚFLOAT)r   Z
param_infor!   r!   r"   Úadd_post_sync_opsó  s
    
z Train.<locals>.add_post_sync_ops)Úinput_builder_funÚforward_pass_builder_funZoptimizer_builder_funÚpost_sync_builder_funÚdevicesÚ
rendezvousZoptimize_gradient_memoryÚ
cpu_deviceZideepZshared_modelZcombine_spatial_bnz----- Create test net ----)rn   ro   )rp   rn   ro   Ú_test)rX   rq   Zinit_paramsÚtest_reader)r    r1   c          
      s$   t | ˆˆˆ jˆ jdˆ jˆ jd� d S )NT)r   r   r    r   r   r   )r#   r<   r    r¡   r¢   )r   )r?   rž   r¬   r!   r"   Útest_input_fn,  s    zTrain.<locals>.test_input_fn)r¥   r¦   r§   Zparam_update_builder_funr¨   rª   re   z.mdlé   zReset epoch to {}z,The format of load_model_path doesn't match!z%s_gpu%d_b%d_L%d_lr%.2f_v2z%s/%s_rQ   éüÿÿÿ)Cr   Ústrr€   ÚgpusÚsplitÚlenÚlistrU   Únum_gpusrE   rF   r4   r   rI   r¡   r;   r¢   r\   rS   rT   r3   Zcudnn_workspace_limit_mbr   ZModelHelperrs   Zdistributed_interfacesÚosÚgetenvÚdictZdistributed_transportZ
redis_hostr   ZRunOperatorOncer   ZCreateOperatorZ
redis_portZrun_idr=   Z
train_dataZCreateDBr1   r	   ZParallelizeZuse_cpuZOptimizeGradientMemoryÚsetrH   r)   Z	CreateNetr8   Z	test_dataZload_model_pathrL   ZFinalizeAfterCheckpointÚendswithÚwarningr:   rš   r   ZModelTrainerLogrR   rg   rD   r>   rK   ÚisfileÚremove)r?   Z
model_namer±   rµ   Zglobal_batch_sizer_   Ztrain_arg_scoper@   rs   Z
interfacesr©   rx   r�   r‘   rœ   rŸ   r¤   rZ   Ztest_arg_scoper­   rA   Zlast_strr]   r^   rC   r!   )r?   rž   r\   r   r¬   r[   r"   ÚTrain  sH   












		








r¾   c              C   sÎ  t jdd�} | jdtd ddd� | jdtdd	d
� | jdtddd
� | jdtddd
� | jdtd dd
� | jdtddd� | jdtddd� | jdtddd
� | jdtddd
� | jdtdd � | jd!tdd"d
� | jd#td$d%d
� | jd&td'd(d
� | jd)td*d+d
� | jd,td-d.d
� | jd/td0d1d
� | jd2td*d3d
� | jd4td5d6d
� | jd7td8d9d
� | jd:tdd;d
� | jd<tdd=d
� | jd>td?d@d
� | jdAtdBd � | jdCtdDd � | jdEtdFdGd
� | jdHtdIdJd
� | jdKtdLdMd
� | jdNtd dOd
� | jdPdQdRdS� | jdTtdUdVd
� | jdWdXdXdYgdZd[� | jd\dQd]dS� | jd^dQd_dS� | jd`tdadbd
� | jdctddded
� | jdftdgdhd
� | jditdjdkd
� | jdldmdmdndodmgdpdq� | jƒ }t|ƒ d S )rNzCaffe2: ImageNet Trainer)Údescriptionz--train_dataTz-Path to training data (or 'null' to simulate))ÚtypeÚdefaultÚrequiredÚhelpz--num_layersé2   z'The number of layers in ResNe(X)t model)rÀ   rÁ   rÃ   z--resnext_num_groupsr   zThe cardinality of resnextz--resnext_width_per_groupé@   z--test_datazPath to test dataz--image_mean_per_channelú+z#The per channel mean for the images)rÀ   ÚnargsrÃ   z--image_std_per_channelz1The per channel standard deviation for the imagesz--test_epoch_sizeiPÃ  zNumber of test imagesz	--db_typeZlmdbz'Database type (such as lmdb or leveldb)z--gpusz*Comma separated list of GPU devices to use)rÀ   rÃ   z
--num_gpusz)Number of GPU devices (instead of --gpus)z--num_channelsr&   zNumber of color channelsz--image_sizeéà   zInput image size (to crop to)z--num_labelsiè  zNumber of labelsz--batch_sizeé    zBatch size, total over all GPUsz--epoch_sizei`ã z/Number of images/epoch, total over all machinesz--num_epochszNum epochs.z--base_learning_rategš™™™™™¹?zInitial learning rate.z--weight_decayg-Cëâ6?z Weight decay (L2 regularization)z--cudnn_workspace_limit_mbzCuDNN workspace limit in MBsz--num_shardsz%Number of machines in distributed runz
--shard_idr   z	Shard id.z--run_idz!Unique run identifier (e.g. uuid)z--redis_hostz%Host of Redis server (for rendezvous)z--redis_portië  z%Port of Redis server (for rendezvous)z--file_store_pathz/tmpz'Path to directory to use for rendezvousz--save_model_nameZresnext_modelz&Save the trained model to a given namez--load_model_pathz0Load previously saved model to continue trainingz	--use_cpuÚ
store_truezUse CPU instead of GPU)ÚactionrÃ   z--use_ideepFz	Use ideepz--dtypeÚfloatr$   zData type used for training)rÁ   ÚchoicesrÃ   z--float16_computez"Use float 16 compute, if availablez--enable_tensor_corez+Enable Tensor Core math for Conv and FC opsz--distributed_transportZtcpz2Transport to use for distributed run [tcp|ibverbs]z--distributed_interfacesr%   z-Network interfaces to use for distributed runz--first_iter_timeoutiX  z<Timeout (secs) of the first iteration (default: %(default)s)z	--timeouté<   zJTimeout (secs) of each (except the first) iteration (default: %(default)s)z--modelrh   ú?ri   zList of models which can be run)rÁ   ÚconstrÇ   rÍ   rÃ   )	ÚargparseÚArgumentParserÚadd_argumentr°   rS   rÌ   ÚboolÚ
parse_argsr¾   )Úparserr?   r!   r!   r"   Úmainy  s¦    




























r×   Ú__main__Zcaffe2z--caffe2_log_level=2)NN)8Ú
__future__r   r   r   r   rÑ   ÚloggingZnumpyrY   rW   r¶   Zcaffe2.pythonr   r   r   r	   r
   r   r   r   r   Zcaffe2.protor   Zcaffe2.python.models.resnetÚpythonÚmodelsr‰   Zcaffe2.python.models.shufflenetri   Z#caffe2.python.modeling.initializersr   r   Z*caffe2.python.predictor.predictor_exporterZ	predictorZpredictor_exporterr7   Z*caffe2.python.predictor.predictor_py_utilsZpredictor_py_utilsrG   Z!caffe2.python.predictor_constantsr   ÚbasicConfigÚ	getLoggerrE   ÚsetLevelÚDEBUGZInitOpsLibraryr#   r,   rD   rL   rg   r¾   r×   Ú__name__Z
GlobalInitr!   r!   r!   r"   Ú<module>   sH   



 
 T  y\
