3
H4W^ÁF  ã               @   s  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZ d dlmZ d dlm	Z	 d dlm
Z
 d	d
lmZ d	dlmZ d	dlmZ d dlZd dlZd dlZd dlZejƒ  ejdƒZejejƒ dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZG dd„ deƒZ dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)Úabstractmethod)Ú	workspace)Útimeout_guard)Údata_parallel_modelé   )Ú
checkpoint)ÚModuleRegister)Ú
module_mapNÚAnyExpc             C   sÜ   t jdddgƒ | d d dks8| d d dks8tdƒ‚| d d dk}|rX| d d n
| d d }|rt| d d	 n
| d d
 }|rˆdnd| d d< |o¢| d d | d d< || d d< || d d< i | d< i | d d< | S )NZcaffe2z--caffe2_log_level=2z--caffe2_gpu_memory_tracking=0ÚdistributedZnum_gpusr   Znum_cpuszCNeed to specify num_gpus or num_cpus to decide which device to use.Zfirst_cpu_idZfirst_gpu_idÚcpuZgpuÚdeviceÚmodel_paramÚcombine_spatial_bnÚnum_xpusÚfirst_xpu_idÚtemp_varÚmetrics_output)r   Z
GlobalInitÚAssertionError)ÚoptsZtrainWithCPUr   Ú	first_xpu© r   úA/tmp/pip-build-e66j97b2/torch/caffe2/contrib/playground/AnyExp.pyÚinitOpts   s    r   c               C   s   t tƒ d S )N)ÚregisterModuleMapr   r   r   r   r   ÚinitDefaultModuleMap6   s    r   c             C   s   t j| ƒ d S )N)r   r   )r   r   r   r   r   :   s    r   c             C   s   t j| d d ƒ}|j| ƒS )NÚinputZinput_name_py)r   Z	getModuleÚget_input_dataset)r   ZmyAquireDataModuler   r   r   ÚaquireDatasets>   s    r"   c             C   s   t jt| ƒS )N)r   ZconstructTrainerClassÚAnyExpTrainer)r   r   r   r   ÚcreateTrainerClassC   s    r$   c             C   s   t j| |ƒS )N)r   ÚoverrideAdditionalMethods)ZmyTrainerClassr   r   r   r   r%   G   s    r%   c              O   s   t j| |ŽS )N)r   Úinitialize_params_from_file)ÚargsÚkwargsr   r   r   r&   K   s    r&   c               @   sp  e Zd Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZd	d
„ Zdd„ Z	dd„ Z
dd„ Zedd„ ƒZdd„ Zedd„ ƒZdd„ Zdd„ Zedd„ ƒZedd„ ƒZdd „ Zd!d"„ Zed#d$„ ƒZed%d&„ ƒZed'd(„ ƒZed)d*„ ƒZd+d,„ Zed-d.„ ƒZed/d0„ ƒZed1d2„ ƒZed3d4„ ƒZed5d6„ ƒZed7d8„ ƒZed9d:„ ƒZ ed;d<„ ƒZ!d=d>„ Z"d?d@„ Z#dAdB„ Z$dCdD„ Z%dEdF„ Z&dGS )Hr#   c             C   s€  dd l }|jƒ  |jdƒ}|j|jƒ || _|| _d | _d | _d | _	d | _
i | _g | _g | _g | _g | _|d d | _|d d }|d d }t||| ƒ| _| jd d	 | jd d  | jd d
  | _| jd d | j | _t|d d ƒdk�r0|d d d | _	t|d d ƒdk�r0|d d d | _
|d d | _|d d | _|d d | _|d d | _|jdjt|ƒƒƒ d S )Nr   r   r   r   r   r   r   Ú
epoch_iterZbatch_per_deviceZ
num_shardsZnum_train_sample_per_epochr    Zdatasetsé   r
   Úshard_idÚstart_epochÚepochÚnum_epochs_per_flow_schedulezopts: {})ÚloggingÚbasicConfigÚ	getLoggerÚsetLevelÚDEBUGÚlogr   Útrain_datasetÚtest_datasetZtrain_dfZtest_dfÚmetricsÚplotsIngredientsÚrecord_epochsÚsamples_per_secÚsecs_per_trainr   ÚrangeÚxpusÚtotal_batch_sizeÚepoch_iterationsÚlenr+   r,   r-   Zepochs_to_runÚinfoÚformatÚstr)Úselfr   r/   r4   r   r   r   r   r   Ú__init__Q   s@    

zAnyExpTrainer.__init__c             C   s   d S )Nr   )rD   r   r   r   r   r!   ‚   s    zAnyExpTrainer.get_input_datasetc             C   s   d S )Nr   )rD   r   r   r   Úget_model_input_fun†   s    z!AnyExpTrainer.get_model_input_func             C   s   d S )Nr   )rD   r   r   r   Ú
init_modelŠ   s    zAnyExpTrainer.init_modelc             C   s\   | j d d }xH|D ]@}| j|d ƒ}|f d| j i|d —Ž}| j|d ||d ƒ qW d S )NÚoutputr7   Zmeter_pyr   Zmeter_kargsÚnameÚis_train)r   ÚgetMeterClassÚ
add_metric)rD   r7   ÚmetricZ
meterClassZmeterInstancer   r   r   Úinit_metricsŽ   s
    
zAnyExpTrainer.init_metricsc             C   s   t j||ƒS )N)r   ZgetClassFromModule)rD   Z	meterNamer   r   r   rK   —   s    zAnyExpTrainer.getMeterClassc             C   s6   | j }i ||< ||| d< ||| d< g || d< d S )NÚ
calculatorrJ   rH   )r7   )rD   rI   rO   rJ   r7   r   r   r   rL   š   s
    zAnyExpTrainer.add_metricc             C   s¬   | j }|sN| j|d< | j|d< | j|d< x~| jjƒ D ]\}}|d ||< q4W nZ|d j| jƒ |d j| jƒ |d j| jƒ x(| jjƒ D ]\}}|| j|d ƒ qŠW d S )NZepochsr:   r;   rH   )r   r9   r:   r;   r7   ÚitemsÚextend)rD   r   rM   Úvaluer   r   r   ÚextendMetricsOutput¡   s    


z!AnyExpTrainer.extendMetricsOutputc             C   s   d S )Nr   )rD   r   r   r   Ú
init_plots°   s    zAnyExpTrainer.init_plotsc             C   s8   | j }i }||d< ||d< ||d< ||d< |j|ƒ d S )NÚxÚx_titleÚysÚy_title)r8   Úappend)rD   rU   rV   rW   rX   r8   ZaPlotIngredientsr   r   r   Úadd_plot´   s    zAnyExpTrainer.add_plotc             C   s   d S )Nr   )rD   r   r   r   Ú	init_logs½   s    zAnyExpTrainer.init_logsc             C   s0   t | jd d | j| jd d  ƒ}t| j|ƒS )Nr)   Ú
num_epochsr.   )Úminr   r-   r<   )rD   Ziter_end_pointr   r   r   Úlist_of_epochsÁ   s    zAnyExpTrainer.list_of_epochsc             C   s   t d| jƒS )Nr   )r<   r?   )rD   r   r   r   Úlist_of_epoch_itersÇ   s    z!AnyExpTrainer.list_of_epoch_itersc             C   s   d S )Nr   )rD   r-   r   r   r   Úfun_per_epoch_b4RunNetÊ   s    z$AnyExpTrainer.fun_per_epoch_b4RunNetc             C   s   d S )Nr   )rD   r-   r   r   r   Úfun_per_epoch_aftRunNetÎ   s    z%AnyExpTrainer.fun_per_epoch_aftRunNetc             C   s2   t jd| j| jd|d ƒ|d | jtdƒƒ| _d S )NTr
   z-inf)r   Zsave_model_paramsÚtrain_modelÚgen_checkpoint_pathr   ÚfloatÚ
model_path)rD   r-   r   r   r   r   Ò   s    zAnyExpTrainer.checkpointc             C   s&   |rdj |ƒ}nd}| jd d | S )Nzmodel_checkpoint_epoch{}.pklzmodel_final.pklrH   Zcheckpoint_folder)rB   r   )rD   Zis_checkpointr-   Úfilenamer   r   r   rc   ×   s    z!AnyExpTrainer.gen_checkpoint_pathc             C   s   d S )Nr   )rD   r-   r)   r   r   r   Úfun_per_iter_b4RunNetâ   s    z#AnyExpTrainer.fun_per_iter_b4RunNetc             C   s   d S )Nr   )rD   r-   r)   r   r   r   Úfun_per_iter_aftRunNetB4Testæ   s    z*AnyExpTrainer.fun_per_iter_aftRunNetB4Testc             C   s   d S )Nr   )rD   r-   r)   r   r   r   Úfun_per_iter_aftRunNetAftTestê   s    z+AnyExpTrainer.fun_per_iter_aftRunNetAftTestc             C   s   d S )Nr   )rD   r   r   r   r   Úfun_conclude_operatorî   s    z#AnyExpTrainer.fun_conclude_operatorc             C   s   | j ƒ  | j| _d S )N)rS   re   Zmodel_output)rD   r   r   r   ÚcreateMetricsPlotsModelsOutputsò   s    z-AnyExpTrainer.createMetricsPlotsModelsOutputsc             C   s   d S )Nr   )rD   r   r   r   ÚassembleAllOutputsö   s    z AnyExpTrainer.assembleAllOutputsc             C   s   d S )Nr   )rD   ÚmodelÚdatasetrJ   r   r   r   Úgen_input_builder_funú   s    z#AnyExpTrainer.gen_input_builder_func             C   s   d S )Nr   )rD   rm   rn   rJ   r   r   r   Úgen_forward_pass_builder_funþ   s    z*AnyExpTrainer.gen_forward_pass_builder_func             C   s   d S )Nr   )rD   rm   rn   rJ   r   r   r   Úgen_param_update_builder_fun  s    z*AnyExpTrainer.gen_param_update_builder_func             C   s   d S )Nr   )rD   rm   rn   rJ   r   r   r   Úgen_optimizer_fun  s    zAnyExpTrainer.gen_optimizer_func             C   s   d S )Nr   )rD   rm   rn   rJ   r   r   r   Úgen_rendezvous_ctx
  s    z AnyExpTrainer.gen_rendezvous_ctxc             C   s   d S )Nr   )rD   r   r   r   Úrun_training_net  s    zAnyExpTrainer.run_training_netc          	   C   s@   | j d krd S d}tj|ƒ� tj| j jjƒ jƒ W d Q R X d S )Ng     @Ÿ@)Ú
test_modelr   ZCompleteInTimeOrDier   ZRunNetÚnetÚProtorI   )rD   Útimeoutr   r   r   Úrun_testing_net  s
    
zAnyExpTrainer.run_testing_netc             C   s   | j ƒ  | jƒ  | jƒ  d S )N)rN   rT   r[   )rD   r   r   r   Úplanning_output  s    zAnyExpTrainer.planning_outputc             C   s(   | j | j| jdƒ | j | j| jdƒ d S )NTF)Úprep_a_data_parallel_modelrb   r5   ru   r6   )rD   r   r   r   Úprep_data_parallel_models   s    
z'AnyExpTrainer.prep_data_parallel_modelsc             C   s0  |d krd S t jdƒ | jd k	r.| j|||ƒnd }t jdƒ | jd k	rT| j|||ƒnd }t jdƒ | jd d }tj|| j|||ƒ| j|||ƒ||| j	| j
|||ƒd| jd d | jd d || jd	 d
 dk| jd d d� t jdƒ tj|jƒ t jdƒ t jdj|jjƒ ƒƒ tj|jƒ t jdƒ d S )Nzin prep_a_data_parallel_modelz0in prep_a_data_parallel_model param_update done z-in prep_a_data_parallel_model optimizer done r   Úmax_concurrent_distributed_opsFZmemongerZ	cuda_ncclr   r   r   r   )Zinput_builder_funZforward_pass_builder_funZparam_update_builder_funZoptimizer_builder_funZdevicesZ
rendezvousZbroadcast_computed_paramsZoptimize_gradient_memoryZuse_ncclr}   Z
cpu_devicer   z/in prep_a_data_parallel_model Parallelize done z.in prep_a_data_parallel_model RunNetOnce done zmodel.net.Proto() {}z-in prep_a_data_parallel_model CreateNet done )r4   rA   rq   rr   r   r	   ZParallelizero   rp   r=   rs   r   Z
RunNetOnceZparam_init_netrB   rv   rw   Z	CreateNet)rD   rm   rn   rJ   Zparam_updateZ	optimizerZmax_opsr   r   r   r{   &  s:    




z(AnyExpTrainer.prep_a_data_parallel_modelc       	      C   sÈ   | j }|d d }|d d }|d d }|d k	rntjj|ƒr¸tjdj|ƒƒ tj| j	|||ddd	�\}}}nJ|d k	r¸tjj|ƒr¸tjd
j|ƒƒ tj| j	|||d|d d d	�\}}}t
j| j	ƒ d S )Nr   Zcheckpoint_modelÚpretrained_modelr   r   zLoad previous checkpoint:{}TF)rm   Zweights_filer   r   Zbroadcast_computed_paramÚreset_epochzLoad pretrained model: {}r   r   )r   ÚosÚpathÚexistsr4   rA   rB   r   r&   rb   r	   ZFinalizeAfterCheckpoint)	rD   r   Zprevious_checkpointr~   r   r,   Zprev_checkpointed_lrZ_best_metricZbest_metricr   r   r   ÚloadCheckpoint^  s2    zAnyExpTrainer.loadCheckpointc             C   sØ  t jdjt|ƒƒƒ t jdjt| ƒƒƒ t jdjt| ƒƒƒ t jdj| jƒƒ t jdjtj| j	ƒƒƒ t jdjtj| j
ƒƒƒ t jdjtj| jƒƒƒ | jd k	rºt jdjtj| jƒƒƒ nt jd	jtj| jƒƒƒ t jd
jtj| jƒƒƒ t jdjtj| jƒƒƒ | jƒ  | jƒ  | jƒ  | jƒ  | jƒ  �x’| jƒ D �]„}t jdj|ƒƒ | j|ƒ �xL| jƒ D �]>}tjƒ | _| j||ƒ | jd k	�r’| jƒ  | j||ƒ tjƒ | _||d d  dk�r”| j| j }| jj|ƒ | j }|| }| j!j|ƒ |t"|ƒ| j#  | _$| j%j| j$ƒ xJ| j&D ]@}| j&| }|d �s6�q|d j'ƒ  |d j|d j(ƒ ƒ �qW tjƒ | _)xZt*d|d d ƒD ]D}	| j+ƒ  x4| j&D ]*}| j&| }|d �r¬�q�|d j'ƒ  �q�W �q|W tjƒ | _,| j,| j) | _-x8| j&j.ƒ D ]*}|d �rú�qè|d j|d j(ƒ ƒ �qèW dj| j$| j/d d || j#|ƒ}
|
dj||d d | j-ƒ7 }
x0| j&j0ƒ D ]"\}}|
dj||d d ƒ7 }
�q^W t jdj|
ƒƒ | j1||ƒ �qbW | j2|ƒ | j3|ƒ �q6W | j4ƒ  | j5ƒ  | jƒ S )Nz(in buildModelAndTrain, trainer_input: {}zcheck type self: {}zcheck self dir: {}zcheck self source: {}z(check self get_input_dataset methods: {}z+check self gen_input_builder_fun method: {}z2check self gen_forward_pass_builder_fun method: {}z2check self gen_param_update_builder_fun method: {}z'check self gen_optimizer_fun method: {}z(check self assembleAllOutputs method: {}z/check self prep_data_parallel_models method: {}zstart training epoch {}r)   Znum_train_iteration_per_testr   rJ   rO   rH   Znum_test_iterz)epoch:{}/{} iter:{}/{} secs_per_train:{} r\   z-samples_per_sec:{} loop {} tests takes {} secz {}:{} r
   zIter Stats: {}éÿÿÿÿ)6r4   rA   rB   rC   ÚtypeÚdirÚ__dict__ÚinspectÚ	getsourcer!   ro   rp   rq   rr   rl   r|   rF   rG   rz   rƒ   r^   r`   r_   ÚtimeZiter_start_timerg   rb   rt   rh   Ziter_end_timer;   rY   r>   r:   rd   r?   Zfract_epochr9   r7   ÚAddZComputeZtest_loop_start_timer<   ry   Ztest_loop_end_timeZsec_per_test_loopÚvaluesr   rP   ri   r   ra   rj   rk   )rD   r   r-   r)   r;   Zsample_trainedr:   ÚkeyrM   Z
_test_iterZlogStrrR   r   r   r   ÚbuildModelAndTrain  sž    












z AnyExpTrainer.buildModelAndTrainN)'Ú__name__Ú
__module__Ú__qualname__rE   r   r!   rF   rG   rN   rK   rL   rS   rT   rZ   r[   r^   r_   r`   ra   r   rc   rg   rh   ri   rj   rk   rl   ro   rp   rq   rr   rs   rt   ry   rz   r|   r{   rƒ   rŽ   r   r   r   r   r#   O   sF   1			8!r#   )!Ú
__future__r   r   r   r   Úabcr   Zcaffe2.pythonr   r   r	   Ú r   r   r   r€   rˆ   rŠ   r/   r0   r1   r4   r2   r3   r   r   r   r"   r$   r%   r&   Úobjectr#   r   r   r   r   Ú<module>   s2   
