3
I4W^î4  ã               @   s<  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZ d dlm	Z	m
Z
mZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZmZ d dlmZ d dlmZmZmZm Z m!Z! d dl"m#Z# d dl$m%Z% d dl&Z'd dl(Z(d dl)Z)d dl*Z*dd„ Z+ddddgZ,dd„ Z-G dd„ deƒZ.G dd„ de#ƒZ/dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚStructÚConstRecord)ÚcoreÚ	workspaceÚmodel_helper)ÚLocalSession)ÚDataset)Úpipe)ÚCheckpointManagerÚMultiNodeCheckpointManagerÚJobÚ	JobRunnerÚepoch_limiterÚUploadTaskGroupBuilderÚdb_name)Úops)ÚNodeÚTaskÚ	TaskGroupÚWorkspaceTypeÚCluster)ÚTestCase)ÚReaderWithLimitNc                sÆ   t d|  ƒ�® tjƒ j�b tƒ �P tdtjtt	dƒƒƒfƒ}t
t|ƒ}t|d|  d�}|jtƒ}tjdgƒ‰ W d Q R X W d Q R X ‡ fdd„}t|d	d
�}t||d� tjƒ j|jƒ ƒ W d Q R X ˆ gS )Nz
trainer_%dÚvalé
   z
dataset:%d)Únameéd   c                s   t jˆ | jƒ gˆ gƒ d S )N)r   ÚAddr   )Zrec)Útotal© ú>/tmp/pip-build-e66j97b2/torch/caffe2/python/checkpoint_test.pyÚ	inc_total"   s    z!build_pipeline.<locals>.inc_totalé   )Znum_iter)Ú	processor)r   r   ÚcurrentÚ
init_groupr   r   ÚnpÚarrayÚlistÚranger   r   r   ÚreaderZConstr   r   Zadd_stop_conditionZdata_finished)Únode_idZdata_arrÚdataZdsZfull_readerr%   Zepoch_readerr#   )r"   r$   Úbuild_pipeline   s    

 r1   ég   és   éˆ   é‘   c                s   ‡ ‡fdd„}|S )Nc                s   t jˆˆ ƒ d S )N)ÚshutilÚcopyfile)ÚinputsÚoutputs)ÚdestÚsrcr#   r$   Úcopy_op/   s    zlocal_copy_op.<locals>.copy_opr#   )r;   r:   r<   r#   )r:   r;   r$   Úlocal_copy_op.   s    r=   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚUploadToLocalFilec             C   s
   || _ d S )N)Údest_dir)Úselfr?   r#   r#   r$   Ú__init__5   s    zUploadToLocalFile.__init__c             C   s–   t tjƒ�‚}xz|jD ]p\}}tt|ƒƒ�V tƒ �D t||j|j	ƒ}t
jj| jt|ƒƒ}tjt||gi fƒg g ƒ W d Q R X W d Q R X qW W d Q R X |S )N)r   r   ÚGLOBALZ_node_managersr   Ústrr   r   Z
_node_nameZ
_db_prefixÚosÚpathÚjoinr?   r   ÚPythonr=   )r@   ÚepochZcheckpoint_managerZupload_task_groupÚnodeÚmanagerZsrc_pathZ	dest_pathr#   r#   r$   Úbuild8   s    6zUploadToLocalFile.buildN)Ú__name__Ú
__module__Ú__qualname__rA   rK   r#   r#   r#   r$   r>   4   s   r>   c               @   sD   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dS )ÚTestCheckpointc       
         s$  t ƒ �� tƒ �}tdd�}W d Q R X ttjdƒ|d�‰ ‡ fdd„}|ƒ \}}|jtƒ t||ƒj	|ƒ}| j
|ttƒƒ | j
||ƒtd	 ƒ xHtd|d ƒD ]6}|ƒ \}}t|||d�j	|ƒ | j
||ƒtd
 ƒ qœW x>td|d ƒD ],}	|j|j|	ƒƒ | j
||ƒt|	d  ƒ qæW W d Q R X d S )Nr   )r/   Úempty)Ústepr9   c                s   | j ˆ ƒ ˆ jƒ d jƒ S )Nr   )Úrunr9   Úfetch)Úsession)Úoutput_fetcherr#   r$   Úfetch_totalJ   s    
z,TestCheckpoint.run_with.<locals>.fetch_totalé   )Úresume_from_epochéÿÿÿÿrY   )r   r   r1   r   r   ÚNetÚcompiler   r   ÚtrainÚassertEqualsÚlenÚEXPECTED_TOTALSr-   rR   Úload)
r@   ÚbuilderÚjobr9   rV   rT   Ú
checkpointÚ
num_epochsÚinitial_epochrH   r#   )rU   r$   Úrun_withD   s*    





zTestCheckpoint.run_withc                sd   z"t jƒ ‰ ‡ fdd„}| j|ƒ W d tjˆ ƒ X z"t jƒ ‰ ‡ fdd„}| j|ƒ W d tjˆ ƒ X d S )Nc                 s&   t jjƒ } t| ƒ}tˆ ddƒ}||fS )NZ	temp_nodeÚminidb)r	   ÚCÚ	Workspacer   r   )ÚwsrT   rc   )Útmpdirr#   r$   ra   f   s    
z6TestCheckpoint.test_single_checkpoint.<locals>.builderc                 s$   t jjƒ } t| ƒ}tˆ dƒ}||fS )Nrg   )r	   rh   ri   r   r   )rj   rT   rc   )rk   r#   r$   ra   t   s    

)ÚtempfileÚmkdtemprf   r6   Úrmtree)r@   ra   r#   )rk   r$   Útest_single_checkpointa   s    z%TestCheckpoint.test_single_checkpointc          *   C   s˜  �z„d}t jƒ }t|dƒ}tƒ �Œ tƒ � }xt|ƒD ]}t|ƒ q4W W d Q R X |jtƒ |j	|j
ƒ ƒ x@t|ƒD ]4}d}d| }|d | d }| j|j||ƒ|ƒ qpW W d Q R X tj|ƒ t jƒ }tjƒ  x˜t|ƒD ]Œ}tjjƒ }	t|	ƒ}
t|dƒ}tƒ �> tƒ �}t|ƒ W d Q R X |jtƒ t||ƒ}|j|
ƒ}W d Q R X | j|ttƒƒ | jt|	jƒdƒ qÖW tjjƒ }	t|	ƒ}
| jt|	jƒdƒ d	d
g}t|dƒ}tƒ �Ü tƒ �"}xt|ƒD ]}t|ƒ �q¶W W d Q R X |jtƒ t||ƒ}|j|d|
d� xltddƒD ]^}| j|j|||
d�ƒ x@|D ]8}| j|	j|ƒƒ | j|	j|ƒtjt|d  gƒƒ �q"W �qW | j|j|d|
d�ƒ W d Q R X W d tj|ƒ X d S )Nr&   rg   é   z
trainer_%dú/z.5é   r   z'trainer_1/task_2/GivenTensorInt64Fill:0z'trainer_2/task_2/GivenTensorInt64Fill:0rW   )Z
blob_namesrH   rT   )rl   rm   r   r   r   r-   r1   r[   r   ÚinitZnodes_to_checkpointr]   Zget_ckpt_db_namer6   rn   r	   ÚResetWorkspacerh   ri   r   r\   r^   r_   ZblobsZload_blobs_from_checkpointsÚ
assertTrueZhas_blobÚ
fetch_blobr*   r+   ÚassertFalse)r@   Ú	num_nodesrk   rc   rb   r/   rH   Ú	node_nameZexpected_db_namerj   rT   Ú
job_runnerrd   Zmodel_blob_namesZ	blob_namer#   r#   r$   Ú(test_ckpt_name_and_load_model_from_ckpts~   sx    













"z7TestCheckpoint.test_ckpt_name_and_load_model_from_ckptsc             C   sF  �z2t jƒ }tjj|dƒ}tj|ƒ d}x8t|ƒD ],}d| }tjj||ƒ}| jtjj|ƒƒ q2W x’tdƒD ]†}t	j
jƒ }t|ƒ}t|dƒ}	tƒ �Z tƒ �}
t|ƒ W d Q R X |
jtƒ t|ƒ}t|
|	|d�}|j|ƒ}| j|ttƒƒ W d Q R X qlW x:t|ƒD ].}d| }tjj||ƒ}| jtjj|ƒƒ �q W W d tj|ƒ X d S )NÚuploadr&   z
trainer_%drg   )Zupload_task_group_builder)rl   rm   rD   rE   rF   Úmkdirr-   rw   Úexistsr	   rh   ri   r   r   r   r   r1   r[   r>   r   r\   r]   r^   r_   ru   r6   rn   )r@   rk   Z
upload_dirrx   r/   ry   Zupload_pathrj   rT   rc   rb   Zlocal_upload_builderrz   rd   r#   r#   r$   Útest_upload_checkpointË   s8    




z%TestCheckpoint.test_upload_checkpointc       
      C   sœ   d}d}t jƒ  x†t|ƒD ]z}t jjƒ }t|ƒ}t|dƒ}tƒ �> tƒ �}t	|ƒ W d Q R X |j
tƒ t||ƒ}|j|ƒ}	W d Q R X | j|	ttƒƒ qW d S )Nr&   z/tmp/path_does_not_exist/rg   )r	   rt   r-   rh   ri   r   r   r   r   r1   r[   r   r\   r]   r^   r_   )
r@   rx   rk   r/   rj   rT   rc   rb   rz   rd   r#   r#   r$   Útest_ckpt_save_failureñ   s    



z%TestCheckpoint.test_ckpt_save_failurec       	   6   C   s~  t jdd�}tjdƒ}x&dD ]}|jjg |gdgd	d
d� qW |jjddgdgƒ |jdgdgƒ t	ƒ �¤}t
dƒ�� |j� t|jd� W dQ R X |j�: tƒ �( tjdƒ� tj|jƒ W dQ R X W dQ R X W dQ R X |j� t|d� W dQ R X t|dƒ W dQ R X W dQ R X tjjƒ }t|ƒ}t|ƒ}|j|ƒ tjddƒjtjƒ}| jtj||jdƒƒƒ | jtj||jdƒƒƒ dS )z}
        A simple test that ensures we have download task group
        executed between epoch_group and exit_group.
        Z
test_model)r   Údownload_netÚinput1Úinput2ÚoutputÚdownload_resulté   g      ð?r   )ÚshapeÚvalueZrun_oncez	trainer:0)rQ   NrW   g       @)r‚   rƒ   r„   r…   )r
   ZModelHelperr   rZ   Zparam_init_netZConstantFillÚnetr!   ZCopyr   r   r)   r   Zepoch_groupr   ZloopZdownload_groupr   r	   rh   ri   r   r   r\   r*   ÚfullZastypeZfloat32ru   Zarray_equalrv   )	r@   Úmodelr�   r   rb   rj   rT   rz   Zexpected_resultr#   r#   r$   Útest_download_group_simple	  s<    


*



z)TestCheckpoint.test_download_group_simplec       
         sÎ   z¼t jƒ }tjjƒ }t|ƒ}t|dƒ}tƒ �}tdd�}W dQ R X t	t
jdƒ|d�‰ |jtƒ ‡ fdd„}t||ƒj|ƒ}x>td	|d	 ƒD ],}	t|||	d
�j|ƒ | j||ƒtd ƒ qŠW W dtj|ƒ X dS )zf
        A simple test that ensures we can reuse a MultiNodeCheckpointManager
        object.
        rg   r   )r/   NrP   )rQ   r9   c                s   | j ˆ ƒ ˆ jƒ d jƒ S )Nr   )rR   r9   rS   )rT   )rU   r#   r$   rV   D  s    
zATestCheckpoint.test_reuse_checkpoint_manager.<locals>.fetch_totalrW   )rX   rY   )rl   rm   r	   rh   ri   r   r   r   r1   r   r   rZ   r[   r   r\   r-   r]   r_   r6   rn   )
r@   rk   rj   rT   rc   rb   r9   rV   rd   re   r#   )rU   r$   Útest_reuse_checkpoint_manager4  s&    


z,TestCheckpoint.test_reuse_checkpoint_managerN)
rL   rM   rN   rf   ro   r{   r   r€   rŒ   r�   r#   r#   r#   r$   rO   C   s   M&+rO   )0Ú
__future__r   r   r   r   Zcaffe2.python.schemar   r   Zcaffe2.pythonr   r	   r
   Zcaffe2.python.sessionr   Zcaffe2.python.datasetr   Zcaffe2.python.pipeliner   Zcaffe2.python.checkpointr   r   r   r   r   r   r   Zcaffe2.python.net_builderr   Zcaffe2.python.taskr   r   r   r   r   Zcaffe2.python.test_utilr   Zcaffe2.python.dataior   Znumpyr*   rD   r6   rl   r1   r_   r=   r>   rO   r#   r#   r#   r$   Ú<module>   s,   $