3
H4W^Hb  ã               @   s  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZ d dljZ	d dl
mZmZ d dlZd dlZd dlZd dlZd dlZd dlmZmZmZ d dljjZd d	lmZ ejd
ƒ ejdƒ ejdƒ ejdƒ ejdƒ dZG dd„ dƒZG dd„ dej ƒZ!e"dk�rd dl#Z#e#j$ƒ  dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚgivenN)ÚProcessÚQueue)ÚcoreÚ	workspaceÚdyndep)ÚIoErrorz2@/caffe2/caffe2/distributed:file_store_handler_opsz3@/caffe2/caffe2/distributed:redis_store_handler_opsz%@/caffe2/caffe2/distributed:store_opsz%@/caffe2/caffe2/contrib/gloo:gloo_opsz)@/caffe2/caffe2/contrib/gloo:gloo_ops_gpuZGLOOc               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTemporaryDirectoryc             C   s   t jƒ | _| jS )N)ÚtempfileÚmkdtempÚtmpdir)Úself© r   ú>/tmp/pip-build-e66j97b2/torch/caffe2/contrib/gloo/gloo_test.pyÚ	__enter__    s    
zTemporaryDirectory.__enter__c             C   s   t j| jƒ d S )N)ÚshutilÚrmtreer   )r   ÚtypeÚvalueÚ	tracebackr   r   r   Ú__exit__$   s    zTemporaryDirectory.__exit__N)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r   r   r      s   r   c               @   s8  e Zd ZdZdZd6dd„Zd7dd„Zd8dd„Zd9d	d
„Zd:dd„Z	e
ejddd�ejddd�ejddd�ejejgƒejƒ d�dd„ ƒZd;dd„Zd<dd„Ze
ejddd�ejddd�ejddd�ejejgƒejƒ d�dd„ ƒZd=dd„Ze
ejddd�ejddd�ejddd�ejejgƒejƒ d�d d!„ ƒZd>d"d#„Ze
ejddd�ejddd�ejddd�ejejgƒejƒ d�d$d%„ ƒZe
ejejgƒd&�d'd(„ ƒZd?d)d*„Ze
ejddd�ejejgƒd+�d,d-„ ƒZd@d.d/„Ze
ejddd�ejejgƒd+�d0d1„ ƒZdAd2d3„Ze
ejddd�ejejgƒd+�d4d5„ ƒZdS )BÚTestCaser   Nc       	         sÀ   t ƒ ‰‡ ‡‡fdd„}g }x:t|d ƒD ]*}||d< t||d�}|jƒ  |j|ƒ q(W xdt|ƒdkrº|jdƒ}x|jƒ r„|jdƒ qpW | j	ˆj
ƒ dƒ ˆjƒ }t|tƒr®|‚qX| j|ƒ qXW d S )	Nc                 sh   y6t jˆ ƒ�" ˆ| |Ž tjƒ  ˆjdƒ W d Q R X W n, tk
rb } zˆj|ƒ W Y d d }~X nX d S )NT)r	   ÚDeviceScoper
   ÚResetWorkspaceÚputÚ	Exception)ÚargsÚkwargsÚex)Údevice_optionÚfnÚqueuer   r   Úrun_fn1   s    
z)TestCase.run_test_locally.<locals>.run_fnÚ	comm_sizeÚ	comm_rank)Útargetr$   r   é
   zJob failed without a result)r   Úranger   ÚstartÚappendÚlenÚpopÚis_aliveÚjoinZassertFalseÚemptyÚgetÚ
isinstancer"   Z
assertTrue)	r   r'   r&   r$   r)   ZprocsÚiÚprocÚor   )r&   r'   r(   r   Úrun_test_locally,   s&    



zTestCase.run_test_locallyc          
   K   sl   t jdƒ}| j|ƒ t jdƒ}| j|ƒ t|ƒ|d< t|ƒ|d< tj|ƒ� |f |Ž tjƒ  W d Q R X d S )NÚ	COMM_RANKZ	COMM_SIZEr+   r*   )ÚosÚgetenvZassertIsNotNoneÚintr	   r   r
   r    )r   r'   r&   r$   r+   r*   r   r   r   Úrun_test_distributedV   s    




zTestCase.run_test_distributedc       	   
   C   sÖ   d}|d krxt jdƒ}tt jddƒƒ}|d k	rXtjtjdg |gttj	ƒd ||d�ƒ ntjtjdg |g|d	�ƒ d
}nt|ƒd }|d k	r¬tjtjd|g|gdt
d�ƒ n"tjtjd|g|g||dt
d�ƒ ||fS )NÚstore_handlerZ
REDIS_HOSTZ
REDIS_PORTië  ZRedisStoreHandlerCreateú/)ÚprefixÚhostÚportZFileStoreHandlerCreate)ÚpathÚcommon_worldz.forkedZCloneCommonWorldT)ÚsyncÚengineZCreateCommonWorld)ÚsizeZrankrH   rI   )r=   r>   r?   r
   ÚRunOperatorOncer	   ÚCreateOperatorÚstrr   Útest_counterÚ	op_engine)	r   r+   r*   r   Úexisting_cwrA   Z
redis_hostZ
redis_portrG   r   r   r   Úcreate_common_worlda   sN    


zTestCase.create_common_worldc             C   sv   t  jd7  _djt jƒ}|dkrNtj|tj|ƒƒ tjtj	d||gg ƒƒ ntjtj	d|g|gƒƒ tj
tj|ƒƒS )Né   zsync_{}r   ZStoreSetZStoreGet)r   Úsync_counterÚformatr
   ÚFeedBlobÚpickleÚdumpsrK   r	   rL   ÚloadsÚ	FetchBlob)r   rA   r   r+   Úblobr   r   r   Úsynchronize�   s    

zTestCase.synchronizeFc             C   s0  | j |||d�\}}| j|||d�}| j|||d�}xöt|ƒD ]ê}	g }
xVt|ƒD ]J}dj|ƒ}|| | }tj|||r|tjntjƒ}tj	||ƒ |
j
|ƒ qPW tjdƒ}|j|g|
 |
|	td� tj|ƒ tj|jƒ ƒ x,t|ƒD ] }tjjtj|
| ƒ|	| ƒ qâW x tdƒD ]}tj|jƒ ƒ �qW q>W d S )N)r+   r*   r   )r+   zblob_{}Ú	broadcast)ÚrootrI   é   )rQ   r[   r.   rT   ÚnpÚfullÚfloat16Úfloat32r
   rU   r0   r	   ÚNetZ	BroadcastrO   Ú	CreateNetÚRunNetÚNameÚtestingÚassert_array_equalrY   )r   r+   r*   Ú	blob_sizeÚ	num_blobsr   Úuse_float16rA   rG   r8   ÚblobsÚjrZ   Úoffsetr   ÚnetÚ_tmpr   r   r   Ú_test_broadcast¢   sF    


zTestCase._test_broadcasté   é   )Z	min_valueZ	max_valueg     @�@g    €„.ArR   r^   )r*   ri   rj   r&   rk   c             C   sd   t  jd7  _tjdƒd k	r4| j| j||||d� n,tƒ � }| j| j||||||d� W d Q R X d S )NrR   r<   )ri   rj   rk   r&   )r*   ri   rj   r&   r   rk   )r   rN   r=   r>   r@   rq   r   r;   )r   r*   ri   rj   r&   rk   r   r   r   r   Útest_broadcast×   s"    
zTestCase.test_broadcastc             C   s(  | j |||d�\}}| j|||d�}| j|||d�}g }	xRt|ƒD ]F}
dj|
ƒ}tj||| |
 |rjtjntjƒ}tj	||ƒ |	j
|ƒ qBW tjdƒ}|j|g|	 |	td� tj|ƒ tj|jƒ ƒ x<t|ƒD ]0}
tjjtj|	|
 ƒ|| || d  d ƒ qÎW x tdƒD ]}tj|jƒ ƒ �qW d S )	N)r+   r*   r   )r+   zblob_{}Ú	allreduce)rI   rR   rr   r^   )rQ   r[   r.   rT   r_   r`   ra   rb   r
   rU   r0   r	   rc   Ú	AllreducerO   rd   re   rf   rg   rh   rY   )r   r+   r*   ri   rj   r   rk   rA   rG   rl   r8   rZ   r   ro   rp   r   r   r   Ú_test_allreduceñ   s@    


zTestCase._test_allreducec             C   s  | j |||d�\}}| j ||||d�\}}tdƒ}d}	xÆ||gD ]º}
g }xHt|	ƒD ]<}dj|ƒ}tj|||	 | tjƒ}tj||ƒ |j	|ƒ qRW t
jdƒ}|j|
g| |td� tj|ƒ x<t|	ƒD ]0}tjjtj|| ƒ|	| |	| d  d	 ƒ qÆW q@W d S )
N)r+   r*   r   )r+   r*   r   rP   g     ˆÃ@r^   zblob_{}Zallreduce_multicw)rI   rR   rr   )rQ   r?   r.   rT   r_   r`   rb   r
   rU   r0   r	   rc   rv   rO   Z
RunNetOncerg   rh   rY   )r   r+   r*   r   Z_store_handlerrG   Ú_Zcommon_world2ri   rj   Zcwrl   r8   rZ   r   ro   r   r   r   Ú_test_allreduce_multicw#  s8    


z TestCase._test_allreduce_multicwc             C   sd   t  jd7  _tjdƒd k	r4| j| j||||d� n,tƒ � }| j| j||||||d� W d Q R X d S )NrR   r<   )ri   rj   rk   r&   )r*   ri   rj   r&   r   rk   )r   rN   r=   r>   r@   rw   r   r;   )r   r*   ri   rj   r&   rk   r   r   r   r   Útest_allreduceJ  s"    
zTestCase.test_allreducec             C   s¨  | j |||d�\}}| j|||d�}| j|||d�}g }	xRt|ƒD ]F}
dj|
ƒ}tj||| |
 |rjtjntjƒ}tj	||ƒ |	j
|ƒ qBW tj|tjd�}|}|| d | }x2t|ƒD ]&}
t||ƒ||
< ||krÜ|| nd}qºW d}tj	||ƒ |	j
|ƒ tjdƒ}|j|g|	 |	td	� tj|ƒ tj|jƒ ƒ xJt|ƒD ]>}
tjjtjtj|	|
 ƒ|| ƒ|| || d  d
 ƒ �q@W x tdƒD ]}tj|jƒ ƒ �qŒW d S )N)r+   r*   r   )r+   zblob_{})ZdtyperR   r   Z
recvCountsZreduce_scatter)rI   rr   r^   )rQ   r[   r.   rT   r_   r`   ra   rb   r
   rU   r0   ÚzerosZint32Úminr	   rc   ZReduceScatterrO   rd   re   rf   rg   rh   ÚresizerY   )r   r+   r*   ri   rj   r   rk   rA   rG   rl   r8   rZ   r   Zrecv_countsÚ	remainingÚ
chunk_sizeZrecv_counts_blobro   rp   r   r   r   Ú_test_reduce_scatterd  sR    



 zTestCase._test_reduce_scatterc             C   sd   t  jd7  _tjdƒd k	r4| j| j||||d� n,tƒ � }| j| j||||||d� W d Q R X d S )NrR   r<   )ri   rj   rk   r&   )r*   ri   rj   r&   r   rk   )r   rN   r=   r>   r@   r€   r   r;   )r   r*   ri   rj   r&   rk   r   r   r   r   Útest_reduce_scatter¢  s"    
zTestCase.test_reduce_scatterc             C   s`  | j |||d�\}}| j|||d�}| j|||d�}g }	xRt|ƒD ]F}
dj|
ƒ}tj||| |
 |rjtjntjƒ}tj	||ƒ |	j
|ƒ qBW tjdƒ}|j|g|	 dgtd� tj|ƒ tj|jƒ ƒ tjg ƒ}xTt|ƒD ]H}
xBt|ƒD ]6}tj||
| | |�rtjntjƒ}tj||fƒ}qèW qÚW tjjtjdƒ|ƒ x tdƒD ]}tj|jƒ ƒ �qDW d S )N)r+   r*   r   )r+   zblob_{}Z	allgatherZGathered)rI   r^   )rQ   r[   r.   rT   r_   r`   ra   rb   r
   rU   r0   r	   rc   Z	AllgatherrO   rd   re   rf   ÚarrayZconcatenaterg   rh   rY   )r   r+   r*   ri   rj   r   rk   rA   rG   rl   r8   rZ   r   ro   Zexpected_outputrm   rp   r   r   r   Ú_test_allgather¼  sH    



zTestCase._test_allgatherc             C   sd   t  jd7  _tjdƒd k	r4| j| j||||d� n,tƒ � }| j| j||||||d� W d Q R X d S )NrR   r<   )ri   rj   rk   r&   )r*   ri   rj   r&   r   rk   )r   rN   r=   r>   r@   rƒ   r   r;   )r   r*   ri   rj   r&   rk   r   r   r   r   Útest_allgatherò  s"    
zTestCase.test_allgather)r&   c             C   sX   t  jd7  _tjdƒd k	r.| j| j|d� n&tƒ �}| j| jd||d� W d Q R X d S )NrR   r<   )r&   rr   )r*   r&   r   )r   rN   r=   r>   r@   ry   r   r;   )r   r&   r   r   r   r   Útest_forked_cw  s    
zTestCase.test_forked_cwc             C   sl   | j |||d�\}}tjdƒ}|j|gg td� tj|ƒ tj|jƒ ƒ xt	dƒD ]}tj|jƒ ƒ qRW d S )N)r+   r*   r   Zbarrier)rI   r^   )
rQ   r	   rc   ÚBarrierrO   r
   rd   re   rf   r.   )r   r+   r*   r   rA   rG   ro   rp   r   r   r   Ú_test_barrier!  s    

zTestCase._test_barrier)r*   r&   c             C   sX   t  jd7  _tjdƒd k	r.| j| j|d� n&tƒ �}| j| j|||d� W d Q R X d S )NrR   r<   )r&   )r*   r&   r   )r   rN   r=   r>   r@   r‡   r   r;   )r   r*   r&   r   r   r   r   Útest_barrier9  s    
zTestCase.test_barrierc       	      C   s†   ||d kf}| j |||d�\}}tjdƒ}|sD|j|gg td� n&|j|g|gtd� ddl}|jdƒ tj	|ƒ tj
|jƒ ƒ dS )z„
        One node calls close connection, others wait it on barrier.
        Test will check that all will exit eventually.
        rr   )r+   r*   r   Zbarrier_or_close)rI   r   Ngš™™™™™¹?)rQ   r	   rc   r†   rO   ZDestroyCommonWorldÚtimeÚsleepr
   rd   re   rf   )	r   r+   r*   r   ÚcloserrA   rG   ro   r‰   r   r   r   Ú_test_close_connectionI  s    



zTestCase._test_close_connectionc             C   s|   dd l }|j ƒ }t jd7  _tjdƒd k	r>| j| j|d� n&tƒ �}| j| j|||d� W d Q R X | j	|j ƒ | dƒ d S )Nr   rR   r<   )r&   )r*   r&   r   g      4@)
r‰   r   rN   r=   r>   r@   rŒ   r   r;   Z
assertLess)r   r*   r&   r‰   Ú
start_timer   r   r   r   Útest_close_connectionk  s    
zTestCase.test_close_connectionc             C   s®   | j |||d�\}}|dkrªd}d}g }xHt|ƒD ]<}	dj|	ƒ}
tj||| |	 tjƒ}tj|
|ƒ |j|
ƒ q2W t	j
dƒ}|j|g| |td� tj|ƒ tj|jƒ ƒ dS )	zV
        Only one node will participate in allreduce, resulting in an IoError
        )r+   r*   r   r   iè  rR   zblob_{}ru   )rI   N)rQ   r.   rT   r_   r`   rb   r
   rU   r0   r	   rc   rv   rO   rd   re   rf   )r   r+   r*   r   rA   rG   ri   rj   rl   r8   rZ   r   ro   r   r   r   Ú_test_io_error„  s*    	


zTestCase._test_io_errorc             C   sn   t  jd7  _| jtƒ�L tjdƒd k	r:| j| j|d� n&tƒ �}| j	| j|||d� W d Q R X W d Q R X d S )NrR   r<   )r&   )r*   r&   r   )
r   rN   ZassertRaisesr   r=   r>   r@   r�   r   r;   )r   r*   r&   r   r   r   r   Útest_io_error¨  s    
zTestCase.test_io_error)N)N)NN)N)NNNNNF)NNNNNF)NNN)NNNNNF)NNNNNF)NNN)NNN)NNN) r   r   r   rN   rS   r;   r@   rQ   r[   rq   r   ÚstZintegersZsampled_fromÚhuZcpu_doZbooleansrt   rw   ry   rz   r€   r�   rƒ   r„   r…   r‡   rˆ   rŒ   rŽ   r�   r�   r   r   r   r   r   (   sŠ   
*

/
     
/     
-  
$     
8     
0  
  
  
 r   Ú__main__)%Ú
__future__r   r   r   r   Z
hypothesisr   Zhypothesis.strategiesZ
strategiesr‘   Úmultiprocessingr   r   Znumpyr_   r=   rV   r   r   Zcaffe2.pythonr	   r
   r   Z"caffe2.python.hypothesis_test_utilÚpythonZhypothesis_test_utilr’   Zgloo.pythonr   ZInitOpsLibraryrO   r   ZHypothesisTestCaser   r   ZunittestÚmainr   r   r   r   Ú<module>   s<   





	     
