3
I4W^±  ã               @   sb   d dl jjZd dlmZ dd„ Zdd„ Zdd„ Zd	d
„ Zddd„Z	dd„ Z
ddd„Zddd„ZdS )é    N)Ú_get_device_indexc             C   s   dd l }t| |jjƒS )Nr   )Ú	torch.jitÚ
isinstanceÚjitÚScriptModule)ÚmoduleÚtorch© r	   ú</tmp/pip-build-e66j97b2/torch/torch/nn/parallel/replicate.pyÚ_is_script_module   s    r   c             C   s   dd l }t| |jjƒS )Nr   )r   r   Z_CZScriptMethod)r   r   r	   r	   r
   Ú_is_script_method
   s    r   c              C   s   dd l } | jjƒ S )Nr   )r   r   r   )r   r	   r	   r
   Ú_init_script_module   s    r   c              C   s   dd l } | jjS )Nr   )r   r   Z_enabled)r   r	   r	   r
   Ú_is_jit_enabled   s    r   c             C   s„   dd„ }t ƒ sdS |d kr tƒ }|j| ƒ t| ƒrV|j|| ƒƒ tdd„ || ƒD ƒƒS x(| jƒ D ]}||krnq`t||ƒs`dS q`W dS )Nc             S   s   | j ƒ }t|ƒ |S )N)ÚmodulesÚnext)r   Úgenr	   r	   r
   Údescendant_modules#   s    z0_replicatable_module.<locals>.descendant_modulesTc             s   s   | ]}t |ƒV  qd S )N)r   )Ú.0Z
descendantr	   r	   r
   ú	<genexpr>1   s   z'_replicatable_module.<locals>.<genexpr>F)r   ÚsetÚaddr   ÚupdateÚallÚchildrenÚ_replicatable_module)r   Úmemor   Úchildr	   r	   r
   r       s     

r   c             C   sP   xJt | ƒD ]>\}}t|ƒsq
|| }x"|jjƒ D ]}|jj|j|ƒ q0W q
W d S )N)Ú	enumerater   Ú_cZ_method_namesZclone_method)r   Úmodule_copiesÚmodule_indicesÚir   ÚreplicaÚmethod_namer	   r	   r
   Ú_copy_scriptmodule_methods?   s    r$   Fc                sd   ddl m} |rtjˆ|ƒS tˆƒdkr\|j|fˆžŽ ‰ ‡ ‡fdd„tdtˆ ƒtˆƒƒD ƒS g S d S )Né   )Ú	Broadcastr   c                s    g | ]}ˆ ||t ˆƒ … ‘qS r	   )Úlen)r   r!   )Útensor_copiesÚtensorsr	   r
   ú
<listcomp>P   s   z0_broadcast_coalesced_reshape.<locals>.<listcomp>)Z
_functionsr&   ÚcommZbroadcast_coalescedr'   ÚapplyÚrange)r)   ÚdevicesÚdetachr&   r	   )r(   r)   r
   Ú_broadcast_coalesced_reshapeH   s    r0   c       "         sR  t | ƒstdƒ‚ttdd„ |ƒƒ}t|ƒ}t| jƒ ƒ}dd„ t|ƒD ƒ}t|||ƒ}t| jƒ ƒ}g }g }	x.|D ]&}
|
j	rŠ| rŠ|j
|
ƒ qn|	j
|
ƒ qnW dd„ t|ƒD ƒ}dd„ t|	ƒD ƒ}t|||d�}t|	|d	d�}t| jƒ ƒ}d
d„ |D ƒ‰ i }dddddh}�x4t|ƒD �]&\}}|||< �xt|ƒD �]}t|ƒ�rätƒ }tdd„ |jjƒ D ƒƒ}t|jjƒ ƒ| | }x.|D ]&}t|j| ƒ�sx|j| |j|< �qxW x~|jjƒ D ]0\}}}||jjƒ k�rÌ�q®|jj|||ƒ �q®W n>|jt|ƒƒ}|jjƒ |_|jjƒ |_|jjƒ |_|jjƒ |_ˆ | j
|ƒ �q.W �qW �xØt|ƒD �]Ê\}}xˆ|jjƒ D ]z\}}|d k�rœxdt|ƒD ]}ˆ | | }d |j|< �qzW n:|| }x0t|ƒD ]$}ˆ | | }ˆ | | |j|< �q®W �q^W xˆ|jjƒ D ]z\}}|d k�r&xdt|ƒD ]}ˆ | | }d |j|< �qW n:|| }x0t|ƒD ]$}ˆ | | }|| | |j|< �q8W �qèW xª|jjƒ D ]œ\}}
|
d k�r°x†t|ƒD ]}ˆ | | }d |j|< �qŽW n\|
j	�rÎ| �rÎ|} ||
 }!n|} ||
 }!x0t|ƒD ]$}ˆ | | }| | |! |j|< �qäW �qrW �qHW x"t|ƒD ]}t|ˆ | |ƒ �q"W ‡ fdd„t|ƒD ƒS )NzKCannot replicate network where python modules are childrens of ScriptModulec             S   s
   t | dƒS )NT)r   )Úxr	   r	   r
   Ú<lambda>[   s    zreplicate.<locals>.<lambda>c             S   s   i | ]\}}||“qS r	   r	   )r   ÚidxÚparamr	   r	   r
   ú
<dictcomp>_   s    zreplicate.<locals>.<dictcomp>c             S   s   i | ]\}}||“qS r	   r	   )r   r3   Úbufr	   r	   r
   r5   k   s    c             S   s   i | ]\}}||“qS r	   r	   )r   r3   r6   r	   r	   r
   r5   l   s    )r/   Tc             S   s   g | ]}g ‘qS r	   r	   )r   Zdevicer	   r	   r
   r*   r   s    zreplicate.<locals>.<listcomp>Ú_parametersÚ_buffersÚ_modulesZforwardr   c             s   s   | ]}|d  V  qdS )r   Nr	   )r   Úentryr	   r	   r
   r   ~   s    zreplicate.<locals>.<genexpr>c                s   g | ]}ˆ | d  ‘qS )r   r	   )r   Új)r   r	   r
   r*   ¹   s    )r   ÚRuntimeErrorÚlistÚmapr'   Ú
parametersr   r0   ÚbuffersZrequires_gradÚappendr   r-   r   r   r   r   Z_get_attributesÚ__dict__Úkeysr   r8   Z_register_attributeÚ__new__ÚtypeÚcopyr7   r9   Úitemsr$   )"Únetworkr.   r/   Znum_replicasÚparamsZparam_indicesZparam_copiesr@   Z
buffers_rgZbuffers_not_rgr6   Zbuffer_indices_rgZbuffer_indices_not_rgZbuffer_copies_rgZbuffer_copies_not_rgr   r    Zscriptmodule_skip_attrr!   r   r;   r"   Zattribute_namesrC   ÚkeyÚnameZthe_typeÚvaluer   Z
module_idxr4   Z	param_idxZbuffer_copiesZ
buffer_idxr	   )r   r
   Ú	replicateV   sš    






$rM   )N)F)F)Ztorch.cuda.commZcudar+   Ztorch.cuda._utilsr   r   r   r   r   r   r$   r0   rM   r	   r	   r	   r
   Ú<module>   s   
	
