3
I4W^÷D ã               @   s  d dl mZ d dl mZ d dl mZ d dlmZ d dlmZmZm	Z	 d dl
Z
d dlZd dlmZ d dlmZmZmZmZmZmZmZ d d	lmZ d dlZd dlZejd
ƒ ejƒ d krÊejdƒ ejdƒ e
jdƒZeje
j ƒ dZ!dZ"dd„ Z#dd„ Z$dd„ Z%ddddddddddddddddddde"fdd„Z&dd„ Z'dd „ Z(d!dddddddddddddde"fd"d#„Z)d‚d$d%„Z*d&d'„ Z+d(d)„ Z,d*d+„ Z-d,d-„ Z.d.d/„ Z/e"fd0d1„Z0dƒd2d3„Z1d„d4d5„Z2d6d7„ Z3d8d9„ Z4d:d;„ Z5d…d<d=„Z6d>d?„ Z7d†d@dA„Z8d‡dBdC„Z9dˆdDdE„Z:d‰dFdG„Z;dHdI„ Z<dJdK„ Z=dLdM„ Z>dNdO„ Z?dPdQ„ Z@dRdS„ ZAG dTdU„ dUeBƒZCdVdW„ ZDdXdY„ ZEdŠdZd[„ZFd‹d\d]„ZGdŒd^d_„ZHd`da„ ZIdbdc„ ZJddde„ ZKdfdg„ ZLdhdi„ ZMdjdk„ ZNdldm„ ZOdndo„ ZPdpdq„ ZQdrds„ ZRdtdu„ ZSdvdw„ ZTd�dxdy„ZUdŽdzd{„ZVd|d}„ ZWd~d„ ZXd€d�„ ZYdS )�é    )Úabsolute_import)Údivision)Úprint_function)ÚOrderedDict)Ú	viewitemsÚviewkeysÚ
viewvaluesN)Ú	cpu_count)Úmodel_helperÚdyndepÚscopeÚ	workspaceÚcoreÚmemongerÚutils)Ú
caffe2_pb2z%@/caffe2/caffe2/contrib/gloo:gloo_opsz%@/caffe2/caffe2/contrib/nccl:nccl_opsz)@/caffe2/caffe2/contrib/gloo:gloo_ops_gpuZdata_parallel_modelé   i,  c              O   s   d|d< t | |Ž d S )NFÚ
cpu_device)ÚParallelize)ÚargsÚkwargs© r   úB/tmp/pip-build-e66j97b2/torch/caffe2/python/data_parallel_model.pyÚParallelize_GPU%   s    r   c              O   s   d|d< t | |Ž d S )NTr   )r   )r   r   r   r   r   ÚParallelize_CPU*   s    r   c              O   s   d|d< t | |Ž d S )NTÚideep)r   )r   r   r   r   r   ÚParallelize_iDeep.   s    r   ZdagTFé   é   c       /         s  t jƒ dks.t jƒ jtjks.tdjt jƒ ƒƒ‚|dkrb|p<|sRttdt	j
ƒ ƒƒ}nttdtƒ ƒƒ}|ph|sÆx0|D ](}|t	jƒ krptjdjt	jƒ |ƒƒ P qpW t	j| _d| _d| _d}|dksÄtdƒ‚nd|røtj| _d	| _d
}|| _|rö|	dk	rödsöt‚n2tj| _d| _d}|| _|�r*|	dk	�r*d�s*t‚tjdj|ƒƒ |	dk	�rHdnd}t|ƒ| | }t||d ƒ}|| jjƒ _|
| jjƒ _|| _|	| _d| _d| _g | _t | t!j"ƒ�s²t‚t#j#| j$ƒ}tjdƒ i }|	dk�rÚdn|	d }dt|ƒ|  }|dk	�p|dk	}|dk	�o|dk	 �s"tdƒ‚| �rŒ| j%�rŒtjdƒ tjdƒ tjdj| j&| ƒƒ tjdƒ tjdƒ tjdƒ tjdƒ tjdƒ x¾|D ]¶} t'j(| j| ƒ}!t'j)|!ƒ�’ t'j*dj| j| ƒƒ�r tjdj|| ƒƒ || ƒ || |ƒ}"|�r*t |"tƒ�stdƒ‚x$|"D ]}#t |#t'j+ƒ�s
tdƒ‚�q
W |"|| < W dQ R X W dQ R X �q’W t,| j$ƒ t-| || j$|ƒ| _.t-| || j/dƒg ƒ}$| j.j0|$ƒ tt1| j.ƒƒ| _2tt1|$ƒƒ| _3|�r´|| ƒ |�rÐtjd ƒ t4|| |ƒ |�rì|| t|ƒ| j| jƒ |�stjd!ƒ t5| ƒ dS |�r<|�std"ƒ‚t6| ƒ |�r4t7| ƒ nt8| ƒ t,| j$ƒ | j9‰ ‡ fd#d$„| j$D ƒ}%‡ fd%d$„|D ƒ}&t-| ||%|&ƒ}'| j.j0|'ƒ tt1|'ƒƒ| _|| _:t5| ƒ tjd&ƒ |�rÆt;|| |	|ƒ t| jƒdk�rt<| ƒ}(t|(ƒdk�sðt‚t=|(|| | j|	||ƒ n
tjd'ƒ tjd(ƒ |	dk�r*dn|	d }t>| j?dƒƒ})|�rNt@| ƒ |dk	�r¶xt|D ]R} t'j(| j| ƒ}!t'j)|!ƒ�. t'j*dj| j| ƒƒ� || ƒ W dQ R X W dQ R X �q^W ntjd)ƒ || ƒ}*|*| _AtB| ƒ\}+},t-| ||+g ƒ}-| j.j0|-ƒ t5| ƒ tC| ƒ | jƒ jDjEƒ }.d*|._&d|._Ftjd+ƒ tG|| | jH| jH|	|,dd,� |dk	�r¨xZ|D ]R} t'j(| j| ƒ}!t'j)|!ƒ�. t'j*dj| j| ƒƒ� || ƒ W dQ R X W dQ R X �qRW |�o°| �s¾td-ƒ‚|�rÐtI| ||ƒ |�râtJ| ||ƒ | jHg| _K| jg| _LtM| |ƒ |�rtN| |)ƒ dS ).a­  
    Function to create a model that can run on many GPUs or CPUs.
      model_helper_obj: an object of ModelHelper
      input_builder_fun:
                         Function that adds the input operators
                         Note: Remember to instantiate reader outside of this
                         function so all devices share same reader object.
                         Signature:  input_builder_fun(model)
      forward_pass_builder_fun:
                        Function to add the operators to the model.
                        Must return list of loss-blob references that
                        are used to build the gradient. Loss scale parameter
                        is passed, as you should scale the loss of your model
                        by 1.0 / the total number of devices.
                        Signature: forward_pass_builder_fun(model, loss_scale)
      param_update_builder_fun:
                        Function that adds operators that are run after
                        gradient update, such as updating the weights and
                        weight decaying. This is called for each GPU separately.
                        Signature: param_update_builder_fun(model)
      optimizer_builder_fun:
                        Alternative to param_update_builder_fun, allows one
                        to add an optimizer for the whole model. Called only
                        once, without name or devicescope.
      net_transformer_fun:
                        Optional function to transform the network after the
                        network is built. It will be called once (NOT once per
                        GPU.)
                        Signature:
                        net_transformer_fun(
                            model, num_devices, device_prefix, device_type)
      pre_grad_net_transformer_fun:
                        Optional function to transform the network similar to
                        net_transformer_fun, but happens before gradient ops
                        been add.
                        Signature: pre_grad_net_transformer_fun(model)
      post_sync_builder_fun:
                        Function applied after initial parameter sync has been
                        completed, such as keeping multi-precision parameters
                        in sync.
                        Signature: post_sync_builder_fun(model)
      devices:          List of GPU ids, such as [0, 1, 2, 3],
      rendezvous:       used for rendezvous in distributed computation, if None
                        then only one node is used. To create rendezvous,
                        use <TBD>.
      net_type:         Network type
      optimize_gradient_memory: whether to apply 'memonger' to share blobs
      shared_model      (only for CPU) use same parameters on each device
                        in gradient computation to reduce memory footprint.
      dynamic_memory_management: Whether to apply dynamic memory optimization
                        by freeing unused blobs. The underlying (de)allocation
                        uses cached allocator. For GPU training PLEASE MAKE SURE
                        caffe2_cuda_memory_pool is set.
      blobs_to_keep :   A list of blob names to keep and don't free during
                        dynamic memory optimization (for example loss blob).
      cpu_device        Use CPU instead of GPU.
      ideep             Use ideep.
      combine_spatial_bn:
                        When set to True, applies batch normalization across
                        all devices within the node. If False, batch
                        normalization will be done separately for each device.
                        This option is currently only supported on the CPU.
      barrier_net_timeout_sec:
                        The timeout in seconds of the barrier net, which is run
                        to synchronize shards before a training epoch starts.
                        Defaults to 300 seconds.
    NzMParallelize must be called without device-scope,         device scope was: {}r   z,** Only {} GPUs available, GPUs {} requestedÚgpuFZGPUz"Shared model only supported on CPUr   ÚIDEEPz4Shared model only supported on single-node currentlyÚcpuÚCPUz#Parallelizing model for devices: {}é   é   z)Create input and model training operatorsÚ
num_shardsg      ð?zGCan only specify one of param_update_builder_fun, optimizer_builder_funÚ z############## WARNING #############z.Model {}/{} is used for testing/validation butzhas init_params=True!z&This can conflict with model training.z4Please ensure model = ModelHelper(init_params=False)z$####################################z{}_{}zModel for {} : {}z5Model builder function must return list of loss blobsz1Model builder func must return list of loss blobszAdding gradient operatorsz6Parameter update function not defined --> only forwardz6combine_spatial_bn should only be used for train modelc                s   g | ]}|ˆ krˆ | ‘qS r   r   )Ú.0Úp)Úparam_to_gradr   r   ú
<listcomp>)  s    zParallelize.<locals>.<listcomp>c                s   g | ]}ˆ | ‘qS r   r   )r'   r(   )r)   r   r   r*   +  s    z$Add gradient all-reduces for SyncSGDz;NOTE: Param builder function did not create any parameters.z,Post-iteration operators for updating paramsz"Calling optimizer builder functionZfirst_iter_only_one_workerzAdd initial parameter sync)Úmax_concurrent_distributed_opszcIt is not advised to use gradient optimization ('memonger')
        with dynamic memory management.)Or   ÚCurrentDeviceScopeÚdevice_typer   r"   ÚAssertionErrorÚformatÚlistÚranger   ZNumCudaDevicesr	   ÚNumGpuDevicesÚlogÚwarningÚGpuDeviceTypeÚ_device_typeÚ_device_prefixÚ_shared_modelr    ÚinfoÚlenÚminÚnetÚProtoÚnum_workersÚtypeÚ_devicesÚ_rendezvousÚ_sync_barrier_netÚ_broadcast_contextÚ_grad_namesÚ
isinstancer
   ÚModelHelperÚcopyÚparamsZinit_paramsÚnamer   ÚDeviceOptionÚDeviceScopeÚ	NameScopeÚBlobReferenceÚ_ValidateParamsÚ_GroupByDeviceÚ_device_grouped_blobsÚGetComputedParamsÚupdater   Ú_param_namesÚ_computed_param_namesÚ_AddGradientOperatorsÚ_InferBlobDeviceÚ_InterleaveOpsÚ!_CPUInterDeviceBatchNormalizationÚ!_GPUInterDeviceBatchNormalizationr)   Ú_losses_by_gpuÚ_BroadcastComputedParamsÚ_GetReverseOrderedGradsÚ_AllReduceBlobsÚsetÚ	GetParamsÚ_PruneParametersForSharingÚ
_optimizerÚ_ComputeBlobsToSyncÚ_AnalyzeOperatorsÚargÚaddÚiÚ_SyncAllParamsÚparam_init_netÚ_OptimizeGradientMemorySimpleÚ_AddDynamicMemoryOptimizationÚ_data_parallel_model_init_netsÚ_data_parallel_model_netsÚ_AddBarrierToModelNetsÚ"_RemapParameterBlobsForSharedModel)/Úmodel_helper_objÚinput_builder_funÚforward_pass_builder_funÚparam_update_builder_funZoptimizer_builder_funZpost_sync_builder_funZpre_grad_net_transformer_funZnet_transformer_funÚdevicesÚ
rendezvousÚnet_typeZbroadcast_computed_paramsÚoptimize_gradient_memoryZdynamic_memory_managementÚblobs_to_keepÚuse_ncclr+   r   r   Únum_threads_per_deviceZshared_modelZcombine_spatial_bnÚbarrier_net_timeout_secr   Zdevice_nameZextra_workersr>   Únon_datapar_paramsÚlosses_by_gpur%   Ú
loss_scaleZhas_parameter_updatesÚdeviceÚ
device_optZlossesÚlossZcomputed_params_groupedZgrads_orderedZnon_datapar_gradsZgradients_groupedZreverse_ordered_gradsÚ
all_paramsZ	optimizerZ
sync_blobsÚ
sync_namesZsync_blobs_groupedrd   r   )r)   r   r   2   s~   \
















"











$



"


r   c              O   s   d|d< t | |Ž d S )NFr   )ÚParallelize_BMUF)r   r   r   r   r   ÚParallelize_GPU_BMUF¢  s    r„   c              O   s   d|d< t | |Ž d S )NTr   )rƒ   )r   r   r   r   r   ÚParallelize_CPU_BMUF§  s    r…   g      ð?c       %         sÊ  t jƒ dks.t jƒ jtjks.tdjt jƒ ƒƒ‚tˆtj	ƒs>t‚|dkrXt
tdtjƒ ƒƒ}|	dkrh|d }	|s®x0|D ](}|tjƒ krrtjdjtjƒ |ƒƒ P qrW tjˆ_dˆ_ntjˆ_dˆ_|ˆ_|ˆ_dˆ_dˆ_dˆ_tjˆj|	ƒ}|rô|d nd	}t|ƒ| }|t|ƒ }|�r|d
7 }d| ‰|dk�r<dd|  }t||d	 ƒ}|ˆjjƒ _|ˆjjƒ _tj dƒˆ_!|ˆj!jƒ _|ˆj!jƒ _tj dƒˆ_"|ˆj"jƒ _|ˆj"jƒ _dd„ }dd„ }dd„ }t#j#ˆj$ƒ}i ˆ_%‡ ‡‡‡fdd„}t&||ˆjˆjdd� t'ˆj$ƒ t(ˆ|ˆj$|ƒˆ_)t
t*ˆj)ƒƒˆ_+t,|ˆˆj%ƒ t'ˆj$ƒ t-ˆƒ ‡‡fdd„}t&||ˆjˆjdd� t
t*ˆj)ƒƒ}|dk	�r|ˆ_.tj dƒˆ_/|ˆj/jƒ _|ˆj/jƒ _t0|ˆˆj1ˆj/|||ƒ xJt*ˆj)ƒD ]<} ˆj)|  |	 }!tj2|ƒ� ˆj/j3|!||!ƒƒ W dQ R X �qÒW x|t*ˆj)ƒD ]n} ˆj)|  |	 }!tj2|ƒ�J ˆj!j4|!||!ƒdd� ˆj!j3|!||!ƒƒ |�r€ˆj!j4|!||!ƒdd� W dQ R X �qW t5||ˆˆj"||
|ƒ �xJ|D �]@} ˆj)|  |	 }!tj2|ƒ�� ˆj"j6|!|!d| d� ˆj"j7|!||!ƒg|!ƒ ˆj"j6|!|!|d� ˆj"j6||!ƒ||!ƒ|d� ˆj"j8||!ƒ|!g||!ƒƒ ˆj"j8||!ƒ||!ƒg||!ƒƒ |�rÒˆj"j7||!ƒ||!ƒg||!ƒƒ ˆj"j6||!ƒ||!ƒ|d� ˆj"j7||!ƒ||!ƒg||!ƒƒ ˆj"j3||!ƒ||!ƒƒ ˆj"j3||!ƒ|!ƒ W dQ R X �q®W t0|ˆˆj1ˆj"|||ƒ |dk	�r&t9ˆ|ˆj"d� |�rˆdd „ ˆjjƒ j:D ƒ}"xD|"D ]<}#|#j;d	 }$tj2|#j<ƒ� ˆj"j4|$g|$dd� W dQ R X �qHW |�rœt=ˆˆj%|ƒ ˆj1ˆj!gˆ_>ˆjˆj"d	fgˆ_?t@ˆ|ƒ dS )!aê  
    Function to create model that run on many GPUs and creates a net for
    parameter_updates that can be run independently for number of iterations
    then followed by another net that runs once to compute the final parameter
    updates according to block wise model update filtering rule described
    in : Scalable Training of Deep Learning Machines by Incremental Block
    Training with Intra-block Parallel Optimization and Blockwise Model-Update
    Filtering (ICASSP 2016).
    NzMParallelize must be called without device-scope,         device scope was: {}r   z,** Only {} GPUs available, GPUs {} requestedr   r!   Fr%   r$   r#   g      ð?Zglobal_model_initZglobal_modelc             S   s
   dj | ƒS )Nz{}_v)r/   )Úparamr   r   r   Ú_v  s    zParallelize_BMUF.<locals>._vc             S   s
   dj | ƒS )Nz{}_g)r/   )r†   r   r   r   Ú_g  s    zParallelize_BMUF.<locals>._gc             S   s
   dj | ƒS )Nz{}_prev)r/   )r†   r   r   r   Ú_v_prev  s    z!Parallelize_BMUF.<locals>._v_prevc                s    ˆˆƒ ˆ ˆˆƒ}|ˆj | < d S )N)rZ   )Úgpu_idr€   )rq   rp   r}   ro   r   r   Ú_InitializeModels  s    
z+Parallelize_BMUF.<locals>._InitializeModelsT)r-   Údevice_prefixÚscopedc                s   ˆˆ ƒ d S )Nr   )rŠ   )ro   rr   r   r   Ú_InitializeParamUpdate6  s    z0Parallelize_BMUF.<locals>._InitializeParamUpdatezwarmup-broadcastg        )Úvalue)Zscale)r<   c             S   s   g | ]}|j d kr|‘qS )ZMomentumSGDUpdate)r?   )r'   Úopr   r   r   r*   ¹  s    z$Parallelize_BMUF.<locals>.<listcomp>)Ar   r,   r-   r   r"   r.   r/   rE   r
   rF   r0   r1   r   r2   r3   r4   r5   r6   r7   r@   rA   rB   rC   r8   r   rJ   r:   r;   r<   r=   r>   r?   ÚNetZ_global_model_init_netZ_global_model_param_updates_netrG   rH   rZ   Ú_ForEachDevicerN   rO   rP   r   rS   rU   rV   Ú_warmup_iterationsÚ_warmup_broadcastrg   rh   rK   ÚCopyÚConstantFillr]   ZScaleÚSubÚAddÚAddBlobSyncr�   ÚinputÚdevice_optionri   rk   rl   rm   )%ro   rp   rq   rr   Zblock_learning_rateZblock_momentumrs   rt   ru   Úmaster_devicerx   Znesterovrv   Zreset_momentum_sgdZwarmup_iterationsr+   Zadd_blobs_to_syncry   r   rz   r   Zmaster_dev_optr%   Únum_devicesr>   r‡   rˆ   r‰   r{   r‹   rŽ   Zmodel_parameter_namesÚ
param_namer†   Zmomentum_opsr�   Zmomentum_blobr   )rq   rp   r}   ro   rr   r   rƒ   ¬  s@   





" 




rƒ   c             C   s@   x:| j D ]0}t|tƒr*tj|d |d� qtj||d� qW d S )Nr   )Ú	overwrite)rl   rE   Útupler   Ú	CreateNet)ÚmodelrŸ   Z	net_itersr   r   r   r¡   Ò  s    
r¡   c             C   s&   x| j D ]}tj|ƒ qW t| ƒ d S )N)rk   r   Ú
RunNetOncer¡   )r¢   Úinit_netr   r   r   Ú
RunInitNetÚ  s    r¥   c             C   s    t j| j| jƒ t j| jƒ d S )N)r   ÚRunNetr<   r“   r£   r”   )r¢   r   r   r   Ú	RunWarmupà  s    r§   c             C   sF   x@| j D ]6}t|tƒr2tj|d jƒ j|d ƒ qtj||ƒ qW d S )Nr   r$   )rl   rE   r    r   r¦   r=   rI   )r¢   Znum_iterationsZnet_iterr   r   r   r¦   å  s    
r¦   c             C   sZ   | j d k	rV| j d dkrVtjdƒ| _t| | jd|ƒ| _| jjd| jƒ | jjd| jƒ d S )NÚengineÚGLOOÚbarrier_init_netZpre_trainingr   )	rA   r   r‘   Z_barrier_init_netÚ_CreateBarrierNetÚ_barrier_netrk   Úinsertrl   )r¢   rz   r   r   r   rm   í  s    	
rm   c             C   s`   t jdƒ | jd dks tdƒ‚t||d | j|d�}tj|d ƒ}|j|gg | jd d� |S )	NzCreating barrier netr¨   r©   zEngine does not support barrierZ_barrier_cw)rt   Útimeout_secr¬   )ÚinputsÚoutputsr¨   )r3   r9   rA   r.   Ú_CreateOrCloneCommonWorldr   r‘   ÚBarrier)r¢   r¤   Zname_prefixr®   Ú
comm_worldZbarrier_netr   r   r   r«     s    
r«   c             C   s¢   t jdtdd� | jd ks(| jd dkr,d S | jd krltjdƒ}t| |d|ƒ| _tj	|ƒ tj
| jƒ || _| j|ksˆtdj| j|ƒƒ‚tjd	ƒ tj| jƒ d S )
Na  The Synchronize API has been deprecated.  We now have a barrier net which runs before training to ensure all hosts wait before training starts.  The default timeout for the barrier is 300s and it can be overridden using the barrier_net_timeout_sec parameter when calling Parallelize.é   )ÚcategoryÚ
stacklevelr%   r$   Zsync_barrier_init_netÚsyncz Must use fixed timeout, {} != {}zSynchronize run barrier net.)ÚwarningsÚwarnÚDeprecationWarningrA   rB   r   r‘   r«   r   r£   r¡   Z_sync_barrier_net_timeoutr.   r/   r3   r9   r¦   )r¢   r®   rª   r   r   r   ÚSynchronize  s     




r»   c             C   sT  t j| ƒ}|dkrtjƒ }tj|jƒr,d}n|jtjkr>d}nd}dj	||j
ƒ}xž|jƒ jD ]�}d|jkrnd‚x$t|jƒD ]\}}|| |j|< qzW x$t|jƒD ]\}}|| |j|< q W x$t|jƒD ]\}}	||	 |j|< qÆW |jj|ƒ q\W x.t|jƒ jƒD ]\}}
||
 |jƒ j|< �q W x.t|jƒ jƒD ]\}}|| |jƒ j|< �q0W |S )a¶  
    Converts all blobs in the net to have namescope gpu_X, and correct
    device scope. You can use this to enable AppendNet with a
    forward_pass_builder_fun:

       def builder_fun(model):
          ...
          model.net.AppendNet(
             data_parallel_model.ConvertNetForDevice(othermodel.net))
          model.param_init_net.AppendNet(
             data_parallel_model.ConvertNetForDevice(othermodel.param_init_net))
    Nr   r   r!   z{}_{}/ÚRecurrentNetworkz-RecurrentNetwork conversion not yet supported)rG   Údeepcopyr   r,   r   ÚIsGPUDeviceTyper-   r   r    r/   Ú	device_idr=   r�   r?   Ú	enumeraterš   ÚoutputÚcontrol_inputr›   ZCopyFromZexternal_inputZexternal_output)r<   r~   ZmnetrŒ   Ú	namescoper�   rf   ZinputbZoutputbÚblobZeinpZeoutpr   r   r   ÚConvertNetForDevice.  s0    

rÅ   c       	      O   sx   xr| D ]j}t j||ƒ}t j|ƒ�J |rVt jdj||ƒƒ� ||f|ž|Ž W d Q R X n||f|ž|Ž W d Q R X qW d S )Nz{}_{})r   rJ   rK   rL   r/   )	rs   Úfr-   rŒ   r�   r   r   r~   r   r   r   r   r’   X  s    
r’   c       	         sz   ‡ fdd„}i }xZ| D ]R}t jˆ j|ƒ}t j|ƒ�0 x(|| D ]}||ƒ}t|ƒ|t|ƒ< q>W W d Q R X qW ˆ j|ƒ d S )Nc                s   ˆ j | t| ƒd dd�S )NZ_gradg      ð?)r�   )r–   Ústr)Zlossp)r¢   r   r   Úcreate_grade  s    z*_AddGradientOperators.<locals>.create_grad)r   rJ   r6   rK   rÇ   ZAddGradientOperators)	rs   r¢   r|   rÈ   Z	loss_gradrŠ   r~   ÚlZlgr   )r¢   r   rU   d  s    
"rU   c       	   
      sz   | j d }dj| j|ƒ‰ ‡ fdd„|D ƒ}‡ fdd„|D ƒ}tj| jjƒ |||dd„ t|| || ƒD ƒd�\}}||fS )	zX
    Returns (net, params) that can be exported to be used as a prediction
    net.
    r   z{}_{}/c                s   g | ]}ˆ t |ƒ ‘qS r   )rÇ   )r'   Úb)Úprefixr   r   r*   {  s    z'ExtractPredictorNet.<locals>.<listcomp>c                s   g | ]}ˆ t |ƒ ‘qS r   )rÇ   )r'   rÊ   )rË   r   r   r*   |  s    c             S   s   i | ]\}}||“qS r   r   )r'   ÚarÊ   r   r   r   ú
<dictcomp>‚  s   z'ExtractPredictorNet.<locals>.<dictcomp>)Z	net_protoÚinput_blobsZoutput_blobsr~   Úrenames)r@   r/   r7   r
   ÚExtractPredictorNetr<   r=   Úzip)	r¢   r¯   r°   r~   rœ   Zprefix_inputsZprefix_outputsZpredictor_netZexport_blobsr   )rË   r   rÐ   t  s    
rÐ   c                s€   t ˆ ƒ\}}‡ fdd„|D ƒ}tƒ }xPˆ jjƒ jD ]@}|jdksJ|jdkr2|jd jdjˆ j	ƒƒs2|j
|jd ƒ q2W |j|ƒS )z†
    Returns a set of blobs that are needed for a complete check point.
    They are blobs for the first gpu and iteration blobs.
    c                s.   h | ]&}t |ƒjd jˆ jˆ jd ƒƒr|’qS )z{}_{}/r   )rÇ   Ú
startswithr/   r7   r@   )r'   rÊ   )r¢   r   r   ú	<setcomp>‘  s   z&GetCheckpointParams.<locals>.<setcomp>ÚIterZ
AtomicIterr   z{}_)rb   r^   r<   r=   r�   r?   rÁ   rÒ   r/   r7   re   Úunion)r¢   Z	all_blobsÚ_Zfirst_gpu_blobsZiteration_blobsr�   r   )r¢   r   ÚGetCheckpointParams‹  s    
r×   c          	      s  t ˆ dƒsö|dkr tˆ ƒ\}}ndd„ |D ƒ}tjdƒ ˆ jƒ }x4|D ],‰ˆˆ jkrF‡ ‡fdd„|D ƒ}|ˆ jˆ< qFW tjdƒˆ _|s�ˆ jj	ƒ  d}ˆ j
dk	rÂˆ j
d	 d
krÂtjdƒ}|sÂ|j	ƒ  t|ˆ |ˆ jˆ j
|d
d� |rêtj|ƒ tjˆ jƒ tjdƒ tjˆ jjƒ jƒ dS )zr
    This function should be called after loading parameters from a
    checkpoint / initial parameters file.
    Ú_checkpoint_netNc             S   s   g | ]}t |ƒ‘qS r   )ÚstripBlobName)r'   r(   r   r   r   r*   ­  s    z+FinalizeAfterCheckpoint.<locals>.<listcomp>z'Creating checkpoint synchronization netc                s(   i | ] }t jd jˆ j|tjˆƒƒ|“qS )z	{}_{}{}{})r   rM   r/   r7   r   Ú_NAMESCOPE_SEPARATOR)r'   Úd)r¢   rI   r   r   rÍ   µ  s   z+FinalizeAfterCheckpoint.<locals>.<dictcomp>Zcheckpoint_sync_netr%   r$   Úcheckpoint_init_net)r+   zRun checkpoint net)Úhasattrrb   r3   r9   Z
GetDevicesrP   r   r‘   rØ   ZRunAllOnGPUrA   rg   r   r£   r¡   r¦   r=   rI   )r¢   ÚblobsZcpu_moderÖ   Zuniq_blob_namesrs   ÚgroupedrÜ   r   )r¢   rI   r   ÚFinalizeAfterCheckpoint£  s@    







rà   c                sž   ˆ j dk	rbˆ jtjks"ˆ jtjkr0ˆ j jdƒgS tjˆ jƒrP‡ fdd„ˆ jD ƒS t	dj
ˆ jƒƒ‚n8g }x.ˆ jjƒ jD ]}|jdkrt|j|jdƒƒ qtW |S dS )zL
    Returns a list of learning rates blob names used in the optimizer.
    NÚlrc                s   g | ]}ˆ j jd |dƒ‘qS )rá   r&   )ra   Zget_gpu_blob_name)r'   r   )r¢   r   r   r*   ä  s   z,GetLearningRateBlobNames.<locals>.<listcomp>zUnsupported device type : {}ZLearningRater   )ra   r6   r   r"   r    Zget_cpu_blob_namer   r¾   r@   Ú	Exceptionr/   r<   r=   r�   r?   ÚappendrÁ   )r¢   Zlr_blob_namesr�   r   )r¢   r   ÚGetLearningRateBlobNamesÜ  s    



rä   c       	      C   sö   | d }|rft ||ƒrftj|j|ƒ}tj|ƒ�2 |jtt|j| ƒƒtt|j| ƒƒdd� d S Q R X xŠ| dd … D ]z}t ||ƒr’tjt	j
|ƒ}n&t||ƒrªtjtjdƒntjtjdƒ}tj|ƒ�& |j|j| | |j| | ƒ W d Q R X qtW d S )Nr   )Úrootr$   )Ú
_IsGPUBlobr   rJ   r6   rK   ZNCCLBroadcastr0   r   rP   r   r5   Ú_IsIDEEPBlobr   r    r"   r•   )	rs   r¢   r<   r†   rx   Z
master_devÚmaster_device_optZdev_idxr   r   r   r   Ú
_Broadcastò  s&    


ré   c                s²  t tˆjˆ ƒƒ‰ ˆjtjkr6|r6ˆjˆ ˆ |d� d S ˆjtjkrLtj	ƒ ‰nd ‰‡ ‡‡‡‡fdd„}t
| ƒdkrôx&tdƒD ]}||d |d d ƒ qzW x&tdƒD ]}||d |d d ƒ q¢W x&tdƒD ]}||d |d d ƒ qÊW |d	dƒ n¬t
| ƒdk�rbx(tdƒD ]}||d |d d ƒ �qW x(tdƒD ]}||d |d d ƒ �q6W |d	dƒ n>t
| ƒdk�r�|d	dƒ |dd
ƒ |d	dƒ n|tt
| ƒƒŽ  t| ˆˆˆƒ d S )N)rÂ   c                 sÈ   ‡fdd„| D ƒ}‡ fdd„| D ƒ}t jˆj|d ƒ}t j|ƒ�~ x`t|ƒD ]T\}}|dkr^qLˆdk	rLˆjrLˆ|d |f  rLˆj|| dj|d ˆ|ƒƒ||< qLW ˆj||d gdd� W dQ R X dS )	a  Create a Sum op for 2 or more blobs on different devices.
        Saves the result on the first device.

        Arguments:
        dev_indices -- a list of device indices, which can be translated into
                       CUDA identifiers with model._devices
        c                s   g | ]}ˆ j | ‘qS r   )r@   )r'   Úidx)r¢   r   r   r*   '  s    z,_AllReduce.<locals>.sumN.<locals>.<listcomp>c                s   g | ]}ˆ | ‘qS r   r   )r'   rê   )Úblobs_groupr   r   r*   (  s    r   Nzgpu_{}/{}_gpu{}_copyÚdpm)rI   )	r   rJ   r6   rK   rÀ   Úsizer•   r/   ÚSum)Zdev_indicesrs   rÞ   r   rf   Zpeer)rë   r¢   r<   Úp2p_access_patternr†   r   r   ÚsumN  s    z_AllReduce.<locals>.sumNr   r#   r´   r$   r   r   é   )r0   r   rP   r6   r   ZCUDAÚNCCLAllreducer   r5   ZGetGpuPeerAccessPatternr:   r1   ré   )rs   r¢   r<   r†   rx   rÂ   rð   Újr   )rë   r¢   r<   rï   r†   r   Ú
_AllReduce  s:    


rô   c             C   s<   |d ks|d dkr$t | |||ƒ nt| ||||||ƒ d S )Nr%   r$   )Ú_SyncAllParamsSingleHostÚ_SyncAllParamsDistributed)rs   r¢   r¤   r<   rt   Úunique_param_namesr+   r   r   r   rg   Q  s    	rg   c                sˆ   t |ƒdkrdS |dkrˆjn|}xD|D ]<‰ ˆ jˆjƒ sHtdjˆ ƒƒ‚‡ ‡fdd„ˆjD ƒˆjˆ < q(W tˆjˆˆj	|ˆj
t|ƒƒ dS )z.
    Sync a blob across devices and hosts
    r   Nz Provide unprefixed blob name: {}c                s$   i | ]}t jd jˆj|ˆ ƒƒ|“qS )z{}_{}/{})r   rM   r/   r7   )r'   rÛ   )rÊ   r¢   r   r   rÍ   r  s   zAddBlobSync.<locals>.<dictcomp>)r:   r<   rÒ   r7   r.   r/   r@   rP   rg   rh   rA   r^   )r¢   rÞ   r<   r   )rÊ   r¢   r   r™   h  s    
r™   c             C   sZ   | j dkrdS djdd„ |D ƒƒ}t| jd| | j d�}| jj|g| || j d d� dS )	z=
    Sync blobs across machines (but not across devices)
    NrÖ   c             S   s   g | ]}t |ƒ‘qS r   )rÇ   )r'   rÊ   r   r   r   r*   †  s    z*AddDistributedBlobSync.<locals>.<listcomp>Zblob_sync_cw_)rt   r¨   )r¯   r°   r¨   )rA   Újoinr±   rh   r<   Ú	Allreduce)r¢   rÞ   Z
synth_namer³   r   r   r   ÚAddDistributedBlobSync€  s    

rú   c                sp  ˆd dkst ‚tj|j| d ƒ}tjtjƒ}tjtjƒ}	|jd krTtd||ˆƒ|_|j‰ �xt	|ƒD �] ‰|j
ˆ | d  }
tt|j
ˆ ƒƒ}‡ ‡‡‡fdd„}t|ˆƒr°|nt|ˆƒr¾|	n|}ˆd dkrîtj|ƒ� ||ƒ W d Q R X qftj|ƒ� ˆj|
t|
ƒd	 ƒ}W d Q R X tj|ƒ� ||gƒ W d Q R X tj|ƒ� ˆj||
ƒ W d Q R X t| |ˆˆƒ qfW d S )
Nr%   r$   r   Ú	broadcastc                s0   ˆ j | ƒ\}}ˆj|g|  | ˆˆd |d� d S )Nr¨   )r¯   r°   rI   r¨   rÂ   )Úget_control_and_contextZ	Broadcast)rH   r³   rÂ   )Úcontextr<   rž   rt   r   r   rû   °  s    z,_SyncAllParamsDistributed.<locals>.broadcastr¨   r©   r!   )r.   r   rJ   r6   r   r"   r    rC   ÚCollectivesConcurrencyControlÚsortedrP   r0   r   ræ   rç   rK   ÚCopyGPUToCPUrÇ   ÚCopyCPUToGPUré   )rs   r¢   r¤   r<   rt   r÷   r+   Zgpu_device_optZcpu_device_optZideep_device_optZmaster_paramZparams_grouprû   r   Z	param_cpur   )rý   r<   rž   rt   r   rö   ”  s<    	

rö   c             C   s    x|D ]}t | |||ƒ qW d S )N)ré   )rs   r¢   r<   r÷   r†   r   r   r   rõ   Ñ  s    
rõ   c             C   s<   |d ks|d dkr&t | ||||ƒ nt| |||||ƒ d S )Nr%   r$   )Ú_AllReduceBlobsSingleHostÚ_AllReduceBlobsDistributed)Ú
blob_namesrs   r¢   r<   rt   rx   r+   r   r   r   r]   Ö  s    r]   c                s|   ˆ j s
t‚djˆ jˆ jd ƒ}ˆ j|ƒˆ _tˆ jƒ‰‡ ‡fdd„ˆ jD ƒˆ _‡fdd„ˆ j	D ƒˆ _	‡fdd„ˆ j
D ƒˆ _
d S )Nz{}_{}/r   c                s    i | ]}|ˆkrˆ j | |“qS r   )r)   )r'   r(   )r¢   Úparamsetr   r   rÍ   ô  s   z._PruneParametersForSharing.<locals>.<dictcomp>c                s   g | ]}|ˆ kr|‘qS r   r   )r'   Úw)r  r   r   r*   ø  s    z._PruneParametersForSharing.<locals>.<listcomp>c                s   g | ]}|ˆ kr|‘qS r   r   )r'   r  )r  r   r   r*   ù  s    )r8   r.   r/   r7   r@   r_   rH   r^   r)   ÚweightsZbiases)r¢   Úmaster_prefixr   )r¢   r  r   r`   ë  s    

r`   c                sb   | j s
t‚dj| j| jd ƒ‰tjdjˆƒƒ t| jƒ ƒ‰‡ ‡‡fdd„}|| j	ƒ || j
ƒ d S )Nz{}_{}/r   z%Remapping param blobs to master -> {}c                sÆ   g }xœ| j ƒ jD ]Ž}d}x:|jD ]0}|ˆ kr |ˆkr d}tjdj|t|ƒƒƒ P q W |rZqx8t|jƒD ]*\}}|ˆ krf|ˆkrfˆt	|ƒ |j|< qfW |j
|ƒ qW | j ƒ jd d …= | j ƒ jj|ƒ d S )NFTzDelete b/c {}:  {})r=   r�   rÁ   r3   Údebugr/   rÇ   rÀ   rš   rÙ   rã   Úextend)r<   Úopsr�   Z	delete_opZoutpró   Úinp)r�   Úmaster_paramsr  r   r   Ú
modify_ops  s     z6_RemapParameterBlobsForSharedModel.<locals>.modify_ops)r8   r.   r/   r7   r@   r3   r9   r^   r_   rh   r<   )r¢   r�   r  r   )r�   r  r  r   rn   ü  s    

rn   c               @   s    e Zd ZdZdd„ Zdd„ ZdS )rþ   z²
    Creates common worlds (up to max_concurrent_context) and manage the
    sequential execution of collectives that shares the same context with
    cyclic control inputs.
    c             C   s.   || _ || _|| _d| _g | _g | _|| _d S )Nr   )rI   rh   Úmax_concurrent_contextÚcounterÚcommon_worldsÚcontrol_inputsrt   )ÚselfrI   r  rh   rt   r   r   r   Ú__init__"  s    z&CollectivesConcurrencyControl.__init__c             C   s�   d d  }}| j | j }t| jƒ| jk r\t| jdj| j|ƒ| jd�}| jj	|ƒ | j
j	|ƒ n| j| }| j
| }|| j
|< |  j d7  _ ||fS )Nz{}_{}_cw)rt   r$   )r  r  r:   r  r±   rh   r/   rI   rt   rã   r  )r  Zcontrol_output_blobZcommon_worldrÂ   Zcurrent_slotr   r   r   rü   1  s    




z5CollectivesConcurrencyControl.get_control_and_contextN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r  rü   r   r   r   r   rþ     s   rþ   c                s\  |j jƒ j}|dkstdƒ‚|d ‰ tj|j|d ƒ}|‰td||j|ƒ‰d }�x| D ]þ‰|j	ˆ |d  }	t
t|j	ˆ ƒƒ}
|	|
ksŠt‚t|	ƒd }‡ ‡‡‡‡fdd„}|d d	krÐ||
|jd
d ƒdkd� qVtj|ƒ�: |j|	|dd� ˆj|
|
|d� |
d }ˆj|	|ƒ W d Q R X ||gƒ tj|ƒ� ˆj||	ƒ W d Q R X t||ˆˆƒ qVW d S )Nr$   z!Please specify more than 1 workerr¨   r   Ú	allreduceZ_redc                sN   t jˆƒ�: ˆj| d ƒ\}}ˆjf |g|  | ˆˆ |dœ|—Ž W d Q R X d S )Nr   )r¯   r°   rI   r¨   rÂ   )r   rK   rü   rù   )rÞ   r   r³   rÂ   )Úall_reduce_engineÚ	blob_namerý   r<   Úreducing_device_optr   r   r  g  s    z-_AllReduceBlobsDistributed.<locals>.allreducer©   Ú	transportZibverbs)Z
gpu_directg        )r�   )rÂ   )r<   r=   r>   r.   r   rJ   r6   rþ   rh   rP   r0   r   rÇ   ÚgetrK   r–   rò   r•   ré   )r  rs   r¢   r<   rt   r+   r>   rè   Znccl_control_blobZmaster_blobrë   Zreduced_blobr  r   )r  r  rý   r<   r  r   r  D  sB    
r  c             C   sô  t |ƒdkrdS tj|j|d ƒ}d}tƒ }�xÀ| D �]¶}tt|j| ƒƒ}	t |	ƒdkrZq4t |	ƒt |ƒksztdj	||ƒƒ‚t
||ƒ�rtj|ƒ��€ t|	d tjƒsÂt||||||ƒ |	d }�nLdj	|j|d ƒ}
d}x|	D ]}|j|krÞd}qÞW |�sŠ|jdd	„ |	D ƒd
j	|
|ƒdj	|
|ƒgddd�\}}xZt|j| ƒD ]H\}}tj|j|ƒ}tj|ƒ�  |j||jƒ |j|jƒ W dQ R X �q>W |jdd	„ |	D ƒdj	|
|ƒdj	|
|ƒgddd�\}}xNt|j| ƒD ]<\}}tj|j|ƒ}tj|ƒ� |j||jƒ W dQ R X �qÎW W dQ R X q4t||ƒ�rŠt|	d tjƒ �sBtdƒ‚tjtjtjƒƒ�. |j|	|	d gƒ |j�s~t||||ƒ W dQ R X q4t|	d tjƒ �s¦tdƒ‚tjtjtjƒƒ�. |j|	|	d gƒ |j�sât||||ƒ W dQ R X q4W dS )z<Performs NCCL AllReduce to distribute blobs to all the GPUs.r$   Nr   z+Each GPU from {}, should have a copy of {}.z{}_{}FTc             S   s   g | ]
}|j ‘qS r   )Úindices)r'   Úgr   r   r   r*   Ã  s    z-_AllReduceBlobsSingleHost.<locals>.<listcomp>z{}/{}_index_concatz{}/{}_index_splitinfoznote:data_parallel_model)ZaxisrI   c             S   s   g | ]
}|j ‘qS r   )Úvalues)r'   r   r   r   r   r*   Ð  s    z{}/{}_val_concatz{}/{}_val_splitinfoz+Synchronizing gradient slices not supported)r:   r   rJ   r6   r^   r0   r   rP   r.   r/   ræ   rK   rE   ÚGradientSlicerô   r7   r  ÚConcatr   r•   re   r!  rç   r   r    rî   r8   ré   r"   )r  rs   r¢   r<   rx   rè   Zlast_outZconcatenated_idxr  rë   Z	master_nsZskip_idx_concatr   Zgrad_idx_concatrÖ   r   r   Zgrad_val_concatr   r   r   r  ”  sr    




*r  c             C   s(   |d krt | ||ƒ nt| |||ƒ d S )N)Ú"_BroadcastComputedParamsSingleHostÚ#_BroadcastComputedParamsDistributed)rs   r¢   rt   rx   r   r   r   r[   ì  s    r[   c             C   s   t | ||ƒ tjdƒ d S )Nz?Distributed broadcast of computed params is not implemented yet)r$  r3   r¹   )rs   r¢   rt   rx   r   r   r   r%  ó  s    r%  c             C   s6   t | ƒdkrdS x |jD ]}t| ||j||ƒ qW dS )z2
    Average computed params over all devices
    r$   N)r:   rT   ré   r<   )rs   r¢   rx   rž   r   r   r   r$  ý  s    r$  c             C   s   t t| jƒƒS )zq
    Returns the gradients in reverse order (namespace stripped),
    for the optimal synchronization order.
    )r0   ÚreversedrD   )r¢   r   r   r   r\   
  s    r\   c             C   sD   t | tjƒr$t| jƒd t| jƒ S t| ƒ}||jtj	ƒd d … S )Nú:r$   )
rE   r   r"  rÙ   r  r!  rÇ   Úindexr   rÚ   )r†   rI   r   r   r   rÙ     s    rÙ   c          	   C   sì   xæ| j ƒ jD ]Ø}d|jksd|jksd|jkr0qd|jkrF|jdkrFqd|jkr\d|jkr\q|j}|j}tj|j	ƒsvqdj
| j|ƒ}x^t|jƒt|jƒ D ]F}|jd	j
| jƒƒrš|j|ƒ rštd
j
||jdj
| j|ƒt|ƒƒƒ‚qšW qW dS )zR
    Look at all the operators and check that they do not cross device scopes
    ZNCCLr•   r#  rî   rì   rù   r©   z{}_{}/z{}_z2Blob {} of op {}, should have namescope {}. Op: {}N)r=   r�   r?   rI   r¨   r›   r¿   r   r¾   r-   r/   r7   r0   rš   rÁ   rÒ   râ   rÇ   )r¢   r�   Zop_devZop_gpurÃ   r  r   r   r   rc     s*    rc   c                s8   i ‰‡ ‡fdd„‰ ˆ | j jƒ ƒ ˆ | jjƒ ƒ ˆ| _dS )zI
    Assign blob to device option based on the operator outputing it
    c                s–   x�| j D ]†}|j}|jdkr,tjƒ }tj|_x,t|jƒt|j	ƒ D ]}|ˆkrB|ˆ|< qBW |jj
dƒrdd„ |jD ƒ}x|D ]}ˆ |jƒ q|W qW d S )NrÔ   r¼   c             S   s   g | ]}|j jd ƒr|‘qS )Zstep_net)rI   Úendswith)r'   rÌ   r   r   r   r*   N  s    z5_InferBlobDevice.<locals>.map_ops.<locals>.<listcomp>)r�   r›   r?   r   rJ   r"   r-   r0   rš   rÁ   rÒ   rd   Ún)Úprotor�   r›   rÊ   Z	step_argsZstep_arg)Úmap_opsÚmappingr   r   r,  C  s    

z!_InferBlobDevice.<locals>.map_opsN)rh   r=   r<   Ú_blob_to_device)r¢   r   )r,  r-  r   rV   =  s
    rV   c             C   s^   || j kr| j | jtjkS dj| j| jd |ƒ}|| j krH| jtjkS | j | jtjkS d S )Nz{}_{}/{}r   )r.  r-   r   r    r/   r7   r@   r6   )r¢   r  r   r   r   rç   U  s    

rç   c             C   s^   || j krtj| j | jƒS dj| j| jd |ƒ}|| j krHtj| jƒS tj| j | jƒS d S )Nz{}_{}/{}r   )r.  r   r¾   r-   r/   r7   r@   r6   )r¢   r  r   r   r   ræ   `  s    

ræ   c       	      C   sf  t ƒ }|t|ƒd… }�xHt|ƒD �]:\}}t|tjƒsRt|tjƒsRtdj|ƒƒ‚t	|ƒ}d}t|tjƒrºt
|jƒ jdƒd jdƒd ƒ}dj| j|ƒ|jƒ ks¸tdjt|ƒ| j|ƒƒ‚n†t
|jjƒ jdƒd jdƒd ƒ}dj| j|ƒ|jjƒ k�std	jt|ƒ| j|ƒƒ‚dj| j|ƒ|jjƒ k�s@td
jt|ƒ| j|ƒƒ‚||k�rRi ||< ||| |< q"W |S )z“
    Groups blobs by device, returning a map of [blobname] = {0: BlobRef, 1: ..}.
    Returns ordered dictionary, ensuring the original order.
    Nz.Param {} is not BlobReference or GradientSlicerÖ   r$   ú/r   z{}_{}/z+Param {} expected to have namescope '{}_{}'z-Indices {} expected to have namescope '{}_{}'z,Values {} expected to have namescope '{}_{}')r   r:   rÀ   rE   r   rM   r"  r.   r/   rÙ   ÚintZGetNameScopeÚsplitr7   rÇ   r  r!  )	r¢   rs   rH   Znon_data_paramsrß   Z_ir(   rI   Zgpuidr   r   r   rO   l  s*     "
rO   c             C   s~   t | ƒ}t| ƒt|ƒkrzg }t| ƒ}x6t|ƒD ]*\}}|dkr.||d  |kr.|j|ƒ q.W t| ƒt|ƒksztdj|ƒƒ‚d S )Nr   r$   zDuplicate entries in params: {})r^   r:   rÿ   rÀ   rã   r.   r/   )rH   Z
set_paramsZdupesÚspró   r(   r   r   r   rN   �  s    rN   c                sæ   t ƒ }ˆ jr0dd„ ˆ jdƒD ƒ}dd„ |D ƒ}nrg }xFˆ jjƒ jD ]6}‡ fdd„|jD ƒ}|jdd„ |D ƒƒ |j|ƒ qBW t ˆ j	ƒ| }|t ƒ ks¢t
dj|ƒƒ‚tˆ jƒd ‰‡fd	d
„}ttt |ƒƒ|d�}dd„ |D ƒ}||fS )zv
    We sync all blobs that are generated by param init net and
    are 'data parallel', i.e assigned to a device
    c             S   s   g | ]}t |ƒ‘qS r   )rÇ   )r'   r(   r   r   r   r*   ¥  s    z'_ComputeBlobsToSync.<locals>.<listcomp>r&   c             S   s   g | ]}t |ƒ‘qS r   )rÙ   )r'   r(   r   r   r   r*   ¦  s    c                s"   g | ]}|j d jˆ jƒƒr|‘qS )z{}_)rÒ   r/   r7   )r'   Úo)r¢   r   r   r*   ¬  s    c             S   s   g | ]}t |ƒ‘qS r   )rÙ   )r'   r3  r   r   r   r*   ¯  s    z2Some params not instantiated in param init net: {}r$   c                s    t | ˆ | jtjƒ… ƒ}|| fS )N)r0  r(  r   rÚ   )rÊ   Zdeviceid)Ú	prefixlenr   r   Úextract_sort_keyº  s    z-_ComputeBlobsToSync.<locals>.extract_sort_key)Úkeyc             S   s   g | ]}t j|ƒ‘qS r   )r   rM   )r'   rÊ   r   r   r   r*   Ã  s    )r^   r8   rQ   rh   r=   r�   rÁ   rR   r
  rS   r.   r/   r:   r7   rÿ   r0   )r¢   r‚   Zblobs_to_syncr�   Z
dp_outputsZdiffr5  r   )r¢   r4  r   rb   œ  s&    
rb   c             C   sR   t jdƒ xB|D ]:}dj| j|ƒ}tj| j|| tt| j	ƒƒ|dd�| j_
qW d S )Nz#------- DEPRECATED API, please use z3data_parallel_model.OptimizeGradientMemory() ----- z{}_{}/F)Úshare_activationszV------- DEPRECATED API, please use data_parallel_model.OptimizeGradientMemory() ----- )r3   r4   r/   r7   r   Úshare_grad_blobsr<   r^   r   r)   Ú_net)r¢   r|   rs   r~   rÃ   r   r   r   ri   Ç  s    
ri   c             C   s~   t ƒ }|d k	r@x0|D ](}x"|D ]}|jdj| j||ƒƒ qW qW | jd k	rd|jdd„ t| jƒD ƒƒ tj	| j
jƒ |ƒ| j
_d S )Nz{}_{}/{}c             S   s   g | ]}t |ƒ‘qS r   )rÇ   )r'   rÊ   r   r   r   r*   ã  s    z1_AddDynamicMemoryOptimization.<locals>.<listcomp>)r^   re   r/   r7   rA   rR   r   r)   r   Zrelease_blobs_when_usedr<   r=   r9  )r¢   rw   rs   Zblobs_to_keep_all_devicesr~   r  r   r   r   rj   Õ  s    


rj   c          
      sÊ   |dk	r`i }x8t |ƒD ],\}}x"| jD ]}||dj| j||ƒ< q&W qW tj| j| jg|ƒ\}}	nd}x`| jD ]V}
dj| j|
ƒ‰ t‡ fdd„|D ƒƒ}t	j
| j| j|
 tt| jƒƒˆ |||d�| j_qlW dS )aÁ  
    Optimize memory usage of the backward pass by recycling blobs for gradient
    inputs that have been 'used'.
    input_shapes:  dict of blob name to shape for the inputs of the model.
                   Pass empty dictionary if not known.
    excluded_blobs: list of blobs that cannot be recycled. These are blobs
                   that you will access externally.
    recycle_activations: whether to also recycle forward pass activations
    Nz{}_{}/{}z{}_{}/c             3   s   | ]}ˆ | V  qd S )Nr   )r'   rÊ   )rÃ   r   r   ú	<genexpr>	  s    z)OptimizeGradientMemory.<locals>.<genexpr>)Zdont_share_blobsr7  Zblob_shapes)r   r@   r/   r7   r   ZInferShapesAndTypesrh   r<   r^   r   r8  rZ   r   r)   r9  )r¢   Zinput_shapesZexcluded_blobsZrecycle_activationsZinput_shapes_all_devicesrÊ   ZshprÛ   ZshapesÚtypesr~   Zexcluded_blobs_by_devicer   )rÃ   r   ÚOptimizeGradientMemoryì  s(    
r<  c             C   s"  |d krt }|d }d }xZ| jƒ jD ]L}|jdkr4q$d}x |jD ]}	|	jdkr@|	j}P q@W ||krdq$|jd }P q$W |d kr†dj|ƒ}|d k	r¨| j	|g|||d d�}
nvt
ƒ }d	|krÂ|d	 |d	< d
|krÖ|d
 |d
< d|krê|d |d< | j|d pøg |f||d |d |d |dœ|—Ž}
|
S )Niè  ÚCreateCommonWorldr$   Ú
timeout_msr   z{}_opr¨   )rI   r¨   r  Z	interfaceÚmpi_rendezvousÚ
kv_handlerr%   Úshard_id)rI   rí   Úrankr¨   r>  éÿÿÿÿ)Ú_DEFAULT_TIMEOUT_SECr=   r�   r?   rd   rI   rf   rÁ   r/   ZCloneCommonWorldÚdictr=  )r<   Zcommon_world_blobrt   rI   r®   r>  Úexistingr�   Zop_timeout_msrd   r³   r   r   r   r   r±     sR    




r±   c             C   s‚  |d kr| j | }tj|ƒ��Z | j}|d ks:|d dkr>dS tj|d ƒjtjƒ}d||d < tj	d|ƒ tj
dƒ}tƒ }d|kr�|d |d< |j|d pžg d	f| jjƒ jd
 |d |d |d dœ|—Ž}|d }|j|g|gdd� |d }	|jd|g|	dd� |j||	g|	g|d d� tj|ƒ tj|	ƒ}
|
d }x6t|d ƒD ]&}|
| |k�sJtdj|d ƒƒ‚�qJW dS Q R X d S )Nr%   r$   TrA  Zcompare_arrZallcompare_netr?  r@  Zinitial_syncz.cw_master_selectr¨   )rI   rí   rB  r¨   Z	_checksumF)ZaverageZ_gather)r¯   r°   rû   )r¯   r°   r¨   r   zallcompare failed on shard {}.)r.  r   rK   rA   ÚnpÚzerosZastypeZfloat32r   ZFeedBlobr‘   rE  r=  r<   r=   rI   ZSumSqrElementsZMulrù   r£   Z	FetchBlobr1   r.   r/   )r¢   r  r~   rt   Ztest_data_arrZcomparison_netr   r³   Zblob_name_checksumZblob_name_gatherZ
gather_arrZbaselinerf   r   r   r   Ú_RunComparisonW  sP    





rI  c       
      C   s  t | jjƒ jƒ}t| jƒ}t|ƒ| }|| t|ƒks>tdƒ‚g }dd„ t|ƒD ƒ}x|D ]}||jj	 j
|ƒ qZW xxt|ƒD ]l}d}xb| jD ]X}	|dkr¨||	 | j}|j
||	 | ƒ ||	 | j|ksŽtdj|||	 | jƒƒ‚qŽW q~W | jjƒ jdd…= | jjƒ jj|ƒ dS )a½  
    Data Parallel Model creates a net with ops in one device grouped together.
    This will interleave the ops so that each op for each device is next
    to each other in the net. Kind of like combining decks of cards. This
    ensures that progress is made along the critical path roughly concurrently
    for each device, which is important due to the extra intra-node
    synchronization required for multi-device batch normalization.
    z7Number of ops per device in original net is not uniformc             S   s   i | ]
}g |“qS r   r   )r'   rÛ   r   r   r   rÍ   ž  s    z"_InterleaveOps.<locals>.<dictcomp>NzType mismatch {} / {})r0   r<   r=   r�   r:   r@   r.   r1   r›   r¿   rã   r?   r/   r
  )
r¢   Úorig_opsr�   Znum_ops_per_devÚnew_opsr  r�   ró   ÚtprÛ   r   r   r   rW   �  s&    	

 rW   c          	   C   sØ  t | jjƒ jƒ}g }t| jƒ}g }g }d}g }g }g }	d }
d}g }g }dd„ }�xN|D �]D}|jdkon|jdk�rB|rÔ|j|ƒ |jt	j
d||
d ƒƒ |jt	j
d||
d ƒƒ |j|ƒ g }g }g }g }d}d }
nb|�r6|j|ƒ |j|t|d	 ƒ||ƒƒ |j|t|d	 ƒ||ƒƒ |j|ƒ g }g }g }g }d}|j|ƒ qV|jdk�rîd
}|
d k�rf|jd	 }
|jd	 }	|jt	j
d|	|	d |	d gƒƒ |j|	d ƒ |j|	d ƒ |jj|
d ƒ |jj|
d ƒ |jjtjd|ƒgƒ |j|ƒ qV|jdkrVd
}|jt	j
d|jd	 |jd |jd |jd g|jd |jd gƒƒ |j|jd ƒ |j|jd ƒ |jjtjd|ƒgƒ |jj|jd |jd gƒ |j|ƒ qVW | �s°tdƒ‚| jjƒ jd d …= | jjƒ jj|ƒ d S )NFc             S   sJ   g }d|  d }|j tjd||ƒƒ x |D ]}|j tjd||ƒƒ q*W |S )a’  
        Reduce results from multiple cpus and distributes the results back
        to each device. This is done by copying values to cpu_0 and summing
        them. The cpu_0 result is then copied back to each of the devices.

        param: the name of the data (blobs) to reduce
        input_blobs: the list of blobs to reduce
        destination_blobs: list of blobs to copy the result to
        zcpu_0/Z	_combinedrî   r•   )rã   r   ÚCreateOperator)r†   rÎ   Údestination_blobsÚ	added_opsZresult_blobrÄ   r   r   r   Ú
_cpuReduceÁ  s    

z5_CPUInterDeviceBatchNormalization.<locals>._cpuReduceÚ	SpatialBNÚSpatialBNGradientrî   Ú_sums_combinedÚ_sumsq_combinedr   TÚChannelStatsÚ_sumsÚ_sumsqÚnum_batchesÚChannelBackpropStatsrñ   r   r´   r$   z@Net modification for cpu inter-device batch normalization failed)r0   r<   r=   r�   r:   r@   r?   r
  rã   r   rM  rÙ   rš   rd   r   ÚMakeArgumentrÁ   r.   )r¢   rJ  rK  r�   Úbatch_norm_opsÚinjected_opsÚspatial_bn_phaseÚ
sums_blobsÚsumsq_blobsrI   Úinput_blob_nameÚspatial_bn_gradient_phaseÚscale_grad_blobsÚbias_grad_blobsrP  r�   r   r   r   rX   °  s¤    












rX   c          	      s$  t ˆjjƒ jƒ}g }tˆjƒ}g }g }d}g }g }g }	d }
d}g }g }d}tjtj	ƒ‰ d‡ ‡fdd„	}�x‚|D �]x}|j
dko†|j
dk�rb|rð|j|ƒ |j|t|
ƒd ||ƒƒ |j|t|
ƒd ||ƒƒ |j|ƒ g }g }g }g }d}d }
nf|�rV|j|ƒ |j|t|d	 ƒ|||ƒƒ |j|t|d	 ƒ|||ƒƒ |j|ƒ g }g }g }g }d}|j|ƒ qn|j
dk�r$d
}|
d k�r†|jd	 }
|jd	 }	tjˆj|jjƒ}|jtjd|	|	d |	d g|d�ƒ |j|	d ƒ |j|	d ƒ |jj|	d ƒ |jj|	d ƒ |jjtjd|ƒgƒ |j|ƒ qn|j
dkrnd
}tjˆj|jjƒ}|jtjd|jd	 |jd |jd |jd g|jd |jd g|d�ƒ |j|jd ƒ |j|jd ƒ |jjtjd|ƒgƒ |jj|jd |jd gƒ |j|ƒ qnW | �sütdƒ‚ˆjjƒ jd d …= ˆjjƒ jj|ƒ d S )NFZcpu_0c       	         sø   g }g }g }|dkr*‡ fdd„t |ƒD ƒ}xdt |ƒD ]X}tjˆj|ƒ}|jdj|ˆ ƒƒ |jdj|ˆ |ƒƒ |jtjd|| || |d�ƒ q4W |jtjd|d	j|ˆ ƒˆd�ƒ xBt |ƒD ]6}tjˆj|ƒ}|jtjd
d	j|ˆ ƒ|| |d�ƒ qºW |S )aå  
        Reduces results from multiple gpus and distributes the results back
        to each device. This is done by copying values to the master device
        and summing them. The master device result is then copied back to
        each of the devices.

        param: the name of the data (blobs) to reduce
        num_devices: the number of devices
        master_device: the device to copy/compute values on
        result_blobs: optional list of result blobs to copy to
        Nc                s   g | ]}d j |ˆ ƒ‘qS )zgpu_{}/{}_combined)r/   )r'   rf   )r†   r   r   r*   =  s    zI_GPUInterDeviceBatchNormalization.<locals>._gpuReduce.<locals>.<listcomp>z	gpu_{}/{}z{}/{}_gpu_{}_copyr   )r›   rî   z{}/{}_combinedr  )r1   r   rJ   r6   rã   r/   rM  )	r†   r�   rœ   Zresult_blobsrO  Zsource_blobsrN  rf   r›   )Úmaster_device_optionr¢   )r†   r   Ú
_gpuReduce,  s>    


z5_GPUInterDeviceBatchNormalization.<locals>._gpuReducerQ  rR  rV  rW  r   TrU  )r›   rS  rT  rX  rY  rñ   r   r´   r$   z@Net modification for gpu inter-device batch normalization failed)N)r0   r<   r=   r�   r:   r@   r   rJ   r   r"   r?   r
  rÙ   rã   rš   r6   r›   r¿   rM  rd   r   rZ  rÁ   r.   )r¢   rJ  rK  r�   r[  r\  r]  r^  r_  rI   r`  ra  rb  rc  rœ   re  r�   r›   r   )rd  r¢   r   rY     s¼    
.

















rY   )F)N)F)NF)F)FN)r   )N)F)F)F)NN)N)ZÚ
__future__r   r   r   Úcollectionsr   Zfuture.utilsr   r   r   ÚloggingrG   Úmultiprocessingr	   Zcaffe2.pythonr
   r   r   r   r   r   r   Zcaffe2.protor   ZnumpyrG  r¸   ZInitOpsLibraryr2   Ú	getLoggerr3   ÚsetLevelÚINFOrD  Z _DEFAULT_BARRIER_NET_TIMEOUT_SECr   r   r   r   r„   r…   rƒ   r¡   r¥   r§   r¦   rm   r«   r»   rÅ   r’   rU   rÐ   r×   rà   rä   ré   rô   rg   r™   rú   rö   rõ   r]   r`   rn   Úobjectrþ   r  r  r[   r%  r$  r\   rÙ   rc   rV   rç   ræ   rO   rN   rb   ri   rj   r<  r±   rI  rW   rX   rY   r   r   r   r   Ú<module>   sà   $



  [
  

*

9

G

= (PX


		!#+- 
=
8!i