3
I4W^Ã  ã               @   sj   d Z ddlZddlmZ ddlmZ dd„ Zdd„ Zdd
d„Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ ZdS )aJ  muji.py does multi-gpu training for caffe2 with no need to change the c++
side code. Everything is defined on the computation graph level.

We support the following use cases:
  - 2 gpus, where peer access is enabled between them.
  - 4 gpus, where peer access are enabled between all of them.
  - 4 gpus, where peer access are enabled in two groups,
    between {1, 2} and {3, 4}
  - 8 gpus, where peer access are enabled in two groups,
    between {1, 2, 3, 4} and {5, 6, 7, 8}.
If above cases are not satisfied, a fallback function which does not rely on
peer access will be called.
é    N)Ú
caffe2_pb2)Ú	workspacec             C   s   t jƒ }tj|_| |_|S )zVA utility function that returns a device option protobuf of the
  specified gpu id.
  )r   ÚDeviceOptionr   ZGpuDeviceTypeÚdevice_typeZ	device_id)Zgpu_idÚdevice_option© r   ú3/tmp/pip-build-e66j97b2/torch/caffe2/python/muji.pyÚOnGPU   s    r	   c              C   s   t jƒ } t j| _| S )N)r   r   ZCPUr   )r   r   r   r   ÚOnCPU!   s    r
   Ú_reducedc             C   sŠ  |dkrt tt|ƒƒƒ}t|ƒt|ƒkr@tdt|ƒt|ƒf ƒ‚tjƒ }t|ƒdkrŠ|jd dkrŠtj|dd…dd…f ƒrŠt	| |||ƒS t|ƒdkrÌ|jd dkrÌtj|dd…dd…f ƒrÌt
| |||ƒS t|ƒdk�r0|jd dk�r0tj|dd…dd…f ƒ�r0tj|dd…dd…f ƒ�r0t| |||ƒS t|ƒdk�rx|jd dk�rxtj|dd…dd…f ƒ�rxt| |||ƒS t| |||ƒS dS )zºThe general Allreduce interface that reroutes the function calls.
    CPUs and AMD GPUs are not supported because
    GetGpuPeerAccessPattern is called to get gpu peer access pattern.
  Nz6gpu_indices length and blobs length mismatch: %d vs %dé   r   é   é   )ÚlistÚrangeÚlenÚRuntimeErrorr   ZGetGpuPeerAccessPatternÚshapeÚnpÚallÚ
Allreduce2Ú
Allreduce4ÚAllreduce4Group2Ú
Allreduce8ÚAllreduceFallback)ÚnetÚblobsÚreduced_affixÚgpu_indicesÚpatternr   r   r   Ú	Allreduce'   s     44V:r    c       
      C   sL   |\}}|\}}| j ||g|| t|ƒd�}|jg || t|ƒd�}	||	fS )zUAllreduce for 2 gpus.

  Algorithm: 0r <- 0 + 1, 1r <- 0r, where r means "reduced"
  )r   )ÚAddr	   ÚCopy)
r   r   r   r   ÚaÚbÚgpu_aÚgpu_bÚ	a_reducedÚ	b_reducedr   r   r   r   @   s    r   c             C   sÄ   |\}}}}|\}}	}
}| j ||gt|ƒ| t|ƒd�}| j ||gt|ƒ| t|
ƒd�}|j ||t|ƒd�}|jg |t|
ƒd�}|jg t|ƒ| t|	ƒd�}|jg t|ƒ| t|ƒd�}||||fS )z•Allreduce for 4 gpus.

  Algorithm: 2 level reduction.
      0r <- 0 + 1, 2r <- 2 + 3
      0r <- 0r + 2r
      2r <- 0r,
      1r <- 0r, 3r <- 2r
  )r   )r!   Ústrr	   r"   )r   r   r   r   r#   r$   ÚcÚdr%   r&   Úgpu_cÚgpu_dr'   Ú	c_reducedr(   Ú	d_reducedr   r   r   r   P   s*    	



r   c             C   sà   |\}}}}|\}}	}
}| j ||gt|ƒ| t|ƒd�}| j ||gt|ƒ| t|
ƒd�}|jg t|ƒd t|ƒd�}|j ||t|ƒd�}|jg |t|
ƒd�}|jg t|ƒ| t|	ƒd�}|jg t|ƒ| t|ƒd�}||||fS )zÅAllreduce for 4 gpus where peer access are enabled in {0,1} and {2,3}

  Algorithm: 2 level reduction.
      0r <- 0 + 1, 2r <- 2 + 3
      0r <- 0r + 2r
      2r <- 0r,
      1r <- 0r, 3r <- 2r
  )r   Ú_copy)r!   r)   r	   r"   )r   r   r   r   r#   r$   r*   r+   r%   r&   r,   r-   r'   r.   Zc_reduced_copyr(   r/   r   r   r   r   x   s2    	




r   c             C   sz  dgd }x@dD ]8}| j || ||d  g|| | t|| ƒd�||< qW xDdD ]<}| j || ||d  gt|| ƒ| t|| ƒd�||< qRW |d jg t|d ƒd	 t|d ƒd�}|d j ||d t|d ƒd�|d< |d jg |d t|d ƒd�|d< x6dD ].}||d  jg || t|| ƒd�||< �qW x:dD ]2}||d  jg || | t|| ƒd�||< �q@W |S )zþAllreduce for 8 gpus.

  Algorithm: 3 level reduction.
      0r <- 0 + 1, 2r <- 2 + 3, 4r <- 4 + 5, 6r <- 6 + 7
      0r <- 0r + 2r, 4r <- 4r + 6r
      0r <- 0r + 4r
      4r <- 0r
      2r <- 0r, 6r <- 4r
      1r <- 0r, 3r <- 2r, 5r <- 4r, 7r <- 6r
  Nr   r   r   r   é   é   )r   r0   é   é   é   )r   r   r   r1   )r   r   )r   r1   )r2   r3   r4   r5   )r!   r	   r)   r"   )r   r   r   r   ÚreducedÚiZreduced_4_copyr   r   r   r   ¦   sD    






r   c             C   sø   dgt |ƒ }|dkr@| j|d |d | t|d ƒd�|d< n|d |d< |d d }xZtdt |ƒƒD ]H}| j|| |t|d ƒd�}| j||d g|d t|d ƒd�|d< qhW x>tdt |ƒƒD ],}| j|d || | t|| ƒd�||< qÄW |S )zÇA fallback option for Allreduce with no assumption on p2p.

  Algorithm: a flat operation on gpu 0
      0r <- 0
      0r <- 0r + i for i in gpu_indices[1:]
      ir <- 0r for i in gpu_indices[1:]
  NÚ r   )r   Z
_temp_copyr2   )r   r"   r	   r   r!   )r   r   r   r   r6   Z	temp_namer7   Útempr   r   r   r   â   s.    


r   )r   N)Ú__doc__Znumpyr   Zcaffe2.protor   Zcaffe2.pythonr   r	   r
   r    r   r   r   r   r   r   r   r   r   Ú<module>   s   

(.<