3
I4W^z  ã               @   sj   d dl Z ddlmZ d dlmZmZmZmZ dd„ Zddd	„Z	dd
d„Z
ddd„Zddd„Zddd„ZdS )é    Né   )Únccl)Ú_take_tensorsÚ_flatten_dense_tensorsÚ_unflatten_dense_tensorsÚ_reorder_tensors_asc             C   s   t jj| |ƒS )aË  Broadcasts a tensor to a number of GPUs.

    Arguments:
        tensor (Tensor): tensor to broadcast.
        devices (Iterable): an iterable of devices among which to broadcast.
          Note that it should be like (src, dst1, dst2, ...), the first element
          of which is the source device to broadcast from.

    Returns:
        A tuple containing copies of the ``tensor``, placed on devices
        corresponding to indices from ``devices``.
    )ÚtorchÚ_CZ
_broadcast)ÚtensorÚdevices© r   ú0/tmp/pip-build-e66j97b2/torch/torch/cuda/comm.pyÚ	broadcast   s    r   é    c             C   s   t jj| ||ƒS )a‡  Broadcasts a sequence tensors to the specified GPUs.
    Small tensors are first coalesced into a buffer to reduce the number
    of synchronizations.

    Arguments:
        tensors (sequence): tensors to broadcast.
        devices (Iterable): an iterable of devices among which to broadcast.
          Note that it should be like (src, dst1, dst2, ...), the first element
          of which is the source device to broadcast from.
        buffer_size (int): maximum size of the buffer used for coalescing

    Returns:
        A tuple containing copies of the ``tensor``, placed on devices
        corresponding to indices from ``devices``.
    )r   r	   Z_broadcast_coalesced)Útensorsr   Úbuffer_sizer   r   r   Úbroadcast_coalesced   s    r   c             C   s>  |dkrt jjƒ }| d jƒ }d}x|t| ƒD ]p\}}|jsBtdƒ‚|jƒ |krR|}|jƒ |kr,djdd„ |jƒ D ƒƒ}djdd„ |D ƒƒ}t	dj
|||ƒƒ‚q,W |dkr°td	ƒ‚|j|d
�j|ƒjƒ }tj| ƒoÞ| d jƒ |k�r|gdd„ | dd… D ƒ }	tj| |	|d� |S x&| D ]}|j|jƒ ƒ}
|j|
ƒ �qW |S )a–  Sums tensors from multiple GPUs.

    All inputs should have matching shapes.

    Arguments:
        inputs (Iterable[Tensor]): an iterable of tensors to add.
        destination (int, optional): a device on which the output will be
            placed (default: current device).

    Returns:
        A tensor containing an elementwise sum of all inputs, placed on the
        ``destination`` device.
    Nr   z+reduce_add expects all inputs to be on GPUsÚxc             s   s   | ]}t |ƒV  qd S )N)Ústr)Ú.0r   r   r   r   ú	<genexpr>C   s    zreduce_add.<locals>.<genexpr>c             s   s   | ]}t |ƒV  qd S )N)r   )r   r   r   r   r   r   D   s    z2input {} has invalid size: got {}, but expected {}zLreduce_add expects destination to be on the same GPU with one of the tensors)Zdevicec             S   s   g | ]}|j |jƒ ƒ‘qS r   )ÚnewÚsize)r   Útr   r   r   ú
<listcomp>L   s    zreduce_add.<locals>.<listcomp>r   )Úroot)r   ZcudaZcurrent_devicer   Ú	enumerateZis_cudaÚAssertionErrorZ
get_deviceÚjoinÚ
ValueErrorÚformatÚRuntimeErrorr   Z
resize_as_Zzero_r   Zis_availableÚreduceZadd_)ÚinputsÚdestinationZ
input_sizeZ	nccl_rootÚiÚinpÚgotÚexpectedÚresultÚoutputsZinput_correct_gpur   r   r   Ú
reduce_add*   s0    

r+   c                s  dd„ | D ƒ}g }g }xˆt | Ž D ]|}tdd„ |D ƒƒrZt||ƒ}|j|ƒ |j|d ƒ q x.t ||ƒD ] \}}	|j|	jr€|	jƒ n|	ƒ qfW |j|d d	 ƒ q W ‡ fdd„|D ƒ}
xLt |
Ž D ]@}dd„ |D ƒ}t||ƒ}x"t||d ƒD ]}	|j|	jƒ qèW q¼W tt	||ƒƒS )
ab  Sums tensors from multiple GPUs.

    Small tensors are first coalesced into a buffer to reduce the number
    of synchronizations.

    Arguments:
        inputs (Iterable[Iterable[Tensor]]): iterable of iterables that
            contain tensors from a single device.
        destination (int, optional): a device on which the output will be
            placed (default: current device).
        buffer_size (int): maximum size of the buffer used for coalescing

    Returns:
        A tuple of tensors containing an elementwise sum of each group of
        inputs, placed on the ``destination`` device.
    c             S   s   g | ]}g ‘qS r   r   )r   Ú_r   r   r   r   h   s    z(reduce_add_coalesced.<locals>.<listcomp>c             s   s   | ]}|j V  qd S )N)Ú	is_sparse)r   r   r   r   r   r   m   s    z'reduce_add_coalesced.<locals>.<genexpr>r   r   c                s   g | ]}t |ˆ ƒ‘qS r   )r   )r   r   )r   r   r   r   u   s    c             S   s   g | ]}t |ƒ‘qS r   )r   )r   Úchunkr   r   r   r   x   s    éÿÿÿÿ)
ÚzipÚallr+   Úappendr-   Zto_denser   ÚdataÚtupler   )r#   r$   r   Zdense_tensorsÚoutputZ	ref_orderZtensor_at_gpusr)   Zcollr   ZitrsÚchunksZflat_tensorsZflat_resultr   )r   r   Úreduce_add_coalescedU   s$    


r7   c             C   s   t tjj| ||||ƒƒS )a¡  Scatters tensor across multiple GPUs.

    Arguments:
        tensor (Tensor): tensor to scatter.
        devices (Iterable[int]): iterable of ints, specifying among which
            devices the tensor should be scattered.
        chunk_sizes (Iterable[int], optional): sizes of chunks to be placed on
            each device. It should match ``devices`` in length and sum to
            ``tensor.size(dim)``. If not specified, the tensor will be divided
            into equal chunks.
        dim (int, optional): A dimension along which to chunk the tensor.

    Returns:
        A tuple containing chunks of the ``tensor``, spread across given
        ``devices``.
    )r4   r   r	   Z_scatter)r
   r   Zchunk_sizesÚdimZstreamsr   r   r   Úscatter‚   s    r9   c             C   s   t jj| ||ƒS )aÿ  Gathers tensors from multiple GPUs.

    Tensor sizes in all dimension different than ``dim`` have to match.

    Arguments:
        tensors (Iterable[Tensor]): iterable of tensors to gather.
        dim (int): a dimension along which the tensors will be concatenated.
        destination (int, optional): output device (-1 means CPU, default:
            current device)

    Returns:
        A tensor located on ``destination`` device, that is a result of
        concatenating ``tensors`` along ``dim``.
    )r   r	   Z_gather)r   r8   r$   r   r   r   Úgather–   s    r:   )r   )N)Nr   )Nr   N)r   N)r   Ú r   Ztorch._utilsr   r   r   r   r   r   r+   r7   r9   r:   r   r   r   r   Ú<module>   s   

+
-
