3
I4W^l  ã               @   s�   d dl Z d dlZdddddgZd Zdd„ Zd	d
„ Zdd„ Zdd„ Zdeddfdd„Z	dd eddfdd„Z
ddd„Zddd„Zeddfdd„ZdS )é    NÚ
all_reduceÚreduceÚ	broadcastÚ
all_gatherÚreduce_scatterc             C   sp   t ƒ }xJ| D ]B}|jrdS |jƒ s&dS |js0dS |jƒ }||krDdS |j|ƒ qW ttjdƒslt	j
dƒ dS dS )NFÚ_nccl_all_reducez)PyTorch is not compiled with NCCL supportT)ÚsetZ	is_sparseZis_contiguousZis_cudaZ
get_deviceÚaddÚhasattrÚtorchÚ_CÚwarningsÚwarn)ZtensorsZdevicesZtensorZdevice© r   ú0/tmp/pip-build-e66j97b2/torch/torch/cuda/nccl.pyÚis_available	   s     

r   c               C   s
   t jjƒ S )N)r   r   Z_nccl_versionr   r   r   r   Úversion   s    r   c               C   s
   t jjƒ S )N)r   r   Z_nccl_unique_idr   r   r   r   Ú	unique_id"   s    r   c             C   s   t jj| ||ƒS )N)r   r   Z_nccl_init_rank)Z	num_ranksÚuidZrankr   r   r   Ú	init_rank&   s    r   c             C   s$   |d kr| }t jj| ||||ƒ d S )N)r   r   r   )ÚinputsÚoutputsÚopÚstreamsÚcommsr   r   r   r   *   s    c             C   s&   |d kr| }t jj| |||||ƒ d S )N)r   r   Z_nccl_reduce)r   r   Úrootr   r   r   r   r   r   r   0   s    c             C   s   t jj| |||ƒ d S )N)r   r   Z_nccl_broadcast)r   r   r   r   r   r   r   r   6   s    c             C   s   t jj| |||ƒ d S )N)r   r   Z_nccl_all_gather)r   r   r   r   r   r   r   r   :   s    c             C   s   t jj| ||||ƒ d S )N)r   r   Z_nccl_reduce_scatter)r   r   r   r   r   r   r   r   r   >   s    )r   NN)NN)r   Z
torch.cudar   Ú__all__ZSUMr   r   r   r   r   r   r   r   r   r   r   r   r   Ú<module>   s   

