3
O4W^ù'  ã               @   sb   d dl mZmZmZmZ d dlZd dlZd dlZd dlm	Z	m
Z
 dd„ Zdd„ Zedkr^eƒ  dS )	é    )Úabsolute_importÚdivisionÚprint_functionÚunicode_literalsN)ÚArgumentParserÚ	REMAINDERc              C   s°   t dd�} | jdtddd� | jdtdd	d� | jd
tddd� | jddtdd� | jddtdd� | jddddd� | jdddddd� | jdtdd� | jdtd� | jƒ S ) zU
    Helper function parsing the command line options
    @retval ArgumentParser
    zcPyTorch distributed training launch helper utilty that will spawn up multiple distributed processes)Údescriptionz--nnodesé   z3The number of nodes to use for distributed training)ÚtypeÚdefaultÚhelpz--node_rankr   z8The rank of the node for multi-node distributed trainingz--nproc_per_nodez·The number of processes to launch on each node, for GPU training, this is recommended to be set to the number of GPUs in your system so that each process can be bound to a single GPU.z--master_addrz	127.0.0.1z©Master node (rank 0)'s address, should be either the IP address or the hostname of node 0, for single node multi-proc training, the --master_addr can simply be 127.0.0.1)r   r
   r   z--master_porti<s  zdMaster node (rank 0)'s free port that needs to be used for communciation during distributed trainingz	--use_envFÚ
store_truezÂUse environment variable to pass 'local rank'. For legacy reasons, the default value is False. If set to True, the script will not pass --local_rank as argument, and will instead set LOCAL_RANK.)r   Úactionr   z-mz--modulezwChanges each process to interpret the launch script as a python module, executing with the same behavior as'python -m'.Útraining_scriptz‰The full path to the single GPU training program/script to be launched in parallel, followed by all the arguments for the training script)r
   r   Útraining_script_args)Únargs)r   Úadd_argumentÚintÚstrr   Ú
parse_args)Úparser© r   ú9/tmp/pip-build-e66j97b2/torch/torch/distributed/launch.pyr   •   s&    






r   c              C   sL  t ƒ } | j| j }tjjƒ }| j|d< t| jƒ|d< t|ƒ|d< g }dtjkrv| jdkrvtdƒ|d< t	dj
|d ƒƒ xœtd| jƒD ]Œ}| j| j | }t|ƒ|d< t|ƒ|d	< tjd
g}| jrÊ|jdƒ |j| jƒ | jsì|jdj
|ƒƒ |j| jƒ tj||d�}|j|ƒ q„W x2|D ]*}|jƒ  |jdk�rtj|j|d�‚�qW d S )NZMASTER_ADDRZMASTER_PORTZ
WORLD_SIZEZOMP_NUM_THREADSr	   a&  *****************************************
Setting OMP_NUM_THREADS environment variable for each process to be {} in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************r   ZRANKZ
LOCAL_RANKz-uz-mz--local_rank={})Úenv)Ú
returncodeÚcmd)r   Znproc_per_nodeZnnodesÚosÚenvironÚcopyZmaster_addrr   Zmaster_portÚprintÚformatÚrangeZ	node_rankÚsysÚ
executableÚmoduleÚappendr   Zuse_envÚextendr   Ú
subprocessÚPopenÚwaitr   ÚCalledProcessError)ÚargsZdist_world_sizeZcurrent_envZ	processesZ
local_rankZ	dist_rankr   Úprocessr   r   r   ÚmainÈ   s:    




r-   Ú__main__)Ú
__future__r   r   r   r   r"   r'   r   Úargparser   r   r   r-   Ú__name__r   r   r   r   Ú<module>   s    34