3
O4W^ƒ!  ã               @   sð   d dl mZmZmZ d dl mZ d dl mZ d dl mZ d dlmZm	Z	 d dl
mZmZ ddlZddlZdd	lmZ dad
d„ Zdd„ Zedd„ ƒZG dd„ deƒZejddddfdd„Zeddd„ƒZdd„ Zed dd„ƒZed!dd„ƒZdS )"é   )Úinvoke_rpc_builtinÚinvoke_rpc_python_udfÚinvoke_remote_builtin)Úinit_rref_context)ÚProcessGroupAgent)ÚWorkerId)Ú	serializeÚ	PythonUDF)Úis_rpc_backend_registeredÚinit_rpc_backendé    N)ÚEnumc                s   ‡ fdd„}|S )Nc                 s   t d krtdƒ‚ˆ | |ŽS )Nz8RPC has not been initialized. Call init_rpc(name) first.)Ú_agentÚRuntimeError)ÚargsÚkwargs)Úfunc© ú6/tmp/pip-build-e66j97b2/torch/torch/distributed/rpc.pyÚwrapper   s    z%_require_initialized.<locals>.wrapperr   )r   r   r   )r   r   Ú_require_initialized   s    r   c               C   s   t rt jƒ  da dS )zÛ
    Block until all local and remote RPC processes reach this method, process
    (send and receive) all pending messages, and then destroy local RPC agent.
    Every RPC process must call this method before exit.
    N)r   Újoinr   r   r   r   Újoin_rpc   s    r   c               C   s   t jƒ  dS )a  
    Block until all local and remote RPC processes reach this method and finish
    sending all pending RPCs. As this method synchronizes at the process
    level, if multiple threads are spawned, only one of them should call this
    method at a time.
    N)r   Úsyncr   r   r   r   Úsync_rpc(   s    	r   c               @   s   e Zd ZdZdS )Ú
RpcBackendr   N)Ú__name__Ú
__module__Ú__qualname__ÚPROCESS_GROUPr   r   r   r   r   3   s   r   é   c             C   s¦   t jd
k rtdƒ‚trtdƒ‚| tjkrxddlm} |ƒ }|dkrb||jƒ krbtdj	||jƒ ƒƒ‚t
|||ƒattƒ n*t| ƒr˜t| ||d�attƒ n
td	| ƒ‚d S )Né   r   z%RPC package does not support Python2.zRPC is already initializedr   )Ú_get_default_groupz.self_rank argument {} doesn't match pg rank {})Ú	self_nameÚinit_methodzUnrecognized RPC backend )r!   r   éÿÿÿÿ)ÚsysÚversion_infor   r   r   r   Zdistributed_c10dr"   ZrankÚformatr   r   r
   r   )Úbackendr#   Z	self_rankr$   Znum_send_recv_threadsr"   Úgroupr   r   r   Ú	_init_rpc8   s&    



r+   c             C   s   | rt j| ƒS t jƒ S dS )a7  
    Get worker id of a given worker name. Use this worker id to avoid passing
    an expensive string to ``rpc`` on every invocation.

    Arguments:
        worker_name (str): the string name of a worker. If ``None``, return the
                           the id of the current worker. (default ``None``)
    N)r   Úget_worker_id)Zworker_namer   r   r   r,   Z   s    

r,   c             C   s2   t | tƒr| S t | tƒr t| ƒS tdj| ƒƒ‚d S )Nz!Unsupported RPC worker ID type {})Ú
isinstancer   Ústrr,   Ú
ValueErrorr(   )Z
name_or_idr   r   r   Ú_to_worker_idj   s
    

r0   c             C   s<   t jj|ƒ}|r|nf }|r |ni }ttt| ƒ|f|ž|ŽS )a  
    Make a ``remote`` call to run ``func`` on worker ``to``, and returns an
    ``RRef`` to the result value immediately. Worker ``to`` will be the owner
    of the return ``RRef``, and this worker is a user. The owner manages the
    global reference count of its ``RRef``s, and the owner ``RRef`` is only
    destructed when globally there is no living references to it.

    Arguments:
        to (int or str): id or name of the destination worker.
        func (callable): builtin functions (like ``torch.add``).
        args (tuple): the argument tuple for the ``func`` invocation.
        kwargs (dict): is a dictionary of keyword arguments for the ``func``
                       invocation.

    Returns:
        A user ``RRef`` instance to the result value. Use the blocking API
        ``RRef.to_here()`` to retrieve the result value locally.

    Example::

        On worker 0:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=0, world_size=2)
        >>> dist.init_rpc("worker0")
        >>> worker1 = dist.get_worker_id("worker1")
        >>> rref1 = dist.remote(worker1, torch.add, args=(torch.ones(2), 3))
        >>> rref2 = dist.remote(worker1, torch.add, args=(torch.ones(2), 1))
        >>> x = rref1.to_here() + rref2.to_here()
        >>> dist.join_rpc()

        One worker 1:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=1, world_size=2)
        >>> dist.init_rpc("worker1")
        >>> dist.join_rpc()
    )ÚtorchÚjitÚ_find_builtinr   r   r0   )Útor   r   r   Úqualified_namer   r   r   Úremotes   s
    &r6   Fc             C   s†   t |ƒstdƒ‚tjj|ƒ}|r$|nf }|r0|ni }|dk	rVttt| ƒ|f|ž|Ž}nttt| ƒt	t
|||ƒƒƒ}|rz|S |jƒ S dS )a}
  
    Make an RPC call to run function ``func`` on worker ``to``. By default, it
    blocks until the return value is locally available. RPC messages are sent
    and received in parallel to execution of Python code. This method is
    thread-safe.

    Arguments:
        to (int or str): id or name of the destination worker.
        func (callable): any callable function. builtin functions (like
                         ``torch.add``) can be sent over RPC more efficiently.
        args (tuple): the argument tuple for the ``func`` invocation.
        kwargs (dict): is a dictionary of keyword arguments for the ``func``
                       invocation.
        async_call (bool): If set to ``True``, this will be an asynchronous RPC,
                           and returns a ``torch.distributed.FutureMessage``
                           object immediately. Otherwise, this RPC will block
                           until the return value is locally available.
                           (default: ``False``)

    Returns:
        If ``async_call`` is ``False``, returns the result of running ``func``
        on ``args`` and ``kwargs``. If ``async_call`` is ``True``, returns a
        ``torch.distributed.FutureMessage`` object that can be waited on. When
        completed, the return value of ``func`` on ``args`` and ``kwargs`` can
        be retrieved from the ``FutureMessage`` object.

    Example::

        Synchronous example:

        On worker 0:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=0, world_size=2)
        >>> dist.init_model_parallel("worker0")
        >>> ret = dist.rpc("worker1", torch.add, args=(torch.ones(2), 3))
        >>> dist.join_rpc()

        One worker 1:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=1, world_size=2)
        >>> dist.init_model_parallel("worker1")
        >>> dist.join_rpc()

        Asynchronous example:

        On worker 0:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=0, world_size=2)
        >>> dist.init_model_parallel("worker0")
        >>> worker1 = dist.get_worker_id("worker1")
        >>> fut1 = dist.rpc(worker1, torch.add, args=(torch.ones(2), 3), async_call=True)
        >>> fut2 = dist.rpc(worker1, min, args=(1, 2), async_call=True)
        >>> result = fut1.wait() + fut2.wait()
        >>> dist.join_rpc()

        One worker 1:
        >>> import torch.distributed as dist
        >>> dist.init_process_group(backend='gloo', rank=1, world_size=2)
        >>> dist.init_model_parallel("worker1")
        >>> dist.join_rpc()
    zfunction should be callable.N)ÚcallableÚ	TypeErrorr1   r2   r3   r   r   r0   r   r   r	   Úwait)r4   r   r   r   Z
async_callr5   Zfutr   r   r   Úrpc¢   s    ?r:   r%   )N)NN)NNF)Ú r   r   r   r   r   r   Zinternal_rpc_utilsr   r	   Zrpc_backend_registryr
   r   r&   r1   Úenumr   r   r   r   r   r   r   r+   r,   r0   r6   r:   r   r   r   r   Ú<module>   s2   		.