3
I4W^Ž(  ã               @   sp   d dl mZmZmZmZ d dlZd dlZdd„ Zdd„ Zdd„ Z	d	d
„ Z
G dd„ dejjƒZdd„ Zdd„ ZdS )é    )Úabsolute_importÚdivisionÚprint_functionÚunicode_literalsNc             C   sj   t | tƒrVg }x>| D ]6}t |tjƒs0|j|ƒ q|jƒ }|j|_|j|ƒ qW t|ƒS tdt| ƒj	ƒ‚d S )Nz@Only tuple of tensors is supported. Got Unsupported input type: )
Ú
isinstanceÚtupleÚtorchÚTensorÚappendÚdetachÚrequires_gradÚRuntimeErrorÚtypeÚ__name__)ÚinputsÚoutÚinpÚx© r   ú7/tmp/pip-build-e66j97b2/torch/torch/utils/checkpoint.pyÚdetach_variable   s    


r   c             C   s    t dd„ | D ƒƒstjdƒ d S )Nc             s   s    | ]}t |tjƒr|jV  qd S )N)r   r   r	   r   )Ú.0r   r   r   r   ú	<genexpr>   s    z*check_backward_validity.<locals>.<genexpr>zBNone of the inputs have requires_grad=True. Gradients will be None)ÚanyÚwarningsÚwarn)r   r   r   r   Úcheck_backward_validity   s    r   c           
   G   sX   t tdd„ | D ƒƒƒ}g }x4|D ],}tjj|ƒ� |jtjjƒ ƒ W d Q R X q W ||fS )Nc             s   s(   | ] }t |tjƒr|jr|jƒ V  qd S )N)r   r   r	   Zis_cudaZ
get_device)r   Úargr   r   r   r   &   s    z$get_device_states.<locals>.<genexpr>)ÚlistÚsetr   ÚcudaÚdevicer
   Úget_rng_state)ÚargsÚfwd_gpu_devicesÚfwd_gpu_statesr!   r   r   r   Úget_device_states#   s    
r&   c          
   C   s@   x:t | |ƒD ],\}}tjj|ƒ� tjj|ƒ W d Q R X qW d S )N)Úzipr   r    r!   Úset_rng_state)ÚdevicesZstatesr!   Ústater   r   r   Úset_device_states1   s    r+   c               @   s$   e Zd Zedd„ ƒZedd„ ƒZdS )ÚCheckpointFunctionc          	   G   sp   t |ƒ || _|| _|rFtjƒ | _d| _tjjrFd| _t	|Ž \| _
| _| j|Ž  tjƒ � ||Ž }W d Q R X |S )NFT)r   Úrun_functionÚpreserve_rng_stater   r"   Úfwd_cpu_stateÚhad_cuda_in_fwdr    Ú_initializedr&   r$   r%   Zsave_for_backwardZno_grad)Úctxr-   r.   r#   Úoutputsr   r   r   Úforward9   s    


zCheckpointFunction.forwardc             G   sÒ   t jjƒ stdƒ‚| j}g }| jr.| jr.| j}t jj	|| jd��R | jrht j
| jƒ | jrht| j| jƒ t|ƒ}t jƒ � | j|Ž }W d Q R X W d Q R X t|t jƒrª|f}t jj||ƒ tdd„ |D ƒƒ}d| S )NzPCheckpointing is not compatible with .grad(), please use .backward() if possible)r)   Zenabledc             s   s$   | ]}t |tjƒr|jn|V  qd S )N)r   r   r	   Zgrad)r   r   r   r   r   r   d   s   z.CheckpointFunction.backward.<locals>.<genexpr>)NN)r   ÚautogradZ_is_checkpoint_validr   Zsaved_tensorsr.   r0   r$   ÚrandomZfork_rngr(   r/   r+   r%   r   Zenable_gradr-   r   r	   Úbackwardr   )r2   r#   r   Zrng_devicesZdetached_inputsr3   Zgradsr   r   r   r7   M   s(    


zCheckpointFunction.backwardN)r   Ú
__module__Ú__qualname__Ústaticmethodr4   r7   r   r   r   r   r,   7   s   r,   c             O   s>   |j ddƒ}|r,tddjdd„ |D ƒƒ ƒ‚tj| |f|žŽ S )ax  Checkpoint a model or part of the model

    Checkpointing works by trading compute for memory. Rather than storing all
    intermediate activations of the entire computation graph for computing
    backward, the checkpointed part does **not** save intermediate activations,
    and instead recomputes them in backward pass. It can be applied on any part
    of a model.

    Specifically, in the forward pass, :attr:`function` will run in
    :func:`torch.no_grad` manner, i.e., not storing the intermediate
    activations. Instead, the forward pass saves the inputs tuple and the
    :attr:`function` parameter. In the backwards pass, the saved inputs and
    :attr:`function` is retreived, and the forward pass is computed on
    :attr:`function` again, now tracking the intermediate activations, and then
    the gradients are calculated using these activation values.

    .. warning::
        Checkpointing doesn't work with :func:`torch.autograd.grad`, but only
        with :func:`torch.autograd.backward`.

    .. warning::
        If :attr:`function` invocation during backward does anything different
        than the one during forward, e.g., due to some global variable, the
        checkpointed version won't be equivalent, and unfortunately it can't be
        detected.

    .. warning:
        At least one of the inputs needs to have :code:`requires_grad=True` if
        grads are needed for model inputs, otherwise the checkpointed part of the
        model won't have gradients.

    Args:
        function: describes what to run in the forward pass of the model or
            part of the model. It should also know how to handle the inputs
            passed as the tuple. For example, in LSTM, if user passes
            ``(activation, hidden)``, :attr:`function` should correctly use the
            first input as ``activation`` and the second input as ``hidden``
        preserve_rng_state(bool, optional, default=True):  Omit stashing and restoring
            the RNG state during each checkpoint.
        args: tuple containing inputs to the :attr:`function`

    Returns:
        Output of running :attr:`function` on :attr:`*args`
    r.   TzUnexpected keyword arguments: ú,c             s   s   | ]
}|V  qd S )Nr   )r   r   r   r   r   r   ™   s    zcheckpoint.<locals>.<genexpr>)ÚpopÚ
ValueErrorÚjoinr,   Úapply)Úfunctionr#   ÚkwargsÚpreserver   r   r   Ú
checkpointi   s    .rC   c       	      O   sú   |j ddƒ}|r,tddjdd„ |D ƒƒ ƒ‚|s>tjdtƒ nt|ƒdkrVtjd	tƒ d
d„ }t| tj	j
ƒrxt| jƒ ƒ} t| ƒ| }d}xTtd||d  |ƒD ]<}|| d }t|||| ƒf|žd|iŽ}t|tƒsž|f}qžW ||d t| ƒd | ƒ|Ž S )a  A helper function for checkpointing sequential models.

    Sequential models execute a list of modules/functions in order
    (sequentially). Therefore, we can divide such a model in various segments
    and checkpoint each segment. All segments except the last will run in
    :func:`torch.no_grad` manner, i.e., not storing the intermediate
    activations. The inputs of each checkpointed segment will be saved for
    re-running the segment in the backward pass.

    See :func:`~torch.utils.checkpoint.checkpoint` on how checkpointing works.

    .. warning::
        Checkpointing doesn't work with :func:`torch.autograd.grad`, but only
        with :func:`torch.autograd.backward`.

    .. warning:
        At least one of the inputs needs to have :code:`requires_grad=True` if
        grads are needed for model inputs, otherwise the checkpointed part of the
        model won't have gradients.

    Args:
        functions: A :class:`torch.nn.Sequential` or the list of modules or
            functions (comprising the model) to run sequentially.
        segments: Number of chunks to create in the model
        inputs: tuple of Tensors that are inputs to :attr:`functions`
        preserve_rng_state(bool, optional, default=True):  Omit stashing and restoring
            the RNG state during each checkpoint.

    Returns:
        Output of running :attr:`functions` sequentially on :attr:`*inputs`

    Example:
        >>> model = nn.Sequential(...)
        >>> input_var = checkpoint_sequential(model, chunks, input_var)
    r.   TzUnexpected keyword arguments: r;   c             s   s   | ]
}|V  qd S )Nr   )r   r   r   r   r   r   È   s    z(checkpoint_sequential.<locals>.<genexpr>zjGiving no input to checkpoint_sequential has been deprecated, a TypeError will be raised after PyTorch 1.3é   zjmultiple inputs to checkpoint_sequential has been deprecated, a TypeError will be raised after PyTorch 1.3c                s   ‡ ‡‡fdd„}|S )Nc                 s@   x:t ˆˆ d ƒD ](}t| tƒr,ˆ| | Ž } qˆ| | ƒ} qW | S )NrD   )Úranger   r   )r   Új)ÚendÚ	functionsÚstartr   r   r4   ×   s
    
z<checkpoint_sequential.<locals>.run_function.<locals>.forwardr   )rI   rG   rH   r4   r   )rG   rH   rI   r   r-   Ö   s    z+checkpoint_sequential.<locals>.run_functionr   éÿÿÿÿ)r<   r=   r>   r   r   ÚDeprecationWarningÚlenr   r   ÚnnZ
Sequentialr   ÚchildrenrE   rC   r   )	rH   Úsegmentsr   rA   rB   r-   Zsegment_sizerG   rI   r   r   r   Úcheckpoint_sequential¡   s*    %


rP   )Ú
__future__r   r   r   r   r   r   r   r   r&   r+   r5   ZFunctionr,   rC   rP   r   r   r   r   Ú<module>   s   28