3
I4W^ 8  ã               @   sZ  d dl Z d dljZd dljjZd dlmZ d dlZd dl	Z	d dl
mZ G dd„ deƒZG dd„ deƒZG dd	„ d	ejeeƒZG d
d„ deƒZG dd„ deeejeeƒƒZdd„ Zdd„ ZG dd„ deƒZd.dd„Zdd„ Zd/dd„Zdd„ Zedd„ dd �Zed!d„ d"ed#�Zed$d„ d%d&d'�Zed(d„ d)d �Zed*d„ d+d„ d"d �ZG d,d-„ d-eƒZ dS )0é    N)Úwith_metaclass)ÚOrderedDictc               @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
Ú_ContextMethodMixinc             G   s
   || _ dS )aÕ  Saves given tensors for a future call to :func:`~Function.backward`.

        **This should be called at most once, and only from inside the**
        :func:`forward` **method.**

        Later, saved tensors can be accessed through the :attr:`saved_tensors`
        attribute. Before returning them to the user, a check is made to ensure
        they weren't used in any in-place operation that modified their content.

        Arguments can also be ``None``.
        N)Úto_save)ÚselfZtensors© r   ú8/tmp/pip-build-e66j97b2/torch/torch/autograd/function.pyÚsave_for_backward   s    z%_ContextMethodMixin.save_for_backwardc             G   s
   || _ dS )aÏ  Marks given tensors as modified in an in-place operation.

        **This should be called at most once, only from inside the**
        :func:`forward` **method, and all arguments should be inputs.**

        Every tensor that's been modified in-place in a call to :func:`forward`
        should be given to this function, to ensure correctness of our checks.
        It doesn't matter whether the function is called before or after
        modification.
        N)Údirty_tensors)r   Úargsr   r   r   Ú
mark_dirty   s    z_ContextMethodMixin.mark_dirtyc             G   s   t jdƒ d S )Nz†mark_shared_storage is deprecated. Tensors with shared storages are automatically tracked. Note that calls to `set_()` are not tracked)ÚwarningsÚwarn)r   Úpairsr   r   r   Úmark_shared_storage'   s    z'_ContextMethodMixin.mark_shared_storagec             G   s
   || _ dS )aR  Marks outputs as non-differentiable.

        **This should be called at most once, only from inside the**
        :func:`forward` **method, and all arguments should be outputs.**

        This will mark outputs as not requiring gradients, increasing the
        efficiency of backward computation. You still need to accept a gradient
        for each output in :meth:`~Function.backward`, but it's always going to
        be a zero tensor with the same shape as the shape of a corresponding
        output.

        This is used e.g. for indices returned from a max :class:`Function`.
        N)Únon_differentiable)r   r   r   r   r   Úmark_non_differentiable-   s    z+_ContextMethodMixin.mark_non_differentiableN)Ú__name__Ú
__module__Ú__qualname__r	   r   r   r   r   r   r   r   r   
   s   r   c               @   s   e Zd Zedd„ ƒZdS )Ú
_HookMixinc             C   s*   | d krt ƒ } tj| ƒ}|| |j< | |fS )N)r   ÚhooksZRemovableHandleÚid)Zbackward_hooksÚhookÚhandler   r   r   Ú_register_hook@   s
    

z_HookMixin._register_hookN)r   r   r   Ústaticmethodr   r   r   r   r   r   >   s   r   c               @   s   e Zd ZdZdd„ ZdS )ÚBackwardCFunctionFc             G   s   | j j| f|žŽ S )N)Ú_forward_clsÚbackward)r   r   r   r   r   ÚapplyL   s    zBackwardCFunction.applyN)r   r   r   Ú
_is_legacyr    r   r   r   r   r   I   s   r   c                   s    e Zd ZdZ‡ fdd„Z‡  ZS )ÚFunctionMetaaG  Function metaclass.

    This metaclass sets up the following properties:
        _is_legacy: True if forward is not defined as a static method.
        _backward_cls: The Function class corresponding to the differentiated
            version of this function (which is generated on the fly by this
            metaclass).
    c                sŒ   x:| j ƒ D ].}|jjdƒ}|d k	r
t|tƒp4t|tƒ}P q
W | | _|s\tt| ƒj	|||ƒS t
|d tfd| iƒ}|| _tt| ƒj	|||ƒS )NÚforwardZBackwardr   )ÚmroÚ__dict__ÚgetÚ
isinstancer   Úclassmethodr!   Úsuperr"   Ú__init__Útyper   Z_backward_cls)ÚclsÚnameÚbasesÚattrsZ	super_clsr#   Zhas_static_forwardZbackward_fn)Ú	__class__r   r   r*   Z   s    zFunctionMeta.__init__)r   r   r   Ú__doc__r*   Ú__classcell__r   r   )r0   r   r"   P   s   r"   c               @   s4   e Zd ZdZejjZdZe	dd„ ƒZ
e	dd„ ƒZdS )ÚFunctiona÷  Records operation history and defines formulas for differentiating ops.

    Every operation performed on :class:`Tensor` s creates a new function
    object, that performs the computation, and records that it happened.
    The history is retained in the form of a DAG of functions, with edges
    denoting data dependencies (``input <- output``). Then, when backward is
    called, the graph is processed in the topological ordering, by calling
    :func:`backward` methods of each :class:`Function` object, and passing
    returned gradients on to next :class:`Function` s.

    Normally, the only way users interact with functions is by creating
    subclasses and defining new operations. This is a recommended way of
    extending torch.autograd.

    Each function object is meant to be used only once (in the forward pass).

    Examples::

        >>> class Exp(Function):
        >>>
        >>>     @staticmethod
        >>>     def forward(ctx, i):
        >>>         result = i.exp()
        >>>         ctx.save_for_backward(result)
        >>>         return result
        >>>
        >>>     @staticmethod
        >>>     def backward(ctx, grad_output):
        >>>         result, = ctx.saved_tensors
        >>>         return grad_output * result
    Fc             O   s   t ‚dS )aP  Performs the operation.

        This function is to be overridden by all subclasses.

        It must accept a context ctx as the first argument, followed by any
        number of arguments (tensors or other types).

        The context can be used to store tensors that can be then retrieved
        during the backward pass.
        N)ÚNotImplementedError)Úctxr   Úkwargsr   r   r   r#   ”   s    zFunction.forwardc             G   s   t ‚dS )a|  Defines a formula for differentiating the operation.

        This function is to be overridden by all subclasses.

        It must accept a context :attr:`ctx` as the first argument, followed by
        as many outputs did :func:`forward` return, and it should return as many
        tensors, as there were inputs to :func:`forward`. Each argument is the
        gradient w.r.t the given output, and each returned value should be the
        gradient w.r.t. the corresponding input.

        The context can be used to retrieve tensors saved during the forward
        pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple
        of booleans representing whether each input needs gradient. E.g.,
        :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the
        first input to :func:`forward` needs gradient computated w.r.t. the
        output.
        N)r4   )r5   Zgrad_outputsr   r   r   r   ¢   s    zFunction.backwardN)r   r   r   r1   Ú_CÚ_FunctionBaseÚ_do_forwardÚ__call__Úis_traceabler   r#   r   r   r   r   r   r3   m   s
   r3   c                s   t jˆ ƒ‡ fdd„ƒ}|S )Nc                sŠ   t jƒ � ˆ| f|žŽ }W d Q R X t jƒ s.|S tdd„ |D ƒƒ}|sH|S t|tƒsX|f}t jjjdt	|ƒƒ}dd„ ‰ |‡ fdd„|D ƒŽ S )Nc             s   s    | ]}t |tjƒo|jV  qd S )N)r'   ÚtorchÚTensorÚrequires_grad)Ú.0Úargr   r   r   ú	<genexpr>Ë   s   z7once_differentiable.<locals>.wrapper.<locals>.<genexpr>sQ   trying to differentiate twice a function that was markedwith @once_differentiablec             S   s   | d k	r| j ƒ } d| _| S )NT)Údetachr>   )Úvarr   r   r   Úfake_requires_gradÚ   s    z@once_differentiable.<locals>.wrapper.<locals>.fake_requires_gradc                s   g | ]}ˆ |ƒ‘qS r   r   )r?   Úv)rD   r   r   ú
<listcomp>à   s    z8once_differentiable.<locals>.wrapper.<locals>.<listcomp>)
r<   Zno_gradZis_grad_enabledÚanyr'   Útupler7   Z
_functionsZDelayedErrorÚlen)r5   r   Úoutputsr>   Zerr_fn)Úfn)rD   r   Úwrapperº   s    



z$once_differentiable.<locals>.wrapper)Ú	functoolsÚwraps)rK   rL   r   )rK   r   Úonce_differentiable¸   s    'rO   c             C   s
   d| _ | S )aÙ  Marks Function as traceable for the JIT.

    Traceable functions have additional restrictions - they can't pass any
    data-dependent values to backward (e.g. Prod passes the output, which makes
    it non-traceable), and their backward should be implemented entirely in terms
    of operations on autograd Tensors in all cases.

    DON'T USE THIS DECORATOR. IT IS FOR INTERNAL USE ONLY AND SHOULD BE HANDLED WITH
    CARE (or can give incorrect results otherwise).
    T)r;   )Zfn_clsr   r   r   Ú	traceableä   s    rP   c                   s   e Zd Zd‡ fdd„	Z‡  ZS )ÚInplaceFunctionFc                s   t t| ƒjƒ  || _d S )N)r)   rQ   r*   Úinplace)r   rR   )r0   r   r   r*   õ   s    zInplaceFunction.__init__)F)r   r   r   r*   r2   r   r   )r0   r   rQ   ó   s   rQ   c                s   ‡ ‡‡‡fdd„‰ ˆ S )Nc                sŒ   ˆˆ ƒrˆˆ ƒS ˆ d krd S t ˆ ttfƒrDtˆ ƒ‡fdd„ˆ D ƒƒS t ˆ tƒrb‡‡ fdd„ˆ D ƒS tdtjˆ ƒ ˆr€dˆ d nd ƒ‚d S )	Nc             3   s   | ]}ˆ |ƒV  qd S )Nr   )r?   Úx)Ú_mapr   r   rA     s    z,_nested_map.<locals>._map.<locals>.<genexpr>c                s   i | ]}ˆ ˆ| ƒ|“qS r   r   )r?   rS   )rT   Úobjr   r   ú
<dictcomp>  s    z-_nested_map.<locals>._map.<locals>.<dictcomp>zAAuto nesting doesn't know how to process an input object of type z. Accepted types: z, or lists/tuples of themÚ )r'   ÚlistrH   r+   ÚdictÚ
ValueErrorr<   Útypename)rU   )rT   Ú	conditionÚcondition_msgrK   )rU   r   rT   û   s    
z_nested_map.<locals>._mapr   )r\   rK   r]   r   )rT   r\   r]   rK   r   Ú_nested_mapú   s    r^   c             C   s   t | dƒr| jƒ S | S )NÚ_jit_unwrap)Úhasattrr_   )rU   r   r   r   Ú_jit_unwrap_structured  s    
ra   Fc                s   ‡ ‡‡‡‡fdd„‰ ˆ S )Nc             3   sÎ   ˆd k	rˆ| ƒ} ˆ| ƒr | V  nª| d kr,d S t | ttfƒrbxŽ| D ]}xˆ |ƒD ]
}|V  qNW q@W nht | tƒr˜x\| jƒ D ]}xˆ |ƒD ]
}|V  q„W qvW n2ˆr¤| V  n&tdtj| ƒ ˆrÂdˆ d nd ƒ‚d S )NzAAuto nesting doesn't know how to process an input object of type z. Accepted types: z, or lists/tuples of themrW   )r'   rX   rH   rY   ÚvaluesrZ   r<   r[   )rU   ÚorC   )Ú_iterÚallow_unknownr\   r]   Ú
conversionr   r   rd     s&    

z_iter_filter.<locals>._iterr   )r\   re   r]   rf   r   )rd   re   r\   r]   rf   r   Ú_iter_filter  s    rg   c                s   ‡ fdd„‰ ˆ | |ƒd S )Nc                s„   g }t |dƒr|j| ƒS t|ttfƒs:| d | dd … fS x8|D ]0}|d krX|j|ƒ q@ˆ | |ƒ\}} |j|ƒ q@W t|ƒ|ƒ| fS )NÚ	_jit_wrapr   é   )r`   rh   r'   rX   rH   Úappendr+   )ÚinputÚprotoÚresÚeZres_e)Úunflatten_helperr   r   ro   5  s    


z$_unflatten.<locals>.unflatten_helperr   r   )rk   rl   r   )ro   r   Ú
_unflatten2  s    rp   c             C   s   | d kpt | tjjƒS )N)r'   r<   r7   ÚValue)rc   r   r   r   Ú<lambda>F  s    rr   zjit's Values or None)r]   c             C   s   t | tjƒS )N)r'   r<   r=   )rS   r   r   r   rr   H  s    ZTensors)r]   rf   c             C   s   t | tjƒS )N)r'   r<   r=   )rS   r   r   r   rr   J  s    TzTensors (permissive))re   r]   c             C   s   | d kpt | tjƒS )N)r'   r<   r=   )rc   r   r   r   rr   M  s    zTensors or Nonec             C   s   t | tjƒS )N)r'   r<   r=   )rS   r   r   r   rr   O  s    c             C   s   | j S )N)Údata)rc   r   r   r   rr   O  s    c                   st   e Zd Z‡ fdd„Z‡ fdd„Zdd„ ZeZdd„ Zd	d
„ Ze	‡ fdd„ƒZ
dd„ Zdd„ Zdd„ Zdd„ Z‡  ZS )ÚNestedIOFunctionc                s8   || _ tt|ƒƒ}tt| ƒj|Ž }| j}t|| jƒ}|S )N)Ú_nested_inputrH   Ú_iter_tensorsr)   rt   r9   Ú_nested_outputrp   )r   rk   Z
flat_inputZflat_outputZnested_outputÚnested_tensors)r0   r   r   r9   U  s    zNestedIOFunction._do_forwardc                s(   || _ tt| ƒj||ƒ}|s$| `| `|S )N)Úretain_variablesr)   rt   Ú_do_backwardrw   Ú_to_save_nested)r   Ú	gradientsry   Úresult)r0   r   r   rz   ]  s    zNestedIOFunction._do_backwardc             G   s"   t || jƒ}| j|Ž }tt|ƒƒS )N)rp   rw   Úbackward_extendedrH   Ú_iter_None_tensors)r   r|   Znested_gradientsr}   r   r   r   r   e  s    
zNestedIOFunction.backwardc             G   s*   t | jƒ}| j|Ž }| `|| _tt|ƒƒS )N)Ú_map_tensor_dataru   Úforward_extendedrw   rH   rv   )r   r   rx   r}   r   r   r   r#   l  s
    

zNestedIOFunction.forwardc             G   s   t t|ƒƒ| _|| _d S )N)rH   rv   r   r{   )r   r   r   r   r   r	   s  s    z"NestedIOFunction.save_for_backwardc                s   t t| ƒj}t|| jƒS )N)r)   rt   Úsaved_tensorsrp   r{   )r   Zflat_tensors)r0   r   r   r‚   w  s    zNestedIOFunction.saved_tensorsc             O   s   t t||fƒƒ| _d S )N)rH   rv   r
   )r   r   r6   r   r   r   r   |  s    zNestedIOFunction.mark_dirtyc             O   s   t t||fƒƒ| _d S )N)rH   rv   r   )r   r   r6   r   r   r   r     s    z(NestedIOFunction.mark_non_differentiablec             G   s   t ‚d S )N)r4   )r   rk   r   r   r   r�   ‚  s    z!NestedIOFunction.forward_extendedc             G   s   t ‚d S )N)r4   )r   Zgrad_outputr   r   r   r~   …  s    z"NestedIOFunction.backward_extended)r   r   r   r9   rz   r   r:   r#   r	   Úpropertyr‚   r   r   r�   r~   r2   r   r   )r0   r   rt   S  s   rt   )N)FNN)!r<   Ztorch._Cr7   Ztorch.utils.hooksÚutilsr   Z
torch._sixr   rM   r   Úcollectionsr   Úobjectr   r   r8   r   r+   r"   r3   rO   rP   rQ   r^   ra   rg   rp   Z_iter_jit_valuesrv   Z_iter_tensors_permissiver   r€   rt   r   r   r   r   Ú<module>   s>   
4K,
 

