3
I4W^¿´  ã               @   sv   d dl mZmZ d dlZd dlZd dlZddlmZ d dlj	j
Z
G dd„ deddd	gƒƒZd
d„ ZG dd„ deƒZdS )é    )ÚOrderedDictÚ
namedtupleNé   )Ú	Parameterc                   s    e Zd Z‡ fdd„ZeZ‡  ZS )Ú_IncompatibleKeysc                s"   | j  r| j rdS tt| ƒjƒ S )Nz<All keys matched successfully>)Úmissing_keysÚunexpected_keysÚsuperr   Ú__repr__)Úself)Ú	__class__© ú8/tmp/pip-build-e66j97b2/torch/torch/nn/modules/module.pyr
      s    z_IncompatibleKeys.__repr__)Ú__name__Ú
__module__Ú__qualname__r
   Ú__str__Ú__classcell__r   r   )r   r   r   
   s   r   ZIncompatibleKeysr   r   c                sP   | j dƒ}t|ƒdkr| S |jdƒ}‡ fdd„|D ƒ}dj|ƒ}|d | }|S )NÚ
é   r   c                s   g | ]}ˆ d  | ‘qS )ú r   )Ú.0Úline)Ú	numSpacesr   r   ú
<listcomp>   s    z_addindent.<locals>.<listcomp>)ÚsplitÚlenÚpopÚjoin)Zs_r   ÚsÚfirstr   )r   r   Ú
_addindent   s    


r!   c               @   s®  e Zd ZdZdZdZdd„ Zdd„ Zdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdgdd„Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ Zd)d*„ Zd+d,„ Zd-d.„ Zd/d0„ Zd1d2„ Zd3d4„ Zd5d6„ Zd7d8„ Zdhd:d;„Z d<d=„ Z!d>d?„ Z"didAdB„Z#djdCdD„Z$dkdEdF„Z%dldGdH„Z&dmdIdJ„Z'dndKdL„Z(dMdN„ Z)dOdP„ Z*dQdR„ Z+dodSdT„Z,dpdUdV„Z-dWdX„ Z.dqdYdZ„Z/d[d\„ Z0d]d^„ Z1d_d`„ Z2dadb„ Z3dcdd„ Z4dedf„ Z5dS )rÚModulea  Base class for all neural network modules.

    Your models should also subclass this class.

    Modules can also contain other Modules, allowing to nest them in
    a tree structure. You can assign the submodules as regular attributes::

        import torch.nn as nn
        import torch.nn.functional as F

        class Model(nn.Module):
            def __init__(self):
                super(Model, self).__init__()
                self.conv1 = nn.Conv2d(1, 20, 5)
                self.conv2 = nn.Conv2d(20, 20, 5)

            def forward(self, x):
                x = F.relu(self.conv1(x))
                return F.relu(self.conv2(x))

    Submodules assigned in this way will be registered, and will have their
    parameters converted too when you call :meth:`to`, etc.
    Fr   c             C   sV   t jjdƒ d| _tƒ | _tƒ | _tƒ | _tƒ | _tƒ | _	tƒ | _
tƒ | _tƒ | _dS )z_
        Initializes internal Module state, shared by both nn.Module and ScriptModule.
        zpython.nn_moduleTN)ÚtorchÚ_CZ_log_api_usage_onceÚtrainingr   Ú_parametersÚ_buffersÚ_backward_hooksÚ_forward_hooksÚ_forward_pre_hooksÚ_state_dict_hooksÚ_load_state_dict_pre_hooksÚ_modules)r   r   r   r   Ú__init__F   s    zModule.__init__c             G   s   t ‚dS )a¨  Defines the computation performed at every call.

        Should be overridden by all subclasses.

        .. note::
            Although the recipe for forward pass needs to be defined within
            this function, one should call the :class:`Module` instance afterwards
            instead of this since the former takes care of running the
            registered hooks while the latter silently ignores them.
        N)ÚNotImplementedError)r   Úinputr   r   r   ÚforwardV   s    zModule.forwardc             C   s¼   d| j krtdƒ‚n¤t|tjjƒs8tdjtj|ƒƒƒ‚n€d|krJt	dƒ‚nn|dkr\t	dƒ‚n\t
| |ƒr€|| jkr€t	dj|ƒƒ‚n8|d	k	r®t|tjƒ r®td
jtj|ƒ|ƒƒ‚n
|| j|< d	S )ax  Adds a persistent buffer to the module.

        This is typically used to register a buffer that should not to be
        considered a model parameter. For example, BatchNorm's ``running_mean``
        is not a parameter, but is part of the persistent state.

        Buffers can be accessed as attributes using given names.

        Args:
            name (string): name of the buffer. The buffer can be accessed
                from this module using the given name
            tensor (Tensor): buffer to be registered.

        Example::

            >>> self.register_buffer('running_mean', torch.zeros(num_features))

        r'   z2cannot assign buffer before Module.__init__() callz&buffer name should be a string. Got {}Ú.zbuffer name can't contain "."Ú z$buffer name can't be empty string ""zattribute '{}' already existsNzHcannot assign '{}' object to buffer '{}' (torch Tensor or None required))Ú__dict__ÚAttributeErrorÚ
isinstancer#   Ú_sixÚstring_classesÚ	TypeErrorÚformatÚtypenameÚKeyErrorÚhasattrr'   ÚTensor)r   ÚnameÚtensorr   r   r   Úregister_bufferc   s     


zModule.register_bufferc             C   sØ   d| j krtdƒ‚njt|tjjƒs8tdjtj|ƒƒƒ‚nFd|krJt	dƒ‚n4|dkr\t	dƒ‚n"t
| |ƒr~|| jkr~t	dj|ƒƒ‚|d	kr’d	| j|< nBt|tƒs´td
jtj|ƒ|ƒƒ‚n |jrÊtdj|ƒƒ‚n
|| j|< d	S )aJ  Adds a parameter to the module.

        The parameter can be accessed as an attribute using given name.

        Args:
            name (string): name of the parameter. The parameter can be accessed
                from this module using the given name
            param (Parameter): parameter to be added to the module.
        r&   z5cannot assign parameter before Module.__init__() callz)parameter name should be a string. Got {}r2   z parameter name can't contain "."r3   z'parameter name can't be empty string ""zattribute '{}' already existsNzQcannot assign '{}' object to parameter '{}' (torch.nn.Parameter or None required)z»Cannot assign non-leaf Tensor to parameter '{0}'. Model parameters must be created explicitly. To express '{0}' as a function of another Tensor, compute the value in the forward() method.)r4   r5   r6   r#   r7   r8   r9   r:   r;   r<   r=   r&   r   Úgrad_fnÚ
ValueError)r   r?   Úparamr   r   r   Úregister_parameter‰   s,    





zModule.register_parameterc             C   s¢   t |tƒ r*|dk	r*tdjtj|ƒƒƒ‚njt |tjjƒsNtdjtj|ƒƒƒ‚nFt| |ƒrr|| j	krrt
dj|ƒƒ‚n"d|kr„t
dƒ‚n|dkr”t
dƒ‚|| j	|< dS )	a]  Adds a child module to the current module.

        The module can be accessed as an attribute using the given name.

        Args:
            name (string): name of the child module. The child module can be
                accessed from this module using the given name
            module (Module): child module to be added to the module.
        Nz{} is not a Module subclassz&module name should be a string. Got {}zattribute '{}' already existsr2   zmodule name can't contain "."r3   z$module name can't be empty string "")r6   r"   r9   r:   r#   r;   r7   r8   r=   r-   r<   )r   r?   Úmoduler   r   r   Ú
add_module°   s    

zModule.add_modulec       
      C   s:  x| j ƒ D ]}|j|ƒ q
W dd„ }xÞ| jjƒ D ]Ð\}}|d k	r0tjƒ � ||ƒ}W d Q R X |||ƒ}|rr||_n*t|tƒs€t	‚|j
sŠt	‚t||jƒ| j|< |jd k	r0tjƒ � ||jƒ}W d Q R X ||j|ƒ}|rÞ||j_q0|jj
sêt	‚|j|jjƒ| j| _q0W x0| jjƒ D ]"\}}	|	d k	�r||	ƒ| j|< �qW | S )Nc             S   s    t j| |ƒrt jjƒ  S dS d S )NF)r#   Z!_has_compatible_shallow_copy_typeÚ
__future__Z)get_overwrite_module_params_on_conversion)r@   Ztensor_appliedr   r   r   Úcompute_should_use_set_dataÌ   s    	z2Module._apply.<locals>.compute_should_use_set_data)ÚchildrenÚ_applyr&   Úitemsr#   Zno_gradÚdatar6   r   ÚAssertionErrorZis_leafÚrequires_gradÚgradÚrequires_grad_r'   )
r   ÚfnrF   rI   ÚkeyrD   Zparam_appliedZshould_use_set_dataZgrad_appliedÚbufr   r   r   rK   È   s2    






zModule._applyc             C   s(   x| j ƒ D ]}|j|ƒ q
W || ƒ | S )aH  Applies ``fn`` recursively to every submodule (as returned by ``.children()``)
        as well as self. Typical use includes initializing the parameters of a model
        (see also :ref:`torch-nn-init`).

        Args:
            fn (:class:`Module` -> None): function to be applied to each submodule

        Returns:
            Module: self

        Example::

            >>> def init_weights(m):
            >>>     print(m)
            >>>     if type(m) == nn.Linear:
            >>>         m.weight.data.fill_(1.0)
            >>>         print(m.weight)
            >>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))
            >>> net.apply(init_weights)
            Linear(in_features=2, out_features=2, bias=True)
            Parameter containing:
            tensor([[ 1.,  1.],
                    [ 1.,  1.]])
            Linear(in_features=2, out_features=2, bias=True)
            Parameter containing:
            tensor([[ 1.,  1.],
                    [ 1.,  1.]])
            Sequential(
              (0): Linear(in_features=2, out_features=2, bias=True)
              (1): Linear(in_features=2, out_features=2, bias=True)
            )
            Sequential(
              (0): Linear(in_features=2, out_features=2, bias=True)
              (1): Linear(in_features=2, out_features=2, bias=True)
            )
        )rJ   Úapply)r   rR   rF   r   r   r   rU   ù   s    %zModule.applyNc                s   | j ‡ fdd„ƒS )a²  Moves all model parameters and buffers to the GPU.

        This also makes associated parameters and buffers different objects. So
        it should be called before constructing optimizer if the module will
        live on GPU while being optimized.

        Arguments:
            device (int, optional): if specified, all parameters will be
                copied to that device

        Returns:
            Module: self
        c                s
   | j ˆ ƒS )N)Úcuda)Út)Údevicer   r   Ú<lambda>1  s    zModule.cuda.<locals>.<lambda>)rK   )r   rX   r   )rX   r   rV   #  s    zModule.cudac             C   s   | j dd„ ƒS )zfMoves all model parameters and buffers to the CPU.

        Returns:
            Module: self
        c             S   s   | j ƒ S )N)Úcpu)rW   r   r   r   rY   9  s    zModule.cpu.<locals>.<lambda>)rK   )r   r   r   r   rZ   3  s    z
Module.cpuc                s   | j ‡ fdd„ƒS )zµCasts all parameters and buffers to :attr:`dst_type`.

        Arguments:
            dst_type (type or string): the desired type

        Returns:
            Module: self
        c                s
   | j ˆ ƒS )N)Útype)rW   )Údst_typer   r   rY   D  s    zModule.type.<locals>.<lambda>)rK   )r   r\   r   )r\   r   r[   ;  s    	zModule.typec             C   s   | j dd„ ƒS )zvCasts all floating point parameters and buffers to float datatype.

        Returns:
            Module: self
        c             S   s   | j ƒ r| jƒ S | S )N)Úis_floating_pointÚfloat)rW   r   r   r   rY   L  s    zModule.float.<locals>.<lambda>)rK   )r   r   r   r   r^   F  s    zModule.floatc             C   s   | j dd„ ƒS )z{Casts all floating point parameters and buffers to ``double`` datatype.

        Returns:
            Module: self
        c             S   s   | j ƒ r| jƒ S | S )N)r]   Údouble)rW   r   r   r   rY   T  s    zModule.double.<locals>.<lambda>)rK   )r   r   r   r   r_   N  s    zModule.doublec             C   s   | j dd„ ƒS )zyCasts all floating point parameters and buffers to ``half`` datatype.

        Returns:
            Module: self
        c             S   s   | j ƒ r| jƒ S | S )N)r]   Úhalf)rW   r   r   r   rY   \  s    zModule.half.<locals>.<lambda>)rK   )r   r   r   r   r`   V  s    zModule.halfc                sL   t jjj||Ž\‰ ‰‰ˆdk	r2ˆjs2tdjˆƒƒ‚‡ ‡‡fdd„}| j|ƒS )a
  Moves and/or casts the parameters and buffers.

        This can be called as

        .. function:: to(device=None, dtype=None, non_blocking=False)

        .. function:: to(dtype, non_blocking=False)

        .. function:: to(tensor, non_blocking=False)

        Its signature is similar to :meth:`torch.Tensor.to`, but only accepts
        floating point desired :attr:`dtype` s. In addition, this method will
        only cast the floating point parameters and buffers to :attr:`dtype`
        (if given). The integral parameters and buffers will be moved
        :attr:`device`, if that is given, but with dtypes unchanged. When
        :attr:`non_blocking` is set, it tries to convert/move asynchronously
        with respect to the host if possible, e.g., moving CPU Tensors with
        pinned memory to CUDA devices.

        See below for examples.

        .. note::
            This method modifies the module in-place.

        Args:
            device (:class:`torch.device`): the desired device of the parameters
                and buffers in this module
            dtype (:class:`torch.dtype`): the desired floating point type of
                the floating point parameters and buffers in this module
            tensor (torch.Tensor): Tensor whose dtype and device are the desired
                dtype and device for all parameters and buffers in this module

        Returns:
            Module: self

        Example::

            >>> linear = nn.Linear(2, 2)
            >>> linear.weight
            Parameter containing:
            tensor([[ 0.1913, -0.3420],
                    [-0.5113, -0.2325]])
            >>> linear.to(torch.double)
            Linear(in_features=2, out_features=2, bias=True)
            >>> linear.weight
            Parameter containing:
            tensor([[ 0.1913, -0.3420],
                    [-0.5113, -0.2325]], dtype=torch.float64)
            >>> gpu1 = torch.device("cuda:1")
            >>> linear.to(gpu1, dtype=torch.half, non_blocking=True)
            Linear(in_features=2, out_features=2, bias=True)
            >>> linear.weight
            Parameter containing:
            tensor([[ 0.1914, -0.3420],
                    [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1')
            >>> cpu = torch.device("cpu")
            >>> linear.to(cpu)
            Linear(in_features=2, out_features=2, bias=True)
            >>> linear.weight
            Parameter containing:
            tensor([[ 0.1914, -0.3420],
                    [-0.5112, -0.2324]], dtype=torch.float16)

        NzInn.Module.to only accepts floating point dtypes, but got desired dtype={}c                s   | j ˆ | jƒ rˆnd ˆƒS )N)Útor]   )rW   )rX   ÚdtypeÚnon_blockingr   r   Úconvert§  s    zModule.to.<locals>.convert)r#   r$   Z_nnZ	_parse_tor]   r9   r:   rK   )r   ÚargsÚkwargsrd   r   )rX   rb   rc   r   ra   ^  s    Bz	Module.toc             C   s   t j| jƒ}|| j|j< |S )a  Registers a backward hook on the module.

        The hook will be called every time the gradients with respect to module
        inputs are computed. The hook should have the following signature::

            hook(module, grad_input, grad_output) -> Tensor or None

        The :attr:`grad_input` and :attr:`grad_output` may be tuples if the
        module has multiple inputs or outputs. The hook should not modify its
        arguments, but it can optionally return a new gradient with respect to
        input that will be used in place of :attr:`grad_input` in subsequent
        computations.

        Returns:
            :class:`torch.utils.hooks.RemovableHandle`:
                a handle that can be used to remove the added hook by calling
                ``handle.remove()``

        .. warning ::

            The current implementation will not have the presented behavior
            for complex :class:`Module` that perform many operations.
            In some failure cases, :attr:`grad_input` and :attr:`grad_output` will only
            contain the gradients for a subset of the inputs and outputs.
            For such :class:`Module`, you should use :func:`torch.Tensor.register_hook`
            directly on a specific input or output to get the required gradients.

        )ÚhooksÚRemovableHandler(   Úid)r   ÚhookÚhandler   r   r   Úregister_backward_hook¬  s    zModule.register_backward_hookc             C   s   t j| jƒ}|| j|j< |S )a“  Registers a forward pre-hook on the module.

        The hook will be called every time before :func:`forward` is invoked.
        It should have the following signature::

            hook(module, input) -> None or modified input

        The hook can modify the input. User can either return a tuple or a
        single modified value in the hook. We will wrap the value into a tuple
        if a single value is returned(unless that value is already a tuple).

        Returns:
            :class:`torch.utils.hooks.RemovableHandle`:
                a handle that can be used to remove the added hook by calling
                ``handle.remove()``
        )rg   rh   r*   ri   )r   rj   rk   r   r   r   Úregister_forward_pre_hookÍ  s    z Module.register_forward_pre_hookc             C   s   t j| jƒ}|| j|j< |S )aq  Registers a forward hook on the module.

        The hook will be called every time after :func:`forward` has computed an output.
        It should have the following signature::

            hook(module, input, output) -> None or modified output

        The hook can modify the output. It can modify the input inplace but
        it will not have effect on forward since this is called after
        :func:`forward` is called.

        Returns:
            :class:`torch.utils.hooks.RemovableHandle`:
                a handle that can be used to remove the added hook by calling
                ``handle.remove()``
        )rg   rh   r)   ri   )r   rj   rk   r   r   r   Úregister_forward_hookâ  s    zModule.register_forward_hookc             C   s:   |j s
d S |j d }x |jƒ D ]\}}|| kr|S qW d S )Nr   éÿÿÿÿ)Ú_traced_module_stackÚnamed_children)r   Útracing_staterF   r?   Úchildr   r   r   Ú_tracing_name÷  s    
zModule._tracing_namec          
   O   s”   t jjƒ }|s| j||ŽS t|dƒs*g |_| j|ƒ}|rP|jd| jƒ |f ƒ n|j| jƒ ƒ |jj	| ƒ z| j||Ž}W d |j
ƒ  |jjƒ  X |S )Nrp   z%s[%s])r#   r$   Ú_get_tracing_stater1   r=   rp   rt   Z
push_scopeÚ	_get_nameÚappendZ	pop_scoper   )r   r0   rf   rr   r?   Úresultr   r   r   Ú_slow_forward   s    


zModule._slow_forwardc       	      O   s$  x8| j jƒ D ]*}|| |ƒ}|d k	rt|tƒs2|f}|}qW tjjƒ rR| j||Ž}n| j||Ž}x*| j	jƒ D ]}|| ||ƒ}|d k	rj|}qjW t
| jƒdk�r |}x:t|tjƒsØt|tƒrÎtdd„ |jƒ D ƒƒ}q |d }q W |j}|d k	�r x4| jjƒ D ]&}tj|| ƒ}tj||ƒ |j|ƒ qöW |S )Nr   c             s   s   | ]}t |tjƒr|V  qd S )N)r6   r#   r>   )r   Úvr   r   r   ú	<genexpr>&  s    z"Module.__call__.<locals>.<genexpr>)r*   Úvaluesr6   Útupler#   r$   ru   ry   r1   r)   r   r(   r>   ÚdictÚnextrB   Ú	functoolsÚpartialÚupdate_wrapperÚregister_hook)	r   r0   rf   rj   rx   Úhook_resultÚvarrB   Úwrapperr   r   r   Ú__call__  s4    




zModule.__call__c             C   sF   | j j|ƒ d| j krtƒ | _d| j kr0tƒ | _d| j krBtƒ | _d S )Nr*   r+   r,   )r4   Úupdater   r*   r+   r,   )r   Ústater   r   r   Ú__setstate__1  s    


zModule.__setstate__c             C   s†   d| j kr$| j d }||kr$|| S d| j krH| j d }||krH|| S d| j krl| j d }||krl|| S tdjt| ƒj|ƒƒ‚d S )Nr&   r'   r-   z!'{}' object has no attribute '{}')r4   r5   r:   r[   r   )r   r?   r&   r'   Úmodulesr   r   r   Ú__getattr__;  s    





zModule.__getattr__c                sz  ‡ fdd„}| j jdƒ}t|tƒrT|d kr2tdƒ‚|| j | j| jƒ | jˆ |ƒ �n"|d k	r�ˆ |kr�|d k	r‚tdj	t
j|ƒˆ ƒƒ‚| jˆ |ƒ næ| j jdƒ}t|tƒrÒ|d kr¶tdƒ‚|| j | j| jƒ ||ˆ < n¤|d k	oàˆ |k�r|d k	�rtdj	t
j|ƒˆ ƒƒ‚||ˆ < nh| j jd	ƒ}|d k	�rhˆ |k�rh|d k	�r^t|t
jƒ �r^td
j	t
j|ƒˆ ƒƒ‚||ˆ < ntj| ˆ |ƒ d S )Nc                 s    x| D ]}ˆ |kr|ˆ = qW d S )Nr   )ZdictsÚd)r?   r   r   Úremove_fromL  s    
z'Module.__setattr__.<locals>.remove_fromr&   z6cannot assign parameters before Module.__init__() callzJcannot assign '{}' as parameter '{}' (torch.nn.Parameter or None expected)r-   z2cannot assign module before Module.__init__() callzJcannot assign '{}' as child module '{}' (torch.nn.Module or None expected)r'   zAcannot assign '{}' as buffer '{}' (torch.Tensor or None expected))r4   Úgetr6   r   r5   r'   r-   rE   r9   r:   r#   r;   r"   r&   r>   ÚobjectÚ__setattr__)r   r?   ÚvaluerŽ   Úparamsr‹   Úbuffersr   )r?   r   r‘   K  s@    





zModule.__setattr__c             C   sL   || j kr| j |= n4|| jkr(| j|= n || jkr<| j|= ntj| |ƒ d S )N)r&   r'   r-   r�   Ú__delattr__)r   r?   r   r   r   r•   w  s    





zModule.__delattr__c             C   s   t j| jƒ}|| j|j< |S )aV  These hooks will be called with arguments: `self`, `state_dict`,
        `prefix`, `local_metadata`, after the `state_dict` of `self` is set.
        Note that only parameters and buffers of `self` or its children are
        guaranteed to exist in `state_dict`. The hooks may modify `state_dict`
        inplace or return a new one.
        )rg   rh   r+   ri   )r   rj   rk   r   r   r   Ú_register_state_dict_hook�  s    z Module._register_state_dict_hookc             C   sp   x4| j jƒ D ]&\}}|dk	r|r$|n|j||| < qW x4| jjƒ D ]&\}}|dk	rB|rZ|n|j||| < qBW dS )a  Saves module state to `destination` dictionary, containing a state
        of the module, but not its descendants. This is called on every
        submodule in :meth:`~torch.nn.Module.state_dict`.

        In rare cases, subclasses can achieve class-specific behavior by
        overriding this method with custom logic.

        Arguments:
            destination (dict): a dict where state will be stored
            prefix (str): the prefix for parameters and buffers used in this
                module
        N)r&   rL   rM   r'   )r   ÚdestinationÚprefixÚ	keep_varsr?   rD   rT   r   r   r   Ú_save_to_state_dictŒ  s    zModule._save_to_state_dictr3   c       	      C   s¬   |dkrt ƒ }t ƒ |_t| jd� |j|dd… < }| j|||ƒ x6| jjƒ D ](\}}|dk	rN|j||| d |d� qNW x,| jj	ƒ D ]}|| |||ƒ}|dk	r†|}q†W |S )a�  Returns a dictionary containing a whole state of the module.

        Both parameters and persistent buffers (e.g. running averages) are
        included. Keys are corresponding parameter and buffer names.

        Returns:
            dict:
                a dictionary containing a whole state of the module

        Example::

            >>> module.state_dict().keys()
            ['bias', 'weight']

        N)Úversionr   r2   )r™   ro   )
r   Ú	_metadatar~   Ú_versionrš   r-   rL   Ú
state_dictr+   r|   )	r   r—   r˜   r™   Úlocal_metadatar?   rF   rj   r„   r   r   r   rž      s    zModule.state_dictc             C   s   t j| jƒ}|| j|j< |S )a#  These hooks will be called with arguments: `state_dict`, `prefix`,
        `local_metadata`, `strict`, `missing_keys`, `unexpected_keys`,
        `error_msgs`, before loading `state_dict` into `self`. These arguments
        are exactly the same as those of `_load_from_state_dict`.
        )rg   rh   r,   ri   )r   rj   rk   r   r   r   Ú"_register_load_state_dict_pre_hook¾  s    z)Module._register_load_state_dict_pre_hookc             C   s”  x&| j jƒ D ]}||||||||ƒ qW tj| jjƒ | jjƒ ƒ}	dd„ |	D ƒ}
xÚ|
jƒ D ]Î\}}|| }||k�r|| }t|jƒdkržt|jƒdkrž|d }|j|jkrÄ|j	dj
||j|jƒƒ qXt|tƒrÔ|j}y|j|ƒ W n2 tk
�r   |j	dj
||jƒ |jƒ ƒƒ Y nX qX|rX|j	|ƒ qXW |�r�x^|jƒ D ]R}|j|ƒ�r:|t|ƒd… }|jddƒd }|| jk�r:||
k�r:|j	|ƒ �q:W dS )	a~  Copies parameters and buffers from :attr:`state_dict` into only
        this module, but not its descendants. This is called on every submodule
        in :meth:`~torch.nn.Module.load_state_dict`. Metadata saved for this
        module in input :attr:`state_dict` is provided as :attr:`local_metadata`.
        For state dicts without metadata, :attr:`local_metadata` is empty.
        Subclasses can achieve class-specific backward compatible loading using
        the version number at `local_metadata.get("version", None)`.

        .. note::
            :attr:`state_dict` is not the same object as the input
            :attr:`state_dict` to :meth:`~torch.nn.Module.load_state_dict`. So
            it can be modified.

        Arguments:
            state_dict (dict): a dict containing parameters and
                persistent buffers.
            prefix (str): the prefix for parameters and buffers used in this
                module
            local_metadata (dict): a dict containing the metadata for this module.
                See
            strict (bool): whether to strictly enforce that the keys in
                :attr:`state_dict` with :attr:`prefix` match the names of
                parameters and buffers in this module
            missing_keys (list of str): if ``strict=True``, add missing keys to
                this list
            unexpected_keys (list of str): if ``strict=True``, add unexpected
                keys to this list
            error_msgs (list of str): error messages should be added to this
                list, and will be reported together in
                :meth:`~torch.nn.Module.load_state_dict`
        c             S   s    i | ]\}}|d k	r|j |“qS )N)rM   )r   Úkrz   r   r   r   ú
<dictcomp>í  s    z0Module._load_from_state_dict.<locals>.<dictcomp>r   r   zfsize mismatch for {}: copying a param with shape {} from checkpoint, the shape in current model is {}.z{While copying the parameter named "{}", whose dimensions in the model are {} and whose dimensions in the checkpoint are {}.Nr2   )r,   r|   Ú	itertoolsÚchainr&   rL   r'   r   Úshaperw   r:   r6   r   rM   Zcopy_Ú	ExceptionÚsizeÚkeysÚ
startswithr   r-   )r   rž   r˜   rŸ   Ústrictr   r   Ú
error_msgsrj   Zlocal_name_paramsZlocal_stater?   rD   rS   Zinput_paramZ
input_namer   r   r   Ú_load_from_state_dictÈ  s<    !

zModule._load_from_state_dictTc                sâ   g ‰g ‰g ‰ t ˆddƒ‰ˆjƒ ‰ˆdk	r.ˆˆ_d‡ ‡‡‡‡‡fdd„	‰ˆ| ƒ d‰|r²tˆƒdkr„ˆ jddjdjd	d
„ ˆD ƒƒƒƒ tˆƒdkr²ˆ jddjdjdd
„ ˆD ƒƒƒƒ tˆ ƒdkrØtdj| jj	djˆ ƒƒƒ‚t
ˆˆƒS )a’  Copies parameters and buffers from :attr:`state_dict` into
        this module and its descendants. If :attr:`strict` is ``True``, then
        the keys of :attr:`state_dict` must exactly match the keys returned
        by this module's :meth:`~torch.nn.Module.state_dict` function.

        Arguments:
            state_dict (dict): a dict containing parameters and
                persistent buffers.
            strict (bool, optional): whether to strictly enforce that the keys
                in :attr:`state_dict` match the keys returned by this module's
                :meth:`~torch.nn.Module.state_dict` function. Default: ``True``

        Returns:
            ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
                * **missing_keys** is a list of str containing the missing keys
                * **unexpected_keys** is a list of str containing the unexpected keys
        rœ   Nr3   c                sl   ˆd kri nˆj |d d… i ƒ}| jˆ||dˆˆˆ ƒ x0| jjƒ D ]"\}}|d k	rBˆ||| d ƒ qBW d S )Nr   Tr2   ro   )r�   r¬   r-   rL   )rF   r˜   rŸ   r?   rs   )r«   ÚloadÚmetadatar   rž   r   r   r   r­   0  s     z$Module.load_state_dict.<locals>.loadr   z%Unexpected key(s) in state_dict: {}. z, c             s   s   | ]}d j |ƒV  qdS )z"{}"N)r:   )r   r¡   r   r   r   r{   ?  s    z)Module.load_state_dict.<locals>.<genexpr>z"Missing key(s) in state_dict: {}. c             s   s   | ]}d j |ƒV  qdS )z"{}"N)r:   )r   r¡   r   r   r   r{   C  s    z*Error(s) in loading state_dict for {}:
	{}z
	)r3   )ÚgetattrÚcopyrœ   r   Úinsertr:   r   ÚRuntimeErrorr   r   r   )r   rž   rª   r   )r«   r­   r®   r   rž   r   r   Úload_state_dict  s.    zModule.load_state_dictc             c   sŠ   t ƒ }|r| j|d�n|| fg}xd|D ]\\}}||ƒ}xJ|D ]B\}	}
|
dks<|
|krVq<|j|
ƒ ||rjdnd |	 }||
fV  q<W q&W dS )z>Helper method for yielding various names + members of modules.)r˜   Nr2   r3   )ÚsetÚnamed_modulesÚadd)r   Zget_members_fnr˜   ÚrecurseÚmemor‹   Zmodule_prefixrF   Úmembersr¡   rz   r?   r   r   r   Ú_named_membersJ  s    
zModule._named_membersc             c   s$   x| j |d�D ]\}}|V  qW dS )aa  Returns an iterator over module parameters.

        This is typically passed to an optimizer.

        Args:
            recurse (bool): if True, then yields parameters of this module
                and all submodules. Otherwise, yields only parameters that
                are direct members of this module.

        Yields:
            Parameter: module parameter

        Example::

            >>> for param in model.parameters():
            >>>     print(type(param.data), param.size())
            <class 'torch.FloatTensor'> (20L,)
            <class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)

        )r·   N)Únamed_parameters)r   r·   r?   rD   r   r   r   Ú
parametersW  s    zModule.parametersc             c   s,   | j dd„ ||d�}x|D ]
}|V  qW dS )aŸ  Returns an iterator over module parameters, yielding both the
        name of the parameter as well as the parameter itself.

        Args:
            prefix (str): prefix to prepend to all parameter names.
            recurse (bool): if True, then yields parameters of this module
                and all submodules. Otherwise, yields only parameters that
                are direct members of this module.

        Yields:
            (string, Parameter): Tuple containing the name and parameter

        Example::

            >>> for name, param in self.named_parameters():
            >>>    if name in ['bias']:
            >>>        print(param.size())

        c             S   s
   | j jƒ S )N)r&   rL   )rF   r   r   r   rY   „  s    z)Module.named_parameters.<locals>.<lambda>)r˜   r·   N)rº   )r   r˜   r·   ÚgenÚelemr   r   r   r»   o  s
    

zModule.named_parametersc             c   s$   x| j |d�D ]\}}|V  qW dS )a  Returns an iterator over module buffers.

        Args:
            recurse (bool): if True, then yields buffers of this module
                and all submodules. Otherwise, yields only buffers that
                are direct members of this module.

        Yields:
            torch.Tensor: module buffer

        Example::

            >>> for buf in model.buffers():
            >>>     print(type(buf.data), buf.size())
            <class 'torch.FloatTensor'> (20L,)
            <class 'torch.FloatTensor'> (20L, 1L, 5L, 5L)

        )r·   N)Únamed_buffers)r   r·   r?   rT   r   r   r   r”   ‰  s    zModule.buffersc             c   s,   | j dd„ ||d�}x|D ]
}|V  qW dS )a�  Returns an iterator over module buffers, yielding both the
        name of the buffer as well as the buffer itself.

        Args:
            prefix (str): prefix to prepend to all buffer names.
            recurse (bool): if True, then yields buffers of this module
                and all submodules. Otherwise, yields only buffers that
                are direct members of this module.

        Yields:
            (string, torch.Tensor): Tuple containing the name and buffer

        Example::

            >>> for name, buf in self.named_buffers():
            >>>    if name in ['running_var']:
            >>>        print(buf.size())

        c             S   s
   | j jƒ S )N)r'   rL   )rF   r   r   r   rY   ´  s    z&Module.named_buffers.<locals>.<lambda>)r˜   r·   N)rº   )r   r˜   r·   r½   r¾   r   r   r   r¿   Ÿ  s
    

zModule.named_buffersc             c   s    x| j ƒ D ]\}}|V  q
W dS )zqReturns an iterator over immediate children modules.

        Yields:
            Module: a child module
        N)rq   )r   r?   rF   r   r   r   rJ   ¹  s    zModule.childrenc             c   sF   t ƒ }x:| jjƒ D ],\}}|dk	r||kr|j|ƒ ||fV  qW dS )aŠ  Returns an iterator over immediate children modules, yielding both
        the name of the module as well as the module itself.

        Yields:
            (string, Module): Tuple containing a name and child module

        Example::

            >>> for name, module in model.named_children():
            >>>     if name in ['conv4', 'conv5']:
            >>>         print(module)

        N)r´   r-   rL   r¶   )r   r¸   r?   rF   r   r   r   rq   Â  s
    
zModule.named_childrenc             c   s    x| j ƒ D ]\}}|V  q
W dS )aÀ  Returns an iterator over all modules in the network.

        Yields:
            Module: a module in the network

        Note:
            Duplicate modules are returned only once. In the following
            example, ``l`` will be returned only once.

        Example::

            >>> l = nn.Linear(2, 2)
            >>> net = nn.Sequential(l, l)
            >>> for idx, m in enumerate(net.modules()):
                    print(idx, '->', m)

            0 -> Sequential(
              (0): Linear(in_features=2, out_features=2, bias=True)
              (1): Linear(in_features=2, out_features=2, bias=True)
            )
            1 -> Linear(in_features=2, out_features=2, bias=True)

        N)rµ   )r   r?   rF   r   r   r   r‹   Ö  s    zModule.modulesc             c   s€   |dkrt ƒ }| |kr||j| ƒ || fV  xP| jjƒ D ]B\}}|dkrHq6||rRdnd | }x|j||ƒD ]
}|V  qjW q6W dS )a)  Returns an iterator over all modules in the network, yielding
        both the name of the module as well as the module itself.

        Yields:
            (string, Module): Tuple of name and module

        Note:
            Duplicate modules are returned only once. In the following
            example, ``l`` will be returned only once.

        Example::

            >>> l = nn.Linear(2, 2)
            >>> net = nn.Sequential(l, l)
            >>> for idx, m in enumerate(net.named_modules()):
                    print(idx, '->', m)

            0 -> ('', Sequential(
              (0): Linear(in_features=2, out_features=2, bias=True)
              (1): Linear(in_features=2, out_features=2, bias=True)
            ))
            1 -> ('0', Linear(in_features=2, out_features=2, bias=True))

        Nr2   r3   )r´   r¶   r-   rL   rµ   )r   r¸   r˜   r?   rF   Zsubmodule_prefixÚmr   r   r   rµ   ñ  s    

zModule.named_modulesc             C   s&   || _ x| jƒ D ]}|j|ƒ qW | S )aê  Sets the module in training mode.

        This has any effect only on certain modules. See documentations of
        particular modules for details of their behaviors in training/evaluation
        mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,
        etc.

        Args:
            mode (bool): whether to set training mode (``True``) or evaluation
                         mode (``False``). Default: ``True``.

        Returns:
            Module: self
        )r%   rJ   Útrain)r   ÚmoderF   r   r   r   rÁ     s    zModule.trainc             C   s
   | j dƒS )a¤  Sets the module in evaluation mode.

        This has any effect only on certain modules. See documentations of
        particular modules for details of their behaviors in training/evaluation
        mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,
        etc.

        This is equivalent with :meth:`self.train(False) <torch.nn.Module.train>`.

        Returns:
            Module: self
        F)rÁ   )r   r   r   r   Úeval+  s    zModule.evalc             C   s    x| j ƒ D ]}|j|ƒ q
W | S )a'  Change if autograd should record operations on parameters in this
        module.

        This method sets the parameters' :attr:`requires_grad` attributes
        in-place.

        This method is helpful for freezing part of the module for finetuning
        or training parts of a model individually (e.g., GAN training).

        Args:
            requires_grad (bool): whether autograd should record operations on
                                  parameters in this module. Default: ``True``.

        Returns:
            Module: self
        )r¼   rQ   )r   rO   Úpr   r   r   rQ   :  s    zModule.requires_grad_c             C   s4   x.| j ƒ D ]"}|jdk	r
|jjƒ  |jjƒ  q
W dS )z/Sets gradients of all model parameters to zero.N)r¼   rP   Zdetach_Zzero_)r   rÄ   r   r   r   Ú	zero_gradO  s    

zModule.zero_gradc             C   s   | j dd„ ƒS )Nc             S   s   | j ƒ S )N)Zshare_memory_)rW   r   r   r   rY   W  s    z%Module.share_memory.<locals>.<lambda>)rK   )r   r   r   r   Úshare_memoryV  s    zModule.share_memoryc             C   s   | j jS )N)r   r   )r   r   r   r   rv   Y  s    zModule._get_namec             C   s   dS )zãSet the extra representation of the module

        To print customized extra information, you should reimplement
        this method in your own modules. Both single-line and multi-line
        strings are acceptable.
        r3   r   )r   r   r   r   Ú
extra_repr\  s    zModule.extra_reprc       	      C   s¸   g }| j ƒ }|r|jdƒ}g }x>| jjƒ D ]0\}}t|ƒ}t|dƒ}|jd| d | ƒ q*W || }| jƒ d }|r¬t|ƒdkr–| r–||d 7 }n|ddj	|ƒ d 7 }|d7 }|S )	Nr   r   ú(z): r   r   z
  ú))
rÇ   r   r-   rL   Úreprr!   rw   rv   r   r   )	r   Zextra_linesrÇ   Zchild_linesrS   rF   Zmod_strÚlinesZmain_strr   r   r   r
   e  s"    

zModule.__repr__c             C   sl   t | jƒ}t| jjƒ ƒ}t| jjƒ ƒ}t| jjƒ ƒ}t| jjƒ ƒ}|| | | | }dd„ |D ƒ}t|ƒS )Nc             S   s   g | ]}|d  j ƒ s|‘qS )r   )Úisdigit)r   rS   r   r   r   r   ‡  s    z"Module.__dir__.<locals>.<listcomp>)	Údirr   Úlistr4   r¨   r&   r-   r'   Úsorted)r   Zmodule_attrsÚattrsr¼   r‹   r”   r¨   r   r   r   Ú__dir__~  s    
zModule.__dir__)N)Nr3   F)T)r3   T)T)r3   T)T)r3   T)Nr3   )T)T)6r   r   r   Ú__doc__Zdump_patchesr�   r.   r1   rA   rE   rG   rK   rU   rV   rZ   r[   r^   r_   r`   ra   rl   rm   rn   rt   ry   r‡   rŠ   rŒ   r‘   r•   r–   rš   rž   r    r¬   r³   rº   r¼   r»   r”   r¿   rJ   rq   r‹   rµ   rÁ   rÃ   rQ   rÅ   rÆ   rv   rÇ   r
   rÑ   r   r   r   r   r"      sf   &'1*
N!	
,


L
6




	
&

	r"   )Úcollectionsr   r   r€   r£   r#   Z	parameterr   Ztorch.utils.hooksÚutilsrg   r   r!   r�   r"   r   r   r   r   Ú<module>   s   	