3
I4W^«<  ã               @   s:  d dl mZ d dlZd dlZd dlZdd„ Zdd„ Zdd„ Zd	d
„ Zd3dd„Z	d4dd„Z
d5dd„Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd6dd „Zd7d!d"„Zd#d$„ Zd8d'd(„Zd9d)d*„Zd:d,d-„Zd;d/d0„Zd1d2„ Zee
ƒZeeƒZeeƒZeeƒZeeƒZeeƒZeeƒZ eeƒZ!eeƒZ"eeƒZ#eeƒZ$dS )<é    )ÚdivisionNc          
   C   s    t jƒ � | j||ƒS Q R X d S )N)ÚtorchÚno_gradÚuniform_)ÚtensorÚaÚb© r	   ú./tmp/pip-build-e66j97b2/torch/torch/nn/init.pyÚ_no_grad_uniform_   s    
r   c          
   C   s    t jƒ � | j||ƒS Q R X d S )N)r   r   Únormal_)r   ÚmeanÚstdr	   r	   r
   Ú_no_grad_normal_   s    
r   c          	   C   s   t jƒ � | j|ƒS Q R X d S )N)r   r   Zfill_)r   Úvalr	   r	   r
   Ú_no_grad_fill_   s    
r   c          	   C   s   t jƒ �
 | jƒ S Q R X d S )N)r   r   Úzero_)r   r	   r	   r
   Ú_no_grad_zero_   s    
r   c             C   s¶   dddddddg}| |ks"| dkr&d	S | d
kr2dS | dkrDt jdƒS | dkr¤|dkrZd}n4t|tƒ rpt|tƒszt|tƒr€|}ntdj|ƒƒ‚t jdd	|d   ƒS tdj| ƒƒ‚dS )ad  Return the recommended gain value for the given nonlinearity function.
    The values are as follows:

    ================= ====================================================
    nonlinearity      gain
    ================= ====================================================
    Linear / Identity :math:`1`
    Conv{1,2,3}D      :math:`1`
    Sigmoid           :math:`1`
    Tanh              :math:`\frac{5}{3}`
    ReLU              :math:`\sqrt{2}`
    Leaky Relu        :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}`
    ================= ====================================================

    Args:
        nonlinearity: the non-linear function (`nn.functional` name)
        param: optional parameter for the non-linear function

    Examples:
        >>> gain = nn.init.calculate_gain('leaky_relu', 0.2)  # leaky_relu with negative_slope=0.2
    ZlinearZconv1dZconv2dZconv3dZconv_transpose1dZconv_transpose2dZconv_transpose3dZsigmoidé   Útanhg      @é   Zrelug       @Ú
leaky_reluNg{®Gáz„?z$negative_slope {} not a valid numberé   zUnsupported nonlinearity {}g«ªªªªªú?)ÚmathÚsqrtÚ
isinstanceÚboolÚintÚfloatÚ
ValueErrorÚformat)ÚnonlinearityÚparamZ
linear_fnsZnegative_sloper	   r	   r
   Úcalculate_gain    s    
 r#   ç        ç      ð?c             C   s   t | ||ƒS )ad  Fills the input Tensor with values drawn from the uniform
    distribution :math:`\mathcal{U}(a, b)`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the lower bound of the uniform distribution
        b: the upper bound of the uniform distribution

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.uniform_(w)
    )r   )r   r   r   r	   r	   r
   r   J   s    r   c             C   s   t | ||ƒS )az  Fills the input Tensor with values drawn from the normal
    distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        mean: the mean of the normal distribution
        std: the standard deviation of the normal distribution

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.normal_(w)
    )r   )r   r   r   r	   r	   r
   r   [   s    r   c             C   s
   t | |ƒS )zÿFills the input Tensor with the value :math:`\text{val}`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        val: the value to fill the tensor with

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.constant_(w, 0.3)
    )r   )r   r   r	   r	   r
   Ú	constant_l   s    r&   c             C   s
   t | dƒS )z¿Fills the input Tensor with the scalar value `1`.

    Args:
        tensor: an n-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.ones_(w)
    g      ð?)r   )r   r	   r	   r
   Úones_{   s    r'   c             C   s   t | ƒS )zÀFills the input Tensor with the scalar value `0`.

    Args:
        tensor: an n-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.zeros_(w)
    )r   )r   r	   r	   r
   Úzeros_‰   s    r(   c             C   sB   | j ƒ dkrtdƒ‚tjƒ � tj| j| | jdœŽ W dQ R X | S )a=  Fills the 2-dimensional input `Tensor` with the identity
    matrix. Preserves the identity of the inputs in `Linear` layers, where as
    many inputs are preserved as possible.

    Args:
        tensor: a 2-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.eye_(w)
    r   z,Only tensors with 2 dimensions are supported)ÚoutÚrequires_gradN)Ú
ndimensionr   r   r   ÚeyeÚshaper*   )r   r	   r	   r
   Úeye_—   s
    
 r.   c             C   sä   | j ƒ }|d	krtdƒ‚| jƒ }t|d |d ƒ}tjƒ �  | jƒ  x�t|ƒD ]„}|dkrtd| ||| jdƒd f< qN|dkr¢d| ||| jdƒd | jdƒd f< qNd| ||| jdƒd | jdƒd | jdƒd f< qNW W dQ R X | S )
aj  Fills the {3, 4, 5}-dimensional input `Tensor` with the Dirac
    delta function. Preserves the identity of the inputs in `Convolutional`
    layers, where as many input channels are preserved as possible.

    Args:
        tensor: a {3, 4, 5}-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 16, 5, 5)
        >>> nn.init.dirac_(w)
    r   é   é   z5Only tensors with 3, 4, or 5 dimensions are supportedr   r   r   N)r   r/   r0   )r+   r   ÚsizeÚminr   r   r   Úrange)r   Ú
dimensionsÚsizesZmin_dimÚdr	   r	   r
   Údirac_«   s    
&>r7   c             C   s‚   | j ƒ }|dk rtdƒ‚|dkr6| jdƒ}| jdƒ}nD| jdƒ}| jdƒ}d}| j ƒ dkrj| d d jƒ }|| }|| }||fS )Nr   zNFan in and fan out can not be computed for tensor with fewer than 2 dimensionsr   r   )Zdimr   r1   Únumel)r   r4   Úfan_inÚfan_outZnum_input_fmapsZnum_output_fmapsZreceptive_field_sizer	   r	   r
   Ú_calculate_fan_in_and_fan_outÊ   s    


r;   c             C   sB   t | ƒ\}}|tjdt|| ƒ ƒ }tjdƒ| }t| | |ƒS )a©  Fills the input `Tensor` with values according to the method
    described in `Understanding the difficulty of training deep feedforward
    neural networks` - Glorot, X. & Bengio, Y. (2010), using a uniform
    distribution. The resulting tensor will have values sampled from
    :math:`\mathcal{U}(-a, a)` where

    .. math::
        a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}}

    Also known as Glorot initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        gain: an optional scaling factor

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.xavier_uniform_(w, gain=nn.init.calculate_gain('relu'))
    g       @g      @)r;   r   r   r   r   )r   Úgainr9   r:   r   r   r	   r	   r
   Úxavier_uniform_Þ   s    r=   c             C   s2   t | ƒ\}}|tjdt|| ƒ ƒ }t| d|ƒS )a•  Fills the input `Tensor` with values according to the method
    described in `Understanding the difficulty of training deep feedforward
    neural networks` - Glorot, X. & Bengio, Y. (2010), using a normal
    distribution. The resulting tensor will have values sampled from
    :math:`\mathcal{N}(0, \text{std}^2)` where

    .. math::
        \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}}

    Also known as Glorot initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        gain: an optional scaling factor

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.xavier_normal_(w)
    g       @g        )r;   r   r   r   r   )r   r<   r9   r:   r   r	   r	   r
   Úxavier_normal_ú   s    r>   c             C   sD   |j ƒ }ddg}||kr(tdj||ƒƒ‚t| ƒ\}}|dkr@|S |S )Nr9   r:   z+Mode {} not supported, please use one of {})Úlowerr   r    r;   )r   ÚmodeZvalid_modesr9   r:   r	   r	   r
   Ú_calculate_correct_fan  s    rA   r9   r   c          
   C   sR   t | |ƒ}t||ƒ}|tj|ƒ }tjdƒ| }tjƒ � | j| |ƒS Q R X dS )až  Fills the input `Tensor` with values according to the method
    described in `Delving deep into rectifiers: Surpassing human-level
    performance on ImageNet classification` - He, K. et al. (2015), using a
    uniform distribution. The resulting tensor will have values sampled from
    :math:`\mathcal{U}(-\text{bound}, \text{bound})` where

    .. math::
        \text{bound} = \sqrt{\frac{6}{(1 + a^2) \times \text{fan\_in}}}

    Also known as He initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the negative slope of the rectifier used after this layer (0 for ReLU
            by default)
        mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
            preserves the magnitude of the variance of the weights in the
            forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
            backwards pass.
        nonlinearity: the non-linear function (`nn.functional` name),
            recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.kaiming_uniform_(w, mode='fan_in', nonlinearity='relu')
    g      @N)rA   r#   r   r   r   r   r   )r   r   r@   r!   Úfanr<   r   Zboundr	   r	   r
   Úkaiming_uniform_  s    


rC   c          
   C   sB   t | |ƒ}t||ƒ}|tj|ƒ }tjƒ � | jd|ƒS Q R X dS )a�  Fills the input `Tensor` with values according to the method
    described in `Delving deep into rectifiers: Surpassing human-level
    performance on ImageNet classification` - He, K. et al. (2015), using a
    normal distribution. The resulting tensor will have values sampled from
    :math:`\mathcal{N}(0, \text{std}^2)` where

    .. math::
        \text{std} = \sqrt{\frac{2}{(1 + a^2) \times \text{fan\_in}}}

    Also known as He initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the negative slope of the rectifier used after this layer (0 for ReLU
            by default)
        mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
            preserves the magnitude of the variance of the weights in the
            forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
            backwards pass.
        nonlinearity: the non-linear function (`nn.functional` name),
            recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.kaiming_normal_(w, mode='fan_out', nonlinearity='relu')
    r   N)rA   r#   r   r   r   r   r   )r   r   r@   r!   rB   r<   r   r	   r	   r
   Úkaiming_normal_B  s
    


rD   r   c       	   	   C   sº   | j ƒ dk rtdƒ‚| jdƒ}| jƒ | }| j||ƒjddƒ}||k rN|jƒ  tj|ƒ\}}tj	|dƒ}|j
ƒ }||9 }||k rˆ|jƒ  tjƒ �  | j|ƒj|ƒ | j|ƒ W dQ R X | S )a!  Fills the input `Tensor` with a (semi) orthogonal matrix, as
    described in `Exact solutions to the nonlinear dynamics of learning in deep
    linear neural networks` - Saxe, A. et al. (2013). The input tensor must have
    at least 2 dimensions, and for tensors with more than 2 dimensions the
    trailing dimensions are flattened.

    Args:
        tensor: an n-dimensional `torch.Tensor`, where :math:`n \geq 2`
        gain: optional scaling factor

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.orthogonal_(w)
    r   z4Only tensors with 2 or more dimensions are supportedr   r   N)r+   r   r1   r8   Únewr   Zt_r   ZqrZdiagÚsignr   Zview_asZcopy_Zmul_)	r   r<   ÚrowsÚcolsZ	flattenedÚqÚrr6   Úphr	   r	   r
   Úorthogonal_d  s"    

rL   ç{®Gáz„?c       	      C   sˆ   | j ƒ dkrtdƒ‚| j\}}ttj|| ƒƒ}tjƒ �F | jd|ƒ x2t	|ƒD ]&}tj
|ƒ}|d|… }d| ||f< qPW W dQ R X | S )aN  Fills the 2D input `Tensor` as a sparse matrix, where the
    non-zero elements will be drawn from the normal distribution
    :math:`\mathcal{N}(0, 0.01)`, as described in `Deep learning via
    Hessian-free optimization` - Martens, J. (2010).

    Args:
        tensor: an n-dimensional `torch.Tensor`
        sparsity: The fraction of elements in each column to be set to zero
        std: the standard deviation of the normal distribution used to generate
            the non-zero values

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.sparse_(w, sparsity=0.1)
    r   z,Only tensors with 2 dimensions are supportedr   N)r+   r   r-   r   r   Úceilr   r   r   r3   Zrandperm)	r   Zsparsityr   rG   rH   Z	num_zerosZcol_idxZrow_indicesZzero_indicesr	   r	   r
   Úsparse_�  s    


rO   c                s<   ˆ j ‰ˆd d… ‰‡ ‡‡fdd„}djˆˆd�|_ˆ|_ |S )Nr   c                 s    t jdjˆˆƒdd� ˆ | |ŽS )Nz4nn.init.{} is now deprecated in favor of nn.init.{}.r   )Ú
stacklevel)ÚwarningsÚwarnr    )ÚargsÚkwargs)ÚmethÚnew_nameÚold_namer	   r
   Údeprecated_init±  s    z(_make_deprecate.<locals>.deprecated_initz²
    {old_name}(...)

    .. warning::
        This method is now deprecated in favor of :func:`torch.nn.init.{new_name}`.

    See :func:`~torch.nn.init.{new_name}` for details.)rW   rV   éÿÿÿÿ)Ú__name__r    Ú__doc__)rU   rX   r	   )rU   rV   rW   r
   Ú_make_deprecate­  s    r\   )N)r$   r%   )r$   r%   )r%   )r%   )r   r9   r   )r   r9   r   )r   )rM   )%Ú
__future__r   r   rQ   r   r   r   r   r   r#   r   r   r&   r'   r(   r.   r7   r;   r=   r>   rA   rC   rD   rL   rO   r\   ÚuniformZnormalZconstantr,   ZdiracZxavier_uniformZxavier_normalZkaiming_uniformZkaiming_normalZ
orthogonalÚsparser	   r	   r	   r
   Ú<module>   sF   
*





#
"
)
 