3
I4W^�}  ã               @   sè  d dl Z d dlZddlmZ d dlmZ d dlmZ d dlmZ d dlm	Z	 ddl
mZ d	d
lmZ G dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd „ d eƒZG d!d"„ d"eƒZG d#d$„ d$eƒZG d%d&„ d&eƒZG d'd(„ d(eƒZG d)d*„ d*eƒZG d+d,„ d,eƒZG d-d.„ d.eƒZG d/d0„ d0eƒZ G d1d2„ d2eƒZ!G d3d4„ d4eƒZ"G d5d6„ d6eƒZ#G d7d8„ d8eƒZ$G d9d:„ d:eƒZ%dS );é    Né   )ÚLinear)Úxavier_uniform_)Ú	constant_)Úxavier_normal_)Ú	Parameter)ÚModuleé   )Ú
functionalc                   s<   e Zd ZdZdddgZd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )Ú	Thresholdaº  Thresholds each element of the input Tensor.

    Threshold is defined as:

    .. math::
        y =
        \begin{cases}
        x, &\text{ if } x > \text{threshold} \\
        \text{value}, &\text{ otherwise }
        \end{cases}

    Args:
        threshold: The value to threshold at
        value: The value to replace with
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    Examples::

        >>> m = nn.Threshold(0.1, 20)
        >>> input = torch.randn(2)
        >>> output = m(input)
    Ú	thresholdÚvalueÚinplaceFc                s$   t t| ƒjƒ  || _|| _|| _d S )N)Úsuperr   Ú__init__r   r   r   )Úselfr   r   r   )Ú	__class__© ú</tmp/pip-build-e66j97b2/torch/torch/nn/modules/activation.pyr   *   s    zThreshold.__init__c             C   s   t j|| j| j| jƒS )N)ÚFr   r   r   )r   Úinputr   r   r   Úforward1   s    zThreshold.forwardc             C   s    | j r
dnd}dj| j| j|ƒS )Nz, inplace=TrueÚ zthreshold={}, value={}{})r   Úformatr   r   )r   Úinplace_strr   r   r   Ú
extra_repr4   s    zThreshold.extra_repr)F)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__constants__r   r   r   Ú__classcell__r   r   )r   r   r      s
   
r   c                   s8   e Zd ZdZdgZd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )ÚReLUaÈ  Applies the rectified linear unit function element-wise:

    :math:`\text{ReLU}(x)= \max(0, x)`

    Args:
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/ReLU.png

    Examples::

        >>> m = nn.ReLU()
        >>> input = torch.randn(2)
        >>> output = m(input)


      An implementation of CReLU - https://arxiv.org/abs/1603.05201

        >>> m = nn.ReLU()
        >>> input = torch.randn(2).unsqueeze(0)
        >>> output = torch.cat((m(input),m(-input)))
    r   Fc                s   t t| ƒjƒ  || _d S )N)r   r"   r   r   )r   r   )r   r   r   r   Y   s    zReLU.__init__c             C   s   t j|| jd�S )N)r   )r   Zrelur   )r   r   r   r   r   r   ]   s    zReLU.forwardc             C   s   | j r
dnd}|S )Nzinplace=Truer   )r   )r   r   r   r   r   r   `   s    zReLU.extra_repr)F)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r"   ;   s
   r"   c                   s<   e Zd ZdZdddgZd‡ fd	d
„	Zdd„ Zdd„ Z‡  ZS )ÚRReLUa½  Applies the randomized leaky rectified liner unit function, element-wise,
    as described in the paper:

    `Empirical Evaluation of Rectified Activations in Convolutional Network`_.

    The function is defined as:

    .. math::
        \text{RReLU}(x) =
        \begin{cases}
            x & \text{if } x \geq 0 \\
            ax & \text{ otherwise }
        \end{cases}

    where :math:`a` is randomly sampled from uniform distribution
    :math:`\mathcal{U}(\text{lower}, \text{upper})`.

     See: https://arxiv.org/pdf/1505.00853.pdf

    Args:
        lower: lower bound of the uniform distribution. Default: :math:`\frac{1}{8}`
        upper: upper bound of the uniform distribution. Default: :math:`\frac{1}{3}`
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    Examples::

        >>> m = nn.RReLU(0.1, 0.3)
        >>> input = torch.randn(2)
        >>> output = m(input)

    .. _`Empirical Evaluation of Rectified Activations in Convolutional Network`:
        https://arxiv.org/abs/1505.00853
    ÚlowerÚupperr   g      ð?é   é   Fc                s$   t t| ƒjƒ  || _|| _|| _d S )N)r   r#   r   r$   r%   r   )r   r$   r%   r   )r   r   r   r   Ž   s    zRReLU.__init__c             C   s   t j|| j| j| j| jƒS )N)r   Zrrelur$   r%   Útrainingr   )r   r   r   r   r   r   ”   s    zRReLU.forwardc             C   s    | j r
dnd}dj| j| j|ƒS )Nz, inplace=Truer   zlower={}, upper={}{})r   r   r$   r%   )r   r   r   r   r   r   —   s    zRReLU.extra_reprç      À?çUUUUUUÕ?)r)   r*   F)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r#   e   s
   &
r#   c                   s<   e Zd ZdZdddgZd‡ fdd	„	Zd
d„ Zdd„ Z‡  ZS )ÚHardtanha?  Applies the HardTanh function element-wise

    HardTanh is defined as:

    .. math::
        \text{HardTanh}(x) = \begin{cases}
            1 & \text{ if } x > 1 \\
            -1 & \text{ if } x < -1 \\
            x & \text{ otherwise } \\
        \end{cases}

    The range of the linear region :math:`[-1, 1]` can be adjusted using
    :attr:`min_val` and :attr:`max_val`.

    Args:
        min_val: minimum value of the linear region range. Default: -1
        max_val: maximum value of the linear region range. Default: 1
        inplace: can optionally do the operation in-place. Default: ``False``

    Keyword arguments :attr:`min_value` and :attr:`max_value`
    have been deprecated in favor of :attr:`min_val` and :attr:`max_val`.

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Hardtanh.png

    Examples::

        >>> m = nn.Hardtanh(-2, 2)
        >>> input = torch.randn(2)
        >>> output = m(input)
    Úmin_valÚmax_valr   ç      ð?FNc                s`   t t| ƒjƒ  |d k	r$tjdƒ |}|d k	r:tjdƒ |}|| _|| _|| _| j| jks\t‚d S )Nz>keyword argument min_value is deprecated and rename to min_valz>keyword argument max_value is deprecated and rename to max_val)	r   r+   r   ÚwarningsÚwarnr,   r-   r   ÚAssertionError)r   r,   r-   r   Z	min_valueZ	max_value)r   r   r   r   Â   s    

zHardtanh.__init__c             C   s   t j|| j| j| jƒS )N)r   Zhardtanhr,   r-   r   )r   r   r   r   r   r   Ð   s    zHardtanh.forwardc             C   s    | j r
dnd}dj| j| j|ƒS )Nz, inplace=Truer   zmin_val={}, max_val={}{})r   r   r,   r-   )r   r   r   r   r   r   Ó   s    zHardtanh.extra_reprç      ð¿)r2   r.   FNN)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r+   œ   s
   #
r+   c                   s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )ÚReLU6a  Applies the element-wise function:

    .. math::
        \text{ReLU6}(x) = \min(\max(0,x), 6)

    Args:
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/ReLU6.png

    Examples::

        >>> m = nn.ReLU6()
        >>> input = torch.randn(2)
        >>> output = m(input)
    Fc                s   t t| ƒjdd|ƒ d S )Ng        g      @)r   r3   r   )r   r   )r   r   r   r   ñ   s    zReLU6.__init__c             C   s   | j r
dnd}|S )Nzinplace=Truer   )r   )r   r   r   r   r   r   ô   s    zReLU6.extra_repr)F)r   r   r   r   r   r   r!   r   r   )r   r   r3   Ú   s   r3   c               @   s   e Zd ZdZdd„ ZdS )ÚSigmoida´  Applies the element-wise function:

    .. math::
        \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}


    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Sigmoid.png

    Examples::

        >>> m = nn.Sigmoid()
        >>> input = torch.randn(2)
        >>> output = m(input)
    c             C   s
   t j|ƒS )N)ÚtorchZsigmoid)r   r   r   r   r   r     s    zSigmoid.forwardN)r   r   r   r   r   r   r   r   r   r4   ù   s   r4   c               @   s   e Zd ZdZdd„ ZdS )ÚTanhaÁ  Applies the element-wise function:

    .. math::
        \text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}}

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Tanh.png

    Examples::

        >>> m = nn.Tanh()
        >>> input = torch.randn(2)
        >>> output = m(input)
    c             C   s
   t j|ƒS )N)r5   Útanh)r   r   r   r   r   r   &  s    zTanh.forwardN)r   r   r   r   r   r   r   r   r   r6     s   r6   c                   s:   e Zd ZdZddgZd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )ÚELUac  Applies the element-wise function:

    .. math::
        \text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))

    Args:
        alpha: the :math:`\alpha` value for the ELU formulation. Default: 1.0
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/ELU.png

    Examples::

        >>> m = nn.ELU()
        >>> input = torch.randn(2)
        >>> output = m(input)
    Úalphar   ç      ð?Fc                s   t t| ƒjƒ  || _|| _d S )N)r   r8   r   r9   r   )r   r9   r   )r   r   r   r   C  s    zELU.__init__c             C   s   t j|| j| jƒS )N)r   Zelur9   r   )r   r   r   r   r   r   H  s    zELU.forwardc             C   s   | j r
dnd}dj| j|ƒS )Nz, inplace=Truer   z
alpha={}{})r   r   r9   )r   r   r   r   r   r   K  s    zELU.extra_repr)r:   F)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r8   *  s
   r8   c                   s:   e Zd ZdZddgZd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )ÚCELUa>  Applies the element-wise function:

    .. math::
        \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))

    More details can be found in the paper `Continuously Differentiable Exponential Linear Units`_ .

    Args:
        alpha: the :math:`\alpha` value for the CELU formulation. Default: 1.0
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/CELU.png

    Examples::

        >>> m = nn.CELU()
        >>> input = torch.randn(2)
        >>> output = m(input)

    .. _`Continuously Differentiable Exponential Linear Units`:
        https://arxiv.org/abs/1704.07483
    r9   r   ç      ð?Fc                s   t t| ƒjƒ  || _|| _d S )N)r   r;   r   r9   r   )r   r9   r   )r   r   r   r   n  s    zCELU.__init__c             C   s   t j|| j| jƒS )N)r   Zcelur9   r   )r   r   r   r   r   r   s  s    zCELU.forwardc             C   s   | j r
dnd}dj| j|ƒS )Nz, inplace=Truer   z
alpha={}{})r   r   r9   )r   r   r   r   r   r   v  s    zCELU.extra_repr)r<   F)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r;   P  s
   r;   c                   s8   e Zd ZdZdgZd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )ÚSELUaN  Applied element-wise, as:

    .. math::
        \text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))

    with :math:`\alpha = 1.6732632423543772848170429916717` and
    :math:`\text{scale} = 1.0507009873554804934193349852946`.

    More details can be found in the paper `Self-Normalizing Neural Networks`_ .

    Args:
        inplace (bool, optional): can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/SELU.png

    Examples::

        >>> m = nn.SELU()
        >>> input = torch.randn(2)
        >>> output = m(input)

    .. _Self-Normalizing Neural Networks: https://arxiv.org/abs/1706.02515
    r   Fc                s   t t| ƒjƒ  || _d S )N)r   r=   r   r   )r   r   )r   r   r   r   š  s    zSELU.__init__c             C   s   t j|| jƒS )N)r   Zselur   )r   r   r   r   r   r   ž  s    zSELU.forwardc             C   s   | j r
dnd}|S )Nzinplace=Truer   )r   )r   r   r   r   r   r   ¡  s    zSELU.extra_repr)F)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r=   {  s
   r=   c                   s8   e Zd ZdZdgZd‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )ÚGLUa3  Applies the gated linear unit function
    :math:`{GLU}(a, b)= a \otimes \sigma(b)` where :math:`a` is the first half
    of the input matrices and :math:`b` is the second half.

    Args:
        dim (int): the dimension on which to split the input. Default: -1

    Shape:
        - Input: :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`

    Examples::

        >>> m = nn.GLU()
        >>> input = torch.randn(4, 2)
        >>> output = m(input)
    Údimr   c                s   t t| ƒjƒ  || _d S )N)r   r>   r   r?   )r   r?   )r   r   r   r   »  s    zGLU.__init__c             C   s   t j|| jƒS )N)r   Zglur?   )r   r   r   r   r   r   ¿  s    zGLU.forwardc             C   s   dj | jƒS )Nzdim={})r   r?   )r   r   r   r   r   Â  s    zGLU.extra_repréÿÿÿÿ)r@   )	r   r   r   r   r    r   r   r   r!   r   r   )r   r   r>   ¦  s
   r>   c                   s8   e Zd ZdZdgZd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )Ú
Hardshrinka°  Applies the hard shrinkage function element-wise:

    .. math::
        \text{HardShrink}(x) =
        \begin{cases}
        x, & \text{ if } x > \lambda \\
        x, & \text{ if } x < -\lambda \\
        0, & \text{ otherwise }
        \end{cases}

    Args:
        lambd: the :math:`\lambda` value for the Hardshrink formulation. Default: 0.5

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Hardshrink.png

    Examples::

        >>> m = nn.Hardshrink()
        >>> input = torch.randn(2)
        >>> output = m(input)
    Úlambdç      à?c                s   t t| ƒjƒ  || _d S )N)r   rA   r   rB   )r   rB   )r   r   r   r   ã  s    zHardshrink.__init__c             C   s   t j|| jƒS )N)r   Z
hardshrinkrB   )r   r   r   r   r   r   ç  s    zHardshrink.forwardc             C   s   dj | jƒS )Nz{})r   rB   )r   r   r   r   r   ê  s    zHardshrink.extra_repr)rC   )	r   r   r   r   r    r   r   r   r!   r   r   )r   r   rA   Æ  s
   rA   c                   s:   e Zd ZdZddgZd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )Ú	LeakyReLUaB  Applies the element-wise function:

    .. math::
        \text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)


    or

    .. math::
        \text{LeakyRELU}(x) =
        \begin{cases}
        x, & \text{ if } x \geq 0 \\
        \text{negative\_slope} \times x, & \text{ otherwise }
        \end{cases}

    Args:
        negative_slope: Controls the angle of the negative slope. Default: 1e-2
        inplace: can optionally do the operation in-place. Default: ``False``

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/LeakyReLU.png

    Examples::

        >>> m = nn.LeakyReLU(0.1)
        >>> input = torch.randn(2)
        >>> output = m(input)
    r   Únegative_slopeç{®Gáz„?Fc                s   t t| ƒjƒ  || _|| _d S )N)r   rD   r   rE   r   )r   rE   r   )r   r   r   r     s    zLeakyReLU.__init__c             C   s   t j|| j| jƒS )N)r   Z
leaky_relurE   r   )r   r   r   r   r   r     s    zLeakyReLU.forwardc             C   s   | j r
dnd}dj| j|ƒS )Nz, inplace=Truer   znegative_slope={}{})r   r   rE   )r   r   r   r   r   r     s    zLeakyReLU.extra_repr)rF   F)	r   r   r   r   r    r   r   r   r!   r   r   )r   r   rD   î  s
    rD   c               @   s   e Zd ZdZdd„ ZdS )Ú
LogSigmoidaÐ  Applies the element-wise function:

    .. math::
        \text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/LogSigmoid.png

    Examples::

        >>> m = nn.LogSigmoid()
        >>> input = torch.randn(2)
        >>> output = m(input)
    c             C   s
   t j|ƒS )N)r   Z
logsigmoid)r   r   r   r   r   r   2  s    zLogSigmoid.forwardN)r   r   r   r   r   r   r   r   r   rG     s   rG   c                   s:   e Zd ZdZddgZd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )ÚSoftplusau  Applies the element-wise function:

    .. math::
        \text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))

    SoftPlus is a smooth approximation to the ReLU function and can be used
    to constrain the output of a machine to always be positive.

    For numerical stability the implementation reverts to the linear function
    for inputs above a certain value.

    Args:
        beta: the :math:`\beta` value for the Softplus formulation. Default: 1
        threshold: values above this revert to a linear function. Default: 20

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Softplus.png

    Examples::

        >>> m = nn.Softplus()
        >>> input = torch.randn(2)
        >>> output = m(input)
    Úbetar   r   é   c                s   t t| ƒjƒ  || _|| _d S )N)r   rH   r   rI   r   )r   rI   r   )r   r   r   r   U  s    zSoftplus.__init__c             C   s   t j|| j| jƒS )N)r   ZsoftplusrI   r   )r   r   r   r   r   r   Z  s    zSoftplus.forwardc             C   s   dj | j| jƒS )Nzbeta={}, threshold={})r   rI   r   )r   r   r   r   r   ]  s    zSoftplus.extra_repr)r   rJ   )	r   r   r   r   r    r   r   r   r!   r   r   )r   r   rH   6  s
   rH   c                   s8   e Zd ZdZdgZd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )Ú
SoftshrinkaÆ  Applies the soft shrinkage function elementwise:

    .. math::
        \text{SoftShrinkage}(x) =
        \begin{cases}
        x - \lambda, & \text{ if } x > \lambda \\
        x + \lambda, & \text{ if } x < -\lambda \\
        0, & \text{ otherwise }
        \end{cases}

    Args:
        lambd: the :math:`\lambda` value for the Softshrink formulation. Default: 0.5

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Softshrink.png

    Examples::

        >>> m = nn.Softshrink()
        >>> input = torch.randn(2)
        >>> output = m(input)
    rB   ç      à?c                s   t t| ƒjƒ  || _d S )N)r   rK   r   rB   )r   rB   )r   r   r   r   ~  s    zSoftshrink.__init__c             C   s   t j|| jƒS )N)r   Z
softshrinkrB   )r   r   r   r   r   r   ‚  s    zSoftshrink.forwardc             C   s
   t | jƒS )N)ÚstrrB   )r   r   r   r   r   …  s    zSoftshrink.extra_repr)rL   )	r   r   r   r   r    r   r   r   r!   r   r   )r   r   rK   a  s
   rK   c                   s4   e Zd ZdZd‡ fdd„	Zdd	„ Zdd
d„Z‡  ZS )ÚMultiheadAttentiona‹  Allows the model to jointly attend to information
    from different representation subspaces.
    See reference: Attention Is All You Need

    .. math::
        \text{MultiHead}(Q, K, V) = \text{Concat}(head_1,\dots,head_h)W^O
        \text{where} head_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V)

    Args:
        embed_dim: total dimension of the model.
        num_heads: parallel attention heads.
        dropout: a Dropout layer on attn_output_weights. Default: 0.0.
        bias: add bias as module parameter. Default: True.
        add_bias_kv: add bias to the key and value sequences at dim=0.
        add_zero_attn: add a new batch of zeros to the key and
                       value sequences at dim=1.
        kdim: total number of features in key. Default: None.
        vdim: total number of features in key. Default: None.

        Note: if kdim and vdim are None, they will be set to embed_dim such that
        query, key, and value have the same number of features.

    Examples::

        >>> multihead_attn = nn.MultiheadAttention(embed_dim, num_heads)
        >>> attn_output, attn_output_weights = multihead_attn(query, key, value)
    ç        TFNc	       	         sZ  t t| ƒjƒ  || _|d k	r |n|| _|d k	r2|n|| _| j|koJ| j|k| _|| _|| _|| | _	| j	| | jks|t
dƒ‚ttjd| |ƒƒ| _| jdkrÖttj||ƒƒ| _ttj|| jƒƒ| _ttj|| jƒƒ| _|rðttjd| ƒƒ| _n| jdd ƒ t|||d�| _|�r<ttjdd|ƒƒ| _ttjdd|ƒƒ| _nd  | _| _|| _| jƒ  d S )Nz(embed_dim must be divisible by num_headsr'   FÚin_proj_bias)Úbiasr   )r   rN   r   Ú	embed_dimÚkdimÚvdimÚ_qkv_same_embed_dimÚ	num_headsÚdropoutZhead_dimr1   r   r5   ÚemptyÚin_proj_weightÚTensorÚq_proj_weightÚk_proj_weightÚv_proj_weightrP   Zregister_parameterr   Úout_projÚbias_kÚbias_vÚadd_zero_attnÚ_reset_parameters)	r   rR   rV   rW   rQ   Zadd_bias_kvra   rS   rT   )r   r   r   r   ¦  s0    

zMultiheadAttention.__init__c             C   s€   | j rt| jƒ nt| jƒ t| jƒ t| jƒ | jd k	rTt| jdƒ t| jj	dƒ | j
d k	rht| j
ƒ | jd k	r|t| jƒ d S )Ng        )rU   r   rY   r[   r\   r]   rP   r   r^   rQ   r_   r   r`   )r   r   r   r   rb   É  s    






z$MultiheadAttention._reset_parametersc             C   sÈ   t | dƒrh| jdkrhtj|||| j| j| j| j| j| j	| j
| j| jj| jj| j|||d| j| j| jd�S t | dƒs~tjdtƒ tj|||| j| j| j| j| j| j	| j
| j| jj| jj| j|||d�S dS )aE  
    Args:
        query, key, value: map a query and a set of key-value pairs to an output.
            See "Attention Is All You Need" for more details.
        key_padding_mask: if provided, specified padding elements in the key will
            be ignored by the attention. This is an binary mask. When the value is True,
            the corresponding value on the attention layer will be filled with -inf.
        need_weights: output attn_output_weights.
        attn_mask: mask that prevents attention to certain positions. This is an additive mask
            (i.e. the values will be added to the attention layer).  

    Shape:
        - Inputs:
        - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
          the embedding dimension.
        - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - key_padding_mask: :math:`(N, S)`, ByteTensor, where N is the batch size, S is the source sequence length.
        - attn_mask: :math:`(L, S)` where L is the target sequence length, S is the source sequence length.

        - Outputs:
        - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
          E is the embedding dimension.
        - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
          L is the target sequence length, S is the source sequence length.
        rU   FT)r(   Úkey_padding_maskÚneed_weightsÚ	attn_maskZuse_separate_proj_weightr[   r\   r]   zƒA new version of MultiheadAttention module has been implemented.                     Please re-train your model with the new module)r(   rc   rd   re   N)ÚhasattrrU   r   Zmulti_head_attention_forwardrR   rV   rY   rP   r_   r`   ra   rW   r^   ÚweightrQ   r(   r[   r\   r]   r/   r0   ÚUserWarning)r   ÚqueryÚkeyr   rc   rd   re   r   r   r   r   Ù  s.    

zMultiheadAttention.forward)rO   TFFNN)NTN)r   r   r   r   r   rb   r   r!   r   r   )r   r   rN   ‰  s
   # rN   c                   s8   e Zd ZdZdgZd‡ fdd„	Zdd„ Zd	d
„ Z‡  ZS )ÚPReLUaÏ  Applies the element-wise function:

    .. math::
        \text{PReLU}(x) = \max(0,x) + a * \min(0,x)

    or

    .. math::
        \text{PReLU}(x) =
        \begin{cases}
        x, & \text{ if } x \geq 0 \\
        ax, & \text{ otherwise }
        \end{cases}

    Here :math:`a` is a learnable parameter. When called without arguments, `nn.PReLU()` uses a single
    parameter :math:`a` across all input channels. If called with `nn.PReLU(nChannels)`,
    a separate :math:`a` is used for each input channel.


    .. note::
        weight decay should not be used when learning :math:`a` for good performance.

    .. note::
        Channel dim is the 2nd dim of input. When input has dims < 2, then there is
        no channel dim and the number of channels = 1.

    Args:
        num_parameters (int): number of :math:`a` to learn.
            Although it takes an int as input, there is only two values are legitimate:
            1, or the number of channels at input. Default: 1
        init (float): the initial value of :math:`a`. Default: 0.25

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    Attributes:
        weight (Tensor): the learnable weights of shape (:attr:`num_parameters`).

    .. image:: scripts/activation_images/PReLU.png

    Examples::

        >>> m = nn.PReLU()
        >>> input = torch.randn(2)
        >>> output = m(input)
    Únum_parametersr   ç      Ð?c                s.   || _ tt| ƒjƒ  ttj|ƒj|ƒƒ| _d S )N)	rl   r   rk   r   r   r5   rZ   Zfill_rg   )r   rl   Úinit)r   r   r   r   E  s    zPReLU.__init__c             C   s   t j|| jƒS )N)r   Zprelurg   )r   r   r   r   r   r   J  s    zPReLU.forwardc             C   s   dj | jƒS )Nznum_parameters={})r   rl   )r   r   r   r   r   M  s    zPReLU.extra_repr)r   rm   )	r   r   r   r   r    r   r   r   r!   r   r   )r   r   rk     s
   0rk   c               @   s   e Zd ZdZdd„ ZdS )ÚSoftsigna²  Applies the element-wise function:

    .. math::
        \text{SoftSign}(x) = \frac{x}{ 1 + |x|}

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Softsign.png

    Examples::

        >>> m = nn.Softsign()
        >>> input = torch.randn(2)
        >>> output = m(input)
    c             C   s
   t j|ƒS )N)r   Zsoftsign)r   r   r   r   r   r   e  s    zSoftsign.forwardN)r   r   r   r   r   r   r   r   r   ro   Q  s   ro   c               @   s   e Zd ZdZdd„ ZdS )Ú
Tanhshrinka¸  Applies the element-wise function:

    .. math::
        \text{Tanhshrink}(x) = x - \text{Tanh}(x)

    Shape:
        - Input: :math:`(N, *)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(N, *)`, same shape as the input

    .. image:: scripts/activation_images/Tanhshrink.png

    Examples::

        >>> m = nn.Tanhshrink()
        >>> input = torch.randn(2)
        >>> output = m(input)
    c             C   s
   t j|ƒS )N)r   Z
tanhshrink)r   r   r   r   r   r   }  s    zTanhshrink.forwardN)r   r   r   r   r   r   r   r   r   rp   i  s   rp   c                   s0   e Zd ZdZdgZd‡ fdd„	Zdd„ Z‡  ZS )	ÚSoftmina9  Applies the Softmin function to an n-dimensional input Tensor
    rescaling them so that the elements of the n-dimensional output Tensor
    lie in the range `[0, 1]` and sum to 1.

    Softmin is defined as:

    .. math::
        \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)}

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    Arguments:
        dim (int): A dimension along which Softmin will be computed (so every slice
            along dim will sum to 1).

    Returns:
        a Tensor of the same dimension and shape as the input, with
        values in the range [0, 1]

    Examples::

        >>> m = nn.Softmin()
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    r?   Nc                s   t t| ƒjƒ  || _d S )N)r   rq   r   r?   )r   r?   )r   r   r   r      s    zSoftmin.__init__c             C   s   t j|| jdd�S )Né   )Ú_stacklevel)r   Zsoftminr?   )r   r   r   r   r   r   ¤  s    zSoftmin.forward)N)r   r   r   r   r    r   r   r!   r   r   )r   r   rq   �  s   rq   c                   s@   e Zd ZdZdgZd‡ fdd„	Zdd„ Zdd	„ Zd
d„ Z‡  Z	S )ÚSoftmaxa   Applies the Softmax function to an n-dimensional input Tensor
    rescaling them so that the elements of the n-dimensional output Tensor
    lie in the range [0,1] and sum to 1.

    Softmax is defined as:

    .. math::
        \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [0, 1]

    Arguments:
        dim (int): A dimension along which Softmax will be computed (so every slice
            along dim will sum to 1).

    .. note::
        This module doesn't work directly with NLLLoss,
        which expects the Log to be computed between the Softmax and itself.
        Use `LogSoftmax` instead (it's faster and has better numerical properties).

    Examples::

        >>> m = nn.Softmax(dim=1)
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    r?   Nc                s   t t| ƒjƒ  || _d S )N)r   rt   r   r?   )r   r?   )r   r   r   r   Ì  s    zSoftmax.__init__c             C   s    | j j|ƒ t| dƒsd | _d S )Nr?   )Ú__dict__Úupdaterf   r?   )r   Ústater   r   r   Ú__setstate__Ð  s    
zSoftmax.__setstate__c             C   s   t j|| jdd�S )Nrr   )rs   )r   Úsoftmaxr?   )r   r   r   r   r   r   Õ  s    zSoftmax.forwardc             C   s   dj | jd�S )Nz	dim={dim})r?   )r   r?   )r   r   r   r   r   Ø  s    zSoftmax.extra_repr)N)
r   r   r   r   r    r   rx   r   r   r!   r   r   )r   r   rt   ¨  s   !rt   c               @   s   e Zd ZdZdd„ ZdS )Ú	Softmax2daQ  Applies SoftMax over features to each spatial location.

    When given an image of ``Channels x Height x Width``, it will
    apply `Softmax` to each location :math:`(Channels, h_i, w_j)`

    Shape:
        - Input: :math:`(N, C, H, W)`
        - Output: :math:`(N, C, H, W)` (same shape as input)

    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [0, 1]

    Examples::

        >>> m = nn.Softmax2d()
        >>> # you softmax over the 2nd dimension
        >>> input = torch.randn(2, 3, 12, 13)
        >>> output = m(input)
    c             C   s$   |j ƒ dkstdƒ‚tj|ddd�S )Né   z'Softmax2d requires a 4D tensor as inputr   rr   )rs   )r?   r1   r   ry   )r   r   r   r   r   r   ò  s    zSoftmax2d.forwardN)r   r   r   r   r   r   r   r   r   rz   Ü  s   rz   c                   s8   e Zd ZdZdgZd
‡ fdd„	Zdd„ Zdd	„ Z‡  ZS )Ú
LogSoftmaxaÜ  Applies the :math:`\log(\text{Softmax}(x))` function to an n-dimensional
    input Tensor. The LogSoftmax formulation can be simplified as:

    .. math::
        \text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)

    Shape:
        - Input: :math:`(*)` where `*` means, any number of additional
          dimensions
        - Output: :math:`(*)`, same shape as the input

    Arguments:
        dim (int): A dimension along which LogSoftmax will be computed.

    Returns:
        a Tensor of the same dimension and shape as the input with
        values in the range [-inf, 0)

    Examples::

        >>> m = nn.LogSoftmax()
        >>> input = torch.randn(2, 3)
        >>> output = m(input)
    r?   Nc                s   t t| ƒjƒ  || _d S )N)r   r|   r   r?   )r   r?   )r   r   r   r     s    zLogSoftmax.__init__c             C   s    | j j|ƒ t| dƒsd | _d S )Nr?   )ru   rv   rf   r?   )r   rw   r   r   r   rx     s    
zLogSoftmax.__setstate__c             C   s   t j|| jdd�S )Nrr   )rs   )r   Zlog_softmaxr?   )r   r   r   r   r   r     s    zLogSoftmax.forward)N)	r   r   r   r   r    r   rx   r   r!   r   r   )r   r   r|   ÷  s
   r|   )&r/   r5   r   r   Ztorch.nn.initr   r   r   Ztorch.nn.parameterr   Úmoduler   r
   r   r   r"   r#   r+   r3   r4   r6   r8   r;   r=   r>   rA   rD   rG   rH   rK   rN   rk   ro   rp   rq   rt   rz   r|   r   r   r   r   Ú<module>   sB   /*7>&++ (0+( 
?'4