3
I4W^¸M ã            
   @   s  d Z ddlmZ ddlZddlZddlZddlmZmZ ddl	m
Z ddlmZ ddlmZmZmZmZ dd	l	mZ dd
l	mZ ddlmZmZ eejdƒZeejdƒZeejdƒZeejdƒZeejdƒZeejdƒZeejdƒZeejdƒZeej j!j"dƒZ"eej j!j#dƒZ#�ddd„Z$�d	dd„Z%eddde$e%e&dd�Z'�d
d d!„Z(�dd"d#„Z)eddde(e)e&d$d�Z*�dd%d&„Z+�dd'd(„Z,edd)de+e,e&d*d�Z-�dd+d,„Z.�dd-d.„Z/edd)de.e/e&d/d�Z0�dd0d1„Z1�dd2d3„Z2edd)de1e2e&d4d�Z3d5d6„ Z4�dd7d8„Z5�dd9d:„Z6�dd;d<„Z7�dd=d>„Z8�dd?d@„Z9�ddAdB„Z:�ddCdD„Z;eddde:e;e&dEd�Z<�ddFdG„Z=�ddHdI„Z>eddde=e>e&dJd�Z?�ddKdL„Z@�ddMdN„ZAeddde@eAe&dOd�ZBeejCdPƒZCdQdR„ ZDdSdT„ ZE�ddWdX„ZF�ddYdZ„ZG�dd[d\„ZH�d d]d^„ZI�d!d_d`„ZJ�d"dadb„ZKeejLdcƒZL�d#ddde„ZMeejNdfƒZN�d%dgdh„ZO�d'djdk„ZPeej j!jQdlƒZQ�d(dmdn„ZR�d)dodp„ZSeej j!jTdqƒZT�d*drds„ZUeejVdtƒZV�d+dudv„ZWeejXdwƒZX�d,dydz„ZYeej j!jZd{ƒZZd|d}„ Z[�d/d€d�„Z\eej]d‚ƒZ]eej j!j^dƒƒZ_d„d…„ Z`�d0d†d‡„Zadˆd‰„ ZbdŠd‹„ Zceej j!jddŒƒZdd�dŽ„ Ze�d1d�d�„Zf�d2d‘d’„Zg�d4d”d•„Zh�d5d–d—„Zieej j!jjd˜ƒZjd™dš„ Zkd›dœ„ Zl�d6d�dž„Zm�d7dŸd „Znd¡d¢„ Zo�d8d¤d¥„Zp�d9d§d¨„Zq�d:d«d¬„Zr�d;d­d®„Zs�d<d¯d°„Zt�d=d±d²„Zu�d>dµd¶„Zv�d?d·d¸„Zw�dAdºd»„Zx�dBd½d¾„Zy�dCd¿dÀ„Zz�dEdÁdÂ„Z{�dFdÃdÄ„Z|�dGdÅdÆ„Z}�dHdÇdÈ„Z~dÉdÊ„ Z�dIdËdÌ„Z€�dJdÍdÎ„Z��dKdÏdÐ„Z‚�dLdÑdÒ„Zƒ�dMdÓdÔ„Z„�dNdÕdÖ„Z…�dOd×dØ„Z†�dPdÙdÚ„Z‡�dQdÛdÜ„Zˆ�dRdÝdÞ„Z‰eejŠdßƒZŠ�dSdádâ„Z‹�dTdãdä„ZŒ�dUdådæ„Z��dVdçdè„ZŽdddéœZ�ddddêœZ��dWdìdí„Z‘�dXdîdï„Z’�dYdñdò„Z“�dZdôdõ„Z”eej•döƒZ•eej–d÷ƒZ–eej j!j—døƒZ—�d[dùdú„Z˜�d\düdý„Z™dþdÿ„ Zš�d]�d �d„Z›�d^�d�d„Zœ�d�d„ Z��d_�d�d„ZždS (`  zFunctional interfaceé    )ÚdivisionN)Ú_infer_sizeÚ_add_docstré   )Ú
_reduction)Úutils)Ú_singleÚ_pairÚ_tripleÚ_list_with_default)Úgrad)Ú_VFé   )Úboolean_dispatchÚLista¢  
conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor

Applies a 1D convolution over an input signal composed of several input
planes.

See :class:`~torch.nn.Conv1d` for details and output shape.

.. include:: cudnn_deterministic.rst

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)`
    weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kW)`
    bias: optional bias of shape :math:`(\text{out\_channels})`. Default: ``None``
    stride: the stride of the convolving kernel. Can be a single number or
      a one-element tuple `(sW,)`. Default: 1
    padding: implicit paddings on both sides of the input. Can be a
      single number or a one-element tuple `(padW,)`. Default: 0
    dilation: the spacing between kernel elements. Can be a single number or
      a one-element tuple `(dW,)`. Default: 1
    groups: split input into groups, :math:`\text{in\_channels}` should be divisible by
      the number of groups. Default: 1

Examples::

    >>> filters = torch.randn(33, 16, 3)
    >>> inputs = torch.randn(20, 16, 50)
    >>> F.conv1d(inputs, filters)
aÎ  
conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor

Applies a 2D convolution over an input image composed of several input
planes.

See :class:`~torch.nn.Conv2d` for details and output shape.

.. include:: cudnn_deterministic.rst

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)`
    weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kH , kW)`
    bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: ``None``
    stride: the stride of the convolving kernel. Can be a single number or a
      tuple `(sH, sW)`. Default: 1
    padding: implicit paddings on both sides of the input. Can be a
      single number or a tuple `(padH, padW)`. Default: 0
    dilation: the spacing between kernel elements. Can be a single number or
      a tuple `(dH, dW)`. Default: 1
    groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the
      number of groups. Default: 1

Examples::

    >>> # With square kernels and equal stride
    >>> filters = torch.randn(8,4,3,3)
    >>> inputs = torch.randn(1,4,5,5)
    >>> F.conv2d(inputs, filters, padding=1)
a»  
conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor

Applies a 3D convolution over an input image composed of several input
planes.

See :class:`~torch.nn.Conv3d` for details and output shape.

.. include:: cudnn_deterministic.rst

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)`
    weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kT , kH , kW)`
    bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: None
    stride: the stride of the convolving kernel. Can be a single number or a
      tuple `(sT, sH, sW)`. Default: 1
    padding: implicit paddings on both sides of the input. Can be a
      single number or a tuple `(padT, padH, padW)`. Default: 0
    dilation: the spacing between kernel elements. Can be a single number or
      a tuple `(dT, dH, dW)`. Default: 1
    groups: split input into groups, :math:`\text{in\_channels}` should be divisible by
      the number of groups. Default: 1

Examples::

    >>> filters = torch.randn(33, 16, 3, 3, 3)
    >>> inputs = torch.randn(20, 16, 50, 10, 20)
    >>> F.conv3d(inputs, filters)
aÖ  
conv_transpose1d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor

Applies a 1D transposed convolution operator over an input signal
composed of several input planes, sometimes also called "deconvolution".

See :class:`~torch.nn.ConvTranspose1d` for details and output shape.

.. include:: cudnn_deterministic.rst

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)`
    weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kW)`
    bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None
    stride: the stride of the convolving kernel. Can be a single number or a
      tuple ``(sW,)``. Default: 1
    padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both
      sides of each dimension in the input. Can be a single number or a tuple
      ``(padW,)``. Default: 0
    output_padding: additional size added to one side of each dimension in the
      output shape. Can be a single number or a tuple ``(out_padW)``. Default: 0
    groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the
      number of groups. Default: 1
    dilation: the spacing between kernel elements. Can be a single number or
      a tuple ``(dW,)``. Default: 1

Examples::

    >>> inputs = torch.randn(20, 16, 50)
    >>> weights = torch.randn(16, 33, 5)
    >>> F.conv_transpose1d(inputs, weights)
a5  
conv_transpose2d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor

Applies a 2D transposed convolution operator over an input image
composed of several input planes, sometimes also called "deconvolution".

See :class:`~torch.nn.ConvTranspose2d` for details and output shape.

.. include:: cudnn_deterministic.rst

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)`
    weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kH , kW)`
    bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None
    stride: the stride of the convolving kernel. Can be a single number or a
      tuple ``(sH, sW)``. Default: 1
    padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both
      sides of each dimension in the input. Can be a single number or a tuple
      ``(padH, padW)``. Default: 0
    output_padding: additional size added to one side of each dimension in the
      output shape. Can be a single number or a tuple ``(out_padH, out_padW)``.
      Default: 0
    groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the
      number of groups. Default: 1
    dilation: the spacing between kernel elements. Can be a single number or
      a tuple ``(dH, dW)``. Default: 1

Examples::

    >>> # With square kernels and equal stride
    >>> inputs = torch.randn(1, 4, 5, 5)
    >>> weights = torch.randn(4, 8, 3, 3)
    >>> F.conv_transpose2d(inputs, weights, padding=1)
a'  
conv_transpose3d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor

Applies a 3D transposed convolution operator over an input image
composed of several input planes, sometimes also called "deconvolution"

See :class:`~torch.nn.ConvTranspose3d` for details and output shape.

.. include:: cudnn_deterministic.rst

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)`
    weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kT , kH , kW)`
    bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None
    stride: the stride of the convolving kernel. Can be a single number or a
      tuple ``(sT, sH, sW)``. Default: 1
    padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both
      sides of each dimension in the input. Can be a single number or a tuple
      ``(padT, padH, padW)``. Default: 0
    output_padding: additional size added to one side of each dimension in the
      output shape. Can be a single number or a tuple
      ``(out_padT, out_padH, out_padW)``. Default: 0
    groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the
      number of groups. Default: 1
    dilation: the spacing between kernel elements. Can be a single number or
      a tuple `(dT, dH, dW)`. Default: 1

Examples::

    >>> inputs = torch.randn(20, 16, 50, 10, 20)
    >>> weights = torch.randn(16, 33, 3, 3, 3)
    >>> F.conv_transpose3d(inputs, weights)
aÔ  
Applies a 1-dimensional sequence convolution over an input sequence.
Input and output dimensions are (Time, Batch, Channels) - hence TBC.

Args:
    input: input tensor of shape :math:`(\text{sequence length} \times batch \times \text{in\_channels})`
    weight: filter of shape (:math:`\text{kernel width} \times \text{in\_channels} \times \text{out\_channels}`)
    bias: bias of shape (:math:`\text{out\_channels}`)
    pad: number of timesteps to pad. Default: 0
az  
avg_pool1d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True) -> Tensor

Applies a 1D average pooling over an input signal composed of several
input planes.

See :class:`~torch.nn.AvgPool1d` for details and output shape.

Args:
    input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)`
    kernel_size: the size of the window. Can be a single number or a
      tuple `(kW,)`
    stride: the stride of the window. Can be a single number or a tuple
      `(sW,)`. Default: :attr:`kernel_size`
    padding: implicit zero paddings on both sides of the input. Can be a
      single number or a tuple `(padW,)`. Default: 0
    ceil_mode: when True, will use `ceil` instead of `floor` to compute the
        output shape. Default: ``False``
    count_include_pad: when True, will include the zero-padding in the
        averaging calculation. Default: ``True``

Examples::

    >>> # pool of square window of size=3, stride=2
    >>> input = torch.tensor([[[1, 2, 3, 4, 5, 6, 7]]], dtype=torch.float32)
    >>> F.avg_pool1d(input, kernel_size=3, stride=2)
    tensor([[[ 2.,  4.,  6.]]])

a¸  
avg_pool2d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor

Applies 2D average-pooling operation in :math:`kH \times kW` regions by step size
:math:`sH \times sW` steps. The number of output features is equal to the number of
input planes.

See :class:`~torch.nn.AvgPool2d` for details and output shape.

Args:
    input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)`
    kernel_size: size of the pooling region. Can be a single number or a
      tuple `(kH, kW)`
    stride: stride of the pooling operation. Can be a single number or a
      tuple `(sH, sW)`. Default: :attr:`kernel_size`
    padding: implicit zero paddings on both sides of the input. Can be a
      single number or a tuple `(padH, padW)`. Default: 0
    ceil_mode: when True, will use `ceil` instead of `floor` in the formula
        to compute the output shape. Default: ``False``
    count_include_pad: when True, will include the zero-padding in the
        averaging calculation. Default: ``True``
    divisor_override: if specified, it will be used as divisor, otherwise
         size of the pooling region will be used. Default: None
aÖ  
avg_pool3d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor

Applies 3D average-pooling operation in :math:`kT \times kH \times kW` regions by step
size :math:`sT \times sH \times sW` steps. The number of output features is equal to
:math:`\lfloor\frac{\text{input planes}}{sT}\rfloor`.

See :class:`~torch.nn.AvgPool3d` for details and output shape.

Args:
    input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iT \times iH , iW)`
    kernel_size: size of the pooling region. Can be a single number or a
      tuple `(kT, kH, kW)`
    stride: stride of the pooling operation. Can be a single number or a
      tuple `(sT, sH, sW)`. Default: :attr:`kernel_size`
    padding: implicit zero paddings on both sides of the input. Can be a
      single number or a tuple `(padT, padH, padW)`, Default: 0
    ceil_mode: when True, will use `ceil` instead of `floor` in the formula
        to compute the output shape
    count_include_pad: when True, will include the zero-padding in the
        averaging calculation
    divisor_override: if specified, it will be used as divisor, otherwise
        size of the pooling region will be used. Default: None
Fc             C   s    |dkr|dkrt dƒ‚|dkr`|dk	s,t‚t|ƒ}t| jdƒ|d  ƒt| jdƒ|d  ƒg}|dkrŒtj| jdƒ| jdƒd| j| jd�}tj	j
j| |||ƒS )ab  Applies 2D fractional max pooling over an input signal composed of several input planes.

    Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham

    The max-pooling operation is applied in :math:`kH \times kW` regions by a stochastic
    step size determined by the target output size.
    The number of output features is equal to the number of input planes.

    Args:
        kernel_size: the size of the window to take a max over.
                     Can be a single number :math:`k` (for a square kernel of :math:`k \times k`)
                     or a tuple `(kH, kW)`
        output_size: the target output size of the image of the form :math:`oH \times oW`.
                     Can be a tuple `(oH, oW)` or a single number :math:`oH` for a square image :math:`oH \times oH`
        output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given.
                      This has to be a number or tuple in the range (0, 1)
        return_indices: if ``True``, will return the indices along with the outputs.
                        Useful to pass to :func:`~torch.nn.functional.max_unpool2d`.

    Examples::
        >>> input = torch.randn(20, 16, 50, 32)
        >>> # pool of square window of size=3, and target output size 13x12
        >>> F.fractional_max_pool2d(input, 3, output_size=(13, 12))
        >>> # pool of square window and target output size being half of input image size
        >>> F.fractional_max_pool2d(input, 3, output_ratio=(0.5, 0.5))

    .. _Fractional MaxPooling:
        http://arxiv.org/abs/1412.6071
    NzRfractional_max_pool2d requires specifying either an output_size or an output_ratior   r   é   r   )ÚdtypeÚdevice)Ú
ValueErrorÚAssertionErrorr	   ÚintÚsizeÚtorchÚrandr   r   Ú_CÚ_nnÚfractional_max_pool2d)ÚinputÚkernel_sizeÚoutput_sizeÚoutput_ratioÚreturn_indicesÚ_random_samplesÚ_output_ratio© r$   ú4/tmp/pip-build-e66j97b2/torch/torch/nn/functional.pyÚ"fractional_max_pool2d_with_indices1  s    !$r&   c             C   s   t | |||||ƒd S )Nr   )r&   )r   r   r   r    r!   r"   r$   r$   r%   Ú_fractional_max_pool2d`  s    r'   r!   é   r   )Úarg_nameZ	arg_indexÚdefaultÚif_trueÚif_falseÚmodule_nameÚ	func_namec             C   s´   |dkr|dkrt dƒ‚|dkrt|dk	s,t‚t|ƒ}t| jdƒ|d  ƒt| jdƒ|d  ƒt| jdƒ|d  ƒg}|dkr tj| jdƒ| jdƒd| j| jd�}tj	j
j| |||ƒS )	a¢  Applies 3D fractional max pooling over an input signal composed of several input planes.

    Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham

    The max-pooling operation is applied in :math:`kT \times kH \times kW` regions by a stochastic
    step size determined by the target output size.
    The number of output features is equal to the number of input planes.

    Args:
        kernel_size: the size of the window to take a max over.
                     Can be a single number :math:`k` (for a square kernel of :math:`k \times k \times k`)
                     or a tuple `(kT, kH, kW)`
        output_size: the target output size of the form :math:`oT \times oH \times oW`.
                     Can be a tuple `(oT, oH, oW)` or a single number :math:`oH` for a cubic output
                      :math:`oH \times oH \times oH`
        output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given.
                      This has to be a number or tuple in the range (0, 1)
        return_indices: if ``True``, will return the indices along with the outputs.
                        Useful to pass to :func:`~torch.nn.functional.max_unpool3d`.

    Examples::
        >>> input = torch.randn(20, 16, 50, 32, 16)
        >>> # pool of cubic window of size=3, and target output size 13x12x11
        >>> F.fractional_max_pool3d(input, 3, output_size=(13, 12, 11))
        >>> # pool of cubic window and target output size being half of input size
        >>> F.fractional_max_pool3d(input, 3, output_ratio=(0.5, 0.5, 0.5))

    .. _Fractional MaxPooling:
        http://arxiv.org/abs/1412.6071
    NzRfractional_max_pool3d requires specifying either an output_size or an output_ratior   r   r   r   r(   )r   r   )r   r   r
   r   r   r   r   r   r   r   r   Úfractional_max_pool3d)r   r   r   r    r!   r"   r#   r$   r$   r%   Ú"fractional_max_pool3d_with_indicesr  s    "$r0   c             C   s   t | |||||ƒd S )Nr   )r0   )r   r   r   r    r!   r"   r$   r$   r%   Ú_fractional_max_pool3d£  s    r1   r/   c             C   s.   |dkrt jjtt g ƒ}t j| |||||ƒS )z‹Applies a 1D max pooling over an input signal composed of several input
    planes.

    See :class:`~torch.nn.MaxPool1d` for details.
    N)r   ÚjitÚannotater   r   Úmax_pool1d_with_indices)r   r   ÚstrideÚpaddingÚdilationÚ	ceil_moder!   r$   r$   r%   r4   µ  s    r4   c             C   s.   |d krt jjtt g ƒ}t j| |||||ƒS )N)r   r2   r3   r   r   Ú
max_pool1d)r   r   r5   r6   r7   r8   r!   r$   r$   r%   Ú_max_pool1dÃ  s    r:   é   r9   c             C   s2   |dkrt jjtt g ƒ}t jjj| |||||ƒS )z‹Applies a 2D max pooling over an input signal composed of several input
    planes.

    See :class:`~torch.nn.MaxPool2d` for details.
    N)r   r2   r3   r   r   r   r   Úmax_pool2d_with_indices)r   r   r5   r6   r7   r8   r!   r$   r$   r%   r<   Õ  s    r<   c             C   s.   |d krt jjtt g ƒ}t j| |||||ƒS )N)r   r2   r3   r   r   Ú
max_pool2d)r   r   r5   r6   r7   r8   r!   r$   r$   r%   Ú_max_pool2dâ  s    r>   r=   c             C   s2   |dkrt jjtt g ƒ}t jjj| |||||ƒS )z‹Applies a 3D max pooling over an input signal composed of several input
    planes.

    See :class:`~torch.nn.MaxPool3d` for details.
    N)r   r2   r3   r   r   r   r   Úmax_pool3d_with_indices)r   r   r5   r6   r7   r8   r!   r$   r$   r%   r?   ô  s    r?   c             C   s.   |d krt jjtt g ƒ}t j| |||||ƒS )N)r   r2   r3   r   r   Ú
max_pool3d)r   r   r5   r6   r7   r8   r!   r$   r$   r%   Ú_max_pool3d  s    rA   r@   c             C   s2  | j ƒ }tjjtt g ƒ}xFtt|ƒƒD ]6}|j||d  d ||  ||  d||   ƒ q(W |d krp|}n¾t|ƒt|ƒd kr�|dd … }t|ƒt|ƒkrÂt	dj
t|ƒt|ƒd t|ƒƒƒ‚xftt|ƒƒD ]V}|| ||  }	|| ||  }
|	||   k �o|
k n  sÐt	dj
|||	|
ƒƒ‚qÐW |}|S )Nr   r   zZoutput_size should be a sequence containing {} or {} elements, but it has a length of '{}'z;invalid output_size "{}" (dim {} must be between {} and {}))r   r   r2   r3   r   r   ÚrangeÚlenÚappendr   Úformat)r   r   r5   r6   r   Z
input_sizeZdefault_sizeÚdÚretZmin_sizeÚmax_sizer$   r$   r%   Ú_unpool_output_size  s,    2rI   c             C   sx   t |ƒ}|dk	rt |ƒ}n|}t |ƒ}t| ||||ƒ}t|tƒrL|dg }n|d }tjjj| jdƒ|jdƒ|ƒj	dƒS )zkComputes a partial inverse of :class:`MaxPool1d`.

    See :class:`~torch.nn.MaxUnpool1d` for details.
    Nr   r   )r   )
r   rI   Ú
isinstanceÚlistr   r   r   Úmax_unpool2dÚ	unsqueezeÚsqueeze)r   Úindicesr   r5   r6   r   Ú_strider$   r$   r%   Úmax_unpool1d1  s    


rQ   c             C   sH   t |ƒ}|dk	rt |ƒ}n|}t |ƒ}t| ||||ƒ}tjjj| ||ƒS )zkComputes a partial inverse of :class:`MaxPool2d`.

    See :class:`~torch.nn.MaxUnpool2d` for details.
    N)r	   rI   r   r   r   rL   )r   rO   r   r5   r6   r   rP   r$   r$   r%   rL   H  s    

rL   c             C   sL   t |ƒ}|dk	rt |ƒ}n|}t |ƒ}t| ||||ƒ}tjjj| ||||ƒS )zkComputes a partial inverse of :class:`MaxPool3d`.

    See :class:`~torch.nn.MaxUnpool3d` for details.
    N)r
   rI   r   r   r   Úmax_unpool3d)r   rO   r   r5   r6   r   rP   r$   r$   r%   rR   Z  s    

rR   c             C   sp   t j|ƒ\}}|dk	r.t| j|ƒ||d|ƒ}nt| j|ƒ|d|d�}tj|ƒttj|ƒƒ j|| ƒjd| ƒS )zóApplies a 2D power-average pooling over an input signal composed of
    several input planes. If the sum of all inputs to the power of `p` is
    zero, the gradient is set to zero as well.

    See :class:`~torch.nn.LPPool2d` for details.
    Nr   )r6   r8   g      ð?)	r   r	   Ú
avg_pool2dÚpowr   ÚsignÚreluÚabsÚmul)r   Ú	norm_typer   r5   r8   ÚkwZkhÚoutr$   r$   r%   Ú	lp_pool2dm  s
    r\   c             C   s^   |dk	r t | j|ƒ||d|ƒ}nt | j|ƒ|d|d�}tj|ƒttj|ƒƒ j|ƒjd| ƒS )zóApplies a 1D power-average pooling over an input signal composed of
    several input planes. If the sum of all inputs to the power of `p` is
    zero, the gradient is set to zero as well.

    See :class:`~torch.nn.LPPool1d` for details.
    Nr   )r6   r8   g      ð?)Ú
avg_pool1drT   r   rU   rV   rW   rX   )r   rY   r   r5   r8   r[   r$   r$   r%   Ú	lp_pool1d~  s    r^   c             C   s   t j| |ƒS )aC  Applies a 1D adaptive max pooling over an input signal composed of
    several input planes.

    See :class:`~torch.nn.AdaptiveMaxPool1d` for details and output shape.

    Args:
        output_size: the target output size (single integer)
        return_indices: whether to return pooling indices. Default: ``False``
    )r   Úadaptive_max_pool1d)r   r   r!   r$   r$   r%   Ú adaptive_max_pool1d_with_indicesŽ  s    r`   c             C   s   t | |ƒd S )Nr   )r`   )r   r   r!   r$   r$   r%   Ú_adaptive_max_pool1dœ  s    ra   r_   c             C   s   t || jƒ ƒ}tjjj| |ƒS )ag  Applies a 2D adaptive max pooling over an input signal composed of
    several input planes.

    See :class:`~torch.nn.AdaptiveMaxPool2d` for details and output shape.

    Args:
        output_size: the target output size (single integer or
            double-integer tuple)
        return_indices: whether to return pooling indices. Default: ``False``
    )r   r   r   r   r   Úadaptive_max_pool2d)r   r   r!   r$   r$   r%   Ú adaptive_max_pool2d_with_indicesª  s    rc   c             C   s   t | |ƒd S )Nr   )rc   )r   r   r!   r$   r$   r%   Ú_adaptive_max_pool2dº  s    rd   rb   c             C   s   t || jƒ ƒ}tjjj| |ƒS )ag  Applies a 3D adaptive max pooling over an input signal composed of
    several input planes.

    See :class:`~torch.nn.AdaptiveMaxPool3d` for details and output shape.

    Args:
        output_size: the target output size (single integer or
            triple-integer tuple)
        return_indices: whether to return pooling indices. Default: ``False``
    )r   r   r   r   r   Úadaptive_max_pool3d)r   r   r!   r$   r$   r%   Ú adaptive_max_pool3d_with_indicesÈ  s    rf   c             C   s   t | |ƒd S )Nr   )rf   )r   r   r!   r$   r$   r%   Ú_adaptive_max_pool3dØ  s    rg   re   a  
adaptive_avg_pool1d(input, output_size) -> Tensor

Applies a 1D adaptive average pooling over an input signal composed of
several input planes.

See :class:`~torch.nn.AdaptiveAvgPool1d` for details and output shape.

Args:
    output_size: the target output size (single integer)
c             C   s   t || jƒ ƒ}tjjj| |ƒS )a"  
    Applies a 2D adaptive average pooling over an input signal composed of
    several input planes.

    See :class:`~torch.nn.AdaptiveAvgPool2d` for details and output shape.

    Args:
        output_size: the target output size (single integer or
            double-integer tuple)
    )r   r   r   r   r   Úadaptive_avg_pool2d)r   r   Ú_output_sizer$   r$   r%   rh   ó  s    rh   c             C   s   t || jƒ ƒ}tjjj| |ƒS )a"  
    Applies a 3D adaptive average pooling over an input signal composed of
    several input planes.

    See :class:`~torch.nn.AdaptiveAvgPool3d` for details and output shape.

    Args:
        output_size: the target output size (single integer or
            triple-integer tuple)
    )r   r   r   r   r   Úadaptive_avg_pool3d)r   r   ri   r$   r$   r%   rj     s    rj   ç      à?Tc             C   s>   |dk s|dkrt dj|ƒƒ‚|r0tj| ||ƒS tj| ||ƒS )aº  
    During training, randomly zeroes some of the elements of the input
    tensor with probability :attr:`p` using samples from a Bernoulli
    distribution.

    See :class:`~torch.nn.Dropout` for details.

    Args:
        p: probability of an element to be zeroed. Default: 0.5
        training: apply dropout if is ``True``. Default: ``True``
        inplace: If set to ``True``, will do this operation in-place. Default: ``False``
    g        g      ð?z9dropout probability has to be between 0 and 1, but got {})r   rE   r   Zdropout_Údropout)r   ÚpÚtrainingÚinplacer$   r$   r%   rl     s
    rl   c             C   s>   |dk s|dkrt dj|ƒƒ‚|r0tj| ||ƒS tj| ||ƒS )z^Applies alpha dropout to the input.

    See :class:`~torch.nn.AlphaDropout` for details.
    g        g      ð?z9dropout probability has to be between 0 and 1, but got {})r   rE   r   Zalpha_dropout_Úalpha_dropout)r   rm   rn   ro   r$   r$   r%   rp   *  s
    rp   c             C   s>   |dk s|dkrt dj|ƒƒ‚|r0tj| ||ƒS tj| ||ƒS )a�  
    Randomly zero out entire channels (a channel is a 2D feature map,
    e.g., the :math:`j`-th channel of the :math:`i`-th sample in the
    batched input is a 2D tensor :math:`\text{input}[i, j]`) of the input tensor).
    Each channel will be zeroed out independently on every forward call with
    probability :attr:`p` using samples from a Bernoulli distribution.

    See :class:`~torch.nn.Dropout2d` for details.

    Args:
        p: probability of a channel to be zeroed. Default: 0.5
        training: apply dropout if is ``True``. Default: ``True``
        inplace: If set to ``True``, will do this operation in-place. Default: ``False``
    g        g      ð?z9dropout probability has to be between 0 and 1, but got {})r   rE   r   Úfeature_dropout_Úfeature_dropout)r   rm   rn   ro   r$   r$   r%   Ú	dropout2d8  s
    rs   c             C   s>   |dk s|dkrt dj|ƒƒ‚|r0tj| ||ƒS tj| ||ƒS )a�  
    Randomly zero out entire channels (a channel is a 3D feature map,
    e.g., the :math:`j`-th channel of the :math:`i`-th sample in the
    batched input is a 3D tensor :math:`\text{input}[i, j]`) of the input tensor).
    Each channel will be zeroed out independently on every forward call with
    probability :attr:`p` using samples from a Bernoulli distribution.

    See :class:`~torch.nn.Dropout3d` for details.

    Args:
        p: probability of a channel to be zeroed. Default: 0.5
        training: apply dropout if is ``True``. Default: ``True``
        inplace: If set to ``True``, will do this operation in-place. Default: ``False``
    g        g      ð?z9dropout probability has to be between 0 and 1, but got {})r   rE   r   rq   rr   )r   rm   rn   ro   r$   r$   r%   Ú	dropout3dP  s
    rt   c             C   s>   |dk s|dkrt dj|ƒƒ‚|r0tj| ||ƒS tj| ||ƒS )Ng        g      ð?z9dropout probability has to be between 0 and 1, but got {})r   rE   r   Zfeature_alpha_dropout_Úfeature_alpha_dropout)r   rm   rn   ro   r$   r$   r%   ru   j  s
    ru   c             C   s&   |rt j| ||ƒ}nt j| ||ƒ}|S )ziThresholds each element of the input Tensor.

    See :class:`~torch.nn.Threshold` for more details.
    )r   Ú
threshold_Ú	threshold)r   rw   Úvaluero   Úresultr$   r$   r%   rw   t  s    rw   zX
threshold_(input, threshold, value) -> Tensor

In-place version of :func:`~threshold`.
c             C   s   |rt j| ƒ}n
t j| ƒ}|S )z™relu(input, inplace=False) -> Tensor

    Applies the rectified linear unit function element-wise. See
    :class:`~torch.nn.ReLU` for more details.
    )r   Úrelu_rV   )r   ro   ry   r$   r$   r%   rV   ˆ  s    
rV   z<
relu_(input) -> Tensor

In-place version of :func:`~relu`.
c             C   s$   | j ƒ dkrtdƒ‚tjjj| |ƒS )a  
    glu(input, dim=-1) -> Tensor

    The gated linear unit. Computes:

    .. math ::
        \text{GLU}(a, b) = a \otimes \sigma(b)

    where `input` is split in half along `dim` to form `a` and `b`, :math:`\sigma`
    is the sigmoid function and :math:`\otimes` is the element-wise product between matrices.

    See `Language Modeling with Gated Convolutional Networks <https://arxiv.org/abs/1612.08083>`_.

    Args:
        input (Tensor): input tensor
        dim (int): dimension on which to split the input. Default: -1
    r   z?glu does not suppport scalars because halving size must be even)ÚdimÚRuntimeErrorr   r   r   Úglu)r   r{   r$   r$   r%   r}   �  s    r}   ç      ð?c             C   s.   |rt jjj| ||ƒ}nt jjj| ||ƒ}|S )z²
    hardtanh(input, min_val=-1., max_val=1., inplace=False) -> Tensor

    Applies the HardTanh function element-wise. See :class:`~torch.nn.Hardtanh` for more
    details.
    )r   r   r   Ú	hardtanh_Úhardtanh)r   Zmin_valZmax_valro   ry   r$   r$   r%   r€   µ  s    r€   z]
hardtanh_(input, min_val=-1., max_val=1.) -> Tensor

In-place version of :func:`~hardtanh`.
c             C   s   t | dd|ƒS )z³relu6(input, inplace=False) -> Tensor

    Applies the element-wise function :math:`\text{ReLU6}(x) = \min(\max(0,x), 6)`.

    See :class:`~torch.nn.ReLU6` for more details.
    g        g      @)r€   )r   ro   r$   r$   r%   Úrelu6Ë  s    r�   c             C   s*   |rt jjj| |ƒ}nt jjj| |ƒ}|S )z•Applies element-wise,
    :math:`\text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))`.

    See :class:`~torch.nn.ELU` for more details.
    )r   r   r   Úelu_Úelu)r   Úalpharo   ry   r$   r$   r%   rƒ   Ö  s    rƒ   zD
elu_(input, alpha=1.) -> Tensor

In-place version of :func:`~elu`.
c             C   s   |rt j| ƒ}n
t j| ƒ}|S )a>  selu(input, inplace=False) -> Tensor

    Applies element-wise,
    :math:`\text{SELU}(x) = scale * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))`,
    with :math:`\alpha=1.6732632423543772848170429916717` and
    :math:`scale=1.0507009873554804934193349852946`.

    See :class:`~torch.nn.SELU` for more details.
    )r   Úselu_Úselu)r   ro   ry   r$   r$   r%   r†   ë  s    
r†   z<
selu_(input) -> Tensor

In-place version of :func:`~selu`.
c             C   s"   |rt j| |ƒ}nt j| |ƒ}|S )zÒcelu(input, alpha=1., inplace=False) -> Tensor

    Applies element-wise,
    :math:`\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))`.

    See :class:`~torch.nn.CELU` for more details.
    )r   Úcelu_Úcelu)r   r„   ro   ry   r$   r$   r%   rˆ     s    	rˆ   zF
celu_(input, alpha=1.) -> Tensor

In-place version of :func:`~celu`.
ç{®Gáz„?c             C   s*   |rt jjj| |ƒ}nt jjj| |ƒ}|S )zï
    leaky_relu(input, negative_slope=0.01, inplace=False) -> Tensor

    Applies element-wise,
    :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)`

    See :class:`~torch.nn.LeakyReLU` for more details.
    )r   r   r   Úleaky_relu_Ú
leaky_relu)r   Znegative_slopero   ry   r$   r$   r%   r‹     s    
r‹   z]
leaky_relu_(input, negative_slope=0.01) -> Tensor

In-place version of :func:`~leaky_relu`.
c             C   s   t j| |ƒS )zíprelu(input, weight) -> Tensor

    Applies element-wise the function
    :math:`\text{PReLU}(x) = \max(0,x) + \text{weight} * \min(0,x)` where weight is a
    learnable parameter.

    See :class:`~torch.nn.PReLU` for more details.
    )r   Úprelu)r   Úweightr$   r$   r%   rŒ   2  s    
rŒ   é   r   c             C   s*   |rt j| |||ƒ}nt j| |||ƒ}|S )z¢rrelu(input, lower=1./8, upper=1./3, training=False, inplace=False) -> Tensor

    Randomized leaky ReLU.

    See :class:`~torch.nn.RReLU` for more details.
    )r   Úrrelu_Úrrelu)r   ÚlowerÚupperrn   ro   ry   r$   r$   r%   r�   ?  s    r�   zf
rrelu_(input, lower=1./8, upper=1./3, training=False) -> Tensor

In-place version of :func:`~rrelu`.
z´
logsigmoid(input) -> Tensor

Applies element-wise :math:`\text{LogSigmoid}(x_i) = \log \left(\frac{1}{1 + \exp(-x_i)}\right)`

See :class:`~torch.nn.LogSigmoid` for more details.
c             C   s   t jjj| ƒS )a  gelu(input) -> Tensor

    Applies element-wise the function
    :math:`\text{GeLU}(x) = x * \Phi(x)`

    where :math:`\Phi(x)` is the Cumulative Distribution Function for Gaussian Distribution.

    See `Gaussian Error Linear Units (GELUs) <https://arxiv.org/abs/1606.08415>`_.
    )r   r   r   Úgelu)r   r$   r$   r%   r“   \  s    
r“   c             C   s   t j| |ƒS )zŸ
    hardshrink(input, lambd=0.5) -> Tensor

    Applies the hard shrinkage function element-wise

    See :class:`~torch.nn.Hardshrink` for more details.
    )r   Ú
hardshrink)r   Úlambdr$   r$   r%   r”   i  s    	r”   c             C   s   | | j ƒ  S )z¦tanhshrink(input) -> Tensor

    Applies element-wise, :math:`\text{Tanhshrink}(x) = x - \text{Tanh}(x)`

    See :class:`~torch.nn.Tanhshrink` for more details.
    )Útanh)r   r$   r$   r%   Ú
tanhshrinku  s    r—   c             C   s   | | j ƒ d  S )z¬softsign(input) -> Tensor

    Applies element-wise, the function :math:`\text{SoftSign}(x) = \frac{x}{1 + |x|}`

    See :class:`~torch.nn.Softsign` for more details.
    r   )rW   )r   r$   r$   r%   Úsoftsign  s    r˜   z1
softplus(input, beta=1, threshold=20) -> Tensor
c             C   s:   t jdj| ƒ|d� |dks,|dks,|dkr2d}nd}|S )NzfImplicit dimension choice for {} has been deprecated. Change the call to include dim=X as an argument.)Ú
stacklevelr   r   r   )ÚwarningsÚwarnrE   )ÚnameÚndimr™   rG   r$   r$   r%   Ú_get_softmax_dimŽ  s    rž   c             C   sB   |dkrt d| jƒ |ƒ}|dkr.|  j|ƒ}n|  j||d�}|S )a{  Applies a softmin function.

    Note that :math:`\text{Softmin}(x) = \text{Softmax}(-x)`. See softmax definition for mathematical formula.

    See :class:`~torch.nn.Softmin` for more details.

    Arguments:
        input (Tensor): input
        dim (int): A dimension along which softmin will be computed (so every slice
            along dim will sum to 1).
        dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor.
          If specified, the input tensor is casted to :attr:`dtype` before the operation
          is performed. This is useful for preventing data type overflows. Default: None.
    NÚsoftmin)r   )rž   r{   Úsoftmax)r   r{   Ú_stacklevelr   rG   r$   r$   r%   rŸ   ™  s    rŸ   c             C   s>   |dkrt d| jƒ |ƒ}|dkr,| j|ƒ}n| j||d�}|S )a¦  Applies a softmax function.

    Softmax is defined as:

    :math:`\text{Softmax}(x_{i}) = \frac{exp(x_i)}{\sum_j exp(x_j)}`

    It is applied to all slices along dim, and will re-scale them so that the elements
    lie in the range `[0, 1]` and sum to 1.

    See :class:`~torch.nn.Softmax` for more details.

    Arguments:
        input (Tensor): input
        dim (int): A dimension along which softmax will be computed.
        dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor.
          If specified, the input tensor is casted to :attr:`dtype` before the operation
          is performed. This is useful for preventing data type overflows. Default: None.

    .. note::
        This function doesn't work directly with NLLLoss,
        which expects the Log to be computed between the Softmax and itself.
        Use log_softmax instead (it's faster and has better numerical properties).

    Nr    )r   )rž   r{   r    )r   r{   r¡   r   rG   r$   r$   r%   r    ²  s    r    ç»½×Ùß|Û=c       
      C   s€   |dkrt jdƒ tj| ƒjƒ jƒ  }| | | }|j|ƒ}|rx|j|dd�d }tj| ƒj	||dƒ}||j
ƒ  | }	n|}	|	S )a  
    Samples from the Gumbel-Softmax distribution (`Link 1`_  `Link 2`_) and optionally discretizes.

    Args:
      logits: `[..., num_features]` unnormalized log probabilities
      tau: non-negative scalar temperature
      hard: if ``True``, the returned samples will be discretized as one-hot vectors,
            but will be differentiated as if it is the soft sample in autograd
      dim (int): A dimension along which softmax will be computed. Default: -1.

    Returns:
      Sampled tensor of same shape as `logits` from the Gumbel-Softmax distribution.
      If ``hard=True``, the returned samples will be one-hot, otherwise they will
      be probability distributions that sum to 1 across `dim`.

    .. note::
      This function is here for legacy reasons, may be removed from nn.Functional in the future.

    .. note::
      The main trick for `hard` is to do  `y_hard - y_soft.detach() + y_soft`

      It achieves two things:
      - makes the output value exactly one-hot
      (since we add then subtract y_soft value)
      - makes the gradient equal to y_soft gradient
      (since we strip all other gradients)

    Examples::
        >>> logits = torch.randn(20, 32)
        >>> # Sample soft categorical using reparametrization trick:
        >>> F.gumbel_softmax(logits, tau=1, hard=False)
        >>> # Sample hard categorical using "Straight-through" trick:
        >>> F.gumbel_softmax(logits, tau=1, hard=True)

    .. _Link 1:
        https://arxiv.org/abs/1611.00712
    .. _Link 2:
        https://arxiv.org/abs/1611.01144
    g»½×Ùß|Û=z0`eps` parameter is deprecated and has no effect.T)Úkeepdimr   g      ð?)rš   r›   r   Z
empty_likeZexponential_Úlogr    ÚmaxZ
zeros_likeZscatter_Údetach)
ZlogitsÚtauÚhardÚepsr{   ZgumbelsZy_softÚindexZy_hardrG   r$   r$   r%   Úgumbel_softmaxÕ  s    *

r«   c             C   s>   |dkrt d| jƒ |ƒ}|dkr,| j|ƒ}n| j||d�}|S )aÔ  Applies a softmax followed by a logarithm.

    While mathematically equivalent to log(softmax(x)), doing these two
    operations separately is slower, and numerically unstable. This function
    uses an alternative formulation to compute the output and gradient correctly.

    See :class:`~torch.nn.LogSoftmax` for more details.

    Arguments:
        input (Tensor): input
        dim (int): A dimension along which log_softmax will be computed.
        dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor.
          If specified, the input tensor is casted to :attr:`dtype` before the operation
          is performed. This is useful for preventing data type overflows. Default: None.
    NÚlog_softmax)r   )rž   r{   r¬   )r   r{   r¡   r   rG   r$   r$   r%   r¬     s    r¬   zŽ
softshrink(input, lambd=0.5) -> Tensor

Applies the soft shrinkage function elementwise

See :class:`~torch.nn.Softshrink` for more details.
c             C   s   t jdƒ | jƒ S )z¾tanh(input) -> Tensor

    Applies element-wise,
    :math:`\text{Tanh}(x) = \tanh(x) = \frac{\exp(x) - \exp(-x)}{\exp(x) + \exp(-x)}`

    See :class:`~torch.nn.Tanh` for more details.
    z9nn.functional.tanh is deprecated. Use torch.tanh instead.)rš   r›   r–   )r   r$   r$   r%   r–   4  s    
r–   c             C   s   t jdƒ | jƒ S )z­sigmoid(input) -> Tensor

    Applies the element-wise function :math:`\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}`

    See :class:`~torch.nn.Sigmoid` for more details.
    z?nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.)rš   r›   Úsigmoid)r   r$   r$   r%   r­   @  s    
r­   c             C   sN   | j ƒ dkr(|dk	r(tj|| |jƒ ƒ}n"| j|jƒ ƒ}|dk	rF||7 }|}|S )a^  
    Applies a linear transformation to the incoming data: :math:`y = xA^T + b`.

    Shape:

        - Input: :math:`(N, *, in\_features)` where `*` means any number of
          additional dimensions
        - Weight: :math:`(out\_features, in\_features)`
        - Bias: :math:`(out\_features)`
        - Output: :math:`(N, *, out\_features)`
    r   N)r{   r   ZaddmmÚtÚmatmul)r   r�   ÚbiasrG   Úoutputr$   r$   r%   ÚlinearK  s    r²   c             C   s   t j| |||ƒS )aÕ  
    Applies a bilinear transformation to the incoming data:
    :math:`y = x_1 A x_2 + b`

    Shape:

        - input1: :math:`(N, *, H_{in1})` where :math:`H_{in1}=\text{in1\_features}`
          and :math:`*` means any number of additional dimensions.
          All but the last dimension of the inputs should be the same.
        - input2: :math:`(N, *, H_{in2})` where :math:`H_{in2}=\text{in2\_features}`
        - weight: :math:`(\text{out\_features}, \text{in1\_features},
          \text{in2\_features})`
        - bias: :math:`(\text{out\_features})`
        - output: :math:`(N, *, H_{out})` where :math:`H_{out}=\text{out\_features}`
          and all but the last dimension are the same shape as the input.
    )r   Úbilinear)Úinput1Úinput2r�   r°   r$   r$   r%   r³   c  s    r³   c             C   s(   t jƒ � t j| |||ƒ W d Q R X d S )N)r   Zno_gradZembedding_renorm_)r�   r   Úmax_normrY   r$   r$   r%   Ú_no_grad_embedding_renorm_x  s    
r·   ç       @c             C   sŒ   |dk	rX|dkr(||j dƒk sVtdƒ‚q\|dk r\||j dƒ ksHtdƒ‚|j dƒ| }nd}|dk	rz| jƒ } t|| ||ƒ tj|| |||ƒS )a  A simple lookup table that looks up embeddings in a fixed dictionary and size.

    This module is often used to retrieve word embeddings using indices.
    The input to the module is a list of indices, and the embedding matrix,
    and the output is the corresponding word embeddings.

    See :class:`torch.nn.Embedding` for more details.

    Args:
        input (LongTensor): Tensor containing indices into the embedding matrix
        weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1,
            and number of columns equal to the embedding size
        padding_idx (int, optional): If given, pads the output with the embedding vector at :attr:`padding_idx`
                                         (initialized to zeros) whenever it encounters the index.
        max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm`
                                    is renormalized to have norm :attr:`max_norm`.
                                    Note: this will modify :attr:`weight` in-place.
        norm_type (float, optional): The p of the p-norm to compute for the :attr:`max_norm` option. Default ``2``.
        scale_grad_by_freq (boolean, optional): If given, this will scale gradients by the inverse of frequency of
                                                the words in the mini-batch. Default ``False``.
        sparse (bool, optional): If ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under
                                 :class:`torch.nn.Embedding` for more details regarding sparse gradients.

    Shape:
        - Input: LongTensor of arbitrary shape containing the indices to extract
        - Weight: Embedding matrix of floating point type with shape `(V, embedding_dim)`,
                            where V = maximum index + 1 and embedding_dim = the embedding size
        - Output: `(*, embedding_dim)`, where `*` is the input shape

    Examples::

        >>> # a batch of 2 samples of 4 indices each
        >>> input = torch.tensor([[1,2,4,5],[4,3,2,9]])
        >>> # an embedding matrix containing 10 tensors of size 3
        >>> embedding_matrix = torch.rand(10, 3)
        >>> F.embedding(input, embedding_matrix)
        tensor([[[ 0.8490,  0.9625,  0.6753],
                 [ 0.9666,  0.7761,  0.6108],
                 [ 0.6246,  0.9751,  0.3618],
                 [ 0.4161,  0.2419,  0.7383]],

                [[ 0.6246,  0.9751,  0.3618],
                 [ 0.0237,  0.7794,  0.0528],
                 [ 0.9666,  0.7761,  0.6108],
                 [ 0.3385,  0.8612,  0.1867]]])

        >>> # example with padding_idx
        >>> weights = torch.rand(10, 3)
        >>> weights[0, :].zero_()
        >>> embedding_matrix = weights
        >>> input = torch.tensor([[0,2,0,5]])
        >>> F.embedding(input, embedding_matrix, padding_idx=0)
        tensor([[[ 0.0000,  0.0000,  0.0000],
                 [ 0.5609,  0.5384,  0.8720],
                 [ 0.0000,  0.0000,  0.0000],
                 [ 0.6262,  0.2438,  0.7471]]])
    Nr   z)Padding_idx must be within num_embeddingsr   éÿÿÿÿ)r   r   Ú
contiguousr·   r   Ú	embedding)r   r�   Zpadding_idxr¶   rY   Úscale_grad_by_freqÚsparser$   r$   r%   r»   ~  s    <r»   Úmeanc	             C   s  |j tjkr(| jƒ r(tjdƒ | | }} |dk	rT| jƒ |jƒ krTtdj|j	| j	ƒƒ‚| j
ƒ dkrº|dk	rztdjt|ƒƒƒ‚tjd| jƒ | jdƒtj| jd�}| jdƒ} |dk	r¸|jdƒ}n¤| j
ƒ dk�rL|dkrØtd	ƒ‚|j
ƒ dkrìtd
ƒ‚t|d ƒdk�rtdj|d jƒ ƒƒ‚t|d ƒ| jdƒk�r^tdj| jdƒ|d jƒ ƒƒ‚ntdj| j
ƒ ƒƒ‚|dk�rnd}	nD|dk�r~d}	n4|dk�rªd}	|�rštdƒ‚|�r²tdƒ‚ntdƒ‚|dk	�rÊt|| ||ƒ |dk	�rì|dk�rìtdj|ƒƒ‚tj|| |||	||ƒ\}
}}}|
S )a4  Computes sums, means or maxes of `bags` of embeddings, without instantiating the
    intermediate embeddings.

    See :class:`torch.nn.EmbeddingBag` for more details.

    .. include:: cuda_deterministic_backward.rst

    Args:
        input (LongTensor): Tensor containing bags of indices into the embedding matrix
        weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1,
            and number of columns equal to the embedding size
        offsets (LongTensor, optional): Only used when :attr:`input` is 1D. :attr:`offsets` determines
                             the starting index position of each bag (sequence) in :attr:`input`.
        max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm`
                                    is renormalized to have norm :attr:`max_norm`.
                                    Note: this will modify :attr:`weight` in-place.
        norm_type (float, optional): The ``p`` in the ``p``-norm to compute for the :attr:`max_norm` option.
                                     Default ``2``.
        scale_grad_by_freq (boolean, optional): if given, this will scale gradients by the inverse of frequency of
                                                the words in the mini-batch. Default ``False``.
                                                Note: this option is not supported when ``mode="max"``.
        mode (string, optional): ``"sum"``, ``"mean"`` or ``"max"``. Specifies the way to reduce the bag.
                                 Default: ``"mean"``
        sparse (bool, optional): if ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under
                                 :class:`torch.nn.Embedding` for more details regarding sparse gradients.
                                 Note: this option is not supported when ``mode="max"``.
        per_sample_weights (Tensor, optional): a tensor of float / double weights, or None
            to indicate all weights should be taken to be 1. If specified, :attr:`per_sample_weights`
            must have exactly the same shape as input and is treated as having the same
            :attr:`offsets`, if those are not None.


    Shape:

        - :attr:`input` (LongTensor) and :attr:`offsets` (LongTensor, optional)

          - If :attr:`input` is 2D of shape `(B, N)`,

            it will be treated as ``B`` bags (sequences) each of fixed length ``N``, and
            this will return ``B`` values aggregated in a way depending on the :attr:`mode`.
            :attr:`offsets` is ignored and required to be ``None`` in this case.

          - If :attr:`input` is 1D of shape `(N)`,

            it will be treated as a concatenation of multiple bags (sequences).
            :attr:`offsets` is required to be a 1D tensor containing the
            starting index positions of each bag in :attr:`input`. Therefore,
            for :attr:`offsets` of shape `(B)`, :attr:`input` will be viewed as
            having ``B`` bags. Empty bags (i.e., having 0-length) will have
            returned vectors filled by zeros.

        - :attr:`weight` (Tensor): the learnable weights of the module of
          shape `(num_embeddings, embedding_dim)`

        - :attr:`per_sample_weights` (Tensor, optional). Has the same shape as
          :attr:`input`.

        - :attr:`output`: aggregated embedding values of shape `(B, embedding_dim)`

    Examples::

        >>> # an Embedding module containing 10 tensors of size 3
        >>> embedding_matrix = torch.rand(10, 3)
        >>> # a batch of 2 samples of 4 indices each
        >>> input = torch.tensor([1,2,4,5,4,3,2,9])
        >>> offsets = torch.tensor([0,4])
        >>> F.embedding_bag(embedding_matrix, input, offsets)
        tensor([[ 0.3397,  0.3552,  0.5545],
                [ 0.5893,  0.4386,  0.5882]])
    zªArgument order of nn.functional.embedding_bag was changed. Usage `embedding_bag(weight, input, ...)` is deprecated, and should now be `embedding_bag(input, weight, ...)`.Nziembedding_bag: If per_sample_weights ({}) is not None, then it must have the same shape as the input ({})r   z�if input is 2D, then offsets has to be None, as input is treated is a mini-batch of fixed length sequences. However, found offsets of type {}r   r   )r   r   z*offsets has to be a 1D Tensor but got Nonezoffsets has to be a 1D Tensorzpoffsets[0] has to be 0, i.e., the first sequence in the mini-batch has to start from position 0. However, got {}zRoffsets[-1] can not be greater than input's length ({}), but got offsets[-1] of {}z?input has to be 1D or 2D Tensor, but got Tensor of dimension {}Úsumr¾   r¥   z?max mode does not support scaling the gradient by the frequencyz(max mode does not support sparse weightsz&mode has to be one of sum, mean or maxz�embedding_bag: per_sample_weights was not None. per_sample_weights is only supported for mode='sum' (got mode='{}'). Please open a feature request on GitHub.r¹   r¹   r¹   r¹   )r   r   ÚlongÚis_floating_pointrš   r›   r   r   rE   Úshaper{   ÚtypeZarangeÚnumelr   Zreshaper   Úitemr·   ÚNotImplementedErrorÚembedding_bag)r   r�   Úoffsetsr¶   rY   r¼   Úmoder½   Zper_sample_weightsÚ	mode_enumrG   Ú_r$   r$   r%   rÇ   Ï  sj    M







rÇ   çš™™™™™¹?çñhãˆµøä>c          
   C   st   |rT| j ƒ }|d }	x(tt|ƒd ƒD ]}
|	||
d  9 }	q&W |	dkrTtdj|ƒƒ‚tj| |||||||tjjj	ƒ	S )zÂApplies Batch Normalization for each channel across a batch of data.

    See :class:`~torch.nn.BatchNorm1d`, :class:`~torch.nn.BatchNorm2d`,
    :class:`~torch.nn.BatchNorm3d` for details.
    r   r   r   zGExpected more than 1 value per channel when training, got input size {})
r   rB   rC   r   rE   r   Ú
batch_normÚbackendsÚcudnnÚenabled)r   Úrunning_meanÚrunning_varr�   r°   rn   Úmomentumr©   r   Z
size_prodsÚir$   r$   r%   rÎ   k  s    

rÎ   c          
   C   s    t j| |||||||t jjjƒ	S )zÚApplies Instance Normalization for each channel in each data sample in a
    batch.

    See :class:`~torch.nn.InstanceNorm1d`, :class:`~torch.nn.InstanceNorm2d`,
    :class:`~torch.nn.InstanceNorm3d` for details.
    )r   Úinstance_normrÏ   rÐ   rÑ   )r   rÒ   rÓ   r�   r°   Zuse_input_statsrÔ   r©   r$   r$   r%   rÖ   Š  s    	
rÖ   c             C   s   t j| ||||t jjjƒS )zzApplies Layer Normalization for last certain number of dimensions.

    See :class:`~torch.nn.LayerNorm` for details.
    )r   Ú
layer_normrÏ   rÐ   rÑ   )r   Znormalized_shaper�   r°   r©   r$   r$   r%   r×   ™  s    r×   c             C   s   t j| ||||t jjjƒS )zzApplies Group Normalization for last certain number of dimensions.

    See :class:`~torch.nn.GroupNorm` for details.
    )r   Ú
group_normrÏ   rÐ   rÑ   )r   Z
num_groupsr�   r°   r©   r$   r$   r%   rØ   £  s    rØ   ç-Cëâ6?ç      è?c          	   C   sø   | j ƒ }|dk rtdj|ƒƒ‚| j| ƒjdƒ}|dkrnt|dd|d |d d fƒ}t||dfdd�jdƒ}nl| jƒ }|j	|d d|d |d dƒ}t|dddd|d |d d fƒ}t
||ddfdd�jdƒ}|j	|ƒ}|j|ƒj|ƒj|ƒ}| | S )zöApplies local response normalization over an input signal composed of
    several input planes, where channels occupy the second dimension.
    Applies normalization across channels.

    See :class:`~torch.nn.LocalResponseNorm` for details.
    r   zWExpected 3D or higher dimensionality                          input (got {} dimensions)r   r   r   )r5   r¹   )r{   r   rE   rX   rM   ÚpadrS   rN   r   ÚviewÚ
avg_pool3dÚaddrT   )r   r   r„   ÚbetaÚkr{   ÚdivÚsizesr$   r$   r%   Úlocal_response_norm­  s    "
rã   c             C   s   t j| ||||tj|ƒ|ƒS )a¬  The Connectionist Temporal Classification loss.

    See :class:`~torch.nn.CTCLoss` for details.

    .. include:: cudnn_deterministic.rst
    .. include:: cuda_deterministic_backward.rst

    Args:
        log_probs: :math:`(T, N, C)` where `C = number of characters in alphabet including blank`,
            `T = input length`, and `N = batch size`.
            The logarithmized probabilities of the outputs
            (e.g. obtained with :func:`torch.nn.functional.log_softmax`).
        targets: :math:`(N, S)` or `(sum(target_lengths))`.
            Targets cannot be blank. In the second form, the targets are assumed to be concatenated.
        input_lengths: :math:`(N)`.
            Lengths of the inputs (must each be :math:`\leq T`)
        target_lengths: :math:`(N)`.
            Lengths of the targets
        blank (int, optional):
            Blank label. Default :math:`0`.
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,
            ``'mean'``: the output losses will be divided by the target lengths and
            then the mean over the batch is taken, ``'sum'``: the output will be
            summed. Default: ``'mean'``
        zero_infinity (bool, optional):
            Whether to zero infinite losses and the associated gradients.
            Default: ``False``
            Infinite losses mainly occur when the inputs are too short
            to be aligned to the targets.

    Example::

        >>> log_probs = torch.randn(50, 16, 20).log_softmax(2).detach().requires_grad_()
        >>> targets = torch.randint(1, 20, (16, 30), dtype=torch.long)
        >>> input_lengths = torch.full((16,), 50, dtype=torch.long)
        >>> target_lengths = torch.randint(10,30,(16,), dtype=torch.long)
        >>> loss = F.ctc_loss(log_probs, targets, input_lengths, target_lengths)
        >>> loss.backward()
    )r   Úctc_lossÚ
_ReductionÚget_enum)Z	log_probsÚtargetsZinput_lengthsZtarget_lengthsZblankÚ	reductionZzero_infinityr$   r$   r%   rä   É  s    +rä   éd   c             C   sŒ  |dk	s|dk	rt j||ƒ}| jƒ }|dk r:tdj|ƒƒ‚| jdƒ|jdƒkrjtdj| jdƒ|jdƒƒƒ‚|dkr�tjjj	| ||t j
|ƒ|ƒ}nø|dkr¶tjjj| ||t j
|ƒ|ƒ}nÒ| jdƒ}	| jdƒ}
|	f| jƒ dd…  }|jƒ dd… | jƒ dd… k�rtdj||jƒ ƒƒ‚| jƒ j|	|
dd
ƒ} |jƒ j|	ddƒ}t j
|ƒ}|d	k�rhtjjj| ||||ƒ}n tjjj| ||||ƒ}|j|ƒ}|S )aì	  The negative log likelihood loss.

    See :class:`~torch.nn.NLLLoss` for details.

    Args:
        input: :math:`(N, C)` where `C = number of classes` or :math:`(N, C, H, W)`
            in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)` where :math:`K \geq 1`
            in the case of K-dimensional loss.
        target: :math:`(N)` where each value is :math:`0 \leq \text{targets}[i] \leq C-1`,
            or :math:`(N, d_1, d_2, ..., d_K)` where :math:`K \geq 1` for
            K-dimensional loss.
        weight (Tensor, optional): a manual rescaling weight given to each
            class. If given, has to be a Tensor of size `C`
        size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,
            the losses are averaged over each loss element in the batch. Note that for
            some losses, there multiple elements per sample. If the field :attr:`size_average`
            is set to ``False``, the losses are instead summed for each minibatch. Ignored
            when reduce is ``False``. Default: ``True``
        ignore_index (int, optional): Specifies a target value that is ignored
            and does not contribute to the input gradient. When :attr:`size_average` is
            ``True``, the loss is averaged over non-ignored targets. Default: -100
        reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the
            losses are averaged or summed over observations for each minibatch depending
            on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per
            batch element instead and ignores :attr:`size_average`. Default: ``True``
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,
            ``'mean'``: the sum of the output will be divided by the number of
            elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`. Default: ``'mean'``

    Example::

        >>> # input is of size N x C = 3 x 5
        >>> input = torch.randn(3, 5, requires_grad=True)
        >>> # each element in target has to have 0 <= value < C
        >>> target = torch.tensor([1, 0, 4])
        >>> output = F.nll_loss(F.log_softmax(input), target)
        >>> output.backward()
    Nr   z&Expected 2 or more dimensions (got {})r   z?Expected input batch_size ({}) to match target batch_size ({}).r(   r   zExpected target size {}, got {}Únoner¹   r¹   )rå   Úlegacy_get_stringr{   r   rE   r   r   r   r   Únll_lossræ   Z
nll_loss2drº   rÜ   )r   Útargetr�   Úsize_averageÚignore_indexÚreducerè   r{   rG   ÚnÚcZout_sizeÚreduction_enumr[   r$   r$   r%   rì   ø  s8    ,

"


rì   ç:Œ0âŽyE>c       	      C   sb   |dk	s|dk	rt j||ƒ}|dkrD|dkrD|dkrD| }t|d ƒ‚tj| ||||t j|ƒƒ}|S )aŒ  Poisson negative log likelihood loss.

    See :class:`~torch.nn.PoissonNLLLoss` for details.

    Args:
        input: expectation of underlying Poisson distribution.
        target: random sample :math:`target \sim \text{Poisson}(input)`.
        log_input: if ``True`` the loss is computed as
            :math:`\exp(\text{input}) - \text{target} * \text{input}`, if ``False`` then loss is
            :math:`\text{input} - \text{target} * \log(\text{input}+\text{eps})`. Default: ``True``
        full: whether to compute full loss, i. e. to add the Stirling
            approximation term. Default: ``False``
            :math:`\text{target} * \log(\text{target}) - \text{target} + 0.5 * \log(2 * \pi * \text{target})`.
        size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,
            the losses are averaged over each loss element in the batch. Note that for
            some losses, there multiple elements per sample. If the field :attr:`size_average`
            is set to ``False``, the losses are instead summed for each minibatch. Ignored
            when reduce is ``False``. Default: ``True``
        eps (float, optional): Small value to avoid evaluation of :math:`\log(0)` when
            :attr:`log_input`=``False``. Default: 1e-8
        reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the
            losses are averaged or summed over observations for each minibatch depending
            on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per
            batch element instead and ignores :attr:`size_average`. Default: ``True``
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,
            ``'mean'``: the sum of the output will be divided by the number of
            elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`. Default: ``'mean'``

    Nrê   r¾   r¿   z is not valid)rå   rë   r   r   Úpoisson_nll_lossræ   )	r   rí   Z	log_inputÚfullrî   r©   rð   rè   rG   r$   r$   r%   rõ   F  s    #rõ   c             C   s„   |dk	s|dk	rt j||ƒ}n0|dkr0tjdƒ |dkrDt jdƒ}n
t j|ƒ}tj| ||ƒ}|dkr€| jƒ dkr€|| jƒ d  }|S )a|  The `Kullback-Leibler divergence`_ Loss.

    See :class:`~torch.nn.KLDivLoss` for details.

    Args:
        input: Tensor of arbitrary shape
        target: Tensor of the same shape as input
        size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,
            the losses are averaged over each loss element in the batch. Note that for
            some losses, there multiple elements per sample. If the field :attr:`size_average`
            is set to ``False``, the losses are instead summed for each minibatch. Ignored
            when reduce is ``False``. Default: ``True``
        reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the
            losses are averaged or summed over observations for each minibatch depending
            on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per
            batch element instead and ignores :attr:`size_average`. Default: ``True``
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'batchmean'`` | ``'sum'`` | ``'mean'``.
            ``'none'``: no reduction will be applied
            ``'batchmean'``: the sum of the output will be divided by the batchsize
            ``'sum'``: the output will be summed
            ``'mean'``: the output will be divided by the number of elements in the output
            Default: ``'mean'``

    .. note::
        :attr:`size_average` and :attr:`reduce` are in the process of being deprecated,
        and in the meantime, specifying either of those two args will override :attr:`reduction`.

    .. note::
        :attr:``reduction`` = ``'mean'`` doesn't return the true kl divergence value, please use
        :attr:``reduction`` = ``'batchmean'`` which aligns with KL math definition.
        In the next major release, ``'mean'`` will be changed to be the same as 'batchmean'.
    Nr¾   zÿreduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.Z	batchmeanr¿   r   )	rå   Úlegacy_get_enumrš   r›   ræ   r   Úkl_divr{   r   )r   rí   rî   rð   rè   ró   Zreducedr$   r$   r%   rø   s  s    #

rø   c             C   s6   |dk	s|dk	rt j||ƒ}tt| dƒ||d|d|ƒS )aÎ	  This criterion combines `log_softmax` and `nll_loss` in a single
    function.

    See :class:`~torch.nn.CrossEntropyLoss` for details.

    Args:
        input (Tensor) : :math:`(N, C)` where `C = number of classes` or :math:`(N, C, H, W)`
            in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)` where :math:`K \geq 1`
            in the case of K-dimensional loss.
        target (Tensor) : :math:`(N)` where each value is :math:`0 \leq \text{targets}[i] \leq C-1`,
            or :math:`(N, d_1, d_2, ..., d_K)` where :math:`K \geq 1` for
            K-dimensional loss.
        weight (Tensor, optional): a manual rescaling weight given to each
            class. If given, has to be a Tensor of size `C`
        size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,
            the losses are averaged over each loss element in the batch. Note that for
            some losses, there multiple elements per sample. If the field :attr:`size_average`
            is set to ``False``, the losses are instead summed for each minibatch. Ignored
            when reduce is ``False``. Default: ``True``
        ignore_index (int, optional): Specifies a target value that is ignored
            and does not contribute to the input gradient. When :attr:`size_average` is
            ``True``, the loss is averaged over non-ignored targets. Default: -100
        reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the
            losses are averaged or summed over observations for each minibatch depending
            on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per
            batch element instead and ignores :attr:`size_average`. Default: ``True``
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,
            ``'mean'``: the sum of the output will be divided by the number of
            elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`. Default: ``'mean'``

    Examples::

        >>> input = torch.randn(3, 5, requires_grad=True)
        >>> target = torch.randint(5, (3,), dtype=torch.int64)
        >>> loss = F.cross_entropy(input, target)
        >>> loss.backward()
    Nr   )rå   rë   rì   r¬   )r   rí   r�   rî   rï   rð   rè   r$   r$   r%   Úcross_entropy¬  s    +rù   c             C   s¶   |dk	s|dk	rt j||ƒ}n
t j|ƒ}|jƒ | jƒ krVtjdj|jƒ | jƒ ƒdd� | jƒ |jƒ kr~tdj|jƒ | jƒ ƒƒ‚|dk	r¢t	|jƒ |jƒ ƒ}|j
|ƒ}tjjj| |||ƒS )aƒ  Function that measures the Binary Cross Entropy
    between the target and the output.

    See :class:`~torch.nn.BCELoss` for details.

    Args:
        input: Tensor of arbitrary shape
        target: Tensor of the same shape as input
        weight (Tensor, optional): a manual rescaling weight
                if provided it's repeated to match input tensor shape
        size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,
            the losses are averaged over each loss element in the batch. Note that for
            some losses, there multiple elements per sample. If the field :attr:`size_average`
            is set to ``False``, the losses are instead summed for each minibatch. Ignored
            when reduce is ``False``. Default: ``True``
        reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the
            losses are averaged or summed over observations for each minibatch depending
            on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per
            batch element instead and ignores :attr:`size_average`. Default: ``True``
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,
            ``'mean'``: the sum of the output will be divided by the number of
            elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`. Default: ``'mean'``

    Examples::

        >>> input = torch.randn((3, 2), requires_grad=True)
        >>> target = torch.rand((3, 2), requires_grad=False)
        >>> loss = F.binary_cross_entropy(F.sigmoid(input), target)
        >>> loss.backward()
    NzwUsing a target size ({}) that is different to the input size ({}) is deprecated. Please ensure they have the same size.r   )r™   zcTarget and input must have the same number of elements. target nelement ({}) != input nelement ({}))rå   r÷   ræ   r   rš   r›   rE   rÄ   r   r   Úexpandr   r   r   Úbinary_cross_entropy)r   rí   r�   rî   rð   rè   ró   Znew_sizer$   r$   r%   rû   Ü  s    $

rû   c             C   sb   |dk	s|dk	rt j||ƒ}n
t j|ƒ}|jƒ | jƒ ksPtdj|jƒ | jƒ ƒƒ‚tj| ||||ƒS )a	  Function that measures Binary Cross Entropy between target and output
    logits.

    See :class:`~torch.nn.BCEWithLogitsLoss` for details.

    Args:
        input: Tensor of arbitrary shape
        target: Tensor of the same shape as input
        weight (Tensor, optional): a manual rescaling weight
            if provided it's repeated to match input tensor shape
        size_average (bool, optional): Deprecated (see :attr:`reduction`). By default,
            the losses are averaged over each loss element in the batch. Note that for
            some losses, there multiple elements per sample. If the field :attr:`size_average`
            is set to ``False``, the losses are instead summed for each minibatch. Ignored
            when reduce is ``False``. Default: ``True``
        reduce (bool, optional): Deprecated (see :attr:`reduction`). By default, the
            losses are averaged or summed over observations for each minibatch depending
            on :attr:`size_average`. When :attr:`reduce` is ``False``, returns a loss per
            batch element instead and ignores :attr:`size_average`. Default: ``True``
        reduction (string, optional): Specifies the reduction to apply to the output:
            ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied,
            ``'mean'``: the sum of the output will be divided by the number of
            elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average`
            and :attr:`reduce` are in the process of being deprecated, and in the meantime,
            specifying either of those two args will override :attr:`reduction`. Default: ``'mean'``
        pos_weight (Tensor, optional): a weight of positive examples.
                Must be a vector with length equal to the number of classes.

    Examples::

         >>> input = torch.randn(3, requires_grad=True)
         >>> target = torch.empty(3).random_(2)
         >>> loss = F.binary_cross_entropy_with_logits(input, target)
         >>> loss.backward()
    Nz4Target size ({}) must be the same as input size ({}))rå   r÷   ræ   r   r   rE   r   Ú binary_cross_entropy_with_logits)r   rí   r�   rî   rð   rè   Z
pos_weightró   r$   r$   r%   rü     s    &
rü   c             C   s^   |j r8| ||ƒ}|dkr|S |dkr.tj|ƒS tj|ƒS tj||ƒ\}}|||tj|ƒƒS d S )Nrê   r¾   )Úrequires_gradr   r¾   r¿   Úbroadcast_tensorsrå   ræ   )r•   Zlambd_optimizedr   rí   rè   rF   Úexpanded_inputÚexpanded_targetr$   r$   r%   Ú_pointwise_lossE  s    
r  c             C   s,   t j| | ƒ}t j|dk d|d  |d ƒS )Nr   g      à?r   )r   rW   Úwhere)r   rí   r®   r$   r$   r%   Ú_smooth_l1_lossP  s    r  c             C   s¬   |j ƒ | j ƒ ks.tjdj|j ƒ | j ƒ ƒdd� |dk	s>|dk	rJtj||ƒ}|jr€t| |ƒ}|dkr¨|dkrttj	|ƒntj
|ƒ}n(tj| |ƒ\}}tjjj||tj|ƒƒ}|S )z«Function that uses a squared term if the absolute
    element-wise error falls below 1 and an L1 term otherwise.

    See :class:`~torch.nn.SmoothL1Loss` for details.
    z©Using a target size ({}) that is different to the input size ({}). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.r   )r™   Nrê   r¾   )r   rš   r›   rE   rå   rë   rý   r  r   r¾   r¿   rþ   r   r   Úsmooth_l1_lossræ   )r   rí   rî   rð   rè   rG   rÿ   r   r$   r$   r%   r  V  s    
r  c             C   s°   |j ƒ | j ƒ ks.tjdj|j ƒ | j ƒ ƒdd� |dk	s>|dk	rJtj||ƒ}|jr„tj| | ƒ}|dkr¬|dkrxtj	|ƒntj
|ƒ}n(tj| |ƒ\}}tjjj||tj|ƒƒ}|S )zÑl1_loss(input, target, size_average=None, reduce=None, reduction='mean') -> Tensor

    Function that takes the mean element-wise absolute value difference.

    See :class:`~torch.nn.L1Loss` for details.
    z©Using a target size ({}) that is different to the input size ({}). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.r   )r™   Nrê   r¾   )r   rš   r›   rE   rå   rë   rý   r   rW   r¾   r¿   rþ   r   r   Úl1_lossræ   )r   rí   rî   rð   rè   rG   rÿ   r   r$   r$   r%   r  n  s    r  c             C   s®   |j ƒ | j ƒ ks.tjdj|j ƒ | j ƒ ƒdd� |dk	s>|dk	rJtj||ƒ}|jr‚| | d }|dkrª|dkrvtj|ƒntj	|ƒ}n(tj
| |ƒ\}}tjjj||tj|ƒƒ}|S )z¼mse_loss(input, target, size_average=None, reduce=None, reduction='mean') -> Tensor

    Measures the element-wise mean squared error.

    See :class:`~torch.nn.MSELoss` for details.
    z©Using a target size ({}) that is different to the input size ({}). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.r   )r™   Nrê   r¾   )r   rš   r›   rE   rå   rë   rý   r   r¾   r¿   rþ   r   r   Úmse_lossræ   )r   rí   rî   rð   rè   rG   rÿ   r   r$   r$   r%   r  ‡  s    r  c             C   s|   |dk	s|dk	rt j||ƒ}n
t j|ƒ}| jƒ dksL|jƒ dksL|jƒ dkrjtdj| jƒ |jƒ |jƒ ƒƒ‚tj| ||||ƒS )z±margin_ranking_loss(input1, input2, target, margin=0, size_average=None, reduce=None, reduction='mean') -> Tensor

    See :class:`~torch.nn.MarginRankingLoss` for details.
    Nr   z\margin_ranking_loss does not support scalars, got sizes: input1: {}, input2: {}, target: {} )	rå   r÷   ræ   r{   r|   rE   r   r   Úmargin_ranking_loss)r´   rµ   rí   Úmarginrî   rð   rè   ró   r$   r$   r%   r     s    
$r  c             C   s8   |dk	s|dk	rt j||ƒ}n
t j|ƒ}tj| |||ƒS )z¬hinge_embedding_loss(input, target, margin=1.0, size_average=None, reduce=None, reduction='mean') -> Tensor

    See :class:`~torch.nn.HingeEmbeddingLoss` for details.
    N)rå   r÷   ræ   r   Úhinge_embedding_loss)r   rí   r  rî   rð   rè   ró   r$   r$   r%   r	  ±  s    
r	  c             C   s:   |dk	s|dk	rt j||ƒ}n
t j|ƒ}tjjj| ||ƒS )z¤multilabel_margin_loss(input, target, size_average=None, reduce=None, reduction='mean') -> Tensor

    See :class:`~torch.nn.MultiLabelMarginLoss` for details.
    N)rå   r÷   ræ   r   r   r   Úmultilabel_margin_loss)r   rí   rî   rð   rè   ró   r$   r$   r%   r
  ¿  s    
r
  c             C   s:   |dk	s|dk	rt j||ƒ}n
t j|ƒ}tjjj| ||ƒS )z˜soft_margin_loss(input, target, size_average=None, reduce=None, reduction='mean') -> Tensor

    See :class:`~torch.nn.SoftMarginLoss` for details.
    N)rå   r÷   ræ   r   r   r   Úsoft_margin_loss)r   rí   rî   rð   rè   ró   r$   r$   r%   r  Ì  s    
r  c             C   s¨   |dk	s|dk	rt j||ƒ}|t| ƒ d| t|  ƒ   }|dk	rL|| }|jdd�| jdƒ }|dkrp|}n4|dkr‚|jƒ }n"|dkr”|jƒ }n| }t|d ƒ‚|S )z›multilabel_soft_margin_loss(input, target, weight=None, size_average=None) -> Tensor

    See :class:`~torch.nn.MultiLabelSoftMarginLoss` for details.
    Nr   )r{   rê   r¾   r¿   z is not valid)rå   rë   Ú
logsigmoidr¿   r   r¾   r   )r   rí   r�   rî   rð   rè   ZlossrG   r$   r$   r%   Úmultilabel_soft_margin_lossÙ  s     

r  c             C   s:   |dk	s|dk	rt j||ƒ}n
t j|ƒ}tj| ||||ƒS )zµcosine_embedding_loss(input1, input2, target, margin=0, size_average=None, reduce=None, reduction='mean') -> Tensor

    See :class:`~torch.nn.CosineEmbeddingLoss` for details.
    N)rå   r÷   ræ   r   Úcosine_embedding_loss)r´   rµ   rí   r  rî   rð   rè   ró   r$   r$   r%   r  ö  s    
r  c       	      C   st   |dk	s|dk	rt j||ƒ}n
t j|ƒ}|dkr@|dkr@tdƒ‚|dk	r\|jƒ dkr\tdƒ‚tjjj| |||||ƒS )zÐmulti_margin_loss(input, target, p=1, margin=1, weight=None, size_average=None,
                          reduce=None, reduction='mean') -> Tensor

    See :class:`~torch.nn.MultiMarginLoss` for details.
    Nr   r   z only p == 1 and p == 2 supportedzweight must be one-dimensional)	rå   r÷   ræ   r   r{   r   r   r   Úmulti_margin_loss)	r   rí   rm   r  r�   rî   rð   rè   ró   r$   r$   r%   r  	  s    
r  aÖ  
Rearranges elements in a tensor of shape :math:`(*, C \times r^2, H, W)` to a
tensor of shape :math:`(*, C, H \times r, W \times r)`.

See :class:`~torch.nn.PixelShuffle` for details.

Args:
    input (Tensor): the input tensor
    upscale_factor (int): factor to increase spatial resolution by

Examples::

    >>> input = torch.randn(1, 9, 4, 4)
    >>> output = torch.nn.functional.pixel_shuffle(input, 3)
    >>> print(output.size())
    torch.Size([1, 1, 12, 12])
Únearestc             C   s   t jdƒ t| ||||ƒS )a  Upsamples the input to either the given :attr:`size` or the given
    :attr:`scale_factor`

    .. warning::
        This function is deprecated in favor of :func:`torch.nn.functional.interpolate`.
        This is equivalent with ``nn.functional.interpolate(...)``.

    .. include:: cuda_deterministic_backward.rst

    The algorithm used for upsampling is determined by :attr:`mode`.

    Currently temporal, spatial and volumetric upsampling are supported, i.e.
    expected inputs are 3-D, 4-D or 5-D in shape.

    The input dimensions are interpreted in the form:
    `mini-batch x channels x [optional depth] x [optional height] x width`.

    The modes available for upsampling are: `nearest`, `linear` (3D-only),
    `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only)

    Args:
        input (Tensor): the input tensor
        size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]):
            output spatial size.
        scale_factor (float or Tuple[float]): multiplier for spatial size. Has to be an integer.
        mode (string): algorithm used for upsampling:
            ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` |
            ``'trilinear'``. Default: ``'nearest'``
        align_corners (bool, optional): Geometrically, we consider the pixels of the
            input and output as squares rather than points.
            If set to ``True``, the input and output tensors are aligned by the
            center points of their corner pixels, preserving the values at the corner pixels.
            If set to ``False``, the input and output tensors are aligned by the corner
            points of their corner pixels, and the interpolation uses edge value padding
            for out-of-boundary values, making this operation *independent* of input size
            when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode`
            is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``.
            Default: ``False``

    .. note::
        With ``mode='bicubic'``, it's possible to cause overshoot, in other words it can produce
        negative values or values greater than 255 for images.
        Explicitly call ``result.clamp(min=0, max=255)`` if you want to reduce the overshoot
        when displaying the image.

    .. warning::
        With ``align_corners = True``, the linearly interpolating modes
        (`linear`, `bilinear`, and `trilinear`) don't proportionally align the
        output and input pixels, and thus the output values can depend on the
        input size. This was the default behavior for these modes up to version
        0.3.1. Since then, the default behavior is ``align_corners = False``.
        See :class:`~torch.nn.Upsample` for concrete examples on how this
        affects the outputs.

    zLnn.functional.upsample is deprecated. Use nn.functional.interpolate instead.)rš   r›   Úinterpolate)r   r   Úscale_factorrÉ   Úalign_cornersr$   r$   r%   Úupsample,	  s    8
r  c                sî  ddl m‰ ‡‡fdd„‰ ‡ ‡‡‡‡fdd„}|dkrH|d	k	rdtd
ƒ‚n|d	krdtjdj|ƒƒ d}ˆjƒ dkrŒ|dkrŒtjj	j
ˆ|dƒƒS ˆjƒ dkr´|dkr´tjj	jˆ|dƒƒS ˆjƒ dkrÜ|dkrÜtjj	jˆ|dƒƒS ˆjƒ dkoî|dk�r tˆ|dƒƒS ˆjƒ dk�r&|dk�r&tˆ|dƒƒS ˆjƒ dk�rL|dk�rLtˆ|dƒƒS ˆjƒ dk�rz|dk�rztjj	jˆ|dƒ|ƒS ˆjƒ dk�rž|dk�ržtdƒ‚�nLˆjƒ dk�rÂ|dk�rÂtdƒ‚�n(ˆjƒ dk�ræ|dk�rætdƒ‚�nˆjƒ dk�r|dk�rtjj	jˆ|dƒ|ƒS ˆjƒ dk�r6|dk�r6tdƒ‚n´ˆjƒ dk�rX|dk�rXtdƒ‚n’ˆjƒ dk�rz|dk�rztdƒ‚npˆjƒ dk�r¨|dk�r¨tjj	jˆ|dƒ|ƒS ˆjƒ dk�rÖ|dk�rÖtjj	jˆ|dƒ|ƒS tdjˆjƒ |ƒƒ‚d	S )a‹
  Down/up samples the input to either the given :attr:`size` or the given
    :attr:`scale_factor`

    The algorithm used for interpolation is determined by :attr:`mode`.

    Currently temporal, spatial and volumetric sampling are supported, i.e.
    expected inputs are 3-D, 4-D or 5-D in shape.

    The input dimensions are interpreted in the form:
    `mini-batch x channels x [optional depth] x [optional height] x width`.

    The modes available for resizing are: `nearest`, `linear` (3D-only),
    `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only), `area`

    Args:
        input (Tensor): the input tensor
        size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]):
            output spatial size.
        scale_factor (float or Tuple[float]): multiplier for spatial size. Has to match input size if it is a tuple.
        mode (str): algorithm used for upsampling:
            ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` |
            ``'trilinear'`` | ``'area'``. Default: ``'nearest'``
        align_corners (bool, optional): Geometrically, we consider the pixels of the
            input and output as squares rather than points.
            If set to ``True``, the input and output tensors are aligned by the
            center points of their corner pixels, preserving the values at the corner pixels.
            If set to ``False``, the input and output tensors are aligned by the corner
            points of their corner pixels, and the interpolation uses edge value padding
            for out-of-boundary values, making this operation *independent* of input size
            when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode`
            is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``.
            Default: ``False``

    .. note::
        With ``mode='bicubic'``, it's possible to cause overshoot, in other words it can produce
        negative values or values greater than 255 for images.
        Explicitly call ``result.clamp(min=0, max=255)`` if you want to reduce the overshoot
        when displaying the image.

    .. warning::
        With ``align_corners = True``, the linearly interpolating modes
        (`linear`, `bilinear`, and `trilinear`) don't proportionally align the
        output and input pixels, and thus the output values can depend on the
        input size. This was the default behavior for these modes up to version
        0.3.1. Since then, the default behavior is ``align_corners = False``.
        See :class:`~torch.nn.Upsample` for concrete examples on how this
        affects the outputs.

    .. include:: cuda_deterministic_backward.rst
    r   )Ú_ntuplec                sf   ˆd krˆ d krt dƒ‚ˆd k	r0ˆ d k	r0t dƒ‚ˆ d k	rbtˆ tƒrbtˆ ƒ| krbt dj| tˆ ƒƒƒ‚d S )Nz-either size or scale_factor should be definedz2only one of size or scale_factor should be definedzPscale_factor shape must match input shape. Input is {}D, scale_factor size is {})r   rJ   ÚtuplerC   rE   )r{   )r  r   r$   r%   Ú_check_size_scale_factor�	  s    z-interpolate.<locals>._check_size_scale_factorc                s^   ˆ| ƒ ˆd k	rˆS ˆ| ƒˆƒ‰ t jjƒ rB‡‡ fdd„t| ƒD ƒS ‡‡ fdd„t| ƒD ƒS d S )Nc                s<   g | ]4}t jˆ j|d  ƒjƒ t jˆ| t jd� jƒ ƒ‘qS )r   )r   )r   Úfloorr   ÚfloatZtensorZfloat32)Ú.0rÕ   )r   Úscale_factorsr$   r%   ú
<listcomp>°	  s   z5interpolate.<locals>._output_size.<locals>.<listcomp>c                s0   g | ](}t tjtˆ j|d  ƒƒˆ|  ƒƒ‘qS )r   )r   Úmathr  r  r   )r  rÕ   )r   r  r$   r%   r  ³	  s    )r   r   Z_get_tracing_staterB   )r{   )r  r  r   r  r   )r  r%   ri   §	  s    
z!interpolate.<locals>._output_sizer  ÚareaNzjalign_corners option can only be set with the interpolating modes: linear | bilinear | bicubic | trilinearzËDefault upsampling behavior when mode={} is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.Fr   r(   r   é   r²   r³   z.Got 3D input, but bilinear mode needs 4D inputZ	trilinearz/Got 3D input, but trilinear mode needs 5D inputz,Got 4D input, but linear mode needs 3D inputz/Got 4D input, but trilinear mode needs 5D inputz,Got 5D input, but linear mode needs 3D inputz.Got 5D input, but bilinear mode needs 4D inputZbicubicz‹Input Error: Only 3D, 4D and 5D input Tensors supported (got {}D) for the modes: nearest | linear | bilinear | bicubic | trilinear (got {}))r  r  )Úmodules.utilsr  r   rš   r›   rE   r{   r   r   r   Zupsample_nearest1dZupsample_nearest2dZupsample_nearest3dÚadaptive_avg_pool1drh   rj   Zupsample_linear1drÆ   Zupsample_bilinear2dZupsample_trilinear3dZupsample_bicubic2d)r   r   r  rÉ   r  ri   r$   )r  r  r   r  r   r%   r  h	  sX    3




r  c             C   s   t jdƒ t| ||dd�S )a{  Upsamples the input, using nearest neighbours' pixel values.

    .. warning::
        This function is deprecated in favor of :func:`torch.nn.functional.interpolate`.
        This is equivalent with ``nn.functional.interpolate(..., mode='nearest')``.

    Currently spatial and volumetric upsampling are supported (i.e. expected
    inputs are 4 or 5 dimensional).

    Args:
        input (Tensor): input
        size (int or Tuple[int, int] or Tuple[int, int, int]): output spatia
            size.
        scale_factor (int): multiplier for spatial size. Has to be an integer.

    .. include:: cuda_deterministic_backward.rst
    zTnn.functional.upsample_nearest is deprecated. Use nn.functional.interpolate instead.r  )rÉ   )rš   r›   r  )r   r   r  r$   r$   r%   Úupsample_nearestç	  s    
r"  c             C   s   t jdƒ t| ||ddd�S )ag  Upsamples the input, using bilinear upsampling.

    .. warning::
        This function is deprecated in favor of :func:`torch.nn.functional.interpolate`.
        This is equivalent with
        ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)``.

    Expected inputs are spatial (4 dimensional). Use `upsample_trilinear` fo
    volumetric (5 dimensional) inputs.

    Args:
        input (Tensor): input
        size (int or Tuple[int, int]): output spatial size.
        scale_factor (int or Tuple[int, int]): multiplier for spatial size

    .. include:: cuda_deterministic_backward.rst
    zUnn.functional.upsample_bilinear is deprecated. Use nn.functional.interpolate instead.r³   T)rÉ   r  )rš   r›   r  )r   r   r  r$   r$   r%   Úupsample_bilinearþ	  s    
r#  )r³   r  )ÚzerosÚborderÚ
reflectionr$  c             C   sž   |dkr|dkrt dj|ƒƒ‚|dkrD|dkrD|dkrDt dj|ƒƒ‚|dkrRd}nd	}|dkrdd}n|dkrrd	}nd
}|dkrŒtjdƒ d}tj| ||||ƒS )a	  Given an :attr:`input` and a flow-field :attr:`grid`, computes the
    ``output`` using :attr:`input` values and pixel locations from :attr:`grid`.

    Currently, only spatial (4-D) and volumetric (5-D) :attr:`input` are
    supported.

    In the spatial (4-D) case, for :attr:`input` with shape
    :math:`(N, C, H_\text{in}, W_\text{in})` and :attr:`grid` with shape
    :math:`(N, H_\text{out}, W_\text{out}, 2)`, the output will have shape
    :math:`(N, C, H_\text{out}, W_\text{out})`.

    For each output location ``output[n, :, h, w]``, the size-2 vector
    ``grid[n, h, w]`` specifies :attr:`input` pixel locations ``x`` and ``y``,
    which are used to interpolate the output value ``output[n, :, h, w]``.
    In the case of 5D inputs, ``grid[n, d, h, w]`` specifies the
    ``x``, ``y``, ``z`` pixel locations for interpolating
    ``output[n, :, d, h, w]``. :attr:`mode` argument specifies ``nearest`` or
    ``bilinear`` interpolation method to sample the input pixels.

    :attr:`grid` specifies the sampling pixel locations normalized by the
    :attr:`input` spatial dimensions. Therefore, it should have most values in
    the range of ``[-1, 1]``. For example, values ``x = -1, y = -1`` is the
    left-top pixel of :attr:`input`, and values  ``x = 1, y = 1`` is the
    right-bottom pixel of :attr:`input`.

    If :attr:`grid` has values outside the range of ``[-1, 1]``, the corresponding
    outputs are handled as defined by :attr:`padding_mode`. Options are

        * ``padding_mode="zeros"``: use ``0`` for out-of-bound grid locations,
        * ``padding_mode="border"``: use border values for out-of-bound grid locations,
        * ``padding_mode="reflection"``: use values at locations reflected by
          the border for out-of-bound grid locations. For location far away
          from the border, it will keep being reflected until becoming in bound,
          e.g., (normalized) pixel location ``x = -3.5`` reflects by border ``-1``
          and becomes ``x' = 1.5``, then reflects by border ``1`` and becomes
          ``x'' = -0.5``.

    .. note::
        This function is often used in conjunction with :func:`affine_grid`
        to build `Spatial Transformer Networks`_ .
    .. include:: cuda_deterministic_backward.rst

    Args:
        input (Tensor): input of shape :math:`(N, C, H_\text{in}, W_\text{in})` (4-D case)
                        or :math:`(N, C, D_\text{in}, H_\text{in}, W_\text{in})` (5-D case)
        grid (Tensor): flow-field of shape :math:`(N, H_\text{out}, W_\text{out}, 2)` (4-D case)
                       or :math:`(N, D_\text{out}, H_\text{out}, W_\text{out}, 3)` (5-D case)
        mode (str): interpolation mode to calculate output values
            ``'bilinear'`` | ``'nearest'``. Default: ``'bilinear'``
        padding_mode (str): padding mode for outside grid values
            ``'zeros'`` | ``'border'`` | ``'reflection'``. Default: ``'zeros'``
        align_corners (bool, optional): Geometrically, we consider the pixels of the
            input  as squares rather than points.
            If set to ``True``, the extrema (``-1`` and ``1``) are considered as referring
            to the center points of the input's corner pixels. If set to ``False``, they
            are instead considered as referring to the corner points of the input's corner
            pixels, making the sampling more resolution agnostic.
            This option parallels the ``align_corners`` option in
            :func:`interpolate`, and so whichever option is used here
            should also be used there to resize the input image before grid sampling.
            Default: ``True``

    Returns:
        output (Tensor): output Tensor

    .. _`Spatial Transformer Networks`:
        https://arxiv.org/abs/1506.02025

    .. warning::
        When ``align_corners = True``, the grid positions depend on the pixel
        size relative to the input image size, and so the locations sampled by
        :func:`grid_sample` will differ for the same input given at different
        resolutions (that is, after being upsampled or downsampled).
        The default behavior up to version 1.3.x is ``align_corners = True``.
        The default behavior will be changed to ``align_corners = False`` from
        v1.4.0, in order to bring it in line with the default for :func:`interpolate`.
    r³   r  zWnn.functional.grid_sample(): expected mode to be 'bilinear' or 'nearest', but got: '{}'r$  r%  r&  zjnn.functional.grid_sample(): expected padding_mode to be 'zeros', 'border', or 'reflection', but got: '{}'r   r   r   Nz‘Default grid_sample and affine_grid behavior will be changed to align_corners=False from 1.4.0. See the documentation of grid_sample for details.T)r   rE   rš   r›   r   Zgrid_sampler)r   ZgridrÉ   Zpadding_moder  rÊ   Zpadding_mode_enumr$   r$   r%   Úgrid_sample!
  s$    O
r'  c             C   s.  |dkrt jdƒ d}| jƒ s.tdj| jƒƒ‚t|ƒdkr‚| jƒ dksb| jd dksb| jd dkrttd	j|| jƒƒ‚|dd… }nbt|ƒd
krÖ| jƒ dks¶| jd dks¶| jd dkrÈtdj|| jƒƒ‚|dd… }nt	dj|ƒƒ‚|�rt
|ƒdk�rt jdƒ nt
|ƒdk�r tdj|ƒƒ‚tj| ||ƒS )a|	  Generates a 2D or 3D flow field (sampling grid), given a batch of
    affine matrices :attr:`theta`.

    .. note::
        This function is often used in conjunction with :func:`grid_sample`
        to build `Spatial Transformer Networks`_ .

    Args:
        theta (Tensor): input batch of affine matrices with shape
            (:math:`N \times 2 \times 3`) for 2D or
            (:math:`N \times 3 \times 4`) for 3D
        size (torch.Size): the target output image size.
            (:math:`N \times C \times H \times W` for 2D or
            :math:`N \times C \times D \times H \times W` for 3D)
            Example: torch.Size((32, 3, 24, 24))
        align_corners (bool, optional): if ``True``, consider ``-1`` and ``1``
            to refer to the centers of the corner pixels rather than the image corners.
            Refer to :func:`grid_sample` for a more complete description.
            A grid generated by :func:`affine_grid` should be passed to :func:`grid_sample`
            with the same setting for this option.
            Default: ``True``

    Returns:
        output (Tensor): output Tensor of size (:math:`N \times H \times W \times 2`)

    .. _`Spatial Transformer Networks`:
        https://arxiv.org/abs/1506.02025

    .. warning::
        When ``align_corners = True``, the grid positions depend on the pixel
        size relative to the input image size, and so the locations sampled by
        :func:`grid_sample` will differ for the same input given at different
        resolutions (that is, after being upsampled or downsampled).
        The default behavior up to version 1.3.x is ``align_corners = True``.
        The default behavior will be changed to ``align_corners = False`` from
        v1.4.0, in order to bring it in line with the default for :func:`interpolate`.
    .. warning::
        When ``align_corners = True``, 2D affine transforms on 1D data and
        3D affine transforms on 2D data (that is, when one of the spatial
        dimensions has unit size) are ill-defined, and not an intended use case.
        This is not a problem when ``align_corners = False``.
        Up to version 1.2.0, all grid points along a unit dimension were
        considered arbitrarily to be at ``-1``.
        From version 1.3.0, under ``align_corners = True`` all grid points
        along a unit dimension are condsidered to be at ```0``
        (the center of the input image).
    Nz‘Default grid_sample and affine_grid behavior will be changed to align_corners=False from 1.4.0. See the documentation of grid_sample for details.Tz6Expected theta to have floating point type, but got {}r(   r   r   r   zJExpected a batch of 2D affine matrices of shape Nx2x3 for size {}. Got {}.r  zJExpected a batch of 3D affine matrices of shape Nx3x4 for size {}. Got {}.zfaffine_grid only supports 4D and 5D sizes, for 2D and 3D affine transforms, respectively. Got size {}.zÅSince version 1.3.0, affine_grid behavior has changed for unit-size grids when align_corners=True. This is not an intended use case of affine_grid. See the documentation of affine_grid for details.r   z/Expected non-zero, positive output size. Got {}éþÿÿÿr¹   r(  r(  r¹   éýÿÿÿ)rš   r›   rÁ   r   rE   r   rC   r{   rÂ   rÆ   Úminr   Zaffine_grid_generator)Úthetar   r  Zspatial_sizer$   r$   r%   Úaffine_grid�
  s0    1

((r,  Úconstantc             C   s¬  t |ƒd dkstdƒ‚t |ƒd | jƒ ks4tdƒ‚|dkrJtj| ||ƒS |dks`tdj|ƒƒ‚| jƒ dkrÈt |ƒdks€tdƒ‚|d	kr˜tjjj	| |ƒS |d
kr°tjjj
| |ƒS |dkrÂt| |ƒS t‚nà| jƒ dk�r8t |ƒdksêtdƒ‚|d	k�rtjjj| |ƒS |d
k�rtjjj| |ƒS |dk�r2t| |ƒS t‚np| jƒ dk�r t |ƒdk�s\tdƒ‚|d	k�rlt‚n2|d
k�r†tjjj| |ƒS |dk�ršt| |ƒS t‚ntdƒ‚dS )a2
  Pads tensor.

    Padding size:
        The padding size by which to pad some dimensions of :attr:`input`
        are described starting from the last dimension and moving forward.
        :math:`\left\lfloor\frac{\text{len(pad)}}{2}\right\rfloor` dimensions
        of ``input`` will be padded.
        For example, to pad only the last dimension of the input tensor, then
        :attr:`pad` has the form
        :math:`(\text{padding\_left}, \text{padding\_right})`;
        to pad the last 2 dimensions of the input tensor, then use
        :math:`(\text{padding\_left}, \text{padding\_right},`
        :math:`\text{padding\_top}, \text{padding\_bottom})`;
        to pad the last 3 dimensions, use
        :math:`(\text{padding\_left}, \text{padding\_right},`
        :math:`\text{padding\_top}, \text{padding\_bottom}`
        :math:`\text{padding\_front}, \text{padding\_back})`.

    Padding mode:
        See :class:`torch.nn.ConstantPad2d`, :class:`torch.nn.ReflectionPad2d`, and
        :class:`torch.nn.ReplicationPad2d` for concrete examples on how each of the
        padding modes works. Constant padding is implemented for arbitrary dimensions.
        Replicate padding is implemented for padding the last 3 dimensions of 5D input
        tensor, or the last 2 dimensions of 4D input tensor, or the last dimension of
        3D input tensor. Reflect padding is only implemented for padding the last 2
        dimensions of 4D input tensor, or the last dimension of 3D input tensor.

    .. include:: cuda_deterministic_backward.rst

    Args:
        input (Tensor): N-dimensional tensor
        pad (tuple): m-elements tuple, where
            :math:`\frac{m}{2} \leq` input dimensions and :math:`m` is even.
        mode: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``.
            Default: ``'constant'``
        value: fill value for ``'constant'`` padding. Default: ``0``

    Examples::

        >>> t4d = torch.empty(3, 3, 4, 2)
        >>> p1d = (1, 1) # pad last dim by 1 on each side
        >>> out = F.pad(t4d, p1d, "constant", 0)  # effectively zero padding
        >>> print(out.data.size())
        torch.Size([3, 3, 4, 4])
        >>> p2d = (1, 1, 2, 2) # pad last dim by (1, 1) and 2nd to last by (2, 2)
        >>> out = F.pad(t4d, p2d, "constant", 0)
        >>> print(out.data.size())
        torch.Size([3, 3, 8, 4])
        >>> t4d = torch.empty(3, 3, 4, 2)
        >>> p3d = (0, 1, 2, 1, 3, 3) # pad by (0, 1), (2, 1), and (3, 3)
        >>> out = F.pad(t4d, p3d, "constant", 0)
        >>> print(out.data.size())
        torch.Size([3, 9, 7, 3])

    r   r   z%Padding length must be divisible by 2zPadding length too larger-  z1Padding mode "{}"" doesn't take in value argumentr   z&3D tensors expect 2 values for paddingZreflectZ	replicateZcircularr(   z&4D tensors expect 4 values for paddingr  r;   z&5D tensors expect 6 values for paddingzGOnly 3D, 4D, 5D padding with non-constant padding are supported for nowN)rC   r   r{   r   Zconstant_pad_ndrE   r   r   r   Zreflection_pad1dZreplication_pad1dÚ_pad_circularrÆ   Zreflection_pad2dZreplication_pad2dZreplication_pad3d)r   rÛ   rÉ   rx   r$   r$   r%   rÛ   ä
  sB    9








rÛ   ç�íµ ÷Æ°>c             C   s   t j| ||||ƒS )z<
    See :class:`torch.nn.PairwiseDistance` for details
    )r   Úpairwise_distance)Úx1Zx2rm   r©   r£   r$   r$   r%   r0  I  s    r0  a„  
pdist(input, p=2) -> Tensor

Computes the p-norm distance between every pair of row vectors in the input.
This is identical to the upper triangular portion, excluding the diagonal, of
`torch.norm(input[:, None] - input, dim=2, p=p)`. This function will be faster
if the rows are contiguous.

If input has shape :math:`N \times M` then the output will have shape
:math:`\frac{1}{2} N (N - 1)`.

This function is equivalent to `scipy.spatial.distance.pdist(input,
'minkowski', p=p)` if :math:`p \in (0, \infty)`. When :math:`p = 0` it is
equivalent to `scipy.spatial.distance.pdist(input, 'hamming') * M`.
When :math:`p = \infty`, the closest scipy function is
`scipy.spatial.distance.pdist(xn, lambda x, y: np.abs(x - y).max())`.

Args:
    input: input tensor of shape :math:`N \times M`.
    p: p value for the p-norm distance to calculate between each vector pair
        :math:`\in [0, \infty]`.
a  
cosine_similarity(x1, x2, dim=1, eps=1e-8) -> Tensor

Returns cosine similarity between x1 and x2, computed along dim.

.. math ::
    \text{similarity} = \dfrac{x_1 \cdot x_2}{\max(\Vert x_1 \Vert _2 \cdot \Vert x_2 \Vert _2, \epsilon)}

Args:
    x1 (Tensor): First input.
    x2 (Tensor): Second input (of size matching x1).
    dim (int, optional): Dimension of vectors. Default: 1
    eps (float, optional): Small value to avoid division by zero.
        Default: 1e-8

Shape:
    - Input: :math:`(\ast_1, D, \ast_2)` where D is at position `dim`.
    - Output: :math:`(\ast_1, \ast_2)` where 1 is at position `dim`.

Example::

    >>> input1 = torch.randn(100, 128)
    >>> input2 = torch.randn(100, 128)
    >>> output = F.cosine_similarity(input1, input2)
    >>> print(output)
aa  
one_hot(tensor, num_classes=-1) -> LongTensor

Takes LongTensor with index values of shape ``(*)`` and returns a tensor
of shape ``(*, num_classes)`` that have zeros everywhere except where the
index of last dimension matches the corresponding value of the input tensor,
in which case it will be 1.

See also `One-hot on Wikipedia`_ .

.. _One-hot on Wikipedia:
    https://en.wikipedia.org/wiki/One-hot

Arguments:
    tensor (LongTensor): class values of any shape.
    num_classes (int):  Total number of classes. If set to -1, the number
        of classes will be inferred as one greater than the largest class
        value in the input tensor.

Returns:
    LongTensor that has one more dimension with 1 values at the
    index of last dimension indicated by the input, and 0 everywhere
    else.

Examples:
    >>> F.one_hot(torch.arange(0, 5) % 3)
    tensor([[1, 0, 0],
            [0, 1, 0],
            [0, 0, 1],
            [1, 0, 0],
            [0, 1, 0]])
    >>> F.one_hot(torch.arange(0, 5) % 3, num_classes=5)
    tensor([[1, 0, 0, 0, 0],
            [0, 1, 0, 0, 0],
            [0, 0, 1, 0, 0],
            [1, 0, 0, 0, 0],
            [0, 1, 0, 0, 0]])
    >>> F.one_hot(torch.arange(0, 6).view(3,2) % 3)
    tensor([[[1, 0, 0],
             [0, 1, 0]],
            [[0, 0, 1],
             [1, 0, 0]],
            [[0, 1, 0],
             [0, 0, 1]]])
c
          	   C   s@   |dk	s|dk	rt j||ƒ}
n
t j|	ƒ}
tj| |||||||
ƒS )z>
    See :class:`~torch.nn.TripletMarginLoss` for details
    N)rå   r÷   ræ   r   Útriplet_margin_loss)ÚanchorZpositiveÚnegativer  rm   r©   Zswaprî   rð   rè   ró   r$   r$   r%   r2  ´  s
    
r2  çê-�™—q=c             C   sX   |dkr*| j ||dƒj|ƒj| ƒ}| | S | j ||dƒj|ƒj| ƒ}tj| ||d�S dS )aL  Performs :math:`L_p` normalization of inputs over specified dimension.

    For a tensor :attr:`input` of sizes :math:`(n_0, ..., n_{dim}, ..., n_k)`, each
    :math:`n_{dim}` -element vector :math:`v` along dimension :attr:`dim` is transformed as

    .. math::
        v = \frac{v}{\max(\lVert v \rVert_p, \epsilon)}.

    With the default arguments it uses the Euclidean norm over vectors along dimension :math:`1` for normalization.

    Args:
        input: input tensor of any shape
        p (float): the exponent value in the norm formulation. Default: 2
        dim (int): the dimension to reduce. Default: 1
        eps (float): small value to avoid division by zero. Default: 1e-12
        out (Tensor, optional): the output tensor. If :attr:`out` is used, this
                                operation won't be differentiable.
    NT)r[   )ZnormZ	clamp_minZ	expand_asr   rá   )r   rm   r{   r©   r[   Zdenomr$   r$   r%   Ú	normalizeÂ  s
    r6  c             C   s(   t | tƒs$t| ƒdks$t|j|ƒƒ‚d S )Nr   )rJ   r   rC   r   rE   )Úargr)   Úmessager$   r$   r%   Úassert_int_or_pairÞ  s    r9  c             C   s|   | j ƒ dkrfd}t|d|ƒ t|d|ƒ t|d|ƒ t|d|ƒ tjjj| t|ƒt|ƒt|ƒt|ƒƒS tdj| j ƒ ƒƒ‚dS )	a
  Extracts sliding local blocks from an batched input tensor.

    .. warning::
        Currently, only 4-D input tensors (batched image-like tensors) are
        supported.

    .. warning::

        More than one element of the unfolded tensor may refer to a single
        memory location. As a result, in-place operations (especially ones that
        are vectorized) may result in incorrect behavior. If you need to write
        to the tensor, please clone it first.


    See :class:`torch.nn.Unfold` for details
    r(   z&{} must be int or 2-tuple for 4D inputr   r7   r6   r5   z:Input Error: Only 4D input Tensors are supported (got {}D)N)	r{   r9  r   r   r   Zim2colr	   rÆ   rE   )r   r   r7   r6   r5   Úmsgr$   r$   r%   Úunfoldâ  s    r;  c             C   sŽ   | j ƒ dkrxd}t|d|ƒ t|d|ƒ t|d|ƒ t|d|ƒ t|d|ƒ tjjj| t|ƒt|ƒt|ƒt|ƒt|ƒƒS tdj| j ƒ ƒƒ‚d	S )
zïCombines an array of sliding local blocks into a large containing
    tensor.

    .. warning::
        Currently, only 4-D output tensors (batched image-like tensors) are
        supported.

    See :class:`torch.nn.Fold` for details
    r   z&{} must be int or 2-tuple for 3D inputr   r   r7   r6   r5   z:Input Error: Only 3D input Tensors are supported (got {}D)N)	r{   r9  r   r   r   Zcol2imr	   rÆ   rE   )r   r   r   r7   r6   r5   r:  r$   r$   r%   Úfold  s    r<  c          
   C   sz  t j| | dd…dd…d|d
 …f gdd�} t j| dd…dd…|d |d   |d  …f | gdd�} t|ƒdkrèt j| | dd…dd…dd…d|d …f gdd�} t j| dd…dd…dd…|d |d   |d  …f | gdd�} t|ƒdk�rvt j| | dd…dd…dd…dd…d|d …f gdd�} t j| dd…dd…dd…dd…|d |d   |d  …f | gdd�} | S )aI  
    Arguments
        :param input: tensor of shape :math:`(N, C_{	ext{in}}, H, [W, D]))`
        :param padding: (tuple): m-elem tuple where m is the degree of convolution
    Returns
        :return: tensor of shape :math:`(N, C_{	ext{in}}, [D + 2 * padding[0],
                 H + 2 * padding[1]], W + 2 * padding[2]))`
    Nr   r   r   )r{   r   r(   r  r;   r¹   r¹   r(  r¹   r)  r)  éüÿÿÿr)  éûÿÿÿr>  iúÿÿÿr>  )r   ÚcatrC   )r   r6   r$   r$   r%   r.    s    ,<2B8Hr.  c       -      C   sÀ  t j| |ƒot j||ƒ}t j||ƒ}| jƒ \}}}||ks>t‚t| jƒ ƒ|||gksXt‚|jƒ |jƒ kslt‚|| }|| |ksˆtdƒ‚t|ƒd }|dk	�rP|rÂt| ||ƒjddd�\}}} �qf|�r~|}!d}"|}#||"|#…dd…f }$|!dk	� rþ|!|"|#… }!t| |$|!ƒ}|dk�r,|dk�s"t‚d}d} nP|}!|}"d}#||"d…dd…f }$|!dk	�rb|!|"d… }!t||$|!ƒjd	dd�\}} nÎ|}!d}"|}#||"|#…dd…f }$|!dk	�r´|!|"|#… }!t| |$|!ƒ}|}!|}"|d	 }#||"|#…dd…f }$|!dk	�rú|!|"|#… }!t||$|!ƒ}|}!|d	 }"d}#||"d…dd…f }$|!dk	�r@|!|"d… }!t||$|!ƒ} �nt jj	|ƒ}%|%jƒ \}&}'|&|k�r‚|'| jdƒk�s†t‚t jj	|ƒ}(|(jƒ \}&}'|&|k�r¸|'|jdƒk�s¼t‚t jj	|ƒ})|)jƒ \}&}'|&|k�rî|'|jdƒk�sòt‚|dk	�rBt| |%|d|… ƒ}t||(|||d	 … ƒ}t||)||d	 d… ƒ} n$t| |%|ƒ}t||(|ƒ}t||)|ƒ} || }|dk	�rZ|dk	�rZ|dk�r4|dk�r4t j
||jd|dƒgƒ}t j
| |jd|dƒgƒ} |dk	�rüt j
|t j|jdƒdf|j|jd
�gdd�}|dk	�rXt j
|t j|jdƒdf|j|jd
�gdd�}n$|dk�sFtdƒ‚|dk�svtdƒ‚n|dk�sht‚|dk�svt‚|jƒ j||| |ƒjddƒ}|dk	�r¼|jƒ jd|| |ƒjddƒ}| dk	�rä| jƒ jd|| |ƒjddƒ} |dk	�r|jdƒ|| k�st‚|jd	ƒ|k�st‚|}|dk	�rX|jdƒ|| k�s@t‚|jd	ƒ|k�sTt‚|} |jdƒ}*|dk	�r”|jdƒ|k�s€t‚|jdƒ|*k�s”t‚|	�r†|*d7 }*t j
|t j|jdƒdf|jƒ d	d…  |j|jd
�gdd�}t j
| t j| jdƒdf| jƒ d	d…  | j| jd
�gdd�} |dk	�rPt j
|t j|jdƒdf|j|jd
�gdd�}|dk	�r†t j
|t j|jdƒdf|j|jd
�gdd�}t j||jdd	ƒƒ}+t|+jƒ ƒ|| ||*gk�sºt‚|dk	�rÖ|jdƒ}|+|7 }+|dk	�r|+j||||*ƒ}+|+j|jdƒjd	ƒtdƒƒ}+|+j|| ||*ƒ}+t|+dd�}+t|+|
|d�}+t j|+| ƒ},t|,jƒ ƒ|| ||gk�sdt‚|,jddƒjƒ j|||ƒ},t|,||ƒ},|�r´|+j||||*ƒ}+|,|+jdd�| fS |,dfS dS )aš  
    Args:
        query, key, value: map a query and a set of key-value pairs to an output.
            See "Attention Is All You Need" for more details.
        embed_dim_to_check: total dimension of the model.
        num_heads: parallel attention heads.
        in_proj_weight, in_proj_bias: input projection weight and bias.
        bias_k, bias_v: bias of the key and value sequences to be added at dim=0.
        add_zero_attn: add a new batch of zeros to the key and
                       value sequences at dim=1.
        dropout_p: probability of an element to be zeroed.
        out_proj_weight, out_proj_bias: the output projection weight and bias.
        training: apply dropout if is ``True``.
        key_padding_mask: if provided, specified padding elements in the key will
            be ignored by the attention. This is an binary mask. When the value is True,
            the corresponding value on the attention layer will be filled with -inf.
        need_weights: output attn_output_weights.
        attn_mask: mask that prevents attention to certain positions. This is an additive mask
            (i.e. the values will be added to the attention layer).
        use_separate_proj_weight: the function accept the proj. weights for query, key,
            and value in differnt forms. If false, in_proj_weight will be used, which is
            a combination of q_proj_weight, k_proj_weight, v_proj_weight.
        q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias.
        static_k, static_v: static key and value used for attention operators.


    Shape:
        Inputs:
        - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
          the embedding dimension.
        - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
          the embedding dimension.
        - key_padding_mask: :math:`(N, S)`, ByteTensor, where N is the batch size, S is the source sequence length.
        - attn_mask: :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
        - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
          N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.
        - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length,
          N is the batch size, E is the embedding dimension. E/num_heads is the head dimension.

        Outputs:
        - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
          E is the embedding dimension.
        - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
          L is the target sequence length, S is the source sequence length.
    z(embed_dim must be divisible by num_headsg      à?Tr   r   )r{   r   Nr   )r   r   z#bias cannot be added to static key.z%bias cannot be added to static value.z-inf)rm   rn   g      à¿r¹   r¹   r¹   r¹   r¹   r¹   r¹   r¹   )r   Úequalr   r   rK   r  r²   Úchunkr2   Z_unwrap_optionalr?  Úrepeatr$  r   r   rº   rÜ   Z	transposeZbmmrM   Zmasked_fillr    rl   r¿   )-ÚqueryÚkeyrx   Zembed_dim_to_checkZ	num_headsZin_proj_weightZin_proj_biasZbias_kZbias_vZadd_zero_attnZ	dropout_pZout_proj_weightZout_proj_biasrn   Zkey_padding_maskZneed_weightsZ	attn_maskZuse_separate_proj_weightZq_proj_weightZk_proj_weightZv_proj_weightZstatic_kZstatic_vZqkv_sameZkv_sameZtgt_lenZbszZ	embed_dimZhead_dimZscalingÚqrà   ÚvÚ_bZ_startZ_endÚ_wZq_proj_weight_non_optZlen1Zlen2Zk_proj_weight_non_optZv_proj_weight_non_optZsrc_lenZattn_output_weightsZattn_outputr$   r$   r%   Úmulti_head_attention_forward4  s   I
 














<<

 




 rI  )NNFN)NNFN)NNFN)NNFN)Nr   r   FF)Nr   r   FF)Nr   r   FF)Nr   r   FF)Nr   r   FF)Nr   r   FF)Nr   N)Nr   N)Nr   N)NF)NF)F)F)F)F)F)F)rk   TF)rk   FF)rk   TF)rk   TF)rk   FF)F)Fr¹   )r¹   ç      ð¿)rJ  r~   F)F)r~   F)F)r~   F)r‰   Fç      À?çUUUUUUÕ?)rK  rL  FF)rk   )Nr   N)Nr   Nr¹   )r   Fr¢   r¹   )Nr   N)N)N)NNr¸   FF)NNr   Fr¾   FN)NNFrÌ   rÍ   )NNNNTrÌ   rÍ   )NNrÍ   )NNrÍ   )rÙ   rÚ   r~   )r   r¾   Féœÿÿÿ)NNrM  Nr¾   )TFNrô   Nr¾   )NNr¾   éœÿÿÿ)NNrN  Nr¾   )NNNr¾   )NNNr¾   N)r¾   )NNr¾   )NNr¾   )NNr¾   )r   NNr¾   )r~   NNr¾   )NNr¾   )NNr¾   )NNNr¾   )r   NNr¾   )r   r~   NNNr¾   )NNr  N)NNr  N)NN)NN)r³   r$  N)N)r-  r   )r¸   r/  F)r~   r   r/  FNNr¾   )r   r   r5  N)r   r   r   )r   r   r   )
TNTNFNNNNN)ŸÚ__doc__Ú
__future__r   rš   r  r   Ztorch._Cr   r   Ú r   rå   Úmodulesr   r   r   r	   r
   r   r   r   Z_jit_internalr   r   Zconv1dZconv2dZconv3dZconv_transpose1dZconv_transpose2dZconv_transpose3dZconv_tbcr]   r   r   rS   rÝ   r&   r'   Ú__name__r   r0   r1   r/   r4   r:   r9   r<   r>   r=   r?   rA   r@   rI   rQ   rL   rR   r\   r^   r`   ra   r_   rc   rd   rb   rf   rg   re   r!  rh   rj   rl   rp   rs   rt   ru   rw   rv   rV   rz   r}   r€   r   r�   rƒ   r‚   r†   r…   rˆ   r‡   r‹   rŠ   rŒ   r�   r�   Zlog_sigmoidr  r“   r”   r—   r˜   Zsoftplusrž   rŸ   r    r«   r¬   Z
softshrinkr–   r­   r²   r³   r·   r»   rÇ   rÎ   rÖ   r×   rØ   rã   rä   rì   rõ   rø   rù   rû   rü   r  r  r  r  r  r  r	  r
  r  r  r  r  Zpixel_shuffler  r  r"  r#  ZGRID_SAMPLE_INTERPOLATION_MODESZGRID_SAMPLE_PADDING_MODESr'  r,  rÛ   r0  ZpdistZcosine_similarityZone_hotr2  r6  r9  r;  r<  r.  rI  r$   r$   r$   r%   Ú<module>   s  ! 	

  -    /           








#<
 P     

 . M ,9 / 7 0     <lWe
,  &         