3
I4W^òµ  ã               @   s  d dl Z d dlZd dlZd dlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddl
mZ d	d
lmZ ejejdœZddd„ZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZdS )é    Né   )ÚModuleé   )Ú	Parameter)ÚPackedSequence)Úinit)Ú_VFé   )Ú_parameter_list)ÚRNN_TANHÚRNN_RELUc             C   s   | j ||ƒS )N)Zindex_select)ZtensorÚpermutationÚdim© r   ú5/tmp/pip-build-e66j97b2/torch/torch/nn/modules/rnn.pyÚapply_permutation   s    r   c                   sÆ   e Zd ZddddddddgZd/‡ fdd„	Zdd„ Z‡ fdd„Zdd„ Zdd„ Ze	eƒdd„ ƒZ
dd„ Zdd„ Zd0dd„Zd d!„ Zd"d#„ Zd1d%d&„Zd'd(„ Z‡ fd)d*„Zed+d,„ ƒZed-d.„ ƒZ‡  ZS )2ÚRNNBaseÚmodeÚ
input_sizeÚhidden_sizeÚ
num_layersÚbiasÚbatch_firstÚdropoutÚbidirectionalr   TFç        c	                s  t t| ƒjƒ  || _|| _|| _|| _|| _|| _t	|ƒ| _
|| _|rJdnd}	t|tjƒ s€d|  kondkn   s€t|tƒrˆtdƒ‚|dkrª|dkrªtjdj||ƒƒ |dkr¼d| }
n:|dkrÎd	| }
n(|d
krÜ|}
n|dkrê|}
ntd| ƒ‚g | _xöt|ƒD ]ê‰ xât|	ƒD ]Ö}ˆ dk�r&|n||	 }ttj|
|ƒƒ}ttj|
|ƒƒ}ttj|
ƒƒ}ttj|
ƒƒ}||||f}|dk�r„dnd‰ddg}|�r¢|ddg7 }‡ ‡fdd„|D ƒ}x$t||ƒD ]\}}t| ||ƒ �qÂW | jj|ƒ �qW �qW | jƒ  | jƒ  d S )Nr   r   r   zbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz›dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout={} and num_layers={}ÚLSTMé   ÚGRUr	   r   r   zUnrecognized RNN mode: Ú_reverseÚ zweight_ih_l{}{}zweight_hh_l{}{}zbias_ih_l{}{}zbias_hh_l{}{}c                s   g | ]}|j ˆ ˆƒ‘qS r   )Úformat)Ú.0Úx)ÚlayerÚsuffixr   r   ú
<listcomp>Q   s    z$RNNBase.__init__.<locals>.<listcomp>)Úsuperr   Ú__init__r   r   r   r   r   r   Úfloatr   r   Ú
isinstanceÚnumbersÚNumberÚboolÚ
ValueErrorÚwarningsÚwarnr!   Ú_all_weightsÚranger   ÚtorchÚTensorÚzipÚsetattrÚappendÚflatten_parametersÚreset_parameters)Úselfr   r   r   r   r   r   r   r   Únum_directionsZ	gate_sizeÚ	directionZlayer_input_sizeZw_ihZw_hhZb_ihZb_hhZlayer_paramsZparam_namesÚnameÚparam)Ú	__class__)r$   r%   r   r(      sX    
(



zRNNBase.__init__c             C   sÌ   t | jƒ ƒj}|j s&tjjj|ƒ r*dS | j}t	dd„ |D ƒƒ}t
|ƒt
|ƒkrVdS tjj|ƒ�` ddljjj}tjƒ �> tj|| jrŒdnd| j|j| jƒ| j| j| jt| jƒƒ W dQ R X W dQ R X dS )zÅResets parameter data pointer so that they can use faster code paths.

        Right now, this works only if the module is on the GPU and cuDNN is enabled.
        Otherwise, it's a no-op.
        Nc             s   s   | ]}|j ƒ V  qd S )N)Zdata_ptr)r"   Úpr   r   r   ú	<genexpr>i   s    z-RNNBase.flatten_parameters.<locals>.<genexpr>r   r   r   )ÚnextÚ
parametersÚdataZis_cudar3   ÚbackendsZcudnnZis_acceptableÚ_flat_weightsÚsetÚlenZcudaZ	device_ofZtorch.backends.cudnn.rnnÚrnnZno_gradZ_cudnn_rnn_flatten_weightr   r   Zget_cudnn_moder   r   r   r   r-   r   )r:   Z	any_paramÚall_weightsZunique_data_ptrsrI   r   r   r   r8   Z   s    
zRNNBase.flatten_parametersc                s   t t| ƒj|ƒ}| jƒ  |S )N)r'   r   Ú_applyr8   )r:   ÚfnÚret)r?   r   r   rK   z   s    zRNNBase._applyc             C   s6   dt j| jƒ }x | jƒ D ]}tj|| |ƒ qW d S )Ng      ð?)ÚmathÚsqrtr   rC   r   Úuniform_)r:   ÚstdvÚweightr   r   r   r9      s    zRNNBase.reset_parametersc             C   s   dd„ | j D ƒS )Nc             S   s   g | ]}|D ]}|‘qqS r   r   )r"   ÚweightsrR   r   r   r   r&   …   s    z3RNNBase._get_flat_weights_names.<locals>.<listcomp>)r1   )r:   r   r   r   Ú_get_flat_weights_names„   s    zRNNBase._get_flat_weights_namesc             C   s   | j S )N)rF   )r:   r   r   r   Ú_get_flat_weights‡   s    zRNNBase._get_flat_weightsc             C   s\   |d k	rdnd}|j ƒ |kr0tdj||j ƒ ƒƒ‚| j|jdƒkrXtdj| j|jdƒƒƒ‚d S )Nr   r	   z%input must have {} dimensions, got {}r   z?input.size(-1) must be equal to input_size. Expected {}, got {}éÿÿÿÿrV   )r   ÚRuntimeErrorr!   r   Úsize)r:   ÚinputÚbatch_sizesZexpected_input_dimr   r   r   Úcheck_input‹   s    zRNNBase.check_inputc             C   sX   |d k	r|d }t |ƒ}n| jr*|jdƒn|jdƒ}| jr>dnd}| j| || jf}|S )Nr   r   r   )Úintr   rX   r   r   r   )r:   rY   rZ   Z
mini_batchr;   Úexpected_hidden_sizer   r   r   Úget_expected_hidden_size—   s    

z RNNBase.get_expected_hidden_sizeúExpected hidden size {}, got {}c             C   s(   |j ƒ |kr$t|j|t|j ƒ ƒƒƒ‚d S )N)rX   rW   r!   Útuple)r:   Úhxr]   Úmsgr   r   r   Úcheck_hidden_size£   s    zRNNBase.check_hidden_sizec             C   s(   | j ||ƒ | j||ƒ}| j||ƒ d S )N)r[   r^   rc   )r:   rY   ÚhiddenrZ   r]   r   r   r   Úcheck_forward_args¨   s    zRNNBase.check_forward_argsc             C   s   |d kr|S t ||ƒS )N)r   )r:   ra   r   r   r   r   Úpermute_hidden¯   s    zRNNBase.permute_hiddenNc          
   C   s:  t |tƒ}|r,|\}}}}|d }t|ƒ}n&d }| jr@|jdƒn|jdƒ}d }d }|d krŠ| jrddnd}tj| j| || j	|j
|jd�}n| j||ƒ}| j|||ƒ t| j }	|d krà|	||| jƒ | j| j| j| j| j| jƒ	}
n&|	|||| jƒ | j| j| j| j| jƒ	}
|
d }|
d }|�r*t||||ƒ}|| j||ƒfS )Nr   r   r   )ÚdtypeÚdevice)r*   r   r\   r   rX   r   r3   Úzerosr   r   rg   rh   rf   re   Ú
_rnn_implsr   rU   r   r   Útraining)r:   rY   ra   Z	is_packedrZ   Úsorted_indicesÚunsorted_indicesÚmax_batch_sizer;   Ú_implÚresultÚoutputrd   r   r   r   Úforwardµ   s6    


zRNNBase.forwardc             C   sl   d}| j dkr|d7 }| jdk	r(|d7 }| jdk	r:|d7 }| jdkrL|d	7 }| jdk	r^|d
7 }|jf | jŽS )Nz{input_size}, {hidden_size}r   z, num_layers={num_layers}Tz, bias={bias}Fz, batch_first={batch_first}r   z, dropout={dropout}z, bidirectional={bidirectional})r   r   r   r   r   r!   Ú__dict__)r:   Úsr   r   r   Ú
extra_reprÚ   s    




zRNNBase.extra_reprc                sÜ   t t| ƒj|ƒ d|kr"|d | _t| jd d tƒr:d S | j}| jrJdnd}g | _x‚t|ƒD ]v‰ xpt|ƒD ]d}|dkr|dnd‰ddd	d
g}‡ ‡fdd„|D ƒ}| j	r¸|  j|g7  _ql|  j|d d… g7  _qlW q^W d S )NrJ   r   r   r   r   r    zweight_ih_l{}{}zweight_hh_l{}{}zbias_ih_l{}{}zbias_hh_l{}{}c                s   g | ]}|j ˆ ˆƒ‘qS r   )r!   )r"   r#   )r$   r%   r   r   r&   õ   s    z(RNNBase.__setstate__.<locals>.<listcomp>)
r'   r   Ú__setstate__r1   r*   Ústrr   r   r2   r   )r:   Údr   r;   r<   rS   )r?   )r$   r%   r   rv   è   s     
zRNNBase.__setstate__c             C   s   dd„ | j D ƒS )Nc             S   s   g | ]}|D ]}|‘qqS r   r   )r"   Zlayerparamsr@   r   r   r   r&   ý   s    z)RNNBase._flat_weights.<locals>.<listcomp>)rJ   )r:   r   r   r   rF   û   s    zRNNBase._flat_weightsc                s   ‡ fdd„ˆ j D ƒS )Nc                s   g | ]}‡ fd d„|D ƒ‘qS )c                s   g | ]}t ˆ |ƒ‘qS r   )Úgetattr)r"   rR   )r:   r   r   r&     s    z2RNNBase.all_weights.<locals>.<listcomp>.<listcomp>r   )r"   rS   )r:   r   r   r&     s    z'RNNBase.all_weights.<locals>.<listcomp>)r1   )r:   r   )r:   r   rJ   ÿ   s    zRNNBase.all_weights)r   TFr   F)r_   )N)Ú__name__Ú
__module__Ú__qualname__Ú__constants__r(   r8   rK   r9   rT   r
   rU   r[   r^   rc   re   rf   rr   ru   rv   ÚpropertyrF   rJ   Ú__classcell__r   r   )r?   r   r      s&   

 < 

%r   c                   s    e Zd ZdZ‡ fdd„Z‡  ZS )ÚRNNas  Applies a multi-layer Elman RNN with :math:`tanh` or :math:`ReLU` non-linearity to an
    input sequence.


    For each element in the input sequence, each layer computes the following
    function:

    .. math::
        h_t = \text{tanh}(W_{ih} x_t + b_{ih} + W_{hh} h_{(t-1)} + b_{hh})

    where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is
    the input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the
    previous layer at time `t-1` or the initial hidden state at time `0`.
    If :attr:`nonlinearity` is ``'relu'``, then `ReLU` is used instead of `tanh`.

    Args:
        input_size: The number of expected features in the input `x`
        hidden_size: The number of features in the hidden state `h`
        num_layers: Number of recurrent layers. E.g., setting ``num_layers=2``
            would mean stacking two RNNs together to form a `stacked RNN`,
            with the second RNN taking in outputs of the first RNN and
            computing the final results. Default: 1
        nonlinearity: The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'``
        bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`.
            Default: ``True``
        batch_first: If ``True``, then the input and output tensors are provided
            as `(batch, seq, feature)`. Default: ``False``
        dropout: If non-zero, introduces a `Dropout` layer on the outputs of each
            RNN layer except the last layer, with dropout probability equal to
            :attr:`dropout`. Default: 0
        bidirectional: If ``True``, becomes a bidirectional RNN. Default: ``False``

    Inputs: input, h_0
        - **input** of shape `(seq_len, batch, input_size)`: tensor containing the features
          of the input sequence. The input can also be a packed variable length
          sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence`
          or :func:`torch.nn.utils.rnn.pack_sequence`
          for details.
        - **h_0** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the initial hidden state for each element in the batch.
          Defaults to zero if not provided. If the RNN is bidirectional,
          num_directions should be 2, else it should be 1.

    Outputs: output, h_n
        - **output** of shape `(seq_len, batch, num_directions * hidden_size)`: tensor
          containing the output features (`h_t`) from the last layer of the RNN,
          for each `t`.  If a :class:`torch.nn.utils.rnn.PackedSequence` has
          been given as the input, the output will also be a packed sequence.

          For the unpacked case, the directions can be separated
          using ``output.view(seq_len, batch, num_directions, hidden_size)``,
          with forward and backward being direction `0` and `1` respectively.
          Similarly, the directions can be separated in the packed case.
        - **h_n** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the hidden state for `t = seq_len`.

          Like *output*, the layers can be separated using
          ``h_n.view(num_layers, num_directions, batch, hidden_size)``.

    Shape:
        - Input1: :math:`(L, N, H_{in})` tensor containing input features where
          :math:`H_{in}=\text{input\_size}` and `L` represents a sequence length.
        - Input2: :math:`(S, N, H_{out})` tensor
          containing the initial hidden state for each element in the batch.
          :math:`H_{out}=\text{hidden\_size}`
          Defaults to zero if not provided. where :math:`S=\text{num\_layers} * \text{num\_directions}`
          If the RNN is bidirectional, num_directions should be 2, else it should be 1.
        - Output1: :math:`(L, N, H_{all})` where :math:`H_{all}=\text{num\_directions} * \text{hidden\_size}`
        - Output2: :math:`(S, N, H_{out})` tensor containing the next hidden state
          for each element in the batch

    Attributes:
        weight_ih_l[k]: the learnable input-hidden weights of the k-th layer,
            of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is
            `(hidden_size, num_directions * hidden_size)`
        weight_hh_l[k]: the learnable hidden-hidden weights of the k-th layer,
            of shape `(hidden_size, hidden_size)`
        bias_ih_l[k]: the learnable input-hidden bias of the k-th layer,
            of shape `(hidden_size)`
        bias_hh_l[k]: the learnable hidden-hidden bias of the k-th layer,
            of shape `(hidden_size)`

    .. note::
        All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`
        where :math:`k = \frac{1}{\text{hidden\_size}}`

    .. include:: cudnn_persistent_rnn.rst

    Examples::

        >>> rnn = nn.RNN(10, 20, 2)
        >>> input = torch.randn(5, 3, 10)
        >>> h0 = torch.randn(2, 3, 20)
        >>> output, hn = rnn(input, h0)
    c                sf   d|krF|d dkrd}n$|d dkr,d}nt dj|d ƒƒ‚|d= nd}tt| ƒj|f|ž|Ž d S )NÚnonlinearityÚtanhr   Úrelur   zUnknown nonlinearity '{}')r.   r!   r'   r€   r(   )r:   ÚargsÚkwargsr   )r?   r   r   r(   e  s    zRNN.__init__)rz   r{   r|   Ú__doc__r(   r   r   r   )r?   r   r€     s   _r€   c                   sz   e Zd ZdZdddgiZ‡ fdd„Zdd„ Zd	d
„ Zdd„ Ze	j
jddd„ƒZe	j
jddd„ƒZe	j
jddd„ƒZ‡  ZS )r   a·  Applies a multi-layer long short-term memory (LSTM) RNN to an input
    sequence.


    For each element in the input sequence, each layer computes the following
    function:

    .. math::
        \begin{array}{ll} \\
            i_t = \sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\
            f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\
            g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{(t-1)} + b_{hg}) \\
            o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\
            c_t = f_t * c_{(t-1)} + i_t * g_t \\
            h_t = o_t * \tanh(c_t) \\
        \end{array}

    where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the cell
    state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}`
    is the hidden state of the layer at time `t-1` or the initial hidden
    state at time `0`, and :math:`i_t`, :math:`f_t`, :math:`g_t`,
    :math:`o_t` are the input, forget, cell, and output gates, respectively.
    :math:`\sigma` is the sigmoid function, and :math:`*` is the Hadamard product.

    In a multilayer LSTM, the input :math:`x^{(l)}_t` of the :math:`l` -th layer
    (:math:`l >= 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous layer multiplied by
    dropout :math:`\delta^{(l-1)}_t` where each :math:`\delta^{(l-1)}_t` is a Bernoulli random
    variable which is :math:`0` with probability :attr:`dropout`.

    Args:
        input_size: The number of expected features in the input `x`
        hidden_size: The number of features in the hidden state `h`
        num_layers: Number of recurrent layers. E.g., setting ``num_layers=2``
            would mean stacking two LSTMs together to form a `stacked LSTM`,
            with the second LSTM taking in outputs of the first LSTM and
            computing the final results. Default: 1
        bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`.
            Default: ``True``
        batch_first: If ``True``, then the input and output tensors are provided
            as (batch, seq, feature). Default: ``False``
        dropout: If non-zero, introduces a `Dropout` layer on the outputs of each
            LSTM layer except the last layer, with dropout probability equal to
            :attr:`dropout`. Default: 0
        bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False``

    Inputs: input, (h_0, c_0)
        - **input** of shape `(seq_len, batch, input_size)`: tensor containing the features
          of the input sequence.
          The input can also be a packed variable length sequence.
          See :func:`torch.nn.utils.rnn.pack_padded_sequence` or
          :func:`torch.nn.utils.rnn.pack_sequence` for details.
        - **h_0** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the initial hidden state for each element in the batch.
          If the LSTM is bidirectional, num_directions should be 2, else it should be 1.
        - **c_0** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the initial cell state for each element in the batch.

          If `(h_0, c_0)` is not provided, both **h_0** and **c_0** default to zero.


    Outputs: output, (h_n, c_n)
        - **output** of shape `(seq_len, batch, num_directions * hidden_size)`: tensor
          containing the output features `(h_t)` from the last layer of the LSTM,
          for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been
          given as the input, the output will also be a packed sequence.

          For the unpacked case, the directions can be separated
          using ``output.view(seq_len, batch, num_directions, hidden_size)``,
          with forward and backward being direction `0` and `1` respectively.
          Similarly, the directions can be separated in the packed case.
        - **h_n** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the hidden state for `t = seq_len`.

          Like *output*, the layers can be separated using
          ``h_n.view(num_layers, num_directions, batch, hidden_size)`` and similarly for *c_n*.
        - **c_n** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the cell state for `t = seq_len`.

    Attributes:
        weight_ih_l[k] : the learnable input-hidden weights of the :math:`\text{k}^{th}` layer
            `(W_ii|W_if|W_ig|W_io)`, of shape `(4*hidden_size, input_size)` for `k = 0`.
            Otherwise, the shape is `(4*hidden_size, num_directions * hidden_size)`
        weight_hh_l[k] : the learnable hidden-hidden weights of the :math:`\text{k}^{th}` layer
            `(W_hi|W_hf|W_hg|W_ho)`, of shape `(4*hidden_size, hidden_size)`
        bias_ih_l[k] : the learnable input-hidden bias of the :math:`\text{k}^{th}` layer
            `(b_ii|b_if|b_ig|b_io)`, of shape `(4*hidden_size)`
        bias_hh_l[k] : the learnable hidden-hidden bias of the :math:`\text{k}^{th}` layer
            `(b_hi|b_hf|b_hg|b_ho)`, of shape `(4*hidden_size)`

    .. note::
        All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`
        where :math:`k = \frac{1}{\text{hidden\_size}}`

    .. include:: cudnn_persistent_rnn.rst

    Examples::

        >>> rnn = nn.LSTM(10, 20, 2)
        >>> input = torch.randn(5, 3, 10)
        >>> h0 = torch.randn(2, 3, 20)
        >>> c0 = torch.randn(2, 3, 20)
        >>> output, (hn, cn) = rnn(input, (h0, c0))
    rr   Úforward_packedÚforward_tensorc                s   t t| ƒjd|ž|Ž d S )Nr   )r   )r'   r   r(   )r:   r„   r…   )r?   r   r   r(   ë  s    zLSTM.__init__c             C   s@   | j ||ƒ | j||ƒ}| j|d |dƒ | j|d |dƒ d S )Nr   z"Expected hidden[0] size {}, got {}r   z"Expected hidden[1] size {}, got {})r[   r^   rc   )r:   rY   rd   rZ   r]   r   r   r   re   î  s    zLSTM.check_forward_argsc             C   s(   |d kr|S t |d |ƒt |d |ƒfS )Nr   r   )r   )r:   ra   r   r   r   r   rf   ø  s    zLSTM.permute_hiddenc          
   C   sÒ   |d kr@| j rdnd}tj| j| || j|j|jd�}||f}n| j||ƒ}| j|||ƒ |d krŽt	j
||| jƒ | j| j| j| j| j | jƒ	}n(t	j
|||| jƒ | j| j| j| j| j ƒ	}|d }	|dd … }
|	|
fS )Nr   r   )rg   rh   r   )r   r3   ri   r   r   rg   rh   rf   re   r   ZlstmrU   r   r   rk   r   )r:   rY   ra   rZ   rn   rl   r;   ri   rp   rq   rd   r   r   r   Úforward_implþ  s     
zLSTM.forward_implNc       	      C   sL   d }| j r|jdƒn|jdƒ}d }d }| j|||||ƒ\}}|| j||ƒfS )Nr   r   )r   rX   r‰   rf   )	r:   rY   ra   rZ   rn   rl   rm   rq   rd   r   r   r   rˆ     s    zLSTM.forward_tensorc       	      C   sP   |\}}}}|d }t |ƒ}| j|||||ƒ\}}t||||ƒ}|| j||ƒfS )Nr   )r\   r‰   r   rf   )	r:   rY   ra   rZ   rl   rm   rn   rq   rd   r   r   r   r‡   #  s    zLSTM.forward_packedc             C   s&   t |tƒr| j||ƒS | j||ƒS d S )N)r*   r   r‡   rˆ   )r:   rY   ra   r   r   r   rr   /  s    
zLSTM.forward)N)N)N)rz   r{   r|   r†   Ú__overloads__r(   re   rf   r‰   r3   Ú_jit_internalÚexportrˆ   r‡   Úignorerr   r   r   r   )r?   r   r   �  s   g
r   c                   sr   e Zd ZdZdddgiZ‡ fdd„Zdd„ Zd	d
„ Zej	j
ddd„ƒZej	j
ddd„ƒZej	jddd„ƒZ‡  ZS )r   aí  Applies a multi-layer gated recurrent unit (GRU) RNN to an input sequence.


    For each element in the input sequence, each layer computes the following
    function:

    .. math::
        \begin{array}{ll}
            r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\
            z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\
            n_t = \tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)}+ b_{hn})) \\
            h_t = (1 - z_t) * n_t + z_t * h_{(t-1)}
        \end{array}

    where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input
    at time `t`, :math:`h_{(t-1)}` is the hidden state of the layer
    at time `t-1` or the initial hidden state at time `0`, and :math:`r_t`,
    :math:`z_t`, :math:`n_t` are the reset, update, and new gates, respectively.
    :math:`\sigma` is the sigmoid function, and :math:`*` is the Hadamard product.

    In a multilayer GRU, the input :math:`x^{(l)}_t` of the :math:`l` -th layer
    (:math:`l >= 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous layer multiplied by
    dropout :math:`\delta^{(l-1)}_t` where each :math:`\delta^{(l-1)}_t` is a Bernoulli random
    variable which is :math:`0` with probability :attr:`dropout`.

    Args:
        input_size: The number of expected features in the input `x`
        hidden_size: The number of features in the hidden state `h`
        num_layers: Number of recurrent layers. E.g., setting ``num_layers=2``
            would mean stacking two GRUs together to form a `stacked GRU`,
            with the second GRU taking in outputs of the first GRU and
            computing the final results. Default: 1
        bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`.
            Default: ``True``
        batch_first: If ``True``, then the input and output tensors are provided
            as (batch, seq, feature). Default: ``False``
        dropout: If non-zero, introduces a `Dropout` layer on the outputs of each
            GRU layer except the last layer, with dropout probability equal to
            :attr:`dropout`. Default: 0
        bidirectional: If ``True``, becomes a bidirectional GRU. Default: ``False``

    Inputs: input, h_0
        - **input** of shape `(seq_len, batch, input_size)`: tensor containing the features
          of the input sequence. The input can also be a packed variable length
          sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence`
          for details.
        - **h_0** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the initial hidden state for each element in the batch.
          Defaults to zero if not provided. If the RNN is bidirectional,
          num_directions should be 2, else it should be 1.

    Outputs: output, h_n
        - **output** of shape `(seq_len, batch, num_directions * hidden_size)`: tensor
          containing the output features h_t from the last layer of the GRU,
          for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been
          given as the input, the output will also be a packed sequence.
          For the unpacked case, the directions can be separated
          using ``output.view(seq_len, batch, num_directions, hidden_size)``,
          with forward and backward being direction `0` and `1` respectively.

          Similarly, the directions can be separated in the packed case.
        - **h_n** of shape `(num_layers * num_directions, batch, hidden_size)`: tensor
          containing the hidden state for `t = seq_len`

          Like *output*, the layers can be separated using
          ``h_n.view(num_layers, num_directions, batch, hidden_size)``.

    Shape:
        - Input1: :math:`(L, N, H_{in})` tensor containing input features where
          :math:`H_{in}=\text{input\_size}` and `L` represents a sequence length.
        - Input2: :math:`(S, N, H_{out})` tensor
          containing the initial hidden state for each element in the batch.
          :math:`H_{out}=\text{hidden\_size}`
          Defaults to zero if not provided. where :math:`S=\text{num\_layers} * \text{num\_directions}`
          If the RNN is bidirectional, num_directions should be 2, else it should be 1.
        - Output1: :math:`(L, N, H_{all})` where :math:`H_{all}=\text{num\_directions} * \text{hidden\_size}`
        - Output2: :math:`(S, N, H_{out})` tensor containing the next hidden state
          for each element in the batch

    Attributes:
        weight_ih_l[k] : the learnable input-hidden weights of the :math:`\text{k}^{th}` layer
            (W_ir|W_iz|W_in), of shape `(3*hidden_size, input_size)` for `k = 0`.
            Otherwise, the shape is `(3*hidden_size, num_directions * hidden_size)`
        weight_hh_l[k] : the learnable hidden-hidden weights of the :math:`\text{k}^{th}` layer
            (W_hr|W_hz|W_hn), of shape `(3*hidden_size, hidden_size)`
        bias_ih_l[k] : the learnable input-hidden bias of the :math:`\text{k}^{th}` layer
            (b_ir|b_iz|b_in), of shape `(3*hidden_size)`
        bias_hh_l[k] : the learnable hidden-hidden bias of the :math:`\text{k}^{th}` layer
            (b_hr|b_hz|b_hn), of shape `(3*hidden_size)`

    .. note::
        All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`
        where :math:`k = \frac{1}{\text{hidden\_size}}`

    .. include:: cudnn_persistent_rnn.rst

    Examples::

        >>> rnn = nn.GRU(10, 20, 2)
        >>> input = torch.randn(5, 3, 10)
        >>> h0 = torch.randn(2, 3, 20)
        >>> output, hn = rnn(input, h0)
    rr   r‡   rˆ   c                s   t t| ƒjd|ž|Ž d S )Nr   )r   )r'   r   r(   )r:   r„   r…   )r?   r   r   r(   ¡  s    zGRU.__init__c          
   C   s`   |d kr4t j||| jƒ | j| j| j| j| j| jƒ	}n(t j|||| jƒ | j| j| j| j| jƒ	}|S )N)	r   ZgrurU   r   r   r   rk   r   r   )r:   rY   ra   rZ   rp   r   r   r   Úrun_impl¤  s    zGRU.run_implc       
      C   sx   |d kr8| j rdnd}tj| j| || j|j|jd�}n| j||ƒ}| j|||ƒ | j	|||ƒ}|d }|d }	||	fS )Nr   r   )rg   rh   r   )
r   r3   ri   r   r   rg   rh   rf   re   rŽ   )
r:   rY   ra   rZ   rn   rl   r;   rp   rq   rd   r   r   r   r‰   ®  s    zGRU.forward_implNc       	      C   sP   |\}}}}|d }t |ƒ}| j|||||ƒ\}}t||||ƒ}|| j||ƒfS )Nr   )r\   r‰   r   rf   )	r:   rY   ra   rZ   rl   rm   rn   rq   rd   r   r   r   r‡   À  s    zGRU.forward_packedc       	      C   sL   d }| j r|jdƒn|jdƒ}d }d }| j|||||ƒ\}}|| j||ƒfS )Nr   r   )r   rX   r‰   rf   )	r:   rY   ra   rZ   rn   rl   rm   rq   rd   r   r   r   rˆ   Ê  s    zGRU.forward_tensorc             C   s&   t |tƒr| j||ƒS | j||ƒS d S )N)r*   r   r‡   rˆ   )r:   rY   ra   r   r   r   rr   Ô  s    
zGRU.forward)N)N)N)rz   r{   r|   r†   rŠ   r(   rŽ   r‰   r3   r‹   rŒ   r‡   rˆ   r�   rr   r   r   r   )r?   r   r   7  s   g
		r   c                   sH   e Zd ZdddgZ‡ fdd„Zdd„ Zdd	„ Zddd„Zdd„ Z‡  Z	S )ÚRNNCellBaser   r   r   c                sž   t t| ƒjƒ  || _|| _|| _ttj|| |ƒƒ| _	ttj|| |ƒƒ| _
|rzttj|| ƒƒ| _ttj|| ƒƒ| _n| jdd ƒ | jdd ƒ | jƒ  d S )NÚbias_ihÚbias_hh)r'   r�   r(   r   r   r   r   r3   r4   Ú	weight_ihÚ	weight_hhr�   r‘   Zregister_parameterr9   )r:   r   r   r   Ú
num_chunks)r?   r   r   r(   ß  s    zRNNCellBase.__init__c             C   sJ   d}d| j kr | jdk	r |d7 }d| j kr<| jdkr<|d7 }|jf | j ŽS )Nz{input_size}, {hidden_size}r   Tz, bias={bias}r�   r‚   z, nonlinearity={nonlinearity})rs   r   r�   r!   )r:   rt   r   r   r   ru   î  s    zRNNCellBase.extra_reprc             C   s,   |j dƒ| jkr(tdj|j dƒ| jƒƒ‚d S )Nr   z6input has inconsistent input_size: got {}, expected {})rX   r   rW   r!   )r:   rY   r   r   r   Úcheck_forward_inputö  s    zRNNCellBase.check_forward_inputr    c             C   s`   |j dƒ|j dƒkr2tdj|j dƒ||j dƒƒƒ‚|j dƒ| jkr\tdj||j dƒ| jƒƒ‚d S )Nr   z8Input batch size {} doesn't match hidden{} batch size {}r   z:hidden{} has inconsistent hidden_size: got {}, expected {})rX   rW   r!   r   )r:   rY   ra   Zhidden_labelr   r   r   Úcheck_forward_hiddenü  s    z RNNCellBase.check_forward_hiddenc             C   s6   dt j| jƒ }x | jƒ D ]}tj|| |ƒ qW d S )Ng      ð?)rN   rO   r   rC   r   rP   )r:   rQ   rR   r   r   r   r9     s    zRNNCellBase.reset_parameters)r    )
rz   r{   r|   r}   r(   ru   r•   r–   r9   r   r   r   )r?   r   r�   Ü  s   

r�   c                   s8   e Zd ZdZddddgZd‡ fdd	„	Zddd„Z‡  ZS )ÚRNNCellaê  An Elman RNN cell with tanh or ReLU non-linearity.

    .. math::

        h' = \tanh(W_{ih} x + b_{ih}  +  W_{hh} h + b_{hh})

    If :attr:`nonlinearity` is `'relu'`, then ReLU is used in place of tanh.

    Args:
        input_size: The number of expected features in the input `x`
        hidden_size: The number of features in the hidden state `h`
        bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`.
            Default: ``True``
        nonlinearity: The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'``

    Inputs: input, hidden
        - **input** of shape `(batch, input_size)`: tensor containing input features
        - **hidden** of shape `(batch, hidden_size)`: tensor containing the initial hidden
          state for each element in the batch.
          Defaults to zero if not provided.

    Outputs: h'
        - **h'** of shape `(batch, hidden_size)`: tensor containing the next hidden state
          for each element in the batch

    Shape:
        - Input1: :math:`(N, H_{in})` tensor containing input features where
          :math:`H_{in}` = `input_size`
        - Input2: :math:`(N, H_{out})` tensor containing the initial hidden
          state for each element in the batch where :math:`H_{out}` = `hidden_size`
          Defaults to zero if not provided.
        - Output: :math:`(N, H_{out})` tensor containing the next hidden state
          for each element in the batch

    Attributes:
        weight_ih: the learnable input-hidden weights, of shape
            `(hidden_size, input_size)`
        weight_hh: the learnable hidden-hidden weights, of shape
            `(hidden_size, hidden_size)`
        bias_ih: the learnable input-hidden bias, of shape `(hidden_size)`
        bias_hh: the learnable hidden-hidden bias, of shape `(hidden_size)`

    .. note::
        All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`
        where :math:`k = \frac{1}{\text{hidden\_size}}`

    Examples::

        >>> rnn = nn.RNNCell(10, 20)
        >>> input = torch.randn(6, 3, 10)
        >>> hx = torch.randn(3, 20)
        >>> output = []
        >>> for i in range(6):
                hx = rnn(input[i], hx)
                output.append(hx)
    r   r   r   r�   Tr‚   c                s"   t t| ƒj|||dd� || _d S )Nr   )r”   )r'   r—   r(   r�   )r:   r   r   r   r�   )r?   r   r   r(   I  s    zRNNCell.__init__Nc             C   s¦   | j |ƒ |d kr0tj|jdƒ| j|j|jd�}| j||dƒ | jdkrft	j
||| j| j| j| jƒ}n<| jdkrŽt	j||| j| j| j| jƒ}n|}tdj| jƒƒ‚|S )Nr   )rg   rh   r    r‚   rƒ   zUnknown nonlinearity: {})r•   r3   ri   rX   r   rg   rh   r–   r�   r   Zrnn_tanh_cellr’   r“   r�   r‘   Zrnn_relu_cellrW   r!   )r:   rY   ra   rM   r   r   r   rr   M  s$    


zRNNCell.forward)Tr‚   )N)rz   r{   r|   r†   r}   r(   rr   r   r   r   )r?   r   r—     s   8r—   c                   s,   e Zd ZdZd‡ fdd„	Zd	dd„Z‡  ZS )
ÚLSTMCellaK	  A long short-term memory (LSTM) cell.

    .. math::

        \begin{array}{ll}
        i = \sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\
        f = \sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\
        g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\
        o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\
        c' = f * c + i * g \\
        h' = o * \tanh(c') \\
        \end{array}

    where :math:`\sigma` is the sigmoid function, and :math:`*` is the Hadamard product.

    Args:
        input_size: The number of expected features in the input `x`
        hidden_size: The number of features in the hidden state `h`
        bias: If ``False``, then the layer does not use bias weights `b_ih` and
            `b_hh`. Default: ``True``

    Inputs: input, (h_0, c_0)
        - **input** of shape `(batch, input_size)`: tensor containing input features
        - **h_0** of shape `(batch, hidden_size)`: tensor containing the initial hidden
          state for each element in the batch.
        - **c_0** of shape `(batch, hidden_size)`: tensor containing the initial cell state
          for each element in the batch.

          If `(h_0, c_0)` is not provided, both **h_0** and **c_0** default to zero.

    Outputs: (h_1, c_1)
        - **h_1** of shape `(batch, hidden_size)`: tensor containing the next hidden state
          for each element in the batch
        - **c_1** of shape `(batch, hidden_size)`: tensor containing the next cell state
          for each element in the batch

    Attributes:
        weight_ih: the learnable input-hidden weights, of shape
            `(4*hidden_size, input_size)`
        weight_hh: the learnable hidden-hidden weights, of shape
            `(4*hidden_size, hidden_size)`
        bias_ih: the learnable input-hidden bias, of shape `(4*hidden_size)`
        bias_hh: the learnable hidden-hidden bias, of shape `(4*hidden_size)`

    .. note::
        All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`
        where :math:`k = \frac{1}{\text{hidden\_size}}`

    Examples::

        >>> rnn = nn.LSTMCell(10, 20)
        >>> input = torch.randn(6, 3, 10)
        >>> hx = torch.randn(3, 20)
        >>> cx = torch.randn(3, 20)
        >>> output = []
        >>> for i in range(6):
                hx, cx = rnn(input[i], (hx, cx))
                output.append(hx)
    Tc                s   t t| ƒj|||dd� d S )Nr   )r”   )r'   r˜   r(   )r:   r   r   r   )r?   r   r   r(   £  s    zLSTMCell.__init__Nc             C   sx   | j |ƒ |d kr8tj|jdƒ| j|j|jd�}||f}| j||d dƒ | j||d dƒ tj	||| j
| j| j| jƒS )Nr   )rg   rh   z[0]r   z[1])r•   r3   ri   rX   r   rg   rh   r–   r   Z	lstm_cellr’   r“   r�   r‘   )r:   rY   ra   ri   r   r   r   rr   ¦  s    
zLSTMCell.forward)T)N)rz   r{   r|   r†   r(   rr   r   r   r   )r?   r   r˜   f  s   ;r˜   c                   s,   e Zd ZdZd‡ fdd„	Zd	dd„Z‡  ZS )
ÚGRUCellaP	  A gated recurrent unit (GRU) cell

    .. math::

        \begin{array}{ll}
        r = \sigma(W_{ir} x + b_{ir} + W_{hr} h + b_{hr}) \\
        z = \sigma(W_{iz} x + b_{iz} + W_{hz} h + b_{hz}) \\
        n = \tanh(W_{in} x + b_{in} + r * (W_{hn} h + b_{hn})) \\
        h' = (1 - z) * n + z * h
        \end{array}

    where :math:`\sigma` is the sigmoid function, and :math:`*` is the Hadamard product.

    Args:
        input_size: The number of expected features in the input `x`
        hidden_size: The number of features in the hidden state `h`
        bias: If ``False``, then the layer does not use bias weights `b_ih` and
            `b_hh`. Default: ``True``

    Inputs: input, hidden
        - **input** of shape `(batch, input_size)`: tensor containing input features
        - **hidden** of shape `(batch, hidden_size)`: tensor containing the initial hidden
          state for each element in the batch.
          Defaults to zero if not provided.

    Outputs: h'
        - **h'** of shape `(batch, hidden_size)`: tensor containing the next hidden state
          for each element in the batch

    Shape:
        - Input1: :math:`(N, H_{in})` tensor containing input features where
          :math:`H_{in}` = `input_size`
        - Input2: :math:`(N, H_{out})` tensor containing the initial hidden
          state for each element in the batch where :math:`H_{out}` = `hidden_size`
          Defaults to zero if not provided.
        - Output: :math:`(N, H_{out})` tensor containing the next hidden state
          for each element in the batch

    Attributes:
        weight_ih: the learnable input-hidden weights, of shape
            `(3*hidden_size, input_size)`
        weight_hh: the learnable hidden-hidden weights, of shape
            `(3*hidden_size, hidden_size)`
        bias_ih: the learnable input-hidden bias, of shape `(3*hidden_size)`
        bias_hh: the learnable hidden-hidden bias, of shape `(3*hidden_size)`

    .. note::
        All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})`
        where :math:`k = \frac{1}{\text{hidden\_size}}`

    Examples::

        >>> rnn = nn.GRUCell(10, 20)
        >>> input = torch.randn(6, 3, 10)
        >>> hx = torch.randn(3, 20)
        >>> output = []
        >>> for i in range(6):
                hx = rnn(input[i], hx)
                output.append(hx)
    Tc                s   t t| ƒj|||dd� d S )Nr	   )r”   )r'   r™   r(   )r:   r   r   r   )r?   r   r   r(   ó  s    zGRUCell.__init__Nc             C   sZ   | j |ƒ |d kr0tj|jdƒ| j|j|jd�}| j||dƒ tj	||| j
| j| j| jƒS )Nr   )rg   rh   r    )r•   r3   ri   rX   r   rg   rh   r–   r   Zgru_cellr’   r“   r�   r‘   )r:   rY   ra   r   r   r   rr   ö  s    
zGRUCell.forward)T)N)rz   r{   r|   r†   r(   rr   r   r   r   )r?   r   r™   µ  s   <r™   )r   )rN   r3   r/   r+   Úmoduler   Z	parameterr   Z	utils.rnnr   r    r   r   r‹   r
   Zrnn_tanhZrnn_relurj   r   r   r€   r   r   r�   r—   r˜   r™   r   r   r   r   Ú<module>   s.   

 m} 7 &2XO