3
I4W^o  ã               @   s(   d dl Z ddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s8   e Zd ZdZd‡ fdd„	Z‡ fd	d
„Zddd„Z‡  ZS )ÚRMSpropa7  Implements RMSprop algorithm.

    Proposed by G. Hinton in his
    `course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.

    The centered version first appears in `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.

    The implementation here takes the square root of the gradient average before
    adding epsilon (note that TensorFlow interchanges these two operations). The effective
    learning rate is thus :math:`\alpha/(\sqrt{v} + \epsilon)` where :math:`\alpha`
    is the scheduled learning rate and :math:`v` is the weighted moving average
    of the squared gradient.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        momentum (float, optional): momentum factor (default: 0)
        alpha (float, optional): smoothing constant (default: 0.99)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        centered (bool, optional) : if ``True``, compute the centered RMSProp,
            the gradient is normalized by an estimation of its variance
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)

    ç{®Gáz„?ç®Gáz®ï?ç:Œ0âŽyE>r   Fc       	         s˜   d|kst dj|ƒƒ‚d|ks,t dj|ƒƒ‚d|ksBt dj|ƒƒ‚d|ksXt dj|ƒƒ‚d|ksnt dj|ƒƒ‚t||||||d�}tt| ƒj||ƒ d S )Ng        zInvalid learning rate: {}zInvalid epsilon value: {}zInvalid momentum value: {}zInvalid weight_decay value: {}zInvalid alpha value: {})ÚlrÚmomentumÚalphaÚepsÚcenteredÚweight_decay)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)	ÚselfÚparamsr   r
   r   r   r	   r   Údefaults)Ú	__class__© ú4/tmp/pip-build-e66j97b2/torch/torch/optim/rmsprop.pyr   "   s    zRMSprop.__init__c                s<   t t| ƒj|ƒ x&| jD ]}|jddƒ |jddƒ qW d S )Nr	   r   r   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroup)r   r   r   r   1   s    zRMSprop.__setstate__Nc             C   sÊ  d}|dk	r|ƒ }�x°| j D �]¤}�xœ|d D �]Ž}|jdkr@q.|jj}|jrVtdƒ‚| j| }t|ƒdkr¸d|d< tj|jƒ|d< |d dkr tj|jƒ|d< |d	 r¸tj|jƒ|d
< |d }|d }|d  d7  < |d dkrö|j	|d |jƒ}|j
|ƒjd| ||ƒ |d	 �rT|d
 }	|	j
|ƒjd| |ƒ |jd|	|	ƒjƒ j|d ƒ}
n|jƒ j|d ƒ}
|d dk�r¨|d }|j
|d ƒj||
ƒ |jj|d  |ƒ q.|jj|d  ||
ƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z)RMSprop does not support sparse gradientsr   ÚstepÚ
square_avgr	   Zmomentum_bufferr   Úgrad_avgr
   r   r   r   r   éÿÿÿÿ)r   ÚgradÚdataZ	is_sparseÚRuntimeErrorr   ÚlenÚtorchZ
zeros_likeÚaddZmul_Zaddcmul_Zadd_ZaddcmulZsqrt_ÚsqrtZaddcdiv_)r   ZclosureZlossr   Úpr"   r   r   r
   r    ÚavgÚbufr   r   r   r   7   sF    


zRMSprop.step)r   r   r   r   r   F)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Ú__classcell__r   r   )r   r   r      s   r   )r&   Z	optimizerr   r   r   r   r   r   Ú<module>   s   