3
I4W^,  ã               @   s0   d dl Z d dlZddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s8   e Zd ZdZd‡ fdd	„	Z‡ fd
d„Zddd„Z‡  ZS )ÚAdama  Implements Adam algorithm.

    It has been proposed in `Adam: A Method for Stochastic Optimization`_.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)

    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ
    çü©ñÒMbP?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>r   Fc                s°   d|kst dj|ƒƒ‚d|ks,t dj|ƒƒ‚d|d   koBdk n  sZt dj|d ƒƒ‚d|d   kopdk n  sˆt dj|d ƒƒ‚t|||||d	�}tt| ƒj||ƒ d S )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚepsÚweight_decayÚamsgrad)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr	   r
   r   r   r   Údefaults)Ú	__class__© ú1/tmp/pip-build-e66j97b2/torch/torch/optim/adam.pyr      s    
zAdam.__init__c                s0   t t| ƒj|ƒ x| jD ]}|jddƒ qW d S )Nr   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroup)r   r   r   r   ,   s    zAdam.__setstate__Nc             C   sÒ  d}|dk	r|ƒ }�x¸| j D �]¬}�x¤|d D �]–}|jdkr@q.|jj}|jrVtdƒ‚|d }| j| }t|ƒdkr°d|d< tj|jƒ|d< tj|jƒ|d< |r°tj|jƒ|d	< |d |d  }}	|rÎ|d	 }
|d
 \}}|d  d7  < d||d   }d||d   }|d dk�r*|j	|d |jƒ |j
|ƒj	d| |ƒ |	j
|ƒjd| ||ƒ |�rŒtj|
|	|
d� |
jƒ tj|ƒ j	|d ƒ}n|	jƒ tj|ƒ j	|d ƒ}|d | }|jj| ||ƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r   ÚstepÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr
   r   r   )Úoutr   r	   )r   ÚgradÚdataZ	is_sparseÚRuntimeErrorr   ÚlenÚtorchZ
zeros_likeZadd_Zmul_Zaddcmul_ÚmaxÚsqrtÚmathZaddcdiv_)r   ZclosureZlossr   Úpr#   r   r   r   r    r!   Zbeta1Zbeta2Zbias_correction1Zbias_correction2ZdenomZ	step_sizer   r   r   r   1   sH    

z	Adam.step©r   r   )r   r,   r   r   F)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Ú__classcell__r   r   )r   r   r      s
    r   )r*   r'   Z	optimizerr   r   r   r   r   r   Ú<module>   s   