3
I4W^"  ã               @   s0   d dl Z d dlZddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s8   e Zd ZdZd‡ fdd	„	Z‡ fd
d„Zddd„Z‡  ZS )ÚAdamWaÃ  Implements AdamW algorithm.

    The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_.
    The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay coefficient (default: 1e-2)
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)

    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ
    çü©ñÒMbP?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>ç{®Gáz„?Fc                s°   d|kst dj|ƒƒ‚d|ks,t dj|ƒƒ‚d|d   koBdk n  sZt dj|d ƒƒ‚d|d   kopdk n  sˆt dj|d ƒƒ‚t|||||d	�}tt| ƒj||ƒ d S )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚepsÚweight_decayÚamsgrad)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr
   r   r   r   r   Údefaults)Ú	__class__© ú2/tmp/pip-build-e66j97b2/torch/torch/optim/adamw.pyr   !   s    
zAdamW.__init__c                s0   t t| ƒj|ƒ x| jD ]}|jddƒ qW d S )Nr   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroup)r   r   r   r   /   s    zAdamW.__setstate__Nc             C   sÎ  d}|dk	r|ƒ }�x´| j D �]¨}�x |d D �]’}|jdkr@q.|jjd|d |d   ƒ |jj}|jrrtdƒ‚|d }| j| }t|ƒdkrÌd|d	< tj	|jƒ|d
< tj	|jƒ|d< |rÌtj	|jƒ|d< |d
 |d  }}	|rê|d }
|d \}}|d	  d7  < d||d	   }d||d	   }|j|ƒj
d| |ƒ |	j|ƒjd| ||ƒ |�rˆtj|
|	|
d� |
jƒ tj|ƒ j
|d ƒ}n|	jƒ tj|ƒ j
|d ƒ}|d | }|jj| ||ƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r
   r   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r   ÚstepÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr   )Úoutr   )r   ÚgradÚdataZmul_Z	is_sparseÚRuntimeErrorr   ÚlenÚtorchZ
zeros_likeZadd_Zaddcmul_ÚmaxÚsqrtÚmathZaddcdiv_)r   ZclosureZlossr   Úpr$   r   r   r    r!   r"   Zbeta1Zbeta2Zbias_correction1Zbias_correction2ZdenomZ	step_sizer   r   r   r   4   sF    

z
AdamW.step©r   r   )r   r-   r   r	   F)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Ú__classcell__r   r   )r   r   r      s
    r   )r+   r(   Z	optimizerr   r   r   r   r   r   Ú<module>   s   