3
I4W^a  ã               @   s(   d dl Z ddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s,   e Zd ZdZd‡ fdd„	Zdd
d„Z‡  ZS )ÚAdamaxaÈ  Implements Adamax algorithm (a variant of Adam based on infinity norm).

    It has been proposed in `Adam: A Method for Stochastic Optimization`__.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)

    __ https://arxiv.org/abs/1412.6980
    çü©ñÒMb`?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>r   c                sÄ   d|kst dj|ƒƒ‚d|ks,t dj|ƒƒ‚d|d   koBdk n  sZt dj|d ƒƒ‚d|d   kopdk n  sˆt dj|d ƒƒ‚d|ksžt d	j|ƒƒ‚t||||d
�}tt| ƒj||ƒ d S )Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {}zInvalid weight_decay value: {})ÚlrÚbetasÚepsÚweight_decay)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr	   r
   r   r   Údefaults)Ú	__class__© ú3/tmp/pip-build-e66j97b2/torch/torch/optim/adamax.pyr      s    zAdamax.__init__Nc             C   s‚  d}|dk	r|ƒ }�xh| j D �]\}�xT|d D �]F}|jdkr@q.|jj}|jrVtdƒ‚| j| }t|ƒdkr”d|d< tj|jƒ|d< tj|jƒ|d< |d |d  }}|d \}	}
|d	 }|d  d
7  < |d dkrè|j	|d |jƒ}|j
|	ƒjd
|	 |ƒ tj|j
|
ƒjdƒ|jƒ j|ƒjdƒgdƒ}tj|dd||jƒ jƒ fd� d
|	|d   }|d | }|jj| ||ƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z(Adamax does not support sparse gradientsr   ÚstepÚexp_avgÚexp_infr
   r   r   r   F)ZkeepdimÚoutr	   )Zparam_groupsÚgradÚdataZ	is_sparseÚRuntimeErrorÚstateÚlenÚtorchZ
zeros_likeÚaddZmul_Zadd_ÚcatZ	unsqueezeÚabsZ
unsqueeze_ÚmaxÚnewÚlongZaddcdiv_)r   ZclosureZlossÚgroupÚpr   r   r   r   Zbeta1Zbeta2r   Znorm_bufZbias_correctionZclrr   r   r   r   '   s>    

zAdamax.step©r   r   )r   r*   r   r   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   Ú__classcell__r   r   )r   r   r      s    r   )r!   Z	optimizerr   r   r   r   r   r   Ú<module>   s   