3
I4W^ó  ã               @   s0   d dl Z d dlZddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s,   e Zd ZdZd‡ fdd„	Zdd	d
„Z‡  ZS )Ú
SparseAdama  Implements lazy version of Adam algorithm suitable for sparse tensors.

    In this variant, only moments that show up in the gradient get updated, and
    only those portions of the gradient get applied to the parameters.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)

    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    çü©ñÒMbP?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>c                s¬   d|k st dj|ƒƒ‚d|k s,t dj|ƒƒ‚d|d   koBdk n  sZt dj|d ƒƒ‚d|d   kopdk n  sˆt dj|d ƒƒ‚t|||d	�}tt| ƒj||ƒ d S )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚeps)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr	   r
   r   Údefaults)Ú	__class__© ú8/tmp/pip-build-e66j97b2/torch/torch/optim/sparse_adam.pyr      s    zSparseAdam.__init__Nc                sà  d}|dk	r|ƒ }�xÆ| j D �]º}�x²|d D �]¤}|jdkr@q.|jj‰ ˆ jsVtdƒ‚| j| }t|ƒdkr”d|d< tj|jƒ|d< tj|jƒ|d< |d  d7  < ˆ j	ƒ ‰ ˆ j
ƒ ‰ˆ jƒ }ˆ jƒ ‰‡ ‡‡fd	d
„}|d |d  }}	|d \}
}|jˆ ƒjƒ }|j|ƒjd|
 ƒ}|j||ƒƒ |	jˆ ƒjƒ }|jdƒj|ƒjd| ƒ}|	j||ƒƒ |j|ƒ}|j|ƒ |jƒ j|d ƒ}~~d|
|d   }d||d   }|d tj|ƒ | }|jj|| |j|ƒ ƒƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   zISparseAdam does not support dense gradients, please consider Adam insteadr   ÚstepÚexp_avgÚ
exp_avg_sqr   c                s6   ˆ j }ˆjƒ dks| jƒ dkr*|ƒ jˆ ƒS |ˆ| ˆƒS )Nr   )ÚnewZdimZ
resize_as_)ÚvaluesÚconstructor)ÚgradÚgrad_indicesÚsizer   r   Úmake_sparseI   s    z$SparseAdam.step.<locals>.make_sparser
   é   r   r	   )Zparam_groupsr   ÚdataZ	is_sparseÚRuntimeErrorÚstateÚlenÚtorchZ
zeros_likeZcoalesceZ_indicesZ_valuesr   Zsparse_maskÚsubZmul_Zadd_ÚpowZsub_Zsqrt_ÚmathÚsqrtZdiv_)r   ZclosureZlossÚgroupÚpr$   Zgrad_valuesr    r   r   Zbeta1Zbeta2Zold_exp_avg_valuesZexp_avg_update_valuesZold_exp_avg_sq_valuesZexp_avg_sq_update_valuesZnumerZdenomZbias_correction1Zbias_correction2Z	step_sizer   )r   r   r   r   r   %   sL    



$zSparseAdam.step©r   r   )r   r-   r   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   Ú__classcell__r   r   )r   r   r      s   r   )r)   r&   Z	optimizerr   r   r   r   r   r   Ú<module>   s   