3
I4W^/  ã               @   s(   d dl Z ddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s4   e Zd ZdZd‡ fdd„	Zdd„ Zdd
d„Z‡  ZS )ÚAdagradaâ  Implements Adagrad algorithm.

    It has been proposed in `Adaptive Subgradient Methods for Online Learning
    and Stochastic Optimization`_.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lr_decay (float, optional): learning rate decay (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-10)

    .. _Adaptive Subgradient Methods for Online Learning and Stochastic
        Optimization: http://jmlr.org/papers/v12/duchi11a.html
    ç{®Gáz„?r   ç»½×Ùß|Û=c                sÜ   d|kst dj|ƒƒ‚d|ks,t dj|ƒƒ‚d|ksBt dj|ƒƒ‚d|ksXt dj|ƒƒ‚d|ksnt dj|ƒƒ‚t|||||d�}tt| ƒj||ƒ xD| jD ]:}x4|d D ](}	| j|	 }
d	|
d
< tj	|	j
|ƒ|
d< q¨W qšW d S )Ng        zInvalid learning rate: {}zInvalid lr_decay value: {}zInvalid weight_decay value: {}z+Invalid initial_accumulator_value value: {}zInvalid epsilon value: {})ÚlrÚlr_decayÚepsÚweight_decayÚinitial_accumulator_valueÚparamsr   ÚstepÚsum)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__Úparam_groupsÚstateÚtorchZ	full_likeÚdata)Úselfr   r   r   r
   r   r	   ÚdefaultsÚgroupÚpr   )Ú	__class__© ú4/tmp/pip-build-e66j97b2/torch/torch/optim/adagrad.pyr      s$    

zAdagrad.__init__c             C   s<   x6| j D ],}x&|d D ]}| j| }|d jƒ  qW qW d S )Nr   r   )r   r   Zshare_memory_)r   r   r   r   r   r   r   Úshare_memory.   s    
zAdagrad.share_memoryNc                s†  d}|dk	r|ƒ }�xl| j D �]`}�xX|d D �]J}|jdkr@q.|jj‰ | j| }|d  d7  < |d dkr’|jjjr€tdƒ‚ˆ j|d |jƒ‰ |d d|d d |d	    }ˆ j�r@ˆ jƒ ‰ ˆ jƒ ‰ˆ j	ƒ }ˆ j
ƒ ‰‡ ‡‡fd
d„}|d j||jdƒƒƒ |d jˆ ƒ}	|	j	ƒ jƒ j|d ƒ}
|jj| |||
 ƒƒ q.|d jdˆ ˆ ƒ |d jƒ j|d ƒ}	|jj| ˆ |	ƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r
   r   z;weight_decay option is not compatible with sparse gradientsr   r   c                s6   ˆ j }ˆjƒ dks| jƒ dkr*|ƒ jˆ ƒS |ˆ| ˆƒS )Nr   )ÚnewZdimZ
resize_as_)ÚvaluesÚconstructor)ÚgradÚgrad_indicesÚsizer   r   Úmake_sparseV   s    z!Adagrad.step.<locals>.make_sparser   é   r	   )r   r#   r   r   Z	is_sparseÚRuntimeErrorÚaddZcoalesceZ_indicesZ_valuesr%   Zadd_ÚpowZsparse_maskZsqrt_Zaddcmul_ÚsqrtZaddcdiv_)r   ZclosureZlossr   r   r   ZclrZgrad_valuesr&   ZstdZ
std_valuesr   )r#   r$   r%   r   r   4   s:    


 zAdagrad.step)r   r   r   r   r   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Ú__classcell__r   r   )r   r   r      s   r   )r   Z	optimizerr   r   r   r   r   r   Ú<module>   s   