3
I4W^Ñ  ã               @   s0   d dl Z d dlZddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s,   e Zd ZdZd‡ fdd„	Zdd
d„Z‡  ZS )ÚASGDaÚ  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)

    .. _Acceleration of stochastic approximation by averaging:
        http://dl.acm.org/citation.cfm?id=131098
    ç{®Gáz„?ç-Cëâ6?ç      è?ç    €„.Ar   c                sT   d|kst dj|ƒƒ‚d|ks,t dj|ƒƒ‚t|||||d�}tt| ƒj||ƒ d S )Ng        zInvalid learning rate: {}zInvalid weight_decay value: {})ÚlrÚlambdÚalphaÚt0Úweight_decay)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr	   r
   r   r   r   Údefaults)Ú	__class__© ú1/tmp/pip-build-e66j97b2/torch/torch/optim/asgd.pyr      s    
zASGD.__init__Nc             C   s˜  d}|dk	r|ƒ }�x~| j D �]r}�xj|d D �]\}|jdkr@q.|jj}|jrVtdƒ‚| j| }t|ƒdkr˜d|d< |d |d< d|d	< tj|jƒ|d
< |d  d7  < |d dkrÆ|j	|d |jƒ}|jj
d|d |d   ƒ |jj|d  |ƒ |d	 dk�r*|d
 j|jj|d
 ƒj|d	 ƒƒ n|d
 j|jƒ |d tjd|d |d  |d   |d ƒ |d< dtd|d |d  ƒ |d	< q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z&ASGD does not support sparse gradientsr   Ústepr	   Úetar   ÚmuZaxr   r
   r   r   )Zparam_groupsÚgradÚdataZ	is_sparseÚRuntimeErrorÚstateÚlenÚtorchZ
zeros_likeÚaddZmul_Zadd_ÚsubÚmulZcopy_ÚmathÚpowÚmax)r   ZclosureZlossÚgroupÚpr   r   r   r   r   r   #   s8    

&.&z	ASGD.step)r   r   r   r   r   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   Ú__classcell__r   r   )r   r   r      s   
r   )r%   r!   Z	optimizerr   r   r   r   r   r   Ú<module>   s   