3
I4W^„  ã               @   s(   d dl Z ddlmZ G dd„ deƒZdS )é    Né   )Ú	Optimizerc                   s,   e Zd ZdZd‡ fdd„	Zdd	d
„Z‡  ZS )ÚAdadeltaaÙ  Implements Adadelta algorithm.

    It has been proposed in `ADADELTA: An Adaptive Learning Rate Method`__.

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        rho (float, optional): coefficient used for computing a running average
            of squared gradients (default: 0.9)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-6)
        lr (float, optional): coefficient that scale delta before it is applied
            to the parameters (default: 1.0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)

    __ https://arxiv.org/abs/1212.5701
    ç      ð?çÍÌÌÌÌÌì?ç�íµ ÷Æ°>r   c                sŽ   d|kst dj|ƒƒ‚d|  ko(dkn  s<t dj|ƒƒ‚d|ksRt dj|ƒƒ‚d|ksht dj|ƒƒ‚t||||d�}tt| ƒj||ƒ d S )Ng        zInvalid learning rate: {}g      ð?zInvalid rho value: {}zInvalid epsilon value: {}zInvalid weight_decay value: {})ÚlrÚrhoÚepsÚweight_decay)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr   r	   r
   r   Údefaults)Ú	__class__© ú5/tmp/pip-build-e66j97b2/torch/torch/optim/adadelta.pyr      s    zAdadelta.__init__Nc             C   s^  d}|dk	r|ƒ }�xD| j D �]8}�x0|d D �]"}|jdkr@q.|jj}|jrVtdƒ‚| j| }t|ƒdkr”d|d< tj|jƒ|d< tj|jƒ|d< |d |d  }}|d |d	  }	}
|d  d
7  < |d dkræ|j	|d |jƒ}|j
|	ƒjd
|	 ||ƒ |j	|
ƒjƒ }|j	|
ƒjƒ j|ƒj
|ƒ}|jj|d  |ƒ |j
|	ƒjd
|	 ||ƒ q.W qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z*Adadelta does not support sparse gradientsr   ÚstepÚ
square_avgÚ	acc_deltar	   r
   r   r   r   )Zparam_groupsÚgradÚdataZ	is_sparseÚRuntimeErrorÚstateÚlenÚtorchZ
zeros_likeÚaddZmul_Zaddcmul_Zsqrt_Zdiv_Zadd_)r   ZclosureZlossÚgroupÚpr   r   r   r   r	   r
   ZstdÚdeltar   r   r   r   &   s4    

 zAdadelta.step)r   r   r   r   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   Ú__classcell__r   r   )r   r   r      s   r   )r   Z	optimizerr   r   r   r   r   r   Ú<module>   s   