3
I4W^ÿ  ã               @   s,   d dl Z ddlmZmZ G dd„ deƒZdS )é    Né   )Ú	OptimizerÚrequiredc                   sB   e Zd ZdZeddddf‡ fdd„	Z‡ fdd„Zdd	d
„Z‡  ZS )ÚSGDa•  Implements stochastic gradient descent (optionally with momentum).

    Nesterov momentum is based on the formula from
    `On the importance of initialization and momentum in deep learning`__.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float): learning rate
        momentum (float, optional): momentum factor (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        dampening (float, optional): dampening for momentum (default: 0)
        nesterov (bool, optional): enables Nesterov momentum (default: False)

    Example:
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> optimizer.zero_grad()
        >>> loss_fn(model(input), target).backward()
        >>> optimizer.step()

    __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf

    .. note::
        The implementation of SGD with Momentum/Nesterov subtly differs from
        Sutskever et. al. and implementations in some other frameworks.

        Considering the specific case of Momentum, the update can be written as

        .. math::
                  v_{t+1} = \mu * v_{t} + g_{t+1} \\
                  p_{t+1} = p_{t} - lr * v_{t+1}

        where p, g, v and :math:`\mu` denote the parameters, gradient,
        velocity, and momentum respectively.

        This is in contrast to Sutskever et. al. and
        other frameworks which employ an update of the form

        .. math::
             v_{t+1} = \mu * v_{t} + lr * g_{t+1} \\
             p_{t+1} = p_{t} - v_{t+1}

        The Nesterov version is analogously modified.
    r   Fc                sŽ   |t k	r|dk rtdj|ƒƒ‚|dk r4tdj|ƒƒ‚|dk rJtdj|ƒƒ‚t|||||d�}|rx|dksp|dkrxtdƒ‚tt| ƒj||ƒ d S )Ng        zInvalid learning rate: {}zInvalid momentum value: {}zInvalid weight_decay value: {})ÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovr   z8Nesterov momentum requires a momentum and zero dampening)r   Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)ÚselfÚparamsr   r   r   r	   r
   Údefaults)Ú	__class__© ú0/tmp/pip-build-e66j97b2/torch/torch/optim/sgd.pyr   3   s    
zSGD.__init__c                s0   t t| ƒj|ƒ x| jD ]}|jddƒ qW d S )Nr
   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroup)r   r   r   r   B   s    zSGD.__setstate__Nc             C   sü   d}|dk	r|ƒ }xä| j D ]Ú}|d }|d }|d }|d }x´|d D ]¨}|jdkrXqH|jj}	|dkrv|	j||jƒ |dkrÜ| j| }
d|
kr¨tj|	ƒjƒ  }|
d< n|
d }|j|ƒjd	| |	ƒ |rØ|	j	||ƒ}	n|}	|jj|d
  |	ƒ qHW qW |S )z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr	   r   r   r
   r   r   Zmomentum_bufferr   r   )
r   ZgradÚdataZadd_r   ÚtorchÚcloneÚdetachZmul_Úadd)r   ZclosureZlossr   r	   r   r   r
   ÚpZd_pZparam_stateÚbufr   r   r   ÚstepG   s2    

zSGD.step)N)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r"   Ú__classcell__r   r   )r   r   r      s
   ,r   )r   Z	optimizerr   r   r   r   r   r   r   Ú<module>   s   