3
I4W^†«  ã               @   sô   d dl Z d dlZd dlmZ d dlmZ d dlZd dlZd dlm	Z	 ddl
mZ G dd„ deƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZdS )é    N)Úinf)Úwraps)Úbisect_righté   )Ú	Optimizerc               @   s8   e Zd Zddd„Zdd„ Zdd„ Zdd	„ Zddd„Zd
S )Ú_LRSchedulerr   c             C   sÐ   t |tƒstdjt|ƒjƒƒ‚|| _|dkrRx|jD ]}|jd|d ƒ q4W d}n.x,t	|jƒD ]\}}d|kr^t
dj|ƒƒ‚q^W ttdd„ |jƒƒ| _|| _d	d
„ }|| jjƒ| j_d| j_d| _| j|ƒ d S )Nz{} is not an Optimizerr   Ú
initial_lrÚlrr   zRparam 'initial_lr' is not specified in param_groups[{}] when resuming an optimizerc             S   s   | d S )Nr   © )Úgroupr
   r
   ú9/tmp/pip-build-e66j97b2/torch/torch/optim/lr_scheduler.pyÚ<lambda>   s    z'_LRScheduler.__init__.<locals>.<lambda>c                sN   t | ddƒr| S tj| jƒ‰| j‰ˆƒ j‰ ~ tˆƒ‡ ‡‡fdd„ƒ}d|_|S )NÚ_with_counterFc                 s*   ˆƒ }| j d7  _ ˆj|ˆ ƒ}|| |ŽS )Nr   )Ú_step_countÚ__get__)ÚargsÚkwargsÚinstanceÚwrapped)ÚclsÚfuncÚinstance_refr
   r   Úwrapper.   s    z<_LRScheduler.__init__.<locals>.with_counter.<locals>.wrapperT)ÚgetattrÚweakrefÚrefÚ__self__Ú__func__Ú	__class__r   r   )Úmethodr   r
   )r   r   r   r   Úwith_counter!   s    	z+_LRScheduler.__init__.<locals>.with_counteréÿÿÿÿ)Ú
isinstancer   Ú	TypeErrorÚformatÚtypeÚ__name__Ú	optimizerÚparam_groupsÚ
setdefaultÚ	enumerateÚKeyErrorÚlistÚmapÚbase_lrsÚ
last_epochÚstepr   )Úselfr'   r/   r   Úir    r
   r
   r   Ú__init__   s&    
z_LRScheduler.__init__c             C   s   dd„ | j jƒ D ƒS )z¥Returns the state of the scheduler as a :class:`dict`.

        It contains an entry for every variable in self.__dict__ which
        is not the optimizer.
        c             S   s   i | ]\}}|d kr||“qS )r'   r
   )Ú.0ÚkeyÚvaluer
   r
   r   ú
<dictcomp>E   s    z+_LRScheduler.state_dict.<locals>.<dictcomp>)Ú__dict__Úitems)r1   r
   r
   r   Ú
state_dict?   s    z_LRScheduler.state_dictc             C   s   | j j|ƒ dS )z¸Loads the schedulers state.

        Arguments:
            state_dict (dict): scheduler state. Should be an object returned
                from a call to :meth:`state_dict`.
        N)r8   Úupdate)r1   r:   r
   r
   r   Úload_state_dictG   s    z_LRScheduler.load_state_dictc             C   s   t ‚d S )N)ÚNotImplementedError)r1   r
   r
   r   Úget_lrP   s    z_LRScheduler.get_lrNc             C   s�   | j dkr>t| jjdƒs&tjdtƒ n| jj dk r>tjdtƒ |  j d7  _ |d kr^| jd }|| _x&t| jj	| j
ƒ ƒD ]\}}||d< qxW d S )Nr   r   a  Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rateaw  Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule.See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rater	   )r   Úhasattrr'   r0   ÚwarningsÚwarnÚUserWarningr/   Úzipr(   r>   )r1   ÚepochÚparam_groupr	   r
   r
   r   r0   S   s    

z_LRScheduler.stepr!   )r!   )N)r&   Ú
__module__Ú__qualname__r3   r:   r<   r>   r0   r
   r
   r
   r   r      s
   
2	r   c                   s:   e Zd ZdZd‡ fdd„	Zdd„ Zdd„ Zd	d
„ Z‡  ZS )ÚLambdaLRaP  Sets the learning rate of each parameter group to the initial lr
    times a given function. When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        lr_lambda (function or list): A function which computes a multiplicative
            factor given an integer parameter epoch, or a list of such
            functions, one for each group in optimizer.param_groups.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # Assuming optimizer has two groups.
        >>> lambda1 = lambda epoch: epoch // 30
        >>> lambda2 = lambda epoch: 0.95 ** epoch
        >>> scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2])
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r   c                s„   || _ t|tƒ r2t|tƒ r2|gt|jƒ | _n6t|ƒt|jƒkr^tdjt|jƒt|ƒƒƒ‚t|ƒ| _|| _	t
t| ƒj||ƒ d S )Nz"Expected {} lr_lambdas, but got {})r'   r"   r,   ÚtupleÚlenr(   Ú
lr_lambdasÚ
ValueErrorr$   r/   ÚsuperrH   r3   )r1   r'   Z	lr_lambdar/   )r   r
   r   r3   „   s    
zLambdaLR.__init__c             C   sb   dd„ | j jƒ D ƒ}dgt| jƒ |d< x4t| jƒD ]&\}}t|tjƒs4|j jƒ |d |< q4W |S )a2  Returns the state of the scheduler as a :class:`dict`.

        It contains an entry for every variable in self.__dict__ which
        is not the optimizer.
        The learning rate lambda functions will only be saved if they are callable objects
        and not if they are functions or lambdas.
        c             S   s   i | ]\}}|dkr||“qS )r'   rK   )r'   rK   r
   )r4   r5   r6   r
   r
   r   r7   ˜   s    z'LambdaLR.state_dict.<locals>.<dictcomp>NrK   )	r8   r9   rJ   rK   r*   r"   ÚtypesÚFunctionTypeÚcopy)r1   r:   ÚidxÚfnr
   r
   r   r:   �   s    zLambdaLR.state_dictc             C   sJ   |j dƒ}| jj|ƒ x.t|ƒD ]"\}}|dk	r | j| jj|ƒ q W dS )z¸Loads the schedulers state.

        Arguments:
            state_dict (dict): scheduler state. Should be an object returned
                from a call to :meth:`state_dict`.
        rK   N)Úpopr8   r;   r*   rK   )r1   r:   rK   rQ   rR   r
   r
   r   r<   ¡   s
    
zLambdaLR.load_state_dictc                s   ‡ fdd„t ˆ jˆ jƒD ƒS )Nc                s   g | ]\}}||ˆ j ƒ ‘qS r
   )r/   )r4   ZlmbdaÚbase_lr)r1   r
   r   ú
<listcomp>°   s   z#LambdaLR.get_lr.<locals>.<listcomp>)rC   rK   r.   )r1   r
   )r1   r   r>   ¯   s    
zLambdaLR.get_lrr!   )r!   )	r&   rF   rG   Ú__doc__r3   r:   r<   r>   Ú__classcell__r
   r
   )r   r   rH   n   s
   rH   c                   s*   e Zd ZdZd	‡ fdd„	Zdd„ Z‡  ZS )
ÚStepLRa`  Sets the learning rate of each parameter group to the initial lr
    decayed by gamma every step_size epochs. When last_epoch=-1, sets
    initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        step_size (int): Period of learning rate decay.
        gamma (float): Multiplicative factor of learning rate decay.
            Default: 0.1.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.05     if epoch < 30
        >>> # lr = 0.005    if 30 <= epoch < 60
        >>> # lr = 0.0005   if 60 <= epoch < 90
        >>> # ...
        >>> scheduler = StepLR(optimizer, step_size=30, gamma=0.1)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    çš™™™™™¹?r   c                s"   || _ || _tt| ƒj||ƒ d S )N)Ú	step_sizeÚgammarM   rX   r3   )r1   r'   rZ   r[   r/   )r   r
   r   r3   Í   s    zStepLR.__init__c                s   ‡ fdd„ˆ j D ƒS )Nc                s"   g | ]}|ˆ j ˆ jˆ j   ‘qS r
   )r[   r/   rZ   )r4   rT   )r1   r
   r   rU   Ó   s   z!StepLR.get_lr.<locals>.<listcomp>)r.   )r1   r
   )r1   r   r>   Ò   s    
zStepLR.get_lrr!   )rY   r!   )r&   rF   rG   rV   r3   r>   rW   r
   r
   )r   r   rX   ´   s   rX   c                   s*   e Zd ZdZd	‡ fdd„	Zdd„ Z‡  ZS )
ÚMultiStepLRa�  Set the learning rate of each parameter group to the initial lr decayed
    by gamma once the number of epoch reaches one of the milestones. When
    last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        milestones (list): List of epoch indices. Must be increasing.
        gamma (float): Multiplicative factor of learning rate decay.
            Default: 0.1.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.05     if epoch < 30
        >>> # lr = 0.005    if 30 <= epoch < 80
        >>> # lr = 0.0005   if epoch >= 80
        >>> scheduler = MultiStepLR(optimizer, milestones=[30,80], gamma=0.1)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    çš™™™™™¹?r   c                s<   t |ƒt|ƒkstd|ƒ‚|| _|| _tt| ƒj||ƒ d S )Nz:Milestones should be a list of increasing integers. Got {})r,   ÚsortedrL   Ú
milestonesr[   rM   r\   r3   )r1   r'   r_   r[   r/   )r   r
   r   r3   ï   s    zMultiStepLR.__init__c                s   ‡ fdd„ˆ j D ƒS )Nc                s$   g | ]}|ˆ j tˆ jˆ jƒ  ‘qS r
   )r[   r   r_   r/   )r4   rT   )r1   r
   r   rU   ø   s   z&MultiStepLR.get_lr.<locals>.<listcomp>)r.   )r1   r
   )r1   r   r>   ÷   s    
zMultiStepLR.get_lrr!   )r]   r!   )r&   rF   rG   rV   r3   r>   rW   r
   r
   )r   r   r\   ×   s   r\   c                   s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )	ÚExponentialLRaS  Set the learning rate of each parameter group to the initial lr decayed
    by gamma every epoch. When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        gamma (float): Multiplicative factor of learning rate decay.
        last_epoch (int): The index of last epoch. Default: -1.
    r   c                s   || _ tt| ƒj||ƒ d S )N)r[   rM   r`   r3   )r1   r'   r[   r/   )r   r
   r   r3     s    zExponentialLR.__init__c                s   ‡ fdd„ˆ j D ƒS )Nc                s   g | ]}|ˆ j ˆ j  ‘qS r
   )r[   r/   )r4   rT   )r1   r
   r   rU     s   z(ExponentialLR.get_lr.<locals>.<listcomp>)r.   )r1   r
   )r1   r   r>   
  s    
zExponentialLR.get_lrr!   )r!   )r&   rF   rG   rV   r3   r>   rW   r
   r
   )r   r   r`   ü   s   r`   c                   s*   e Zd ZdZd	‡ fdd„	Zdd„ Z‡  ZS )
ÚCosineAnnealingLRa–  Set the learning rate of each parameter group using a cosine annealing
    schedule, where :math:`\eta_{max}` is set to the initial lr and
    :math:`T_{cur}` is the number of epochs since the last restart in SGDR:

    .. math::
        \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})(1 +
        \cos(\frac{T_{cur}}{T_{max}}\pi))

    When last_epoch=-1, sets initial lr as lr.

    It has been proposed in
    `SGDR: Stochastic Gradient Descent with Warm Restarts`_. Note that this only
    implements the cosine annealing part of SGDR, and not the restarts.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        T_max (int): Maximum number of iterations.
        eta_min (float): Minimum learning rate. Default: 0.
        last_epoch (int): The index of last epoch. Default: -1.

    .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:
        https://arxiv.org/abs/1608.03983
    r   r   c                s"   || _ || _tt| ƒj||ƒ d S )N)ÚT_maxÚeta_minrM   ra   r3   )r1   r'   rb   rc   r/   )r   r
   r   r3   (  s    zCosineAnnealingLR.__init__c                s   ‡ fdd„ˆ j D ƒS )Nc                s<   g | ]4}ˆ j |ˆ j  d tjtjˆ j ˆ j ƒ  d  ‘qS )r   é   )rc   ÚmathÚcosÚpir/   rb   )r4   rT   )r1   r
   r   rU   .  s   z,CosineAnnealingLR.get_lr.<locals>.<listcomp>)r.   )r1   r
   )r1   r   r>   -  s    
zCosineAnnealingLR.get_lrr!   )r   r!   )r&   rF   rG   rV   r3   r>   rW   r
   r
   )r   r   ra     s   ra   c            	   @   s`   e Zd ZdZdd
d„Zdd„ Zddd„Zdd„ Zedd„ ƒZ	dd„ Z
dd„ Zdd„ Zdd„ ZdS )ÚReduceLROnPlateauaú	  Reduce learning rate when a metric has stopped improving.
    Models often benefit from reducing the learning rate by a factor
    of 2-10 once learning stagnates. This scheduler reads a metrics
    quantity and if no improvement is seen for a 'patience' number
    of epochs, the learning rate is reduced.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        mode (str): One of `min`, `max`. In `min` mode, lr will
            be reduced when the quantity monitored has stopped
            decreasing; in `max` mode it will be reduced when the
            quantity monitored has stopped increasing. Default: 'min'.
        factor (float): Factor by which the learning rate will be
            reduced. new_lr = lr * factor. Default: 0.1.
        patience (int): Number of epochs with no improvement after
            which learning rate will be reduced. For example, if
            `patience = 2`, then we will ignore the first 2 epochs
            with no improvement, and will only decrease the LR after the
            3rd epoch if the loss still hasn't improved then.
            Default: 10.
        verbose (bool): If ``True``, prints a message to stdout for
            each update. Default: ``False``.
        threshold (float): Threshold for measuring the new optimum,
            to only focus on significant changes. Default: 1e-4.
        threshold_mode (str): One of `rel`, `abs`. In `rel` mode,
            dynamic_threshold = best * ( 1 + threshold ) in 'max'
            mode or best * ( 1 - threshold ) in `min` mode.
            In `abs` mode, dynamic_threshold = best + threshold in
            `max` mode or best - threshold in `min` mode. Default: 'rel'.
        cooldown (int): Number of epochs to wait before resuming
            normal operation after lr has been reduced. Default: 0.
        min_lr (float or list): A scalar or a list of scalars. A
            lower bound on the learning rate of all param groups
            or each group respectively. Default: 0.
        eps (float): Minimal decay applied to lr. If the difference
            between new and old lr is smaller than eps, the update is
            ignored. Default: 1e-8.

    Example:
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> scheduler = ReduceLROnPlateau(optimizer, 'min')
        >>> for epoch in range(10):
        >>>     train(...)
        >>>     val_loss = validate(...)
        >>>     # Note that step should be called after validate()
        >>>     scheduler.step(val_loss)
    Úminçš™™™™™¹?é
   Fç-Cëâ6?Úrelr   ç:Œ0âŽyE>c             C   sü   |dkrt dƒ‚|| _t|tƒs4tdjt|ƒjƒƒ‚|| _t|	t	ƒsNt|	t
ƒr†t|	ƒt|jƒkrzt djt|jƒt|	ƒƒƒ‚t	|	ƒ| _n|	gt|jƒ | _|| _|| _|| _d| _|| _|| _|| _d | _d | _d | _|
| _d| _| j|||d� | jƒ  d S )	Ng      ð?zFactor should be < 1.0.z{} is not an Optimizerzexpected {} min_lrs, got {}r   r   )ÚmodeÚ	thresholdÚthreshold_moder!   )rL   Úfactorr"   r   r#   r$   r%   r&   r'   r,   rI   rJ   r(   Úmin_lrsÚpatienceÚverboseÚcooldownÚcooldown_counterro   rp   rq   ÚbestÚnum_bad_epochsÚ
mode_worseÚepsr/   Ú_init_is_betterÚ_reset)r1   r'   ro   rr   rt   ru   rp   rq   rv   Úmin_lrr{   r
   r
   r   r3   d  s8    
zReduceLROnPlateau.__init__c             C   s   | j | _d| _d| _dS )z3Resets num_bad_epochs counter and cooldown counter.r   N)rz   rx   rw   ry   )r1   r
   r
   r   r}   ‰  s    zReduceLROnPlateau._resetNc             C   s’   t |ƒ}|d kr | jd  }| _|| _| j|| jƒrB|| _d| _n|  jd7  _| jrj|  jd8  _d| _| j| jkrŽ| j|ƒ | j	| _d| _d S )Nr   r   )
Úfloatr/   Ú	is_betterrx   ry   Úin_cooldownrw   rt   Ú
_reduce_lrrv   )r1   ZmetricsrD   Úcurrentr
   r
   r   r0   �  s    
zReduceLROnPlateau.stepc             C   sn   xht | jjƒD ]X\}}t|d ƒ}t|| j | j| ƒ}|| | jkr||d< | jrt	dj
|||ƒƒ qW d S )Nr	   z:Epoch {:5d}: reducing learning rate of group {} to {:.4e}.)r*   r'   r(   r   Úmaxrr   rs   r{   ru   Úprintr$   )r1   rD   r2   rE   Zold_lrZnew_lrr
   r
   r   r‚   ¥  s    zReduceLROnPlateau._reduce_lrc             C   s
   | j dkS )Nr   )rw   )r1   r
   r
   r   r�   ¯  s    zReduceLROnPlateau.in_cooldownc             C   sˆ   | j dkr*| jdkr*d| j }||| k S | j dkrL| jdkrL||| j k S | j dkrv| jdkrv| jd }||| kS ||| j kS d S )Nri   rm   g      ð?Úabsr„   )ro   rq   rp   )r1   Úarx   Zrel_epsilonr
   r
   r   r€   ³  s    

zReduceLROnPlateau.is_betterc             C   s^   |dkrt d| d ƒ‚|d	kr0t d| d ƒ‚|dkr@t| _nt | _|| _|| _|| _d S )
Nri   r„   zmode z is unknown!rm   r†   zthreshold mode >   r„   ri   >   r†   rm   )rL   r   rz   ro   rp   rq   )r1   ro   rp   rq   r
   r
   r   r|   Â  s    z!ReduceLROnPlateau._init_is_betterc             C   s   dd„ | j jƒ D ƒS )Nc             S   s   i | ]\}}|d kr||“qS )r'   r
   )r4   r5   r6   r
   r
   r   r7   Ò  s    z0ReduceLROnPlateau.state_dict.<locals>.<dictcomp>)r8   r9   )r1   r
   r
   r   r:   Ñ  s    zReduceLROnPlateau.state_dictc             C   s&   | j j|ƒ | j| j| j| jd� d S )N)ro   rp   rq   )r8   r;   r|   ro   rp   rq   )r1   r:   r
   r
   r   r<   Ô  s    z!ReduceLROnPlateau.load_state_dict)	ri   rj   rk   Frl   rm   r   r   rn   )N)r&   rF   rG   rV   r3   r}   r0   r‚   Úpropertyr�   r€   r|   r:   r<   r
   r
   r
   r   rh   3  s   /  
#

rh   c            
       sJ   e Zd ZdZd‡ fdd„	Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z	‡  Z
S )ÚCyclicLRa¬  Sets the learning rate of each parameter group according to
    cyclical learning rate policy (CLR). The policy cycles the learning
    rate between two boundaries with a constant frequency, as detailed in
    the paper `Cyclical Learning Rates for Training Neural Networks`_.
    The distance between the two boundaries can be scaled on a per-iteration
    or per-cycle basis.

    Cyclical learning rate policy changes the learning rate after every batch.
    `step` should be called after a batch has been used for training.

    This class has three built-in policies, as put forth in the paper:
    "triangular":
        A basic triangular cycle w/ no amplitude scaling.
    "triangular2":
        A basic triangular cycle that scales initial amplitude by half each cycle.
    "exp_range":
        A cycle that scales initial amplitude by gamma**(cycle iterations) at each
        cycle iteration.

    This implementation was adapted from the github repo: `bckenstler/CLR`_

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        base_lr (float or list): Initial learning rate which is the
            lower boundary in the cycle for each parameter group.
        max_lr (float or list): Upper learning rate boundaries in the cycle
            for each parameter group. Functionally,
            it defines the cycle amplitude (max_lr - base_lr).
            The lr at any cycle is the sum of base_lr
            and some scaling of the amplitude; therefore
            max_lr may not actually be reached depending on
            scaling function.
        step_size_up (int): Number of training iterations in the
            increasing half of a cycle. Default: 2000
        step_size_down (int): Number of training iterations in the
            decreasing half of a cycle. If step_size_down is None,
            it is set to step_size_up. Default: None
        mode (str): One of {triangular, triangular2, exp_range}.
            Values correspond to policies detailed above.
            If scale_fn is not None, this argument is ignored.
            Default: 'triangular'
        gamma (float): Constant in 'exp_range' scaling function:
            gamma**(cycle iterations)
            Default: 1.0
        scale_fn (function): Custom scaling policy defined by a single
            argument lambda function, where
            0 <= scale_fn(x) <= 1 for all x >= 0.
            If specified, then 'mode' is ignored.
            Default: None
        scale_mode (str): {'cycle', 'iterations'}.
            Defines whether scale_fn is evaluated on
            cycle number or cycle iterations (training
            iterations since start of cycle).
            Default: 'cycle'
        cycle_momentum (bool): If ``True``, momentum is cycled inversely
            to learning rate between 'base_momentum' and 'max_momentum'.
            Default: True
        base_momentum (float or list): Lower momentum boundaries in the cycle
            for each parameter group. Note that momentum is cycled inversely
            to learning rate; at the peak of a cycle, momentum is
            'base_momentum' and learning rate is 'max_lr'.
            Default: 0.8
        max_momentum (float or list): Upper momentum boundaries in the cycle
            for each parameter group. Functionally,
            it defines the cycle amplitude (max_momentum - base_momentum).
            The momentum at any cycle is the difference of max_momentum
            and some scaling of the amplitude; therefore
            base_momentum may not actually be reached depending on
            scaling function. Note that momentum is cycled inversely
            to learning rate; at the start of a cycle, momentum is 'max_momentum'
            and learning rate is 'base_lr'
            Default: 0.9
        last_epoch (int): The index of the last batch. This parameter is used when
            resuming a training job. Since `step()` should be invoked after each
            batch instead of after each epoch, this number represents the total
            number of *batches* computed, not the total number of epochs computed.
            When last_epoch=-1, the schedule is started from the beginning.
            Default: -1

    Example:
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1)
        >>> data_loader = torch.utils.data.DataLoader(...)
        >>> for epoch in range(10):
        >>>     for batch in data_loader:
        >>>         train_batch(...)
        >>>         scheduler.step()


    .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186
    .. _bckenstler/CLR: https://github.com/bckenstler/CLR
    éÐ  NÚ
triangularç      ð?ÚcycleTçš™™™™™é?çÍÌÌÌÌÌì?r   c                sÈ  t |tƒstdjt|ƒjƒƒ‚|| _| jd||ƒ}|dkr\x t||j	ƒD ]\}}||d< qHW | jd||ƒ| _
t|ƒ}|d k	r„t|ƒn|}|| | _|| j | _|dkr¶|d kr¶td	ƒ‚|| _|| _|d k�r| jdkræ| j| _d
| _n6| jdk�r| j| _d
| _n| jdk�r*| j| _d| _n|| _|	| _|
| _|
�r¬d|jk�rJtdƒ‚| jd||ƒ}|dk�r†x"t||j	ƒD ]\}}||d< �qpW ttdd„ |j	ƒƒ| _| jd||ƒ| _tt| ƒj||ƒ || _d S )Nz{} is not an OptimizerrT   r   r	   Úmax_lrr‹   Útriangular2Ú	exp_rangez$mode is invalid and scale_fn is Noner�   Z
iterationsÚmomentumzDoptimizer must support momentum with `cycle_momentum` option enabledÚbase_momentumc             S   s   | d S )Nr“   r
   )r   r
   r
   r   r   u  s    z#CyclicLR.__init__.<locals>.<lambda>Úmax_momentumr!   )r‹   r‘   r’   r!   ) r"   r   r#   r$   r%   r&   r'   Ú_format_paramrC   r(   Úmax_lrsr   Ú
total_sizeÚ
step_ratiorL   ro   r[   Ú_triangular_scale_fnÚscale_fnÚ
scale_modeÚ_triangular2_scale_fnÚ_exp_range_scale_fnÚcycle_momentumÚdefaultsr,   r-   Úbase_momentumsÚmax_momentumsrM   r‰   r3   r.   )r1   r'   rT   r�   Ústep_size_upÚstep_size_downro   r[   r›   rœ   rŸ   r”   r•   r/   r.   r	   r   r¡   r“   )r   r
   r   r3   7  sT    




zCyclicLR.__init__c             C   sT   t |ttfƒr@t|ƒt|jƒkr<tdjt|jƒ|t|ƒƒƒ‚|S |gt|jƒ S dS )z<Return correctly formatted lr/momentum for each param group.z!expected {} values for {}, got {}N)r"   r,   rI   rJ   r(   rL   r$   )r1   Únamer'   Úparamr
   r
   r   r–   {  s    zCyclicLR._format_paramc             C   s   dS )Ng      ð?r
   )r1   Úxr
   r
   r   rš   …  s    zCyclicLR._triangular_scale_fnc             C   s   dd|d   S )Nr   g       @r
   )r1   r§   r
   r
   r   r�   ˆ  s    zCyclicLR._triangular2_scale_fnc             C   s
   | j | S )N)r[   )r1   r§   r
   r
   r   rž   ‹  s    zCyclicLR._exp_range_scale_fnc             C   sV  t jd| j| j  ƒ}d| j| j  | }|| jkr@|| j }n|d | jd  }g }xbt| j| jƒD ]P\}}|| | }| jdkr˜||| j	|ƒ  }n||| j	| jƒ  }|j
|ƒ qfW | j�rRg }	xdt| j| jƒD ]R\}
}||
 | }| jdk�r
||| j	|ƒ  }n||| j	| jƒ  }|	j
|ƒ qÖW x$t| jj|	ƒD ]\}}||d< �q<W |S )zúCalculates the learning rate at batch index. This function treats
        `self.last_epoch` as the last batch index.

        If `self.cycle_momentum` is ``True``, this function has a side effect of
        updating the optimizer's momentum.
        r   g      ð?r�   r“   )re   Úfloorr/   r˜   r™   rC   r.   r—   rœ   r›   ÚappendrŸ   r¡   r¢   r'   r(   )r1   r�   r§   Zscale_factorÚlrsrT   r�   Zbase_heightr	   Z	momentumsr”   r•   r“   rE   r
   r
   r   r>   Ž  s.    

zCyclicLR.get_lrr!   )
rŠ   Nr‹   rŒ   Nr�   TrŽ   r�   r!   )r&   rF   rG   rV   r3   r–   rš   r�   rž   r>   rW   r
   r
   )r   r   r‰   Ù  s    \         7
r‰   c                   s4   e Zd ZdZd‡ fdd„	Zdd„ Zdd	d
„Z‡  ZS )ÚCosineAnnealingWarmRestartsaW  Set the learning rate of each parameter group using a cosine annealing
    schedule, where :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}`
    is the number of epochs since the last restart and :math:`T_{i}` is the number
    of epochs between two warm restarts in SGDR:

    .. math::
        \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})(1 +
        \cos(\frac{T_{cur}}{T_{i}}\pi))

    When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`.
    When :math:`T_{cur}=0`(after restart), set :math:`\eta_t=\eta_{max}`.

    It has been proposed in
    `SGDR: Stochastic Gradient Descent with Warm Restarts`_.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        T_0 (int): Number of iterations for the first restart.
        T_mult (int, optional): A factor increases :math:`T_{i}` after a restart. Default: 1.
        eta_min (float, optional): Minimum learning rate. Default: 0.
        last_epoch (int, optional): The index of last epoch. Default: -1.

    .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:
        https://arxiv.org/abs/1608.03983
    r   r   c                sz   |dkst |tƒ r"tdj|ƒƒ‚|dk s6t |tƒ rDtdj|ƒƒ‚|| _|| _|| _|| _tt	| ƒj
||ƒ | j| _d S )Nr   z)Expected positive integer T_0, but got {}r   z(Expected integer T_mult >= 1, but got {})r"   ÚintrL   r$   ÚT_0ÚT_iÚT_multrc   rM   r«   r3   r/   ÚT_cur)r1   r'   r­   r¯   rc   r/   )r   r
   r   r3   Ï  s    z$CosineAnnealingWarmRestarts.__init__c                s   ‡ fdd„ˆ j D ƒS )Nc                s<   g | ]4}ˆ j |ˆ j  d tjtjˆ j ˆ j ƒ  d  ‘qS )r   rd   )rc   re   rf   rg   r°   r®   )r4   rT   )r1   r
   r   rU   Ü  s   z6CosineAnnealingWarmRestarts.get_lr.<locals>.<listcomp>)r.   )r1   r
   )r1   r   r>   Û  s    
z"CosineAnnealingWarmRestarts.get_lrNc             C   s&  |dkrH| j d }| jd | _| j| jkrì| j| j | _| j| j | _n¤|dk r^tdj|ƒƒ‚|| jkrÞ| jdkr€|| j | _qìttj	|| j | jd  d | jƒƒ}|| j| j| d  | jd   | _| j| j|  | _n| j| _|| _tj
|ƒ| _ x(t| jj| jƒ ƒD ]\}}||d< �qW dS )aï  Step could be called after every batch update

        Example:
            >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
            >>> iters = len(dataloader)
            >>> for epoch in range(20):
            >>>     for i, sample in enumerate(dataloader):
            >>>         inputs, labels = sample['inputs'], sample['labels']
            >>>         scheduler.step(epoch + i / iters)
            >>>         optimizer.zero_grad()
            >>>         outputs = net(inputs)
            >>>         loss = criterion(outputs, labels)
            >>>         loss.backward()
            >>>         optimizer.step()

        This function can be called in an interleaved way.

        Example:
            >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
            >>> for epoch in range(20):
            >>>     scheduler.step()
            >>> scheduler.step(26)
            >>> scheduler.step() # scheduler.step(27), instead of scheduler(20)
        Nr   r   z'Expected non-negative epoch, but got {}r	   )r/   r°   r®   r¯   rL   r$   r­   r¬   re   Úlogr¨   rC   r'   r(   r>   )r1   rD   ÚnrE   r	   r
   r
   r   r0   ß  s&    


&$z CosineAnnealingWarmRestarts.stepr!   )r   r   r!   )N)r&   rF   rG   rV   r3   r>   r0   rW   r
   r
   )r   r   r«   ´  s   r«   c                   sB   e Zd ZdZd‡ fdd„	Zdd„ Zdd„ Zdd„ Zdd„ Z‡  Z	S )Ú
OneCycleLRa8  Sets the learning rate of each parameter group according to the
    1cycle learning rate policy. The 1cycle policy anneals the learning
    rate from an initial learning rate to some maximum learning rate and then
    from that maximum learning rate to some minimum learning rate much lower
    than the initial learning rate.
    This policy was initially described in the paper `Super-Convergence:
    Very Fast Training of Neural Networks Using Large Learning Rates`_.

    The 1cycle learning rate policy changes the learning rate after every batch.
    `step` should be called after a batch has been used for training.

    This scheduler is not chainable.

    This class has two built-in annealing strategies:
    "cos":
        Cosine annealing
    "linear":
        Linear annealing

    Note also that the total number of steps in the cycle can be determined in one
    of two ways (listed in order of precedence):
    1) A value for total_steps is explicitly provided.
    2) A number of epochs (epochs) and a number of steps per epoch
       (steps_per_epoch) are provided.
       In this case, the number of total steps is inferred by
       total_steps = epochs * steps_per_epoch
    You must either provide a value for total_steps or provide a value for both
    epochs and steps_per_epoch.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        max_lr (float or list): Upper learning rate boundaries in the cycle
            for each parameter group.
        total_steps (int): The total number of steps in the cycle. Note that
            if a value is provided here, then it must be inferred by providing
            a value for epochs and steps_per_epoch.
            Default: None
        epochs (int): The number of epochs to train for. This is used along
            with steps_per_epoch in order to infer the total number of steps in the cycle
            if a value for total_steps is not provided.
            Default: None
        steps_per_epoch (int): The number of steps per epoch to train for. This is
            used along with epochs in order to infer the total number of steps in the
            cycle if a value for total_steps is not provided.
            Default: None
        pct_start (float): The percentage of the cycle (in number of steps) spent
            increasing the learning rate.
            Default: 0.3
        anneal_strategy (str): {'cos', 'linear'}
            Specifies the annealing strategy.
            Default: 'cos'
        cycle_momentum (bool): If ``True``, momentum is cycled inversely
            to learning rate between 'base_momentum' and 'max_momentum'.
            Default: True
        base_momentum (float or list): Lower momentum boundaries in the cycle
            for each parameter group. Note that momentum is cycled inversely
            to learning rate; at the peak of a cycle, momentum is
            'base_momentum' and learning rate is 'max_lr'.
            Default: 0.85
        max_momentum (float or list): Upper momentum boundaries in the cycle
            for each parameter group. Functionally,
            it defines the cycle amplitude (max_momentum - base_momentum).
            Note that momentum is cycled inversely
            to learning rate; at the start of a cycle, momentum is 'max_momentum'
            and learning rate is 'base_lr'
            Default: 0.95
        div_factor (float): Determines the initial learning rate via
            initial_lr = max_lr/div_factor
            Default: 25
        final_div_factor (float): Determines the minimum learning rate via
            min_lr = initial_lr/final_div_factor
            Default: 1e4
        last_epoch (int): The index of the last batch. This parameter is used when
            resuming a training job. Since `step()` should be invoked after each
            batch instead of after each epoch, this number represents the total
            number of *batches* computed, not the total number of epochs computed.
            When last_epoch=-1, the schedule is started from the beginning.
            Default: -1

    Example:
        >>> data_loader = torch.utils.data.DataLoader(...)
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01, steps_per_epoch=len(data_loader), epochs=10)
        >>> for epoch in range(10):
        >>>     for batch in data_loader:
        >>>         train_batch(...)
        >>>         scheduler.step()


    .. _Super-Convergence\: Very Fast Training of Neural Networks Using Large Learning Rates:
        https://arxiv.org/abs/1708.07120
    Nç333333Ó?rf   Tç333333ë?çffffffî?ç      9@ç     ˆÃ@r   c                s˜  t |tƒstdjt|ƒjƒƒ‚|| _|d krF|d krF|d krFtdƒ‚n€|d k	rx|dksbt |tƒ rptdj|ƒƒ‚|| _	nN|dksŒt |tƒ rštdj|ƒƒ‚|dks®t |tƒ r¼tdj|ƒƒ‚|| | _	t
|| j	 ƒd | _t
| j	| j ƒd | _|dk �s|dk�st |t
ƒ �r tdj|ƒƒ‚|dk�r:tdj|ƒƒ‚n&|d	k�rN| j| _n|d
k�r`| j| _| jd| j|ƒ}|dk�rÂxFt| jjƒD ]6\}}|| | |d< || |d< |d | |d< �qˆW || _| j�r‚d| jjk�rôd| jjk�rôtdƒ‚d| jjk| _| jd||
ƒ}| jd||	ƒ}|dk�r‚xXt|||jƒD ]F\}}}| j�rd|d \}}||f|d< n||d< ||d< ||d< �q8W tt| ƒj||ƒ d S )Nz{} is not an OptimizerzBYou must define either total_steps OR (epochs AND steps_per_epoch)r   z5Expected non-negative integer total_steps, but got {}z0Expected non-negative integer epochs, but got {}z9Expected non-negative integer steps_per_epoch, but got {}r   z4Expected float between 0 and 1 pct_start, but got {}rf   Úlinearz@anneal_strategy must by one of 'cos' or 'linear', instead got {}r�   r	   r~   r“   ÚbetaszDoptimizer must support momentum with `cycle_momentum` option enabledr•   r”   )rf   r¹   r!   r!   )r"   r   r#   r$   r%   r&   r'   rL   r¬   Útotal_stepsr   r£   r¤   Ú_annealing_cosÚanneal_funcÚ_annealing_linearr–   r*   r(   rŸ   r    Ú	use_beta1rC   rM   r³   r3   )r1   r'   r�   r»   ZepochsZsteps_per_epochZ	pct_startZanneal_strategyrŸ   r”   r•   Z
div_factorZfinal_div_factorr/   r—   rQ   r   r¢   r¡   Z
m_momentumZ
b_momentumÚ_Úbeta2)r   r
   r   r3   l  s^    


"





zOneCycleLR.__init__c             C   sT   t |ttfƒr@t|ƒt|jƒkr<tdjt|jƒ|t|ƒƒƒ‚|S |gt|jƒ S dS )z<Return correctly formatted lr/momentum for each param group.z!expected {} values for {}, got {}N)r"   r,   rI   rJ   r(   rL   r$   )r1   r¥   r'   r¦   r
   r
   r   r–   ¹  s    zOneCycleLR._format_paramc             C   s(   t jt j| ƒd }||| d |  S )z@Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.r   g       @)re   rf   rg   )r1   ÚstartÚendÚpctZcos_outr
   r
   r   r¼   Ã  s    zOneCycleLR._annealing_cosc             C   s   || | | S )zBLinearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.r
   )r1   rÂ   rÃ   rÄ   r
   r
   r   r¾   È  s    zOneCycleLR._annealing_linearc       	      C   s
  g }| j }|| jkr*tdj|d | jƒƒ‚xÚ| jjD ]Î}|| jkr‚| j|d |d || j ƒ}| jrÊ| j|d |d || j ƒ}nH|| j }| j|d |d || j	 ƒ}| jrÊ| j|d |d || j	 ƒ}|j
|ƒ | jr4| jrú|d \}}||f|d< q4||d	< q4W |S )
NzATried to step {} times. The specified number of total steps is {}r   r   r�   r•   r”   r~   rº   r“   )r/   r»   rL   r$   r'   r(   r£   r½   rŸ   r¤   r©   r¿   )	r1   rª   Zstep_numr   Zcomputed_lrZcomputed_momentumZdown_step_numrÀ   rÁ   r
   r
   r   r>   Ì  s.    



zOneCycleLR.get_lrr!   )NNNr´   rf   Trµ   r¶   r·   r¸   r!   )
r&   rF   rG   rV   r3   r–   r¼   r¾   r>   rW   r
   r
   )r   r   r³     s    \          @
r³   )rN   re   Z
torch._sixr   Ú	functoolsr   r@   r   Úbisectr   r'   r   Úobjectr   rH   rX   r\   r`   ra   rh   r‰   r«   r³   r
   r
   r
   r   Ú<module>   s&   bF#%$ ' \[