3
I4W^|A  ã               @   sH   d dl Z d dlmZ ddlmZ ddd„Zddd„ZG dd„ deƒZdS )é    N)Úreduceé   )Ú	Optimizerc             C   sä   |d k	r|\}}n| |kr"| |fn|| f\}}|| d||  | |   }	|	d ||  }
|
dkrÔ|
j ƒ }| |krœ|||  || |	 || d|     }n(| | | || |	 || d|     }tt||ƒ|ƒS || d S d S )Né   é   r   g       @)ÚsqrtÚminÚmax)Úx1Úf1Zg1Zx2Úf2Zg2ÚboundsZ
xmin_boundZ
xmax_boundZd1Z	d2_squareZd2Zmin_pos© r   ú2/tmp/pip-build-e66j97b2/torch/torch/optim/lbfgs.pyÚ_cubic_interpolate   s    
	*(r   ç-Cëâ6?çÍÌÌÌÌÌì?ç•Ö&è.>é   c       !   	   C   s   |j ƒ jƒ }|jƒ }| |||ƒ\}}d}|j|ƒ}d|||f\}}}}d}d}�x$||
k �rt|||| |  ks€|dkr¦||kr¦||g}||g}||jƒ g}||g}P t |ƒ| | krÐ|g}|g}|g}d}P |dkrþ||g}||g}||jƒ g}||g}P |d||   }|d }|}t||||||||fd�}|}|}|jƒ }|}| |||ƒ\}}|d7 }|j|ƒ}|d7 }qRW ||
k�r˜d|g}||g}||g}d}|d |d	 k�r²d
nd\}}�x| �rÚ||
k �rÚt|d |d |d |d |d |d ƒ}dt|ƒt|ƒ  } tt|ƒ| |t|ƒ ƒ| k �r˜|�sP|t|ƒk�sP|t|ƒk�r’t |t|ƒ ƒt |t|ƒ ƒk �r€t|ƒ|  }nt|ƒ|  }d}nd}nd}| |||ƒ\}}|d7 }|j|ƒ}|d7 }|||| |  k�sê||| k�r.|||< |||< |jƒ ||< |||< |d |d k�r$dnd\}}nˆt |ƒ| | k�rHd}nJ||| ||   dk�r’|| ||< || ||< || ||< || ||< |||< |||< |jƒ ||< |||< t |d |d  ƒ| |	k �r¾P �q¾W || }|| }|| }||||fS )Nr   r   FTg{®Gáz„?é
   )r   gš™™™™™¹?éÿÿÿÿ)r   r   )r   r   )r   r   )r   r   )Úabsr	   ÚcloneÚdotr   r   )!Úobj_funcÚxÚtÚdÚfÚgÚgtdZc1Úc2Útolerance_changeZmax_lsZd_normZf_newZg_newÚls_func_evalsZgtd_newZt_prevZf_prevZg_prevZgtd_prevÚdoneZls_iterZbracketZ	bracket_fZ	bracket_gZbracket_gtdZmin_stepZmax_stepÚtmpZinsuf_progressZlow_posZhigh_posZepsr   r   r   Ú_strong_wolfe"   s¶    
$

	 ""
$ r&   c                   sZ   e Zd ZdZd‡ fdd	„	Zd
d„ Zdd„ Zdd„ Zdd„ Zdd„ Z	dd„ Z
dd„ Z‡  ZS )ÚLBFGSa  Implements L-BFGS algorithm, heavily inspired by `minFunc
    <https://www.cs.ubc.ca/~schmidtm/Software/minFunc.html>`.

    .. warning::
        This optimizer doesn't support per-parameter options and parameter
        groups (there can be only one).

    .. warning::
        Right now all parameters have to be on a single device. This will be
        improved in the future.

    .. note::
        This is a very memory intensive optimizer (it requires additional
        ``param_bytes * (history_size + 1)`` bytes). If it doesn't fit in memory
        try reducing the history size, or use a different algorithm.

    Arguments:
        lr (float): learning rate (default: 1)
        max_iter (int): maximal number of iterations per optimization step
            (default: 20)
        max_eval (int): maximal number of function evaluations per optimization
            step (default: max_iter * 1.25).
        tolerance_grad (float): termination tolerance on first order optimality
            (default: 1e-5).
        tolerance_change (float): termination tolerance on function
            value/parameter changes (default: 1e-9).
        history_size (int): update history size (default: 100).
        line_search_fn (str): either 'strong_wolfe' or None (default: None).
    r   é   NçH¯¼šò×z>ç•Ö&è.>éd   c	       
   	      sl   |d kr|d d }t |||||||d�}	tt| ƒj||	ƒ t| jƒdkrRtdƒ‚| jd d | _d | _d S )Né   é   )ÚlrÚmax_iterÚmax_evalÚtolerance_gradr"   Úhistory_sizeÚline_search_fnr   z>LBFGS doesn't support per-parameter options (parameter groups)r   Úparams)	ÚdictÚsuperr'   Ú__init__ÚlenÚparam_groupsÚ
ValueErrorÚ_paramsÚ_numel_cache)
Úselfr4   r.   r/   r0   r1   r"   r2   r3   Údefaults)Ú	__class__r   r   r7   Ö   s    	zLBFGS.__init__c             C   s$   | j d krtdd„ | jdƒ| _ | j S )Nc             S   s   | |j ƒ  S )N)Únumel)ÚtotalÚpr   r   r   Ú<lambda>ô   s    zLBFGS._numel.<locals>.<lambda>r   )r<   r   r;   )r=   r   r   r   Ú_numelò   s    
zLBFGS._numelc             C   sn   g }x\| j D ]R}|jd kr.|j|jƒ ƒjƒ }n&|jjrH|jjƒ jdƒ}n|jjdƒ}|j|ƒ qW t	j
|dƒS )Nr   r   r   r   )r;   ZgradÚnewr@   Zzero_Z	is_sparseZto_denseÚviewÚappendÚtorchÚcat)r=   ZviewsrB   rF   r   r   r   Ú_gather_flat_grad÷   s    
zLBFGS._gather_flat_gradc             C   sZ   d}x@| j D ]6}|jƒ }|jj||||| … j|jƒƒ ||7 }qW || jƒ ksVt‚d S )Nr   )r;   r@   ÚdataÚadd_Zview_asrD   ÚAssertionError)r=   Z	step_sizeÚupdateÚoffsetrB   r@   r   r   r   Ú	_add_grad  s    "zLBFGS._add_gradc             C   s   dd„ | j D ƒS )Nc             S   s   g | ]}|j ƒ ‘qS r   )r   )Ú.0rB   r   r   r   ú
<listcomp>  s    z&LBFGS._clone_param.<locals>.<listcomp>)r;   )r=   r   r   r   Ú_clone_param  s    zLBFGS._clone_paramc             C   s*   x$t | j|ƒD ]\}}|jj|ƒ qW d S )N)Úzipr;   rK   Úcopy_)r=   Zparams_datarB   Zpdatar   r   r   Ú
_set_param  s    zLBFGS._set_paramc             C   s0   | j ||ƒ t|ƒ ƒ}| jƒ }| j|ƒ ||fS )N)rP   ÚfloatrJ   rV   )r=   Úclosurer   r   r   ÚlossÚ	flat_gradr   r   r   Ú_directional_evaluate  s
    

zLBFGS._directional_evaluatec       &         sz  t ˆjƒdkst‚ˆjd }|d }|d }|d }|d }|d }|d }|d	 }	ˆjˆjd  }
|
jd
dƒ |
jddƒ ˆ ƒ }t|ƒ}d}|
d
  d7  < ˆjƒ }|jƒ j	ƒ |k}|r¾|S |
j
dƒ}|
j
dƒ}|
j
dƒ}|
j
dƒ}|
j
dƒ}|
j
dƒ}|
j
dƒ}|
j
dƒ}d}�x ||k �r4|d7 }|
d  d7  < |
d dk�rb|jƒ }g }g }g }d}�nR|j|ƒ}|j|ƒ}|j|ƒ}|dk�ræt |ƒ|	k�r¶|jdƒ |jdƒ |jdƒ |j|ƒ |j|ƒ |jd| ƒ ||j|ƒ }t |ƒ}d|
k�rdg|	 |
d< |
d }|jƒ }xJt|d ddƒD ]6}|| j|ƒ||  ||< |j||  || ƒ �q(W tj||ƒ }} x@t|ƒD ]4}|| j| ƒ||  }!| j|| |! || ƒ �q|W |dk�rÈ|jƒ }n
|j|ƒ |}|
d dk�r tdd|jƒ jƒ  ƒ| }n|}|j|ƒ}"|"| k�rP d}#|dk	�rŽ|dk�r>tdƒ‚n2ˆjƒ }$‡ ‡fdd„}%t|%|$|||||"ƒ\}}}}#ˆj||ƒ |jƒ j	ƒ |k}n<ˆj||ƒ ||k�rÊtˆ ƒ ƒ}ˆjƒ }|jƒ j	ƒ |k}d}#||#7 }|
d
  |#7  < ||k�rîP ||k�rúP |�rP |j|ƒjƒ j	ƒ |k�rP t|| ƒ|k �rP �qW ||
d< ||
d< ||
d< ||
d< ||
d< ||
d< ||
d< ||
d< |S )z¬Performs a single optimization step.

        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        r   r   r.   r/   r0   r1   r"   r3   r2   Z
func_evalsÚn_iterr   r   Úold_dirsÚold_stpsÚroÚH_diagÚprev_flat_gradÚ	prev_lossg»½×Ùß|Û=g      ð?ÚalNZstrong_wolfez only 'strong_wolfe' is supportedc                s   ˆj ˆ | ||ƒS )N)r[   )r   r   r   )rX   r=   r   r   r   ¢  s    zLBFGS.step.<locals>.obj_funcr   r   )r8   r9   rM   Ústater;   Ú
setdefaultrW   rJ   r   r	   ÚgetÚnegÚsubÚmulr   ÚpoprG   ÚrangerL   rH   r   rU   r   ÚsumÚRuntimeErrorrS   r&   rP   )&r=   rX   Úgroupr.   r/   r0   r1   r"   r3   r2   rd   Z	orig_lossrY   Zcurrent_evalsrZ   Zopt_condr   r   r]   r^   r_   r`   ra   rb   r\   ÚyÚsZysZnum_oldrc   ÚqÚiÚrZbe_ir    r#   Zx_initr   r   )rX   r=   r   Ústep  sÚ    





























z
LBFGS.step)r   r(   Nr)   r*   r+   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r7   rD   rJ   rP   rS   rV   r[   rt   Ú__classcell__r   r   )r?   r   r'   ·   s         	r'   )N)r   r   r   r   )rH   Ú	functoolsr   Z	optimizerr   r   r&   r'   r   r   r   r   Ú<module>   s   
#   
 