3
I4W^1Í  ã               @   s>  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZ d dlm	Z	 d dl
Z
d dlZd dlZd dlmZmZmZmZ d d	lmZ d d
lmZ dZedddgƒZeeƒZdddgZe
jeƒZG dd„ deƒZ G dd„ de ƒZ!G dd„ de!ƒZ"G dd„ de!ƒZ#G dd„ de ƒZ$G dd„ de ƒZ%G dd„ de ƒZ&G d d!„ d!e ƒZ'G d"d#„ d#e ƒZ(G d$d%„ d%e ƒZ)G d&d'„ d'e ƒZ*G d(d)„ d)e ƒZ+G d*d+„ d+e ƒZ,d,d-„ Z-dUd.d/„Z.d0d1„ Z/d2d3„ Z0d4d5„ Z1dVd8d9„Z2d:d;„ Z3dWd<d=„Z4dXd>d?„Z5d@dA„ Z6dYdCdD„Z7dZdFdG„Z8d[dHdI„Z9d\dJdK„Z:d]dLdM„Z;d^dNdO„Z<d_dQdR„Z=d`dSdT„Z>dS )aé    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)Ú
namedtupleÚdefaultdict)Ú
basestringN)ÚcoreÚscopeÚutilsÚ	workspace)Úparameter_info)Ú
caffe2_pb2Úlr_injectionÚAuxOptimizerParamsÚlocalÚsharedZSIMD_Q_FP16ZSIMD_Q_STOC_FP16ZSIMD_Q_STOC_MKL_FP16c               @   s”   e Zd Zdd„ Zd$dd„Zdd„ Zd%d	d
„Zdd„ Zedd„ ƒZ	dd„ Z
d&dd„Zdd„ Zd'dd„Zdd„ Zedd„ ƒZdd„ Zd d!„ Zd"d#„ ZdS )(Ú	Optimizerc             C   sF   t g g d�| _t| jj | _t| jj  d7  < d | _d | _d| _d S )N)r   r   é   F)	r   Ú_aux_paramsÚ_optimizer_instance_countÚ	__class__Ú__name__Ú_instance_numÚ_lr_multiplierÚ_local_lr_multiplierÚ_local_lr_multiplier_on_gpu)Úself© r   ú8/tmp/pip-build-e66j97b2/torch/caffe2/python/optimizer.pyÚ__init__    s    zOptimizer.__init__Nc             C   sh   |d kr2t |tjƒs"tdj|ƒƒ‚|jd k	sVt‚n$t |tƒrFtj|ƒ}tjd ||d�}| j	|||ƒ d S )Nz6Expected parameter to be of type ParameterInfo, got {})Zparam_idÚparamÚgrad)
Ú
isinstancer   ZParameterInfoÚAssertionErrorÚformatr"   r   r	   ZBlobReferenceÚ_run)r   ÚnetÚparam_init_netr!   r"   r   r   r   Ú__call__0   s    

zOptimizer.__call__c             C   s   t dƒ‚d S )NzNot Implemented)Ú	Exception)r   r'   r(   Ú
param_infor   r   r   r&   ?   s    zOptimizer._runÚ c             C   s   | j j}d|| j||f S )Nz%s_%d_%s%s_cpu)r   r   r   )r   Úbase_strÚ	node_nameÚ	classnamer   r   r   Úget_cpu_blob_nameB   s    zOptimizer.get_cpu_blob_namec             C   s   | j j}d|| j|||f S )Nz%s_%d_%s%s_gpu%d)r   r   r   )r   r-   Zgpu_idr.   r/   r   r   r   Úget_gpu_blob_nameF   s    zOptimizer.get_gpu_blob_namec             C   s   t j| jƒ}|d= |S )Nr   )ÚcopyÚdeepcopyÚ__dict__)r   Úattrr   r   r   Ú
attributesL   s    zOptimizer.attributesc             C   sJ   t jƒ }|dkr| j|ƒS tj|jƒr8| j||j|jƒS | j||jƒS dS )zo
        Returns a blob name that will be unique to the current device
        and optimizer instance.
        N)	r
   ÚCurrentDeviceScoper0   r	   ÚIsGPUDeviceTypeÚdevice_typer1   Ú	device_idr.   )r   r-   Úcurrent_scoper   r   r   Úmake_unique_blob_nameS   s    
zOptimizer.make_unique_blob_nameÚfixedr   c             K   sú   |d kr| j dƒ}tj|||d�}|j|ƒsL|j|g|f| |dœ|—Ž}	n
|j|ƒ}	| jd k	rŽ|j| j| j dƒƒ}
|j|	|
g| j dƒdd�}	| j	d k	ròt
jƒ }|d k	rÒtj|jƒrÒ| j rÒ|j| j	| j dƒƒ}n| j	}|j|	|g| j d	ƒdd�}	|	|fS )
NÚlr)Úiter_val)Zbase_lrÚpolicyÚlr_multiplierZ	scaled_lrr   )Ú	broadcastÚlocal_lr_multiplierZlocal_scaled_lr)r<   r   ÚBuildUniqueMutexIterÚBlobIsDefinedZLearningRateZ
GetBlobRefr   ZCopyFromCPUInputÚMulr   r
   r7   r	   r8   r9   r   )r   r'   r(   Úbase_learning_rateZlearning_rate_blobr@   r?   ÚkwargsÚ	iterationr>   rA   r;   rC   r   r   r   Úbuild_lrc   sF    





zOptimizer.build_lrc             C   s
   || _ dS )zé
        Set the global learning rate multiplier. If a multiplier already
        existed, this will overwrite the existing multiplier. The multiplier is
        used for all future calls to _run(), unless it is overwritten.
        N)r   )r   rA   r   r   r   Úadd_lr_multiplier›   s    zOptimizer.add_lr_multiplierFc             C   s   || _ || _dS )a¦  
        Set the local learning rate multiplier. This local multiplier is
        multiplied with the global learning rate multiplier if it exists. As
        with the global learning rate multiplier, this multiplier will be
        used for all future calls to _run(), so please call
        _clear_local_lr_multiplier() at the beginning of the optimizer's _run()
        before optionally calling this function.
        N)r   r   )r   rC   Úis_gpu_blobr   r   r   Ú_add_local_lr_multiplier£   s    	z"Optimizer._add_local_lr_multiplierc             C   s   d | _ d| _d S )NF)r   r   )r   r   r   r   Ú_clear_local_lr_multiplier¯   s    z$Optimizer._clear_local_lr_multiplierc             C   s4   t |tjƒstdj|ƒƒ‚|r,| j||d�S |S d S )Nz,Dedup only works for sparse gradient, got {})Z
aggregator)r#   r	   ÚGradientSlicer$   r%   ZDeduplicateGradientSlices)r'   Úsparse_dedup_aggregatorr"   r   r   r   Údedup³   s    
zOptimizer.dedupc             C   s   | j S )ax  Returns a list of auxiliary parameters.

        Returns:
            aux_params: A namedtuple, AuxParams.

            aux_params.local stores a list of blobs. Each blob is a local
            auxiliary parameter. A local auxiliary parameter is a parameter in
            parallel to a learning rate parameter. Take adagrad as an example,
            the local auxiliary parameter is the squared sum parameter, because
            every learning rate has a squared sum associated with it.

            aux_params.shared also stores a list of blobs. Each blob is a shared
            auxiliary parameter. A shared auxiliary parameter is a parameter
            that is shared across all the learning rate parameters. Take adam as
            an example, the iteration parameter is a shared parameter, because
            all the learning rates share the same iteration parameter.
        )r   )r   r   r   r   Úget_auxiliary_parameters½   s    z"Optimizer.get_auxiliary_parametersc             O   s   t dƒ‚d S )Nz9Optimizer Need to Implement `scale_learning_rate` method.)ÚNotImplementedError)r   ÚargsrH   r   r   r   Úscale_learning_rateÙ   s    zOptimizer.scale_learning_ratec             C   sF   |j g ddg|d�}|j g ddg|d�}|j g ddg|d�}|||fS )NÚweight_decayr   )ÚshapeÚvalueÚtrustÚlr_max)ÚConstantFill)r   r(   rV   rY   rZ   Úwdr   r   r   Úcreate_lars_inputsÝ   s    zOptimizer.create_lars_inputs)N)r,   )Nr=   r   )F)r   Ú
__module__Ú__qualname__r    r)   r&   r0   r1   Úpropertyr6   r<   rJ   rK   rM   rN   ÚstaticmethodrQ   rR   rU   r]   r   r   r   r   r      s    

 
6

r   c                   s.   e Zd Zd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )ÚSgdOptimizerç{®Gáz„?r=   ç        r   Nc                s<   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	d S )N)
Úsuperrb   r    rG   r@   ÚmomentumÚnesterovrP   ÚlarsÚinit_kwargs)r   rG   r@   rf   rg   rP   rh   rH   )r   r   r   r    ç   s    zSgdOptimizer.__init__c             C   sZ  |j }|j}| jdkrd S | jdks4tdj| jƒƒ‚| jƒ  | jd k	ràt|tj	ƒ rà| jdksntdj| jƒƒ‚| j
|ddtjtjƒjƒ\}}}|j|||||g| jt|ƒd ƒ| jdd�}	tjƒ }
| j|	|
d k	oÚtj|
jƒd� | jrêdnd	}| j||f| j| | jd
œ| j—Ž\}}tjƒ }|d k�r4tjtjƒ}|jg dj|j|j|j ƒd	gdd�}| j!j"j#|ƒ | jdk�r˜|j|t|ƒd dd�}| j!j$j#|ƒ t|tj	ƒ�r| j%|| j&|ƒ}| jdk�rð|j'|j(||||j)g|j(||g| j| j*d� n|j+|||j)|j(|g|ƒ nJ| jdk�r>|j,||||g|||g| j| j*d� n|}|j-||||g|ƒ d S )Nr   z*Expect positive base learning rate, got {}z'Lars offset must be nonnegative, got {}g        g      ð?Ú_lars)ÚoffsetÚlr_min)rL   r   )rG   r@   zONE_{}_{}{})rW   rX   Ú	_momentum)rX   )rf   rg   éÿÿÿÿ).Úblobr"   rG   r$   r%   rN   rh   r#   r	   rO   r]   ÚnpÚfinfoÚfloat32ÚmaxÚLarsr<   Ústrr
   r7   rM   r8   r9   rf   rJ   r@   ri   ÚDeviceOptionr   ÚCPUr[   r:   r.   r   r   Úappendr   rQ   rP   ZSparseMomentumSGDUpdateÚvaluesÚindicesrg   ZScatterWeightedSumÚMomentumSGDUpdateÚWeightedSum)r   r'   r(   r+   r!   r"   r\   rY   rZ   Úlr_lars_multiplierr;   Zlr_signr>   Ú_ÚdevÚONEÚmomentum_dataZcoeffr   r   r   r&   ó   s~    





zSgdOptimizer._runc             C   s   |  j |9  _ d S )N)rG   )r   Úscaler   r   r   rU   J  s    z SgdOptimizer.scale_learning_rate)rc   r=   rd   r   NN)r   r^   r_   r    r&   rU   Ú__classcell__r   r   )r   r   rb   æ   s
     
Wrb   c                   s&   e Zd Zd
‡ fdd„	Zdd	„ Z‡  ZS )ÚMultiPrecisionSgdOptimizerçš™™™™™¹?ç        r=   r   Nc                s&   t t| ƒjf |||||dœ|—Ž d S )N)rG   r@   rf   rg   rP   )re   r„   r    )r   rG   rf   r@   rg   rP   rH   )r   r   r   r    P  s    z#MultiPrecisionSgdOptimizer.__init__c             C   s  |j }|jd k	r|jtjj nd }|d kr:tj| |||ƒS |j}| jdkrNd S | jdksht	dj
| jƒƒ‚| j||f| j | jdœ| j—Ž\}}|j|t|ƒd dd�}	| jjj|	ƒ t|tjƒ sÊt	dƒ‚|j||d ƒ}
|j|
|	||g|
|	|g| j| jd	� |j||ƒ d S )
Nr   z*Expect positive base learning rate, got {})rG   r@   rm   g        )rX   z3MultiPrecisionSgd does not support sparse gradientsZ_fp32)rf   rg   )ro   Ú	blob_copyr	   ÚDataTypeÚFLOATrb   r&   r"   rG   r$   r%   rJ   r@   ri   r[   ru   r   r   rx   r#   rO   ZHalfToFloatr{   rf   rg   ÚFloatToHalf)r   r'   r(   r+   r!   Ú
param_fp32r"   r>   r~   r�   Z	grad_fp32r   r   r   r&   \  s6    



zMultiPrecisionSgdOptimizer._run)r…   r†   r=   r   N)r   r^   r_   r    r&   rƒ   r   r   )r   r   r„   O  s    r„   c                   s(   e Zd Zd‡ fdd„	Zdd
d„Z‡  ZS )ÚFP16SgdOptimizerçš™™™™™¹?ç        r=   r   ç-Cëâ6?Nc                s,   t t| ƒjf |||||dœ|—Ž || _d S )N)rG   r@   rf   rg   rP   )re   rŒ   r    rV   )r   rG   rf   r@   rg   rV   rP   rH   )r   r   r   r    ‰  s    zFP16SgdOptimizer.__init__Fc             C   s´  d}t |jƒ}|jdƒdkr d}|r6d}|j}|j}nx|jd krRd}|j}|j}n\tjj|jkrv|j}|jtjj }n8tjj|jkrš|jtjj }|j}nds®tdj	|j
ƒƒ‚|j}	| jdkrÂd S | jdksÜtdj	| jƒƒ‚| j||f| j | jdœ| j—Ž\}
}|j|t |ƒd	 d
d�}|j|t |ƒd ƒ}| jjj|ƒ t|	tjƒ �sTtdƒ‚|dk�rˆ|j|	||
|g|	||g| j| j| jd� n(|j|	||
|g|	||g| j| j| jd� d S )Nr   Zspatbnr   TFzLUnrecognized parameter format to be updated by FP16 Optimizer. Parameter: {}z*Expect positive base learning rate, got {})rG   r@   Z_momentum_fp32g        )rX   rm   z)FP16Sgd does not support sparse gradients)rf   rg   rV   rn   )ru   ro   Úfindr‡   r	   rˆ   r‰   ZFLOAT16r$   r%   Únamer"   rG   rJ   r@   ri   r[   rŠ   r   r   rx   r#   rO   ZFP16MomentumSGDUpdaterf   rg   rV   ZFP32MomentumSGDUpdate)r   r'   r(   r+   Zfp32_updateZfp32_update_flagÚ
param_namer!   r‹   r"   r>   r~   Zmomentum_data_fp32r�   r   r   r   r&   —  sh    







zFP16SgdOptimizer._run)r�   rŽ   r=   r   r�   N)F)r   r^   r_   r    r&   rƒ   r   r   )r   r   rŒ   ˆ  s     rŒ   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚWeightDecayBuilderc             C   s
   || _ d S )N)rV   )r   rV   r   r   r   r    é  s    zWeightDecayBuilder.__init__c             C   s”   t jƒ }|d krtjtjƒ}|jg dj|j|j	ƒdgdd�}|jg dj|j|j	ƒdg| j
d�}t|jtjƒrvtdƒ‚n|j|j||j|g|jƒ d S )Nz	ONE_{}_{}r   g      ð?)rW   rX   zwd_{}_{}z2Weight decay does not yet support sparse gradients)r
   r7   r	   rv   r   rw   r[   r%   r9   r:   rV   r#   r"   rO   Ú
ValueErrorr|   ro   )r   r'   r(   r+   r   r€   ZWDr   r   r   r&   ì  s"    zWeightDecayBuilder._runN)r   r^   r_   r    r&   r   r   r   r   r“   è  s   r“   c            
       s.   e Zd Zd‡ fdd	„	Zd
d„ Zdd„ Z‡  ZS )ÚAdagradOptimizerç{®Gáz„?ç-Cëâ6?r   r=   NFr,   c                sT   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _d S )N)re   r•   r    ÚalphaÚepsilonÚdecayr@   rP   ÚrowWiseÚenginerh   Úoutput_effective_lrÚoutput_effective_lr_and_updateri   )r   r˜   r™   rš   r@   rP   r›   rœ   rh   r�   rž   rH   )r   r   r   r      s    zAdagradOptimizer.__init__c             C   s:  |j }|j}| jdkrd S | jƒ  | jd k	rÆt|tjƒ rÆ| jdksTtdj	| jƒƒ‚| j
|ddtjtjƒjƒ\}}}|j|||||g| jt|ƒd ƒ| jdd�}	tjƒ }
| j|	|
d k	oÀtj|
jƒd� | j||f| j| jdœ| j—Ž\}}| j�rœtjd	j	| jƒƒ tj|gƒ\}}t|ƒ|k�rr|j |t|ƒd
 ƒ}|j!|gt|ƒd dgdgd�}|j"|t|ƒd ddd�}n(|j"g t|ƒd |t|ƒ d gdd�}n†tjdj	| jƒƒ | jt#k�rtj|gƒ\}}t|ƒ|k�sàt|ƒ‚|t|ƒ }|j$g t|ƒd d|d�}n|j"|gt|ƒd dd�}| j%j&j'|ƒ | j�rNt|tjƒ�sNtdƒ‚t|tjƒ�rÀ| j(dk�sptdƒ‚| j)|| j*|ƒ}| j�rŽd}nd}|j+|ƒ|||j,|j-|g||g| j.| jd� nv||g}| j/�rö|j't|ƒd ƒ |j't|ƒd ƒ n| j0�r|j't|ƒd ƒ |j1||||g|| j.t2| j(ƒ| jd� d S )Nr   z'Lars offset must be nonnegative, got {}g        g      ð?rj   )rk   rl   )rL   )rG   r@   z#Using engine {} for rowWise AdagradZ_shapeZ_numrowsr   )ZstartsZendsZ_avg_squared_sum)Zinput_as_shaperX   )rW   rX   z#Using engine {} for regular AdagradZ_squared_sum)rX   rW   )rX   z¨If SparseAdagrad with rowWise=True, gradient must be a gradientslice. PLease ensure that rowWise is not enabled for the dense Adagrad optimizer, as it is not supported.z?Decay is not implemented for SparseAdagrad and must be set to 1ZRowWiseSparseAdagradZSparseAdagrad)r™   rœ   Ú_effective_lrÚ_update)r™   rš   rœ   )3ro   r"   r˜   rN   rh   r#   r	   rO   r$   r%   r]   rp   rq   rr   rs   rt   r<   ru   r
   r7   rM   r8   r9   rJ   r@   ri   r›   ÚloggerÚinforœ   r   ÚInferShapesAndTypesZShapeÚSlicer[   ÚFP16_ENGINESZFloat16ConstantFillr   r   rx   rš   rQ   rP   Ú__getattr__rz   ry   r™   rž   r�   ZAdagradÚfloat)r   r'   r(   r+   r!   r"   r\   rY   rZ   r}   r;   r>   r~   ÚshapesÚtypesrW   Znum_rowsZparam_squared_sumÚopÚoutput_argsr   r   r   r&     s¬    









zAdagradOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   Ž  s    z$AdagradOptimizer.scale_learning_rate)
r–   r—   r   r=   NFr,   NFF)r   r^   r_   r    r&   rU   rƒ   r   r   )r   r   r•     s      vr•   c            	       s.   e Zd Zd‡ fdd	„	Zd
d„ Zdd„ Z‡  ZS )ÚWngradOptimizerç      ð?ç•Ö&è.>r=   Nr,   ç      Y@Fc
                sN   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _d S )N)re   r¬   r    r˜   r™   r@   rP   rœ   Úmoment_initrh   r�   rž   ri   )r   r˜   r™   r@   rP   rœ   r°   rh   r�   rž   rH   )r   r   r   r    ”  s    zWngradOptimizer.__init__c             C   sÐ  |j }|j}| jdkrd S | jƒ  | jd k	rÆt|tjƒ rÆ| jdksTtdj	| jƒƒ‚| j
|ddtjtjƒjƒ\}}}|j|||||g| jt|ƒd ƒ| jdd�}	tjƒ }
| j|	|
d k	oÀtj|
jƒd� | j||f| j| jdœ| j—Ž\}}|jg t|ƒd	 d
g| jd�}| jjj|ƒ t|tjƒ�r^| j|| j |ƒ}|j!|||j"|j#|g||g| j$| j%d� nn||g}| j&�r”|jt|ƒd ƒ |jt|ƒd ƒ n| j'�r®|jt|ƒd ƒ |j(||||g|| j$| j%d� d S )Nr   z'Lars offset must be nonnegative, got {}g        g      ð?rj   )rk   rl   )rL   )rG   r@   Ú_momentr   )rW   rX   )r™   rœ   rŸ   r    ))ro   r"   r˜   rN   rh   r#   r	   rO   r$   r%   r]   rp   rq   rr   rs   rt   r<   ru   r
   r7   rM   r8   r9   rJ   r@   ri   r[   r°   r   r   rx   rQ   rP   ZSparseWngradrz   ry   r™   rœ   rž   r�   ZWngrad)r   r'   r(   r+   r!   r"   r\   rY   rZ   r}   r;   r>   r~   Úmomentr«   r   r   r   r&   ¤  sb    



zWngradOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   å  s    z#WngradOptimizer.scale_learning_rate)	r­   r®   r=   Nr,   r¯   NFF)r   r^   r_   r    r&   rU   rƒ   r   r   )r   r   r¬   “  s      Ar¬   c                   s.   e Zd Zd‡ fdd„	Zd	d
„ Zdd„ Z‡  ZS )ÚAdadeltaOptimizerç{®Gáz„?ç-Cëâ6?çffffffî?r=   Nr,   c                s<   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	dS )au  Constructor function to add Adadelta Optimizer

        Args:
            alpha: learning rate
            epsilon: attribute of Adadelta to avoid numerical issues
            decay: attribute of Adadelta to decay the squared gradient sum
            policy: specifies how learning rate should be applied, options are
              "fixed", "step", "exp", etc.
            sparse_dedup_aggregator: specifies deduplication strategy for
              gradient slices. Works while using sparse gradients. Options
              include "mean" and "sum".
            engine: the engine used, options include "", "CUDNN", etc.
        N)
re   r³   r    r˜   r™   rš   r@   rP   rœ   ri   )r   r˜   r™   rš   r@   rP   rœ   rH   )r   r   r   r    ë  s    zAdadeltaOptimizer.__init__c       
      C   s
  |j }|j}| jdkrd S | j||f| j| jdœ| j—Ž\}}|j|gt|ƒd dd�}|j|gt|ƒd dd�}	| jj	j
|ƒ | jj	j
|	ƒ t|tjƒrÜ| j|| j|ƒ}|j|||	|j|j|g|||	g| j| j| jd� n*|j|||	||g|||	g| j| j| jd� d S )Nr   )rG   r@   Z_squared_momentg        )rX   Z_squared_moment_update)r™   rš   rœ   )ro   r"   r˜   rJ   r@   ri   r[   ru   r   r   rx   r#   r	   rO   rQ   rP   ZSparseAdadeltarz   ry   r™   rš   rœ   ZAdadelta)
r   r'   r(   r+   r!   r"   r>   r~   r²   Zmoment_updater   r   r   r&     s<    

zAdadeltaOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   -  s    z%AdadeltaOptimizer.scale_learning_rate)r´   rµ   r¶   r=   Nr,   )r   r^   r_   r    r&   rU   rƒ   r   r   )r   r   r³   ê  s    *r³   c                   s.   e Zd Zd‡ fdd„	Zdd	„ Zd
d„ Z‡  ZS )ÚFtrlOptimizerç{®Gáz„?ç-Cëâ6?r   Nr,   c                s6   t t| ƒjƒ  || _|| _|| _|| _|| _|| _d S )N)	re   r·   r    r˜   ÚbetaÚlambda1Úlambda2rP   rœ   )r   r˜   rº   r»   r¼   rP   rœ   )r   r   r   r    3  s    zFtrlOptimizer.__init__c          	   C   sÆ   |j }|j}| jdkrd S |j|gt|ƒd dgdd�}| jjj|ƒ t|t	j
ƒr–| j|| j|ƒ}|j|||j|jg||g| j| j| j| j| jd� n,|j|||g||g| j| j| j| j| jd� d S )Nr   Z_ftrl_nzé   g        )Úextra_shaperX   )rœ   r˜   rº   r»   r¼   )ro   r"   r˜   r[   ru   r   r   rx   r#   r	   rO   rQ   rP   Z
SparseFtrlrz   ry   rœ   rº   r»   r¼   ZFtrl)r   r'   r(   r+   r!   r"   Únzr   r   r   r&   =  s8    

zFtrlOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   a  s    z!FtrlOptimizer.scale_learning_rate)r¸   r¹   r   r   Nr,   )r   r^   r_   r    r&   rU   rƒ   r   r   )r   r   r·   2  s    	$r·   c                   s2   e Zd ZdZd‡ fdd„	Zd	d
„ Zdd„ Z‡  ZS )ÚGFtrlOptimizerzGroup Lasso FTRL Optimizer.ç{®Gáz„?ç-Cëâ6?r   Nr,   c                s6   t t| ƒjƒ  || _|| _|| _|| _|| _|| _d S )N)	re   rÀ   r    r˜   rº   r»   r¼   rP   rœ   )r   r˜   rº   r»   r¼   rP   rœ   )r   r   r   r    i  s    zGFtrlOptimizer.__init__c          	   C   sv   |j }|j}| jdkrd S |j|gt|ƒd dgdd�}| jjj|ƒ |j|||g||g| j	| j| j
| j| jd� d S )Nr   Z	_gftrl_nzr½   g        )r¾   rX   )rœ   r˜   rº   r»   r¼   )ro   r"   r˜   r[   ru   r   r   rx   ZGFtrlrœ   rº   r»   r¼   )r   r'   r(   r+   r!   r"   r¿   r   r   r   r&   s  s$    

zGFtrlOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   ‹  s    z"GFtrlOptimizer.scale_learning_rate)rÁ   rÂ   r   r   Nr,   )r   r^   r_   Ú__doc__r    r&   rU   rƒ   r   r   )r   r   rÀ   f  s
    	rÀ   c                   s.   e Zd Zd‡ fdd„	Zdd„ Zdd„ Z‡  ZS )ÚAdamOptimizerçü©ñÒMbP?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>r=   Fç{®Gáz„?TNr,   c                sZ   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _|| _d S )N)re   rÄ   r    r˜   Úbeta1Úbeta2r™   r@   Úuse_lr_adaptionÚlr_alphaÚnormalized_lr_adaptionrP   r›   rœ   ri   )r   r˜   rÊ   rË   r™   r@   rÌ   rÍ   rÎ   rP   r›   rœ   rH   )r   r   r   r    ‘  s    zAdamOptimizer.__init__c             C   sè  |j }|j}| jdkrd S | j||f| j| jdœ| j—Ž\}}|j|g|d dd�}| jrŒtj	|gƒ\}	}
|jg |d |	| d gdd�}n|j|g|d dd�}| j
jj|ƒ | j
jj|ƒ | j
jj|ƒ | jræt|tjƒsætd	ƒ‚|||g}| j�rt|ƒd
 }|j|ƒ t|tjƒ�r˜| j|| j|ƒ}| j�r:d}nd}|j|ƒ||||j|j||g|| j| j| jd� | j�rä|j||j|g|g| j| jd� nL|j||||||g|| j| j| jd� | j�rä|j|||g|g| j| jd� d S )Nr   )rG   r@   Z_first_momentg        )rX   Z_avg_second_moment)rW   rX   Z_second_momentz¢If SparseAdam with rowWise=True, gradient must be a gradientslice. PLease ensure that rowWise is not enabled for the dense Adam optimizer, as it is not supported.Z_effective_gradZRowWiseSparseAdamZ
SparseAdam)rÊ   rË   r™   )rÍ   rÎ   ) ro   r"   r˜   rJ   r@   ri   r[   r›   r   r£   r   r   rx   r   r#   r	   rO   r$   rÌ   ru   rQ   rP   r¦   rz   ry   rÊ   rË   r™   ZLearningRateAdaptionrÍ   rÎ   ZAdam)r   r'   r(   r+   r!   r"   r>   rI   Úm1r¨   r©   Úm2Zoutput_blobsZeffective_gradrª   r   r   r   r&   £  s~    






zAdamOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   ÷  s    z!AdamOptimizer.scale_learning_rate)rÅ   rÆ   rÇ   rÈ   r=   FrÉ   TNFr,   )r   r^   r_   r    r&   rU   rƒ   r   r   )r   r   rÄ   �  s      TrÄ   c                   s2   e Zd ZdZd‡ fd
d„	Zdd„ Zdd„ Z‡  ZS )ÚYellowFinOptimizerz¾YellowFin: An automatic tuner for momentum SGD

    See https://arxiv.org/abs/1706.03471 for more details. This implementation
    has separate learning rate and momentum per each parameter.çš™™™™™¹?ç        ç+‡ÙÎ÷ï?é   Té   r=   Nc	       
         sH   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	|| _
|	| _d S )N)re   rÑ   r    r˜   Úmurº   Úcurv_win_widthÚzero_debiasr™   r@   rP   ri   )
r   r˜   r×   rº   rØ   rÙ   r™   r@   rP   rH   )r   r   r   r      s    
zYellowFinOptimizer.__init__c             C   sš  d}|j }|j}|j|g|d dd�}|jg |d | jgdd�}|j|g|d dd�}	|j|g|d dd�}
|jg |d	 d
g| jd�}|jg |d d
g| jd�}|jg |d |gdd�}| jdksÆt‚t|tj	ƒ sÜtdƒ‚t
j||dd�}| jjj|ƒ | jjj|ƒ | jjj|ƒ | jjj|ƒ | jjj|ƒ | jjj|	ƒ | jjj|
ƒ | jjj|ƒ ||||||	|
|g}|j|||g || j| j| j| jd� d S )Né   r±   g        )rX   Z	_curv_win)rW   rX   Z_g_avgZ_g2_avgZ_lr_avgr   Z_mu_avgZ_scalars_memoryr   z+YellowFin does not support sparse gradients)r?   )rº   r™   rØ   rÙ   )ro   r"   r[   rØ   r˜   r×   r$   r#   r	   rO   r   rD   r   r   rx   r   Z	YellowFinrº   r™   rÙ   )r   r'   r(   r+   ZSCALARS_MEMORY_SIZEr!   r"   r²   Zcurv_winZg_avgZg2_avgZlr_avgZmu_avgZscalars_memoryrI   Zyf_in_out_argsr   r   r   r&     s‚    


zYellowFinOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   n  s    z&YellowFinOptimizer.scale_learning_rateç�íµ ÷Æ°>)rÒ   rÓ   rÔ   rÕ   TrÛ   r=   N)r   r^   r_   rÃ   r    r&   rU   rƒ   r   r   )r   r   rÑ   ü  s          WrÑ   c                   s.   e Zd Zd‡ fdd„	Zd	d
„ Zdd„ Z‡  ZS )ÚRmsPropOptimizerç{®Gáz„?çÍÌÌÌÌÌì?ç        çñhãˆµøä>r=   r,   c                s<   t t| ƒjƒ  || _|| _|| _|| _|| _|| _|| _	d S )N)
re   rÜ   r    r˜   rš   rf   r™   r@   rœ   ri   )r   r˜   rš   rf   r™   r@   rœ   rH   )r   r   r   r    t  s    
zRmsPropOptimizer.__init__c             C   sF  |j }|j}| jdkst‚t|tjƒ s0tdƒ‚tjƒ }|d krLtj	t
jƒ}|jg dj|j|jƒdgdd�}| j||f| j | jdœ| j—Ž\}}	|j|gt|ƒd d	d
�}
|j|gt|ƒd d	d
�}|j|gt|ƒd d	d
�}| jjj|ƒ | jjj|ƒ |j||||g|
||g| j| j| j| jd� |j|
|||g|
||gƒ d S )Nr   z1RmsPropOptimizer doesn't support sparse gradientsz	ONE_{}_{}r   g      ð?)rW   rX   )rG   r@   Z_grad_og        )ry   Z_mean_squaresrm   )rš   rf   r™   rœ   )ro   r"   r˜   r$   r#   r	   rO   r
   r7   rv   r   rw   r[   r%   r9   r:   rJ   r@   ri   ru   r   r   rx   ZRmsProprš   rf   r™   rœ   r{   )r   r'   r(   r+   r!   r"   r   r€   r>   r~   Zgrad_oÚmsZmomr   r   r   r&   ‡  sV    





zRmsPropOptimizer._runc             C   s   |  j |9  _ d S )N)r˜   )r   r‚   r   r   r   rU   Å  s    z$RmsPropOptimizer.scale_learning_rate)rÝ   rÞ   rß   rà   r=   r,   )r   r^   r_   r    r&   rU   rƒ   r   r   )r   r   rÜ   s  s        >rÜ   c             C   s"   t j| jƒ}|jt j| jƒƒ |S )N)r	   ZInferBlobDevicesr'   Úupdater(   )ÚmodelÚparam_to_devicer   r   r   Ú_get_param_to_deviceÊ  s    rå   c             C   sœ   |}|p
i }| |kr||  }ndt |tjƒrj|}t|jƒ|krL|t|jƒ }q‚t|jƒ|kr‚|t|jƒ }nt|ƒ}||kr‚|| }|d k	s˜tdj| ƒƒ‚|S )Nz,Cannot infer device for {}: no op creates it)r#   r	   rO   ru   ry   rz   r$   r%   )r’   r"   rä   Zdefault_deviceÚdeviceZ	grad_namer   r   r   Úget_param_deviceÒ  s     

rç   c               C   s
   t jtƒS )zÈ
    Gets current value for lr_injection, a multiplier for all base
    learning rates.
    Must set allow_lr_injection=True when building optimizer, as it
    relies on synchronization over CPU.
    )r   Z	FetchBlobÚ_LEARNING_RATE_INJECTIONr   r   r   r   Úget_lr_injectionë  s    ré   c             C   s"   t jttjt| ƒgtjd�ƒ dS )z²
    Sets lr_injection, a multiplier for all base learning rates.
    Must set allow_lr_injection=True when building optimizer, as it
    relies on synchronization over CPU.
    )ZdtypeN)r   ZFeedBlobrè   rp   Úarrayr§   rr   )Zlr_injection_valuer   r   r   Úset_lr_injectionõ  s
    rë   c             C   s$  t j|ƒ�� g }xˆt|ƒD ]|\}}tt|jƒ|j|ƒ}t j|ƒ�P t|jt j	ƒsX|jn|jj
}	dj|ƒ}
| jj|	|
ƒ}| jj|ƒ}|j|ƒ W d Q R X qW t jt jtjƒƒ�b | jj|dƒ}| jj|ddd�}| jjg dg t|ƒd�}| jj||gdƒ}| jj||gd	ƒ}|S Q R X W d Q R X d S )
Nzgrad_{}_squared_sumÚgrad_squared_full_sumÚglobal_normg      à?)ÚexponentÚ	clip_norm)rW   rX   Úmax_normÚ
norm_ratio)r	   Z	NameScopeÚ	enumeraterç   ru   ro   r"   ÚDeviceScoper#   rO   ry   r%   r'   ZSumSqrElementsZEnsureCPUOutputrx   rv   r   rw   ZSumÚPowr(   r[   r§   ZMaxÚDiv)rã   ÚparamsZ
name_scoperä   Úmax_gradient_normZgrad_squared_sumsÚir!   ræ   r"   Zgrad_squared_sum_nameZgrad_squared_sumZgrad_squared_sum_cpurì   rí   rï   rð   rñ   r   r   r   Ú_calc_norm_ratio  sJ    
rù   FTc             C   s(  t | ƒ}| jƒ  g }x,| jƒ D ] }|r4|j| jkr4q|j|ƒ qW d }	|d k	r^t| |d||ƒ}	|r®| jjt	ƒs†| j
jg t	dgdd�}
nt	}
|	d kr˜|
}	n| jj|	|
gddd�}	|j|	ƒ xj|D ]b}t|jƒ}t||j|ƒ}tj|ƒ�6 |joî|�r|j| j| j
|ƒ n|| j| j
|ƒ W d Q R X q¾W |S )NZnorm_clipped_grad_updater   g      ð?)rW   rX   rA   )rB   )rå   ZValidateZGetOptimizationParamInforo   Úweightsrx   rù   r'   rE   rè   r(   r[   rF   rK   ru   rç   r"   r	   ró   Ú	optimizer)rã   rû   Úweights_onlyÚuse_param_info_optimr÷   Úallow_lr_injectionrä   rö   r+   rA   r   r’   ræ   r   r   r   Ú_build<  sN    



rÿ   c             C   s   t | t|d�ddd� dS )z”Adds a decay to weights in the model.

    This is a form of L2 regularization.

    Args:
        weight_decay: strength of the regularization
    )rV   TF)rü   rý   N)rÿ   r“   )rã   rV   r   r   r   Úadd_weight_decayz  s
    r   c             K   s   t |f|Ž}t| |||d�S )N)r÷   rþ   )rb   rÿ   )rã   rG   r÷   rþ   rH   Zsgd_optimizerr   r   r   Ú	build_sgdŠ  s    r  c             K   s   t |f|Ž}t| |||d�S )N)r÷   rþ   )r„   rÿ   )rã   rG   r÷   rþ   rH   Zmulti_prec_sgd_optimizerr   r   r   Úbuild_multi_precision_sgdš  s    
r  c             K   s   t |f|Ž}t| |ƒS )N)rŒ   rÿ   )rã   rG   rH   Zfp16_sgd_optimizerr   r   r   Úbuild_fp16_sgd¬  s    
r  ÚSIMDc             K   s@   |dkr$t jdƒst‚t jdƒs$t‚tf d|i|—Ž}t| |ƒS )Nr  ZFtrl_ENGINE_SIMDZSparseFtrl_ENGINE_SIMDrœ   )r	   Ú
IsOperatorr$   r·   rÿ   )rã   rœ   rH   Zftrl_optimizerr   r   r   Ú
build_ftrl³  s
    r  r,   c             K   s2   |dkrt jdƒst‚tf d|i|—Ž}t| |ƒS )Nr  ZGFtrl_ENGINE_SIMDrœ   )r	   r  r$   rÀ   rÿ   )rã   rœ   rH   Zgftrl_optimizerr   r   r   Úbuild_gftrl»  s    r  c             K   s"   t f d|i|—Ž}t| |||d�S )Nr˜   )r÷   rþ   )r•   rÿ   )rã   rG   Ú
parametersr÷   rþ   rH   Zadagrad_optimizerr   r   r   Úbuild_adagradÂ  s    r	  c             K   s"   t f d|i|—Ž}t| |||d�S )Nr˜   )r÷   rþ   )r¬   rÿ   )rã   rG   r  r÷   rþ   rH   Zwngrad_optimizerr   r   r   Úbuild_wngradÓ  s    r
  c             K   s"   t f d|i|—Ž}t| |||d�S )Nr˜   )r÷   rþ   )r³   rÿ   )rã   rG   r  r÷   rþ   rH   Zadadelta_optimizerr   r   r   Úbuild_adadeltaä  s    r  c             K   s"   t f d|i|—Ž}t| |||d�S )Nr˜   )r÷   rþ   )rÄ   rÿ   )rã   rG   r÷   rþ   rH   Zadam_optimizerr   r   r   Ú
build_adamõ  s    r  çš™™™™™¹?c             K   s   t f d|i|—Ž}t| |ƒS )Nr˜   )rÑ   rÿ   )rã   rG   rH   Zyellowfin_optimizerr   r   r   Úbuild_yellowfin  s    r  c             K   s"   t f d|i|—Ž}t| |||d�S )Nr˜   )r÷   rþ   )rÜ   rÿ   )rã   rG   r÷   rþ   rH   Zrms_prop_optimizerr   r   r   Úbuild_rms_prop  s    r  )NN)FTNF)NF)NF)r  )r,   )NNF)NNF)NNF)NF)r  )NF)?Ú
__future__r   r   r   r   Úcollectionsr   r   Zpast.builtinsr   Úloggingr2   Znumpyrp   Zcaffe2.pythonr	   r
   r   r   Zcaffe2.python.modelingr   Zcaffe2.protor   rè   r   Úintr   r¥   Ú	getLoggerr   r¡   Úobjectr   rb   r„   rŒ   r“   r•   r¬   r³   r·   rÀ   rÄ   rÑ   rÜ   rå   rç   ré   rë   rù   rÿ   r   r  r  r  r  r  r	  r
  r  r  r  r  r   r   r   r   Ú<module>   s|   

 Hi9` WH4*lwW

;   
8 
 



  
  
  
 


 