3
I4W^åf  ã               @   s   d dl mZ d dl mZ d dl mZ d dlmZ d dljjZd dlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ d dlmZ d dlmZmZ d d	lmZmZ d d
lmZ d dlZ d dl!m"Z"m#Z# d dl$Z$d dl%Z%G dd„ deeƒZ&G dd„ deeƒZ'G dd„ deeeƒZ(G dd„ deeeƒZ)G dd„ deeƒZ*G dd„ deeƒZ+G dd„ deeeƒZ,G dd„ deeƒZ-G dd„ deeeƒZ.G dd„ deeeƒZ/G d d!„ d!eeeƒZ0G d"d#„ d#eeƒZ1G d$d%„ d%eeeƒZ2G d&d'„ d'eƒZ3G d(d)„ d)eƒZ4G d*d+„ d+eƒZ5dS ),é    )Úabsolute_import)Údivision)Úprint_function)Ú
caffe2_pb2N)Ú	build_sgdÚbuild_multi_precision_sgdÚ
build_ftrlÚbuild_gftrlÚbuild_wngradÚbuild_adagradÚbuild_adadeltaÚ
build_adamÚbuild_yellowfinÚbuild_rms_propÚadd_weight_decayÚSgdOptimizer)ÚUseOptimizer)ÚOptimizerTestBaseÚLRModificationTestBase)ÚcoreÚ	workspace)ÚTestCase)Úassert_allcloseÚassert_equalc               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚTestLarsc             C   s   t jdƒ‚d S )Nzno sparse support)ÚunittestÚSkipTest)Úself© r   ú=/tmp/pip-build-e66j97b2/torch/caffe2/python/optimizer_test.pyÚ
testSparse   s    zTestLars.testSparsec             K   s   d| _ t|fdddœ|—ŽS )NFgš™™™™™¹?g      à?)Úbase_learning_rateÚlars)Ú	_skip_gpur   )r   ÚmodelÚkwargsr   r   r   Úbuild_optimizer   s    zTestLars.build_optimizerc             C   s\   | j |jƒ jƒ | j|jƒ jƒ x6|jƒ jD ](}tj|ƒ}tjj	tj
dgƒ|dd� q,W d S )Ng      ð?gñhãˆµøä>)Úatol)Ú
assertTrueÚget_auxiliary_parametersÚsharedÚassertFalseÚlocalr   Ú	FetchBlobÚnpÚtestingr   Úarray)r   Ú	optimizerÚparamÚtensorr   r   r   Úcheck_optimizer   s
    
zTestLars.check_optimizerN)Ú__name__Ú
__module__Ú__qualname__r    r&   r4   r   r   r   r   r      s   r   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestMomentumSgdc             K   s   d| _ t|fdddœ|—ŽS )NFgš™™™™™¹?)r!   Zmomentum)r#   r   )r   r$   r%   r   r   r   r&   '   s    zTestMomentumSgd.build_optimizerc             C   s\   | j |jƒ jƒ | j |jƒ jƒ x6|jƒ jD ](}tj|ƒ}tjjtj	dgƒ|dd� q,W d S )Ng      ð?gñhãˆµøä>)r'   )
r(   r)   r*   r,   r   r-   r.   r/   r   r0   )r   r1   r2   r3   r   r   r   r4   +   s
    
zTestMomentumSgd.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   r8   &   s   r8   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestSgdc             K   s   d| _ t|fddi|—ŽS )NFr!   gš™™™™™¹?)r#   r   )r   r$   r%   r   r   r   r&   4   s    zTestSgd.build_optimizerc             C   s\   | j |jƒ jƒ | j|jƒ jƒ x6|jƒ jD ](}tj|ƒ}tjj	tj
dgƒ|dd� q,W d S )Ng      ð?gñhãˆµøä>)r'   )r(   r)   r*   r+   r,   r   r-   r.   r/   r   r0   )r   r1   r2   r3   r   r   r   r4   8   s
    
zTestSgd.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   r9   3   s   r9   c                   s<   e Zd Zdd„ Zdd„ Zejej dƒ‡ fdd„ƒZ	‡  Z
S )ÚTestMultiPrecisionSgdc             K   s   d| _ t|fddi|—ŽS )NFr!   gš™™™™™¹?)r#   r   )r   r$   r%   r   r   r   r&   C   s    z%TestMultiPrecisionSgd.build_optimizerc             C   s\   | j |jƒ jƒ | j|jƒ jƒ x6|jƒ jD ](}tj|ƒ}tjj	tj
dgƒ|dd� q,W d S )Ng      ð?gñhãˆµøä>)r'   )r(   r)   r*   r+   r,   r   r-   r.   r/   r   r0   )r   r1   r2   r3   r   r   r   r4   I   s
    
z%TestMultiPrecisionSgd.check_optimizerzNo GPU supportc                s   t t| ƒjtjjƒ d S )N)Úsuperr:   ÚtestGPUDenser   ÚDataTypeZFLOAT16)r   )Ú	__class__r   r   r<   P   s    z"TestMultiPrecisionSgd.testGPUDense)r5   r6   r7   r&   r4   r   ÚskipIfr   Úhas_gpu_supportr<   Ú__classcell__r   r   )r>   r   r:   @   s   r:   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestFtrlc             K   s"   d| _ t|fd dddddœ|—ŽS )NTg      ð?gš™™™™™¹?g        )ÚengineÚalphaÚbetaÚlambda1Úlambda2)r#   r   )r   r$   r%   r   r   r   r&   V   s    zTestFtrl.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   b   s    zTestFtrl.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   rB   U   s   rB   c               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )Ú	TestGFtrlc             C   s   t jdƒ‚d S )Nzno sparse support)r   r   )r   r   r   r   r    j   s    zTestGFtrl.testSparsec             K   s"   d| _ t|fd dddddœ|—ŽS )NTg      ð?gš™™™™™¹?g        )rC   rD   rE   rF   rG   )r#   r	   )r   r$   r%   r   r   r   r&   m   s    zTestGFtrl.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   y   s    zTestGFtrl.check_optimizerN)r5   r6   r7   r    r&   r4   r   r   r   r   rH   i   s   rH   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestAdagradc             K   s   d| _ t|fdddœ|—ŽS )NFg      ð?g      à?)r!   r"   )r#   r   )r   r$   r%   r   r   r   r&   �   s    zTestAdagrad.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   …   s    zTestAdagrad.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   rI   €   s   rI   c               @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
ÚTestRowWiseAdagradc             K   s   d| _ t|fddddœ|—ŽS )NTg      ð?g      à?)r!   r"   ZrowWise)r#   r   )r   r$   r%   r   r   r   r&   �   s    z"TestRowWiseAdagrad.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   “   s    z"TestRowWiseAdagrad.check_optimizerc             C   s   t jdƒ‚d S )Nzno dense support)r   r   )r   r   r   r   Ú	testDense™   s    zTestRowWiseAdagrad.testDensec             C   s   t jdƒ‚d S )Nzno dense support)r   r   )r   r   r   r   r<   œ   s    zTestRowWiseAdagrad.testGPUDenseN)r5   r6   r7   r&   r4   rK   r<   r   r   r   r   rJ   Œ   s   rJ   c               @   s   e Zd Zdd„ Zdd„ ZdS )Ú
TestWngradc             K   s   d| _ t|fddi|—ŽS )NTr!   g      9@)r#   r
   )r   r$   r%   r   r   r   r&   ¡   s    zTestWngrad.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   ¥   s    zTestWngrad.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   rL       s   rL   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestAdadeltac             K   s   d| _ t|fdddœ|—ŽS )NFg      ð?g×£p=
×ï?)r!   Zdecay)r#   r   )r   r$   r%   r   r   r   r&   ­   s    zTestAdadelta.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   ±   s    zTestAdadelta.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   rM   ¬   s   rM   c               @   s   e Zd Zdd„ Zdd„ ZdS )ÚTestAdamc             K   s   d| _ t|fddi|—ŽS )NFr!   gš™™™™™¹?)r#   r   )r   r$   r%   r   r   r   r&   ¹   s    zTestAdam.build_optimizerc             C   s”   | j |jƒ jƒ | j |jƒ jƒ | j tjdƒƒ tjdƒ}tjj	tj
dgƒ|dd� x|jƒ jD ]}tj|ƒ q`W x|jƒ jD ]}tj|ƒ q~W d S )NÚoptimizer_iterationiÐ  gñhãˆµøä>)r'   )r(   r)   r*   r,   r   ÚHasBlobr-   r.   r/   r   r0   )r   r1   Úiteration_tensorr2   r   r   r   r4   ½   s    
zTestAdam.check_optimizerN)r5   r6   r7   r&   r4   r   r   r   r   rN   ¸   s   rN   c               @   s€   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
ejdƒdd„ ƒZejdƒejej dƒdd„ ƒƒZdS )ÚTestYellowFinc             C   s   d| _ t|dd�S )NFgš™™™™™¹?)r!   )r#   r   )r   r$   r   r   r   r&   Î   s    zTestYellowFin.build_optimizerc             C   s”   | j |jƒ jƒ | j |jƒ jƒ | j tjdƒƒ tjdƒ}tjj	tj
dgƒ|dd� x|jƒ jD ]}tj|ƒ q`W x|jƒ jD ]}tj|ƒ q~W d S )NrO   iÐ  gñhãˆµøä>)r'   )r(   r)   r*   r,   r   rP   r-   r.   r/   r   r0   )r   r1   rQ   r2   r   r   r   r4   Ò   s    
zTestYellowFin.check_optimizerc             C   s   t jdƒ‚d S )Nzno sparse support)r   r   )r   r   r   r   r    ß   s    zTestYellowFin.testSparsec             C   s   |r|d||   S |S d S )Ng      ð?r   )r   ÚvalrE   ÚiÚzero_debiasr   r   r   Údebâ   s    zTestYellowFin.debc             C   s"  |dkrN|| }t j|ƒd t j|ƒd  d }dt j|ƒ d | }||fS |d |d  d | }tj|| d| | |  ƒ | d }	|	dkršdndtjtj|	ƒdƒ }
|
|d	 |
  }|d }t|dƒ}|| }tt j|ƒd t j|ƒd  d |d ƒ}dt j|ƒ d | }||fS )Nr   é   é   g      @g      ;@g       @g        g      ð?g      @g�íµ ÷Æ°>gh/¡½„öÂ?g      ð¿gUUUUUUÕ?gé!çýÿï?)r.   ÚsqrtÚmathÚpowÚfabsÚminÚmax)r   ZdistanceZgrad_varÚh_minÚh_maxZdrÚmuZlr_minÚpZw3ÚwÚyÚrootr   r   r   Ú	get_lr_muè   s     ("
*zTestYellowFin.get_lr_muc              C   s<  i }d}d}d}	d}
d}t jdƒ}t jdƒ}tjƒ  t jt jtjƒƒ�@ |jg dd	gd
t j	j
d�}|jg dgƒ}|j||g|gƒ W d Q R X |jg d|g|d�}|r²|j|gdƒ}|j|gdƒ}|j||gddd�}|jg d|gdd�}dd„ }||_||_tj|||	|
||d�j|||ƒ tj|ƒ tj|dd� �x t|ƒD ]ô}tj|ƒ tjdƒ}|d	 }|d }|d }|d }tjdƒ}tjdƒd
 }tjdƒd
 }| j||	|d	 |ƒ}t| j||	|d	 |ƒ|j|ƒ |ƒ}|d
k�r@tj| j||	|d	 |ƒƒtj| j||	|d	 |ƒƒ|| j||	|d	 |ƒ||d œ||< �q@W |S )!Ng      ð?g        g+‡ÙÎ÷ï?é   g�íµ ÷Æ°>ÚnetÚparam_init_netÚ	iterationrW   r   )ÚshapeÚvalueZdtypeZiteration_mutexÚpre_grad)rk   rl   Ziteration_cpuÚiteration_floatÚgradT)Ú	broadcastrc   c               S   s   d S )Nr   r   r   r   r   Ú<lambda>!  s    z0TestYellowFin.caffe2_yellowfin.<locals>.<lambda>)rD   ra   rE   Úcurv_win_widthÚepsilonrU   )Ú	overwriteZw_scalars_memoryrX   é   é   Zw_g_avgZw_lr_avgZw_mu_avg)r`   r_   ÚvarÚdistÚlrra   )r   ZNetr   ZResetWorkspaceÚDeviceScopeÚDeviceOptionr   ÚCPUZConstantFillr=   ZINT64ZCreateMutexZ
AtomicIterZCopyCPUToGPUZCastZMulZblobro   r1   ZYellowFinOptimizerZ_runZ
RunNetOnceZ	CreateNetÚrangeZRunNetr-   rV   r^   Údotr.   Úexp) r   rU   Ú	grad_coefÚn_dimÚn_iterÚgpuZ
caffe2_resrD   ra   rE   rr   rs   rh   ri   rj   Z
iter_mutexrm   rn   ro   rc   Z
param_inforT   Zscalars_memory_blobZg_norm2_avgZg_norm2_min_avgZg_norm2_max_avgZdistance_avgZ
g_avg_blobZres_lrZres_muZg_debZvariancer   r   r   Úcaffe2_yellowfinû   s”    









zTestYellowFin.caffe2_yellowfinc             C   s†  i }d}d}d}	d}
d}d}d}d}�xZt |ƒD �]L}|d | }d|	 dtj|tj|gƒ d ƒ  }	d|
 dtjj|tj|gƒ ƒ  }
d| d|  }d| dtj|d | ƒ  }d| dtjtd|d d ƒ| d | ƒ  }|�r&|	dd|d    |d | dd|d   d   }n|	|d |  }d| d|
 |	  }|dkr0|�r| j|dd|d    |tj	|dd|d    ƒtj	|dd|d    ƒƒ\}}d| d|  }d| d|  }tj	|dd|d    ƒtj	|dd|d    ƒ||dd|d    ||d	œ||< q0| j||tj	|ƒtj	|ƒƒ\}}d| d|  }d| d|  }tj	|ƒtj	|ƒ||||d	œ||< q0W |S )
Ng        g      ð?rW   g+‡ÙÎ÷ï?gü©ñÒMbP?rX   rg   r   )r`   r_   rw   rx   ry   ra   )
r}   r.   ÚsumZonesZlinalgZnormÚlogr^   rf   r   )r   rU   r€   r�   r‚   rƒ   Z	numpy_resZtarget_h_maxZtarget_h_minZtarget_g_norm_squared_avgZtarget_g_norm_avgZtarget_g_avgZtarget_dist_avgZ	target_lrZ	target_murT   Zgrad_valZ
target_varry   ra   r   r   r   Únumpy_yellowfinU  sr     *$ zTestYellowFin.numpy_yellowfinc             C   sÖ   ||||||ƒ}||||||ƒ}	t t|ƒt|	ƒƒ xžtdt|ƒƒD ]Œ}
t ||
 jƒ |	|
 jƒ ƒ xl||
 jƒ D ]\}dt|
ƒ d d | d d t|ƒ d d t|ƒ }t||
 | |	|
 | d|d� qnW qBW d S )	NrW   zi=z,
zfeat=z
grad_coef=zzero_debias=g{®Gáz„?)ZrtolÚerr_msg)r   Úlenr}   ÚkeysÚstrr   )r   Zmodel0Zmodel1rU   r€   r�   r‚   rƒ   Z
model0_resZ
model1_resrT   Zfeatrˆ   r   r   r   Úcompare_yellowfin_modelsš  s    8
z&TestYellowFin.compare_yellowfin_modelsz5Results might vary too much. Only for individual use.c             C   s\   d}d}t jtjƒ}t j|ƒ�4 x,dD ]$\}}| j| j| j||||dd� q&W W d Q R X d S )Ni@B é2   Fç      ð?çš™™™™™¹?ç{®Gáz„?T)rƒ   ©FrŽ   ©Fr�   ©Fr�   ©TrŽ   )r‘   r’   r“   r”   )r   r{   r   r|   rz   rŒ   r„   r‡   )r   r�   r‚   Zcpu_device_optrU   r€   r   r   r   Útest_caffe2_cpu_vs_numpy²  s"       z&TestYellowFin.test_caffe2_cpu_vs_numpyzNo gpu supportc             C   sh   d}d}t jtjdƒ}t j|ƒ�> x6d
D ].}x(dD ] }| j| j| j||||dd	� q2W q(W W d Q R X d S )Ni@B r�   r   FTç      ð?çš™™™™™¹?ç{®Gáz„?)rƒ   )FT)r–   r—   r˜   )r   r{   r   ZGpuDeviceTyperz   rŒ   r„   r‡   )r   r�   r‚   Zgpu_device_optrU   r€   r   r   r   Útest_caffe2_gpu_vs_numpyÈ  s    

z&TestYellowFin.test_caffe2_gpu_vs_numpyN)r5   r6   r7   r&   r4   r    rV   rf   r„   r‡   rŒ   r   Úskipr•   r?   r   r@   r™   r   r   r   r   rR   Ë   s   ZErR   c               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚTestRmsPropc             K   s   d| _ t|fdddœ|—ŽS )NFgš™™™™™¹?)r!   rs   )r#   r   )r   r$   r%   r   r   r   r&   Ý  s    zTestRmsProp.build_optimizerc             C   sB   | j |jƒ jƒ | j|jƒ jƒ x|jƒ jD ]}tj|ƒ q,W d S )N)r+   r)   r*   r(   r,   r   r-   )r   r1   r2   r   r   r   r4   ã  s    zTestRmsProp.check_optimizerc             C   s   t jdƒ‚d S )Nzno sparse support)r   r   )r   r   r   r   r    é  s    zTestRmsProp.testSparseN)r5   r6   r7   r&   r4   r    r   r   r   r   r›   Ü  s   r›   c               @   s   e Zd Zdd„ ZdS )ÚTestMultiOptimizersc          /      sH  ddl m}m}m‰  ddlm} |dd�}|j|dddd	ƒ}|j||d
d	dƒ}|j||dddƒ}|j|dgddgƒ\}}	|j|	gƒ ˆ j	|ƒ‰‡ ‡fdd„}
ˆ j
dd�}ˆ j
dd�}ˆ jƒ }|j|
dƒƒ� ||j|jddƒ W d Q R X |j|
dƒƒ� ||j|jddƒ W d Q R X g }xR|jjƒ jD ]B}|jdk�rF|jd dk�sV|jd dk�r&|j|jd ƒ �q&W | j|d |d ƒ |j|
dƒƒ� ||j|jddƒ W d Q R X |j|
dƒƒ� ||j|jdd ƒ W d Q R X g }xf|jjƒ jD ]V}|jdk�r
|jd dk�s|jd dk�rê| j|jd |kƒ |j|jd ƒ �qêW | j|d |d ƒ |j|
d!ƒƒ� ||j|jd!d"ƒ W d Q R X |j|
d#ƒƒ� ||j|jd#d$ƒ W d Q R X g }xz|jjƒ jD ]j}|jd%k�râ|jd d!k�sò|jd d#k�rÂ| j|jd |kƒ | j|jd |kƒ |j|jd ƒ �qÂW | j|d |d ƒ d S )&Nr   )Úbrewr   r1   )ÚModelHelperÚtest)ÚnameÚdataÚfc1éd   r�   Úfc2é   Zfc3é
   ÚlabelÚsoftmaxÚlossc                s   ˆ j | dj| ƒˆƒS )Nz{}_grad)Zget_param_deviceÚformat)Z	blob_name)r1   Úparam_to_devicer   r   Úinfer_blob_deviceþ  s    zGTestMultiOptimizers.test_multiple_optimizers.<locals>.infer_blob_devicegš™™™™™¹?)r!   gš™™™™™É?Zfc1_wZ
fc1_w_gradZfc1_bZ
fc1_b_gradÚWeightedSumru   rW   Zfc2_wZ
fc2_w_gradZfc2_bZ
fc2_b_gradZfc3_wZ
fc3_w_gradZfc3_bZ
fc3_b_gradZAdagrad)Úcaffe2.pythonr�   r   r1   Úcaffe2.python.model_helperrž   ÚfcÚSoftmaxWithLossÚAddGradientOperatorsZ_get_param_to_devicer   ZAdagradOptimizerrz   rh   ri   ÚProtoÚopÚtypeÚinputÚappendÚassertEqualr(   )r   r�   r   rž   r$   r¢   r¤   Úpredr¨   r©   r¬   Zsgd_1Zsgd_2ZadagradZfc1_lr_blobsr´   Zfc2_lr_blobsZfc3_lr_blobsr   )r1   r«   r   Útest_multiple_optimizersî  s`    

z,TestMultiOptimizers.test_multiple_optimizersN)r5   r6   r7   rº   r   r   r   r   rœ   í  s   rœ   c               @   s   e Zd Zdd„ ZdS )ÚTestWeightDecayc             C   s.  ddl m} ddlm} |dddid�}|j|dd	d
d
dƒ}|j||dddƒ}|j||dddƒ}|j|dgddgƒ\}}|j|gƒ t|dd� t	|dƒ dddh}	xn|j
jƒ jD ]^}
|
jdkoÈd|
jkr²|
jd |	krìtdj|
jd ƒƒ | j|
jd |	kƒ |	j|
jd ƒ q²W | j|	tƒ dj|	ƒƒ d S )Nr   )r�   )rž   rŸ   ÚorderÚNCHW)r    Ú	arg_scoper¡   Úcnvé    rv   Úar£   éÈ   Úbé   r§   r¨   r©   g-Cëâ6?)Úweight_decayg)\�Âõ(¼?Úb_w_gradÚa_w_gradÚ
cnv_w_gradr­   Úwd_0_0z%Unexpected param for weight_decay: {}z Not all weights were decayed: {})r®   r�   r¯   rž   Úconvr°   r±   r²   r   r   rh   r³   r´   rµ   r¶   ÚoutputÚprintrª   r(   Úremover¸   Úset)r   r�   rž   r$   r¿   rÁ   r¹   r¨   r©   Úexpected_weight_gradr´   r   r   r   Útest_weight_decay1  s2    

z!TestWeightDecay.test_weight_decayN)r5   r6   r7   rÐ   r   r   r   r   r»   /  s   r»   c               @   s   e Zd Zdd„ ZdS )ÚTestOptimizerContextc             C   s  ddl m}m} ddlm} |dddid�}|jd }td	ƒ}td
ƒ}tdƒ}t|ƒ� |j|dddddƒ}	W d Q R X t||dœƒ� |j	||	dddƒ}
W d Q R X |j	||
dddƒ}|j
|dgddgƒ\}}|j|gƒ t|dd� t|dƒ dddh}dj|ƒd)dj|d  ƒd*dj|d! ƒd+dj|d" ƒd,i}xÀ|jjƒ jD ]°}|jd#k�r¢d$|jk�r¢|jd |k�r~td%j|jd ƒƒ | j|jd |kƒ |j|jd ƒ |jd&k�r>d}x"|jD ]}|jd'k�rº|j}�qºW | j|||jd  ƒ �q>W | j|tƒ d(j|ƒƒ d S )-Nr   )r�   r1   )rž   rŸ   r¼   r½   )r    r¾   r   g333333Ã?gš™™™™™É?gš™™™™™¹?r¡   r¿   rÀ   rv   )ZWEIGHTZBIASrÁ   r£   rÂ   rÃ   rÄ   r§   r¨   r©   g-Cëâ6?)rÅ   g)\�Âõ(¼?rÆ   rÇ   rÈ   zSgdOptimizer_{}_lr_cpurW   rX   ru   r­   rÉ   z%Unexpected param for weight_decay: {}ZLearningRateZbase_lrz Not all weights were decayed: {}g333333Ã¿gš™™™™™É¿gš™™™™™¹¿g)\�Âõ(¼¿)r®   r�   r1   r¯   rž   Z_optimizer_instance_countr   r   rÊ   r°   r±   r²   r   r   rª   rh   r³   r´   rµ   r¶   rË   rÌ   r(   rÍ   Úargr    ÚfZassertAlmostEqualr¸   rÎ   )r   r�   r1   rž   r$   ÚcountZ	cnv_optimZweight_optimZ
bias_optimr¿   rÁ   r¹   r¨   r©   rÏ   Zexpected_learning_rater´   rS   rÒ   r   r   r   Útest_optimizer_contextY  sV    




z+TestOptimizerContext.test_optimizer_contextN)r5   r6   r7   rÕ   r   r   r   r   rÑ   W  s   rÑ   )6Ú
__future__r   r   r   Zcaffe2.protor   Zcaffe2.python.optimizerÚpythonr1   r   r   r   r	   r
   r   r   r   r   r   r   r   Zcaffe2.python.optimizer_contextr   Z!caffe2.python.optimizer_test_utilr   r   r®   r   r   Zcaffe2.python.test_utilr   Znumpyr.   Znumpy.testingr   r   rZ   r   r   r8   r9   r:   rB   rH   rI   rJ   rL   rM   rN   rR   r›   rœ   r»   rÑ   r   r   r   r   Ú<module>   s@   8

  B(