3
H4W^˜$  ã               @   s¨   d dl mZ d dl mZ d dl mZ d dl mZ d dlZd dlZd dlm	Z	m
Z
mZmZmZ d dlmZ d dlmZmZ d d	lmZ G d
d„ deƒZG dd„ deƒZdS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)ÚbrewÚcoreÚ	workspaceÚcnnÚ	optimizer)Ú
caffe2_pb2)ÚInitializerÚPseudoFP16Initializer)ÚModelHelperc               @   sP   e Zd ZdZejjfdd„Zdd„ Ze	j
ej dƒejjfdd„ƒZd	d
„ ZdS )ÚOptimizerTestBasez³
    This is an abstract base class.
    Don't inherit from unittest.TestCase, and don't name it 'Test*'.
    Do, however, do these things in classes which inherit from this.
    c             C   s"  t jdddddgƒjt jƒ}t jjdƒ |tjjkr:t jnt j	}|tjjkrPt
nt}t jjdd|jfd�j|ƒ}t j||ƒd d …t jf }td	d
did�}tj|dd|jddi fdi fd||d�
}|tjjkrÞ|j||d ƒ}|j|dgƒ}	|j|	dƒ}
|j|
gƒ}| j|d tjƒ ||||fS )Né   é   é   r   é   é{   é   )ÚsizeÚtestÚorderÚNCHW)ÚnameZ	arg_scopeÚdataÚfcÚConstantFill)ÚaxisZWeightInitializerZBiasInitializerZ_fp32ÚlabelÚavg_lossÚfc_w)ÚnpÚarrayÚastypeÚfloat32ÚrandomÚseedr   ÚDataTypeÚFLOATZfloat16r   r   Úrandintr   ÚdotÚnewaxisr   r   r   ÚFLOAT16ÚHalfToFloatÚSquaredL2DistanceÚAveragedLossÚAddGradientOperatorsÚassertIsInstanceZBlobReference)ÚselfÚdtypeÚperfect_modelZnumpy_dtypeZinitializerr   r   ÚmodelÚoutÚsqÚlossÚgrad_map© r;   úB/tmp/pip-build-e66j97b2/torch/caffe2/python/optimizer_test_util.pyÚ_createDense   s,    
zOptimizerTestBase._createDensec             C   sÞ   | j ƒ \}}}}| j|ƒ}tjd|d ƒ tjd|d ƒ tj|jƒ tj|jdƒ xTtdƒD ]H}t	j
j|jd ƒ}tjd|| ƒ tjd|| ƒ tj|jjƒ jƒ q^W t	jj|t	jd d …f tjdƒdd� | j|ƒ d S )	Nr   r   r   TiÐ  r!   g{®Gáz„?)Úatol)r=   Úbuild_optimizerr   ÚFeedBlobÚ
RunNetOnceÚparam_init_netÚ	CreateNetÚnetÚranger"   r&   r*   ÚshapeÚRunNetÚProtor   ÚtestingÚassert_allcloser,   Ú	FetchBlobÚcheck_optimizer)r3   r6   r5   r   r   r
   Ú_Úidxr;   r;   r<   Ú	testDense3   s     
zOptimizerTestBase.testDensezNo gpu supportc             C   sâ   t jtjdƒ}t j|ƒ�j | j|ƒ\}}}}|t jjkrR|jddƒ}|j	|dƒ n|j	ddƒ tj
d|d ƒ tj
d|d ƒ W d Q R X tj|dddd	dd
� | j|ƒ | jr²d S tj|jƒ tj|jdƒ tj|jjƒ jƒ d S )Nr   r   Úfc_fp32_for_hostZfc_cpur   r   Zfc2r   é
   )Zdim_inZdim_outr   T)r   ZDeviceOptionr   ZGpuDeviceTypeZDeviceScoper=   r(   r-   r.   ZCopyGPUToCPUr@   r   r   r?   Z	_skip_gpurA   rB   rC   rD   rG   rH   r   )r3   r4   Z
device_optr6   Z_perfect_modelr   r   rP   r;   r;   r<   ÚtestGPUDenseG   s     
zOptimizerTestBase.testGPUDensec             C   s  d}t jdddddgƒjt jƒ}t jjdƒ t jjdd|j| fd�jt jƒ}t j|t j	||ƒƒ}t
jd	d
d�}|jjg d|jgdd�}|jj|ƒ |jj|dgdƒ}|j|dƒ}|j|dgƒ}	|j|	dƒ}
|j|
gƒ}| j|d tjƒ | j|ƒ}tjdƒ tjdƒ xêt jt jgD ]Ú}tj|jƒ tj|jdƒ x t dƒD ]”}t jj|j!d ƒ}t j	t j"|jƒ|ƒ|| dk }|jdk�r„�q>tj#d|j$|jfƒj|ƒƒ tj#dt j|| ƒjt jƒƒ tj%|jj&ƒ j'ƒ �q>W t j(j)|tj*dƒdd� �qW | j+|ƒ d S )Nr   r   r   r   r   r   r   )r   r   r   )r   Úwg        )rF   ÚvalueÚindicesZgatherÚsumr   r    TiÐ  g{®Gáz„?)r>   ),r"   r#   r$   r%   r&   r'   r*   r   r+   Úrepeatr	   ZCNNModelHelperrB   r   ÚparamsÚappendrD   ZGatherZReduceFrontSumr/   r0   r1   r2   r   ÚGradientSlicer?   r   Z
CreateBlobZint32Zint64rA   rC   rE   rF   Zaranger@   ZreshaperG   rH   r   rI   rJ   rK   rL   )r3   ZDUPLICATIONr5   r   r   r6   rS   Zpickedr7   r8   r9   r:   r
   Zindices_typerM   rN   rU   r;   r;   r<   Ú
testSparseb   sP    


zOptimizerTestBase.testSparseN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r(   r)   r=   rO   ÚunittestZskipIfr   Zhas_gpu_supportrR   r[   r;   r;   r;   r<   r      s   r   c               @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚLRModificationTestBasez³
    This is an abstract base class.
    Don't inherit from unittest.TestCase, and don't name it 'Test*'.
    Do, however, do these things in classes which inherit from this.
    c             C   sŠ   ddl m} d}xV|D ]N}t|j| |jƒs6|j| n
|j| j}tj|ƒ}|tj	tj
j|ƒdƒ7 }qW tj|ƒ}	|}
|
tj|
|	ƒ }|S )Nr   )r   g        g       @)Úcaffe2.pythonr   Ú
isinstanceÚparam_to_gradrZ   Úvaluesr   rK   r"   ÚpowerZlinalgZnormÚsqrtÚmaximum)r3   r6   rX   Úmax_gradient_normr   Zsum_squared_normsÚparamZgradÚvalZglobal_normZ	clip_normÚ
norm_ratior;   r;   r<   Ú_gradient_ratio_referenceŸ   s    


z0LRModificationTestBase._gradient_ratio_referencec             C   s&  d}| j ƒ \}}}}| j||d�}g }x:|jdd�D ]*}||jkr4t|j| tjƒs4|j|ƒ q4W tj	d|d ƒ tj	d|d ƒ tj
|jƒ tj|jdƒ | j|jƒ tjj|jd ƒ}	tj	d||	 ƒ tj	d||	 ƒ tj|jjƒ jƒ | j|||ƒ}
tjdƒ}tjj||
ƒ | j|
dk d	ƒ d S )
Ng      ð?)ri   T)Z	top_scoper   r   r   z#norm_clipped_grad_update/norm_ratioz$Bad test, gradient not being scaled.)r=   r?   Z	GetParamsrd   rc   r   rZ   rY   r   r@   rA   rB   rC   rD   ÚassertIsNotNoneÚ_lr_multiplierr"   r&   r*   rF   rG   rH   r   rm   rK   rI   Zassert_almost_equalZ
assertTrue)r3   ri   r6   r5   r   r   ÚoptrX   rj   rN   Ú	referencerl   r;   r;   r<   Ú(test_global_norm_based_gradient_clipping±   s8    
z?LRModificationTestBase.test_global_norm_based_gradient_clippingc             C   s¸   | j ƒ \}}}}| j|ddd�}tjd|d ƒ tjd|d ƒ tj|jƒ tj|jdƒ | j|j	ƒ | j
tjƒ dƒ tjdƒ | j
tjƒ dƒ tj|jjƒ jƒ | j
tjdƒdƒ d S )Nr   T)ri   Zallow_lr_injectionr   r   r   Zlr_multiplier)r=   r?   r   r@   rA   rB   rC   rD   rn   ro   ZassertEqualr
   Zget_lr_injectionZset_lr_injectionrG   rH   r   rK   )r3   r6   r5   r   r   rp   r;   r;   r<   Útest_lr_injection×   s    
z(LRModificationTestBase.test_lr_injectionN)r\   r]   r^   r_   rm   rr   rs   r;   r;   r;   r<   ra   ˜   s   &ra   )Ú
__future__r   r   r   r   r`   Znumpyr"   rb   r   r   r   r	   r
   Zcaffe2.protor   Z#caffe2.python.modeling.initializersr   r   Zcaffe2.python.model_helperr   Úobjectr   ra   r;   r;   r;   r<   Ú<module>   s    