3
I4W^!  ã               @   sÀ   d dl mZmZmZ d dljjZd dlj	Z
d dlZd dljZd dlmZmZmZmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZmZ d dl m!Z! G d	d
„ d
eƒZ"G dd„ deƒZ#dS )é    )Úabsolute_importÚdivisionÚprint_functionN)ÚcoreÚlayer_model_instantiatorÚregularizerÚschemaÚ	workspace)ÚLayersTestCase)ÚSgdOptimizer)ÚL1NormÚRegularizationBy)ÚRegularizerContextÚUseRegularizer)Úgivenc               @   s*   e Zd Zeejddgd�d�dd„ ƒZdS )ÚTestRegularizerContexté   é   )Údims)ÚXc                sR  t dƒ}t dƒ}||dœ}d}| jtjtjdfƒƒ}tj||gƒ t|ƒ�þ tj	ƒ j
dƒ}tj	ƒ j
dƒ}tdƒ}	||ks€td	ƒ‚||ks�td
ƒ‚| jj|||	|	||d�}
tjd|
fƒ| j_| jtjtj|ffƒ|
ƒ tj| jƒ\}}|jƒ j‰ ‡ fdd„ttˆ ƒƒD ƒ}|jdƒdk�st‚|jdƒdk�s0t‚|jdƒdk�sDt‚W d Q R X d S )Ngš™™™™™É?r   )ÚWEIGHTÚBIASr   r   r   r   g333333Ã?z+fail to get correct weight reg from contextz)fail to get correct bias reg from context)Zweight_optimZ
bias_optimÚ
weight_regÚbias_regÚ	fc_outputc                s   g | ]}ˆ | j ‘qS © )Útype)Ú.0Úi)Úopsr   ú?/tmp/pip-build-e66j97b2/torch/caffe2/python/regularizer_test.pyú
<listcomp>2   s    zCTestRegularizerContext.test_regularizer_context.<locals>.<listcomp>ZLpNormZScaleé   ZLpNormGradient)r   )r   Z
new_recordr   ZScalarÚnpÚfloat32Z
FeedRecordr   r   ÚcurrentZget_regularizerr   ÚAssertionErrorÚmodelZFCÚStructZoutput_schemaZassertEqualr   Zgenerate_training_netsZProtoÚopÚrangeÚlenÚcount)Úselfr   Zweight_reg_outZbias_reg_outZregularizersZoutput_dimsZinput_recordr   r   Zoptimr   Ú_Ú	train_netZops_type_listr   )r   r    Útest_regularizer_context   s8    



z/TestRegularizerContext.test_regularizer_contextN)Ú__name__Ú
__module__Ú__qualname__r   ÚhuÚarraysr0   r   r   r   r    r      s   r   c               @   s.  e Zd Zeejddgejddd�d�d�dd„ ƒZef ejddgejddd�d�ej	ƒ ej	ƒ ejd	d
d�ejd dd�ejd!dd�dœej
—Ždd„ ƒZeejddƒejddƒejddƒd�dd„ ƒZeejddƒejddƒejddƒd�dd„ ƒZeejddƒejddƒejddƒejddƒd�dd„ ƒZdS )"ÚTestRegularizerr   r   g      ð?)Z	min_valueZ	max_value)r   Úelements)r   c       
      C   s¨   t jdƒ}tj||ƒ | jƒ \}}tjdƒ}||||tjd�}||||d tj	d� tj
|ƒ tj
|ƒ dd„ }x4ttj||gƒ||ƒƒD ]\}}	tj||	dd� qˆW d S )	Nr   g      ð?)Úby)Úgradr8   c             S   s<   t jt jt jt j| dd ƒƒ ƒd ƒjt jƒt j| dd ƒfS )Ng•Ö&è.>g      à?)r#   ÚarrayÚsumÚlogÚclipÚastyper$   )r   r   r   r    ÚrefJ   s    &z-TestRegularizer.test_log_barrier.<locals>.refgü©ñÒMbP?)Zrtol)r   ÚBlobReferencer	   ÚFeedBlobÚget_training_netsr   Z
LogBarrierr   ÚON_LOSSÚAFTER_OPTIMIZERÚ
RunNetOnceÚzipZ
FetchBlobsÚnptÚassert_allclose)
r-   r   ÚparamÚtrain_init_netr/   ÚregÚoutputr?   ÚxÚyr   r   r    Útest_log_barrier9   s     




"z TestRegularizer.test_log_barrierg�íµ ÷Æ°>g-Cëâ6?)r   Ú	left_openÚ
right_openÚepsÚubÚlbc	                sÒ   ˆˆr
ˆ nd ˆˆrˆ nd k r$d S t jdƒ}	tj|	|ƒ | jƒ \}
}tjˆˆˆˆˆ d�}|||
|	tjd�}|||
|	d tj	d� tj
|
ƒ tj
|ƒ ‡ ‡‡‡‡fdd„}|d ks´t‚tjtj|	 ||ƒdd	� d S )
Ng        r   )rT   rS   rP   rQ   Úepsilon)r8   )r9   r8   c                s&   t j| ˆˆrˆ nd ˆˆrˆ nd ƒS )Ng        )r#   r=   )r   )rR   rT   rP   rQ   rS   r   r    r?   r   s    z3TestRegularizer.test_bounded_grad_proj.<locals>.refgH¯¼šò×z>)Zatol)r   r@   r	   rA   rB   r   ZBoundedGradientProjectionr   rC   rD   rE   r&   rG   rH   Úblobs)r-   r   rP   rQ   rR   rS   rT   ÚgcÚdcrI   rJ   r/   rK   rL   r?   r   )rR   rT   rP   rQ   rS   r    Útest_bounded_grad_projU   s&    
 



z&TestRegularizer.test_bounded_grad_projé   é
   é   é   r   )Ú
output_dimÚ	input_numÚ
reg_weightc             C   sì   dd„ }t jj||ƒjt jƒ}t jjd|d d�}dg}t j|t jt jjt	d|d ƒ|dd�ƒƒ}t j||gƒ}t j
|ƒ}tjdƒ}	tj|	|ƒ | jƒ \}
}tj|d	 |jƒ ƒ}|||
|	tjd
�}tj|
ƒ tj|ƒ ||||d	 |ƒ dS )zý
        1. create a weight blob
        2. create random group splits
        3. run group_l1_nrom with the weight blob
        4. run equivalent np operations to calculate group l1 norm
        5. compare if the results from 3 and 4 are equal
        c             S   sp   t j| |dd… ƒ}t jdd„ |D ƒƒ}t j|t jdd„ |D ƒƒƒ}t jt j|ƒ|ƒ}tj|tj	| dd� d S )NrZ   c             S   s   g | ]}t jt j|ƒƒ‘qS r   )r#   r;   Úsquare)r   Úgr   r   r    r!   ‰   s    zQTestRegularizer.test_group_l1_norm.<locals>.compare_reference.<locals>.<listcomp>c             S   s   g | ]}t j|jd  ƒ‘qS )rZ   )r#   ÚsqrtÚshape)r   rb   r   r   r    r!   ‹   s    r   )Údecimaléÿÿÿÿ)
r#   Zhsplitrc   Úmultiplyr:   r;   rG   Úassert_almost_equalr	   rV   )ÚweightÚgroup_boundariesZ
reg_lambdarL   Zgroup_splitsZl2_regZl2_normalizedÚresultr   r   r    Úcompare_reference‡   s    z=TestRegularizer.test_group_l1_norm.<locals>.compare_referencerZ   )ÚlowÚhighr   F)ÚreplaceÚweight_blobgš™™™™™¹?)r8   N)r#   ÚrandomÚrandr>   r$   ÚrandintÚappendÚsortÚchoicer*   Zdiffr   r@   r	   rA   rB   r   ZGroupL1NormÚtolistr   rC   rE   )r-   r^   r_   r`   rl   ri   Zfeature_numrj   Z
split_inforp   rJ   r/   rK   rL   r   r   r    Útest_group_l1_normz   s&     



z"TestRegularizer.test_group_l1_normé	   )Ú	param_dimÚkr`   c             C   s°   t jj|ƒjt jƒ}tjdƒ}tj||ƒ | j	ƒ \}}t
j|d |ƒ}||||tjd�}	tj|ƒ tj|ƒ t jt jt j|ƒƒd || … ƒ| d }
tj|
tj|	 dd� d S )Nrp   gš™™™™™¹?)r8   r   )re   )r#   rq   rr   r>   r$   r   r@   r	   rA   rB   r   ZL1NormTrimmedr   rC   rE   r;   ru   ÚabsoluterG   rh   rV   )r-   rz   r{   r`   ri   rp   rJ   r/   rK   rL   rk   r   r   r    Útest_l1_norm_trimmed©   s    


*z$TestRegularizer.test_l1_norm_trimmed)rz   r{   Úl1Úl2c             C   sÖ   t jj|ƒjt jƒ}tjdƒ}tj||ƒ | j	ƒ \}}t
j|d |d |ƒ}	|	|||tjd�}
tj|ƒ tj|ƒ t jt jt j|ƒƒd || … ƒ}t jt j|ƒƒ}|| d || d  }tj|tj|
 dd� d S )Nrp   gš™™™™™¹?)r8   r   )re   )r#   rq   rr   r>   r$   r   r@   r	   rA   rB   r   ZElasticNetL1NormTrimmedr   rC   rE   r;   ru   r|   ra   rG   rh   rV   )r-   rz   r{   r~   r   ri   rp   rJ   r/   rK   rL   Zl1_normZl2_normrk   r   r   r    Útest_elastic_l1_norm_trimmed¾   s    


"z,TestRegularizer.test_elastic_l1_norm_trimmedNg      ð¿g      ð¿g      ð¿g      ð¿)r1   r2   r3   r   r4   r5   ÚstZfloatsrO   ZbooleansZgcs_cpu_onlyrY   Zintegersrx   r}   r€   r   r   r   r    r6   8   s,   *

,




r6   )$Ú
__future__r   r   r   Z"caffe2.python.hypothesis_test_utilÚpythonZhypothesis_test_utilr4   Zhypothesis.strategiesZ
strategiesr�   Znumpyr#   Znumpy.testingZtestingrG   Zcaffe2.pythonr   r   r   r   r	   Zcaffe2.python.layer_test_utilr
   Zcaffe2.python.optimizerr   Zcaffe2.python.regularizerr   r   Z!caffe2.python.regularizer_contextr   r   Z
hypothesisr   r   r6   r   r   r   r    Ú<module>   s   

)