3
I4W^+  ã               @   sª   d dl mZ d dl mZ d dl mZ d dl mZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ ddd„Zddd„Zd dd„Zd!dd„Zd"dd„Zd#dd„Zd$dd„ZdS )%é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)Úscope)ÚParameterTags)Ú
caffe2_pb2)ÚinitializersÚNCHWFc       
      K   sˆ   d|kr|d nt jƒ }|dkp*|jtjk}|rD| rDd|d< |}n|d| d g}| jj||fd|i|—Ž}	|r|| r||	S |	d S dS )	ÚLRNZdevice_optionNÚCUDNNÚengineÚ_Z_scaleÚorderr   )r   ZCurrentDeviceScopeZdevice_typer   ZCPUÚnetr   )
ÚmodelÚblob_inÚblob_outr   Ú	use_cudnnÚkwargsÚdevZis_cpuZ	blobs_outÚlrn© r   úD/tmp/pip-build-e66j97b2/torch/caffe2/python/helpers/normalization.pyr      s    

r   Nc             K   s:   |rd|d< |dk	r&| j j||f|ŽS | j j|f|ŽS dS )zSoftmax.r   r   N)r   ZSoftmax)r   r   r   r   r   r   r   r   Úsoftmax%   s
    r   c                sÄ   ˆ pˆj jƒ ‰ ‡ ‡‡fdd„}|ddƒ|ddƒ }}ˆj|tjƒ ˆj|tjƒ ˆ ˆ d ˆ d g}	d	|kr˜|d	 r˜ˆj j|||gˆ gfd
|i|—Ž}
|
S ˆj j|||g|	fd
|i|—Ž}
|
d S d S )Nc                s   ˆj jg ˆ d | ˆg| d�S )Nr   )ÚshapeÚvalue)Zparam_init_netÚConstantFill)r   Úsuffix)r   Údim_inr   r   r   Ú	init_blob6   s    z instance_norm.<locals>.init_blobg      ð?Úsg        ÚbÚ_smÚ_sivÚis_testr   r   )r   ÚNextNameZAddParameterr   ÚWEIGHTÚBIASZInstanceNorm)r   r   r   r   r   r   r    ÚscaleÚbiasÚ	blob_outsÚblob_outputsr   )r   r   r   r   Úinstance_norm/   s    r-   ç      ð?ç        c             K   sª  |p| j jƒ }| jrŽdd|if}dd|if}dddif}dddif}tj||di fƒ}tj||di fƒ}tj||di fƒ}tj|	|di fƒ}	n tjƒ }tjƒ }tjƒ }tjƒ }	| j|d |g|tjd�}| j|d |g|tj	d�}| j|d |g|tj
d�}| j|d	 |g|	tj
d�}||||d
 |d g}d|k�rr|d �rr| j j||||d |d g|gfd|
i|—Ž}|S | j j||||d |d g|fd|
i|—Ž}|d S d S )Nr   r   g        g      ð?Ú_s)Ú
param_namer   ÚinitializerÚtagsÚ_bÚ_rmÚ_rivr#   r$   r%   é   é   r   r   )r   r&   Úinit_paramsr	   Úupdate_initializerÚExternalInitializerÚcreate_paramr   r'   r(   ÚCOMPUTED_PARAMZ	SpatialBN)r   r   r   r   Ú
init_scaleÚ	init_biasÚScaleInitializerÚBiasInitializerÚRunningMeanInitializerÚRunningVarianceInitializerr   r   Ú
scale_initÚ	bias_initÚrm_initÚriv_initr)   r*   Úrunning_meanÚrunning_inv_varr+   r,   r   r   r   Ú
spatial_bnK   sb    	



rJ   c             K   sÈ   |p| j jƒ }| jrRdd|if}dd|if}tj||di fƒ}tj||di fƒ}ntjƒ }tjƒ }| j|d |g|tjd�}| j|d |g|tj	d�}||d |d g}| j j
|||g|f|Ž}|d S )	zK
    Group normalizes the input, cf. https://arxiv.org/abs/1803.08494.
    r   r   r0   )r1   r   r2   r3   r4   Ú_meanÚ_stdr   )r   r&   r9   r	   r:   r;   r<   r   r'   r(   Z	GroupNorm)r   r   r   r   r>   r?   r@   rA   rB   rC   r   r   rD   rE   r)   r*   r+   r,   r   r   r   Ú
spatial_gnœ   s6    	

rM   r7   ç-Cëâ6?c             C   s¢   | j dj|ƒt|tƒr|gn|tjd|d�tjd�}| j dj|ƒt|tƒrP|gn|tjd|d�tjd�}	| j	j
|||	g||d |d g||dd	�\}
}}|
||fS )
a>  
    Layer normalizes the input, cf. https://arxiv.org/pdf/1607.06450.pdf.

    Args:
        blob_in: The input blob to layer normalize.
        blob_out: The layer normalized output blob.
        dim_in: The dimension of the scale and bias. For example, if blob_in is
            a 2D design matrix and axis is 1, this would be the number of
            columns.
        axis: (optional) The axis to normalize. Typically the feature axis.
            Defaults to 1.
        epsilon: (optional) A small value used for numerical stability in
            calculation. Defaults to 1e-4.
        initial_scale: (optional) The initial value for the learned scale
            parameter. Defaults to 1.0
        initial_bias: (optional) The initial value for the learned bias
            parameter of the layerwise standard deviation. Defaults to 0.0.

    Returns:
        A 3-tuple consisting of:
            - The layer normalized input blob.
            - The mean of the input blob across the given axis.
            - The standard deviation of the input blob acress the given axis.
    z{}_scaler   )r   )r1   r   r2   r3   z{}_biasrK   rL   T)ÚaxisÚepsilonZelementwise_affine)r<   ÚformatÚ
isinstanceÚintr	   ZInitializerr   r'   r(   r   Z	LayerNorm)r   r   r   r   rO   rP   Zinitial_scaleZinitial_biasr)   r*   Ú
normalizedZmeanZstdr   r   r   Ú
layer_normÓ   s*    $

rU   c             K   sú   | j rDdddif}dddif}	tj||di fƒ}tj||	di fƒ}ntjƒ }tjƒ }| j|d |g|tjd�}
| j|d |g|tjd�}|d |d	 g}|d
krÈ| jj|g|fdddg|ddœ|—Ž}n.|dkrö| jj|g|fdddg|ddœ|—Ž}|S )Nr   r   g        g      ð?r5   )r1   r   r2   r3   r6   r#   Z_svr
   r   r8   é   F)Zaxesr   ZkeepdimsZNHWCr7   )	r9   r	   r:   r;   r<   r   r=   r   ZMoments)r   r   r   r   rB   rC   r   r   rF   rG   rH   rI   r+   r,   r   r   r   Úmoments_with_running_stats  s>    

rW   )r
   F)NF)r
   )r.   r/   NNNNr
   )r.   r/   NNNNr
   )r7   rN   r.   r/   )NNr
   )Ú
__future__r   r   r   r   Zcaffe2.pythonr   Z%caffe2.python.modeling.parameter_infor   Zcaffe2.protor   Zcaffe2.python.modelingr	   r   r   r-   rJ   rM   rU   rW   r   r   r   r   Ú<module>   s0   



   
N   
8   
< 