3
H4W^   ã               @   s¤   d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZmZ d dl	m
Z
 d dlZG dd	„ d	eƒZG d
d„ deƒZG dd„ deƒZG dd„ deƒZdd„ ZdS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚDataTypeÚBlobReferenceÚScopedBlobReference)ÚParameterInfoNc               @   s*   e Zd ZdZd	dd„Zdd„ Zdd„ ZdS )
ÚInitializerz¢
    This class abstracts out parameter creation. One can come up with a new
    Initializer in order to implement more complex parameter initializaion logic
    Nc             K   s   || _ || _d S )N)Úoperator_nameÚoperator_kwargs)Úselfr   Úkwargs© r   úD/tmp/pip-build-e66j97b2/torch/caffe2/python/modeling/initializers.pyÚ__init__   s    zInitializer.__init__c             C   s"   | j d k	rtdƒ‚|| _ || _d S )Nz(Operator name overwrites are not allowed)r   Ú	Exceptionr   )r   r   r   r   r   r   Úupdate   s    
zInitializer.updatec             C   s.   |j | jƒg |fd|i| j—Ž}td ||d�S )NÚshape)Úparam_idÚparamr   )Ú__getattr__r   r   r	   )r   Ú
param_nameÚinit_netr   r   r   r   r   Úcreate_param   s    
zInitializer.create_param)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r   r   r
      s   
r
   c               @   s   e Zd ZdZdd„ ZdS )ÚExternalInitializerzÿ
    This class is used in cases when the parameter should not be initialized by
    the initializer, but rather provided in the workspace when param_init_net is
    executed.

    Current version is not doing any real sanity checks to the parameter.
    c             C   sD   t |tƒrtt|ƒ|ƒ}nt |tjƒr2t||ƒ}nd‚td ||d�S )NzUnsupported type for param_name)r   r   r   )Ú
isinstancer   ÚstrÚsixÚstring_typesr   r	   )r   r   r   r   r   r   r   r   r   /   s    
z ExternalInitializer.create_paramN)r   r   r   r   r   r   r   r   r   r   &   s   r   c               @   s    e Zd ZdZdd„ Zdd„ ZdS )ÚPseudoFP16InitializeraØ  
    Used in cases when the parameter should be used at half (16-bit) precision
    for compute purposes (i.e. on the forward and backward pass) but
    needs to be stored and optimized at single (32-bit) precision so tiny
    gradients with small learning rates don't underflow FP16 precision.
    A 32-bit copy of the 16-bit blob is stored in the ParameterInfo.
    This is helpful for mixed-precision training, see
    https://arxiv.org/abs/1710.03740 for details.
    c             C   s"   | j d k	rtdƒ‚|| _ || _d S )Nz(Operator name overwrites are not allowed)r   r   r   )r   r   r   r   r   r   r   H   s    
zPseudoFP16Initializer.updatec             C   sF   |j | jƒg |d fd|i| j—Ž}|j||ƒ}td ||tj|id�S )NZ_fp32r   )r   r   r   Ú	blob_copy)r   r   r   ÚFloatToHalfr	   r   ÚFLOAT)r   r   r   r   Ú
param_fp32r   r   r   r   r   N   s    

z"PseudoFP16Initializer.create_paramN)r   r   r   r   r   r   r   r   r   r   r$   >   s   	r$   c               @   s    e Zd ZdZdd„ Zdd„ ZdS )ÚReversePseudoFP16Initializerz½
    Like PseudoFP16Initializer above, except the primary blob is taken to
    be the 32-bit precision parameter, and the 16-bit version of the blob
    is stored in blob_copy instead.
    c             C   s"   | j d k	rtdƒ‚|| _ || _d S )Nz(Operator name overwrites are not allowed)r   r   r   )r   r   r   r   r   r   r   e   s    
z#ReversePseudoFP16Initializer.updatec             C   sF   |j | jƒg |fd|i| j—Ž}|j||d ƒ}td ||tj|id�S )Nr   Z_fp16)r   r   r   r%   )r   r   r   r&   r	   r   ZFLOAT16)r   r   r   r   r(   Z
param_fp16r   r   r   r   k   s    

z)ReversePseudoFP16Initializer.create_paramN)r   r   r   r   r   r   r   r   r   r   r)   _   s   r)   c                sL   ‡ ‡fdd„}| dk	r0| |ƒ d f|ƒ d Ž}nt |ƒ d f|ƒ d Ž}|S )aV  
    A helper function to convert from operator_name_and_kwargs to new
    object of type initializer_class. This function serves two purposes:

    1. Support for custom initialization operators being passed in
    2. Allow user to specify a custom Initializer without overwriting
       default operators used for initialization

    If initializer_class is None, creates a default initializer using
    the Initializer class and operator_name_and_kwargs provided

    If operator_name_and_kwargs is None, uses default_operator_name_and_kwargs

    returns an instantiated Initializer object
    c                  s   ˆpˆ S )Nr   r   )Ú default_operator_name_and_kwargsÚoperator_name_and_kwargsr   r   Úget_initializer_args�   s    z0update_initializer.<locals>.get_initializer_argsNr   é   )r
   )Zinitializer_classr+   r*   r,   Úinitr   )r*   r+   r   Úupdate_initializer{   s    
r/   )Ú
__future__r   r   r   r   Zcaffe2.python.corer   r   r   Z%caffe2.python.modeling.parameter_infor	   r"   Úobjectr
   r   r$   r)   r/   r   r   r   r   Ú<module>   s   !