3
H4W^R\  ã            -   @   s$  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZmZ d d	lmZ d d
lmZmZ d dlmZ d dlZd dlZdddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9g-ZG d:d;„ d;eƒZd>d<d=„ZdS )?é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚcoreÚscopeÚ	workspaceÚhelpers)Úparameter_info)Úparameter_sharing_context)ÚOptimizerContextÚDEFAULT_OPTIM)ÚRegularizerContext)Ú	viewitemsÚviewkeys)ÚchainNZAccuracyZAdamÚAddZAdagradZSparseAdagradZAdadeltaZSparseAdadeltaZAveragedLossZCastZ
CheckpointZConstantFillZCopyZCopyGPUToCPUZCopyCPUToGPUZDequeueBlobsZEnsureCPUOutputZ
ExpandDimsZFlattenZFlattenToVecZLabelCrossEntropyZLearningRateZMakeTwoClassZMatMulZNCCLAllreduceZ	NHWC2NCHWZPackSegmentsZPrintZPReluZReduceFrontSumZScaleZScatterWeightedSumZSigmoidZSortedSegmentSumZSnapshotZSoftmaxZSoftmaxWithLossZSquaredL2DistanceZSqueezeÚStopGradientZ	SummarizeZTanhZ	TransposeZUnpackSegmentsZWeightedSumZ	YellowFinc               @   s  e Zd ZdZd?dd„Zedd„ ƒZd	d
„ Zdd„ Zdd„ Z	dd„ Z
d@dd„Zdd„ ZdAdd„ZdBdd„Zedd„ ƒZdCdd„Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ ZdDd)d*„Zd+d,„ Zd-d.„ ZdEd/d0„ZdFd1d2„Zd3d4„ Zd5d6„ Zd7d8„ Zd9d:„ Z d;d<„ Z!d=d>„ Z"dS )GÚModelHelpera–  A helper model so we can manange models more easily. It contains net def
    and parameter storages. You can add an Operator yourself, e.g.

        model = model_helper.ModelHelper(name="train_net")
        # init your weight and bias as w and b
        w = model.param_init_net.XavierFill(...)
        b = model.param_init_net.ConstantFill(...)
        fc1 = model.FC([input, w, b], output, **kwargs)

    or you can use helper functions in brew module without manually
    defining parameter initializations and operators.

        model = model_helper.ModelHelper(name="train_net")
        fc1 = brew.fc(model, input, output, dim_in, dim_out, **kwargs)

    NTFc             C   sÊ   |pd| _ tj| j ƒ| _|d k	rJ|j| _|j| _|j| _|j| _|j| _n*tj| j d ƒ| _i | _g | _i | _g | _g | _	g | _
d| _|| _|| _|| _g | _g | _ddddœ| _|d k	rÆ| jj|ƒ d S )NÚmodelÚ_initFZNCHWT)ÚorderZ	use_cudnnZcudnn_exhaustive_search)Únamer   ÚNetÚnetÚparam_init_netÚparam_to_gradÚparamsÚ_parameters_infoÚ_computed_paramsÚ_param_info_deprecatedÚ_devicesÚgradient_ops_addedÚinit_paramsÚallow_not_known_opsÚskip_sparse_optimÚweightsÚbiasesÚ
_arg_scopeÚupdate)Úselfr   r#   r$   r%   Zparam_modelÚ	arg_scope© r,   ú;/tmp/pip-build-e66j97b2/torch/caffe2/python/model_helper.pyÚ__init__^   s4    


zModelHelper.__init__c             C   s   | j S )N)r(   )r*   r,   r,   r-   r+   ‚   s    zModelHelper.arg_scopec             C   s   | j S )N)r   )r*   r,   r,   r-   Úget_name†   s    zModelHelper.get_namec             C   sL   xF| j jƒ jD ]6}t|ƒ|jkrx"|jD ]}|jdkr(t|jƒS q(W qW d S )NÚshape)	r   ÚProtoÚopÚstrÚoutputÚargr   ÚlistÚints)r*   Úparamr2   r5   r,   r,   r-   Ú_infer_param_shape‰   s    
zModelHelper._infer_param_shapec             C   sš   t | jƒt | jƒkst‚x\| jt | jƒd … D ]D}t|tjƒsNtdt|ƒ ƒ‚| jj	t
jt | jƒ|| j|ƒd�ƒ q.W x| jD ]}| jj|jƒ|_q~W d S )Nz!Param %s must be a BlobReference!)Úparam_idr8   r0   )Úlenr    r   ÚAssertionErrorÚ
isinstancer   ÚBlobReferenceÚ
ValueErrorr3   Úappendr
   ÚParameterInfor9   r   Úgetr   Úgrad)r*   r8   Úinfor,   r,   r-   Ú_update_param_info_deprecated‘   s    
z)ModelHelper._update_param_info_deprecatedc             C   s$   |pg }t |tƒrt|ƒS t|gƒS )N)r=   r6   Úset)r*   Útagsr,   r,   r-   Ú_normalize_tagsž   s    zModelHelper._normalize_tagsc       	      C   sô   t |tjƒrt|ƒ}n t |tjƒr.tj|ƒ}ntdƒ‚|| j	kr`| j	| j
|ksTt‚| j	| jS |jtj|ƒ| j|d�}tjƒ }x(| j|ƒD ]}|j|ƒrŒ|j|ƒ|_qŒW |j rÈ|jtƒrÈ|jtƒ|_tjƒ }||_|| j	|< | j|j|ƒ |jS )aõ  
        Creates parameter with a given name and initializer.

        If param_name is instance of BlobRefernce - then this blob will be used
        to store parameter (no any logic will affect it's location).

        If param_name is instance of a string type, then the final blob will
        be created in the CurrentNameScope with the respect of all parameter
        sharing logic, i.e. 'resolved_name_scope/param_name'.

        Parameter sharing logic is going to override CurrentNameScope accoring
        to the rules that are specified through ParameterSharing contexts,
        all ParameterSharing contexts are applied recursively until there are no
        extra overrides present, where on each step the best match will be
        applied first.

        The following examples should clarify the way ParameterSharing logic
        works:

        As an example if this function is called with parameter 'w':
        a. Call from some scope 'global_scope' with no Parameter sharing:
          'global_scope/w'
        b. Call from scope 'scope_b', with override {'scope_b': 'scope_a'}:
          'scope_a/w'
        c. Call from scope 'scope_a', with override {'scope_a': ''}:
          'scope_a/w'
        d. Call from scope 'scope_b/shared', with overrides
          {'scope_b/shared': 'scope_b', 'scope_b': 'scope_a'}:
          'scope_a/w'
        d. Call from scope 'scope_b/unshared', with overrides
          {'scope_b/shared': 'scope_b', 'scope_b': 'scope_a'}:
          'scope_a/unshared/w'
        zUnsupported type for param_name)Ú
param_nameZinit_netr0   )r=   r   r>   r3   ÚsixÚstring_typesr   Zget_parameter_nameÚ	TypeErrorr   r0   r<   ZblobÚcreate_paramr   r   ÚcurrentrH   Zhas_optimizerZget_optimizerZ	optimizerr   r   ZregularizerÚAddParameter)	r*   rI   r0   ZinitializerrG   Ú
param_infoZoptim_contextÚtagZreg_contextr,   r,   r-   rM   ¢   s0    %



zModelHelper.create_paramc             C   s(   t |tjƒstdj|ƒƒ‚| jj|d ƒS )NzParam {} is not a BlobReference)r=   r   r>   r<   Úformatr   rB   )r*   r8   r,   r,   r-   Úget_param_infoë   s    zModelHelper.get_param_infoc             C   s¶   t jdƒ | jƒ  | j|ƒ |d k	rX| jjƒ d k	rX| jjƒ jƒ j|ƒ}| jjƒ jƒ | }|d k	rd|n| j	|ƒ}t
|tjƒsŠtdt|ƒ ƒ‚| jjtjt| jƒ||||d�ƒ | jd S )Nzadd_param method is DEPRECATEDz!Param %s must be a BlobReference!)r:   r8   r0   ÚkeyÚlengthé   éÿÿÿÿ)ÚloggingÚwarningrE   rO   r   Zinput_recordZfield_blobsÚindexÚfield_namesr9   r=   r   r>   r?   r3   r    r@   r
   rA   r;   )r*   r8   rT   r0   rU   Úidxr,   r,   r-   Úadd_param_DEPRECATEDò   s     



z ModelHelper.add_param_DEPRECATEDc             C   st   t |tjƒst‚| j|ƒ}tjj|kr4| jj	|ƒ n| j
j	|ƒ tjj|krX| jj	|ƒ tjj|krp| jj	|ƒ d S )N)r=   r   r>   r<   rH   r
   ZParameterTagsZCOMPUTED_PARAMr   r@   r   ZWEIGHTr&   ZBIASr'   )r*   r8   rG   r,   r,   r-   rO     s    
zModelHelper.AddParameterc             C   s6   | d krt jƒ S | dks$| jt jƒr(| S | t j S d S )NÚ )r   ZCurrentNameScopeÚendswithZ_NAMESCOPE_SEPARATOR)Ú	namescoper,   r,   r-   Ú_NormalizeNamescope  s
    zModelHelper._NormalizeNamescopec                s8   t jˆ ƒ‰ ˆ dkr | jdd… S ‡ fdd„| jD ƒS dS )z9
        Returns the params in current namescope
        r^   Nc                s   g | ]}|j ƒ jˆ ƒr|‘qS r,   )ÚGetNameScopeÚ
startswith)Ú.0Úp)r`   r,   r-   ú
<listcomp>$  s    z)ModelHelper.GetParams.<locals>.<listcomp>)r   ra   r   )r*   r`   Z	top_scoper,   )r`   r-   Ú	GetParams  s    
zModelHelper.GetParamsc             C   s
   | j jƒ S )N)r   r1   )r*   r,   r,   r-   r1   '  s    zModelHelper.Protoc             C   s
   | j jƒ S )N)r   r1   )r*   r,   r,   r-   Ú	InitProto*  s    zModelHelper.InitProtoc             O   s    | j j||Ž | jj||Ž d S )N)r   ÚRunAllOnGPUr   )r*   ÚargsÚkwargsr,   r,   r-   ri   -  s    zModelHelper.RunAllOnGPUc             K   s    | j jg |f||dœ|—Ž}|S )N)ÚdbÚdb_type)r   ÚCreateDB)r*   Úblob_outrl   rm   rk   Zdbreaderr,   r,   r-   rn   1  s    zModelHelper.CreateDBc             O   s„   | j rtdƒ‚| jƒ  d| _ | jj||Ž| _| j| jƒ| _xB| jj	ƒ D ]4\}}| j
|ƒ}|rd||_qFtjd ||d�| j|< qFW | jS )Nz*You cannot run AddGradientOperators twice.T)r:   r8   rC   )r"   ÚRuntimeErrorÚValidater   ÚAddGradientOperatorsÚgrad_mapÚget_param_to_gradr   r   ÚitemsrS   rC   r
   rA   r   )r*   rj   rk   r8   rC   rP   r,   r,   r-   rr   6  s    
z ModelHelper.AddGradientOperatorsc             C   sD   i }| j stdƒ‚x,|D ]$}t|ƒ| jkr| jt|ƒ ||< qW |S )zp
        Given a list of parameters returns a dict from a parameter
        to a corresponding gradient
        z+You need to run AddGradientOperators first.)r"   rp   r3   rs   )r*   r   r   re   r,   r,   r-   rt   N  s    
zModelHelper.get_param_to_gradc                s8   ˆ j stdƒ‚ˆ j}|r"ˆ j|ƒ}‡ fdd„t|ƒD ƒS )zy
        Returns a map for param => grad.
        If params is not specified, all parameters will be considered.
        z'Need to call AddGradientOperators firstc                s0   g | ](\}}ˆ j  s"t|tjƒ rˆ j|ƒ‘qS r,   )r%   r=   r   ZGradientSlicerS   )rd   r8   rC   )r*   r,   r-   rf   k  s    z8ModelHelper.GetOptimizationParamInfo.<locals>.<listcomp>)r"   rp   r   rt   r   )r*   r   r   r,   )r*   r-   ÚGetOptimizationParamInfo^  s    
z$ModelHelper.GetOptimizationParamInfoc             C   sx   dd„ | j D ƒ}t|ƒ}g }t|ƒt|ƒkrtt|ƒ}x>t|ƒD ]2\}}|dkr>||d  |kr>||kr>|j|ƒ q>W |S )z,
        Check for duplicate params
        c             S   s   g | ]}t |ƒ‘qS r,   )r3   )rd   re   r,   r,   r-   rf   v  s    z)ModelHelper._Validate.<locals>.<listcomp>r   rV   )r   rF   r;   ÚsortedÚ	enumerater@   )r*   Zparams_listZ
params_setÚdupesÚjre   r,   r,   r-   Ú	_Validater  s    zModelHelper._Validatec             C   s"   | j ƒ }|g kstdj|ƒƒ‚d S )NzDuplicate params: {})r{   r<   rR   )r*   ry   r,   r,   r-   rq   ƒ  s    zModelHelper.Validatec                s8   t jˆ ƒ‰ ˆ dkr | jdd… S ‡ fdd„| jD ƒS dS )a  
        Returns the computed params in current namescope. 'Computed params'
        are such parameters that are not optimized via gradient descent but are
        directly computed from data, such as the running mean and variance
        of Spatial Batch Normalization.
        r^   Nc                s   g | ]}|j ƒ jˆ ƒr|‘qS r,   )rb   rc   )rd   re   )r`   r,   r-   rf   “  s    z1ModelHelper.GetComputedParams.<locals>.<listcomp>)r   ra   r   )r*   r`   r,   )r`   r-   ÚGetComputedParams‡  s    
zModelHelper.GetComputedParamsc             C   s   | j |ƒ| j|ƒ S )N)rg   r|   )r*   r`   r,   r,   r-   ÚGetAllParams–  s    zModelHelper.GetAllParamsc             K   s,   t |ƒdkstdƒ‚tjj| ||||f|ŽS )zTensorProtosDBInput.r   z†You cannot pass reader to model_helper.TensorProtosDBInput.
               Use model.net.TensorProtosDBInput instead to create the op.)r;   r<   r	   Zdb_input)r*   Zunused_blob_inro   Z
batch_sizerl   rm   rk   r,   r,   r-   ÚTensorProtosDBInput™  s    zModelHelper.TensorProtosDBInputc             C   s   t | jƒdkstdƒ‚| jS )Nr   z6Use data_parallel_model to run model on multiple GPUs.)r;   r!   r<   )r*   r,   r,   r-   Ú
GetDevices¤  s    zModelHelper.GetDevicesc             C   s~   |j dƒrt|ƒ‚tj|ƒsFtd| d d djtjj|ƒƒ d ƒ‚|tkrr| j	sbtdj
|ƒƒ‚tjdj
|ƒƒ | jj|ƒS )	z?Catch-all for all other operators, mostly those without params.Ú__zMethod z is not a registered operator.z Did you mean: [ú,ú]z#Operator {} is not known to be safezCYou are creating an op that the ModelHelper does not recognize: {}.)rc   ÚAttributeErrorr   Z
IsOperatorÚjoinr   ÚCZnearby_opnamesÚ_known_working_opsr$   rR   rX   rY   r   Ú__getattr__)r*   Zop_typer,   r,   r-   r‡   ©  s    

(zModelHelper.__getattr__c             C   s"   t tttt| ƒƒt| jƒtƒƒƒS )N)rw   rF   r   ÚdirÚtyper   Ú__dict__r†   )r*   r,   r,   r-   Ú__dir__½  s    
zModelHelper.__dir__c             C   sh   | j j| jd dd�}x|jƒ jD ]}|jd |_q"W |j| jƒ | jjƒ jdƒrd| jjƒ j	|jƒ _	|S )zr Return param_init_net + net Net.
        Returns:
          'core.Net' containing param_init_net and net
        Z_complete_netT)Úkeep_schemaz/param_init_netr‰   )
r   ÚCloner   r1   r2   Ú
debug_infoZ	AppendNetr   ZHasFieldr‰   )r*   Znew_netr2   r,   r,   r-   ÚGetCompleteNetÄ  s    zModelHelper.GetCompleteNetc             C   s‚   g }g }x<t |jƒ jƒD ]*\}}|jjdƒr8|j|ƒ q|j|ƒ qW |j|jƒ d d|dd�| _|j|jƒ d d|dd�| _	dS )zŽ construct init net and train net from complete_net
        Inputs:
          net: 'core.Net' containing param_init_net and train net
        z/param_init_netz/generated_param_init_netT)rŒ   Z
op_id_maskZupdate_external_listz/generated_netN)
rx   r1   r2   rŽ   r_   r@   r�   ÚNamer   r   )r*   r   Zparam_op_maskZtrain_op_maskr\   r2   r,   r,   r-   ÚConstructInitTrainNetfromNetÔ  s     


z(ModelHelper.ConstructInitTrainNetfromNet)NTTFNN)N)NNN)N)NF)N)N)N)#Ú__name__Ú
__module__Ú__qualname__Ú__doc__r.   Úpropertyr+   r/   r9   rE   rH   rM   rS   r]   rO   Ústaticmethodra   rg   r1   rh   ri   rn   rr   rt   rv   r{   rq   r|   r}   r~   r   r‡   r‹   r�   r‘   r,   r,   r,   r-   r   L   s>    
#
I

	



r   c                sR  t j| jd ƒ}|jƒ }t| jƒ}t| jƒ}	dd„ ˆ D ƒ‰ t|ƒjˆ ƒ}
dd„ ˆD ƒ‰tˆ ƒ}tˆƒ}ˆdkrri ‰|dk	r†|
t|ƒ }
t| j	ƒ‰y$t
‡ ‡fdd„ttˆƒƒD ƒƒ}W n" tk
rÖ   tdjˆ ƒƒ‚Y nX y$t‡‡fd	d„ttˆƒƒD ƒƒ}W n$ tk
�r    td
jˆƒƒ‚Y nX dd„ }‡fdd„}�xtˆ||d … D �]^}|
j|jƒ�rˆ|jdk�rx®|jD ]¤}|jdk�r’|jtdƒƒ n‚|jdk�rrxD|jj	D ]8}||jƒ ||jƒ |dk	�r¨|j|j_|j|j_�q¨W ||jjƒ ||jjƒ |jt|jjƒj|ƒƒ �qrW |dk	�r8|j|j_|j|j_||ƒ |j	j|gƒ |
j|jƒ |jt|jƒj|ƒƒ |jt|jƒj|	ƒƒ n tj dj|jt|jƒj!|
ƒƒƒ �qJW |jj|ƒ |jj|ƒ ||jƒ ||jƒ g }x2ˆ D ]*}|ˆk�r|j"ˆ| ƒ n
|j"|ƒ �qäW x$|j	D ]}||jƒ ||jƒ �qW |tt|jƒt|ƒ ƒfS )aª  
    Takes a model net for training and returns a net which can be
    used for prediction. For example, all gradient operators and
    input operators are removed.
    @param net_proto protobuf of the net you want to process (net.Proto())
    @param input_blobs list/set of blob names that are the inputs of predictor
    @param output_blobs list/set of blob names that are outputs of predictor
    @param device optional device option that is assigned
    @param renames dictionary of blob name to a new name (optional)
    @param disabled_inputs optional set of blobs that are 'switched off'. This
                will cause branches with those blobs as inputs to be removed
    Z_predictc             S   s   h | ]}t |ƒ’qS r,   )r3   )rd   Úbr,   r,   r-   ú	<setcomp>  s    z&ExtractPredictorNet.<locals>.<setcomp>c             S   s   h | ]}t |ƒ’qS r,   )r3   )rd   r˜   r,   r,   r-   r™   
  s    Nc                s.   g | ]&}ˆ j ˆ| jƒrˆ| jd kr|‘qS )r   )ÚintersectionÚinputr‰   )rd   rz   )Úinput_blobsÚopsr,   r-   rf     s    z'ExtractPredictorNet.<locals>.<listcomp>zNo ops with input={}c                s    g | ]}ˆj ˆ | jƒr|‘qS r,   )rš   r4   )rd   rz   )r�   Úoutput_blobsr,   r-   rf   $  s    zNo ops with output={}c             S   s>   x8| j D ].}|jdkr|jdkrtddjt| ƒƒ ƒ‚qW d S )NZis_testr   z4An operator had is_test=0, did you try to extract a z5predictor from a train model (instead of test model)?z Op was: {}ziAn operator had is_test=0, did you try to extract a predictor from a train model (instead of test model)?)r5   r   ÚiÚ	ExceptionrR   r3   )r2   r5   r,   r,   r-   Úvalidate_op+  s
    z(ExtractPredictorNet.<locals>.validate_opc                sN   | d d … }x(t |ƒD ]\}}|ˆ krˆ | ||< qW | d d …= | j|ƒ d S )N)rx   Úextend)Z
proto_listZnew_listrz   r˜   )Úrenamesr,   r-   Úrename_list6  s    
z(ExtractPredictorNet.<locals>.rename_listrV   ZRecurrentNetworkZbackward_step_netÚnZstep_netzOp {} had unknown inputs: {})#r   r   r   r1   rF   Zexternal_inputZexternal_outputÚunionr6   r2   ÚminÚranger;   r?   r    rR   ÚmaxÚ
issupersetr›   r‰   r5   Z
ClearFieldr3   r¥   r4   Zdevice_typeZdevice_optionZ	device_idr)   rš   r¢   rX   ÚdebugÚ
differencer@   )Z	net_protorœ   rž   Zdevicer£   Zdisabled_inputsZpredict_netZpredict_protoZorig_external_inputsZorig_external_outputsZknown_blobsZexternal_inputsZexternal_outputsZfirst_op_with_inputZlast_op_with_outputr¡   r¤   r2   r5   Zstep_opZrenamed_input_blobsr˜   r,   )rœ   r�   rž   r£   r-   ÚExtractPredictorNetï  sŽ    


""











r­   )NNN)Ú
__future__r   r   r   r   Zcaffe2.pythonr   r   r   r	   Zcaffe2.python.modelingr
   Z(caffe2.python.modeling.parameter_sharingr   Zcaffe2.python.optimizer_contextr   r   Z!caffe2.python.regularizer_contextr   Zfuture.utilsr   r   Ú	itertoolsr   rX   rJ   r†   Úobjectr   r­   r,   r,   r,   r-   Ú<module>   s€      *  