3
I4W^ÌR  ã               @   sü   d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZ d dlm	Z	 d dl
mZ d dlmZmZmZmZmZmZmZmZmZ d d	lZd d	lZd d	lZd d	lZd d	lZd d	lZejeƒZd
d„ Z ddd„Z!dd„ Z"dd„ Z#dd„ Z$G dd„ deƒZ%d	S )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚFP16_ENGINESÚ	Optimizer)Úget_current_scope)Úschema)	Úget_categorical_limitÚget_keyÚIdListÚIdScoreListÚIdListWithEvictedÚIdScoreListWithEvictedÚLayerPsParamÚ
ModelLayerÚalmost_equal_schemasNc             C   sf   t | tƒr^t| dƒr^tjdj| jƒƒ | jtkrFtjdj| jƒƒ dS tjdj| jƒƒ dS ndS d S )NÚenginez/Attempting to set trainer version for engine {}z"Setting FP16 trainer for engine {}Úfp16z"Setting FP32 trainer for engine {}Úfp32)Ú
isinstancer   ÚhasattrÚloggerÚinfoÚformatr   r   )Z	optim_def© r   úC/tmp/pip-build-e66j97b2/torch/caffe2/python/layers/sparse_lookup.pyÚ"get_trainer_version_based_on_optim    s    
r   c             C   s”   | dkst dj| ƒƒ‚| dkr�|d k	rz|d k	rz|d k	rz||k rVtjdj|||ƒƒ d} |d d	kr�tjd
j||ƒƒ d} ntdj|| |||ƒƒ‚| S )Nr   r   Úuint8rowwiseÚfused_uint8rowwiseÚfused_uint4rowwisez-Unexpected version of sparse_lookup layer {0}zK{} fall back to uint8 because lookup table size {} < min_blob_size_4bits {}é   é   zM{} fall back to uint8 because lookup table dimension {} is not divisible by 2z When 4 bit quantization is enabled for {}, (i.e., Sparse lookup predictor version:{}), requires arguments blob_size:{}, min_blob_size_4bits:{}, embedding_dim:{}>   r   r   r    r   r   )ÚAssertionErrorr   r   r   Ú
ValueError)ÚversionÚ	blob_sizeÚmin_blob_size_4bitsÚembedding_dimÚsparse_feature_namer   r   r   Ú#get_sparse_lookup_predictor_version/   s6    
r*   c             C   s   | dkst dj| ƒƒ‚| S )Nr   r   z-Unexpected version of sparse_lookup layer {0}>   r   r   )r#   r   )r%   r   r   r   Ú!get_sparse_lookup_trainer_versionb   s    
r+   c             C   s
   t | tƒS )N)r   r   )Úinput_recordr   r   r   Ú_is_id_listg   s    r-   c             C   s   t | tdd�S )NF)Úcheck_field_types)r   r   )r,   r   r   r   Ú_is_id_score_listk   s    r/   c            	       sÀ   e Zd Zddddddddd	g	Zd
dddddd	gZdgZdddd
dgZd,‡ fdd„	Zdd„ Zdd„ Z	dd„ Z
d-dd„Zdd„ Zdd„ Zd.dd „Zd!d"„ Zd#d$„ Zd/d&d'„Zd(d)„ Zd*d+„ Z‡  ZS )0ÚSparseLookupZ
LogMeanExpZ	LogSumExpZMaxÚMeanÚSumÚWeightedSumÚWeightedMeanÚSqrtÚNoneÚPositionWeightedÚRecencyWeightedÚFloat16UniformFillNÚsparse_lookupc	          	      sš  t t| ƒj|||f|	Ž t| jƒƒ | _tjd| j ƒ t|t	ƒrF|g}t|t
ƒspt|tƒsptdjt|ƒ| jƒƒ‚|dkr¨t| jƒsštd jt| jƒ| jƒƒ‚| jjƒ | _n.|dkrÖt| jƒsÊtdj| jƒƒ‚| jjƒ | _|| _t| jƒ}
|
d	k�std
j| j|
ƒƒ‚|
| _|
g| | _t|ƒ| _| jƒ }|�p.|| _d | _tj| jtƒ�s\tj| jtdd��rf| jj | _| jdk�rð| j| j!k�s”tdj| j| j!ƒƒ‚| jd	 dk�r´d| jd f| _| jd	 | j"k�sÞtdj| jd	 | j"ƒƒ‚|d k�sðtdƒ‚| jj#j$�r
| jj#j$j%}nd }| j&d| j| j|t'| j|d�|d�| _(| j�rV| j&d|| j|j)d d�| _*dddif| _+| j&dg | j+|j)d�| _,tj-t.j/|f| j0dƒƒ| _1d S )!Nz"Setup the sparse lookup layer for zHUnexpected type for inner_shape, expected list or tuple, got {0} for {1}r7   z<PositionWeighted only support IdScoreList, but got {} for {}z4please use PositionWeighted layer to convert IdList zto IdScoreListr8   zNRecencyWeighted only supports IdScoreList, while the sparse feature {} is not.r   z0{} should have categorical limit > 0, but got {}F)r.   r   a  Fp16 training is enabled. The reducer specified is not supported. Got {}. Supported reducers: {}. Right now, in general, sum, mean, positional pooling are supported. Attention is not. Please check if there is fp16 trained sparse features using advanced pooling.ÚUniformFillr9   r"   zjFp16 training is enabled. Init op for weight parameter must be fp16 compatibale. Got {}. Supported ops: {}z'Regularizer is not compatible with fp16Úw)Ú
sparse_keyZaverage_length)Ú
param_nameÚshapeÚinitializerÚ	optimizerZps_paramÚregularizerÚ
reinit_vec)r>   r?   r@   rA   rB   ZConstantFillÚvalueg        Ú
scale_bias)r>   r?   r@   rA   ÚoutputzpPositionWeighted only support IdScoreList, but got {} for {}please use PositionWeighted layer to convert IdList z~PositionWeighted only support IdScoreList, but got {} for {}please use PositionWeighted layer to convert IdList to IdScoreList)2Úsuperr0   Ú__init__r   r,   r=   r   r   r   ÚintÚlistÚtupler#   r   Útyper/   ÚreprÚvaluesÚexternal_weightsÚreducerr
   Ú	input_dimr?   r   Útrainer_versionÚ_get_default_init_opÚweight_initÚevicted_valuesr	   Zequal_schemasr   r   Z_evicted_valuesÚ_fp16_compatible_reducersÚ_fp16_compatible_init_op_typesÚlengthsÚmetadataZexpected_valueZcreate_paramr   r<   ZNoOptimrC   Zscale_bias_initrE   ZScalarÚnpZfloat32Zget_next_blob_referenceÚoutput_schema)ÚselfÚmodelr,   Zinner_shaperP   rT   Zweight_optimÚnamerB   ÚkwargsrQ   Zdefault_init_opZ
avg_length)Ú	__class__r   r   rH   ƒ   s’    
 







zSparseLookup.__init__c             C   s   t jtj| jƒd S )Né   )Ú	functoolsÚreduceÚoperatorÚmulr?   )r\   r   r   r   Úget_memory_usageó   s    zSparseLookup.get_memory_usagec             C   s   | j gS )N)r<   )r\   r   r   r   Úget_fp16_compatible_parametersö   s    z+SparseLookup.get_fp16_compatible_parametersc             C   s$   t | jƒdks| jd dk r dS dS )Nr!   r"   é   FT)Úlenr?   )r\   r   r   r   Úsupport_8bitù   s    zSparseLookup.support_8bitTc             C   sH   | j ƒ sg S |r(tjddƒ}|| jƒgS tjddƒ}|| j| jƒgS d S )NÚRowwiseQuantized8BitsWeightr<   zw, scale_bias)rj   ÚcollectionsÚ
namedtupler<   rE   )r\   Zfusedrk   r   r   r   Úget_8bits_compatible_parameters   s    z,SparseLookup.get_8bits_compatible_parametersc             C   s^   t jd| j ƒ}| jdkr,d| |dœf}n.| jdkrHd| |dœf}ntdjt| jƒƒ‚|S )Ng      ð?r   r;   )ÚminÚmaxr   r9   zATrain version {} is not currently supported for sparse feature {})ÚmathÚsqrtrQ   rR   ÚNotImplementedErrorr   r=   )r\   ZscaleZdefault_weight_initr   r   r   rS     s    

z!SparseLookup._get_default_init_opc             C   sÞ   |dkr|j | j|g|ƒS |dkr@|j | j|gdƒ}|j||ƒS |dkr||j | j|gdƒ}|j | j|gdƒ}|j||g|ƒS |dkr¢|j | j|gdƒ}|j||ƒS |dkrÈ|j | j|gdƒ}|j||ƒS dd	j|| jƒ ‚d S )
Nr   r   Ú
gathered_wr   Úgathered_scale_biasr   r    z1Unsupported version of operators in SparseLookup z!layer: {0} for sparse feature {1})	ZGatherr<   ZHalfToFloatrE   ZRowwise8BitQuantizedToFloatZ Fused8BitRowwiseQuantizedToFloatZ Fused4BitRowwiseQuantizedToFloatr   r=   )r\   Únetr%   Ú
in_indicesÚoutrt   ru   r   r   r   Ú_gather_wrapper  s*    zSparseLookup._gather_wrapperr   c       	      C   sþ   | j ||| jjƒ g}d| }|dkrj|dkrN|dkrN|j|| jjƒ |dd� qú|j|ƒ|| jjƒ |d� n�|dkr |jt|ƒ| j	ƒ |j|d	 ƒ|| jjƒ ƒ nZ|d
krÄ|j|d ƒ|| jjƒ ƒ n6|dkrè|j|d ƒ|| jjƒ ƒ nddj
|| jƒ ‚d S )NÚSparseLengthsr   r   r3   ZFP16)Úgrad_on_weightsr   )r{   r   Ú8BitsRowwiser   ÚFused8BitRowwiser    ÚFused4BitRowwisez0Unsupported version of operator in SparseLookUp z!layer: {0} for sparse feature {1})r   r   )r<   r,   rX   ZSparseLengthsWeightedSumr[   Úfield_blobsÚ__getattr__Úinsertri   rE   r   r=   )	r\   rw   ÚweightsrP   rv   r%   r{   Úop_inputÚ
layer_namer   r   r   Ú _sparse_lengths_weighted_reducer=  s:    

z-SparseLookup._sparse_lengths_weighted_reducerc             C   sð  | j | jks tdj| j | jƒƒ‚| j dk�r$| j| jjƒ | jjƒ g}| j dkrVd| _ n| j dkrfd| _ d| j  }|dkr�|j	|ƒ|| j
jƒ ƒ n’|d	krÆ|jt|ƒ| jƒ |j	|d
 ƒ|| j
jƒ ƒ n\|dkrê|j	|d ƒ|| j
jƒ ƒ n8|dk�r|j	|d ƒ|| j
jƒ ƒ nddj|| jƒ ‚nÈ| j dk�rj|j| jjƒ g|jdƒgdd�}| j| jjƒ |d||ƒ n‚| j dk�r”| j||| jjƒ | j
jƒ ƒ nX| j||| jjƒ dƒ}|j| jjƒ |j| jjƒ d ƒƒ}|j	d| j  ƒ||g| j
jƒ ƒ d S )Nz&Unsupported reducer: {} for ID_LIST {}r2   r1   r3   r4   rz   r   r   r   r|   r   r}   r    r~   z0Unsupported version of operator in SparseLookUp z!layer: {0} for sparse feature {1}r5   Zlengths_sqrtg      à?)Úpowerr6   Ú
table_rowsZ_sidZSortedSegmentRange)r2   r1   r3   r4   )r   r   )rP   Ú_id_list_supported_reducersr#   r   r=   r<   r,   ÚitemsrX   r€   r[   r   r�   ri   rE   ZLengthsToWeightsZNextScopedBlobr…   ry   ZLengthsToSegmentIds)r\   rv   r%   rƒ   r„   Zsqrt_weightr‡   Zsegment_idsr   r   r   Ú_add_ops_id_listl  sb    





zSparseLookup._add_ops_id_listc             C   sŒ  | j | jks tdj| j | jƒƒ‚| j dkrN| j| jjƒ | jjƒ | j ||ƒ �n:| j dk�r| j	| jjƒ | jj
ƒ g}d| j  }|dkrœ|j|ƒ|| jjƒ ƒ n€|d	krÀ|j|d
 ƒ|| jjƒ ƒ n\|dkrä|j|d ƒ|| jjƒ ƒ n8|dk�r
|j|d ƒ|| jjƒ ƒ nddj|| jƒ ‚nj| j dk�rJ| j| jjƒ | jd||dd� n>| j dk�rt| j||| jjƒ | jjƒ ƒ nddj| j | jƒ ‚d S )Nz,Unsupported reducer: {} for ID_SCORE_LIST {}r3   r4   r2   r1   rz   r   r   r   r|   r   r}   r    r~   z0Unsupported version of operator in SparseLookUp z!layer: {0} for sparse feature {1}r7   r8   r"   )r{   r6   z9Only Sum, Mean, None are supported for IdScoreList input.z.Trying to create with {} for sparse feature {})r3   r4   )r2   r1   )r   r   )r7   r8   )rP   Ú!_id_score_list_supported_reducersr#   r   r=   r…   r,   ÚkeysrN   r<   rX   r€   r[   r   rO   ry   )r\   rv   r%   rƒ   r„   r   r   r   Ú_add_ops_id_score_listµ  sP    


z#SparseLookup._add_ops_id_score_listr   c             C   sj   | j r&|j| j| j jƒ | jg| jgƒ t| jƒr@| j||d� n&t| jƒrZ| j	||d� ndj
| jƒ‚d S )N)r%   zUnsupported input type {0})rU   ZCopyRowsToTensorr<   ÚgetrC   r-   r,   rŠ   r/   r�   r   )r\   rv   r%   r   r   r   Ú_add_opsí  s    

zSparseLookup._add_opsc             C   s   | j || jƒ d S )N)r�   rR   )r\   rv   r   r   r   Úadd_train_opsø  s    zSparseLookup.add_train_opsc             C   s|   t ƒ jtjddiƒ}| jd | jd  }t|d |d|krB|d nd | jd | jd�}| jƒ  rl|d
krld}| j||ƒ d S )Nr%   r   r   r"   r'   )r&   r'   r(   r)   r   r   r    >   r   r   r    )r   rŽ   r*   Ú__name__r?   r=   rj   r�   )r\   rv   Úversion_infoZlookup_table_blob_sizer%   r   r   r   Úadd_opsû  s    
 zSparseLookup.add_ops)NNr:   N)T)r   )r   )r‘   Ú
__module__Ú__qualname__rˆ   r‹   rW   rV   rH   rf   rg   rj   rn   rS   ry   r…   rŠ   r�   r�   r�   r“   Ú__classcell__r   r   )r`   r   r0   q   s(   

 n
!
-I8
r0   )NNNN)&Ú
__future__r   r   r   r   Zcaffe2.python.optimizerr   r   Zcaffe2.python.helpers.arg_scoper   Zcaffe2.pythonr	   Zcaffe2.python.layers.layersr
   r   r   r   r   r   r   r   r   rl   rb   Úloggingrq   ZnumpyrZ   rd   Ú	getLoggerr‘   r   r   r*   r+   r-   r/   r0   r   r   r   r   Ú<module>   s.   ,
   
.