3
I4W^l  ã               @   s:  d dl mZ d dl mZ d dl mZ d dl mZ d dlZd dlZd dlZd dlZd dl	Z
d dlZd dlZd dlZd dlZd dljjZd dlmZmZmZ d dljjjjZd dlmZ ejeƒZej ej!ƒ ej"ej#ej$ƒƒ ej%dd	d
ddddgƒZ&dd„ Z'G dd„ de(ƒZ)dd„ Z*ddd„Z+dd„ Z,edk�r6e,ƒ  dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)ÚcoreÚ	workspaceÚdata_parallel_model)ÚSeq2SeqModelHelperÚBatchÚencoder_inputsÚencoder_lengthsÚdecoder_inputsÚdecoder_lengthsÚtargetsÚtarget_weightsc          
   C   sz  dd„ | D ƒ}t |ƒ}g }t dd„ | D ƒƒ}g }g }g }g }xÐ| D ]È\}	}
tjg|t|	ƒ  }|jtt|	ƒƒ| ƒ tjg|t|
ƒ  }tjg|
 }|jt|ƒƒ |j|| ƒ |
tjg }|| }|j|ƒ t|	ƒt|
ƒ dkròdgt|ƒ }ndd„ |D ƒ}|j|ƒ qBW t	t
j|t
jd�jƒ t
j|t
jd�t
j|t
jd�jƒ t
j|t
jd�t
j|t
jd�jƒ t
j|t
jd�jƒ d�S )Nc             S   s   g | ]}t |d  ƒ‘qS )r   )Úlen)Ú.0Úentry© r   úC/tmp/pip-build-e66j97b2/torch/caffe2/python/models/seq2seq/train.pyú
<listcomp>'   s    z!prepare_batch.<locals>.<listcomp>c             S   s   g | ]}t |d  ƒ‘qS )é   )r   )r   r   r   r   r   r   *   s    r   c             S   s   g | ]}|t jkrd nd‘qS )r   r   )Úseq2seq_utilÚPAD_ID)r   Útargetr   r   r   r   H   s   )Zdtype)r   r   r   r   r   r   )Úmaxr   r   r   ÚappendÚlistÚreversedZGO_IDZEOS_IDr
   ÚnpÚarrayZint32Z	transposeZfloat32)Úbatchr   Zmax_encoder_lengthr   Zmax_decoder_lengthZbatch_encoder_inputsZbatch_decoder_inputsZbatch_targetsZbatch_target_weightsZ
source_seqZ
target_seqZencoder_padsZdecoder_padsZtarget_seq_with_go_tokenZtarget_seq_with_eosr   r   r   r   r   Úprepare_batch&   sN    
r"   c               @   sª   e Zd Zdd„ Zdd„ Zdd„ Zd*d	d
„Zd+dd„Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zd,dd„Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ ZdS )-ÚSeq2SeqModelCaffe2c                sü   t |d�}ˆ j|ƒ ˆ j|ƒ t |d�}ˆ j|ƒ ˆ j|ƒ ˆ jdkrtˆ j|ƒ}|j|ƒ ˆ j|dd� ˆ j|ƒ nhˆ jˆ j dksˆt	‚t
j|dd„ ˆ jd ttˆ jƒƒd� ‡ fdd	„}t
j|d
d„ ˆ j|ttˆ jƒƒd� ˆ j|dd� |ˆ _|jˆ _d S )N)Úinit_paramsr   Únorm_clipped_grad_update)Úscopec             S   s   d S )Nr   )Úmr   r   r   Ú<lambda>   s    z1Seq2SeqModelCaffe2._build_model.<locals>.<lambda>)Zinput_builder_funZforward_pass_builder_funZparam_update_builder_funZdevicesc                s   ˆ j | dd� d S )Nr%   )r&   )r%   )Úmodel)Úselfr   r   Úclipped_grad_update_bound…   s    zBSeq2SeqModelCaffe2._build_model.<locals>.clipped_grad_update_boundc             S   s   d S )Nr   )r'   r   r   r   r(   �   s    Únorm_clipped_sparse_grad_update)r	   Ú_build_sharedÚ_build_embeddingsÚnum_gpusÚmodel_build_funZAddGradientOperatorsr%   Úforward_model_build_funÚ
batch_sizeÚAssertionErrorr   ZParallelize_GPUr   Úranger,   r)   ÚnetÚforward_net)r*   r$   r)   Zforward_modelZ
loss_blobsr+   r   )r*   r   Ú_build_modele   s@    








zSeq2SeqModelCaffe2._build_modelc             C   sn   | j d }tjtjtjƒƒ�H |jdt|d ƒdd�| _|jdddd�| _	|jdt
j
ƒ dd�| _W d Q R X d S )NÚoptimizer_paramsÚlearning_rateF)ÚnameZ
init_valueZ	trainableÚglobal_stepr   Ú
start_time)Úmodel_paramsr   ÚDeviceScopeÚDeviceOptionÚ
caffe2_pb2ÚCPUZAddParamÚfloatr9   r;   Útimer<   )r*   r)   r8   r   r   r   r-   ™   s    



z Seq2SeqModelCaffe2._build_sharedc             C   s”   t jt jtjƒƒ�x tjdƒ}|jjg d| j	| j
d g| |d�| _|jj| jƒ |jjg d| j| j
d g| |d�| _|jj| jƒ W d Q R X d S )Né   Úencoder_embeddingsÚencoder_embedding_size)ÚshapeÚminr   Údecoder_embeddingsÚdecoder_embedding_size)r   r>   r?   r@   rA   ÚmathÚsqrtÚparam_init_netZUniformFillÚsource_vocab_sizer=   rE   Úparamsr   Útarget_vocab_sizerI   )r*   r)   Zsqrt3r   r   r   r.   ¬   s$    




z$Seq2SeqModelCaffe2._build_embeddingsFNc             C   sÌ  |j jtjƒ d ƒ}|j jtjƒ d ƒ}|j jtjƒ d ƒ}|j jtjƒ d ƒ}|j jtjƒ d ƒ}|j jtjƒ d ƒ}	| jd }
|
d ksŽt‚tj|| jt	| jd ƒ||| j
| j| jd |
dk| jd�
\}}}}}tj|| jd ||||||||| j| j| jd |
d| jd�\}}tj|||| j| jd d�}|j j|gddgd!gd�\}}|j j|	gddgd"gd�\}	}|j j|||	gddgdd�\}}|j j|	gdƒ}|j j||gdƒ}|j j|gdd| j d�}|gS )#Nr   r   r   r   r   r   Ú	attentionÚnoneÚregularÚdotÚdecoder_layer_configsrF   )
r)   Úencoder_paramsZnum_decoder_layersÚinputsÚinput_lengthsÚ
vocab_sizeÚ
embeddingsÚembedding_sizeÚuse_attentionr/   rJ   F)rU   rW   rX   r   Úencoder_outputsÚweighted_encoder_outputsÚfinal_encoder_hidden_statesÚfinal_encoder_cell_statesÚencoder_units_per_layerrY   rZ   r[   Úattention_typeÚforward_onlyr/   Údecoder_softmax_size)r)   Údecoder_outputsÚdecoder_output_sizerP   rd   Ztargets_old_shaper   )rG   Ztarget_weights_old_shapeZOutputProbs_INVALIDÚloss_per_wordT)Z	only_lossÚ	num_wordsÚtotal_loss_scalarÚtotal_loss_scalar_weightedg      ð?)Zscale)rR   rS   rT   éÿÿÿÿrk   )r5   ZAddExternalInputr   ZGetNameScoper=   r3   r   Zbuild_embedding_encoderrV   r   rN   rE   r/   Zbuild_embedding_decoderrP   rI   Zoutput_projectionZReshapeZSoftmaxWithLossÚSumElementsÚMulZScaler2   )r*   r)   rc   Ú
loss_scaler   r   r   r   r   r   rb   r]   r^   r_   r`   ra   re   rf   Zoutput_logitsÚ_rg   rh   ri   rj   r   r   r   r0   Æ   sŽ    
z"Seq2SeqModelCaffe2.model_build_func             C   s   | j |d|d�S )NT)r)   rc   rn   )r0   )r*   r)   rn   r   r   r   r1   +  s    z*Seq2SeqModelCaffe2.forward_model_build_func             C   sü   t j|ƒ�è g }xzt|ƒD ]n\}}tj|ƒ t|j| t jƒsH|j| n
|j| j}|j	j
|gdj|ƒƒ}	|j	j|	dj|ƒƒ}
|j|
ƒ qW |j	j|dƒ}|j	j|ddd�}|jjg dg t| jd ƒd	�}|j	j||gd
ƒ}|j	j||gdƒ}|S Q R X d S )Nzgrad_{}_squaredzgrad_{}_squared_sumÚgrad_squared_full_sumÚglobal_normg      à?)ÚexponentÚ	clip_normÚmax_gradient_norm)rG   ÚvalueÚmax_normÚ
norm_ratio)r   Ú	NameScopeÚ	enumerateÚloggerÚinfoÚ
isinstanceÚparam_to_gradÚGradientSliceÚvaluesr5   ZSqrÚformatrl   r   ZSumÚPowrM   ÚConstantFillrB   r=   ZMaxÚDiv)r*   r)   rO   r&   ÚONEZgrad_squared_sumsÚiÚparamZgradZgrad_squaredZgrad_squared_sumrp   rq   rs   rv   rw   r   r   r   Ú_calc_norm_ratio2  sD    
z#Seq2SeqModelCaffe2._calc_norm_ratioc             C   sœ   x–|D ]Ž}|j | }|jj|gdƒ}	tj|ƒ� |jj|	|gddd�}
W d Q R X t|tjƒr~|j}|jj	|||j
||
g|ƒ q|jj||||
g|ƒ qW d S )NZnegative_learning_rateÚupdate_coeffr   )Ú	broadcast)r}   r5   ZNegativer   rx   rm   r|   r~   r   ZScatterWeightedSumÚindicesZWeightedSum)r*   rw   r)   rO   r9   r&   r„   r†   Z
param_gradZnlrrˆ   Zparam_grad_valuesr   r   r   Ú_apply_norm_ratioa  s2    

z$Seq2SeqModelCaffe2._apply_norm_ratioc             C   s¨   | j dkr| j}n|j| jdƒ}g }x:|jdd�D ]*}||jkr2t|j| tjƒs2|j|ƒ q2W |j	j
g ddgdd�}tjd	ƒ | j||||ƒ}| j||||||ƒ d S )
Nr   ZLRT)Ú	top_scoper„   r   g      ð?)rG   ru   zDense trainable variables: )r/   r9   ZCopyCPUToGPUÚ	GetParamsr}   r|   r   r~   r   rM   r‚   rz   r{   r‡   r‹   )r*   r)   r&   r9   rO   r†   r„   rw   r   r   r   r%   ˆ  s&    


z+Seq2SeqModelCaffe2.norm_clipped_grad_updatec             C   sŽ   | j }g }x:|jdd�D ]*}||jkrt|j| tjƒr|j|ƒ qW |jjg ddgdd�}t	j
dƒ | j||||ƒ}| j||||||ƒ d S )NT)rŒ   r„   r   g      ð?)rG   ru   zSparse trainable variables: )r9   r�   r}   r|   r   r~   r   rM   r‚   rz   r{   r‡   r‹   )r*   r)   r&   r9   rO   r†   r„   rw   r   r   r   r,   ¤  s"    

z2Seq2SeqModelCaffe2.norm_clipped_sparse_grad_updatec             C   sP   | j dkrtjdƒS d}x.t| j ƒD ] }dj|ƒ}tj|ƒ}||7 }q$W |S d S )Nr   ri   zgpu_{}/total_loss_scalar)r/   r   Ú	FetchBlobr4   r€   )r*   Ú
total_lossr…   r:   Zgpu_lossr   r   r   ri   ¼  s    



z$Seq2SeqModelCaffe2.total_loss_scalarc             C   s0   t j| jjƒ dd„ }|| jjƒ || jƒ d S )Nc             S   s   t j| dd„ | jD ƒd� d S )Nc             S   s   g | ]}t |ƒ‘qS r   )Ústr)r   r…   r   r   r   r   Í  s    zFSeq2SeqModelCaffe2._init_model.<locals>.create_net.<locals>.<listcomp>)Zinput_blobs)r   Z	CreateNetZexternal_inputs)r5   r   r   r   Ú
create_netÊ  s    z2Seq2SeqModelCaffe2._init_model.<locals>.create_net)r   Z
RunNetOncer)   rM   r5   r6   )r*   r‘   r   r   r   Ú_init_modelÇ  s    zSeq2SeqModelCaffe2._init_modelr   c             C   sZ   || _ d| _|d | _|| _|| _|| _|| _|d | _tj	dddddt
| jƒ gƒ d S )	NZrnnÚencoder_typer2   Zcaffe2z--caffe2_log_level=0z--v=0z---caffe2_handle_executor_threads_exceptions=1z--caffe2_mkl_num_threads=)r=   r“   rV   rN   rP   r/   Únum_cpusr2   r   Z
GlobalInitr�   )r*   r=   rN   rP   r/   r”   r   r   r   Ú__init__Ó  s    

zSeq2SeqModelCaffe2.__init__c             C   s   | S )Nr   )r*   r   r   r   Ú	__enter__ï  s    zSeq2SeqModelCaffe2.__enter__c             C   s   t jƒ  d S )N)r   ZResetWorkspace)r*   Úexc_typeÚ	exc_valueÚ	tracebackr   r   r   Ú__exit__ò  s    zSeq2SeqModelCaffe2.__exit__c             C   s,   t jdƒ | jdd� | jƒ  t jdƒ d S )Nz3Initializing Seq2SeqModelCaffe2 from scratch: StartT)r$   z4Initializing Seq2SeqModelCaffe2 from scratch: Finish)rz   r{   r7   r’   )r*   r   r   r   Úinitialize_from_scratchõ  s    
z*Seq2SeqModelCaffe2.initialize_from_scratchc             C   s   t j| jƒd S )Nr   )r   rŽ   r;   )r*   r   r   r   Úget_current_stepû  s    z#Seq2SeqModelCaffe2.get_current_stepc             C   s"   t j| jtj| jƒ d gƒƒ d S )Nr   )r   ÚFeedBlobr;   r   r    rœ   )r*   r   r   r   Úinc_current_stepþ  s    z#Seq2SeqModelCaffe2.inc_current_stepc       
      C   sð   | j dk r:t|ƒ}x¬ttj|ƒD ]\}}tj||ƒ q W n†x„t| j ƒD ]v}||d | j … }t|ƒ}xXttj|ƒD ]H\}}dj||ƒ}|dkršt	j
tjƒ}	nt	j
tj|ƒ}	tj|||	d� qpW qFW |rÒtj| jƒ ntj| jjƒ | jƒ  | jƒ S )Nr   z	gpu_{}/{}r   r   )Zdevice_option)r   r   )r/   r"   Úzipr
   Ú_fieldsr   r�   r4   r€   r   r?   r@   rA   ZGpuDeviceTypeZRunNetr6   r)   r5   rž   ri   )
r*   r!   rc   Z	batch_objZbatch_obj_nameZbatch_obj_valuer…   Z	gpu_batchr:   Údevr   r   r   Ústep  s,    
zSeq2SeqModelCaffe2.stepc          
   C   sŠ   dj ||ƒ}tjtjd| jjƒ g d|dd�ƒs2t‚tj	j
tj	j|ƒdƒ}t|dƒ�.}|jd| d	 | d
 ƒ tjd| ƒ W d Q R X |S )Nz{0}-{1}ZSaveTZminidb)Zabsolute_pathÚdbZdb_typeÚ
checkpointÚwzmodel_checkpoint_path: "z"
all_model_checkpoint_paths: "z"
zSaved checkpoint file to )r€   r   ZRunOperatorOncer   ZCreateOperatorr)   ZGetAllParamsr3   ÚosÚpathÚjoinÚdirnameÚopenÚwriterz   r{   )r*   Zcheckpoint_path_prefixZcurrent_stepZcheckpoint_pathZcheckpoint_config_pathZcheckpoint_config_filer   r   r   Úsave'  s$    
zSeq2SeqModelCaffe2.save)FN)N)r   r   )Ú__name__Ú
__module__Ú__qualname__r7   r-   r.   r0   r1   r‡   r‹   r%   r,   ri   r’   r•   r–   rš   r›   rœ   rž   r¢   r¬   r   r   r   r   r#   c   s(   4
e
/' 
#r#   c             C   sJ  t | ƒ�”}t |ƒ�€}g }xtt||ƒD ]f\}	}
tj|	|ƒ}tj|
|ƒ}t|ƒdkr$t|ƒdkr$|d ks|t|ƒ|kr$t|ƒ|kr$|j||fƒ q$W W d Q R X W d Q R X |jdd„ d� g g  }}x0|D ](}|j|ƒ t|ƒ|krÂ|j|ƒ g }qÂW t|ƒdk�r<x"t|ƒ|k �r|j|d ƒ � qþW t|ƒ|k�s2t‚|j|ƒ tj	|ƒ |S )Nr   c             S   s   t | d ƒt | d ƒfS )Nr   r   )r   )Zs_tr   r   r   r(   ^  s    zgen_batches.<locals>.<lambda>)Úkeyr   rk   )
rª   rŸ   r   Zget_numberized_sentencer   r   Úsortr3   ÚrandomÚshuffle)Úsource_corpusÚtarget_corpusÚsource_vocabÚtarget_vocabr2   Ú
max_lengthÚsourcer   Zparallel_sentencesZsource_sentenceZtarget_sentenceZnumerized_source_sentenceZnumerized_target_sentenceÚbatchesr!   Zsentence_pairr   r   r   Úgen_batchesC  s>     





r»   c       
      C   s‚  t j| j| jƒ}t j| j| jƒ}tjdjt|ƒƒƒ tjdjt|ƒƒƒ t	| j| j|||d | j
ƒ}tjdjt|ƒƒƒ t	| j| j|||d | j
ƒ}tjdjt|ƒƒƒ t|t|ƒt|ƒ| jdd��´}|jƒ  x¤t| jƒD ]–}tjdj|ƒƒ d	}x|D ]}	||j|	d
d�7 }qøW tjdj|ƒƒ d	}x |D ]}	||j|	dd�7 }�q,W tjdj|ƒƒ | jd k	rÚ|j| j|ƒ qÚW W d Q R X d S )NzSource vocab size {}zTarget vocab size {}r2   zNumber of training batches {}zNumber of eval batches {}é   )r=   rN   rP   r/   r”   zEpoch {}r   F)r!   rc   z	training loss {}Tz	eval loss {})r   Z	gen_vocabr´   Zunk_thresholdrµ   rz   r{   r€   r   r»   r¸   Zsource_corpus_evalZtarget_corpus_evalr#   r/   r›   r4   Zepochsr¢   r¤   r¬   )
Úargsr=   r¶   r·   rº   Zbatches_evalZ	model_objr…   r�   r!   r   r   r   Úrun_seq2seq_modelo  sP    



r¾   c              C   sB  t jdƒ tjdd�} | jdtd ddd� | jdtd d	dd� | jd
td dd� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jdd d!d"� | jd#d d$d"� | jd%td d&dd� | jd'td d(dd� | jd)td*d+d� | jd,td-d.d� | jd/td*d0d� | jd1td-d2d� | jd3td4d5d� | jd6td*d7d� | jd8td d9d� | jd:td d;d� | jƒ }t	|j
d<�g|j }|j�rà|j
d- dk�sÌt‚|d d=  d-  < t	|jd<�g|j }t|t	|j�rd>nd?|t	||jd@�|jt	|jdA�|j|j|j|jdB�	dC� d S )DNi·z  zCaffe2: Seq2Seq Training)Údescriptionz--source-corpuszcPath to source corpus in a text file format. Each line in the file should contain a single sentenceT)ÚtypeÚdefaultÚhelpÚrequiredz--target-corpusz+Path to target corpus in a text file formatz--max-lengthz+Maximal lengths of train and eval sentences)rÀ   rÁ   rÂ   z--unk-thresholdé2   zCThreshold frequency under which token becomes labeled unknown tokenz--batch-sizeé    zTraining batch sizez--epochsé
   z'Number of iterations over training dataz--learning-rateg      à?zLearning ratez--max-gradient-normg      ð?zbMax global norm of gradients at the end of each backward pass. We do clipping to match the number.z
--num-gpusr   z&Number of GPUs for data parallel modelz--use-bidirectional-encoderÚ
store_truezJSet flag to use bidirectional recurrent network for first layer of encoder)ÚactionrÂ   z--use-attentionz,Set flag to use seq2seq with attention modelz--source-corpus-evalz:Path to source corpus for evaluation in a text file formatz--target-corpus-evalz:Path to target corpus for evaluation in a text file formatz--encoder-cell-num-unitsi   z&Number of cell units per encoder layerz--encoder-num-layersé   zNumber encoder layersz--decoder-cell-num-unitsz)Number of cell units in the decoder layerz--decoder-num-layerszNumber decoder layersz--encoder-embedding-sizeé   z&Size of embedding in the encoder layerz--decoder-embedding-sizez&Size of embedding in the decoder layerz--decoder-softmax-sizez$Size of softmax layer in the decoderz--checkpointzPath to checkpoint)Ú	num_unitsrË   rS   rR   )Úencoder_layer_configsÚuse_bidirectional_encoder)r9   )	rQ   rU   r“   r2   r8   rF   rJ   rd   rt   )r=   )r²   ÚseedÚargparseÚArgumentParserÚadd_argumentr�   ÚintrB   Ú
parse_argsÚdictZencoder_cell_num_unitsZencoder_num_layersrÍ   r3   Zdecoder_cell_num_unitsZdecoder_num_layersr¾   r\   r2   r9   rF   rJ   rd   rt   )Úparserr½   rÌ   rU   r   r   r   Úmain¡  s�    





















rÖ   Ú__main__)N)-Ú
__future__r   r   r   r   rÏ   ÚcollectionsÚloggingrK   Znumpyr   r²   rC   Úsysr¦   Zcaffe2.proto.caffe2_pb2Úprotor@   Zcaffe2.pythonr   r   r   Z)caffe2.python.models.seq2seq.seq2seq_utilÚpythonÚmodelsZseq2seqr   Z1caffe2.python.models.seq2seq.seq2seq_model_helperr	   Ú	getLoggerr­   rz   ÚsetLevelÚINFOÚ
addHandlerÚStreamHandlerÚstderrÚ
namedtupler
   r"   Úobjectr#   r»   r¾   rÖ   r   r   r   r   Ú<module>   sH   
=   c,
2_
