3
I4W^í^  ã               @   s.  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZ d dlZd dl	m
Z
 d dlZd dlZd dlmZ d dlZd d	lmZmZmZ d d
lmZ d dlmZ d dljjjjZejeƒZ e j!ej"ƒ e j#ej$ej%ƒƒ dd„ Z&G dd„ deee'ƒƒZ(G dd„ de(ƒZ)dd„ Z*dd„ Z+edk�r*e+ƒ  dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚABCMetaÚabstractmethodN)Ú	viewitems)Úwith_metaclass)ÚcoreÚrnn_cellÚ	workspace)ÚBeamSearchForwardOnly)ÚSeq2SeqModelHelperc             C   s0   t ||gt|ƒ ƒ}dd„ |D ƒ}| jj||ƒS )Nc             S   s   g | ]}|D ]}|‘qqS © r   )Ú.0Zv_wÚxr   r   úG/tmp/pip-build-e66j97b2/torch/caffe2/python/models/seq2seq/translate.pyú
<listcomp>   s    z!_weighted_sum.<locals>.<listcomp>)ÚzipÚlenÚnetZWeightedSum)ÚmodelÚvaluesÚweightÚoutput_nameZvalues_weightsZvalues_weights_flattenedr   r   r   Ú_weighted_sum   s
    r   c               @   s4   e Zd Zedd„ ƒZedd„ ƒZdd„ Zdd„ Zd	S )
Ú%Seq2SeqModelCaffe2EnsembleDecoderBasec             C   s   d S )Nr   )Úselfr   r   r   r   Úget_model_file&   s    z4Seq2SeqModelCaffe2EnsembleDecoderBase.get_model_filec             C   s   d S )Nr   )r   r   r   r   Úget_db_type*   s    z1Seq2SeqModelCaffe2EnsembleDecoderBase.get_db_typec             C   sD   t j|g|t jd�}d|tj< d|tj< d|tj< || |tj< |S )N)Údtyper   )ÚnpÚfullZfloat32Úseq2seq_utilZPAD_IDÚGO_IDÚEOS_IDZUNK_ID)r   Ú
vocab_sizeÚword_rewardÚ
unk_rewardÚword_rewardsr   r   r   Úbuild_word_rewards.   s    


z8Seq2SeqModelCaffe2EnsembleDecoderBase.build_word_rewardsc          
      s´   d}xªt | j| jƒD ]˜\}‰ ‡ fdd„| jjƒ D ƒ}tjtjdg |g| j	|ƒ| j
ƒ d�ƒsntdj| j	|ƒƒƒ‚tjtjd|g|dˆ d	 d
d�ƒs”t‚tjdjˆ | j	|ƒƒƒ qW d S )NÚreaderc                s   g | ]}t |ƒjˆ ƒr|‘qS r   )ÚstrÚ
startswith)r   Úparam)Ú
scope_namer   r   r   =   s   zESeq2SeqModelCaffe2EnsembleDecoderBase.load_models.<locals>.<listcomp>ZCreateDB)ÚdbZdb_typezFailed to create db {}ÚLoadé   ú/zgpu_0/)Zload_allZ
add_prefixZstrip_prefixz'Model {} is loaded from a checkpoint {})r   ÚmodelsÚdecoder_scope_namesr   ZGetAllParamsr   ZRunOperatorOncer
   ZCreateOperatorr   r   ÚAssertionErrorÚformatÚloggerÚinfo)r   Z	db_readerr   Zparams_for_current_modelr   )r/   r   Úload_models6   s*    
z1Seq2SeqModelCaffe2EnsembleDecoderBase.load_modelsN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r*   r:   r   r   r   r   r   $   s   r   c               @   s<   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )Ú!Seq2SeqModelCaffe2EnsembleDecoderc             C   s   |d S )NÚ
model_filer   )r   r   r   r   r   r   U   s    z0Seq2SeqModelCaffe2EnsembleDecoder.get_model_filec             C   s   dS )NZminidbr   )r   r   r   r   r   X   s    z-Seq2SeqModelCaffe2EnsembleDecoder.get_db_typec             C   s   |d k	r|d | S |S )Nr3   r   )r   r/   Z	blob_namer   r   r   Úscope[   s    z'Seq2SeqModelCaffe2EnsembleDecoder.scopec       %   #      s˜  |d }|d*kst ‚|dk}	tjˆ ƒ�  tj|| j|d ddd�}
W d Q R X tj||d t|d	 ƒ| j| j	| j|
|d |	d
dˆ d�\}}}}}tjˆ ƒ�j |	r´|j
j|d| jdd�}|d k	rÒ|j
j|d| jdd�}tj|| j|d ddd�}|j
j||gdƒ}W d Q R X g }g }xrt|d	 ƒD ]b\}}|d }|j|ƒ |d
k�rL|d }n|d	 |d  d }tjd||ddd�}|j|ƒ �qW tjˆ ƒ�Ä |d k	�räxHtt|ƒƒD ]8}|| d k	�r¨|j
j|| dj|ƒ| jdd�||< �q¨W |d k	�r8xHtt|ƒƒD ]8}|| d k	�rü|j
j|| dj|ƒ| jdd�||< �qüW tj||||||	d�}W d Q R X tj||d+ d | j||d |d, ||ˆ d�
}|j
j‡ fdd„|jƒ D ƒŽ }|j|||||d�\}}dd„ t|||ƒD ƒ}tjˆ ƒ�® |j
j|gdd gd-|jƒ gd!�\}} tj|||jƒ | j|d" d#�}!|j
j|!d$ƒ}"|j
j|"d%ƒ}#|	�rJ|j ƒ }$n:|j
j!| jgd&dd'�}$|j
j"|$d(ƒ}$|j
j|$d)| jd
d�}$W d Q R X ||#|$fS ).NÚ	attentionÚnoneÚregularÚencoder_embedding_sizeÚencoder_embeddingsF)r   r&   Úembedding_sizeÚnameZfreeze_embeddingsÚencoder_typeÚdecoder_layer_configsr   T)r   Zencoder_paramsZnum_decoder_layersÚinputsZinput_lengthsr&   Z
embeddingsrF   Úuse_attentionZnum_gpusÚforward_onlyr@   Zencoder_outputs_tiledr2   )ZtilesZaxisZweighted_encoder_outputs_tiledÚdecoder_embedding_sizeÚdecoder_embeddingsÚembedded_tokens_t_prevÚ	num_unitsg        )rL   Ú
input_sizeZhidden_sizeZforget_biasZmemory_optimizationzfinal_encoder_hidden_tiled_{}zfinal_encoder_cell_tiled_{})r   Úencoder_units_per_layerÚdecoder_units_per_layerÚfinal_encoder_hidden_statesÚfinal_encoder_cell_statesrK   )
Úencoder_outputsZencoder_output_dimÚencoder_lengthsr&   Úattention_typerF   Zdecoder_num_unitsÚdecoder_cellsÚweighted_encoder_outputsrG   c                s   g | ]}d j ˆ |ƒ‘qS )z
{}/{}_prev)r7   )r   Ús)r@   r   r   r   ê   s   zDSeq2SeqModelCaffe2EnsembleDecoder._build_decoder.<locals>.<listcomp>)r   Zinput_tZseq_lengthsÚstatesÚtimestepc          
   S   s:   g | ]2\}}}t j|t j|d dd�t j|ddd�d�‘qS )r   r2   )ZblobÚoffsetZwindow)Úinitial_valueZstate_prev_linkZ
state_link)r   ZStateConfigZ
LinkConfig)r   Zinitial_stateZ
state_prevÚstater   r   r   r   ö   s   Údecoder_outputs_flattenedZ2decoder_outputs_and_contexts_combination_old_shape)ÚshapeÚdecoder_softmax_size)r   Údecoder_outputsZdecoder_output_sizeÚtarget_vocab_sizerc   Úoutput_probsÚoutput_log_probsZzero_attention_weights_tmp_1)ÚvalueZzero_attention_weights_tmp_2Zzero_attention_weights_tmp)rB   rC   éÿÿÿÿri   ri   )#r6   r
   Z	NameScoper#   Zbuild_embeddingsÚsource_vocab_sizeZbuild_embedding_encoderr   Úencoder_inputsrW   r   ZTileÚ	beam_sizere   ZGatherÚ	enumerateÚappendr   ZLSTMCellÚranger7   Z build_initial_rnn_decoder_statesZLSTMWithAttentionDecoderZAddExternalInputsZget_state_namesÚapplyr   ZReshapeZget_output_dimZoutput_projectionZSoftmaxÚLogZget_attention_weightsÚConstantFillZ	Transpose)%r   r   Ú
step_modelÚmodel_paramsr@   Úprevious_tokensr]   Úfake_seq_lengthsrX   rK   rE   rV   rZ   rT   rU   rR   rN   rO   rY   rS   ÚiZlayer_configrP   rQ   ÚcellZinitial_statesZattention_decoderZstates_prevrd   r\   Ústate_configsra   Ú_Zoutput_logitsrf   rg   Úattention_weightsr   )r@   r   Ú_build_decoderb   s    
	








z0Seq2SeqModelCaffe2EnsembleDecoder._build_decoderc          
   C   sö  |d | _ |d }|d | _t| j ƒdks.t‚| j d d }| j d d }x.| j D ]$}|d |ksft‚|d |ksRt‚qRW t|ƒ| _t|ƒ| _dd„ tt| j ƒƒD ƒ| _td	d
�| _	| j	j
jdƒ| _| j	j
jdƒ| _| j	j
jdƒ| _| j	jjg d| jgdtjjd�}t| j| j	tjtjd�}|jƒ }g }	g }
g }xpt| j | jƒD ]^\}}| j| j	||d ||jƒ |jƒ |d�\}}}|	j|ƒ |
j|ƒ |d k	�r@|j|ƒ �q@W t|ƒdk�s´t‚| j	jjg ddtt|ƒƒ dgd�}t |||dd�}| j	jjg ddtt|
ƒƒ dgd�}t ||
|dd�}| j	jjg d| jgdtjj!d�}|j"| j| j|||	g |d�\| _#| _$| _%| _&t'j(| j	jƒ t'j)d| j*| j|d d |d d d �ƒ t'j+| j	j
t,| jƒt,| jƒt,| jƒgd!� t-j.d"ƒ x| j	j/D ]}t-j.|ƒ �qÞW d S )#NÚensemble_modelsÚdecoding_paramsrl   r   Úsource_vocabÚtarget_vocabc             S   s   g | ]}d j |ƒ‘qS )zmodel{})r7   )r   rw   r   r   r   r   O  s    z>Seq2SeqModelCaffe2EnsembleDecoder.__init__.<locals>.<listcomp>T)Zinit_paramsrk   rW   Úmax_output_seq_lenrv   i † )rb   rh   r    )rl   r   Zgo_token_idZeos_token_idrt   )r   rs   rt   r@   ru   r]   rv   Ú num_decoders_with_attention_blobr2   )rh   rb   Úattention_weights_average)r   r   r   r   Únum_decoders_blobÚoutput_log_probs_averager)   g        )rJ   ÚlengthZ	log_probsZ
attentionsry   Zdata_dependenciesr)   r'   r(   )r&   r'   r(   )Zinput_blobszParams created: )0r4   rl   r   r6   rj   re   ro   r5   r   r   r   ZAddExternalInputrk   rW   r�   Zparam_init_netrr   r
   ZDataTypeZINT32r   r#   r$   r%   Zget_step_modelr   r|   Zget_previous_tokensZget_timestepÚextendrn   Úfloatr   ÚFLOATrp   Úoutput_token_beam_listÚoutput_prev_index_beam_listÚoutput_score_beam_listÚ"output_attention_weights_beam_listr   Z
RunNetOnceÚFeedBlobr*   Z	CreateNetr,   r8   r9   Úparams)r   Útranslate_paramsr~   r   r€   r   rv   Zbeam_decoderrs   ry   rg   r{   r/   Zstate_configs_per_decoderZoutput_log_probs_per_decoderZattention_weights_per_decoderr‚   rƒ   r„   r…   r)   r.   r   r   r   Ú__init__<  sÄ    











z*Seq2SeqModelCaffe2EnsembleDecoder.__init__c                sÖ  t j| jtjdd„ tˆ ƒD ƒƒjtjd�ƒ t j| jtjt	ˆ ƒgƒjtjd�ƒ t j| j
tj|gƒjtjd�ƒ t j| jjƒ |}t j| jƒ}t j| jƒ}t j| jƒ}t j| jƒ}|df}xvt|d ƒD ]f}	x`t| jƒD ]R}
||	 |
 d tjksú|	|krØ||	 |
 d ||d  |d  d krØ|	|
f}qØW qÈW |\}	}
g }g }||	 |
 d  }xP|	dk�r¤|j||	 |
 d ƒ |j||	 |
 ƒ ||	 |
 d }
|	d8 }	�qVW t|ƒ}‡ fdd„|D ƒ}tt|ƒƒ}|||fS )Nc             S   s   g | ]
}|g‘qS r   r   )r   Ztoken_idr   r   r   r   Ô  s    z<Seq2SeqModelCaffe2EnsembleDecoder.decode.<locals>.<listcomp>)r    r   r2   c                s$   g | ]}t t|ƒƒd tˆ ƒ… ‘qS )N)ÚlistÚreversedr   )r   r{   )Únumberized_inputr   r   r     s   )r   rŽ   rk   r!   Úarrayr“   ZastypeZint32rW   r   r�   Zint64ZRunNetr   r   Z	FetchBlobrŒ   rŠ   r‹   r�   ro   rl   r#   r%   rn   r’   )r   r”   r�   Z	num_stepsZscore_beam_listZtoken_beam_listZprev_index_beam_listZattention_weights_beam_listZbest_indicesrw   Z	hyp_indexÚoutputZattention_weights_per_tokenZ
best_scorer   )r”   r   ÚdecodeÐ  sR    
z(Seq2SeqModelCaffe2EnsembleDecoder.decodeN)	r;   r<   r=   r   r   r@   r|   r‘   r—   r   r   r   r   r>   S   s    [ r>   c                sÞ   t j| j| jƒ}tjdjt|ƒƒƒ t j| j| jƒ}dd„ t	|ƒD ƒ‰ tjdjt|ƒƒƒ t
tt|||| jd�g|d�d�}|jƒ  xTtjD ]J}t j||ƒ}|j|dt|ƒ d	 ƒ\}}	}
td
j‡ fdd„|D ƒƒƒ qŒW d S )NzSource vocab size {}c             S   s   i | ]\}}||“qS r   r   )r   ÚkÚvr   r   r   ú
<dictcomp>  s    z,run_seq2seq_beam_decoder.<locals>.<dictcomp>zTarget vocab size {})r   r€   rt   r?   )r}   r~   )r�   é   é   ú c                s   g | ]}ˆ | ‘qS r   r   )r   Útid)Úinversed_target_vocabr   r   r   7  s    z,run_seq2seq_beam_decoder.<locals>.<listcomp>)r#   Z	gen_vocabZsource_corpusZunk_thresholdr8   r9   r7   r   Ztarget_corpusr   r>   ÚdictÚ
checkpointr:   ÚsysÚstdinZget_numberized_sentencer—   ÚprintÚjoin)Úargsrt   r~   r   r€   ÚdecoderÚlineZnumerized_source_sentenceÚtranslationÚ	alignmentrz   r   )rŸ   r   Úrun_seq2seq_beam_decoder  s4    
r«   c           
   C   sÜ  t jdd�} | jdtd ddd� | jdtd ddd� | jd	td
dd� | jdddd� | jdddd� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jdtddd� | jd tdd!d� | jd"td d#d� | jd$td%d&d� | jd'td(d)d� | jd*td(d+d� | jd,td d-dd� | jƒ }t|jd.�g|j	 }|j
�rz|jd d/k�sft‚|d/ d0  d  < t|jd.�g|j }t|t|j�r d1nd2|t||j
d3�|j|j|jd4�t|j|j|jd5�d6� d S )7NzCaffe2: Seq2Seq Translation)Údescriptionz--source-corpuszcPath to source corpus in a text file format. Each line in the file should contain a single sentenceT)ÚtypeÚdefaultÚhelpÚrequiredz--target-corpusz+Path to target corpus in a text file formatz--unk-thresholdé2   zCThreshold frequency under which token becomes labeled unknown token)r­   r®   r¯   z--use-bidirectional-encoderÚ
store_truez:Set flag to use bidirectional recurrent network in encoder)Úactionr¯   z--use-attentionz,Set flag to use seq2seq with attention modelz--encoder-cell-num-unitsi   z&Number of cell units per encoder layerz--encoder-num-layersr›   zNumber encoder layersz--decoder-cell-num-unitsz)Number of cell units in the decoder layerz--decoder-num-layerszNumber decoder layersz--encoder-embedding-sizeé   z&Size of embedding in the encoder layerz--decoder-embedding-sizez&Size of embedding in the decoder layerz--decoder-softmax-sizez$Size of softmax layer in the decoderz--beam-sizeé   zSize of beam for the decoderz--word-rewardg        zReward per each word generated.z--unk-rewardzBReward per each UNK token generated. Typically should be negative.z--checkpointzPath to checkpoint)rP   r   rP   rC   rB   )Úencoder_layer_configsÚuse_bidirectional_encoder)rA   rI   rH   rD   rM   rc   )rl   r'   r(   )rt   r~   )ÚargparseÚArgumentParserÚadd_argumentr,   Úintrˆ   Ú
parse_argsr    Zencoder_cell_num_unitsZencoder_num_layersr·   r6   Zdecoder_cell_num_unitsZdecoder_num_layersr«   rK   rD   rM   rc   rl   r'   r(   )Úparserr¦   r¶   rI   r   r   r   Úmain:  sz    
















r¾   Ú__main__),Ú
__future__r   r   r   r   Úabcr   r   r¸   Zfuture.utilsr   ÚloggingZnumpyr!   Úsixr	   r¢   Zcaffe2.pythonr
   r   r   Z(caffe2.python.models.seq2seq.beam_searchr   Z1caffe2.python.models.seq2seq.seq2seq_model_helperr   Z)caffe2.python.models.seq2seq.seq2seq_utilÚpythonr4   Zseq2seqr#   Ú	getLoggerr;   r8   ÚsetLevelÚINFOÚ
addHandlerÚStreamHandlerÚstderrr   Úobjectr   r>   r«   r¾   r   r   r   r   Ú<module>   s6   
	/   D&U
