3
I4W^)*  ã            	   @   sR  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlZd dlZd dlZd dlZejƒ  ejd	ƒZejejƒ d
d„ Zdd„ Zdd„ Ze
jdd„ ƒZdd„ Zedk�rNeƒ jƒ \Z Z!e j"ròdnd Z#ej$ddddj%e#ƒdge! ƒ e	j&e j'�r&ej(nej)dƒZ*e	j+e*ƒ� ee ƒ W dQ R X dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)Ú
caffe2_pb2)Ú	workspaceÚcoreÚutilsÚrnn_cellÚmodel_helper)Ú	recurrentNZ
lstm_benchc             C   s€  t jdj| ƒƒ tjdƒ}|jg dd| d�}|jg dd| d�}tj|ƒ tjdƒ}|j|dgdgƒ |j|d	gd	gƒ t	j
jd
ƒ g }xät| ƒD ]Ø}	|	td| d ƒ dkrºtdj|	| ƒƒ |	dkrè| rèt	j
jd|d ƒg|dd…  n|}
t	j
j|
Ž jt	jƒ}|
d }|| }t	j
j|
d ƒ| jt	jƒ}tjd|ƒ tjd	|ƒ tj|jƒ ƒ |j|
d |
d  ƒ q�W t jdƒ |||fS )z&
    Fill a queue with input data
    z Generating T={} sequence batchesZgenerate_input_initZ
inputqueueé   )Z	num_blobsÚcapacityZ
labelqueueZgenerate_inputZscratchZ	label_scri+
  é
   r   zGenerating data {}/{}NzFinished data generation)ÚlogÚinfoÚformatr   ZNetZCreateBlobsQueuer   Ú
RunNetOnceZEnqueueBlobsÚnpÚrandomÚseedÚrangeÚmaxÚprintÚrandintZrandZastypeÚfloat32Úint32ÚFeedBlobÚProtoÚappend)ÚTÚshapeZ
num_labelsÚfixed_shapeZgenerate_input_init_netÚqueueÚlabel_queueZgenerate_input_netÚentry_countsÚtZrandom_shapeÚXÚ
batch_sizeÚLÚlabels© r+   ú=/tmp/pip-build-e66j97b2/torch/caffe2/python/lstm_benchmark.pyÚgenerate_data   s4    


2
r-   c             C   sj  t jdd�}|jjddƒ\}}|jj|dƒ}|jj|dƒ}g }	| jd#k�r"d }
d| jkrz| jsftd
ƒ‚| j}
t	dj
|
ƒƒ x>t| jƒD ]0}|jjdj
|ƒdj
|ƒƒ\}}|	j||gƒ q†W tj||||	| j| jg| j d| j| jdd|
d�\}}}}d| jk�r‚t	dƒ d|jjƒ _d|jjƒ _n`| jdk�rt|jjddƒ}	|jjg ||d� tj|||	| j| jd| jd�\}}}nd�s‚tdƒ‚|jj|dƒ}|jj|j|ƒ||gddgƒ\}}| j�sÆ|j|gƒ xX|	D ]P}|jj||ƒ | j}| jdk�rú|| j9 }tj |t!j"d| j#|gt!j$d �ƒ �qÌW | j%�rbx8|jjƒ j&D ](}|jj'd!ƒ�r6t(j)|| j*| j+d"� �q6W ||fS )$NZ
LSTM_bench)ÚnameÚseq_lengthsÚtargetZ
input_dataÚlabelÚownÚstaticÚ
static_dagz0Random input length is not static RNN compatiblezUsing static RNN of size {}zhidden_init_{}zcell_init_{}Zlstm1T)ÚmodelÚ
input_blobr/   Úinitial_statesÚdim_inÚdim_outÚscopeÚmemory_optimizationÚforward_onlyZdrop_statesZreturn_last_layer_onlyZstatic_rnn_unroll_sizeZdagzUsing DAG net typeé   ZcudnnÚhidden_initÚ	cell_init)r!   Z	cudnnlstm)r5   r6   r7   r8   r9   r:   Ú
num_layersFzUnknown implementationÚweightsÚsoftmaxÚlossr   )ÚdtypeZRecurrentNetwork)Znum_threadsZmax_cuda_streams)r2   r3   r4   ),r   ZModelHelperÚnetZAddExternalInputsZDequeueBlobsÚimplementationr"   ÚAssertionErrorÚ
seq_lengthr   r   r   r@   Úextendr
   ZLSTMÚ	input_dimÚ
hidden_dimr;   r<   r   ÚtypeZnum_workersÚparam_init_netZConstantFillZ
cudnn_LSTMZUniformFillZSoftmaxWithLossZFlattenZAddGradientOperatorsZCopyr   r   r   Úzerosr(   r   Úrnn_executorÚopÚ
startswithr   Zset_rnn_executor_configZrnn_executor_num_threadsZrnn_executor_max_cuda_streams)Úargsr#   r$   Zinput_shaper5   r/   r0   r6   r*   Z
init_blobsr    Úir>   r?   ÚoutputZlast_hiddenÚ_Z
last_staterA   rB   rC   Z	init_blobÚszrP   r+   r+   r,   Úcreate_modelC   sŠ    




rW   c          
   C   sF  | j | j }| j| j| jg}t|| j || j| jƒ\}}}tjdt	j
| jg| j t	jd�ƒ t| |||ƒ\}}tj|jƒ tj|jƒ tjƒ }|| j }	d}
tjdƒ tj|jjƒ jƒ | jrìtjdƒ tjƒ }tjdj|d d d ƒƒ tjd	ƒ tjƒ }tjƒ }x†td
|	| jƒD ]t}t| j|	| ƒ}|
|7 }
tj|jjƒ j|ƒ tjƒ }tjdj||	t	j|||| … ƒ||  d d ƒƒ |}�qW tjdjt	j|d
d … ƒtjƒ |  d d | j�rÆdndƒƒ | j�r:tjdƒ tjƒ }tjdj|d d d ƒƒ |d |d k�r:tj dj|d |d ƒƒ tj dƒ tjƒ | S )Nr/   )rD   r   z------ Warming up ------zMemory stats:zGPU memory:	{} MBZ	max_totali   z ------ Starting benchmark ------r   z'Iter: {} / {}. Entries Per Second: {}k.éd   r   z/Done. Total EPS excluding 1st iteration: {}k {}z (with RNN executor)Ú Útotalz3Max usage differs from current total usage: {} > {}z-This means that costly deallocations occured.)!Z	data_sizer(   rH   rJ   r-   rK   r"   r   r   r   Úarrayr   rW   r   rM   Z	CreateNetrE   Útimer   r   ZRunNetr   r.   Úgpur	   ZGetGPUMemoryUsageStatsr   r   Ziters_to_reportÚminÚsumrO   Úwarning)rR   r    Zinput_blob_shaper#   r$   r%   r5   rT   Ú
start_timeZ	num_itersZtotal_itersÚstatsZ	last_timeÚ	iterationZ
iters_onceZnew_timer+   r+   r,   Ú
Caffe2LSTM£   s`    




*
$

rd   c             C   s   t | ƒS )N)rd   )rR   r+   r+   r,   Ú	Benchmarké   s    re   c              C   s  t jdd�} | jdtddd� | jdtdd	d� | jd
tddd� | jdtddd� | jdtddd� | jdtddd� | jdddd� | jdtddd� | jdddd� | jdddd� | jd dd!d� | jd"td#d$d� | jd%dd&d� | jd'td d(d� | jd)td d*d� | S )+NzLSTM benchmark.)Údescriptionz--hidden_dimi   zHidden dimension)rL   ÚdefaultÚhelpz--input_dimé(   zInput dimensionz--batch_sizeé€   zThe batch size.z--seq_lengthé   zMax sequence lengthz--data_sizei@B z!Number of data points to generatez--iters_to_reportz&Number of iteration to report progressz--gpuÚ
store_truezRun all on GPU)Úactionrh   z--implementationr2   z('cudnn', 'own', 'static' or 'static_dag'z--fixed_shapezLWhether to randomize shape of input batches. Static RNN requires fixed shapez--memory_optimizationz+Whether to use memory optimized LSTM or notz--forward_onlyz Whether to run only forward passz--num_layersr   zNNumber of LSTM layers. All output dimensions are going to beof hidden_dim sizez--rnn_executorzWhether to use RNN executorz--rnn_executor_num_threadsz*Number of threads used by CPU RNN Executorz--rnn_executor_max_cuda_streamsz:Maximum number of CUDA streams used by RNN executor on GPU)ÚargparseÚArgumentParserÚadd_argumentÚintÚstr)Úparserr+   r+   r,   ÚGetArgumentParserî   s�    rt   Ú__main__r   Zcaffe2z--caffe2_log_level=0z#--caffe2_print_blob_sizes_at_exit=0z--caffe2_rnn_executor={}z--caffe2_gpu_memory_tracking=1r=   ),Ú
__future__r   r   r   r   Zcaffe2.protor   Zcaffe2.pythonr   r   r	   r
   r   r   rn   Znumpyr   r\   ÚloggingÚbasicConfigÚ	getLoggerr   ÚsetLevelÚDEBUGr-   rW   rd   Údebugre   rt   Ú__name__Úparse_known_argsrR   Ú
extra_argsrO   Zrnn_executor_optZ
GlobalInitr   ZDeviceOptionr]   ZGpuDeviceTypeZCPUZdeviceZDeviceScoper+   r+   r+   r,   Ú<module>   s>   
,`F]
