3
H4W^?  ã               @   s8  d dl mZ d dl mZ d dl mZ d dl mZ yd dlZW n ek
rX   d dlZY nX d dlm	Z	 d dl
Z
d dlZd dlZd dlZd dlmZmZmZmZ d dlmZ d d	lmZmZmZmZmZmZ e
jd
ƒZeje
j ƒ dZ!dd„ Z"ddd„Z#G dd„ deƒZ$G dd„ deƒZ%G dd„ deƒZ&e%ƒ a'dd„ Z(dS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)Úchain)Ú	workspaceÚcoreÚscopeÚutils)Ú
caffe2_pb2)ÚMetricsÚStateÚWorkerCoordinatorÚGlobalWorkerCoordinatorÚWorkerÚ
run_workerZdata_workersé<   c             C   s   t td| ƒƒS )Nr   )ÚlistÚrange)Znum_workers© r   ú;/tmp/pip-build-e66j97b2/torch/caffe2/python/data_workers.pyÚget_worker_idsV   s    r   é   Útrainé   FéX  c                sÄ   t jƒ }|d krtjtjd�}t|ƒ‰t| |ˆ|t jƒ |tj	||ƒˆ|	|
|d�‰ dd„ t
|ƒD ƒ}t|||ˆ ƒ‰‡ ‡‡‡‡fdd„|D ƒ}|jtjtdj|t jƒ ƒˆˆ gd�ƒ |ˆ_tjˆƒ tS )N)Zdevice_type)Útimeoutc             S   s   g | ]}t jƒ ‘qS r   )Úglobal_coordinatorZget_new_worker_id)Ú.0Úir   r   r   ú
<listcomp>~   s   z+init_data_input_workers.<locals>.<listcomp>c                s4   g | ],}t jtd j|ƒˆtˆ|ˆˆˆˆ ƒgd�‘qS )zdata_workers fetcher id {})ÚtargetÚnameÚargs)Ú	threadingÚThreadr   ÚformatÚ
DataWorker)r   Ú	worker_id)Úbatch_feederÚ
batch_sizeÚcoordinatorÚ	fetch_funÚmetricsr   r   r    ‡   s   zEnqueuer {} {})r!   r"   r#   )r	   ZCurrentDeviceScoper   ZDeviceOptionZCPUr   ÚBatchFeederZCurrentNameScoper   Ú	get_queuer   r   Úappendr$   r%   Úenqueuerr&   Z_workersÚadd)ÚnetÚinput_blob_namesr,   r*   Znum_worker_threadsÚinput_source_nameÚmax_buffered_batchesZinit_funZexternal_loggersÚdont_rebatchÚbatch_columnsr   Údevice_optionZ
worker_idsÚworkersr   )r)   r*   r+   r,   r-   r   Úinit_data_input_workersZ   s:    

r;   c               @   s€   e Zd Zd dd„Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd!dd„ZdS )"r.   éX  c             C   sš   d| _ || _|| _|| _g | _|| _|| _|| _|| _d| _	| j
|ƒ | j|ƒ d| _d| _tjƒ | _|	| _| jƒ  || _|
d kr�dd„ |D ƒ}
|
| _d S )Nr   é   c             S   s   g | ]}d ‘qS )r   r   )r   Ú_r   r   r   r    ²   s    z(BatchFeeder.__init__.<locals>.<listcomp>)Ú_counterÚ_input_blob_namesÚ_batch_sizeÚ_internal_queueÚ_queuesÚ_device_optionÚ
_namescopeÚ_timeoutÚ_input_source_nameÚ_c2_queue_capacityÚ_create_caffe2_queuesÚ_create_caffe2_opsÚ_inputsÚ_prev_secondsÚtimeÚ_last_warningÚ_dont_rebatchÚ_init_scratchÚ_metricsÚ_batch_columns)Úselfr3   r4   r*   r9   Ú	namescoper5   Úqueuer-   r7   r8   r   r   r   r   Ú__init__›   s*    


zBatchFeeder.__init__c             C   s   d| _ tjƒ | _d S )Nr   )rK   rM   rL   )rS   r   r   r   Ústartµ   s    zBatchFeeder.startc             C   s@   z*x$| j D ]}tjtjd|gg ƒƒ q
W W d | jddd� X d S )NZCloseBlobsQueuer   T)Úforce)rC   r   ÚRunOperatorOncer   ÚCreateOperatorÚ_log_inputs_per_interval)rS   Úqr   r   r   Ústop¹   s
    zBatchFeeder.stopc             C   s$   t j| jjƒ ƒ t j| jjƒ ƒ d S )N)r
   Z
ResetBlobsÚ_scratch_blobÚvaluesÚ_scratch_status)rS   r   r   r   ÚcleanupÂ   s    zBatchFeeder.cleanupc             C   s|   t j ƒ }t j ƒ }xf|jƒ rvy| jjddd�S  tjk
rr   t j ƒ | dkrltjdjt j ƒ | ƒƒ t j ƒ }wY qX qW d S )NTg      à?)Úblockr   g      $@z2** Data input is slow: (still) no data in {} secs.)	rM   Ú	is_activerB   ÚgetÚQueueÚEmptyÚlogÚwarningr&   )rS   Údata_input_coordinatorÚ
start_timeZlast_warningr   r   r   Ú_getÆ   s    
zBatchFeeder._getc             C   sÄ   |d krt jdƒ dS t|ƒt| jƒks0tdƒ‚x |D ]}t|tjƒs6tdƒ‚q6W | js¦d}xH|dd … D ]8}|j	| j
|  |d j	| j
d  ksštdƒ‚|d7 }qjW t|ƒdkrÀt jdƒ dS d	S )
NzFetcher function returned NoneFz"Expecting data blob for each inputz*Fetcher function must return a numpy arrayé   r   z5Each returned input must have equal number of samplesz!Worker provided zero length inputT)rg   rh   Úlenr@   ÚAssertionErrorÚ
isinstanceÚnpZndarrayrO   ÚshaperR   )rS   ÚchunkÚdÚjr   r   r   Ú_validate_chunkÔ   s&    



zBatchFeeder._validate_chunkc             C   sÂ   | j |ƒsd S x®|jƒ r¼y~| jjƒ }|dk r`tjƒ | j tkr`tjddj	|| j
ƒ ƒ tjƒ | _|  jd7  _| jj|ddd� | j|d jd ƒ d S  tjk
r¸   tjd	ƒ wY qX qW d S )
Nr   z&Warning, data loading lagging behind: zqueue size={}, name={}rl   Tg      à?)rb   r   r   z Queue full: stalling fetchers...)ru   rc   rB   ÚqsizerM   rN   ÚLOG_INT_SECSrg   rh   r&   rG   r?   Úputr[   rq   re   ÚFullÚdebug)rS   rr   ri   rv   r   r   r   rx   ì   s     




zBatchFeeder.putc             C   sP   | j |ƒ}|d krd S |jƒ rLx,t| j| j|ƒD ]\}}}| j|||ƒ q0W d S )N)rk   rc   Úzipr@   rC   Ú_enqueue)rS   ri   ÚdataÚbr\   Úcr   r   r   Ú_enqueue_batch_directÿ   s    
z!BatchFeeder._enqueue_batch_directc             C   sö  | j r| j|ƒ dS dd„ | jD ƒ}| jd }xš|d jd dksV|d j| | jk rÈ|jƒ rÈ| j|ƒ}|dkrrq0xRt|ƒD ]F\}}|| jd dkr¤|j	ƒ ||< q|t
j|| || j| d�||< q|W q0W tjƒ }�z|d jd dkoú|d j| | jk�rœg }g }	xHt|ƒD ]<\}}
t
j|
| jg| j| d�\}}|j|ƒ |	j|ƒ �qW |	}y| jj|dd� W n tjk
�r€   Y nX |d j| | jk�sœt‚|jƒ �rÖx.t| j| j|ƒD ]\}
}}| j|
||ƒ �q¸W W d| jjdtjƒ | ƒ X dS )	z“
        This pulls data from the python-side queue and collects them
        into batch-sized pieces, unless dont_rebatch is set to true.
        Nc             S   s   g | ]}t jg ƒ‘qS r   )rp   Úarray)r   rs   r   r   r   r      s    z.BatchFeeder._enqueue_batch.<locals>.<listcomp>r   )ZaxisF)rb   Zenqueue_time)rO   r€   r@   rR   rq   rA   rc   rk   Ú	enumerateÚcopyrp   r0   rM   ÚsplitrB   rx   re   ry   rn   r{   rC   r|   rQ   Ú
put_metric)rS   ri   Z	cur_batchZfirst_batch_colrr   rt   Z
chunk_elemrj   ÚleftoverZtrimmed_batchr~   r   Úlr\   r   r   r   Ú_enqueue_batch  sP    


"


zBatchFeeder._enqueue_batchc             C   s˜   i | _ i | _xF| jD ]<}| j| d | j }tj|ƒ| j |< tj|d ƒ| j|< qW x>t| j jƒ | jjƒ ƒD ]$}t	j
|tjg ƒjtjƒ| jd� qlW d S )NZ	_scratch_Z_status)r9   )r^   r`   r@   rE   rG   r   ZBlobReferencer   r_   r   ÚFeedBlobrp   r�   ZastypeZfloat32rD   )rS   Ú	blob_nameZscratch_namer~   r   r   r   rP   B  s    zBatchFeeder._init_scratchc             C   sT   t j| j| || jd� tjd|| j| g| j| | j| g| jd�}t j|ƒ dS )zM
        Enqueue the correctly sized batch arrays to Caffe2's queue.
        )r9   ZSafeEnqueueBlobsN)r   r‰   r^   rD   r   rZ   r`   rY   )rS   rŠ   rU   Zdata_arrÚopr   r   r   r|   Y  s    

zBatchFeeder._enqueuec             C   sJ   dd„ }x<| j D ]2}|d d | j }||d| jd�}| jj|ƒ qW dS )z/
        Creates queues on caffe2 side
        c             S   s&   t jtjdg | gd|d�ƒ tj| ƒS )NZCreateBlobsQueuerl   )Ú	num_blobsÚcapacity)r   rY   r   rZ   ZScopedBlobReference)Ú
queue_namerŒ   r�   r   r   r   Úcreate_queueo  s    
z7BatchFeeder._create_caffe2_queues.<locals>.create_queueZ_c2queuer>   rl   )rŒ   r�   N)r@   rG   rH   rC   r0   )rS   r3   r�   rŠ   Úqnamer\   r   r   r   rI   k  s    	z!BatchFeeder._create_caffe2_queuesc             C   s6   x0t | j| jƒD ]\}}|j||t| jƒd� qW dS )z4
        Creates dequeue-ops on caffe2 side
        )Ztimeout_secsN)r{   rC   r@   ZDequeueBlobsÚfloatrF   )rS   r3   r\   rŠ   r   r   r   rJ     s    zBatchFeeder._create_caffe2_opsFc             C   sÀ   |  j |7  _ tjƒ }|| j }|tks,|r¼t| j | ƒ}| jjƒ }tjdj	| j
| j|ƒƒ tjdj	|ƒƒ | jjd|dƒ | jjd|dƒ | jjd|dƒ | jjƒ  | jjƒ  d| _ || _d S )Nz{}/{}: {} inputs/secz-- queue: {} batchesÚinputs_per_secFZ
queue_sizeZtime_elapsedr   )rK   rM   rL   rw   ÚintrB   rv   rg   Úinfor&   rG   rE   rQ   r…   Zlog_metricsZreset_metrics)rS   ÚinputsrX   Zcurrent_secondsZdelta_secondsr’   rv   r   r   r   r[   ‡  s(    





z$BatchFeeder._log_inputs_per_intervalN)r<   )F)Ú__name__Ú
__module__Ú__qualname__rV   rW   r]   ra   rk   ru   rx   r€   rˆ   rP   r|   rI   rJ   r[   r   r   r   r   r.   š   s   
	;r.   c               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚGlobalCoordinatorc             C   s   t j| ƒ i | _d S )N)r   rV   rC   )rS   r   r   r   rV   ¡  s    
zGlobalCoordinator.__init__c             C   s4   t |tƒst‚|| jkr*tj|d�| j|< | j| S )N)Úmaxsize)ro   r“   rn   rC   re   )rS   rŽ   r6   r   r   r   r/   ¥  s    
zGlobalCoordinator.get_queuec             C   sP   t jdj||ƒƒ x8| jD ].}|j|kr|jj|kr||j_|jj|ƒ qW d S )Nz$Reset data input {}, batch size {}: )	rg   r”   r&   Z_coordinatorsZ_worker_nameÚ_staterE   rA   rJ   )rS   rT   r"   r3   r*   r   r   r   r   Úreset_data_input«  s
    z"GlobalCoordinator.reset_data_inputN)r–   r—   r˜   rV   r/   rœ   r   r   r   r   r™      s   r™   c               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )r'   c             C   s$   t j| ||||d� || _|| _d S )N)Ú
worker_funr-   )r   rV   rA   Ú_batch_feeder)rS   r+   r(   r�   r-   r*   r)   r   r   r   rV   ´  s    	zDataWorker.__init__c             C   s$   | j | j| jƒ}| jj|| jƒ d S )N)Z_worker_funZ
_worker_idrA   rž   rx   Z_coordinator)rS   Z
input_datar   r   r   ÚrunÂ  s    zDataWorker.runc             C   s   | j jdtjƒ | j ƒ d S )NZfetcher_time)rQ   r…   rM   Z_start_time)rS   r   r   r   ÚfinishÇ  s    zDataWorker.finishN)r–   r—   r˜   rV   rŸ   r    r   r   r   r   r'   ³  s   r'   c             C   s   x| j ƒ r|j| ƒ qW d S )N)rc   rˆ   )r+   r)   r   r   r   r1   Ï  s    
r1   )r   r   r   NNFNr   ))Ú
__future__r   r   r   r   re   ÚImportErrorrU   Ú	itertoolsr   Úloggingr$   Znumpyrp   rM   Zcaffe2.pythonr   r   r	   r
   Zcaffe2.protor   Zcaffe2.python.parallel_workersr   r   r   r   r   r   Ú	getLoggerrg   ÚsetLevelÚINFOrw   r   r;   r.   r™   r'   r   r1   r   r   r   r   Ú<module>   sD   ; 
	       
4  