3
I4W^   ã               @   sü   d dl mZ d dl mZ d dl mZ d dl mZ d dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZ ejdƒZejejƒ dZddd„ZG dd„ deƒZG dd„ dƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZeƒ add„ ZdS )é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)ÚABCMetaÚabstractmethodZparallel_workersé<   é   Útrainc                sT   t |ƒ‰dd„ t|ƒD ƒ}t||||d�‰ ‡ ‡‡fdd„|D ƒ}|ˆ _tjˆ ƒ tS )Nc             S   s   g | ]}t jƒ ‘qS © )Úglobal_coordinatorÚget_new_worker_id)Ú.0Úir   r   ú?/tmp/pip-build-e66j97b2/torch/caffe2/python/parallel_workers.pyú
<listcomp>@   s   z init_workers.<locals>.<listcomp>)Úshutdown_func                s0   g | ](}t jtd j|ƒˆ tˆ |ˆˆƒgd�‘qS )zparallel_workers worker id {})ÚtargetÚnameÚargs)Ú	threadingÚThreadÚ
run_workerÚformatÚWorker)r   Ú	worker_id)ÚcoordinatorÚmetricsÚ
worker_funr   r   r   J   s   )ÚMetricsÚrangeÚWorkerCoordinatorÚ_workersr   Úadd)r   Znum_worker_threadsÚworker_nameÚinit_funÚexternal_loggersr   Ú
worker_idsÚworkersr   )r   r   r   r   Úinit_workers3   s    

r)   c               @   s.   e Zd Zdd„ Zdd„ Zdd„ Zddd	„Zd
S )r   c             C   s   t jdd„ ƒ| _|| _d S )Nc               S   s   dS )Nr   r   r   r   r   r   Ú<lambda>Z   s    z"Metrics.__init__.<locals>.<lambda>)ÚcollectionsÚdefaultdictÚ_metricsÚ_external_loggers)Úselfr&   r   r   r   Ú__init__Y   s    zMetrics.__init__c             C   s   t jdd„ ƒ| _d S )Nc               S   s   dS )Nr   r   r   r   r   r   r*   ^   s    z'Metrics.reset_metrics.<locals>.<lambda>)r+   r,   r-   )r/   r   r   r   Úreset_metrics]   s    zMetrics.reset_metricsc             C   s`   | j s
d S xP| j D ]F}y|j| jƒ W q tk
rV } ztdj|ƒƒ W Y d d }~X qX qW d S )Nz!Failed to call ExternalLogger: {})r.   Úlogr-   Ú	ExceptionÚprintr   )r/   ÚloggerÚer   r   r   Úlog_metrics`   s    zMetrics.log_metricsTc             C   s6   | j |  |7  < |r2dj|ƒ}| j |  d7  < d S )Nz{}_counté   )r-   r   )r/   ÚkeyÚvalueÚcountZ	count_keyr   r   r   Ú
put_metrici   s    
zMetrics.put_metricN)T)Ú__name__Ú
__module__Ú__qualname__r0   r1   r7   r<   r   r   r   r   r   X   s   	r   c               @   s:   e Zd Zejeƒ edd„ ƒZedd„ ƒZedd„ ƒZ	dS )ÚStatec             C   s   d S )Nr   )r/   r   r   r   Ústarts   s    zState.startc             C   s   d S )Nr   )r/   r   r   r   Ústopw   s    z
State.stopc             C   s   d S )Nr   )r/   r   r   r   Úcleanup{   s    zState.cleanupN)
r=   r>   r?   ÚsixÚadd_metaclassr   r   rA   rB   rC   r   r   r   r   r@   p   s   
r@   c               @   sJ   e Zd Zddd„Zdd„ Zdd„ Zdd	„ Zdd
d„Zddd„Zdd„ Z	dS )r!   Nc             C   s4   d| _ d| _g | _|| _|| _|| _|| _|| _d S )NTF)Ú_activeÚ_startedr"   Ú_worker_nameÚ_worker_idsÚ	_init_funÚ_stateÚ_shutdown_fun)r/   r$   r'   r%   Ústater   r   r   r   r0   �   s    zWorkerCoordinator.__init__c             C   s   | j S )N)rF   )r/   r   r   r   Ú	is_activeŽ   s    zWorkerCoordinator.is_activec             C   s"   | j r| j r| }| j ||ƒ d S )N)rJ   rG   )r/   r   Zdata_coordinatorr   r   r   Úinit‘   s    zWorkerCoordinator.initc             C   sH   | j r
d S d| _d| _ | jr&| jjƒ  x| jD ]}d|_|jƒ  q.W d S )NT)rG   rF   rK   rA   r"   Údaemon)r/   Úwr   r   r   Ú_start–   s    
zWorkerCoordinator._startc             C   sL   d| _ |d k	rtjdj|ƒƒ | jr2| jr2| jƒ  | jrB| jjƒ  d| _d S )NFz%Data input failed due to an error: {})rF   r2   Úerrorr   rL   rG   rK   rB   )r/   Úreasonr   r   r   Ú_stop¢   s    
zWorkerCoordinator._stopc             C   sŠ   t dj| jƒƒ x$| jD ]}|tjƒ kr|jdƒ qW d}x(| jD ]}|jƒ rBt dj|ƒƒ d}qBW |rx| jrx| jj	ƒ  t dj|ƒƒ |S )NzWait for workers to die: {}g      @Tz'Worker {} failed to close while waitingFzAll workers terminated: {})
r4   r   rH   r"   r   Úcurrent_threadÚjoinÚisAliverK   rC   )r/   rC   rQ   Úsuccessr   r   r   Ú_wait_finish­   s    

zWorkerCoordinator._wait_finishc             C   s   | j S )N)rI   )r/   r   r   r   Úget_worker_ids¿   s    z WorkerCoordinator.get_worker_ids)NN)N)N)
r=   r>   r?   r0   rN   rO   rR   rU   rZ   r[   r   r   r   r   r!   €   s   


r!   c               @   sL   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dS )ÚGlobalWorkerCoordinatorc             C   s   g | _ d| _g | _| jƒ  d S )Nr   )Ú_coordinatorsÚ_fetcher_id_seqrI   Úregister_shutdown_handler)r/   r   r   r   r0   Ä   s    z GlobalWorkerCoordinator.__init__c             C   s   | j j|ƒ d S )N)r]   Úappend)r/   r   r   r   r   r#   Ê   s    zGlobalWorkerCoordinator.addc             C   s$   | j }| jj|ƒ |  j d7  _ |S )Nr8   )r^   rI   r`   )r/   r   r   r   r   r   Í   s    z)GlobalWorkerCoordinator.get_new_worker_idc             C   s   | j S )N)rI   )r/   r   r   r   r[   Ó   s    z&GlobalWorkerCoordinator.get_worker_idsc             C   s6   x| j D ]}|j| ƒ qW x| j D ]}|jƒ  q"W d S )N)r]   rO   rR   )r/   Úcr   r   r   rA   Ö   s    zGlobalWorkerCoordinator.startc             C   sF   d}x| j D ]}|jƒ  qW x| j D ]}|jƒ }|o6|}q$W g | _ |S )NT)r]   rU   rZ   )r/   Zall_successra   rY   r   r   r   rB   Þ   s    zGlobalWorkerCoordinator.stopc                sD   x(| j D ]}|jˆ kr|jƒ  |jƒ  qW ‡ fdd„| j D ƒ| _ dS )z-
        Stop a specific coordinator
        c                s   g | ]}|j ˆ kr|‘qS r   )rH   )r   ra   )r$   r   r   r   ñ   s    z<GlobalWorkerCoordinator.stop_coordinator.<locals>.<listcomp>N)r]   rH   rU   rZ   )r/   r$   ra   r   )r$   r   Ústop_coordinatorè   s
    
z(GlobalWorkerCoordinator.stop_coordinatorc                s   ‡ fdd„}t j|ƒ d S )Nc                  s   ˆ j ƒ  d S )N)rB   r   )r/   r   r   rC   ö   s    zBGlobalWorkerCoordinator.register_shutdown_handler.<locals>.cleanup)ÚatexitÚregister)r/   rC   r   )r/   r   r_   õ   s    z1GlobalWorkerCoordinator.register_shutdown_handlerN)r=   r>   r?   r0   r#   r   r[   rA   rB   rb   r_   r   r   r   r   r\   Ã   s   
r\   c               @   s6   e Zd Zddd„Zdd„ Zdd„ Zdd	„ Zd
d„ ZdS )r   Nc             C   s   || _ || _|| _|| _d S )N)Ú_coordinatorÚ
_worker_idÚ_worker_funr-   )r/   r   r   r   r   r   r   r   r0   ý   s    zWorker.__init__c             C   s   t j ƒ | _d S )N)ÚtimeÚ_start_time)r/   r   r   r   rA   	  s    zWorker.startc             C   s   | j | jƒ d S )N)rg   rf   )r/   r   r   r   Úrun  s    z
Worker.runc             C   s.   t jƒ  tjd|ƒ | jjdj| j|ƒƒ d S )NzException in workerzException in worker {}: {})Ú	tracebackÚ	print_excÚloggingÚ	exceptionre   rU   r   rf   )r/   r6   r   r   r   Úhandle_exception  s    
zWorker.handle_exceptionc             C   s&   | j jdtjƒ | j ƒ | j jƒ  d S )NZworker_time)r-   r<   rh   ri   r7   )r/   r   r   r   Úfinish  s    zWorker.finish)NN)r=   r>   r?   r0   rA   rj   ro   rp   r   r   r   r   r   ü   s    
r   c             C   sd   x^| j ƒ r^|jƒ  z>y|jƒ  W n, tk
rL } z|j|ƒ W Y d d }~X nX W d |jƒ  X qW d S )N)rN   rA   rj   r3   ro   rp   )r   Zworkerr6   r   r   r   r     s    
 r   )r	   r
   NNN)Ú
__future__r   r   r   r   rm   r   rc   rh   r+   rD   rk   Úabcr   r   Ú	getLoggerr2   ÚsetLevelÚINFOZLOG_INT_SECSr)   Úobjectr   r@   r!   r\   r   r   r   r   r   r   r   Ú<module>   s4   
    
C9 