3
I4W^D  ã               @   s  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZ d dlm	Z	m
Z
 d dlmZmZ d dlmZmZ d d	lmZmZmZ G d
d„ deƒZdZdd„ Zd%dd„Zdd„ Zd&dd„Zd'dd„Zdd„ Zdd„ Zdd„ Zd(dd „Z G d!d"„ d"e	ƒZ!G d#d$„ d$eƒZ"dS ))é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚcoreÚ
queue_util)ÚReaderÚWriter)Ú
NetBuilderÚops)Ú	as_recordÚField)ÚNodeÚTaskÚ	TaskGroupc               @   s   e Zd ZdZddd„ZdS )ÚOutputz¿
    Represents the result of a processor function. A processor can either
    return an Output, or it can return a record, in which case an Output will be
    created for it afterwards.
    Nc             C   s|   t jƒ jƒ }|d ks(t|ƒdks(tdƒ‚|d kr4|}t|tjƒrF|g}|d krRg nt|ƒ| _	|d krhd nt
|ƒ| _|| _d S )Nr   z7Cannot both use `ops` syntax and return a list of nets.)r
   ÚcurrentÚgetÚlenÚAssertionErrorÚ
isinstancer   ÚNetÚlistÚnetsr   ÚrecordÚshould_stop)Úselfr   r   r   Zbuilder_children© r   ú7/tmp/pip-build-e66j97b2/torch/caffe2/python/pipeline.pyÚ__init__   s    zOutput.__init__)NNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r      s   r   éd   c             C   s’   | d kr*t j|d k	r|ntd�}|jƒ }nTt| tƒrN|d ksDtdƒ‚d }| }n0t| dƒrv|d kshtdƒ‚| }| jƒ }ntdƒ‚|j	||ƒ ||fS )N)Úcapacityzcapacity would not be used.Úwriterz)output must be a reader, queue or stream.)
r   ÚQueueÚDEFAULT_QUEUE_CAPACITYr&   r   r	   r   ÚhasattrÚ
ValueErrorÚsetup_ex)Úoutputr%   Úglobal_init_netÚglobal_exit_netÚ	out_queuer&   r   r   r   Ú_init_output%   s     



r0   Nc                sR   ˆ d krdd„ S t ˆ tjƒr$tˆ ƒS ˆd k	rJtˆ dƒrJ‡ ‡fdd„}|ˆ _ˆ S d S )Nc             S   s   | S )Nr   )Úrecr   r   r   Ú<lambda><   s    z make_processor.<locals>.<lambda>Úschema_funcc                  s
   ˆ j ˆƒS )N)r3   r   )Ú	processorÚreaderr   r   Úprocessor_schemaA   s    z(make_processor.<locals>.processor_schema)r   r   r   ÚNetProcessorr)   Úschema)r4   r5   r6   r   )r4   r5   r   Úmake_processor:   s    r9   c             C   s|   t | tƒr| S t | tƒr"t| d�S t | tƒrpt| ƒdkoTt | d tƒoTt | d tjƒ}|rftd| žŽ S t| Ž S nt| ƒS dS )z‰
    Allow for processors to return results in several formats.
    TODO(azzolini): simplify once all processors use NetBuilder API.
    )r   é   r   é   N)N)r   r   r   Útupler   r   ÚBlobReference)r,   Zis_record_and_blobr   r   r   Únormalize_processor_outputH   s    




r>   r;   c       
   	   C   s   t | |||||||ƒ\}}	|S )a[
  
    Given a Reader, Queue or DataStream in `input`, and optionally, a Writer,
    Queue or DataStream in `output`, creates a Task that, when run, will
    pipe the input into the output, using multiple parallel threads.
    Additionally, if a processor is given, it will be called between reading
    and writing steps, allowing it to transform the record.

    Args:
        input:       either a Reader, Queue or DataStream that will be read
                     until a stop is signaled either by the reader or the
                     writer.
        output:      either a Writer, a Queue or a DataStream that will be
                     writen to as long as neither reader nor writer signal
                     a stop condition. If output is not provided or is None,
                     a Queue is created with given `capacity` and writen to.
        num_threads: number of concurrent threads used for processing and
                     piping. If set to 0, no Task is created, and a
                     reader is returned instead -- the reader returned will
                     read from the reader passed in and process it.
                     ** DEPRECATED **. Use `num_runtime_threads` instead.
                     This option will be removed once all readers/processors
                     support `num_runtime_threads`.
        processor:   (optional) function that takes an input record and
                     optionally returns a record; this will be called
                     between read and write steps. If the processor does
                     not return a record, a writer will not be instantiated.
                     Processor can also be a core.Net with input and output
                     records properly set. In that case, a NetProcessor is
                     instantiated, cloning the net for each of the threads.
        name:        (optional) name of the task to be created.
        capacity:    when output is not passed, a queue of given `capacity`
                     is created and written to.
        group:       (optional) explicitly add the created Task to this
                     TaskGroup, instead of using the currently active one.
        num_runtime_threads: Similar to `num_threads`, but instead of expanding
                     the tasks with a `for` loop in python, does that at
                     runtime. This is preferable to `num_threads`, but some
                     processors/readers still require to be called multiple
                     times in python.

    Returns:
        Output Queue, DataStream, Reader, or None, depending on the parameters
        passed.
    )Ú
_pipe_step)
Úinputr,   Únum_threadsr4   Únamer%   ÚgroupÚnum_runtime_threadsÚresultÚ_r   r   r   Úpipec   s    /
rG   c	          
   C   s\   |dkst ‚t| ||||||||ƒ	\}	}
d}|dk	rT|
jƒ }t|ƒttfkrT|d }|	|fS )aq  
    Similar to `pipe`, with the additional ability for the pipe Task to
    return output values to the `Session` once done.

    Returns:
        Tuple (out_queue, *task_outputs)
            out_queue:    same as return value of `pipe`.
            task_outputs: TaskOutput object, fetchable from the client after
                          session.run() returns.
    r   N)r   r?   ÚoutputsÚtyper   r<   )r@   r,   rA   r4   rB   r%   rC   rD   Úfinal_outputsrE   Útaskr   r   r   Úpipe_and_output˜   s    rL   c             C   sT   t | dƒr| jS t | dƒrL| jdkr*| jS t | dƒrFd| jj| jf S | jS | jjS )NrB   Ú	func_namez<lambda>Úim_classz%s.%s)r)   rB   rM   r!   rN   r    Ú	__class__)r4   r   r   r   Úprocessor_name±   s    



rP   c          %   C   sÒ  t tjƒ ƒ}dj|d| tr"ttƒnd|r0t|ƒndƒ}t| |||d���|}	tjdƒ}
tjdƒ}|j	||
ƒ tjdƒ}tjd	ƒ}|j
||ƒ\}}}|jg |gg d
tjjd� |d k	rÚt||||
ƒ\}}|j||||ƒ\}}nd }g }tjƒ � tj|ƒ W d Q R X tjƒ � tj|ƒ W d Q R X tjdƒ}|jg |d�}tjdƒ}|j|g ƒ tjtjd|gt|ƒ t|ƒ |g |d�ƒ tj|ƒ tjƒ � tj|ƒ W d Q R X tjƒ � tj|
ƒ W d Q R X W d Q R X ||	fS )Nz{0}/{1}/{2}/{3}/{4}rG   ÚNoInputÚNoOutput)rB   rC   rH   Znum_instancesz	pipe:exitz	pipe:initzpipe:instance:initzpipe:instance:exitF)ÚshapeÚvalueÚdtypeÚtimer_start)Úcounter_nameÚ	timer_endÚbody)Úshould_stop_blob)Ústrr   r   Úformatr@   rP   r   r   r   r+   Úread_record_exÚConstantFillÚDataTypeÚBOOLr0   Úwrite_record_exr   Z	task_initÚnetZtask_instance_initÚ
TimerBeginÚTimerEndÚexecution_stepr   Ztask_instance_exitZ	task_exit)rB   rC   rJ   r5   rA   r,   r%   Ú	node_nameÚprofiler_namerK   r.   r-   Úinit_netÚexit_netÚ	read_netsÚstatusr1   r/   r&   Ú
write_netsrF   Útimer_start_netÚtimerÚtimer_end_netr   r   r   Ú_runtime_threads_task½   s\    











rp   c             C   s  t tjƒ ƒ}dj|d| tr"ttƒnd|r0t|ƒndƒ}t| ||d���°}	tjdƒ}
tjdƒ}|j	||
ƒ d }d }g }�xPt
|ƒD �]B}td| d	���}tjdƒ}tjdƒ}|j||ƒ\}}}|jg |gg d
tjjd� |d k	�r,|d k�rt|	jd�� t||||
ƒ\}}W d Q R X |j||||ƒ\}}ng }tjdƒ}|jg |d�}tjdƒ}|j|g ƒ tj|ƒ tjtjd|gt|ƒ t|ƒ |g |d�ƒ tj|ƒ tj|ƒ W d Q R X |jtj|ƒƒ q€W tj|ƒ tjtjd|dd�ƒ tj|
ƒ W d Q R X ||	fS )Nz{0}/{1}/{2}/{3}/{4}rG   rQ   rR   )rB   rC   rH   ÚexitÚinitzt:%d)rB   F)rS   rT   rU   )Z	_fullnamerV   )rW   rX   rY   )rZ   T)Zconcurrent_substeps)r[   r   r   r\   r@   rP   r   r   r   r+   Úranger
   r]   r^   r_   r`   rB   r0   ra   rc   rd   r   rb   re   r   ÚappendZto_execution_step)rB   rC   rJ   r5   rA   r,   r%   rf   rg   rK   r.   r-   r/   r&   ZstepsZ	thread_idÚnbrh   ri   rj   rk   r1   rl   rF   rm   rn   ro   r   r   r   Ú_static_threads_taskù   sf    











rv   c	       
      C   s   |dks|dkst dƒ‚t| tƒr(| }	n&t| dƒr<| jƒ }	ntdjt| ƒƒƒ‚|dk	r`t|	|ƒ}	|dksp|dkr„|dks|t ‚|	dfS |dkrœ|dk	rœt	|ƒ}|dkr¸|dk	r¸dt	|ƒ }|dkrÌdt	| ƒ }|dkrèt
||||	|||ƒS t||||	|||ƒS dS )	z
    r;   z;Only one of num_threads or num_runtime_threads must be set.r5   z/Input must be a reader, queue or stream. Got {}Nr   zpipe_into:%szpipe_from:%s)r   r   r   r)   r5   r*   r\   rI   ÚProcessingReaderrP   rv   rp   )
r@   r,   rA   r4   rB   r%   rC   rD   rJ   r5   r   r   r   r?   :  s2    



r?   c               @   s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )rw   zo
    Reader that reads from an upstream reader, calls the processor, and returns
    the processed record.
    c             C   s    t j| ƒ || _t||ƒ| _d S )N)r   r   r5   r9   r4   )r   r5   r4   r   r   r   r   f  s    
zProcessingReader.__init__c             C   s
   | j jƒ S )N)r4   r8   )r   r   r   r   r8   k  s    zProcessingReader.schemac             C   s   | j j||ƒ d S )N)r5   r+   )r   rh   Z
finish_netr   r   r   r+   n  s    zProcessingReader.setup_exc       
   
   C   sØ   | j j||ƒ\}}}tƒ �}t| j|ƒƒ}W d Q R X ||j7 }|jsJ|jr’tj	dƒ}|jrn|j
||jg|gƒ |jrˆ|j
||jg|gƒ |j|ƒ t| jdƒr®|jtj| jƒ | j|jƒ |jrÊ|jjƒ nd }	|||	fS )NÚstop_netÚsetup)r5   r]   r
   r>   r4   r   r   Z
_stop_blobr   r   ÚOrrt   r)   Zadd_attributer   ZLOCAL_SETUPZ_set_schemar   Zfield_blobs)
r   rh   ri   rj   rk   r1   ru   rE   rx   Úfieldsr   r   r   Úread_exq  s     


zProcessingReader.read_exN)r    r!   r"   r#   r   r8   r+   r|   r   r   r   r   rw   a  s
   rw   c               @   s:   e Zd ZdZddd„Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )r7   zø
    Processor that clones a core.Net each time it's called, executing
    the cloned net as the processor. It requires the Net to have input
    and (optionally) output records set, with net.set_input_record() and
    net.set_output_record().
    Nc             C   sb   t |tjƒst‚|d ks(t |tjƒs(t‚|p2t|ƒ| _|p<g | _|| _|| _	g | _
d| _g | _d S )NF)r   r   r   r   r=   r[   rB   Úthread_init_netsrb   Ú_stop_signalÚ
_blob_mapsÚ_frozenÚ_cloned_init_nets)r   rb   Ústop_signalr}   rB   r   r   r   r   ”  s    

zNetProcessor.__init__c             C   s
   | j jƒ S )N)rb   Úoutput_record)r   r   r   r   r8      s    zNetProcessor.schemac             C   s   d| _ | j}g | _|S )NT)r€   r�   )r   rh   Zcloned_init_netsr   r   r   ry   £  s    zNetProcessor.setupc       	      C   sÔ   | j  st‚tjƒ jd }i }x6| jD ],}tj|t|ƒ| ||ƒ\}}| j	j
|ƒ q&W tj| jt| jƒ| |||ƒ\}}| jd krˆd }n.t| jƒ|kr°tj|t| jƒ |d�}n| j}| jj
|ƒ t|g|jƒ |ƒS )Nú/)rb   )r€   r   r
   r   rB   r}   r   Zclone_and_bind_netr[   r�   rt   rb   r~   r=   r   r   rƒ   )	r   r1   ÚprefixZ
blob_remaprb   Znew_netrF   Z
remappingsr‚   r   r   r   Ú__call__©  s$    

zNetProcessor.__call__c             C   s   d| _ | jS )NT)r€   r   )r   r   r   r   Ú	blob_mapsÁ  s    zNetProcessor.blob_maps)NNN)	r    r!   r"   r#   r   r8   ry   r†   r‡   r   r   r   r   r7   �  s   
r7   )N)Nr;   NNNNr;   )Nr;   NNNNr;   N)Nr;   NNNNNN)#Ú
__future__r   r   r   r   Zcaffe2.pythonr   r   Zcaffe2.python.dataior   r	   Zcaffe2.python.net_builderr
   r   Zcaffe2.python.schemar   r   Zcaffe2.python.taskr   r   r   Úobjectr   r(   r0   r9   r>   rG   rL   rP   rp   rv   r?   rw   r7   r   r   r   r   Ú<module>   s0   
 
4 
<B 
%,