3
I4W^P?  ã               @   sv  d dl mZ d dl mZ d dl mZ d dl mZ d dlmZmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZ d dlmZmZmZ d dlmZ d dlm Z  d dl!m"Z"m#Z#m$Z$ d dl%m&Z& d dl'Z(d dl)j*Z+d dl,Z,d dl-Z-d dl.Z.d dl/Z/d dl0Z0ddd„Z1ddd„Z2G dd„ de	ƒZ3G dd„ deƒZ4G dd„ deƒZ5G dd„ deƒZ6dS ) é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literals)ÚCompositeReaderÚCompositeReaderBuilderÚReaderÚReaderBuilderÚReaderWithDelayÚReaderWithLimitÚReaderWithTimeLimit)ÚDataset)ÚDBFileReader)Úpipe)ÚStructÚ	NewRecordÚ
FeedRecord)ÚLocalSession)Ú	TaskGroupÚfinal_outputÚWorkspaceType)ÚTestCase)ÚCachedReader)ÚcoreÚ	workspaceÚschema)ÚopsNéd   c             C   sz   |pd}t jdj|ƒƒ}t j|ƒ�D tdtjt||| ƒƒfƒ}t||ƒ}t	||d�}t
||| ƒ W d Q R X | j|ƒ |S )NÚsrcz{}_initÚlabel)Úname)r   ÚNetÚformatÚ	NameScoper   ÚnpÚarrayÚranger   r   r   Úrun)ÚwsÚsizeÚoffsetr    Zsrc_initZ
src_valuesZ	src_blobsÚsrc_ds© r,   ú:/tmp/pip-build-e66j97b2/torch/caffe2/python/dataio_test.pyÚmake_source_dataset#   s    

r.   c             C   sR   |pd}t jdj|ƒƒ}t j|ƒ� t||d�}|j|ƒ W d Q R X | j|ƒ |S )NÚdstz{}_init)r    )r   r!   r"   r#   r   Z
init_emptyr'   )r(   r   r    Zdst_initÚdst_dsr,   r,   r-   Úmake_destination_dataset/   s    
r1   c               @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
ÚTestReaderBuilderc             C   s0   t jdt jƒ fƒ| _|| _|| _|| _d | _d S )Nr   )r   r   ZScalarÚ_schemaÚ_nameÚ_sizeÚ_offsetÚ_src_ds)Úselfr    r)   r*   r,   r,   r-   Ú__init__:   s    zTestReaderBuilder.__init__c             C   s   | j S )N)r3   )r8   r,   r,   r-   r   C   s    zTestReaderBuilder.schemac             C   s   t || j| j| jd�| _i S )N)r*   r)   r    )r.   r6   r5   r4   r7   )r8   r(   r,   r,   r-   ÚsetupF   s    zTestReaderBuilder.setupc             K   s   | j S )N)r7   )r8   Úkwargsr,   r,   r-   Ú
new_readerK   s    zTestReaderBuilder.new_readerN)Ú__name__Ú
__module__Ú__qualname__r9   r   r:   r<   r,   r,   r,   r-   r2   9   s   	r2   c               @   sD   e Zd Zejejjdƒdƒdd„ ƒZejejjdƒdƒdd„ ƒZ	dS )ÚTestCompositeReaderZJENKINS_URLzFlaky test on Jenkinsc                sT  t jjƒ ‰tˆƒ}d}dd„ t|ƒD ƒ}d‰ ‡ fdd„t|ƒD ƒ}‡ ‡fdd„t||ƒD ƒ}‡fdd„|D ƒ}x,t||ƒD ]\}}tj|t||ˆ  ƒƒ qzW tj	dd„ t||ƒD ƒŽ }	t
ˆ|	ƒ}
tƒ �,}t|d	d„ |D ƒƒ}t||
jƒ dd
� W d Q R X |j|ƒ xLt|ƒD ]@}tˆjt|
jƒ ||  jƒ ƒƒƒ}tj|| |dj|ƒƒ �qW d S )Né   c             S   s   g | ]}d j |ƒ‘qS )zsrc_{})r"   )Ú.0Úir,   r,   r-   ú
<listcomp>U   s    z=TestCompositeReader.test_composite_reader.<locals>.<listcomp>r   c                s   g | ]}|ˆ  ‘qS r,   r,   )rB   rC   )r)   r,   r-   rD   W   s    c                s    g | ]\}}t ˆ|ˆ |d �‘qS ))r*   r)   r    )r.   )rB   r    r*   )r)   r(   r,   r-   rD   X   s   c                s    g | ]}ˆ j t|jd  ƒƒ‘qS )r   )Ú
fetch_blobÚstrZfield_blobs)rB   r   )r(   r,   r-   rD   [   s    c             S   s    g | ]\}}||j ƒ jƒ f‘qS r,   )ÚcontentÚclone_schema)rB   r    r+   r,   r,   r-   rD   c   s   c             S   s   g | ]}|j ƒ ‘qS r,   )Úreader)rB   r+   r,   r,   r-   rD   k   s    )Únum_runtime_threadszi: {})r   ÚCÚ	Workspacer   r&   ÚzipÚnptÚassert_array_equalr   r   r1   r   r   r   Úwriterr'   ÚsortedrE   rF   rG   r   r"   )r8   ÚsessionÚnum_srcsÚnamesÚoffsetsZsrc_dsesÚdataÚdr*   Údst_ds_schemar0   ÚtgrI   rC   Úwritten_datar,   )r)   r(   r-   Útest_composite_readerP   s0    


 z)TestCompositeReader.test_composite_readerc                s  t jjƒ }t|ƒ}d}dd„ t|ƒD ƒ}d‰ ‡ fdd„t|ƒD ƒ}‡ fdd„t||ƒD ƒ}tjdd„ t||ƒD ƒŽ }t||ƒ}t	ƒ �2}	t
||ƒ}
|
j|d� t|
jƒ |jƒ dd	� W d Q R X |j|	ƒ xRt||ƒD ]D\}}t|jt|jƒ | jƒ ƒƒƒ}tjt||ˆ  ƒ|d
j|ƒƒ qÒW d S )NrA   c             S   s   g | ]}d j |ƒ‘qS )zsrc_{})r"   )rB   rC   r,   r,   r-   rD   y   s    zETestCompositeReader.test_composite_reader_builder.<locals>.<listcomp>r   c                s   g | ]}|ˆ  ‘qS r,   r,   )rB   rC   )r)   r,   r-   rD   {   s    c                s   g | ]\}}t |ˆ |d �‘qS ))r*   r)   r    )r2   )rB   r    r*   )r)   r,   r-   rD   }   s   c             S   s   g | ]\}}||j ƒ f‘qS r,   )r   )rB   r    Zsrc_ds_builderr,   r,   r-   rD   „   s   )r(   )rJ   zname: {})r   rK   rL   r   r&   rM   r   r   r1   r   r   r:   r   r<   rP   r'   rQ   rE   rF   rG   r   rN   rO   r"   )r8   r(   rR   rS   rT   rU   Zsrc_ds_buildersrX   r0   rY   Zreader_builderr    r*   rZ   r,   )r)   r-   Útest_composite_reader_buildert   s0    



z1TestCompositeReader.test_composite_reader_builderN)
r=   r>   r?   ÚunittestZskipIfÚosÚenvironÚgetr[   r\   r,   r,   r,   r-   r@   O   s   $r@   c               @   s\   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ ZdS )ÚTestReaderWithLimitc                s,  t jjƒ }t|ƒ}t|ƒ}d gd ‰ ‡ fdd„}tƒ �}t|jƒ d|d� W d Q R X |j|ƒ | j	ˆ d j
ƒ dƒ | j	ˆ d j
ƒ dƒ | j	ˆ d	 j
ƒ dƒ tƒ �<}t|jƒ d	d
�}tt|jƒ dd�dd
�}t||dd� W d Q R X |j|ƒ | j	ˆ d j
ƒ dƒ | j	ˆ d j
ƒ dƒ | j	ˆ d	 j
ƒ dƒ d S )NrA   c          &      s  t jƒ �0 t jg dgƒ}t jg dgƒ}t jg dgƒ}W d Q R X t jƒ � t jg dgƒ}W d Q R X t j|ƒ t j|ƒ t jƒ �6 t jt j|ƒƒ� t j|ƒ W d Q R X t j|ƒ W d Q R X t jƒ �< t	t j|ƒƒˆ d< t	t j|ƒƒˆ d< t	t j|ƒƒˆ d< W d Q R X | S )NZglobal_counterZglobal_counter2Zglobal_counter3Útask_counterr   é   é   )
r   Z	task_initZCreateCounterZtask_instance_initZCountUpZtask_instance_exitZloopZRetrieveCountZ	task_exitr   )ZrecZcounter1Zcounter2Zcounter3rb   )Útotalsr,   r-   Úproc    s"    





z6TestReaderWithLimit.test_runtime_threads.<locals>.procé   )rJ   Ú	processorr   r   rc   rd   )rJ   é   )Únum_iteré   )rh   rJ   )r   rK   rL   r   r.   r   r   rI   r'   ÚassertEqualÚfetchr   )r8   r(   rR   r+   rf   rY   Zq1Zq2r,   )re   r-   Útest_runtime_threadsš   s*    



z(TestReaderWithLimit.test_runtime_threadsc             C   s<   t jjƒ }t|ƒ}t||d�}t||jƒ jƒ ƒ}||||fS )N)r)   )r   rK   rL   r   r.   r1   rG   rH   )r8   r)   r(   rR   r+   r0   r,   r,   r-   Ú_test_limit_reader_init_sharedÌ   s
    
z2TestReaderWithLimit._test_limit_reader_init_sharedc             K   sô   | j |ƒ\}}	}
}ttjd��H}|dkr@|t|
jƒ |ƒf|Ž}n||
jƒ f|Ž}t||jƒ |d� W d Q R X |	j|ƒ t	t
|jt|jƒ jƒ ƒ jƒ ƒƒ}| j||ƒ | jt
|jt|jƒ jƒ ƒ jƒ ƒtt|ƒƒƒ | j|jt|jƒ ƒ jƒ |ƒ d S )N)Zworkspace_typer   )rJ   )ro   r   r   ÚGLOBALr
   rI   r   rP   r'   ÚlenrQ   ÚblobsrF   rG   r   rm   rl   Úlistr&   Zdata_finished)r8   Zreader_classr)   Úexpected_read_lenÚexpected_finishÚnum_threadsÚ
read_delayZlimiter_argsr(   rR   r+   r0   rY   rI   Zread_lenr,   r,   r-   Ú_test_limit_reader_sharedØ   s     

"z-TestReaderWithLimit._test_limit_reader_sharedc          	   C   s   | j tdddddd d� d S )Nr   Trg   r   )r)   rt   ru   rv   rw   rj   )rx   r   )r8   r,   r,   r-   Ú%test_count_limit_reader_without_limitó   s    z9TestReaderWithLimit.test_count_limit_reader_without_limitc          	   C   s   | j tddddddd� d S )Nr   r   Frg   )r)   rt   ru   rv   rw   rj   )rx   r   )r8   r,   r,   r-   Ú'test_count_limit_reader_with_zero_limitý   s    z;TestReaderWithLimit.test_count_limit_reader_with_zero_limitc          	   C   s   | j tddddddd� d S )Nr   é
   Frg   r   )r)   rt   ru   rv   rw   rj   )rx   r   )r8   r,   r,   r-   Ú&test_count_limit_reader_with_low_limit  s    z:TestReaderWithLimit.test_count_limit_reader_with_low_limitc          	   C   s   | j tddddddd� d S )Nr   Trg   r   én   )r)   rt   ru   rv   rw   rj   )rx   r   )r8   r,   r,   r-   Ú'test_count_limit_reader_with_high_limit  s    z;TestReaderWithLimit.test_count_limit_reader_with_high_limitc          	   C   s   | j tddddddd� d S )Nr   Trg   gš™™™™™¹?r   )r)   rt   ru   rv   rw   Úduration)rx   r   )r8   r,   r,   r-   Ú$test_time_limit_reader_without_limit  s    z8TestReaderWithLimit.test_time_limit_reader_without_limitc          	   C   sL   d}d}d}d}t t|| | ƒƒ}|d|  }| jt||d|||d� d S )Né2   é   g      Ð?rc   F)r)   rt   ru   rv   rw   r   )ÚintÚroundrx   r   )r8   r)   rv   Zsleep_durationr   rt   r,   r,   r-   Ú'test_time_limit_reader_with_short_limit%  s    z;TestReaderWithLimit.test_time_limit_reader_with_short_limitc          	   C   s   | j tddddddd� d S )Nr�   Tr‚   g      Ð?rk   )r)   rt   ru   rv   rw   r   )rx   r   )r8   r,   r,   r-   Ú&test_time_limit_reader_with_long_limit7  s    z:TestReaderWithLimit.test_time_limit_reader_with_long_limitN)r=   r>   r?   rn   ro   rx   ry   rz   r|   r~   r€   r…   r†   r,   r,   r,   r-   ra   ™   s   2




ra   c               @   sX   e Zd Zdd„ Zdd„ Zedd„ ƒZdd„ Zed	d
„ ƒZedd„ ƒZ	dd„ Z
dd„ ZdS )ÚTestDBFileReaderc             C   s
   g | _ d S )N)Ú
temp_paths)r8   r,   r,   r-   ÚsetUpC  s    zTestDBFileReader.setUpc             C   s   x| j D ]}| j|ƒ qW d S )N)rˆ   Ú_delete_path)r8   Úpathr,   r,   r-   ÚtearDownF  s    zTestDBFileReader.tearDownc             C   s2   t jj| ƒrt j| ƒ nt jj| ƒr.tj| ƒ d S )N)r^   r‹   ÚisfileÚremoveÚisdirÚshutilÚrmtree)r‹   r,   r,   r-   rŠ   K  s    zTestDBFileReader._delete_pathc          	   C   s*   t jƒ �}|j}W d Q R X | jj|ƒ |S )N)ÚtempfileÚNamedTemporaryFiler    rˆ   Úappend)r8   ÚfZ	temp_pathr,   r,   r-   Ú_make_temp_pathR  s    
z TestDBFileReader._make_temp_pathc             C   s   t | |ƒ}|jƒ S )N)r.   rI   )r(   r)   r+   r,   r,   r-   Ú_build_source_readerY  s    
z%TestDBFileReader._build_source_readerc             C   sZ   t | |jƒ jƒ ƒ}tƒ �}t||jƒ dd� W d Q R X |j|ƒ | jt|j	ƒ j
ƒ ƒ jƒ S )Nrg   )rJ   )r1   r   rH   r   r   rP   r'   rr   rF   rG   r   rm   )r(   rI   rR   r0   rY   r,   r,   r-   Ú_read_all_data^  s
    
zTestDBFileReader._read_all_datac       	      C   s  t jjƒ }t|ƒ}| jƒ }t| j|dƒ|dd�}|jƒ }|j|ƒ | j	|||ƒ}| j
t|ƒttdƒƒƒ t| j|dƒ|ƒ}|jƒ }|j|ƒ | j	|||ƒ}| j
t|ƒttdƒƒƒ | j|ƒ t| j|dƒ|ƒ}|jƒ }|j|ƒ | j	|||ƒ}| j
t|ƒttdƒƒƒ | j|ƒ d S )Nr   F)Z	loop_overéÈ   i,  )r   rK   rL   r   r–   r   r—   Úbuild_cache_stepr'   r˜   rl   rQ   rs   r&   rŠ   )	r8   r(   rR   Údb_pathZcached_reader1rš   rV   Zcached_reader2Zcached_reader3r,   r,   r-   Útest_cached_readerh  s.    




z#TestDBFileReader.test_cached_readerc             C   s‚   t jjƒ }t|ƒ}| jƒ }t| j|dƒ|dd�}|jƒ }|j|ƒ t	|dd�}| j
|||ƒ}| jt|ƒttdƒƒƒ | j|ƒ d S )Nr   ZLevelDB)r›   Zdb_type)r   rK   rL   r   r–   r   r—   rš   r'   r   r˜   rl   rQ   rs   r&   rŠ   )r8   r(   rR   r›   Zcached_readerrš   Zdb_file_readerrV   r,   r,   r-   Útest_db_file_reader�  s    


z$TestDBFileReader.test_db_file_readerN)r=   r>   r?   r‰   rŒ   ÚstaticmethodrŠ   r–   r—   r˜   rœ   r�   r,   r,   r,   r-   r‡   B  s   
'r‡   )r   r   N)N)7Ú
__future__r   r   r   r   Zcaffe2.python.dataior   r   r   r	   r
   r   r   Zcaffe2.python.datasetr   Zcaffe2.python.db_file_readerr   Zcaffe2.python.pipeliner   Zcaffe2.python.schemar   r   r   Zcaffe2.python.sessionr   Zcaffe2.python.taskr   r   r   Zcaffe2.python.test_utilr   Zcaffe2.python.cached_readerr   Zcaffe2.pythonr   r   r   Zcaffe2.python.net_builderr   Znumpyr$   Znumpy.testingZtestingrN   r^   r�   r]   r’   Útimer.   r1   r2   r@   ra   r‡   r,   r,   r,   r-   Ú<module>   s8   $	



J *