3
I4W^aa ã               @   s®  d dl mZ d dl mZ d dl mZ d dlZd dlZd dlZd dlm	Z	m
Z
 d dlmZmZ d dlmZmZmZmZ d dljZd dlZd dlZd dlmZmZmZmZ d dljjZ d d	l!m"Z" ej#d
ƒ dd„ Z$ej%ddd„ƒZ&ddej'dfdd„Z(ej'ej)j*ej+ej)j,ej-ej)j.ej/ej)j0ej1ej)j2ej3ej)j4ej5ej)j6ej7ej)j8ej9ej)j:i	Z;ddd„Z<de j=dde<fdd„Z>de j=de<fdd„Z?G dd„ de j@ƒZAeBdk�rªejCƒ  dS )é    )Úabsolute_import)Údivision)Úprint_functionN)ÚpartialÚreduce)Ú	viewitemsÚviewkeys)ÚassumeÚgivenÚsettingsÚHealthCheck)ÚcoreÚ	workspaceÚtt_coreÚdyndep)Ú
caffe2_pb2z*@/caffe2/caffe2/fb/optimizers:sgd_simd_opsc             C   s   ddt j|  ƒ  S )Ng      ð?)ÚnpÚexp)Úx© r   ú>/tmp/pip-build-e66j97b2/torch/caffe2/python/hypothesis_test.pyÚsigmoid   s    r   é   é   c             K   sx   | t jtjf |Ž||d�ƒ}| t jtjf |Ž||d�ƒ}tt|ƒt|ƒ |k ƒ | tj|| ||ƒƒ| tj|||ƒƒfS )N)Úmin_sizeÚmax_size)ÚstÚlistsÚhuÚdimsr	   ÚlenÚarrays)ZdrawÚdtypeÚelementsÚmin_dimÚmax_dimÚkwargsZdims_Zextra_r   r   r   Ú_tensor_and_prefix   s    r'   c                sB   t jtjf |Ž| |d�}t jddd�}t j||ƒj‡ ‡fdd„ƒS )zF generates a tensor and a list of indices of larger tensor of same dim)r   r   é   é
   )Ú	min_valueÚ	max_valuec          
      sN   t jt j| d ƒtj| d ˆ ˆƒtj| d d tjt jd| d d d�d�ƒS )Nr   r   )r*   r+   )r"   r#   )r   ÚtuplesÚjustr   r!   r   Úint64Úintegers)Úpair)r"   r#   r   r   Ú<lambda>*   s   z%_tensor_and_indices.<locals>.<lambda>)r   r   r   r   r/   r,   Úflatmap)r$   r%   r"   r#   r&   Z
data_dims_Zoriginal_dimr   )r"   r#   r   Ú_tensor_and_indices%   s    r3   c             C   s"   | r| nt jt jt jg} tj| ƒS )N)r   Úint32r.   Úfloat32r   Úsampled_from)Údtypesr   r   r   Ú_dtypes?   s    r8   Fc                sR   t f |ƒ j‡ fdd„ƒtj|r"dndƒdœ|—Žtddd	�‡‡‡fd
d„ƒƒ}|S )Nc                s   t jd| t j| ˆ d�d�S )Nr(   )Úfilter_)Únr"   r#   )r   ÚtensorsÚelements_of_type)r"   )r9   r   r   r1   H   s   z_test_binary.<locals>.<lambda>ÚYÚX1ÚX2)ÚinputsÚouté   éd   )Úmax_examplesÚtimeoutc                sr   t jˆ ddg|gƒ}|\}}| j||||gdgƒ ˆrZ|jtjkrZ| j||||gddgƒ | j||||gˆƒ d S )Nr>   r?   r   )r   ÚCreateOperatorÚassertDeviceChecksr"   r   r5   ÚassertGradientChecksÚassertReferenceChecks)Úselfr@   rA   ÚgcÚdcÚopr>   r?   )ÚnameÚrefÚtest_gradientr   r   Útest_binaryF   s    	z!_test_binary.<locals>.test_binary)r=   r>   r?   )r=   )r
   r2   r   r6   r   )rN   rO   r9   ÚgcsrP   Úallow_inplacer7   rQ   r   )r9   rN   rO   rP   r   Ú_test_binaryD   s    
rT   c                sT   t f |ƒ j‡ fdd„ƒ|r"tjƒ ntjdƒdœ|—Žtddd�‡‡fdd	„ƒƒ}|S )
Nc                s   t | tj| ˆ d�d�S )N)r9   )r"   r#   )r'   r   r<   )r"   )r9   r   r   r1   ]   s   z(_test_binary_broadcast.<locals>.<lambda>F)r@   Úin_placerB   rC   )rD   rE   c       	         sb   t jˆ ddg|rdndgdd�}|\}}| j||||gdgƒ ‡fdd„}| j||||g|ƒ d S )	Nr>   r?   r=   r   )Ú	broadcastr   c                s   t jˆ | |ƒd | jd�fS )Nr   )r"   )r   Úarrayr"   )r   Úy)rO   r   r   Úcast_refi   s    zG_test_binary_broadcast.<locals>.test_binary_broadcast.<locals>.cast_ref)r   rF   rG   rI   )	rJ   r@   rU   rK   rL   rM   r>   r?   rY   )rN   rO   r   r   Útest_binary_broadcast\   s    z5_test_binary_broadcast.<locals>.test_binary_broadcast)r
   r2   r   Úbooleansr-   r   )rN   rO   r9   rR   rS   r7   rZ   r   )r9   rN   rO   r   Ú_test_binary_broadcastZ   s    r\   c               @   s,  e Zd Zdd„ Zef ejdd�ejƒ dœej	—Ždd„ ƒZ
ef dejdddd	�iej—Žd
d„ ƒZef dejdd�iej—Ždd„ ƒZdd„ Zdd„ Zdd„ Zdd„ Zef ejƒ ejƒ dœej	—Ždd„ ƒZef dejƒ iej	—Ždd„ ƒZef dejƒ iej	—Ždd„ ƒZef ejƒ ejƒ dœej	—Ždd „ ƒZeejdd!ejejƒd"�ejƒ d#�d$d%„ ƒZef ejd&d!d'�ejd!d(d'�ejd)d*gƒd+œej	—Žd,d-„ ƒZejej  d.ƒeejd&d/d'�ejd&d/d'�ejƒ ejd0gƒejd1gƒej!d2d2d'�ejdd3d'�ejd&d!d'�ejd&d!d'�d4�	d5d6„ ƒƒZ"ef ejd&d!ƒejd7d/ƒejd7d&ƒejdd!ƒd8œej	—Žd9d:„ ƒZ#ef ejdd!ƒej�d�dgƒd=œej	—Žd>d?„ ƒZ$ef dej%d@dgej!d2dAd'�dB�iej—ŽdCdD„ ƒZ&eeje'j(e'j)e'j*e'j+gƒdE�dFdG„ ƒZ,ef ejdd�ejƒ ej!dHdId'�ejƒ ej!dHdId'�dJœej	—ŽdKdL„ ƒZ-ef ejd/d�ejƒ ej!dHdId'�ej!dHdId'�ej!dHdId'�ej!dMdNd'�dOœej	—ŽdPdQ„ ƒZ.e/dRdS„ ƒZ0ef ejd!d�ejƒ ej!dNdHd'�ej!dHdId'�ej!dTdHd'�ej!dTdHd'�ejdUdVgƒdWœej—ŽdXdY„ ƒZ1e/dZd[„ ƒZ2ef ejd!d�ejƒ ej!dNdHd'�ej!dHdId'�ej!dTdHd'�ej!dTdHd'�ejdUdVgƒdWœej—Žd\d]„ ƒZ3ef ejd!d�ej!dNdHd'�ej!dHdId'�ej!dTdHd'�ej!dTdHd'�ejdUdVgƒd^œej—Žd_d`„ ƒZ4e/dadb„ ƒZ5ef ejd!d�ejƒ ej!dNdHd'�ej!dHdId'�ej!dTdHd'�ej!dTdHd'�ejdUdVgƒdWœej—Ždcdd„ ƒZ6ef ejd!d�ej!dNdHd'�ej!dHdId'�ej!dTdHd'�ej!dTdHd'�ejdUdVgƒd^œej—Ždedf„ ƒZ7ef ejdgd&e'j*ejd7dhd'�di�ejƒ djœej	—Ždkdl„ ƒZ8ef ej%dhd/gej!dmdmd7d&dn�dB�ej%dhge'j*ejd7�dd'�do�ejd&d/d'�dpœej	—Ždqdr„ ƒZ9ef dsej%dhgej!dmdmdNd&dn�dB�iej	—Ždtdu„ ƒZ:ef dvejejd7dhd'�d7dhdw�iej—Ždxdy„ ƒZ;ef dvejejd7dhd'�d7dhdw�iej—Ždzd{„ ƒZ<ef ejŽd|d}„ ƒZ=ef dvejejd7dhd'�d7dhdw�iej—Žd~d„ ƒZ>ef ej%dhd/gej!dmdmd7d&dn�dB�ej%dhge'j*ejd7�d	d'�do�d€œej	—Žd�d‚„ ƒZ?ef dvejejd7dhd'�d7dhdw�iej—Ždƒd„„ ƒZ@ef ejejd&dhd'�d7dhdw�ejd…d2d†d‡gƒdˆœej—Žd‰dŠ„ ƒZAef d‹ej%dhgej!dmdmdŒ�dB�iej	—Žd�dŽ„ ƒZBef d‹ej%dhgej!�d
dhd'�dB�iej	—Žd�d�„ ƒZCef d‹ej%dhgej!�ddhd'�dB�iej	—Žd‘d’„ ƒZDef d‹ej%dhgej!dmdmdŒ�dB�iej	—Žd“d”„ ƒZEef d‹ej%dhgej!d&d•d'�dB�iej—Žd–d—„ ƒZFd˜d™„ ZGeejd&dhƒejd&dšƒejd&d@ƒejd&d/ƒejejHƒd›�dœd�„ ƒZIeejd&dhƒejd&dhƒejd&d@ƒejd&d/ƒejejHƒdž�dŸd „ ƒZJeejd&d@ƒejd@dhƒejd&d@ƒejd&d/ƒd¡�d¢d£„ ƒZKef d¤ejƒ iej—Žd¥d¦„ ƒZLef ejŽd§d¨„ ƒZMef ejd&dhƒejd©dªgej �
rd«gng  ƒd¬œej	—Žd­d®„ ƒZNef ejdd3d¯�ejƒ ejƒ ejƒ ejƒ d°œej—Žd±d²„ ƒZOef d¤ejƒ iej—Žd³d´„ ƒZPef d¤ejƒ iej—Ždµd¶„ ƒZQef d¤ejƒ iej—Žd·d¸„ ƒZReejd7dšƒejd7dšƒd¹�dºd»„ ƒZSd¼d½„ ZTeejd7dšƒejd7dšƒd¾�d¿dÀ„ ƒZUeejd7dšƒejd7dšƒejd7d@ƒdÁ�dÂdÃ„ ƒZVef ejƒ ejeWeXeYƒƒƒejeWeXeYƒƒƒejƒ dÄœej	—ŽdÅdÆ„ ƒZZef ejƒ ej!dÇdNd'�ejej!dNdÈd'�dÉ�ej!dÇdTd'�dÊœej	—ŽdËdÌ„ ƒZ[ef ejej!dÍdÎ�dÉ�ej!�ddhd'�dÏœej	—ŽdÐdÑ„ ƒZ\ef e]e'j*e'j^e'j(e'j+gdÒ�j_dÓdÔ„ ƒejƒ ejƒ ejd&d@ejd&d@ƒd"�dÕœej	—ŽdÖd×„ ƒZ`eejd&d@ƒejd&d@ƒejd&d@ƒdØ�dÙdÚ„ ƒZaebecjdgdÛ�ef ejd&d@ƒejd&d@ƒejd&d@ƒejd&d@ƒejd7dƒejdd/ƒdÜœej	—ŽdÝdÞ„ ƒƒZeebecjdgdÛ�ef ejd&d@ƒejd&d@ƒejd&d@ƒejd&d@ƒejd7dƒejdd/ƒdÜœej	—Ždßdà„ ƒƒZfef ejƒ ejƒ ej!�dd‡d'�dáœej	—Ždâdã„ ƒZgeejhƒ dä�dådæ„ ƒZief ejd7d/ƒejd&dhƒejd&d@ƒejd&d@ƒejd;d<gƒejdçdèdégƒdêœej	—Ždëdì„ ƒZjef ejd7d/ƒejd7d/ƒejd7d/ƒejd7d/ƒejd&dhƒejd&d@ƒejd&d@ƒejd;d<gƒejdçdèdégƒdíœ	ej	—Ždîdï„ ƒZkef ejdðdhƒejd&dhƒejd&d/ƒejd;d<gƒej!dÇdNd'�dñœej—Ždòdó„ ƒZlef dôe]ƒ j_dõdÔ„ ƒiej	—Ždöd÷„ ƒZmef dejdddd	�iej—Ždødù„ ƒZnef ejddhd'�ejddhd'�ejddhd'�ej!dHd2d'�dúœej—Ždûdü„ ƒZoef ejddhd'�ejddhd'�ej!dHd2d'�dýœej—Ždþdÿ„ ƒZpef ejddhd'�ejddhd'��d œej—Ž�d�d„ ƒZqef ejdhdšd'�ejddhd'�ejd&d@d'��dœej—Ž�d�d„ ƒZrdUS (  ÚTestOperatorsc             C   s`   dd„ dd„ dd„ dd„ dœ}x<t |ƒD ]0\}}t||tjd�| ƒ t||tjd�| ƒ q(W d S )Nc             S   s
   | |k gS )Nr   )Úx1Úx2r   r   r   r1   v   s    z3TestOperators.test_comparison_ops.<locals>.<lambda>c             S   s
   | |kgS )Nr   )r^   r_   r   r   r   r1   w   s    c             S   s
   | |kgS )Nr   )r^   r_   r   r   r   r1   x   s    c             S   s
   | |kgS )Nr   )r^   r_   r   r   r   r1   y   s    )ÚLTZLEÚGTÚGE)rR   )r   rT   r   Úgcs_cpu_onlyr\   )rJ   ÚopsrN   rO   r   r   r   Útest_comparison_opsu   s    z!TestOperators.test_comparison_opsr(   )r:   )r@   rU   c             C   sV   t jdddg|sdndgƒ}|\}}| j||||gdgƒ | j||||gddgƒ d S )NÚSumr>   r?   r=   r   )r   rF   rG   rH   )rJ   r@   rU   rK   rL   rM   r>   r?   r   r   r   Útest_sum~   s
    zTestOperators.test_sumr@   )r:   r$   r%   c       
      C   sŒ   t jdddgdgƒ}|\}}|d d …df }dd„ }| j||||gdgƒ x(tdƒD ]}	| j||||g|	dgƒ qTW | j||||g|ƒ d S )	NZRowMulr>   r?   r=   r   c             S   sD   t j| j| jd�}x*t|jƒD ]}| |f ||  ||f< qW |gS )N)Úshaper"   )r   Úzerosrh   r"   ÚrangeÚsize)r   rX   ÚretÚir   r   r   rO   Œ   s    z'TestOperators.test_row_mul.<locals>.refr(   )r   rF   rG   rj   rH   rI   )
rJ   r@   rK   rL   rM   r>   ZXtmpr?   rO   rm   r   r   r   Útest_row_mul†   s    zTestOperators.test_row_mulc       	      C   sÄ   t jdddgdgƒ}|\}}|tj||d k||kƒ  d8  < |tj||d k||kƒ  d7  < | j||||gdgƒ x(tdƒD ]}| j||||g|dgƒ q„W dd	„ }| j||||g|ƒ d S )
NZMaxr>   r?   r=   gš™™™™™©?r   r(   c             S   s   t j| |ƒgS )N)r   Úmaximum)ÚXr=   r   r   r   Úelementwise_max¤   s    z/TestOperators.test_max.<locals>.elementwise_max)r   rF   r   Úlogical_andrG   rj   rH   rI   )	rJ   r@   rK   rL   rM   r>   r?   rm   rq   r   r   r   Útest_max—   s    $$zTestOperators.test_maxc             C   s:   dd„ }dd„ }t d||dd�| ƒ td||d�| ƒ d S )	Nc             S   s   t | tƒst| ƒdk S dS )Nr   é   Ti   @iÿÿÿ?)Ú
isinstanceÚfloatÚabs)r   r   r   r   Únot_overflow©   s    
z,TestOperators.test_add.<locals>.not_overflowc             S   s
   | | fS )Nr   )r   rX   r   r   r   rO   ®   s    z#TestOperators.test_add.<locals>.refÚAddT)r9   rP   )r9   )rT   r\   )rJ   rx   rO   r   r   r   Útest_add¨   s    zTestOperators.test_addc             C   s,   dd„ }t d|dd�| ƒ td|ƒ| ƒ d S )Nc             S   s
   | | fS )Nr   )r   rX   r   r   r   rO   ´   s    z#TestOperators.test_sub.<locals>.refÚSubT)rP   )rT   r\   )rJ   rO   r   r   r   Útest_sub³   s    zTestOperators.test_subc             C   s:   dd„ }dd„ }t d||dd�| ƒ td||d�| ƒ d S )	Nc             S   s   t | tƒst| ƒdk S dS )Nr   é   Ti €  iÿ  )ru   rv   rw   )r   r   r   r   rx   »   s    
z,TestOperators.test_mul.<locals>.not_overflowc             S   s
   | | fS )Nr   )r   rX   r   r   r   rO   À   s    z#TestOperators.test_mul.<locals>.refZMulT)r9   rP   )r9   )rT   r\   )rJ   rx   rO   r   r   r   Útest_mulº   s    zTestOperators.test_mulc             C   s`   dd„ }dd„ }dd„ }t d||d|tjd	�| ƒ t d||d
|d�| ƒ td|||d�| ƒ d S )Nc             S   s
   | | fS )Nr   )r   rX   r   r   r   rO   Æ   s    z#TestOperators.test_div.<locals>.refc             S   s   t | ƒdkS )Ng{®Gáz„?)rw   )r   r   r   r   Únon_zeroÉ   s    z(TestOperators.test_div.<locals>.non_zeroc               S   s   t jtjtjgƒS )N)r   r6   r   r5   Úfloat64r   r   r   r   Ú
div_dtypesÌ   s    z*TestOperators.test_div.<locals>.div_dtypesÚDivT)r9   rP   r7   rR   F)r9   rP   r7   )r9   r7   )rT   r   rc   r\   )rJ   rO   r   r�   r   r   r   Útest_divÅ   s    
zTestOperators.test_div)rp   rU   c             C   sH   t jddg|sdndgƒ}| j|||gdgƒ | j|||gddgƒ d S )NZNegativerp   r=   r   )r   rF   rG   rH   )rJ   rp   rU   rK   rL   rM   r   r   r   Útest_negativeÚ   s    
zTestOperators.test_negativerp   c             C   s<   t jdddƒ}| j|||gdgƒ | j|||gddgƒ d S )NZTanhrp   r=   r   )r   rF   rG   rH   )rJ   rp   rK   rL   rM   r   r   r   Ú	test_tanhá   s    zTestOperators.test_tanhc             C   s@   t jddgdgƒ}| j|||gdgƒ | j|||gddgƒ d S )NÚAveragedLossrp   Úlossr   )r   rF   rG   rH   )rJ   rp   rK   rL   rM   r   r   r   Útest_averaged_lossç   s    z TestOperators.test_averaged_loss)rp   Úinplacec             C   s”   t jddg|rdndgƒ}dd„ }| j|||gdgƒ | j|||g|ƒ |rz| jtƒ� | j|||gddgƒ W d Q R X n| j|||gddgƒ d S )NZSoftsignrp   r=   c             S   s   | dt j| ƒ  fS )Nr   )r   rw   )rp   r   r   r   Úsoftsignñ   s    z-TestOperators.test_softsign.<locals>.softsignr   )r   rF   rG   rI   ÚassertRaisesÚ	ExceptionrH   )rJ   rp   r‰   rK   rL   rM   rŠ   r   r   r   Útest_softsigní   s    "zTestOperators.test_softsignr   )r   r   r#   )Údevice_optionsÚset_seedc                s°   t j|ƒ}ttdd„ |D ƒƒdkƒ |r<x|D ]
}d|_q.W dd„ ‰ ‡ fdd„|D ƒ}xT|dd … D ]D}|r€tjj|d	 |ƒ qd| jt	ƒ� tjj|d	 |ƒ W d Q R X qdW d S )
Nc             S   s   h | ]
}|j ’qS r   )Údevice_type)Ú.0Údor   r   r   ú	<setcomp>  s    z;TestOperators.test_random_seed_behaviour.<locals>.<setcomp>r   iè  c             S   sB   t jƒ  t jjƒ }tjdg dg| dgd�}|j|ƒ |jd jƒ S )NZ
XavierFillr=   r(   )Údevice_optionrh   )	r   ZResetWorkspaceÚCÚ	Workspacer   rF   ÚrunÚblobsÚfetch)r’   ÚwsrM   r   r   r   r—     s    


z5TestOperators.test_random_seed_behaviour.<locals>.runc                s   g | ]}ˆ |ƒ‘qS r   r   )r‘   r’   )r—   r   r   ú
<listcomp>  s    z<TestOperators.test_random_seed_behaviour.<locals>.<listcomp>r   )
ÚcopyÚdeepcopyr	   r    Zrandom_seedr   ÚtestingÚassert_array_equalr‹   ÚAssertionError)rJ   rŽ   r�   r’   ÚysrX   r   )r—   r   Útest_random_seed_behaviourü   s    	


z(TestOperators.test_random_seed_behaviourr   )r*   r+   é   Ú ZPACKED)ÚaxisÚ
num_outputÚenginec             C   sT  t jjdƒ t jjdddddƒjt jƒ}dd„ }|t|jƒ|d … ƒ}|}	t jj|	|ƒjt jƒ}
t jj|	ƒjt jƒ}tj	dddd	gd
g||d�}x4d|fd|
fd	|fgD ]\}}| j
j|ƒj|ƒ q¨W | j
j|ƒ | j
jd
 jƒ }| jt|jƒt|jƒd |… |	g ƒ ||
|g}| j|||dgƒ x*t|ƒD ]\}}| j||||dgƒ �q.W d S )Nr   r(   rB   c             S   s   d}x| D ]}||9 }q
W |S )Nr   r   )ÚxsÚpr   r   r   r   Úprod'  s    
z5TestOperators.test_fully_connected_axis.<locals>.prodZFCrp   ÚWÚbr=   )r§   r¥   r   )r   ÚrandomÚseedÚrandnÚastyper5   Úlistrh   r   rF   rš   Úcreate_blobÚfeedr—   r˜   r™   ÚassertEqualrG   Ú	enumeraterH   )rJ   r¥   r¦   r§   rK   rL   rp   rª   ÚKÚNr«   r¬   rM   rN   Úparamr=   r@   Ú_r   r   r   Útest_fully_connected_axis  s,     &
z'TestOperators.test_fully_connected_axisz$Skipping test due to no gpu present.rB   ÚlstmZlinearg      ð?é   )	Úhidden_sizeÚ
num_layersÚbidirectionalÚrnn_modeÚ
input_modeÚdropoutÚTr·   ÚDc
             C   s   t jrt|dkƒ d}
tjj|
ƒ t jr4tj}d}n
tj}d}||	 }|| }|rZ|d9 }|| }|}|}|rrdnd}|| | | }|| | | }d||d |   }||9 }tjj	|ƒj
tjƒ}| jjdƒj||d� tjd	d
dddgdddddg|||||||
|d�}tjj|||	ƒj
tjƒ}| jjd
ƒj||d� | jjd jƒ }tjj|||| ƒj
tjƒ}|dk�rŽtjj|||| ƒj
tjƒntjdƒj
tjƒ}||||g}|dk�rÈdd„ t|ƒD ƒndddg}x(|D ] }| j||||dgddd� �qØW d S )Nr   iÒ  ZMIOPENZCUDNNr(   r   ZWEIGHT)r”   Z	RecurrentZINPUTZHIDDEN_INPUTZ
CELL_INPUTZOUTPUTZHIDDEN_OUTPUTZCELL_OUTPUTZRNN_SCRATCHZDROPOUT_STATES)r½   r¿   rÀ   rÂ   rÁ   r¾   r®   r§   r»   c             S   s   g | ]\}}|‘qS r   r   )r‘   rm   r¹   r   r   r   r›   ˆ  s    z0TestOperators.test_recurrent.<locals>.<listcomp>r   rB   g{®Gáz„?)ÚstepsizeÚ	threshold)r   )r   Zhas_hip_supportr	   r   r­   r®   r   Zhip_doZgpu_doÚrandr°   r5   rš   r²   r³   r   rF   r¯   r˜   r™   Úemptyrµ   rH   )rJ   r½   r¾   r¿   rÀ   rÁ   rÂ   rÃ   r·   rÄ   r®   r”   r§   Zinput_weight_sizeZupper_layer_input_weight_sizeZrecurrent_weight_sizeZinput_bias_sizeZrecurrent_bias_sizeZnum_directionsZfirst_layer_szZupper_layer_szZtotal_szr«   rM   rp   ÚHr•   r@   Z
input_idxsZ	input_idxr   r   r   Útest_recurrentC  sf    

&&
zTestOperators.test_recurrentr   )Úndimr¥   Úadd_axisÚ
num_inputsc                s   t ˆ|k ƒ ddddgd |… }ddddgd |… }d	ddd
dgd |… }	g }
x>t|ƒD ]2}ˆ dkrp|	| |ˆ< |
jtjj|Ž jtjƒƒ qXW tj	d|ddgˆˆ d�}| j
|||
dgƒ x$t|ƒD ]}| j|||
|dgƒ qÂW ‡ ‡fdd„}| j|||
|ƒ d S )NÚX0r>   r?   ÚX3r(   rB   é   é   r   r   r   ÚConcatr=   ÚY_dims)r¥   rÌ   c                 s`   t | ƒ} ˆ r6x(tt| ƒƒD ]}tj| | ˆƒ| |< qW tj‡fdd„| D ƒƒ}tj| ˆd�|gS )Nc                s   g | ]}t j|ƒˆ  ‘qS r   )r   rh   )r‘   r   )r¥   r   r   r›   ª  s    zITestOperators.test_depth_concat.<locals>.depth_concat.<locals>.<listcomp>)r¥   )r±   rj   r    r   Úexpand_dimsrW   Úconcatenate)r@   rm   Ú
input_dims)rÌ   r¥   r   r   Údepth_concat¥  s    z5TestOperators.test_depth_concat.<locals>.depth_concat)r	   rj   Úappendr   r­   r¯   r°   r5   r   rF   rG   rH   rI   )rJ   rË   r¥   rÌ   rÍ   rK   rL   Úinput_namesrh   Úindividual_dimsr@   rm   rM   r×   r   )rÌ   r¥   r   Útest_depth_concat�  s     
zTestOperators.test_depth_concatÚNCHWÚNHWC)rÍ   Úorderc                sæ   ddddgd |… }ddddg}d	ddd
gd |… }g }x:t |ƒD ].}	||	 |ˆ d	 < |jtjj|Ž jtjƒƒ qBW tjd|ddgˆ d d�}
| j	||
|dgƒ x$t |ƒD ]}	| j
||
||	dgƒ qªW ‡ fdd„}| j||
||ƒ d S )NrÎ   r>   r?   rÏ   r(   rB   rÐ   rÑ   r   r   rÒ   r=   rÓ   r   )rÞ   c                 s:   t | ƒ} ˆd ‰ tj‡ fdd„| D ƒƒ}tj| ˆ d�|gS )Nr   c                s   g | ]}t j|ƒˆ  ‘qS r   )r   rh   )r‘   r   )r¥   r   r   r›   Å  s    z_TestOperators.test_depth_concat_with_order.<locals>.depth_concat_with_order.<locals>.<listcomp>)r¥   )r±   r   rW   rÕ   )r@   rÖ   )rÞ   )r¥   r   Údepth_concat_with_orderÂ  s    zKTestOperators.test_depth_concat_with_order.<locals>.depth_concat_with_order)rj   rØ   r   r­   rÇ   r°   r5   r   rF   rG   rH   rI   )rJ   rÍ   rÞ   rK   rL   rÙ   rh   rÚ   r@   rm   rM   rß   r   )rÞ   r   Útest_depth_concat_with_order¯  s    z*TestOperators.test_depth_concat_with_orderrÐ   g      $@)r   r#   c             C   sú   t jd|ƒ t jdtjdtjd�ƒ t jdƒ tjdƒ}|jdddgddgdd� tj	d	ƒ}|j
tjd	|gd
d�ƒ t j|ƒ t jdƒ}t jdƒ}tjd|jd gƒ}x4t|jd d
 ƒD ]}	||	|jd   ||	d < qºW dd lj}
|
j||dd� d S )NÚinputÚnextr   )r"   ÚoutputÚcollect_netrÑ   )Znum_to_collectZcollect_datar(   )Únum_iterr   rÐ   )Údecimal)r   ÚFeedBlobr   rW   r4   Z
CreateBlobr   ÚNetZLastNWindowCollectorÚPlanÚAddStepÚexecution_stepZRunPlanÚ	FetchBlobri   rh   rj   Znumpy.testingrž   Zassert_almost_equal)rJ   rp   rK   rL   rä   Úplanrã   r@   Z
new_outputrm   Znptr   r   r   Útest_last_n_windowsÊ  s&    







z!TestOperators.test_last_n_windows)r"   c             C   sB   t jjdƒj|ƒ}| jjdƒj|ƒ tjddg ƒ}| jj	|ƒ d S )Nr¼   ÚdataZPrint)
r   r­   Zpermutationr°   rš   r²   r³   r   rF   r—   )rJ   r"   rï   rM   r   r   r   Ú
test_printã  s    zTestOperators.test_printgš™™™™™¹?gÍÌÌÌÌÌì?)r@   rU   ÚmomentumÚnesterovÚlrc                sŽ   |\}}	t j|gt jd�}tjddddg|r0dnd|r:dndgˆ tˆƒ|d�}
| j||
||	|gd	gƒ ‡ ‡fd
d„}| j||
||	|g|ƒ d S )N)r"   ZMomentumSGDÚgradÚmró   Úgrad_oZm_o)rñ   rò   r”   r   c                sP   |d }ˆs$||  ˆ |  }||fS ˆ | ||   }dˆ  | ˆ |  |fS d S )Nr   r   r   )rô   rõ   ró   Zadjusted_gradientZm_new)rñ   rò   r   r   Úmomentum_sgd   s    z5TestOperators.test_momentum_sgd.<locals>.momentum_sgd)r   Úasarrayr5   r   rF   ÚintrG   rI   )rJ   r@   rU   rñ   rò   ró   rK   rL   rô   rõ   rM   r÷   r   )rñ   rò   r   Útest_momentum_sgdê  s    
	zTestOperators.test_momentum_sgdgñhãˆµøä>g{®Gáz„?)r@   rU   Údecayrñ   ró   Úepsilonc	          	      s®   |\}	}
}t j|
ƒd }
t j|gt jd�}tjdddddg|rBdnd|rLdnd	|rVdnd
gˆˆ ˆ|d�}| j|||	|
||gdgƒ ‡ ‡‡fdd„}| j|||	|
||g|ƒ d S )Ng{®Gáz„?)r"   ZRmsProprô   ÚmsÚmomró   rö   Úms_oÚmom_o)rñ   rû   rü   r”   r   c                sN   |d }|dˆ  t j| ƒ|   }ˆ| ||  t jˆ| ƒ  }|}|||fS )Nr   g      ð?)r   ÚsquareÚsqrt)rô   rý   rþ   ró   rÿ   r   rö   )rû   rü   rñ   r   r   Úrmsprop   s
    z/TestOperators.test_rmsprop_sgd.<locals>.rmsprop)r   rw   rø   r5   r   rF   rG   rI   )rJ   r@   rU   rû   rñ   ró   rü   rK   rL   rô   rý   rþ   rM   r  r   )rû   rü   rñ   r   Útest_rmsprop_sgd  s    	



zTestOperators.test_rmsprop_sgdc             C   sÂ   t | tjƒrtj| ƒ} tj|ddd�}tj|ddd�}|| }	tj||	 ƒtj|ƒ |  }
|||
|  7 }||	7 }tj|ƒ| | |tj|ƒ |  |  }d|tj|ƒ|k< |tj||gdd�fS )Nr   r   )r¥   éÿÿÿÿr  r  )	ru   r   ÚndarrayÚasscalarÚtaker  Úsignrw   Ústack)ÚalphaÚbetaÚlambda1Úlambda2ÚwÚnzÚgr:   ÚzÚg2Úsigmar   r   r   Ú_dense_ftrl)  s    
zTestOperators._dense_ftrlgü©ñÒMbP?NZSIMD)r@   rU   r  r  r  r  r§   c
             C   sž   |\}
}}}t j|ƒ}t j||gdd�}tjddddg|r>dnd|rHdndg||||||d	�	}| j|	||
||gd
gƒ | j|||
||gt| j||||ƒƒ d S )Nr   )r¥   ÚFtrlÚvarr  rô   Úvar_oÚnz_o)r  r  r  r  r§   r”   r   r  )	r   rw   r
  r   rF   rG   rI   r   r  )rJ   r@   rU   r  r  r  r  r§   rK   rL   r  r:   r  rô   r  rM   r   r   r   Útest_ftrl_sgd9  s     


zTestOperators.test_ftrl_sgdc             C   sv  t | tjƒrtj| ƒ} |j}tj|ddd�}tj|ddd�}	|jd }
|j|
dƒ}|j|
d	ƒ}|j|
d
ƒ}|	j|
dƒ}	|jd }|| }tj|| ƒtj|ƒ |  }|	|||  7 }	||7 }tjj	|	ddd�}|d }|	|tj|
ƒ | d  |tj|ƒ |  |  }x:t
|ƒD ].}|| |tj|
ƒ k�rd|d d …|f< �qW |j|ƒ}|j|ƒ}|	j|ƒ}	|tj||	gdd�fS )Nr   r   )r¥   r(   g�íµ ÷Æ°>r  r  r  r  r  r  r  )ru   r   r  r  rh   r  Úreshaper  ÚlinalgÚnormrj   r
  )r  r  r  r  r  r  r  Z	old_shaper:   r  Z
output_dimZ	input_dimr  r  Zz_normsrm   r   r   r   Ú_dense_gftrlV  s4    





zTestOperators._dense_gftrlc
             C   sž   |\}
}}}t j|ƒ}t j||gdd�}tjddddg|r>dnd|rHdndg||||||d	�	}| j|	||
||gd
gƒ | j|||
||gt| j||||ƒƒ d S )Nr   )r¥   ZGFtrlr  r  rô   r  r  )r  r  r  r  r§   r”   r   r  )	r   rw   r
  r   rF   rG   rI   r   r  )rJ   r@   rU   r  r  r  r  r§   rK   rL   r  r:   r  rô   r  rM   r   r   r   Útest_gftrl_sgd}  s     


zTestOperators.test_gftrl_sgd)r@   r  r  r  r  r§   c	                sÂ   |\}	}
}}t j|	jd ƒ}||d dk }|| }t j|
ƒ}
t j|
|gdd�}tjddddd	gddgˆ ˆˆˆ||d
�	}ˆj|||	|||gdgƒ ‡ ‡‡‡‡fdd„}ˆj|||	|||g|ƒ d S )Nr   r(   r   )r¥   Ú
SparseFtrlr  r  Úindicesrô   )r  r  r  r  r§   r”   c                s:   ˆj ˆ ˆˆˆ| | || |ƒ\}}|| |< |||< | |fS )N)r  )r  r  rm   r  ÚswÚsnz)r  r  r  r  rJ   r   r   Úftrl´  s
    z0TestOperators.test_sparse_ftrl_sgd.<locals>.ftrlr  )	r   Úarangerh   rw   r
  r   rF   rG   rI   )rJ   r@   r  r  r  r  r§   rK   rL   r  r:   r  rô   r!  r  rM   r$  r   )r  r  r  r  rJ   r   Útest_sparse_ftrl_sgd™  s"    	

z"TestOperators.test_sparse_ftrl_sgdc             C   s   t j|| |||||ƒS )N)r]   r  )r  r  r  r  r  r  r  r   r   r   Ú_dense_ftrl_send_alpha_by_input¾  s    z-TestOperators._dense_ftrl_send_alpha_by_inputc
          
   C   s²   |\}
}}}t j|ƒ}t j||gdd�}t j|ƒjt jƒ}tjdddddg|rRdnd|r\dnd	g|||||d
�}| j|	||
|||gdgƒ | j	|||
|||gt
| j|||ƒƒ d S )Nr   )r¥   r  r  r  rô   r  r  r  )r  r  r  r§   r”   r   r  )r   rw   r
  rW   r°   r5   r   rF   rG   rI   r   r'  )rJ   r@   rU   r  r  r  r  r§   rK   rL   r  r:   r  rô   r  rM   r   r   r   Ú!test_ftrl_sgd_send_alpha_by_inputÃ  s"    



z/TestOperators.test_ftrl_sgd_send_alpha_by_inputc	          
      sÖ   |\}	}
}}t j|	jd ƒ}||d dk }|| }t j|
ƒ}
t j|
|gdd�}t j|ƒjt jƒ}tj	ddddd	d
gddgˆ ˆˆ||d�}ˆj
|||	||||gdgƒ ‡ ‡‡‡fdd„}ˆj|||	||||g|ƒ d S )Nr   r(   r   )r¥   r   r  r  r!  rô   r  )r  r  r  r§   r”   c                s:   ˆj ˆ ˆˆ| | || ||ƒ\}}|| |< |||< | |fS )N)r'  )r  r  rm   r  r  r"  r#  )r  r  r  rJ   r   r   r$  ý  s    zDTestOperators.test_sparse_ftrl_sgd_send_alpha_by_input.<locals>.ftrlr  )r   r%  rh   rw   r
  rW   r°   r5   r   rF   rG   rI   )rJ   r@   r  r  r  r  r§   rK   rL   r  r:   r  rô   r!  r  rM   r$  r   )r  r  r  rJ   r   Ú(test_sparse_ftrl_sgd_send_alpha_by_inputà  s&    

z6TestOperators.test_sparse_ftrl_sgd_send_alpha_by_inputé   r)   )r+   r%   r"   r#   )rá   Úwith_remappingc                s^   t jddgdgˆrdgng  |d�}ˆ j|||gdgƒ d	‡ ‡fdd„	}ˆ j|||g|ƒ d S )
NZUniquerá   ÚuniqueÚ	remapping)r”   r   c                sp   ˆj ˆjtt| ƒƒƒ ˆj tˆƒtt| ƒƒƒ ˆrlˆj ˆ j| jƒ ‡ ‡fdd„tt| ƒƒD ƒ}tjj	|| ƒ d S )Nc                s   g | ]}ˆˆ |  ‘qS r   r   )r‘   rm   )r-  r,  r   r   r›     s    zCTestOperators.test_unique.<locals>.unique_valid.<locals>.<listcomp>)
r´   rk   r    ÚsetÚsortedrh   rj   r   rž   rŸ   )rá   r,  r-  Zremapped)rJ   r+  )r-  r,  r   Úunique_valid  s    z/TestOperators.test_unique.<locals>.unique_valid)N)r   rF   rG   ZassertValidationChecks)rJ   rá   r+  rK   rL   rM   r0  r   )rJ   r+  r   Útest_unique  s    zTestOperators.test_uniqueF)Ú	allow_nanÚallow_infinityr*   r+   )r   r"   r#   )Ú
predictionÚlabelsÚtop_kc             C   sL   |dkrt j}tjdddgdg||d�}dd„ }| j|||||g|d	� d S )
Nr   ZAccuracyr4  r5  Úaccuracy)r6  r”   c       
      S   s�   | j d }d}xrtdt| ƒƒD ]`}tdd„ t| | ƒD ƒdd„ dd�}dd	„ |d|… D ƒ}x |D ]}||| krb|d
7 }qbW qW || }	|	fS )Nr   c             s   s   | ]\}}||gV  qd S )Nr   )r‘   ÚjÚitemr   r   r   ú	<genexpr>=  s    z>TestOperators.test_accuracy.<locals>.op_ref.<locals>.<genexpr>c             S   s   | d S )Nr   r   )r   r   r   r   r1   >  s    z=TestOperators.test_accuracy.<locals>.op_ref.<locals>.<lambda>T)ÚkeyÚreversec             S   s   g | ]}|d  ‘qS )r   r   )r‘   r   r   r   r   r›   A  s    z?TestOperators.test_accuracy.<locals>.op_ref.<locals>.<listcomp>r   )rh   rj   r    r/  rµ   )
r4  r5  r6  r·   Úcorrectrm   Zpred_sortedÚmax_idsrõ   r7  r   r   r   Úop_ref8  s    

z+TestOperators.test_accuracy.<locals>.op_ref)r”   rM   r@   Ú	reference)r   Úcpu_dor   rF   rI   )rJ   r4  r5  r6  rK   rL   rM   r?  r   r   r   Útest_accuracy!  s    zTestOperators.test_accuracyÚtarget_probabilitiesc             C   s2   t jddgdgƒ}dd„ }| j|||g|d� d S )NZ
PerplexityrC  Ú
perplexityc             S   s.   | j d }tj| d| ƒ}tdd„ |ƒ}|fS )Nr   g      ð?c             S   s   | | S )Nr   )r   rX   r   r   r   r1   ^  s    z?TestOperators.test_perplexity.<locals>.op_ref.<locals>.<lambda>g      ð¿)rh   r   Úpowerr   )rC  r·   ZperplexitiesrD  r   r   r   r?  [  s    
z-TestOperators.test_perplexity.<locals>.op_ref)r”   rM   r@   r@  )r   rF   rI   )rJ   rC  rK   rL   rM   r?  r   r   r   Útest_perplexityN  s    zTestOperators.test_perplexityÚlengths)r   r   c             C   s>   t jddgdgƒ}dd„ }| j||tj|tjd�g|d� d S )NZLengthsToSegmentIdsrG  Úsegment_idsc             S   s<   g }x$t | ƒD ]\}}|j||g ƒ qW tj|tjd�fS )N)r"   )rµ   Úextendr   rW   r4   )rG  Úsidsrm   Úlr   r   r   r?  q  s    z9TestOperators.test_lengths_to_segment_ids.<locals>.op_ref)r"   )r”   rM   r@   r@  )r   rF   rI   r   rW   r4   )rJ   rG  rK   rL   rM   r?  r   r   r   Útest_lengths_to_segment_idsg  s    z)TestOperators.test_lengths_to_segment_idsc             C   s>   t jddgdgƒ}dd„ }| j||tj|tjd�g|d� d S )NZLengthsRangeFillrG  Zincreasing_seqc             S   s>   g }x&t | ƒD ]\}}|jtt|ƒƒƒ qW tj|tjd�fS )N)r"   )rµ   rI  r±   rj   r   rW   r4   )rG  rJ  r¹   rK  r   r   r   r?  ‡  s    z5TestOperators.test_lengths_range_fill.<locals>.op_ref)r"   )r”   rM   r@   r@  )r   rF   rI   r   rW   r4   )rJ   rG  rK   rL   rM   r?  r   r   r   Útest_lengths_range_fill}  s    z%TestOperators.test_lengths_range_fillc             C   sX   ddddddg}t jddgdgƒ}d	d
„ }dd„ }| j||tj||ƒtjd�|d� d S )Nr   r¼   rB   r(   r   ZSegmentIdsToRangesrH  Úrangesc             S   s†   t jddgt jd�g}d}xXt| ƒD ]L\}}x.||krZ|d7 }|jt j|dgt jd�ƒ q.W |d d  d7  < q$W t j|t jd�fS )Nr   )r"   r   r  )r   rW   r4   rµ   rØ   )rH  rN  Úprevrm   Zsidr   r   r   r?  ›  s    
z8TestOperators.test_segment_ids_to_ranges.<locals>.op_refc             S   s<   g }x$t | ƒD ]\}}|j||g ƒ qW tj|tjd�fS )N)r"   )rµ   rI  r   rW   r4   )rG  rJ  rm   rK  r   r   r   Úlengths_to_segment_ids¥  s    zHTestOperators.test_segment_ids_to_ranges.<locals>.lengths_to_segment_ids)r"   )r”   rM   r@   r@  )r   rF   rI   r   rW   r4   )rJ   rK   rL   rG  rM   r?  rP  r   r   r   Útest_segment_ids_to_ranges“  s    
z(TestOperators.test_segment_ids_to_rangesc             C   s>   t jddgdgƒ}dd„ }| j||tj|tjd�g|d� d S )NZLengthsToRangesrG  rN  c             S   s<   | j s| jdƒfS tjtjdgtj| ƒd d… fƒ| fƒfS )Nr   r(   r   )r   r(   r  )rk   r  r   Zcolumn_stackrÕ   Zcumsum)r   r   r   r   r?  »  s     z4TestOperators.test_lengths_to_ranges.<locals>.op_ref)r"   )r”   rM   r@   r@  )r   rF   rI   r   rW   r4   )rJ   rG  rK   rL   rM   r?  r   r   r   Útest_lengths_to_ranges±  s    z$TestOperators.test_lengths_to_ranges)r4  r5  c             C   s8   t jdddgddgƒ}dd„ }| j||||g|d� d S )	NZMultiClassAccuracyr4  r5  Ú
accuraciesÚamountsc             S   sÖ   | j d }| j d }tj|td�}|jdƒ tj|td�}|jdƒ tj| dd�}xJtd|ƒD ]<}|| }|| }	||	krŠ||	  d7  < ||	  d7  < q^W x.td|ƒD ] }|| }
|
rª||  |
  < qªW ||fS )Nr   r   )r"   )r¥   )rh   r   rÈ   rv   Úfillrù   Zargmaxrj   )r4  r5  r·   rÄ   rS  rT  r>  rm   Zmax_idZlabel_idÚamountr   r   r   r?  Ø  s$    



z7TestOperators.test_multi_class_accuracy.<locals>.op_ref)r”   rM   r@   r@  )r   rF   rI   )rJ   r4  r5  rK   rL   rM   r?  r   r   r   Útest_multi_class_accuracyÇ  s    
z'TestOperators.test_multi_class_accuracyc             C   sN   t jddgdgƒ}dd„ }||ƒ}dd„ }| j||tj|tjd�g|d	� d S )
NZSegmentIdsToLengthsrH  rG  c             S   s.   g }x$t | ƒD ]\}}|j||g ƒ qW |S )N)rµ   rI  )rG  rJ  rm   rK  r   r   r   Úlengths_to_idsü  s    zATestOperators.test_segment_ids_to_lengths.<locals>.lengths_to_idsc             S   s²   t | ƒ}|dkr"tjg tjd�fS g }d}d}xft|ƒD ]Z}| | }||krŠ|dkr^|j|ƒ x"|d |kr€|jdƒ |d7 }q`W |}d}|d7 }q8W |j|ƒ tj|tjd�fS )Nr   )r"   r   r  )r    r   rW   r4   rj   rØ   )ÚidsZ
ids_lengthrG  Zprev_idZ
tmp_lengthÚidxZcur_idr   r   r   Úids_to_lengths  s&    


zATestOperators.test_segment_ids_to_lengths.<locals>.ids_to_lengths)r"   )r”   rM   r@   r@  )r   rF   rI   r   rW   r4   )rJ   rG  rK   rL   rM   rX  rH  r[  r   r   r   Útest_segment_ids_to_lengthsò  s    z)TestOperators.test_segment_ids_to_lengthsg      à?g      ø?g       @)rG  rE  c                sF   t jddgdgˆ d�}‡ fdd„}| j||tj|tjd�g|d� d S )	NZLengthsToWeightsrG  Úweights)rE  c                s<   g }x&| D ]}|j |dt|ˆ ƒ g ƒ q
W tj|td�fS )Nr   )r"   )rI  Úpowr   rW   rv   )rG  Zweighted_lengthrK  )rE  r   r   Úlengths_to_weights-  s    
zATestOperators.test_lengths_to_weights.<locals>.lengths_to_weights)r"   )r”   rM   r@   r@  )r   rF   rI   r   rW   r4   )rJ   rG  rE  rK   rL   rM   r_  r   )rE  r   Útest_lengths_to_weights!  s    z%TestOperators.test_lengths_to_weightsÚinput_tensor)r2  r3  c             C   s2   t jddgdgƒ}dd„ }| j|||g|d� d S )NZAbsrá   rã   c             S   s   t j| ƒfS )N)r   rw   )ra  r   r   r   Úabs_refE  s    z'TestOperators.test_abs.<locals>.abs_ref)r”   rM   r@   r@  )r   rF   rI   )rJ   ra  rK   rL   rM   rb  r   r   r   Útest_abs:  s    zTestOperators.test_absc             C   s2   t jddgdgƒ}dd„ }| j|||g|d� d S )NZCosrá   rã   c             S   s   t j| ƒfS )N)r   Úcos)ra  r   r   r   Úcos_refY  s    z'TestOperators.test_cos.<locals>.cos_ref)r”   rM   r@   r@  )r   rF   rI   )rJ   ra  rK   rL   rM   re  r   r   r   Útest_cosN  s    zTestOperators.test_cosc             C   s2   t jddgdgƒ}dd„ }| j|||g|d� d S )NZSinrá   rã   c             S   s   t j| ƒfS )N)r   Úsin)ra  r   r   r   Úsin_refm  s    z'TestOperators.test_sin.<locals>.sin_ref)r”   rM   r@   r@  )r   rF   rI   )rJ   ra  rK   rL   rM   rh  r   r   r   Útest_sinb  s    zTestOperators.test_sinc             C   s2   t jddgdgƒ}dd„ }| j|||g|d� d S )NZExprá   rã   c             S   s   t j| ƒfS )N)r   r   )ra  r   r   r   Úexp_ref�  s    z'TestOperators.test_exp.<locals>.exp_ref)r”   rM   r@   r@  )r   rF   rI   )rJ   ra  rK   rL   rM   rj  r   r   r   Útest_expv  s    zTestOperators.test_expi'  c             C   sH   t jddgdgƒ}dd„ }| j|||g|d� | j|||gddgƒ d S )NÚLográ   rã   c             S   s   t j| ƒfS )N)r   Úlog)ra  r   r   r   Úlog_ref•  s    z'TestOperators.test_log.<locals>.log_ref)r”   rM   r@   r@  r   )r   rF   rI   rH   )rJ   ra  rK   rL   rM   rn  r   r   r   Útest_logŠ  s    zTestOperators.test_logc                sr   t jdg dgddd�‰ ˆjjˆ ƒ tjƒ }t jddgdgdd	�‰ ˆjt‡ ‡fd
d„ƒ tjƒ | }ˆj|dƒ d S )NÚCreateBlobsQueueÚqueuerÐ   r   )ÚcapacityÚ	num_blobsÚDequeueBlobsrA   gš™™™™™É?)Ztimeout_secsc                  s   ˆj jˆ ƒS )N)rš   r—   r   )rM   rJ   r   r   r1   ­  s    z:TestOperators.test_blobs_dequeue_timeout.<locals>.<lambda>gR¸…ëQÈ?)r   rF   rš   r—   Útimer‹   ÚRuntimeErrorZassertGreater)rJ   Útr   )rM   rJ   r   Útest_blobs_dequeue_timeoutŸ  s     z(TestOperators.test_blobs_dequeue_timeoutrC   )Únum_threadsÚnum_elementsrr  rs  r’   c                s  ddl ‰
yddl‰W n tk
r0   ddl‰Y nX tjdg dg|ˆˆ d�}ˆ	jj|ƒ ‡fdd„tˆƒD ƒ‰ˆjƒ ‰x(tˆƒD ]‰ˆj	‡fdd„ˆD ƒƒ q~W ‡ ‡‡‡‡	fd	d
„‰x6t|ƒD ]*}x$tˆƒD ]‰ˆ	jj
djˆ|ƒƒ qÊW q¼W ‡‡
fdd„t|ƒD ƒ}x|D ]}	|	jƒ  �qW xHtˆƒD ]<‰‡fdd„tˆƒD ƒ}
tjddg|
ˆ d�}ˆ	jj|ƒ �q$W x|D ]}	|	jƒ  �qjW tjddgg ƒ}ˆ	jj|ƒ ‡‡	fdd„tˆƒD ƒ‰xdtˆƒD ]X‰ˆ	jˆˆ jˆˆ jƒ x8tˆƒD ],‰ˆ	jt‡‡‡‡fdd„tˆƒD ƒƒƒ �qàW �qºW dS )a  
        - Construct matrices of size N x D
        - Start K threads
        - Push all N rows into the queue of capacity C
        - Pull all N rows out of the queue.
        - Verify that the output matrices are permutation of the rows of the
          original matrices.
        r   Nrp  rq  )rr  rs  r”   c                s"   g | ]}t jjˆ d ƒjt jƒ‘qS )rÐ   )r   r­   r¯   r°   r5   )r‘   r¹   )rz  r   r   r›   Ï  s   z<TestOperators.test_blobs_queue_threading.<locals>.<listcomp>c                s   g | ]}|ˆ  ‘qS r   r   )r‘   r   )rm   r   r   r›   Ó  s    c                s–   x�‡ fdd„t ˆƒD ƒ}tjddg| |ˆd�}yFˆjƒ }x,t||ƒD ]\}}ˆjj|ƒj|ˆd� qFW ˆjj|ƒ W q ˆj	k
rŒ   d S X qW d S )Nc                s   g | ]}d j |ˆ ƒ‘qS )zx_{}_{})Úformat)r‘   rm   )rw  r   r   r›   ×  s    zMTestOperators.test_blobs_queue_threading.<locals>.enqueue.<locals>.<listcomp>ZEnqueueBlobsrq  )r”   )
rj   r   rF   Ú
get_nowaitÚziprš   r²   r³   r—   ÚEmpty)rw  Z
feed_blobsrM   ÚelemsÚelemZ	feed_blob)r’   rs  Úqrq  rJ   )rw  r   ÚenqueueÕ  s    z9TestOperators.test_blobs_queue_threading.<locals>.enqueuezx_{}_{}c                s   g | ]}ˆj ˆ |fd �‘qS ))ÚtargetÚargs)ÚThread)r‘   rw  )r‚  Ú	threadingr   r   r›   ì  s   c                s   g | ]}d j |ˆ ƒ‘qS )zy_{}_{})r{  )r‘   rm   )r:   r   r   r›   ò  s    rt  )r”   ÚCloseBlobsQueuec                s*   g | ]"‰ t j‡ ‡fd d„tˆƒD ƒƒ‘qS )c                s$   g | ]}ˆj jd jˆ |ƒ jƒ ‘qS )zy_{}_{})rš   r˜   r{  r™   )r‘   r:   )rm   rJ   r   r   r›   ý  s   zGTestOperators.test_blobs_queue_threading.<locals>.<listcomp>.<listcomp>)r   Zvstackrj   )r‘   )rz  rJ   )rm   r   r›   ý  s   c             3   s*   | ]"}t jˆˆ  ˆ ˆˆ  | ƒV  qd S )N)r   Zarray_equal)r‘   Úk)rm   r8  r¨   r¡   r   r   r:    s   z;TestOperators.test_blobs_queue_threading.<locals>.<genexpr>)r†  rq  ÚImportErrorÚQueuer   rF   rš   r—   rj   Úputr²   r{  ÚstartÚjoinr´   rh   Ú
assertTrueÚany)rJ   ry  rz  rr  rs  r’   rM   rw  ÚthreadsÚthreadZdequeue_blobsr   )r’   r‚  rm   r8  r:   rs  rz  r�  rq  rJ   r†  r¨   r¡   r   Útest_blobs_queue_threading±  s\    


z(TestOperators.test_blobs_queue_threading)Únum_producersÚnum_consumersrr  rs  r’   c                s,  t jdƒ}|jg d||d�}g }d}	x†t|ƒD ]z}
d|
 }t j|ƒ‰ ‡ fdd„t|ƒD ƒ}ˆ jƒ }ˆ j|g| ||g ƒ |
d d }t j|ˆ |d	�}|	|7 }	|j|ƒ q.W t jd
ƒ}|j|gdƒ t jdt jd|dd�t jd|ƒgƒ}g }g }|j	g ddd�}x�t|ƒD ]„}
d|
 }t j|ƒ}|j
|g|d ƒ}|d }t j|d ƒ}|j	g ddd�}|j|ƒ |j||g|ƒ |jt j|||g|d�ƒ �q
W t jd|dd�}t jd|ƒ}t jd||gdd�}t jdƒ}|j|ƒ |j|ƒ | jj|ƒ d}x*|D ]"}|| jjt|ƒ jƒ jƒ 7 }�qöW | j||	ƒ d S )NÚinit_netr   )rr  rs  r   zproducer_%dc                s   g | ]}ˆ j g d ddd�‘qS )r   g      ð?F)ÚvalueÚrun_once)ÚConstantFill)r‘   Útimes)Únetr   r   r›     s   z7TestOperators.test_safe_blobs_queue.<locals>.<listcomp>r)   )rå   Úproducer_exit_netÚproducerÚ	producersT)Úconcurrent_substepsÚproducer_exitg      ð?)r–  zconsumer_%dÚ_counterg        )Úshould_stop_blobÚconsumerÚinitÚworkerÚtestr  )r   rè   rp  rj   ÚNextNameÚSafeEnqueueBlobsrë   rØ   r‡  r˜  ZSafeDequeueBlobsry   ré   rê   rš   r—   r˜   Ústrr™   Útolistr´   )rJ   r“  r”  rr  rs  r’   r•  rq  Úproducer_stepsÚtruthrm   rN   r˜   ÚstatusÚcountÚstepr›  Úproducer_stepZconsumer_stepsZcountersZconst_1Znet1Znet2ÚcounterÚconsumer_stepÚ	init_stepÚworker_steprí   Úvr   )rš  r   Útest_safe_blobs_queue
  sb    










"z#TestOperators.test_safe_blobs_queue)Ú
num_queuesrå   rr  rs  c                s   t d|ƒ tjdƒ‰‡ ‡‡fdd„t|ƒD ƒ}g }g }x¾t|ƒD ]²}d| }	tj|	ƒ‰‡fdd„tˆƒD ƒ}
ˆjƒ }ˆj|| g|
 |
|g ƒ tjd| ƒ}|j|| dƒ |j|ƒ tj|	tjd| ˆg|d	�tjd| |gƒgƒ}|j|ƒ q@W tjd
tjd|dd�gƒ}g ‰‡fdd„}tjdƒ}t	j
jdˆdd�}|j|ˆd t	j
jdd|fd�|d d�}
|
d }|j|ƒ|ƒ tjdtjd|g|d�tjd|ƒgƒ}tjdˆƒ}tjd||gdd�}tjdƒ}|j|ƒ |j|ƒ | jj|ƒ tˆƒ|d k�st‚tˆƒ|| d k�st‚d S )Nr¶  r•  c                s   g | ]}ˆj g d ˆ ˆd�‘qS )r   )rr  rs  )rp  )r‘   r¹   )rr  r•  rs  r   r   r›   T  s   zBTestOperators.test_weighted_sample_blobs_queue.<locals>.<listcomp>zproducer_%dc                s   g | ]}ˆ j g d ddd�‘qS )r   g      ð?F)r–  r—  )r˜  )r‘   r¹   )rš  r   r   r›   _  s   zproducer_exit_%dr   )rå   rœ  r�  T)rž  c                s   ˆ j | ƒ d S )N)rØ   )ZinsZouts)Ú
status_lstr   r   rØ   ~  s    z>TestOperators.test_weighted_sample_blobs_queue.<locals>.appendZweight_sample_dequeue_netr   )ÚlowÚhighrk   g        g      ð?)r]  Útable_idx_blobr¢  )r¡  rŸ  r£  r¤  r¥  r  r  )Úprintr   rè   rj   r¦  r§  r‡  rØ   rë   r   r­   ÚrandintZWeightedSampleDequeueBlobsÚuniformÚPythonré   rê   rš   r—   r    r    )rJ   r¶  rå   rr  rs  Zqueuesrª  Zproducer_exit_netsrm   rN   r˜   r¬  Zexit_netr®  r¯  rØ   Zconsumer_netrº  r±  r²  r³  rí   r   )rr  r•  rš  rs  r·  r   Ú test_weighted_sample_blobs_queueI  sj    








z.TestOperators.test_weighted_sample_blobs_queuerï   c                sj   ddg‰ t |jƒdkr ˆ jdƒ tjddgdgˆ d�}‡ fdd„}‡ fd	d
„}| j|||g|d|d� d S )Nr   r(   Z
ExpandDimsrï   Úexpanded)r   c                s:   t tˆ ƒƒ}|jƒ  | }x|D ]}tj||d�}qW |fS )N)r¥   )r±   r.  Úsortr   rÔ   )rï   r„  ÚkwZinc_dimsÚrÚdim)r   r   r   Úexpand_dims_ref°  s    
z?TestOperators.test_squeeze_expand_dims.<locals>.expand_dims_refc                s>   t tˆ ƒƒ}|jdd� | }x|D ]}tj||d�}q"W |fS )NT)r<  )r¥   )r±   r.  rÁ  r   Zsqueeze)rï   r„  rÂ  Zdec_dimsrÃ  rÄ  )r   r   r   Úsqueeze_ref¸  s    
z;TestOperators.test_squeeze_expand_dims.<locals>.squeeze_ref)r”   rM   r@   r@  Zoutput_to_gradZgrad_reference)r    rh   rØ   r   rF   rI   )rJ   rï   rK   rL   rM   rÅ  rÆ  r   )r   r   Útest_squeeze_expand_dims£  s"    
z&TestOperators.test_squeeze_expand_dimsc             C   s@  d}t jj|ƒ ddddg}ddddg}dddddg}tjddddgd	g|||d
�}t jt jjdƒjt jƒdd�}t j	dgd ƒjt jƒ}	t
j|||ƒ}
| jjdƒj|ƒ | jjdƒj|	ƒ | jjdƒj|
ƒ | jj|ƒ | jjd	 jƒ }|jdgƒ}t j	d d!d"dd#d$dd%ddd&d'dd(d)d*gƒ}| jt jj|| ƒddd� d S )+NiÒ  r(   r   rB   ZTTrp   r¬   Úcoresr=   )Ú	inp_sizesÚ	out_sizesÚtt_ranksé   r   )r¥   g=]–ï¯>gêšÞ�Œµ>g<’ûÅE6“>gÃ¥„~Ã·Ž>gò²”ïtÀ>g)|ö
”¾>g&ÛÒ
>¯>gö^Íj‹>gíUÜ-†¼>gàP¥Â¨»>gFSKìÌ±>g™wr™D{Ÿ>gcPrWª¦q>gÛ£â©°K>gï…LÀŽ>g½ó©~±>g»½×Ùß|Û=)Údeltag=]–ï¯¾gêšÞ�Œµ¾g<’ûÅE6“¾gò²”ïtÀ¾g)|ö
”¾¾gö^Íj‹¾gFSKìÌ±¾g™wr™D{Ÿ¾gÛ£â©°K¾gï…LÀŽ¾g½ó©~±¾)r   r­   r®   r   rF   rÔ   rÇ   r°   r5   rW   r   Zinit_tt_coresrš   r²   r³   r—   r˜   r™   r  ÚassertAlmostEqualr  r  )rJ   rK   rL   r®   rÉ  rÊ  rË  rM   rp   r¬   rÈ  r=   Zgoldenr   r   r   Útest_tt_layerÈ  s>    
zTestOperators.test_tt_layerÚsimpleZdagZ	async_dag)Únum_workersÚnet_typec                sL  ddl m} ddlm} |dd�‰d‰d‰ d‰d}tjjd	ƒ �xttˆƒƒD �] }xútd| ƒD ]ê}	d
j	|d d|	 ƒ}
d
j	|d d|	 d ƒ}dj	|d d|	 ƒ}dj	|d d|	 d ƒ}d
j	||	ƒ}|j
ˆ|
|ˆ ˆ dttjjƒ d�fdttjjƒ d�fd� |j
ˆ||ˆ ˆ dttjjƒ d�fdttjjƒ d�fd� ˆjj||g|ƒ qbW qNW ˆjjddgdƒ ˆjjddƒ ˆjdgƒ‰ˆjƒ jjˆƒ ˆjjƒ jjˆƒ |ˆjƒ _|ˆjƒ _ˆjjˆjƒ ttˆjƒ ƒƒ ‡ ‡‡‡‡‡‡fdd„‰‡fdd„t|ƒD ƒ}xD|dd … D ]4}tjj|d |ƒ ˆjtjtj |ƒƒddd� �qW d S )Nr   )ÚModelHelper)ÚbrewZ
test_model)rN   r)   r(   rÐ   i¥  z{}_{}r   z{}_{}_mr˜  )r–  )Údim_inÚdim_outZweight_initZ	bias_initZ0_0ÚlabelZxentr‡   c                 s²   dd l } | jjdƒ ‡fdd„tdˆ ƒD ƒ}x\|D ]T}ˆjj|ƒj| jjˆˆ ƒj| j	ƒˆd� ˆjjdƒj| jjˆˆ ƒj| j	ƒˆd� q4W ˆjj
ˆjƒ ‡‡fdd„|D ƒ}|S )	Nr   i¥  c                s   g | ]}d j ˆ |ƒ‘qS )z{}_{})r{  )r‘   r8  )Údepthr   r   r›   '  s    zCTestOperators.test_dag_net_forking.<locals>.run.<locals>.<listcomp>r(   )r”   r×  c                s$   g | ]}ˆj jtˆ | ƒ jƒ ‘qS r   )rš   r˜   r¨  r™   )r‘   Ú
input_blob)Úinput_to_gradrJ   r   r   r›   1  s   )Únumpyr­   r®   rj   rš   r²   r³   r¯   r°   r5   r—   rš  )r   Zinput_blobsrÙ  Z	gradients)ÚdrØ  rK   rÚ  rõ   r:   rJ   r   r   r—   $  s    
z/TestOperators.test_dag_net_forking.<locals>.runc                s   g | ]
}ˆ ƒ ‘qS r   r   )r‘   r¹   )r—   r   r   r›   5  s    z6TestOperators.test_dag_net_forking.<locals>.<listcomp>gÜôg?RôV@g{®Gáz„?)rÍ  )!Zcaffe2.python.model_helperrÓ  Úcaffe2.pythonrÔ  r   r­   r®   Úreversedrj   r{  ÚfcÚdictr¯   rš  rf   ZSquaredL2Distancer†   ZAddGradientOperatorsÚProtor”   ZCopyFromÚparam_init_netÚtyperÑ  rš   r—   r»  r¨  rž   rŸ   rÎ  Úsumr  )rJ   rÒ  rÑ  rK   rL   rÓ  rÔ  Úitersrm   r8  Zbottom_1Zbottom_2Zmid_1Zmid_2ÚtopÚoutputsrã   r   )rÜ  rØ  rK   rÚ  rõ   r:   r—   rJ   r   Útest_dag_net_forkingô  sV    


z"TestOperators.test_dag_net_forking)r$   r%   )rá   Ú	slice_dimÚar¬   Úis_emptyc             C   sø   |t |jƒ }|r:tjjdgt|jƒ Ž jtjƒ}|d7 }||j|  }||j|  d }tjt |jƒtjd�}tj	t |jƒtjd�d }	t
||ƒ||< t||ƒ|	|< tjddddgdgƒ}
d	d
„ }| j||
|||	g|ƒ | j||
|||	gddgƒ d S )Nr   r   )r"   ÚSlicerá   rŒ  Úendrã   c             S   s0   t |jƒdkr| S dd„ t||ƒD ƒ}| | fS )Nr   c             S   s&   g | ]\}}t ||dkrdn|ƒ‘qS )r   Nr  )Úslice)r‘   ÚsiÚeir   r   r   r›   U  s    z?TestOperators.test_slice.<locals>.slice_ref.<locals>.<listcomp>)r    rh   r}  )r   ÚsÚeZslcr   r   r   Ú	slice_refR  s    z+TestOperators.test_slice.<locals>.slice_refr  )r    rh   r   r­   rÇ   r±   r°   r4   ri   ÚonesÚminÚmaxr   rF   rI   rH   )rJ   rá   ré  rê  r¬   rë  rK   rL   Z	start_vecZend_vecrM   ró  r   r   r   Ú
test_slice;  s$     zTestOperators.test_slicec             C   s,   t jddgdgƒ}| j|||gdd„ ƒ d S )NÚShaperï   rh   c             S   s   | j fS )N)rh   )r   r   r   r   r1   _  s    z*TestOperators.test_shape.<locals>.<lambda>)r   rF   rI   )rJ   rï   rK   rL   rM   r   r   r   Ú
test_shape\  s    zTestOperators.test_shapec             C   sP   dd„ }t jjt|jƒdd�jƒ }tjddgdg|d�}| j||||g|ƒ d S )	Nc                s   ‡ fdd„|D ƒfS )Nc                s   g | ]}ˆ j | ‘qS r   )rh   )r‘   rm   )r   r   r   r›   d  s    zITestOperators.test_shape_with_axes.<locals>.shape_ref.<locals>.<listcomp>r   )r   rX   r   )r   r   Ú	shape_refc  s    z5TestOperators.test_shape_with_axes.<locals>.shape_refr)   )rk   rø  rï   rh   )Úaxes)	r   r­   r¼  r    rh   r©  r   rF   rI   )rJ   rï   rK   rL   rú  rû  rM   r   r   r   Útest_shape_with_axesa  s    z"TestOperators.test_shape_with_axesc             C   sT   t jddgdgƒ}| j|||gdd„ ƒ t jddgdgƒ}| j|||gdd„ ƒ d S )	NZHasElementsrï   Zhas_elementsc             S   s   t | ƒdkfS )Nr   )r    )r   r   r   r   r1   l  s    z1TestOperators.test_has_elements.<locals>.<lambda>ZIsEmptyrë  c             S   s   t | ƒdkfS )Nr   )r    )r   r   r   r   r1   o  s    )r   rF   rI   )rJ   rï   rK   rL   rM   r   r   r   Útest_has_elementsi  s    zTestOperators.test_has_elements)Úinitial_itersÚ	max_itersc             C   sü   t jdƒ}|jdgdgƒ | jjdƒjtj|gƒjtj	ƒƒ | jjdƒjtj|gƒjtj	ƒƒ t jdƒ}|j
ddgdgƒ |jƒ jjdgƒ t jdƒ}|jt jd||gt jdƒd�ƒ | jj|ƒ | jjd jƒ }| j|jtj	ƒ | j|d	 t||ƒƒ d S )
Nrš  ÚiterÚ	num_itersZcriteriaÚstoprí   r®  )r¡  r   )r   rè   ÚIterrš   r²   r³   r   rø   r°   r.   rb   rá  Úexternal_outputrI  ré   rê   rë   ÚBlobReferencer—   r˜   r™   r´   r"   rö  )rJ   rþ  rÿ  rš  Zcriteria_netrí   rå  r   r   r   Ú/test_should_stop_as_criteria_net_execution_stepq  s"    


z=TestOperators.test_should_stop_as_criteria_net_execution_stepc             C   sà   dd„ }|ddƒ|ddƒ|ddƒg}t jd|d	 t jd
ƒd�t jd|d t jdƒd�t jd|d ƒg}dddg}t jdƒ}|jt jd|dd�ƒ | jj|ƒ x>t|ƒD ]2\}}| j| jj	dj
|d ƒ jƒ d	 || ƒ q¦W d S )Nc       	      S   s  dj | ƒ}dj | ƒ}tjdj | ƒƒ}|jg |gdgdd� |j|g|gdd� tjd	j | ƒƒ}|jg dg|rpd
ndd�}|j|g|gdd� |jƒ jj|gƒ tjdj | ƒƒ}|jg |gdgd
d� tjdj | ƒƒ}|jg dgd
d�}|j|g|gdd� |jƒ jj|gƒ ||||gS )Nzshould_stop_{}z	output_{}zinit_{}r   g        )rh   r–  Úbool)Útozcriterion_{}g      ð?znet_{}zender_{})r{  r   rè   r˜  ÚCastrá  r  rI  )	rm   ÚdisabledZshould_stoprã   r£  Z	criterionÚtmprš  Úenderr   r   r   Ú
createNets‰  s<    

z>TestOperators.test_disabled_execution_step.<locals>.createNetsr   Fr(   TrB   Zstep_1r   Zshould_stop_1)r¡  Zstep_2Zshould_stop_2Zstep_3g      ð?g        rí   Z	all_steps)rå   z	output_{})r   rë   r  ré   rê   rš   r—   rµ   r´   r˜   r{  r™   )rJ   r  ÚnetsZstepsÚexpectedrí   rm   r¹   r   r   r   Útest_disabled_execution_stepˆ  s&    -

z*TestOperators.test_disabled_execution_step)rþ  r  c             C   s¬   t jdƒ}|jdgdgƒ | jjdƒjtj|gƒjtj	ƒƒ t j
d|gƒ}|j|ƒ t jdƒ}|j|ƒ | jj|ƒ | jjd jƒ }| j|jtj	ƒ | j|d || ƒ d S )Nrš  r   r®  rí   r   )r   rè   r  rš   r²   r³   r   rø   r°   r.   ÚExecutionStepZSetIterré   rê   r—   r˜   r™   r´   r"   )rJ   rþ  r  rš  r®  rí   rå  r   r   r   Ú#test_iter_count_with_execution_stepÍ  s    



z1TestOperators.test_iter_count_with_execution_step)rþ  r  Únum_netsc             C   sŒ  t jdƒ}|jg dgƒ}| jjdƒjtj|gƒjtj	ƒƒ t j
d|d�}xJt|ƒD ]>}t jdj|ƒƒ}|j|dgdgƒ t j
d|gƒ}	|j|	ƒ qRW |jdƒ t jd	ƒ}
|
j|ƒ t jd
ƒ}|jg dddgƒ | jj|ƒ | jj|
ƒ | jj|ƒ | jjd jƒ }| j|jtj	ƒ | j|d |||  ƒ || dk�rˆ| jjd jƒ }d}| j||kƒ | jjd jƒ }| j|| |t|ƒj|ƒ ƒ d S )Nr•  Ú
iter_mutexr   Úconcurrent_steps)rå   znet_{}r®  Trí   Ú	stats_netÚ	stats_keyZ	stats_valZstats_tsr   s   atomic_iter/stats/iter/num_iter)r   rè   ZCreateMutexrš   r²   r³   r   rø   r°   r.   r  rj   r{  Z
AtomicIterZ
AddSubstepZSetConcurrentSubstepsré   rê   ZStatRegistryExportr—   r˜   r™   r´   r"   rŽ  r±   Úindex)rJ   rþ  r  r  r•  r  r  rm   rš  r®  rí   r  rå  r  Zatomic_iter_keyZstat_valr   r   r   Ú&test_atomic_iter_with_concurrent_stepsà  s8    




z4TestOperators.test_atomic_iter_with_concurrent_steps)rê  ÚsrcÚdstÚuse_namec                sÆ   |j |ƒ}tjtjg}||krLˆ |krLˆ tjk	rLtjˆ ƒ}tj||j|jƒ}‡ fdd„}	t	ˆ  }
|rvt
jjj|
ƒjƒ }
tjddgdg|
d�}| j|||gdgƒ | j|||g|	ƒ\}| jˆ |jƒ d S )Nc                s   | j ˆ ƒgS )N)r°   )rï   )r  r   r   rO     s    z$TestOperators.test_cast.<locals>.refr	  rp   r=   )r  r   )r°   r   r5   r€   r  ZiinfoÚcliprõ  rö  Ú_NUMPY_TYPE_TO_ENUMr   ZTensorProtoÚDataTypeÚNameÚlowerr   rF   rG   rI   r´   r"   )rJ   rê  r  r  r  rK   rL   ZftypesÚinforO   r  rM   rA   r   )r  r   Ú	test_cast  s    

zTestOperators.test_castg-Cëâ6?g®Gáz®ï?)r#   )rê  ÚepsÚa_gradÚeps_gradc       
   	      s~   ‡ fdd„}t jddgdgˆ d�}| j|||gdgƒ | j|||g|ƒ t jddgdg|d�}	| j||	|gddgdd	d
� d S )Nc                s&   t j| ˆ dˆ  ƒ} t j| d|   ƒfS )Ng      ð?r   )r   r  rm  )rï   )r$  r   r   rO   '  s    z%TestOperators.test_logit.<locals>.refZLogitrp   r=   )r$  r   g{®Gáz¤?gü©ñÒMb`?)rÆ   rÅ   )r   rF   rG   rI   rH   )
rJ   rê  r$  r%  r&  rK   rL   rO   rM   Zop_gradr   )r$  r   Ú
test_logit!  s    zTestOperators.test_logitT)r2  )rê  r–  c                sL   ‡ fdd„}t jddgdgˆ d�}| j|||gdgƒ | j|||g|ƒ d S )Nc                s   t j| ƒ}ˆ |t j| ƒ< |fS )N)r   rœ   Úisnan)rï   rA   )r–  r   r   rO   >  s    
z+TestOperators.test_replace_nan.<locals>.refZ
ReplaceNaNrp   r=   )r–  r   )r   rF   rG   rI   )rJ   rê  r–  rK   rL   rO   rM   r   )r–  r   Útest_replace_nan:  s    zTestOperators.test_replace_nan)r7   c             C   s   t jd| t j| ƒd�S )Nr   )r$   r"   r#   )r   Útensorr<   )r"   r   r   r   r1   H  s   zTestOperators.<lambda>)rï   Ú	has_inputÚhas_extra_shapeÚextra_shapec                sð   |j j‰ |j tj tjƒkr tj‰ |jdƒ‰|j‰|g}tˆ  }|rˆ|rntjddgdg||ˆd�}	ˆt	|ƒ7 ‰q¨tjddgdg|ˆd�}	n tjdg dg|ˆt
ˆƒd�}	g }d
‡ ‡‡fdd	„	}
| j||	|dgƒ | j||	||
ƒ\}| jˆ |j ƒ d S )Nr   r˜  rp   r=   )r"   r-  r–  )r"   r–  )r"   r–  rh   c                s   t jˆˆˆ d�}|gS )N)rh   Z
fill_valuer"   )r   Úfull)r@   rç  )r"   Úgt_shaper–  r   r   rO   m  s    z-TestOperators.test_constant_fill.<locals>.ref)N)r"   rã  r   r  r9  rh   r  r   rF   Útupler±   rG   rI   r´   )rJ   rï   r+  r,  r-  rK   rL   r@   Z	enum_typerM   rO   rA   r   )r"   r/  r–  r   Útest_constant_fillG  s4    


z TestOperators.test_constant_fill)rw  r:   rÜ  c                sÜ  ddl m}m} tjjdƒ |jdd�}|jƒ jj	dddd	d
dgƒ d|jƒ _
|jƒ jj	ddgƒ |j|d	d||dd� |jjddgdgƒ |jjdgdgƒ x|jjƒ jD ]}tj|ƒ q¬W tjj|jƒ jddiƒ\}}	dd„ t|	ƒD ƒ}	tjdƒ}
|
jƒ jd d …= |
jƒ jj	|ƒ |	d dk�s(t‚d,d-d.g}t|Ž \}}}d/d0d1g}t|Ž \}}}|
jƒ jj	dgƒ |
jƒ jj	|jƒ jƒ |
jƒ jj	|jƒ jƒ ddd
ddg‰ dg}tjdˆ dddd gddgddgdd2gdg‡ fd!d"„|D ƒ||||||‡ fd#d"„|jD ƒ|jƒ |
jƒ dgd$�}tjdtjj|||ƒjtj ƒƒ tjdtjjd||ƒjtj ƒƒ tjdtjj!d||fd%�jtj"ƒƒ d&d'„ }| j#t$j%|d(d"„ d3D ƒ|dddgd)� x.d4D ]&}| j&t$j%|d+d"„ d5D ƒ|dgƒ �q®W d S )6Nr   )Úmodel_helperrÔ  i¥  ZElman)rN   Úinput_tÚseq_lengthsZtimestepÚhidden_t_prevÚ	gates_t_wÚ	gates_t_brÐ  Úhidden_tZgates_tr(   )rÕ  rÖ  r¥   Úhidden_t_gradc             S   s   i | ]\}}t |ƒt |ƒ“qS r   )r¨  )r‘   rˆ  r´  r   r   r   ú
<dictcomp>�  s   z>TestOperators.test_elman_recurrent_network.<locals>.<dictcomp>ZElmanBackwardÚgates_t_gradÚhiddenr   rá   Úhidden_t_prev_gradÚhidden_gradÚ
input_gradÚhidden_inputZRecurrentNetworkrã   Zhidden_outputZstep_workspacesc                s   g | ]}ˆ j |ƒ‘qS r   )r  )r‘   rm   )r@   r   r   r›   ±  s    z>TestOperators.test_elman_recurrent_network.<locals>.<listcomp>c                s   g | ]}ˆ j |ƒ‘qS r   )r  )r‘   r©   )r@   r   r   r›   ¹  s    )Z	alias_srcZ	alias_dstZalias_offsetZrecurrent_statesZinitial_recurrent_state_idsÚlink_internalÚlink_externalÚlink_offsetÚbackward_link_internalÚbackward_link_externalÚbackward_link_offsetr¸   Ústep_netÚbackward_step_netZoutputs_with_grads)rk   c             S   s  | j d }| j d }| j d }tj|d ||fd�}|j d |d ksJt‚|j d |ks\t‚|j d |ksnt‚||dd d …d d …f< xnt|ƒD ]b}	| |	 jd||ƒ}
||	 jd||ƒ}tj||jƒ}|jd||ƒ|
jd||ƒ }t|ƒ||	d < qŽW |dd … ||d jd||ƒfS )Nr   r   r(   )rh   r  )	rh   r   ri   r    rj   r  ÚdotrÃ   r   )rá   r4  Zgates_wZgates_br@  rÃ   r·   rÄ   r<  rw  r3  r5  Zgatesr   r   r   r@  Å  s    


z=TestOperators.test_elman_recurrent_network.<locals>.referencec             S   s   g | ]}t j|ƒ‘qS r   )r   rì   )r‘   rN   r   r   r   r›   Ú  s   )Zoutputs_to_checkrB   c             S   s   g | ]}t j|ƒ‘qS r   )r   rì   )r‘   rN   r   r   r   r›   ä  s   )r5  r<  r   )r8  r<  r   )r3  rá   r   )r=  r>  r   )r9  r>  r   )r;  r?  r   r  )rá   r4  r6  r7  r@  )r   r(   rB   )rá   r4  r6  r7  r@  )'rÝ  r2  rÔ  r   r­   r®   rÓ  rá  Zexternal_inputrI  rã  r  rß  rš  rf   ZSigmoidrâ  rM   r   ZRunOperatorOncer   ZGradientRegistryZGetBackwardPassr   rè   r    r}  rF   Úparamsrç   r¯   r°   r5   r¼  r4   rI   r   rA  rH   )rJ   rw  r:   rÜ  r2  rÔ  rG  rM   Zbackward_opsZbackward_mappingrH  ÚlinksrA  rB  rC  Zbackward_linksrD  rE  rF  Zrecurrent_inputsr@  r¸   r   )r@   r   Útest_elman_recurrent_networku  s     






  
 z*TestOperators.test_elman_recurrent_network)Zsuppress_health_check)r:   ÚcÚhr  ÚpadÚ
block_sizec	             C   s�   t |d|  | dkƒ t |d|  | dkƒ tjj||||ƒjtjƒ}	tjddgdg||d�}
| j||
|	gdgƒ | j	||
|	gddgƒ d S )Nr(   r   ZSpaceToBatchrp   r=   )rO  rP  )
r	   r   r­   r¯   r°   r5   r   rF   rG   rH   )rJ   r:   rM  rN  r  rO  rP  rK   rL   rp   rM   r   r   r   Útest_space_to_batchê  s    	
z!TestOperators.test_space_to_batchc	             C   s°   t |d|  | dkƒ t |d|  | dkƒ tjj|| | ||d|  | |d|  | ƒjtjƒ}	tjddgdg||d�}
| j||
|	gdgƒ | j	||
|	gddgƒ d S )Nr(   r   ZBatchToSpacerp   r=   )rO  rP  )
r	   r   r­   r¯   r°   r5   r   rF   rG   rH   )rJ   r:   rM  rN  r  rO  rP  rK   rL   rp   rM   r   r   r   Útest_batch_to_spaceû  s    	

z!TestOperators.test_batch_to_space)rp   rU   Úscalec             C   sL   t jddg|sdndg|d�}| j|||gdgƒ | j|||gddgƒ d S )NZScalerp   r=   )rS  r   )r   rF   rG   rH   )rJ   rp   rU   rS  rK   rL   rM   r   r   r   Ú
test_scale  s
    zTestOperators.test_scale)rñ  c             C   sv   |j ddƒ}| jjdƒj|ƒ | jjd jdƒ}| jjdƒj|ƒ | j|| jjd jƒ ƒ | j|| jjd jƒ ƒ d S )NÚasciiÚignorerê  r¬   )	Úencoderš   r²   r³   r˜   Ú	serializeZdeserializer´   r™   )rJ   rñ  Z
serializedr   r   r   Útest_string_serde  s    zTestOperators.test_string_serdeZconstantZreflectZedge)rO  rk   Úinput_channelsÚ
batch_sizerÞ   Úmodec	                sÈ   t |ˆkƒ tjddgdgˆˆ |d�}	|dkr\tjj||||ƒjtjƒd }
‡ ‡fdd„}n,tjj||||ƒjtjƒd }
‡ ‡fd	d„}| j||	|
g|ƒ | j	||	|
gd
gƒ | j
||	|
gd
d
gƒ d S )NÚPadImagerp   r=   )rO  r\  rÞ   rÝ   g      à?c                s    t j| dˆˆfˆˆfdfˆ ƒfS )Nr   )r   r   )r   r   )r   rO  )r   )r\  rO  r   r   Únumpy_pad_ref;  s    z8TestOperators.test_same_pad_image.<locals>.numpy_pad_refc                s    t j| ddˆˆfˆˆffˆ ƒfS )Nr   )r   r   )r   r   )r   rO  )r   )r\  rO  r   r   r^  C  s    r   )r	   r   rF   r   r­   rÇ   r°   r5   rI   rG   rH   )rJ   rO  rk   rZ  r[  rÞ   r\  rK   rL   rM   rp   r^  r   )r\  rO  r   Útest_same_pad_image$  s$    	z!TestOperators.test_same_pad_image)	Úpad_tÚpad_lÚpad_bÚpad_rrk   rZ  r[  rÞ   r\  c                sä   t |tˆˆˆˆƒkƒ tjddgdgˆˆˆˆˆ |d�	}|dkrrtjj||||ƒjtjƒd }‡ ‡‡‡‡fdd„}n2tjj||||ƒjtjƒd }‡ ‡‡‡‡fd	d„}| j	|
||g|ƒ | j
|||gd
gƒ | j|
||gd
d
gƒ d S )Nr]  rp   r=   )r`  ra  rb  rc  r\  rÞ   rÝ   g      à?c                s    t j| dˆˆfˆˆfdfˆ ƒfS )Nr   )r   r   )r   r   )r   rO  )r   )r\  rb  ra  rc  r`  r   r   r^  h  s    z3TestOperators.test_pad_image.<locals>.numpy_pad_refc                s    t j| ddˆˆfˆˆffˆ ƒfS )Nr   )r   r   )r   r   )r   rO  )r   )r\  rb  ra  rc  r`  r   r   r^  q  s    r   )r	   rö  r   rF   r   r­   rÇ   r°   r5   rI   rG   rH   )rJ   r`  ra  rb  rc  rk   rZ  r[  rÞ   r\  rK   rL   rM   rp   r^  r   )r\  rb  ra  rc  r`  r   Útest_pad_imageK  s*    zTestOperators.test_pad_imagerÑ   )rk   rZ  r[  rÞ   rü   c                sb  t jddddgdg|ˆd�}tjjdƒ tjjˆƒjtjƒd }	tjjˆƒjtjƒd }
tjjˆ ˆˆˆƒjtjƒd }|d	kr’|jd
dƒjddƒ}‡ ‡‡‡fdd„‰‡fdd„}| j	||||	|
g|dkrÌˆn|ƒ t
jjƒ }d|fd|	fd|
fg}x |D ]\}}|j|ƒj|ƒ qøW xtdƒD ]}|j|ƒ �qW x*|D ]"\}}tjj|j| jƒ |ƒ �q8W d S )NZInstanceNormrp   rS  Úbiasr=   )rÞ   rü   i¥  g      à?rÝ   r   r(   rB   c                s”   | j ˆ ˆ ˆˆ ƒ} | | jdƒd d …tjf  }|tj| jdƒˆ ƒd d …tjf  }|j ˆ ˆˆˆƒ}||j dˆddƒ }||j dˆddƒ }|fS )Nr   )r  Zmeanr   Znewaxisr  r  )r   rS  re  rX   )r[  rü   rZ  rk   r   r   Úref_nchw‘  s    &z2TestOperators.test_instance_norm.<locals>.ref_nchwc                s:   | j ddƒj ddƒ} ˆ | ||ƒd }|j ddƒj ddƒfS )Nr(   rB   r   r   )Úswapaxes)r   rS  re  rX   )rf  r   r   Úref_nhwcš  s    z2TestOperators.test_instance_norm.<locals>.ref_nhwcrÜ   rC   )r   rF   r   r­   r®   rÇ   r°   r5   rg  rI   r   r•   r–   r²   r³   rj   r—   rž   rŸ   r˜   r™   )rJ   rk   rZ  r[  rÞ   rü   rK   rL   rM   rS  re  rp   rh  rš   ZfeedsZblobZarrr¹   r   )r[  rü   rZ  rf  rk   r   Útest_instance_normz  s4    	
z TestOperators.test_instance_normÚinpc             C   s   t tjddd�| d�S )Nr   r   )r*   r+   )r#   r"   )r3   r   Úfloats)Údtr   r   r   r1   °  s   c       
      C   sÞ   |\}}}|j tj dƒkr@|jtjtjhkr@tdj|j ƒƒ d S |jtjtjhkr^|jtj	ƒ}tj
|f|jdd …  ƒj|j ƒ}tjddddgdgƒ}d	d
„ }	| j|||||g|	ƒ |jtjƒ}| j|||||gddgƒ d S )Nr5   z
Bailout {}r   ZSparseToDenseÚIrp   rÄ   r=   c             S   s<   t j|jƒ}x(t| ƒD ]\}}||  || 7  < qW |gS )N)r   ri   rh   rµ   )rm  rp   rÄ   ÚOrm   r©   r   r   r   Úsparse_to_denseÂ  s    z;TestOperators.test_sparse_to_dense.<locals>.sparse_to_denser   )r"   r   r�   r   ZCUDAZHIPr»  r{  r°   r4   ri   rh   r   rF   rI   r5   rH   )
rJ   rj  rK   rL   Z	first_dimrp   rm  rÄ   rM   ro  r   r   r   Útest_sparse_to_dense°  s    
""z"TestOperators.test_sparse_to_densec             C   s€   |\}}t jdddgdƒ}dd„ }| j||||g|ƒ | j||||gdgƒ | j||||gddgƒ | j||||gddgƒ d S )	NZ
DotProductrp   r=   rA   c             S   s   t j| | dd�fS )Nr   )r¥   )r   rä  )rp   r=   r   r   r   Ú
dotproductÑ  s    z2TestOperators.test_dot_product.<locals>.dotproductr   r   )r   rF   rI   rG   rH   )rJ   r@   rK   rL   rp   r=   rM   rq  r   r   r   Útest_dot_productÌ  s    zTestOperators.test_dot_product)r·   ÚMr¶   Ú	pad_valuec                sº   t jjˆˆƒjt jƒd }t jjˆˆ ƒjt jƒd }tjdddgdˆd�}	‡ ‡‡‡fdd„}
| j||	||g|
ƒ | j||	||gd	gƒ | j	||	||gd	d	gƒ | j	||	||gd
d	gƒ d S )Ng      à?ÚDotProductWithPaddingrp   r=   rA   )rt  c                s|   t jˆtˆˆ ƒfƒjt jƒˆ }ˆˆ k rP| |d d …d ˆ…f< t j|| dd�fS ||d d …d ˆ …f< t j||  dd�fS d S )Nr   )r¥   )r   rô  rö  r°   r5   rä  )rp   r=   ÚZ)r¶   rs  r·   rt  r   r   rq  ä  s     z?TestOperators.test_dot_product_with_padding.<locals>.dotproductr   r   )
r   r­   rÇ   r°   r5   r   rF   rI   rG   rH   )rJ   r·   rs  r¶   rt  rK   rL   rp   r=   rM   rq  r   )r¶   rs  r·   rt  r   Útest_dot_product_with_paddingÙ  s    	z+TestOperators.test_dot_product_with_padding)r·   rs  rt  c       
         sÀ   dˆ ‰ t jj|ˆƒjt jƒd }t jj|ˆ ƒjt jƒd }tjdddgdd|d�}‡ ‡fd	d
„}	| j||||g|	ƒ | j||||gdgƒ | j	||||gddgƒ | j	||||gddgƒ d S )Nr(   g      à?ru  rp   r=   rA   T)Z	replicatert  c                sb   dd l j}ˆˆ k r8|j| dˆ ˆ ƒ}tj|| dd�fS |j|dˆˆ  ƒ}tj||  dd�fS d S )Nr   r   )r¥   )Znumpy.matlibZmatlibZrepmatr   rä  )rp   r=   Znpmrv  )r¶   rs  r   r   rq  þ  s    
zCTestOperators.test_dot_product_with_rep_padding.<locals>.dotproductr   r   )
r   r­   rÇ   r°   r5   r   rF   rI   rG   rH   )
rJ   r·   rs  rt  rK   rL   rp   r=   rM   rq  r   )r¶   rs  r   Ú!test_dot_product_with_rep_paddingò  s    	z/TestOperators.test_dot_product_with_rep_padding)r·   rs  c             C   s¬   t jj||ƒjt jƒd }tjddgdƒ}| j|||gdd„ ƒ | j|||gdgƒ t jj||ƒjt jƒd }tjddgdƒ}| j|||gdd„ ƒ | j|||gdgƒ d S )	Ng      à?ZEnsureDenserp   c             S   s   | gS )Nr   )r   r   r   r   r1   	  s    z1TestOperators.test_ensure_dense.<locals>.<lambda>r   rA   c             S   s   | gS )Nr   )r   r   r   r   r1   	  s    )	r   r­   rÇ   r°   r5   r   rF   rI   rG   )rJ   r·   rs  rK   rL   rp   rM   r   r   r   Útest_ensure_dense	  s    zTestOperators.test_ensure_dense)r·   rs  Únum_bucketsc       	         sz   t jjˆˆ ƒjt jƒ}d\‰‰tjddgddgˆˆˆd�}‡ ‡‡‡‡fdd	„}| j|||gd
dgƒ | j|||g|ƒ d S )Nçš™™™™™¹?çÍÌÌÌÌÌì?ZAccumulateHistogramrp   Úcur_histÚacc_hist)Úlower_boundÚupper_boundrz  c                sä   t jˆd ft jd�}ˆˆ ˆ }t jˆˆ ft jd�}d|| ˆk < ˆd || ˆk< | | ˆk| ˆk @  ˆ | d jt jƒ|| ˆk| ˆk @ < xFt|jd ƒD ]4}x.t|jd ƒD ]}||| |   d7  < q®W qšW || }}||gS )Nr(   )r"   r   r   )r   ri   r4   r°   rj   rh   )rp   ÚhistZsegmentr=   rm   r8  r}  r~  )rs  r·   r  rz  r€  r   r   Ú	histogram'	  s    " 
z=TestOperators.test_accumulate_histogram_op.<locals>.histogramr   r   )r{  r|  )	r   r­   rÇ   r°   r5   r   rF   rG   rI   )	rJ   r·   rs  rz  rK   rL   rp   rM   r‚  r   )rs  r·   r  rz  r€  r   Útest_accumulate_histogram_op	  s    
z*TestOperators.test_accumulate_histogram_op)rÜ   r   )rÝ   rB   r(   r(   iöÿÿÿiöÿÿÿiöÿÿÿg       À)sÚ__name__Ú
__module__Ú__qualname__re   r
   r   r;   r   r[   rR   rg   rc   rn   rs   rz   r|   r~   rƒ   r*  r„   r…   rˆ   r�   r   r6   Zexpanded_device_optionsr¢   r/   rº   ÚunittestZskipIfr   Zhas_gpu_supportrk  rÊ   rÛ   rà   r!   rî   r   r5   r€   r4   r  rð   rú   r  Ústaticmethodr  r  r  r  r&  r'  r(  r)  r1  rB  rF  rL  rM  rQ  rR  rW  r\  r`  rc  rf  ri  rk  ro  rx  rŽ   r’  rµ  r¿  rÇ  rÏ  rè  r÷  rù  rü  rý  r  r  r  r  r±   r   r  r#  r'  r)  r8   r.   r2   r1  rL  r   r   Zfilter_too_muchrQ  rR  rT  ÚtextrY  r_  rd  ri  rp  rr  rw  rx  ry  rƒ  r   r   r   r   r]   s   sV  	*($&  &!


B

 ('"


#


",










U


;

W#,C
   E
%


'
s









	


!





&

1((r]   Ú__main__)r   r   )N)DÚ
__future__r   r   r   rÛ  r   rœ   ru  Ú	functoolsr   r   Zfuture.utilsr   r   Z
hypothesisr	   r
   r   r   Zhypothesis.strategiesZ
strategiesr   r‡  ÚosrÝ  r   r   r   r   Z"caffe2.python.hypothesis_test_utilÚpythonZhypothesis_test_utilr   Zcaffe2.protor   ZInitOpsLibraryr   Z	compositer'   r5   r3   r  ÚFLOATr4   ZINT32r  ZBOOLZuint8ZUINT8Zint8ZINT8Zuint16ZUINT16Zint16ZINT16r.   ZINT64r€   ZDOUBLEr  r8   rR   rT   r\   ZHypothesisTestCaser]   r„  Úmainr   r   r   r   Ú<module>   sl   











                 Z
