3
I4W^ì1 ã               @   s2  d dl mZ d dl mZ d dl mZ d dl mZ d dlZd dlmZmZ d dl	m
Z
mZmZ d dlZd dlZd dlmZ d d	lmZ d dljjZd d
lmZmZmZ d dlZd dlZd dlZdd„ Zdd„ Zdd„ Zdd„ Zdd„ Z �d“dd„Z!�d”dd„Z"�d•dd„Z#dd„ Z$dd„ Z%dd „ Z&ed!d"ƒd#d$„ ƒZ'ed!d"ƒd%d&„ ƒZ(d'd(„ Z)d)d*„ Z*d+d,„ Z+ed!d!d!d-d-ƒd.d/„ ƒZ,d0d1„ Z-d2d3„ Z.d4d5„ Z/d6d7„ Z0d8d9„ Z1d:d;„ Z2d<d=„ Z3d>d?„ Z4d@dA„ Z5dBdC„ Z6dDdE„ Z7dFdG„ Z8dHdI„ Z9�d–dKdL„Z:dMdN„ Z;�d—dOdP„Z<e<dQdRƒZ=e<dSdTƒZ>e<dUdVdWdX�Z?ed!d"dYƒdZd[„ ƒZ@d\d]„ ZAd^d_„ ZBd`d-„ ZCdadb„ ZDdcdd„ ZEdedf„ ZFed!d!d!d"d"d"d!ƒdgdh„ ƒZGdidj„ ZHed!d"d"ƒdkdl„ ƒZIed!dmƒdndo„ ƒZJdpdq„ ZKdrds„ ZLdtdu„ ZMed!d"d"ƒdvdw„ ƒZNed!dmd"ƒdxdy„ ƒZOed!d"d!ƒdzd{„ ƒZP�d˜d|d}„ZQd~d„ ZRd€d�„ ZSd‚dƒ„ ZTd„d…„ ZUed!d-d-ƒd†d‡„ ƒZV�d™dˆd‰„ZWed!d"ƒdŠd‹„ ƒZXed!d"dYƒ�dšdŒd�„ƒZYed!d-d!ƒdŽd�„ ƒZZd�d‘„ Z[d’d“„ Z\e\d”e
d•dWd–�Z]e\d—ed˜dWd–�Z^e\d™edšdWd–�Z_e\d›e
d•dJd–�Z`e\dœed˜dJd–�Zae\d�edšdJd–�ZbdždŸ„ Zcecd e
ƒZdecd¡eƒZeecd¢eƒZf�d›d£d¤„Zgegd¥d¦e
ƒZhegd§d¦eƒZiegd¨d¦eƒZjegd©dªe
e`ƒZkegd«dªeeaƒZlegd¬dªeebƒZmed!dmd­ƒd®d¯„ ƒZned!dmƒd°d±„ ƒZoed!dmƒd²d³„ ƒZpeoZqeoZreoZsepZtepZuepZvd´dµ„ Zwewd¶dšd·ƒZxewd¸d¹d·ƒZyewdºd»d·ƒZzewd¼dšd½ƒZ{ewd¾d¹d½ƒZ|ewd¿d»d½ƒZ}dÀdÁ„ Z~dÂdÃ„ ZdÄdÅ„ Z€dÆdÇ„ Z�e€dÈdÉ„ ƒZ‚dÊdË„ ZƒdÌdÍ„ Z„dÎdÏ„ Z…dÐdÑ„ Z†e€dÒdÓ„ ƒZ‡e€dÔdÕ„ ƒZˆedÖƒd×dØ„ ƒZ‰edÖƒdÙdÚ„ ƒZŠdÛdÜ„ Z‹ed!d"dYƒ�dœdÝdÞ„ƒZŒed!d!d!dmdmdmd"dmd"d"d"d"ƒdßdà„ ƒZ�ed!d!d!d!d!d"d­d­d"ƒ	dádâ„ ƒZŽed!dmd!d!d­d"ƒdãdä„ ƒZ�ed!d!d!d!d!d"d­d­d"ƒ	dådæ„ ƒZ�ed!d"d"d"ƒdçdè„ ƒZ‘ed!d!d"ƒdédê„ ƒZ’ed!d-d-d-ƒdëdì„ ƒZ“dídî„ Z”ed!d"d!ƒdïdð„ ƒZ•dñdò„ Z–dódô„ Z—dõdö„ Z˜d÷dø„ Z™ed!d!d"d­ƒdùdú„ ƒZšdûdü„ Z›dýdþ„ Zœdÿ�d „ Z��d�d„ Zž�d�d„ ZŸ�d�d„ Z ed!d­ƒ�d�d„ ƒZ¡ed!d­ƒ�d	�d
„ ƒZ¢�d��d�d„Z£�dž�d�d„Z¤�d�d„ Z¥ed!d­d"ƒ�d�d„ ƒZ¦�d�d„ Z§e§�dƒZ¨e§�dƒZ©e§�dƒZªe¦Z«e¨Z¬e©Z­eªZ®ed!d-dmd"ƒ�d�d„ ƒZ¯ed!d!d!d"ƒ�d�d„ ƒZ°ed!d"d"ƒ�d�d„ ƒZ±ed!d"d"d"ƒ�d�d„ ƒZ²x@ej³j´ƒ D ]2\ZµZ¶�d j·eµƒZ¸ed!d"ƒeej¹e¶ƒƒeºƒ e¸< �q*W ed!d"d!d!d!d!ƒ�dŸ�d!�d"„ƒZ»ed!d"d!d!d!d!ƒ�d �d#�d$„ƒZ¼ed!d"d!d!d!ƒ�d¡�d%�d&„ƒZ½ed!d"d!d!d!ƒ�d¢�d'�d(„ƒZ¾ed!d"d!d!d!ƒ�d£�d)�d*„ƒZ¿ed!d"d!d!d!ƒ�d¤�d+�d,„ƒZÀ�d¥�d-�d.„ZÁed!d­d"d!d!d!ƒ�d¦�d/�d0„ƒZÂed!d!d!d!d"ƒ�d1�d2„ ƒZÃed!d­d­ƒ�d3�d4„ ƒZÄ�d5�d6„ ZÅed!d"ƒ�d7�d8„ ƒZÆed!d"d"dYƒ�d§�d9�d:„ƒZÇed!d"d"d"d"dYƒ�d¨�d;�d<„ƒZÈ�d=�d>„ ZÉ�d?�d@„ ZÊed!d"ƒ�dA�dB„ ƒZËed!d"d!d!d­d"ƒ�dC�dD„ ƒZÌ�d©�dE�dF„ZÍed!d!d!d"d"d­d"d"d"ƒ	�dG�dH„ ƒZÎed!d!d!d!d"d"d­d"d"ƒ	�dI�dJ„ ƒZÏ�dK�dL„ ZÐ�dM�dN„ ZÑeÑ�dOƒZÒeÑ�dPƒZÓeÑ�dQƒZÔed!d"ƒ�dR�dS„ ƒZÕ�dT�dU„ ZÖed!d"ƒ�dV�dW„ ƒZ×ed!d!d"ƒ�dX�dY„ ƒZØed!d!d"d-d!ƒ�dZ�d[„ ƒZÙ�d\�d]„ ZÚ�d^�d_„ ZÛ�d`�da„ ZÜed!d­d­d"dYƒ�db�dc„ ƒZÝed!ƒ�dd�de„ ƒZÞed!ƒ�df�dg„ ƒZßed!d"d"ƒ�dh�di„ ƒZàed!ƒ�dj�dk„ ƒZáed!ƒ�dl�dm„ ƒZâed!d"d"d"ƒ�dn�do„ ƒZã�dp�dq„ Zä�dr�ds„ Zåed!d"d!d!ƒ�dt�du„ ƒZæed!d"d!d!ƒ�dv�dw„ ƒZç�dx�dy„ Zè�dz�d{„ Zéed!d"d!d!ƒ�dª�d|�d}„ƒZêed!dm�d~d"ƒ�d�d€„ ƒZë�d��d‚„ Zìed!dmd"ƒ�dƒ�d„„ ƒZí�d…�d†„ Zî�d‡�dˆ„ Zï�d‰�dŠ„ Zðed!dmd"ƒ�d«�d‹�dŒ„ƒZñed!d"�d~d!ƒ�d¬�d��dŽ„ƒZò�d��d�„ Zó�d‘�d’„ ZôdS (­  é    )Úabsolute_import)Údivision)Úprint_function)Úunicode_literalsN)ÚListTypeÚOptionalType)Ú_singleÚ_pairÚ_triple)Úpartial)Úwraps)Ú
parse_argsÚ
_parse_argÚ_unimplementedc             C   s   | j dƒ}|jtjƒ ƒ |S )Nzprim::Constant)ÚopÚsetTyper   ZofTensor)ÚgÚn© r   ú;/tmp/pip-build-e66j97b2/torch/torch/onnx/symbolic_opset9.pyÚunused?   s    
r   c             C   s   | j d|ƒS )NÚShape)r   )r   Úinputr   r   r   Ú_shape_as_tensorD   s    r   c             C   s   | j d||ƒS )NÚReshape)r   )r   r   Úshaper   r   r   Ú_reshape_from_tensorH   s    r   c             C   s   t | ||ƒS )N)Úview)r   Úselfr   r   r   r   ÚreshapeL   s    r   c             C   s   | j d|ƒ}t| ||ƒS )Nr   )r   r   )r   r   Úotherr   r   r   r   Ú
reshape_asP   s    r!   c             C   s0   |r"t jt j|ƒƒdkr"tddƒS | jd||ƒS )Né   Úaddz
alpha != 1ÚAdd)Úsym_helpÚ_scalarÚ_maybe_get_scalarr   r   )r   r   r    Úalphar   r   r   r#   U   s    
r#   c             C   s0   |r"t jt j|ƒƒdkr"tddƒS | jd||ƒS )Nr"   Úsubz
alpha != 1ÚSub)r%   r&   r'   r   r   )r   r   r    r(   r   r   r   r)   \   s    
r)   c             C   s   t | |||d�S )N)r(   )r)   )r   r   r    r(   r   r   r   Úrsubc   s    r+   c             C   s   | j d||ƒS )NÚMul)r   )r   r   r    r   r   r   Úmulg   s    r-   c             C   s   | j d||ƒS )NÚDiv)r   )r   r   r    r   r   r   Údivk   s    r/   c             C   s   | j dtjdƒ|ƒS )Nr.   r"   )r   ÚtorchÚones)r   r   r   r   r   Ú
reciprocalo   s    r2   ÚvÚic             C   s   t j|ƒ}| jd|žd|iŽS )NÚConcatÚaxis_i)r5   )r%   Ú_unpack_listr   )r   Útensor_listÚdimZtensorsr   r   r   Úcats   s    
r:   c                s.   ‡ ‡fdd„t j|ƒD ƒ}ˆjd|ždˆ iŽS )Nc                s   g | ]}ˆj d |ˆ gd�‘qS )Ú	Unsqueeze)Úaxes_i)r   )Ú.0Út)r9   r   r   r   ú
<listcomp>{   s    zstack.<locals>.<listcomp>r5   r6   )r5   )r%   r7   r   )r   r8   r9   Z
unsqueezedr   )r9   r   r   Ústacky   s    r@   c             C   s,   | j dtjdgƒd�}| j d|||ddd�S )NÚConstantr"   )Úvalue_tÚGemmg        g      ð?)Úbeta_fÚalpha_f)r   r0   Útensor)r   r   r    ÚCr   r   r   Úmm   s    rH   c             C   s   | j d||ƒS )NÚMatMul)r   )r   r   r    r   r   r   Úbmm†   s    rJ   c             C   s   | j d||ƒS )NrI   )r   )r   r   r    r   r   r   ÚmatmulŠ   s    rK   r>   c             C   s"   | j d|||tj|ƒtj|ƒd�S )NrC   )rD   rE   )r   r%   r&   )r   r   Zmat1Zmat2Úbetar(   r   r   r   ÚaddmmŽ   s    rM   c             C   s   | j d|ƒS )NZNeg)r   )r   r   r   r   r   Úneg“   s    rN   c             C   s   | j d|ƒS )NÚSqrt)r   )r   r   r   r   r   Úsqrt—   s    rP   c             C   s"   t | tj| tjdƒ|ƒt| |ƒƒS )Nr"   )r/   r%   Ú_if_scalar_type_asr0   r1   rP   )r   r   r   r   r   Úrsqrt›   s    rR   c             C   s   | j d|ƒS )NÚTanh)r   )r   r   r   r   r   ÚtanhŸ   s    rT   c             C   s   | j d|ƒS )NZSin)r   )r   r   r   r   r   Úsin£   s    rU   c             C   s   | j d|ƒS )NZCos)r   )r   r   r   r   r   Úcos§   s    rV   c             C   s   | j d|ƒS )NZTan)r   )r   r   r   r   r   Útan«   s    rW   c             C   s   | j d|ƒS )NZAsin)r   )r   r   r   r   r   Úasin¯   s    rX   c             C   s   | j d|ƒS )NZAcos)r   )r   r   r   r   r   Úacos³   s    rY   c             C   s   | j d|ƒS )NZAtan)r   )r   r   r   r   r   Úatan·   s    rZ   c             C   s   | j d|ƒS )NÚSigmoid)r   )r   r   r   r   r   Úsigmoid»   s    r\   c             C   s   | j d|ƒS )NZSign)r   )r   r   r   r   r   Úsign¿   s    r]   c             C   sP   t |ƒt |ƒkst‚t |ƒdkr<|d dkr<|d dkr<|S | jd||||d�S )Nr"   r   l   ÿÿÿÿ ÚSlice)r<   Zstarts_iZends_i)ÚlenÚAssertionErrorr   )r   r   ÚaxesÚstartsÚendsr   r   r   Ú_sliceÃ   s    $rd   Tc                s   d‡ ‡fdd„	}|S )Nc                sf   |d kr| j ˆ|dd�S ˆ r dnd}tj||dƒtj|ddƒ }}ˆ rJ|n|g}| j ˆ|||d�S d S )Nr   )Ú
keepdims_iÚisr4   r9   Úkeepdim)r<   re   )r   r%   Ú
_get_const)r   r   r9   rg   ÚdescZdim_list)Úallow_multi_dim_supportÚonnx_op_namer   r   ÚsymbolicË   s    z%_reduce_op_symbolic.<locals>.symbolic)NNr   )rk   rj   rl   r   )rj   rk   r   Ú_reduce_op_symbolicÊ   s    
rm   c                s   t ˆ ƒ‡ fdd„ƒ}|S )Nc                sX   ˆ | f|žŽ }d }x0|D ](}|j }t|ƒt|ƒkr|| f|žŽ S qW tdjˆ jƒƒ‚d S )NzUnknown aten::{} signature)Z_arg_descriptorsr_   ÚNotImplementedErrorÚformatÚ__name__)r   ÚargsZ	overloadsZlast_exceptionZoverloadZarg_descriptors)Úfnr   r   ÚwrapperØ   s    
z&overload_by_arg_count.<locals>.wrapper)r   )rr   rs   r   )rr   r   Úoverload_by_arg_count×   s    	rt   c                s$   t | ˆ d�‰t‡ ‡‡fdd„ƒ}|S )N)rj   c                sH   t ddƒ‡‡fdd„ƒ}ˆ r dnd}t d|ddƒ‡‡fdd„ƒ}||fS )	Nr3   Únonec                s$   |j ƒ jƒ dkrtˆ dƒS ˆ| |ƒS )Nzprim::ConstantÚdtype)ÚnodeÚkindr   )r   r   rv   )Únamerl   r   r   Úreduce_nodimè   s    
z8_reduce_with_dtype.<locals>.reduce.<locals>.reduce_nodimrf   r4   c                s(   |j ƒ jƒ dkrtˆ dƒS ˆ| |||ƒS )Nzprim::Constantrv   )rw   rx   r   )r   r   r9   rg   rv   )ry   rl   r   r   Ú
reduce_dimð   s    
z6_reduce_with_dtype.<locals>.reduce.<locals>.reduce_dim)r   )r   rq   Úkwargsrz   Zdim_descr{   )rj   ry   rl   r   r   Úreduceæ   s    z"_reduce_with_dtype.<locals>.reduce)rm   rt   )Zonnx_opry   rj   r}   r   )rj   ry   rl   r   Ú_reduce_with_dtypeã   s    r~   Ú	ReduceSumÚsumÚ
ReduceMeanÚmeanZ
ReduceProdÚprodF)rj   ru   c             C   s,   |j ƒ jƒ dkrttdƒS | jd|d|d�S )Nzprim::Constantrv   ÚATenÚcumsum)Ú
operator_sÚdim_i)rw   rx   r   ry   r   )r   r   r9   rv   r   r   r   r…   ü   s    
r…   c             C   s&   |j ƒ jƒ stddƒS | jd|dd�S )NÚ_sample_dirichletz#We are not able to export generatorr„   )r†   )rw   Ú
mustBeNoner   r   )r   r   Ú	generatorr   r   r   rˆ     s    rˆ   c             C   s&   |j ƒ jƒ stddƒS | jd|dd�S )NÚ_standard_gammaz#We are not able to export generatorr„   )r†   )rw   r‰   r   r   )r   r   rŠ   r   r   r   r‹   
  s    r‹   c             C   s   | j d|dd�S )NÚ	Transposer"   r   )Úperm_i)r"   r   )r   )r   r   r   r   r   r>     s    c             C   s8   t j|dƒ}t j|ƒs*| jdtj|ƒd�}| jd||ƒS )Nrf   rA   )rB   ÚExpand)r%   Ú_maybe_get_constÚ	_is_valuer   r0   Ú
LongTensor)r   r   ÚsizeZimplicitr   r   r   Úexpand  s    
r“   c             C   s   | j d|ƒ}| j d||ƒS )Nr   rŽ   )r   )r   r   r    r   r   r   r   Ú	expand_as  s    r”   c             C   s   | j d||ƒS )NÚGather)r   )r   ÚweightÚindicesZpadding_idxÚscale_grad_by_freqÚsparser   r   r   Ú	embedding!  s    rš   c             C   s0   |j ƒ jƒ stdƒ‚| jd|||dd|||d�	S )NzAUnsupported: ONNX export of embedding_bag with per_sample_weightsr„   Úembedding_bagé   )r†   ÚoutputsZscale_grad_by_freq_iZmode_iZsparse_i)rw   r‰   ÚRuntimeErrorr   )r   Zembedding_matrixr—   Úoffsetsr˜   Úmoder™   Zper_sample_weightsr   r   r   r›   %  s    	r›   c             C   sp   t j|dƒdk rD|jƒ jƒ }|rDt j|dƒ| }| jdtj|ƒd�}| jd|ƒ}t| || jdtjdgƒd�|ƒS )Nr4   r   rA   )rB   r   )r%   r�   Útyper9   r   r0   rF   Úselect)r   r   r9   ÚrankZ
full_shaper   r   r   r’   <  s    r’   c             C   sj   ||kr|S |j ƒ rRtt|jƒ jƒ ƒƒ}|| ||  ||< ||< | jd||d�S | jd|d||d�S d S )NrŒ   )r�   r„   Ú	transpose)r†   Zdim0_iZdim1_i)ÚisCompleteTensorÚlistÚranger¡   r9   r   )r   r   Zdim0Zdim1ra   r   r   r   r¤   F  s    r¤   rf   c             C   s*   |t tdt|ƒƒƒkr|S | jd||d�S )Nr   rŒ   )r�   )r¦   r§   r_   r   )r   r   Údimsr   r   r   ÚpermuteV  s    r©   c             C   s‚   t j|dƒ}t j|ƒr|}nX|jƒ r`|jƒ jƒ }|r`t|ƒdkr`|d |d kr`| jd|dd�S | jdtj	|ƒd�}| jd	||ƒS )
Nrf   é   r   ÚFlattenr"   )r6   rA   )rB   r   )
r%   r�   r�   r¥   r¡   Úsizesr_   r   r0   r‘   )r   r   r’   r   Ú
self_sizesr   r   r   r   ]  s    
 r   c             C   sL   |j ƒ jƒ | }|g||  }|| }|r4|j|ƒ | jd|||t|ƒd�S )NÚSplit)Úsplit_ir6   r�   )r¡   r¬   Úappendr   r_   )r   r   Ú
split_sizer9   r’   ÚsplitsÚleftoverr   r   r   Úprim_ConstantSplitj  s    
r´   c             C   s*   |j ƒ jƒ | | d | }t| |||ƒS )Nr"   )r¡   r¬   r´   )r   r   Úchunksr9   r±   r   r   r   Úprim_ConstantChunkw  s    r¶   c             C   sH   |j ƒ jƒ | }|g||  }|| }|r4|j|ƒ | jd|||dd�S )Nr®   r"   )r¯   r6   r�   )r¡   r¬   r°   r   )r   r   r±   r9   r’   r²   r³   r   r   r   Úsplit|  s    
r·   c             C   s   | j d|||dd�S )Nr®   r"   )r¯   r6   r�   )r   )r   r   Zsplit_sizesr9   r   r   r   Úsplit_with_sizes†  s    r¸   c             C   sX   |dkrBt |dƒ}tj| ||g|g|d gd�}| jd||gd�S | jd|||d�S d S )Nr"   r4   )ra   rb   rc   ÚSqueeze)r<   r•   )r6   )r   r%   Ú_slice_helperr   )r   r   r9   ÚindexZ	index_valZ
slice_noder   r   r   r¢   ‹  s    
r¢   c             C   sÚ   |d kr>g }x¼t |jƒ jƒ ƒD ]\}}|dkr|j|ƒ qW nŒtj|ddƒg}xzt |ƒD ]n\}}|dk rX|jƒ jƒ }|r¼tjdt	|ƒ d d d t	|| ƒ d	 d
 ƒ ||  |7  < qXt
ddƒS qXW | jd||d�S )Nr"   r4   r9   r   z'ONNX export squeeze with negative axis z- might cause the onnx model to be incorrect. z(Negative axis is not supported in ONNX. zAxis is converted to z& based on input shape at export time. zCPassing an tensor of different rank in execution will be incorrect.Úsqueezez%negative axis with unknown input rankr¹   )r<   )Ú	enumerater¡   r¬   r°   r%   rh   r9   ÚwarningsÚwarnÚstrr   r   )r   r   r9   r¨   r4   r’   r£   r   r   r   r¼   ™  s     &r¼   c             C   sT   |j ƒ rF|jƒ jƒ }|rFt|ƒdkrF| jd|ttdt|ƒd ƒƒd�}| jd||ƒS )Nrª   r;   r"   )r<   ÚPRelu)r¥   r¡   r¬   r_   r   r¦   r§   )r   r   r–   r­   r   r   r   Úprelu³  s
    "rÂ   c             C   s   | j d|ƒS )NÚRelu)r   )r   r   r   r   r   Úrelu»  s    rÄ   c             C   s   | j d|ƒS )NZCeil)r   )r   r   r   r   r   Úceil¿  s    rÅ   c             C   s   | j d|ƒS )NZFloor)r   )r   r   r   r   r   ÚfloorÃ  s    rÆ   c             C   s<   t j|ƒdkrtddƒS t j|ƒdkr0tddƒS | jd|ƒS )Nr   Ú	thresholdznon-zero thresholdznon-zero valuerÃ   )r%   r&   r   r   )r   r   rÇ   Úvaluer   r   r   rÇ   Ç  s
    

rÇ   c             C   s$   t j|ddƒ}| jd|t j|ƒd�S )Nr>   Únegative_slopeÚ	LeakyRelu)rE   )r%   rh   r   r&   )r   r   rÉ   Zinplacer   r   r   Ú
leaky_reluÑ  s    rË   c             C   sH   |j ƒ jƒ | d dkst‚| jd||dd�\}}| jd|| jd|ƒƒS )Nrª   r   r®   )r6   r�   r,   r[   )r¡   r¬   r`   r   )r   r   r9   ÚfirstÚsecondr   r   r   ÚgluØ  s    rÎ   c       	      C   sà   |j ƒ jƒ }|rx|dk r || }||d krx| jd||d�}|rt|jƒ jƒ dkrttj|ddƒ}| jd|tj| d	�}|S | jd
|ƒ}| jd||gd�}| jd||ƒ}|rÜ|jƒ jƒ dkrÜtj|ddƒ}| jd|tj| d	�}|S )Nr   r"   ZSoftmax)r6   zprim::Constantr4   rv   ÚCast)Úto_iÚExpr   )r<   r.   )r¡   r9   r   rw   rx   r%   rh   Úscalar_type_to_onnx)	r   r   r9   rv   Z	input_dimÚsoftmaxZparsed_dtypeÚexpr€   r   r   r   rÓ   à  s"    rÓ   c             C   s   |dkrt ddƒS | jd|ƒS )Nr"   rL   zhas to be 1ÚSoftplus)r   r   )r   r   rL   rÇ   r   r   r   Úsoftplus  s    
rÖ   c                s¦   | j ƒ jƒ tˆƒ d … ‰‡‡‡‡fdd„tdtˆƒƒD ƒ‰ ‡ ‡‡‡fdd„tdtˆ ƒƒD ƒ‰ ‡ ‡‡‡‡fdd„tdtˆƒƒD ƒ‰‡‡‡fdd„tdtˆƒƒD ƒ‰ˆS )Nc                sB   g | ]:}t tjˆ | d ˆ|   ˆ|  tˆ| ƒ ƒƒd ‘qS )rª   r"   )ÚintÚmathrÅ   Úfloat)r=   r4   )r9   Úkernel_sizeÚpaddingÚstrider   r   r?     s   z)get_pool_ceil_padding.<locals>.<listcomp>r   c                sD   g | ]<}ˆ | d  ˆ|  ˆ| ˆ|  kr8ˆ | d  nˆ | ‘qS )r"   r   )r=   r4   )Úceiled_output_dimr9   rÛ   rÜ   r   r   r?     s   c                sP   g | ]H}ˆ| d krdn2ˆ| ˆ| dˆ|   ˆ | d  ˆ|  d    ‘qS )r"   r   rª   r   )r=   r4   )rÝ   r9   rÚ   rÛ   rÜ   r   r   r?     s   c                sd   g | ]\}ˆ| d ˆ|   ˆ | krTˆ| ˆ | d k rDt ˆ| ƒq^t ˆ | d ƒn
t ˆ| ƒ‘qS )rª   r"   )r×   )r=   r4   )rÚ   rÛ   Úpadding_ceilr   r   r?     s   )r¡   r¬   r_   r§   )r   rÚ   rÜ   rÛ   r   )rÝ   r9   rÚ   rÛ   rÞ   rÜ   r   Úget_pool_ceil_padding  s    rß   c                s(   t ddddddƒ‡ ‡‡‡fdd„ƒ}|S )Nr3   rf   r4   c                s@  |r|j ƒ  rtˆ dƒS tˆ|ƒƒdhkr4tˆ dƒS |s<|}tˆ|ƒƒ}|rpt||||ƒ}|ttj||ƒƒ }n|d }ˆ|ƒ|ˆ|ƒdœ}ˆ�r | jd|fddi|—Ž\}	}
| jd|ddd	„ tˆƒD ƒd
d	„ tˆƒD ƒd�\}}t	j
| |dd	„ tˆƒD ƒˆdƒˆdƒd�}t| |
|ƒ}
|	|
fS | jd|fddi|—Ž}	|	S d S )Nzinput size not accessibler"   Údilationrª   )Úkernel_shape_iÚpads_iÚ	strides_iÚMaxPoolr�   c             S   s   g | ]}d ‘qS )r"   r   )r=   Ú_r   r   r   r?   I  s    z2_max_pool.<locals>.symbolic_fn.<locals>.<listcomp>c             S   s   g | ]}d ‘qS )r"   r   )r=   rå   r   r   r   r?   J  s    )r�   rá   rã   c             S   s   g | ]}d | ‘qS )rª   r   )r=   r4   r   r   r   r?   L  s    r   )ra   rb   rc   )r¥   r   ÚsetÚtuplerß   Únumpyr#   r   r§   r%   rº   r)   )r   r   rÚ   rÜ   rÛ   rà   Ú	ceil_moderÞ   r|   Úrr—   rå   Zflattened_indicesÚs)ry   ÚndimsÚreturn_indicesÚtuple_fnr   r   Úsymbolic_fn&  s2    


z_max_pool.<locals>.symbolic_fn)r   )ry   rî   rì   rí   rï   r   )ry   rì   rí   rî   r   Ú	_max_pool%  s    $.rð   Ú
max_pool1dr"   )rí   Ú
max_pool2drª   Ú
max_pool3dé   Úmax_pool1d_with_indicesÚmax_pool2d_with_indicesÚmax_pool3d_with_indicesc                s(   t dddddddƒd‡ ‡fdd„	ƒ}|S )Nr3   rf   r4   ru   c       
         sÊ   |r|j ƒ  rtˆ dƒS |r6|jƒ jƒ dkr6tˆ dƒS |s>|}tˆ|ƒƒ}|r\t||||ƒ}|rˆ| jd|d| d ddd	�}dt|ƒ }|r¢|ttj	||ƒƒ }n|d }| jd
|ˆ|ƒˆ|ƒ|d�}	|	S )Nzinput size not accessiblezprim::ConstantÚdivisor_overrideÚPadr   rª   Úconstantg        )râ   Úmode_sÚvalue_fÚAveragePool)rá   rã   râ   )r   )r   r   )r   )
r¥   r   rw   rx   rç   rß   r   r_   rè   r#   )
r   r   rÚ   rÜ   rÛ   ré   Zcount_include_padrø   rÞ   Úoutput)ry   rî   r   r   rï   `  s.    


z_avg_pool.<locals>.symbolic_fn)N)r   )ry   rî   rï   r   )ry   rî   r   Ú	_avg_pool_  s    rÿ   Ú
avg_pool1dÚ
avg_pool2dÚ
avg_pool3dc                s    t ddƒ‡ ‡‡‡fdd„ƒ}|S )Nr3   rf   c          	      s4  ˆdgt ˆƒ kr&ˆdkr&| jd|ƒS |jƒ sZˆdgt ˆƒ krP| jd|ƒd fS tˆdƒS |jƒ jƒ dd … ‰ ‡ ‡fdd„td	t ˆ ƒƒD ƒ}|d	gt |ƒ krÊˆdgt ˆƒ krÀ| jd|ƒd fS tˆd
ƒS ‡ ‡fdd„td	t ˆ ƒƒD ƒ}ˆdk�rˆ| |||dt ˆ ƒ dt ˆ ƒ dƒS | jˆ|ˆ|ƒˆ|ƒd�}|S )Nr"   rý   ZGlobalAveragePoolZGlobalMaxPoolzinput size not accessiblerª   c                s   g | ]}ˆ | ˆ|  ‘qS r   r   )r=   r4   )r9   Úoutput_sizer   r   r?   –  s    z7_adaptive_pool.<locals>.symbolic_fn.<locals>.<listcomp>r   z-output size that are not factor of input sizec                s    g | ]}t ˆ | ˆ|  ƒ‘qS r   )r×   )r=   r4   )r9   r  r   r   r?   ›  s    rä   F)rá   rã   )r   )r"   )r_   r   r¥   r   r¡   r¬   r§   )r   r   r  ÚmodÚkrþ   )rr   ry   rî   r¡   )r9   r  r   rï   ƒ  s&    


$z#_adaptive_pool.<locals>.symbolic_fn)r   )ry   r¡   rî   rr   rï   r   )rr   ry   rî   r¡   r   Ú_adaptive_pool‚  s     r  Úadaptive_avg_pool1drý   Úadaptive_avg_pool2dÚadaptive_avg_pool3dÚadaptive_max_pool1drä   Úadaptive_max_pool2dÚadaptive_max_pool3dÚfc             C   s6   ddl m} d}||jƒ jƒ |ƒ}| jd||||d�S )Nr   )Úprepare_onnx_paddingsrú   rù   )râ   rû   rü   )Útorch.autograd._functions.utilsr  r¡   r9   r   )r   r   rÛ   rÈ   r  r    Úpaddingsr   r   r   Úconstant_pad_nd¯  s    r  c             C   s4   ddl m} d}||jƒ jƒ |ƒ}| jd|||d�S )Nr   )r  Zreflectrù   )râ   rû   )r  r  r¡   r9   r   )r   r   rÛ   r  r    r  r   r   r   Úreflection_pad·  s    r  c             C   s4   ddl m} d}||jƒ jƒ |ƒ}| jd|||d�S )Nr   )r  Zedgerù   )râ   rû   )r  r  r¡   r9   r   )r   r   rÛ   r  r    r  r   r   r   Úreplication_pad¿  s    r  c                s   d‡ ‡‡fdd„	}|S )Nc                sD   t jˆƒ t j|ƒ}|r"tˆdƒS t j| ||ˆ ƒ}| jd||ˆd�S )Nzalign_corners == TrueZUpsample)rû   )r%   Z_interpolate_warningr'   r   Z_interpolate_size_to_scalesr   )r   r   r  Zalign_cornersZscales)r9   Úinterpolate_modery   r   r   rï   Ð  s    


z!_interpolate.<locals>.symbolic_fn)Nr   )ry   r9   r  rï   r   )r9   r  ry   r   Ú_interpolateÏ  s    r  Úupsample_nearest1dZnearestÚupsample_nearest2drœ   Úupsample_nearest3dé   Úupsample_linear1dZlinearÚupsample_bilinear2dÚupsample_trilinear3dc                s   ‡ fdd„}|S )Nc                s   ‡ ‡fdd„}|S )Nc                s   | j dˆ | ||ƒtjˆ d�S )NrÏ   )rÐ   )r   r%   Úcast_pytorch_to_onnx)r   r   r    )rr   Úto_typer   r   Úwrap_with_castä  s    zGwrap_logical_op_with_cast_to.<locals>.decorator.<locals>.wrap_with_castr   )rr   r  )r  )rr   r   Ú	decoratorã  s    z/wrap_logical_op_with_cast_to.<locals>.decoratorr   )r  r   r   )r  r   Úwrap_logical_op_with_cast_toâ  s    r!  c                s   ‡ fdd„}|S )Nc                s   ‡ ‡fdd„}|S )Nc                s@   t ƒ djˆƒ }t|jƒ jƒ ƒˆ ƒ}|| || |dƒ|| |dƒƒS )Nz_cast_{}F)Úglobalsro   r!  r¡   Ú
scalarType)r   r   r    Zto_cast_funcZfrom_cast_func)rr   r  r   r   r  ì  s    zPwrap_logical_op_with_cast_to_and_from.<locals>.decorator.<locals>.wrap_with_castr   )rr   r  )r  )rr   r   r   ë  s    z8wrap_logical_op_with_cast_to_and_from.<locals>.decoratorr   )r  r   r   )r  r   Ú%wrap_logical_op_with_cast_to_and_fromê  s    r$  c                s   ‡ fdd„}|S )Nc                s   | j dˆ | ||ƒƒS )NÚNot)r   )r   r   r    )Úfuncr   r   Úwrap_with_notõ  s    z4wrap_logical_op_with_negation.<locals>.wrap_with_notr   )r&  r'  r   )r&  r   Úwrap_logical_op_with_negationô  s    r(  c             C   s   | j d||ƒS )NÚEqual)r   )r   r   r    r   r   r   Úeqú  s    r*  c             C   s   | j d||ƒS )Nr)  )r   )r   r   r    r   r   r   Úneþ  s    r+  c             C   s   t | ||ƒS )N)Úgt_impl)r   r   r    r   r   r   Úgt  s    r-  c             C   s   | j d||ƒS )NZGreater)r   )r   r   r    r   r   r   r,    s    r,  c             C   s   t | ||ƒS )N)Últ_impl)r   r   r    r   r   r   Últ  s    r/  c             C   s   | j d||ƒS )NZLess)r   )r   r   r    r   r   r   r.    s    r.  c             C   s   t | ||ƒS )N)r.  )r   r   r    r   r   r   Úge  s    r0  c             C   s   t | ||ƒS )N)r,  )r   r   r    r   r   r   Úle  s    r1  ZBoolc             C   s   | j d||ƒS )NÚAnd)r   )r   r   r    r   r   r   Ú__and_  s    r3  c             C   s   | j d||ƒS )NÚOr)r   )r   r   r    r   r   r   Ú__or_"  s    r5  c             C   s   | j d|||ƒS )NÚWhere)r   )r   Ú	conditionr   r    r   r   r   Úwhere'  s    r8  c             C   st   |dk r|j ƒ jƒ | }|j ƒ jƒ |d kr6tddƒS | jd||d�}|rp|jƒ jƒ dkrp| jd|tj| d	�}|S )
Nr   r"   r9   z=ONNX and PyTorch use different strategies to split the input.Z
LogSoftmax)r6   zprim::ConstantrÏ   )rÐ   )r¡   r9   r   r   rw   rx   r%   rÒ   )r   r   r9   rv   Z	return_opr   r   r   Úlog_softmax+  s    
r9  c             C   sÜ   |j ƒ jƒ }||g}|jƒ jƒ  r<|j ƒ jƒ dkr<|j|ƒ |dd … ||| ||	dœ}tdd„ |D ƒƒrŽ|srt‚t|ƒt|ƒks†t‚||d< | j	|ršdndf|ž|Ž}|jƒ jƒ  rÔ|j ƒ jƒ dkrÔ| j	d	||ƒS |S d S )
Nr"   rª   )rá   rã   râ   Zdilations_iZgroup_ic             s   s   | ]}|d kV  qdS )r   Nr   )r=   Úor   r   r   ú	<genexpr>K  s    z_convolution.<locals>.<genexpr>Zoutput_padding_iZConvTransposeZConvr$   )
r¡   r¬   rw   r‰   r9   r°   Úanyr`   r_   r   )r   r   r–   ÚbiasrÜ   rÛ   rà   Z
transposedZoutput_paddingÚgroupsZ	benchmarkZdeterministicÚcudnn_enabledZweight_sizerq   r|   r   r   r   r   Ú_convolution9  s"    

r@  c
             C   s¬  |j ƒ jƒ }
t|
ƒdkr*| jd|dgd�}|d ks>|jƒ jƒ r†t|
ƒdksNt‚tjdg|
d  ƒj d|j ƒ j	ƒ  d ƒ}| jd|d	�}|d ksš|jƒ jƒ rât|
ƒdksªt‚tjd
g|
d  ƒj d|j ƒ j	ƒ  d ƒ}| jd|d	�}| jd||||||d| |�sdndd�	}|�s6t|
ƒdk�r2| jd|dgd�}|S |\}}}}}|j
|j ƒ ƒ |j
|j ƒ ƒ |jd|jƒ  ƒ |jd|jƒ  ƒ t|
ƒdk�r¤| jd|dgd�}|S d S )Nrª   r;   )r<   r"   g      ð?ztorch.ÚTensorrA   )rB   g        ZBatchNormalizationr  )Ú	epsilon_fZ
momentum_fr�   r¹   zbatch_norm_dead_output-)r¡   r¬   r_   r   rw   r‰   r`   r0   rF   r#  r   ZsetDebugNameZ	debugName)r   r   r–   r=  Úrunning_meanÚrunning_varÚtrainingÚmomentumÚepsr?  Úinput_sizesÚweight_valueÚ
bias_valueÚoutÚresZnew_running_meanZnew_running_varZ
saved_meanZ	saved_varr   r   r   Ú
batch_norm[  s:    rM  c          
   C   s  t jtjjjkr*| jd||||||dd�S dd„ tt|ƒddƒD ƒ}| jdtj	d	ƒd
�}| jdtj	|ƒd
�}	| jd||d�}
t
| ||
ƒ}| jdt| ||ƒ|d�}t| t| ||	ƒƒ}t| ||ƒ}|d kpÐ|jƒ jƒ sÞt| ||ƒ}|d kpð|jƒ jƒ �s t| ||ƒ}|S )Nr„   Ú
layer_norm)Znormalized_shape_iÚeps_fZcudnn_enable_ir†   c             S   s   g | ]
}| ‘qS r   r   )r=   r4   r   r   r   r?   …  s    zlayer_norm.<locals>.<listcomp>r   r"   rA   g       @)rB   r�   )r<   éÿÿÿÿ)r%   Ú_operator_export_typer0   ÚonnxÚOperatorExportTypesÚONNX_ATEN_FALLBACKr   r§   r_   rF   r)   ÚpowrP   r#   r/   rw   r‰   r-   )r   r   Znormalized_shaper–   r=  rG  Zcudnn_enablera   Ztwo_cstZeps_cstr‚   Ú	numeratorZvarianceÚdenominatorrN  r   r   r   rN    s     rN  c
             C   sØ   |j ƒ jƒ }
|d ks |jƒ jƒ rht|
ƒdks0t‚tjdg|
d  ƒj d|j ƒ jƒ  d ƒ}| j	d|d�}|d ks||jƒ jƒ rÄt|
ƒdksŒt‚tjdg|
d  ƒj d|j ƒ jƒ  d ƒ}| j	d|d�}| j	d||||d	�S )
Nr"   g      ð?ztorch.rA  rA   )rB   g        ZInstanceNormalization)rB  )
r¡   r¬   rw   r‰   r_   r`   r0   rF   r#  r   )r   r   r–   r=  rC  rD  Zuse_input_statsrF  rG  r?  rH  rI  rJ  r   r   r   Úinstance_normš  s    rX  c             C   s   | j d|d|||d�S )Nr„   Úunfold)r†   Zdimension_iZsize_iZstep_i)r   )r   r   Z	dimensionr’   Ústepr   r   r   rY  ª  s    rY  c             C   s   | j d|||dd�S )Nr„   Ú_weight_norm)r‡   r†   )r   )Úgraphr3   r   r9   r   r   r   r[  ¯  s    r[  c             C   sB   |r|dkrt ddƒS |r,|dkr,t ddƒS | jd|tj|ƒd�S )Ng      ð?Úscalezdoes not support scale in EluÚinput_scalez#does not support input_scale in EluÚElu)rE   )r   r   r%   r&   )r   r   r(   r]  r^  r   r   r   Úelu´  s
    

r`  c             C   s   | j d|ƒS )NZSelu)r   )r   r   r   r   r   Úselu¾  s    ra  c             C   sz   t j|ƒ}|jƒ jƒ }t j|ƒs8| jdtj|gƒd�}n0|d k	rh|dkrh| jd|| jdtjdgƒd�ƒ}| jd|||d�S )NrA   )rB   r   r   r"   r•   )r6   )r%   r'   r¡   r9   r�   r   r0   r‘   )r   r   r9   r»   Zindex_constZ	index_dimr   r   r   Úindex_selectÂ  s    

 rb  c             C   s0   t j|ƒ}|g| ||g }| jd|žddiŽS )Nr„   r†   Ú	index_put)r„   )r%   r7   r   )r   r   Zindices_list_valueÚvaluesÚ
accumulateZindices_listrq   r   r   r   rc  Ô  s    
rc  c       
         sð   t j|dƒ‰ t jtjjjkr2| jd|||ˆ dd�S |jƒ j	ƒ d krLt
ddƒS |jƒ j	ƒ }| jd|‡ fdd„t|ƒD ƒd	�}| jd
| jd|ƒ| jd|dgd	�| jd|ƒdd�}t| ||d ƒ}t j|ƒ}t j| ||ƒ}t| ||d ƒ}	t| ||||	ƒS )Nr4   r„   Ú
index_fill)r‡   r†   zinput rank not accesibler;   c                s   g | ]}|ˆ kr|‘qS r   r   )r=   r4   )Ú	dim_valuer   r   r?   æ  s    zindex_fill.<locals>.<listcomp>)r<   ÚScatterr   r   )r6   )r%   r   rQ  r0   rR  rS  rT  r   r¡   r9   r   r§   r“   r'   rQ   Úscatter)
r   r   r9   r»   rÈ   Úself_dimÚunsqueezed_indexÚexpanded_index_shapeÚexpanded_indexZexpanded_valuer   )rg  r   rf  Ú  s    
""
rf  c       	         sÊ   t j|dƒ‰ t jtjjjkr2| jd|||ˆ dd�S |jƒ j	ƒ d krLt
ddƒS |jƒ j	ƒ }| jd|‡ fdd„t|ƒD ƒd	�}| jd
| jd|ƒ| jd|dgd	�| jd|ƒdd�}t| ||d ƒ}t| ||||ƒS )Nr4   r„   Ú
index_copy)r‡   r†   zinput rank not accesibler;   c                s   g | ]}|ˆ kr|‘qS r   r   )r=   r4   )rg  r   r   r?   û  s    zindex_copy.<locals>.<listcomp>)r<   rh  r   r   )r6   )r%   r   rQ  r0   rR  rS  rT  r   r¡   r9   r   r§   r“   ri  )	r   r   r9   r»   Úsourcerj  rk  rl  rm  r   )rg  r   rn  ò  s    
""rn  c             C   sl   |j ƒ r,|j ƒ r,|jƒ jƒ |jƒ jƒ kr,|S |j ƒ rV|jƒ jƒ }| jd|tj| d�S | jd||dd�S d S )NrÏ   )rÐ   r„   Útype_as)r†   )r¥   r¡   r#  r   r%   r  )r   r   r    Zother_type_namer   r   r   rp    s    (rp  c             C   s   | j d||||dd�S )Nr„   Úcosine_similarity)r‡   rO  r†   )r   )r   Úx1Zx2r9   rG  r   r   r   rq    s    rq  c             C   s   |S )Nr   )r   r   r   r   r   Úclone  s    rs  c             C   s   | j d|ƒS )NÚAbs)r   )r   r   r   r   r   Úabs  s    ru  c             C   s   | j d|ƒS )NÚLog)r   )r   r   r   r   r   Úlog  s    rw  c             C   s"   t | t| tj| tjdƒ|ƒ|ƒƒS )Nr"   )rw  r#   r%   rQ   r0   r1   )r   r   r   r   r   Úlog1p!  s    rx  c             C   s   | j d||ƒS )NÚPow)r   )r   r   Úexponentr   r   r   rU  %  s    rU  c             C   sZ   |j ƒ jƒ rt| ||ƒS |j ƒ jƒ r0t| ||ƒS t|dƒ}t|dƒ}| jd|||d�S d S )Nr  ÚClip)Úmin_fÚmax_f)rw   r‰   Ú	clamp_maxÚ	clamp_minr   r   )r   r   ÚminÚmaxr   r   r   Úclamp)  s    

r‚  c             C   s   | j d||d�S )Nr{  )r|  )r   )r   r   r€  r   r   r   r  6  s    r  c             C   s   | j d||d�S )Nr{  )r}  )r   )r   r   r�  r   r   r   r~  ;  s    r~  c             C   s„   |d kr |d kr | j d|dd�S |d kr6| j d||ƒS tj|ddƒ}tj|ddƒ}| j d||g|d�}| j d	|||d
�}||fS d S )NZ	ReduceMaxr   )re   ZMaxr4   r9   rg   )r<   re   ÚArgMax)r6   re   )r   r%   rh   )r   r   Údim_or_yrg   r9   r�  r—   r   r   r   r�  B  s    r�  c             C   s„   |d kr |d kr | j d|dd�S |d kr6| j d||ƒS tj|ddƒ}tj|ddƒ}| j d||g|d�}| j d	|||d
�}||fS d S )NZ	ReduceMinr   )re   ZMinr4   r9   rg   )r<   re   ÚArgMin)r6   re   )r   r%   rh   )r   r   r„  rg   r9   r€  r—   r   r   r   r€  R  s    r€  c             C   s   | j d|ƒS )NrÑ   )r   )r   r   r   r   r   rÔ   b  s    rÔ   c             C   s,   |s|S t jdƒ | jd||dd�\}}|S )Nz™Dropout is a training op and should not be exported in inference mode. Make sure to call eval() on the model, and to export it with param training=False.ZDropoutrª   )Zratio_fr�   )r¾   r¿   r   )r   r   ÚpÚtrainrê   rå   r   r   r   Údropoutf  s
    
rˆ  c                s   t dddƒ‡ fdd„ƒ}|S )Nr3   r  r4   c                s   |rt ˆ dƒS |S )Nztraining mode)r   )r   r   r†  r‡  )ry   r   r   Úfeature_dropoutq  s    
z-_unsupported_dropout.<locals>.feature_dropout)r   )ry   r‰  r   )ry   r   Ú_unsupported_dropoutp  s    rŠ  r‰  Úalpha_dropoutÚfeature_alpha_dropoutc             C   s<   |dkrt dƒ}n|dkr$t dƒ}ntdƒ‚|| |||d�S )Nr"   ZReduceL1rª   ZReduceL2z)ONNX export only p-norms with p of 1 or 2)r9   rg   )rm   rž   )r   r   r†  r9   rg   r  r   r   r   Únorm…  s    

r�  c             C   s   | j d|||d|d�S )Nr„   Úconv_tbc)r†   Zpad_i)r   )r   r   r–   r=  Úpadr   r   r   rŽ  �  s    rŽ  c             C   s   | j d|d||dd�S )Nr„   Ú_uniquerª   )r†   Úsorted_iÚreturn_inverse_ir�   )r   )r   r   ÚsortedÚreturn_inverser   r   r   r�  •  s    r�  c          	   C   s   | j d|d|||dd�S )Nr„   Ú_unique2rô   )r†   r‘  r’  Zreturn_counts_ir�   )r   )r   r   r“  r”  Zreturn_countsr   r   r   r•  ›  s    r•  z_cast_{}c             C   s   t | |||||ƒS )N)Úzeros)r   r¬   rv   ÚlayoutÚdeviceÚ
pin_memoryÚmemory_formatr   r   r   Úempty§  s    r›  c             C   s   t | |||||ƒS )N)Ú
zeros_like)r   r   rv   r—  r˜  r™  rš  r   r   r   Ú
empty_like¬  s    r�  c             C   s"   | j d|tjdgtj| d�d�S )NÚConstantOfShaper   )rv   )rB   )r   r0   rF   r%   Úscalar_type_to_pytorch_type)r   r¬   rv   r—  r˜  r™  r   r   r   r–  ±  s    r–  c             C   s.   | j d|ƒ}| j d|tjdgtj| d�d�S )Nr   rž  r   )rv   )rB   )r   r0   rF   r%   rŸ  )r   r   rv   r—  r˜  r™  r   r   r   r   rœ  ¸  s    rœ  c             C   s"   | j d|tjdgtj| d�d�S )Nrž  r"   )rv   )rB   )r   r0   rF   r%   rŸ  )r   r¬   rv   r—  r˜  r™  r   r   r   r1   ¿  s    r1   c             C   s.   | j d|ƒ}| j d|tjdgtj| d�d�S )Nr   rž  r"   )rv   )rB   )r   r0   rF   r%   rŸ  )r   r   rv   r—  r˜  r™  r   r   r   r   Ú	ones_likeÅ  s    r   c       	      C   sx   t j|dƒ}t j|ƒrDt| ||||ƒ}t| ||| jdtjdƒd�ƒS t j|ddƒ}| jd|tj|gt j	| d�d�S d S )	Nr>   rA   r"   )rB   r4   rv   rž  )rv   )
r%   r�   r�   r–  r#   r   r0   rF   rh   rŸ  )	r   r¬   rÈ   rv   r—  r˜  r™  Zconst_valueÚtmpr   r   r   ÚfullÌ  s    
r¢  c             C   s.   | j d|ƒ}| j d|tj|gtj| d�d�S )Nr   rž  )rv   )rB   )r   r0   rF   r%   rŸ  )r   r   Z
fill_valuerv   r—  r˜  r™  r   r   r   r   Ú	full_like×  s    r£  c       	      C   sÆ   |dkrt ddƒ |jƒ jƒ dksB|jƒ jƒ dksB|jƒ jƒ dkrŠ| jd|dgd�}| jd|dgd�}| jd|dgd�}| jd||||ƒS t|d	ƒ}t|d	ƒ}t|d	ƒ}tj| ||g|g|gd
�S d S )Nr"   Úslicez"step!=1 is currently not supportedzonnx::Constantr;   r   )r<   ZDynamicSlicer4   )ra   rb   rc   )r   rw   rx   r   r   r%   rº   )	r   r   r9   ÚstartÚendrZ  Zstart_unsqueezedZend_unsqueezedZdim_unsqueezedr   r   r   r¤  Þ  s    
 


r¤  c             C   s   | j d|||d�S )Nr{  )r|  r}  )r   )r   r   Zmin_valZmax_valr   r   r   Úhardtanhï  s    r§  c             C   s   |S )Nr   )r   r   r   r   r   Úaliasô  s    r¨  c             C   sx   |dk rf|j ƒ jƒ }|r\tjdt|ƒ d d d t|| d ƒ d d ƒ || d }n
td	d
ƒS | jd||gd�S )Nr   z)ONNX export unsqueeze with negative axis z- might cause the onnx model to be incorrect. z(Negative axis is not supported in ONNX. zAxis is converted to r"   z& based on input shape at export time. zCPassing an tensor of different rank in execution will be incorrect.Ú	unsqueezez%negative axis with unknown input rankr;   )r<   )r¡   r9   r¾   r¿   rÀ   r   r   )r   r   r9   r£   r   r   r   r©  ø  s    *
r©  c             C   sD   |d k	rt ddƒ |jƒ s$t ddƒS | jd||jƒ jƒ | |dd�S )NZSortz'Out parameter is not supported for sortzinput size not accessibleÚTopKrª   )Úk_ir6   r�   )r   r¥   r   r¡   r¬   )r   r   r9   Z	decendingrK  r   r   r   Úsort
  s
    

r¬  c             C   s4   |d k	rt ddƒ |s t ddƒ | jd|||dd�S )Nrª  z'Out parameter is not supported for topkzAscending TopK is not supportedrª   )r«  r6   r�   )r   r   )r   r   r  r9   Úlargestr“  rK  r   r   r   Útopk  s
    

r®  c             G   sø   t |ƒdkrP|d jƒ jtjƒ ƒr&|S tj|d ddƒ}| jd|tj| d�S n¤t |ƒdkr„tj|d ddƒ}| jd|tj| d�S t |ƒd	kr¸tj|d ddƒ}| jd|tj| d�S t |ƒd
krìtj|d ddƒ}| jd|tj| d�S t	dƒ‚d S )Nrô   r   r4   rv   rÏ   )rÐ   rœ   r"   r  é   zUnknown aten::to signature)
r_   r¡   ÚisSubtypeOfr   ÚofIntsr%   rh   r   rÒ   rn   )r   r   rq   rv   r   r   r   Úto  s    r²  c             C   s†   t j|ƒs| jdtj|ƒd�}t j|dƒ}|jƒ rxt j|ƒ rx|jƒ jƒ }t	|ƒt	|ƒ }|dkrxt
| |dg| | ƒ}| jd||ƒS )NrA   )rB   rf   r   r"   ZTile)r%   r�   r   r0   r‘   r�   r¥   r¡   r¬   r_   r   )r   r   ZrepeatsZconst_repeatsr¬   Z	diff_dimsr   r   r   Úrepeat9  s    
r³  c          
   C   s’   |j ƒ jƒ }t|ƒdkr"tddƒS |d | | }t| |d||||d |d gƒ}| jd|ddddd	dgd
�}t| |d||d | |d | gƒS )Nrœ   Úpixel_shufflezonly support 4d inputr"   rª   rô   rŒ   r   r  )r�   rP  rP  )r¡   r¬   r_   r   r   r   )r   r   Zupscale_factorr¨   Zoutput_channelZ
after_viewZafter_transposer   r   r   r´  F  s    
r´  c          
   C   s   | j d||||||dd�S )Nr„   Ú
group_norm)Znum_groups_irO  Zcudnn_enabled_ir†   )r   )r   r   Z
num_groupsr–   r=  rG  r?  r   r   r   rµ  T  s    rµ  c       )         sF  t jd4ˆ d d d d ƒ ddd	d
dddddddg}ttdd„ |D ƒ|ƒƒ}|rXdnd‰tˆ ƒˆˆ d|	  ksxt‚‡ ‡fdd„tdtˆ ƒˆƒD ƒ‰|
r²ˆjd|dddgd�}|rÄ|rÄtddƒS ˆj	dƒræ|ˆdd … j
ƒ  }d‰ˆ d }|jƒ jƒ d ‰|	 }|}g }ˆdk�s ˆdk�r&|}nˆdk�r<|\}}g }|d k�rNtˆƒn|}ˆdk�rhd5d6d7g‰nˆdk�r|d8d9d:g‰d!d"„ ‰‡‡‡‡‡‡fd#d$„}‡‡fd%d&„}�xtˆƒD �]ø}|�rÜ||ƒ\}}}||d f}nr|d| ƒ\}}} |d| d ƒ\}!}"}#ˆjd'||!dd(�}ˆjd'||"dd(�}ˆjd'| |#dd(�}d| d| d f}|||||g}$|$j||f|žŽ ƒ ˆdk�rŽ|$j||f|žŽ ƒ |�r˜i nd)d*i}%ˆdk�râ|	�rº||g}&n|g}&ˆjd;|$ždˆ|&d+œ|%—Ž\}}'nVˆdk�rˆjd<|$ždˆdd,œ|%—Ž\}}'n*ˆdk�r8ˆjd=|$žd ˆd-œ|%—Ž\}}'}(|	�r|ˆjd|dddd gd�}ˆjd.|ˆjd/tjddd>gƒd0�ƒ}nˆjd1|dgd2�}|j|'ƒ ˆdk�r´|j|(ƒ �q´W |
�rÎˆjd|dddgd�}ˆdk�rÜ|'nˆjd?|žd3diŽ}ˆdk�sˆdk�r||fS ˆdk�rBˆdk�r$|(nˆjd@|žd3diŽ}|||fS d S )ANz:Exporting a model to ONNX with a batch_size other than 1, zwith a variable lenght with z can cause an error z9when running the ONNX model with a different batch size. z4Make sure to save the model with a batch size of 1, z=or define the initial states (h0/c0) as inputs of the model. rÃ   rS   r[   ZAffinerÊ   ZThresholdedReluZ
ScaledTanhZHardSigmoidr_  ZSoftsignrÕ   c             S   s   g | ]}|j ƒ ‘qS r   )Úlower)r=   Zact_funr   r   r   r?   e  s    z _generic_rnn.<locals>.<listcomp>rœ   rª   r"   c                s   g | ]}ˆ ||ˆ … ‘qS r   r   )r=   r4   )Úall_weightsÚweights_per_layerr   r   r?   h  s    r   rŒ   )r�   zRNN/GRU/LSTMzdropout in training modeÚRNNÚGRUÚLSTMrô   c                s*   ‡ ‡‡fdd„|D ƒ}ˆ j d|žddiŽS )Nc          	      s2   g | ]*\}}t jˆ ˆd g|ˆ g|ˆ gd�‘qS )r   )ra   rb   rc   )r%   rº   )r=   ÚxÚy)r   r   Úwr   r   r?   �  s    z8_generic_rnn.<locals>.reform_weights.<locals>.<listcomp>r5   r6   r   )r5   )r   )r   r¾  r   Z	intervalsZslicesr   )r   r   r¾  r   Úreform_weightsŒ  s    z$_generic_rnn.<locals>.reform_weightsc                s|   ˆdkrˆ|  \}}}}n4ˆdks*ˆdkrN‡ ‡‡‡fdd„ˆ|  D ƒ\}}}}ˆ j d||dd�}t‡ fd	d
„|||fD ƒƒS )Nr¹  rº  r»  c                s   g | ]}ˆˆ |ˆˆƒ‘qS r   r   )r=   r¾  )r   Úhidden_sizeÚreform_permutationr¿  r   r   r?   •  s    z;_generic_rnn.<locals>.transform_weights.<locals>.<listcomp>r5   r   )r6   c             3   s    | ]}ˆ j d |dgd�V  qdS )r;   r   )r<   N)r   )r=   r¼  )r   r   r   r;  ˜  s    z:_generic_rnn.<locals>.transform_weights.<locals>.<genexpr>)r   rç   )Zlayer_indexÚ	weight_ihÚ	weight_hhZbias_ihZbias_hhÚbias_concat)r   rÀ  Úlayer_weightsrÁ  r¿  Úvariantr   r   Útransform_weights�  s    $z'_generic_rnn.<locals>.transform_weightsc                s&   ˆdkr| S t jˆ | dg|g|gd�S )Nr"   r   )ra   rb   rc   )r%   rº   )r¼  r¥  r¦  )r   Ú
num_layersr   r   Úretrieve_stateš  s    z$_generic_rnn.<locals>.retrieve_stater5   )r6   Zdirection_sÚbidirectional)r�   Úhidden_size_iZactivations_s)r�   rË  Zlinear_before_reset_i)r�   rË  r   rA   )rB   r¹   )r<   r6   zVExporting a model to ONNX with a batch_size other than 1, with a variable lenght with )r"   rª   )r   r"   )rª   rô   )r   r"   )rô   rœ   )r"   rô   )r¹  )rº  )r»  rP  )r5   )r5   )r¾   r¿   ÚdictÚzipr_   r`   r§   r   r   Ú
startswithr¶  r¡   r¬   r   r°   r0   r‘   ))r   rÆ  r   Zinitial_statesr·  Ú
has_biasesrÈ  rˆ  r‡  rÊ  Úbatch_firstÚbatch_sizesZonnxActivationsZvariantToOnnxActivationMapZnonlinearityZw_hhZunidirectionalZprev_outputZh_outsZh0Zc0Zc_outsZsequence_lensrÇ  rÉ  r4   rÂ  rÃ  rÄ  Zstate_indicesZweight_ih_fZweight_hh_fZbias_fZweight_ih_bZweight_hh_bZbias_bÚinputsÚextra_kwargsZ
activationZh_outZc_outr   )	r·  r   rÀ  rÅ  rÈ  rÁ  r¿  rÆ  r¸  r   Ú_generic_rnnZ  s¢     











&

"
"rÔ  c
             C   s2   t j|ƒt j|ƒ }
}t| d||
|||||||	ƒS )Nr»  )r%   r7   rÔ  )r   r   Úhidden_vÚweight_vrÏ  rÈ  rˆ  r‡  rÊ  rÐ  Úhiddenr–   r   r   r   Ú
_lstm_fullá  s    rØ  c
             C   s4   t j|ƒt j|ƒ }
}t| d||
||||||	|d�S )Nr»  )rÑ  )r%   r7   rÔ  )r   r   rÑ  rÕ  rÖ  rÏ  rÈ  rˆ  r‡  rÊ  r×  r–   r   r   r   Ú_lstm_packedè  s    rÙ  c             G   s.   t j|d ƒrt| f|žŽ S t| f|žŽ S d S )Nrô   )r%   Ú_is_tensor_listrÙ  rØ  )r   rq   r   r   r   Úlstmï  s    rÛ  c          
      sZ   t dddddddddƒ	‡fdd„ƒ‰ t dddddddddƒ	‡fdd„ƒ‰‡ ‡fdd	„}|S )
Nr3   r4   r  c
                s&   t j|ƒ}
t| ˆ |||
||||||	ƒS )N)r%   r7   rÔ  )r   r   r×  rÖ  rÏ  rÈ  rˆ  r‡  rÊ  rÐ  r–   )rx   r   r   Ú	_rnn_full÷  s    
z"_one_hidden_rnn.<locals>._rnn_fullc
                s(   t j|ƒ}
t| ˆ |||
|||||	|d�S )N)rÑ  )r%   r7   rÔ  )r   r   rÑ  r×  rÖ  rÏ  rÈ  rˆ  r‡  rÊ  r–   )rx   r   r   Ú_rnn_packedý  s    
z$_one_hidden_rnn.<locals>._rnn_packedc                s.   t j|d ƒrˆ| f|žŽ S ˆ | f|žŽ S d S )Nrô   )r%   rÚ  )r   rq   )rÜ  rÝ  r   r   rl     s    z!_one_hidden_rnn.<locals>.symbolic)r   )rx   rl   r   )rÜ  rÝ  rx   r   Ú_one_hidden_rnnö  s    $$rÞ  rº  ZRNN_TANHZRNN_RELUc             C   s`   | j d|ƒ}| j d|| j dtj|ƒd�dd�}tjtjjjkrJ| j d|ƒS t| |dd d d ƒS d S )	Nr   r•   rA   )rB   r   )r6   z_caffe2::Rangerœ   )	r   r0   rF   r%   rQ  rR  rS  rT  Úarange)r   Úliker9   Z
like_shapeÚstopr   r   r   Ú_dim_arange  s
    "râ  c             C   s   |S )Nr   )r   r   r   r   r   Údetach  s    rã  c             C   s   |dkrt dƒ‚|S )Nrª   z-onnx memory_format support is not implemented)rž   )r   r   rš  r   r   r   Ú
contiguous!  s    rä  c             C   sf   |r| j d|dddgd�}|jƒ jtjjjƒ ƒs8tdƒ‚|jƒ jƒ dkrTt	| |dƒ}| j d	||dd
�S )NrŒ   r"   r   rª   )r�   z(Lengths must be a Tensor for ONNX exportZIntFzprim::PackPadded)r�   )
r   r¡   r°  r0   Z_CZ
TensorTypeÚgetrž   r#  Z	_cast_Int)r   r   ÚlengthsrÐ  r   r   r   Ú_pack_padded_sequence(  s    rç  c             C   s8   | j d||dd�\}}|r0| j d|dddgd�}||fS )Nzprim::PadPackedrª   )r�   rŒ   r"   r   )r�   )r   )r   ÚdatarÑ  rÐ  Zpadding_valueÚtotal_lengthræ  r   r   r   Ú_pad_packed_sequence9  s    rê  c             G   s&   t |ƒ}tj|d dƒ}| jd|d�S )Nr   rf   ZRandomNormal)Úshape_i)r¦   r%   r�   r   )r   ÚshapesÚshapes_listr   r   r   r   ÚrandnD  s    rî  c             G   s&   t |ƒ}tj|d dƒ}| jd|d�S )Nr   rf   ZRandomUniform)rë  )r¦   r%   r�   r   )r   rì  rí  r   r   r   r   ÚrandJ  s    rï  c             G   s   | j d|ƒS )NZRandomNormalLike)r   )r   r   Zothersr   r   r   Ú
randn_likeP  s    rð  c             C   s    | j d|||d�}| j d||ƒS )NZRandomUniformLike)Zhigh_fZlow_frÁ   )r   )r   r   r¶  ÚupperrE  rŠ   r†  r   r   r   ÚrreluT  s    rò  c             C   s   | j d|ƒ}| j d|ƒS )Nr[   rv  )r   )r   r   r†  r   r   r   Úlog_sigmoidZ  s    ró  c             C   s   | j d|ƒS )NÚErf)r   )r   r   r   r   r   Úerf`  s    rõ  c       
      C   sö   |j ƒ jƒ }|dk r|| }|dkr@||d kr@| jd||d�S |dkrh||d krh| jd||d d�S |jƒ sztddƒS |j ƒ jƒ }g }xFtd|ƒD ]8}||k rÀ||krÀ|| ||  ||< q–|j|| ƒ q–W | jdtj	|ƒd	�}t
| ||ƒ}	|	S )
Nr   r"   r«   )r6   rª   Úflattenzinput size not accessiblerA   )rB   )r¡   r9   r   r¥   r   r¬   r§   r°   r0   r‘   r   )
r   r   Z	start_dimZend_dimr9   Z
input_dimsZoutput_dimsr4   r   r†  r   r   r   rö  e  s$    
rö  c             C   s   t | | jd|ƒƒS )NZNonZero)r>   r   )r   r   r   r   r   Únonzero~  s    r÷  c             C   s   | j d|ƒ}|S )NZIsNaN)r   )r   r   rþ   r   r   r   Úisnanƒ  s    rø  c             C   s   t j| ||g|g|| gd�S )N)ra   rb   rc   )r%   rº   )r   r   r9   r¥  Úlengthr   r   r   Únarrow‰  s    rú  c             C   sT   |j ƒ jƒ r*t| |dƒ}| jd|ddd�S t|dƒ}t|dƒ}| jd|||d�S d S )	Nr"   rƒ  r   F)r6   re   r4   rP  )rP  )rw   r‰   r   r   r   )r   r   r9   rg   Ú	flattenedr   r   r   ÚargmaxŽ  s    

rü  c             C   sT   |j ƒ jƒ r*t| |dƒ}| jd|ddd�S t|dƒ}t|dƒ}| jd|||d�S d S )	Nr"   r…  r   F)r6   re   r4   rP  )rP  )rw   r‰   r   r   r   )r   r   r9   rg   rû  r   r   r   Úargmin˜  s    

rý  c             C   s   | j d||||d�S )Nrh  )r6   )r   )r   r   r9   r»   Úsrcr   r   r   ri  ¢  s    ri  c             C   s|   |j ƒ stddƒS |jƒ jƒ }tjjtj| ƒ}tj| }|jƒ j	ƒ }| j
dtj||d�d�}tj| ||||ƒ}t| ||ƒS )NÚscatter_addzinput size not accessiblerA   )rv   )rB   )r¥   r   r¡   r#  r%   rÒ   r»   r  rŸ  r¬   r   r0   r–  Z_scatter_helperr#   )r   r   r9   r»   rþ  rv   r¬   Zto_addr   r   r   rÿ  §  s    

rÿ  c             C   s*   d}| j dt| |ƒ| j dtj|gƒd�ƒS )Ngï9úþB.æ?r.   rA   )rB   )r   rw  r0   rA  )r   r   Z_ln2r   r   r   Úlog2´  s    r   c             C   s   | j d|ƒS )Nr   )r   )r   r   r   r   r   Ú
prim_shape¹  s    r  c       	   	   C   s²   t j|dƒrtddƒS |jƒ jƒ }| jdtjddgƒd�}t| || jdtj|gƒd�ƒ}| jd| jd	||||d
�t j	| d�}| jd| jd||d gd�|ƒ}| jd||gdd�S )Nr4   Úgatherzsparse_grad == TruerA   r   r"   )rB   rÏ   ZOneHot)r6   )rÐ   r,   r;   )r<   r   )r<   re   )
r%   r�   r   r¡   r#  r   r0   r‘   r’   r  )	r   r   r9   r»   Zsparse_gradrv   rd  Údepthr-   r   r   r   r  ½  s    
& r  Úbc          	      s"  ˆ j ƒ �r| jdˆ ˆ ƒ}|d krN| jd|dd�}| jdˆ dd�}ˆ jƒ jƒ }n6| jd|||d�}| jdˆ ||d�}‡ fdd„|D ƒ}| jd||ƒ}	| jd| jd	||	ƒƒ}
|�rtj|ƒ}| jd|
| jd
tj|tjd�d�ƒ}| jd|| jd
tj|d tjd�d�ƒ}
| jd|
ƒ}|S t	ddƒ d S )Nr,   r�   r   )re   )r<   re   c                s   g | ]}ˆ j ƒ jƒ | ‘qS r   )r¡   r¬   )r=   r4   )r   r   r   r?   Ö  s    z_std.<locals>.<listcomp>rt  r*   rA   )rv   )rB   r.   r"   rO   Ústdz8Unknown input rank. Cannot compute std along dimensions.)
r¥   r   r¡   r¬   rè   rƒ   r0   rF   rÙ   r   )r   r   r9   Zunbiasedrg   ZsqrdZsqrdmeanr‚   Zredudced_dimsZmeansqrdÚvarÚcountr-   r  r   )r   r   Ú_stdË  s$    

$(r  c             G   s>   |d j ƒ jtjƒ ƒr&t| |f|žŽ S t| |d |d d ƒS d S )Nr   )r¡   r°  r   r±  r  )r   r   rq   r   r   r   r  è  s    r  c             C   s   | j d|||d�S )NZReduceLogSumExp)r<   re   )r   )r   r   r9   rg   r   r   r   Ú	logsumexpï  s    r	  c       	      G   s,  t jtjjjkr$| jd|žddiŽS dd„ }t|ƒdkrž||d ƒ}| jd|d	 d	gd
�}| jdt| t	| ||f|dd … žŽ ƒdgd
�}| jd|t j
| d�S t|ƒdk�r@||d ƒ}| jd|d d	gd
�}| jd|d	 d	gd
�}| jd||ƒ}| jd| jdt| t	| ||f|dd … žŽ ƒdgd
�|ƒ}| jd|t j
| d�S t|ƒdk�r||d ƒ}| jd|d d	gd
�}| jd|d d	gd
�}| jd|d	 d	gd
�}| jd| jd||ƒ|ƒ}| jdt| t	| ||f|dd … žŽ ƒdgd
�}| jd| jd||ƒ|ƒ}| jd|t j
| d�S tdtt|ƒƒ d ƒ‚d S )Nr„   r†   rß  c             S   s   t j| dƒ} t j| ƒrd} | S )Nr4   rœ   )r%   r�   r�   )rv   r   r   r   Ú_get_arange_dtypeø  s    
z!arange.<locals>._get_arange_dtyper  r"   r;   r   )r<   r¹   rª   rÏ   )rÐ   r¯  r*   r$   rô   é   r.   rœ   r,   z&Unknown aten::arange signature taking z arguments.)r„   )r%   rQ  r0   rR  rS  rT  r   r_   r÷  r1   rÒ   rn   rÀ   )	r   rq   r
  rv   r¦  Zarange_tensorr¥  Zrange_tensorrZ  r   r   r   rß  ô  s2    .8.rß  c             C   s0   t | |dƒ}tj|ƒ}| jd|tj| ||ƒ|ƒS )NFr6  )Z
_cast_Boolr%   r'   r   rQ   )r   r   ÚmaskrÈ   r   r   r   Úmasked_fill  s    
r  c                s>  t jtjjjkr"ˆjd||dd�S t j|ƒr8t j|ƒ}n|g}‡fdd„‰‡fdd„|D ƒ}t	|ƒdkrzt
ˆ|d	|d	 ƒS d
d„ t|ƒD ƒ‰ t	ˆ ƒd	krœ|S t	ˆ ƒdkrÂt
ˆ|ˆ d	 |ˆ d	  ƒS |jƒ jƒ }|d krÞtd%ƒ‚tjdtt jƒ d d d ƒ |jƒ jƒ }t	ˆ ƒ}tˆ|ƒ‰‡‡fdd„t|ƒD ƒ‰ˆjd|ˆ ‡ fdd„t|ƒD ƒ d�}ˆjd||d�}|ˆ d&  }ˆˆ d'  }xTt|d d(d)ƒD ]@}ˆjd|ˆ |  |ƒ}	ˆjd||	ƒ}ˆjd|ˆˆ |  ƒ}�q”W t
ˆ|d	|ƒ}tˆ|ƒ}
ˆ ttˆ d	 ˆ d* d ƒƒk�r ˆjdtjd+gƒd�g‡ ‡fdd„t|ƒD ƒ }ˆjd,|ždd	iŽ}ˆjd ||ƒ}ttdˆ d	 d ƒƒd	g ttˆ d	 d || d ƒƒ }ˆjd||d�}‡fd!d„tˆ d	 ƒD ƒ|
g ‡ ‡fd"d„tˆ d	 |ƒD ƒ }ˆjd-|ždd	iŽ}n,ˆjd|
f‡ ‡fd#d„t|ƒD ƒždd	iŽ}ˆjd ||ƒS d S ).Nr„   r»   )r†   c                s@   | j ƒ jƒ  r<| jƒ jƒ dkr<tjdƒ tˆ tˆ | ƒdd�} | S )NZBytez—Exporting aten::index operator with indices of type Byte. Only 1-D indices are supported. In any other case, this will produce an incorrect ONNX graph.r"   )r9   )rw   r‰   r¡   r#  r¾   r¿   r¼   r÷  )r»   )r   r   r   Útry_mask_to_index)  s    
z index.<locals>.try_mask_to_indexc                s   g | ]}ˆ |ƒ‘qS r   r   )r=   Úidx)r  r   r   r?   1  s    zindex.<locals>.<listcomp>r"   r   c             S   s    g | ]\}}|j ƒ jƒ s|‘qS r   )rw   r‰   )r=   r4   r  r   r   r   r?   G  s    zQUnsupported aten::index operator of advanced indexing on tensor of unknown rank, z7try turning on shape and type propagate during export: z(torch.onnx._export(..., propagate=True).z=Exporting aten::index operator of advanced indexing in opset z8 is achieved by combination of multiple ONNX operators, z2including Reshape, Transpose, Concat, and Gather. zVIf indices include negative values, the exported graph will produce incorrect results.c          	      s0   g | ](}ˆ j d ˆˆ j dtj|gƒd�dd�‘qS )r•   rA   )rB   r   )r6   )r   r0   r‘   )r=   r9   )r   Úshape_tensorr   r   r?   ^  s    rŒ   c                s   g | ]}|ˆ kr|‘qS r   r   )r=   r4   )Úadv_idx_indicesr   r   r?   a  s    )r�   r«   )r6   rª   r,   r$   rA   )rB   c                s   g | ]}|ˆ krˆ| ‘qS r   r   )r=   r4   )r  Údim_tensor_listr   r   r?   v  s    r5   r6   r   c                s   g | ]}ˆ | ‘qS r   r   )r=   r4   )r  r   r   r?   €  s    c                s   g | ]}|ˆ krˆ| ‘qS r   r   )r=   r4   )r  r  r   r   r?   ‚  s    c                s   g | ]}|ˆ krˆ| ‘qS r   r   )r=   r4   )r  r  r   r   r?   ˆ  s    zˆUnsupported aten::index operator of advanced indexing on tensor of unknown rank, try turning on shape and type propagate during export: z°Unsupported aten::index operator of advanced indexing on tensor of unknown rank, try turning on shape and type propagate during export: torch.onnx._export(..., propagate=True).rP  rP  rP  rP  rP  rP  )r5   )r5   )r%   rQ  r0   rR  rS  rT  r   Z_is_packed_listr7   r_   rb  r½   r¡   r9   rn   r¾   r¿   rÀ   Z_export_onnx_opset_versionr   r§   r¦   r‘   )r   r   r»   r—   r£   Zadv_idx_countZcum_adv_indexZ
multiplierr4   Z	adv_indexZcum_adv_index_shape_tensorZfolded_adv_idx_shape_listZfolded_adv_idx_shapeZadv_idx_permuteZfinal_shape_listZfinal_shaper   )r  r  r   r  r  r   r»      sf    
 
&
 :>r»   c             C   s,   | j d||ƒ}| j d|||d�}| j d|ƒS )Nr,   r   )r<   re   rO   )r   )r   r   r9   rg   ZsqrZsumsqrr   r   r   Úfrobenius_normŽ  s    r  c             C   sZ   |d k	r |j ƒ jƒ  r tddƒ | r8|dkr8tddƒ t| |ƒ}| jd|tjd |d�S )NZMultinomialz*generator is not supported for multinomialr"   zGreplacement=False when num_samples > 1 is not supported for multinomialZLong)Zdtype_iZsample_size_i)rw   r‰   r   rw  r   r%   r  )r   r   Znum_samplesÚreplacementrŠ   Z	log_inputr   r   r   Úmultinomial•  s    


r  c       
      C   s`   |j ƒ jƒ }t| ||ƒ}t| || jd|tj| d�ƒ}t| || jd|tj| d�ƒ}	t| ||	ƒS )NrÏ   )rÐ   )r¡   r#  rK   r-   r   r%   r  r#   )
r   r   Zbatch1Zbatch2rL   r(   rv   Z	batch_mulZmul_aZmul_br   r   r   Úbaddbmm¡  s
    r  c          	   C   sj   d}| j dt| |tj|ƒƒƒ}t| || j dtjdtjd�d�ƒ}t| t| ||ƒ| j dtjdtjd�d�ƒS )NgÍ;fž ö?rô  rA   r"   )rv   )rB   g      à?)r   r/   r0   rF   r#   rÙ   r-   )r   r   Z_sqrt2rõ  Zerf_plusoner   r   r   Úgelu¨  s    "r  )N)N)N)T)T)N)F)N)N)N)NN)NN)FN)FN)F)F)F)F)F)F)N)N)NN)F)NF)FN)õÚ
__future__r   r   r   r   r0   Ztorch._Cr   r   Ztorch.nn.modules.utilsr   r	   r
   Z
torch.onnxZtorch.onnx.utilsÚ	functoolsr   r   Ztorch.onnx.symbolic_helperrR  Zsymbolic_helperr%   r   r   r   rè   rØ   r¾   r   r   r   r   r!   r#   r)   r+   r-   r/   r2   r:   r@   rH   rJ   rK   rM   rN   rP   rR   rT   rU   rV   rW   rX   rY   rZ   r\   r]   rd   rm   rt   r~   r€   r‚   rƒ   r…   rˆ   r‹   r>   r“   r”   rš   r›   r’   r¤   r©   r   r´   r¶   r·   r¸   r¢   r¼   rÂ   rÄ   rÅ   rÆ   rÇ   rË   rÎ   rÓ   rÖ   rß   rð   rñ   rò   ró   rõ   rö   r÷   rÿ   r   r  r  r  r  r  r	  r
  r  r  r  r  r  Zreflection_pad1dZreflection_pad2dZreflection_pad3dZreplication_pad1dZreplication_pad2dZreplication_pad3dr  r  r  r  r  r  r  r!  r$  r(  r*  r+  r-  r,  r/  r.  r0  r1  r3  r5  r8  r9  r@  rM  rN  rX  rY  r[  r`  ra  rb  rc  rf  rn  rp  rq  rs  ru  rw  rx  rU  r‚  r  r~  r�  r€  rÔ   rˆ  rŠ  r‰  r‹  rŒ  Zdropout_Zfeature_dropout_Zalpha_dropout_Zfeature_alpha_dropout_r�  rŽ  r�  r•  r  Úitemsr  r3   ro   ry   Z_cast_func_templater"  r›  r�  r–  rœ  r1   r   r¢  r£  r¤  r§  r¨  r©  r¬  r®  r²  r³  r´  rµ  rÔ  rØ  rÙ  rÛ  rÞ  ZgruZrnn_tanhZrnn_relurâ  rã  rä  rç  rê  rî  rï  rð  rò  ró  rõ  rö  r÷  rø  rú  rü  rý  ri  rÿ  r   r  r  r  r  r	  rß  r  r»   r  r  r  r  r   r   r   r   Ú<module>   sä  (





&2


$

&" $ 






"	 $$




&n
