a
    ö£BeÃd  ã                   @   s‚  U d Z ddlmZ ddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZmZmZmZ ddlmZ ddlmZmZmZ dZejZejZejZejZejZejZejZej Z ej!Z!ej"Z"ej#Z#ej$Z$ej%Z%G d	d
„ d
e
ƒZ&e 'd¡Z(i Z)eee&f e*d< ddœdd„Z+dd„ Z,dddœdd„Z-dd„ Z.dd„ Z/dZ0e.de0 d ƒZ1e.de0 d ƒZ2e.dƒZ3e.dƒZ4d d!„ Z5G d"d#„ d#e
ƒZ6G d$d%„ d%e6ƒZ7G d&d'„ d'ƒZ8d(d)„ Z9d*d+„ Z:d,d-œe;eee<e<f ee7 d.œd/d0„Z=d1d2„ Z>dd,d3d4œee; eee< ee<e<f ee7 d5œd6d7„Z?d8d9„ Z@eAd:k�r~ejBd; ZCeDeCƒ�ZEeE F¡ ZGW d  ƒ n1 �sT0    Y  e=eGed<ƒd=�D ]ZHeIeHƒ �qndS )>a¼  
This tokenizer has been copied from the ``tokenize.py`` standard library
tokenizer. The reason was simple: The standard library tokenizer fails
if the indentation is not right. To make it possible to do error recovery the
    tokenizer needed to be rewritten.

Basically this is a stripped down version of the standard library module, so
you can read the documentation there. Additionally we included some speed and
memory optimizations here.
é    )Úabsolute_importN)ÚBOM_UTF8)Ú
NamedTupleÚTupleÚIteratorÚIterableÚListÚDictÚPatternÚSet)ÚPythonTokenTypes)Úsplit_linesÚPythonVersionInfoÚparse_version_stringu   ô�¿¿c                   @   sb   e Zd ZU eed< ee ed< ee ed< eeef ed< eed< eeef ed< ee ed< dS )	ÚTokenCollectionÚpseudo_tokenÚsingle_quotedÚtriple_quotedÚendpatsÚ
whitespaceÚfstring_pattern_mapÚalways_break_tokensN)	Ú__name__Ú
__module__Ú__qualname__r
   Ú__annotations__r   Ústrr	   r   © r   r   úX/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/parso/python/tokenize.pyr   +   s   
r   zutf-8Ú_token_collection_cacheF©Úcapturec                 O   s*   |rJ ‚d}| s|d7 }|d  |¡ d S )Nú(z?:ú|ú))Újoin)r!   ÚchoicesÚkwargsÚstartr   r   r   Úgroup:   s
    r)   c                  G   s   t | Ž d S )Nú?)r)   )r&   r   r   r   ÚmaybeC   s    r+   ©Úinclude_fstringÚonly_fstringc                 C   sr   dd„ }g d¢}dh}| r<ddg}|r2|}t ƒ }qF||7 }n
|rFt ƒ S |D ]"}t |¡D ]}| ||ƒ¡ qXqJ|S )Nc                 s   s*   t jdd„ | D ƒŽ D ]}d |¡V  qd S )Nc                 S   s   g | ]}||  ¡ f‘qS r   )Úupper)Ú.0Úcr   r   r   Ú
<listcomp>J   ó    zI_all_string_prefixes.<locals>.different_case_versions.<locals>.<listcomp>Ú )Ú
_itertoolsÚproductr%   )ÚprefixÚsr   r   r   Údifferent_case_versionsI   s    z5_all_string_prefixes.<locals>.different_case_versions)ÚbÚrÚuÚbrr4   ÚfÚfr)Úsetr5   ÚpermutationsÚupdate)r-   r.   r9   Zvalid_string_prefixesÚresultr>   r7   Útr   r   r   Ú_all_string_prefixesH   s    
rE   c                 C   s   t  | t j¡S ©N)ÚreÚcompileÚUNICODE)Úexprr   r   r   Ú_compilee   s    rK   c                 C   s>   zt t| ƒ W S  ty8   t| ƒ t t| ƒ< }| Y S 0 d S rF   )r   ÚtupleÚKeyErrorÚ_create_token_collection)Úversion_inforC   r   r   r   Ú_get_token_collectioni   s    ÿrP   z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c           -   	   C   sˆ  d}t |ƒ}d}dt d }d}d}d}d}t||||ƒ}	d	}
td
dƒt|
ƒ }d|
 }t||ƒ}td|d ƒ}t|||	ƒ}tƒ }t|Ž }ttdd�Ž }tddd�}t|Ž }d}d}d}d}t|d |d ƒ}tdddddddƒ}d}g d ¢}| d!k�r| d"d#¡ n| d"d$¡ t|Ž }t|||ƒ}t|d% td&d'ƒ |d( td)d'ƒ ƒ}||g} d*}!|�rp|  |t|!Ž  ¡ td+g| ¢R Ž }"t|dd,�t|"||||dd,� }#i }$|D ]F}%t |ƒ|$|%d& < t |ƒ|$|%d) < t |ƒ|$|%d < t |ƒ|$|%d < �q¨tƒ }&tƒ }'i }(|D ]:})d-D ]}*|& |)|* ¡ �qd.D ]}*|' |)|* ¡ �q&�q|D ]})|!D ]}*|*|(|)|* < �qL�qDd/}+t |#ƒ},t	|,|&|'|$||(t|+ƒƒS )0Nz[ \f\t]*z	#[^\r\n]*u   ([A-Za-z_0-9Â€-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r-   r,   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""ú'''ú"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?ú~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])é   é   r   z:=?ú:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*ú'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*ú")rX   rW   rR   rQ   z\\(?:\r\n?|\n)|\Zr    )rX   rW   )rR   rQ   )ú;ÚimportÚclassÚdefÚtryÚexceptÚfinallyÚwhileÚwithÚreturnÚcontinueÚbreakÚdelÚpassÚglobalÚassertÚnonlocal)
rK   ÚMAX_UNICODEr)   r+   rE   ÚinsertÚappendr@   Úaddr   )-rO   Ú
Whitespacer   ÚCommentÚNameÚ	HexnumberÚ	BinnumberÚ	OctnumberÚ	DecnumberÚ	IntnumberÚExponentÚ
PointfloatÚExpfloatÚFloatnumberÚ
ImagnumberÚNumberZpossible_prefixesÚStringPrefixZStringPrefixWithFZfstring_prefixesZFStringStartÚSingleÚDoubleÚSingle3ÚDouble3ÚTripleÚOperatorZBracketZspecial_argsÚSpecialÚFunnyÚContStrZpseudo_extra_poolZ
all_quotesÚPseudoExtrasÚPseudoTokenr   Ú_prefixr   r   r   rD   ÚquoteZALWAYS_BREAK_TOKENSZpseudo_token_compiledr   r   r   rN   ~   sž    ÿÿ
ý
ÿÿþ
ÿ
þrN   c                   @   sP   e Zd ZU eed< eed< eeef ed< eed< eeeef dœdd„ƒZ	dS )	ÚTokenÚtypeÚstringÚ	start_posr7   )rb   c                 C   sR   t | jƒ}t|ƒdkr0| jd t|ƒ d dfS | jd | jd t| jƒ fS d S ©Né   r   )r   rŒ   Úlenr�   )ÚselfÚlinesr   r   r   Úend_posð   s    
zToken.end_posN)
r   r   r   r   r   r   r   ÚintÚpropertyr“   r   r   r   r   rŠ   ê   s   
rŠ   c                   @   s   e Zd Zdd„ ZdS )ÚPythonTokenc                 C   s   d| j | jjd� S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r‹   )Ú_replacer‹   Úname©r‘   r   r   r   Ú__repr__ú   s    ÿzPythonToken.__repr__N)r   r   r   rš   r   r   r   r   r–   ù   s   r–   c                   @   s<   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )ÚFStringNodec                 C   s"   || _ d| _d| _d | _d| _d S )Nr   r4   )r‰   Úparentheses_countÚprevious_linesÚlast_string_start_posÚformat_spec_count)r‘   r‰   r   r   r   Ú__init__   s
    zFStringNode.__init__c                 C   s   |  j d7  _ d S )Nr�   )rœ   ©r‘   Ú	characterr   r   r   Úopen_parentheses	  s    zFStringNode.open_parenthesesc                 C   s"   |  j d8  _ | j dkrd| _d S rŽ   ©rœ   rŸ   r¡   r   r   r   Úclose_parentheses  s    
zFStringNode.close_parenthesesc                 C   s   t | jƒdkS )NrT   )r�   r‰   r™   r   r   r   Úallow_multiline  s    zFStringNode.allow_multilinec                 C   s   | j | jkS rF   r¤   r™   r   r   r   Ú
is_in_expr  s    zFStringNode.is_in_exprc                 C   s   |   ¡  o| jS rF   )r§   rŸ   r™   r   r   r   Úis_in_format_spec  s    zFStringNode.is_in_format_specN)	r   r   r   r    r£   r¥   r¦   r§   r¨   r   r   r   r   r›   ÿ   s   	r›   c           
   	   C   s–   t | ƒD ]‚\}}| ¡ }t|ƒt|ƒ }| |j¡rtt|j||| f||d |…  d�}	d}|jrhJ ‚| |d …= |	dt|jƒ| f  S qd |dfS )N©r7   r4   r   )Ú	enumerateÚlstripr�   Ú
startswithr‰   r–   ÚFSTRING_ENDr�   )
Úfstring_stackrŒ   Zline_nrÚcolumnÚadditional_prefixZfstring_stack_indexÚnodeZlstripped_stringZ
len_lstripÚtokenr   r   r   Ú_close_fstring_if_necessary  s    
ü

r³   c                 C   sô   |d }|  ¡ }| ¡ r(|r"t}q6t}n|r2t}nt}| ||¡}|d u rT|j|fS |jsd||f|_| 	d¡}	|D ]6}
| |
j
  |	¡}|d urr| 	d¡d t|
j
ƒ … }	qr|}|t|	ƒ7 }|	 d¡sÎ|	 d¡râ| j|	7  _d}	n
|j|	 }	|	|fS )Néÿÿÿÿr   Ú
úr4   )r¦   r¨   Úfstring_format_spec_multi_lineÚfstring_format_spec_single_lineÚfstring_string_multi_lineÚfstring_string_single_lineÚmatchr�   rž   r)   r‰   r�   Úendswith)r   r®   ÚlineÚlnumÚposÚtosr¦   Úregexr»   rŒ   Úfstring_stack_nodeÚ	end_matchÚnew_posr   r   r   Ú_find_fstring_string.  s4    



rÅ   )r�   r   )r�   )ÚcoderO   r�   rb   c                C   s   t | dd�}t|||d�S )z0Generate tokens from a the source code (string).T)Úkeepends)rO   r�   )r   Útokenize_lines)rÆ   rO   r�   r’   r   r   r   ÚtokenizeV  s    rÉ   c                    s   ‡ fdd„}|S )zL
    A small helper function to help debug the tokenize_lines function.
    c                  ?   s&   ˆ | i |¤ŽD ]}t |ƒ |V  qd S rF   )Úprint)Úargsr'   r²   ©Úfuncr   r   Úwrapperb  s    z_print_tokens.<locals>.wrapperr   )rÍ   rÎ   r   rÌ   r   Ú_print_tokens^  s    rÏ   T)Úindentsr�   Úis_first_token)r’   rO   rÐ   r�   rb   c          .   	   #   s  ‡ ‡‡fdd„}t |ƒ\}}}}	}
}}d}ˆ du r8dg‰ d}d}d}d}d}d}|d d ‰g }| D �]}ˆd7 ‰d}t|ƒ}|rÌ| t¡r¤t}|dd… }t|ƒ}d	|d  | }|d }||d 7 }d
}|�r$| |¡}|�r| d¡}tt||d|…  ||ƒV  d}d}n|| }|| }qd||k rd|�rÆ|d }| ¡ �sˆt	|	||ˆ|ƒ\}}|�r|tt
||jdd�V  d|_�q$||k�rˆqd||d… }t||ˆ||ƒ\}}}||7 }|du�rÆ|V  �q$|�rP|} |D ]l}!|!j}"|	|"  ||¡}#|#du�rÔ|# d¡}$t|$ƒt|"ƒ | t| ƒk �rÔ|d|… |$dt|"ƒ …  } �qÔ| | |¡}%n| ||¡}%|%�r¸||% d¡ }d}|% d¡\}&}ˆ|&f‰|% d¡}'|'dk�r®|�s¨J ‚|}qd|'d }(n(|
 ||¡})||) ¡  }(|) ¡ }&ˆ|&f‰|�rR|(dv�rR|(dk�s|%du �rRd
}|dk�rR|�sR|&}*|*ˆ d k�rDttdˆdƒV  ˆ  |*¡ ||*ƒE dH  |%�sÊ|
 ||¡})|�rŒ|dk�rŒ|�sŒ||) ¡ ƒE dH  |) ¡ }d
}tt|| ˆ|f||) d¡ ƒV  d}|d7 }�q$|(|v �sò|(dk�r|'dk�r|'dk�rtt|'ˆ|ƒV  �q$|% d¡du�rž|'|v �rl|�s,|�rlg |dd…< d}t d|d|&… ¡}+|+du�rl||+ ¡ ƒE dH  |' ¡ �rˆtt|'ˆ|ƒV  nt|'ˆ|ƒE dH  �q$|(dv �rütdd„ |D ƒƒ�rÄ| ¡  |�sì|dk�rì|�sìtt|'ˆ|ƒV  n||' }d}�q$|(dk�r\|' d¡�s|' d¡�r"J ‚|�rP|d  ¡ �rPtt|(ˆ|ƒV  |&d }n||' }�q$|'|v �rÂ|	|' }| ||¡}|�r¨| d¡}||&|… }'tt|'ˆ|ƒV  nˆ}||&d… }|}qd�q$|(|v �sð|'dd… |v �sð|'dd… |v �rX|'d dv �rDˆ|&f}|	 |(¡�p.|	 |'d ¡�p.|	 |'d ¡}||&d… }|}qdntt|'ˆ|ƒV  �q$|'|v �r†| t||' ƒ¡ tt|'ˆ|ƒV  nè|(dk�rº||&d… dv �rº||||&d…  7 }qdn´|'dv �rä|�rÚ|d   |'¡ n|d7 }nz|'dv �r|�r|d  !|'¡ n|�r^|d8 }nJ|' d¡�r^|�r^|d j"|d j# dk�r^|d  j#d7  _#d}'|&d }tt$|'ˆ|ƒV  �q$qd|�r¦tt|||ƒV  | d¡�s¢| d¡�r¦d}|�rÒ|d }|j�rÒtt
|j|jdd�V  ˆ|f},ˆ dd… D ]}-ˆ  %¡  tt&d|,dƒV  �qætt'd|,|ƒV  dS )a)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    c                 3   sT   | ˆ d k rP| ˆ d kr6t tdˆ| fdƒV  | ˆ d< qPˆ  ¡  t tdˆdƒV  q d S )Nr´   éþÿÿÿr4   )r–   ÚERROR_DEDENTÚpopÚDEDENT)r(   ©rÐ   r¾   Úsposr   r   Údedent_if_necessaryz  s    z+tokenize_lines.<locals>.dedent_if_necessaryr   NÚ
0123456789r4   Tr�   ú^Fr´   r©   é   z
#ú\Ú.z...rT   z	[ \f\t]*$z
c                 s   s   | ]}|  ¡  V  qd S rF   )r¦   )r0   r>   r   r   r   Ú	<genexpr>  r3   z!tokenize_lines.<locals>.<genexpr>ú#rµ   r¶   )z\
z\
z\z([{z)]}rV   )(rP   r�   r¬   ÚBOM_UTF8_STRINGr»   Úendr–   ÚSTRINGr§   rÅ   ÚFSTRING_STRINGrž   r�   r³   r‰   r)   ÚspanÚINDENTrl   Ú
ERRORTOKENÚNUMBERrG   ÚisidentifierÚNAMEÚ_split_illegal_unicode_nameÚanyÚclearÚNEWLINEr¼   Úgetr›   ÚFSTRING_STARTr£   r¥   rœ   rŸ   ÚOPrÔ   rÕ   Ú	ENDMARKER).r’   rO   rÐ   r�   rÑ   rØ   r   r   r   r   r   r   r   Zparen_levelZmax_ÚnumcharsÚcontstrÚnew_liner7   r°   r®   r½   r¿   ÚendprogÚendmatchZcontstr_startÚcontlinerÀ   rŒ   ÚrestZfstring_end_tokenZquote_lengthZstring_linerÂ   r‰   rÃ   Zend_match_stringÚpseudomatchr(   r²   Úinitialr»   Zindent_startÚmr“   Úindentr   rÖ   r   rÈ   k  s¬   þ



þ
û
û



"



$
þ
ÿÿÿ







ÿþÿ




ÿÿÿûrÈ   c                 #   sÂ   ‡ ‡‡‡fdd„}d‰ d‰|‰t | ƒD ]Š\}}ˆrn| ¡ rd|ƒ V  |‰ d‰d‰|d |d | f‰q°ˆ |7 ‰ q&ˆ | }| ¡ r„|‰ q&ˆ r¨|ƒ V  d‰|d |d | f‰|‰ d‰q&ˆ r¾|ƒ V  d S )Nc                      s   t ˆr
tntˆ ˆˆƒS rF   )r–   ræ   ré   r   ©ÚfoundZ
is_illegalr¿   r7   r   r   Úcreate_token‰  s    z1_split_illegal_unicode_name.<locals>.create_tokenr4   Fr   r�   T)rª   rè   )r²   r�   r7   rÿ   ÚiÚcharZ	new_foundr   rý   r   rê   ˆ  s0    
rê   Ú__main__r�   z3.10)rO   )JÚ__doc__Ú
__future__r   ÚsysrG   Ú	itertoolsr5   Úcodecsr   Útypingr   r   r   r   r   r	   r
   r   Zparso.python.tokenr   Zparso.utilsr   r   r   rj   râ   ré   rç   rð   rí   rå   rÕ   rñ   ræ   rÓ   rï   rã   r­   r   Údecoderà   r   r   r)   r+   rE   rK   rP   Zunicode_character_namerº   r¹   r¸   r·   rN   rŠ   r–   r›   r³   rÅ   r   r”   rÉ   rÏ   rÈ   rê   r   ÚargvÚpathÚopenr>   ÚreadrÆ   r²   rÊ   r   r   r   r   Ú<module>   s�   
(

		ÿÿ
ÿl)ÿþú
ù  !


(