a
    ¨ˆc¸|  ã                   @   sÀ  d Z ddlZddlZddlZddlmZmZ ddlmZ ddl	m
Z
mZmZmZ ddlmZmZmZmZmZmZ ddlmZ g d¢Zg d	¢Zed
d„ ƒZG dd„ deƒZG dd„ ded�ZG dd„ deƒZG dd„ deƒZ G dd„ dƒZ!e!ƒ Z"G dd„ de#ƒZ$G dd„ dƒZ%dd„ Z&G dd„ dƒZ'e'ƒ Z(dd „ Z)G d!d"„ d"ƒZ*G d#d$„ d$eƒZ+G d%d&„ d&eƒZ,G d'd(„ d(ee,d�Z-G d)d*„ d*ƒZ.G d+d,„ d,e-ƒZ/d-d.„ Z0G d/d0„ d0e,ƒZ1G d1d2„ d2e-e1d�Z2dS )3z´
    pygments.lexer
    ~~~~~~~~~~~~~~

    Base lexer classes.

    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
é    N)Úapply_filtersÚFilter)Úget_filter_by_name)ÚErrorÚTextÚOtherÚ
_TokenType)Úget_bool_optÚget_int_optÚget_list_optÚmake_analysatorÚFutureÚguess_decode)Ú	regex_opt)ÚLexerÚ
RegexLexerÚExtendedRegexLexerÚDelegatingLexerÚLexerContextÚincludeÚinheritÚbygroupsÚusingÚthisÚdefaultÚwords))s   ï»¿úutf-8)s   ÿþ  zutf-32)s     þÿzutf-32be)s   ÿþzutf-16)s   þÿzutf-16bec                 C   s   dS )Nç        © ©Úxr   r   úQ/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/pygments/lexer.pyÚ<lambda>!   ó    r"   c                   @   s   e Zd ZdZdd„ ZdS )Ú	LexerMetaz‚
    This metaclass automagically converts ``analyse_text`` methods into
    static methods which always return float values.
    c                 C   s(   d|v rt |d ƒ|d< t | |||¡S )NÚanalyse_text)r   ÚtypeÚ__new__)ZmcsÚnameÚbasesÚdr   r   r!   r'   *   s    zLexerMeta.__new__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   r   r   r   r!   r$   $   s   r$   c                   @   s^   e Zd ZdZdZdZg Zg Zg Zg Z	dZ
dd„ Zdd„ Zdd	„ Zd
d„ Zddd„Zdd„ ZdS )r   aÈ  
    Lexer for a specific language.

    Basic options recognized:
    ``stripnl``
        Strip leading and trailing newlines from the input (default: True).
    ``stripall``
        Strip all leading and trailing whitespace from the input
        (default: False).
    ``ensurenl``
        Make sure that the input ends with a newline (default: True).  This
        is required for some lexers that consume input linewise.

        .. versionadded:: 1.3

    ``tabsize``
        If given and greater than 0, expand tabs in the input (default: 0).
    ``encoding``
        If given, must be an encoding name. This encoding will be used to
        convert the input string to Unicode, if it is not already a Unicode
        string (default: ``'guess'``, which uses a simple UTF-8 / Locale /
        Latin1 detection.  Can also be ``'chardet'`` to use the chardet
        library, if it is installed.
    ``inencoding``
        Overrides the ``encoding`` if given.
    Nr   c                 K   s„   || _ t|ddƒ| _t|ddƒ| _t|ddƒ| _t|ddƒ| _| dd	¡| _| d
¡pZ| j| _g | _	t
|ddƒD ]}|  |¡ qpd S )NÚstripnlTÚstripallFÚensurenlÚtabsizer   ÚencodingÚguessZ
inencodingÚfiltersr   )Úoptionsr	   r/   r0   r1   r
   r2   Úgetr3   r5   r   Ú
add_filter)Úselfr6   Úfilter_r   r   r!   Ú__init__a   s    zLexer.__init__c                 C   s(   | j rd| jj| j f S d| jj S d S )Nz<pygments.lexers.%s with %r>z<pygments.lexers.%s>)r6   Ú	__class__r+   ©r9   r   r   r!   Ú__repr__m   s
    ÿzLexer.__repr__c                 K   s*   t |tƒst|fi |¤Ž}| j |¡ dS )z8
        Add a new stream filter to this lexer.
        N)Ú
isinstancer   r   r5   Úappend)r9   r:   r6   r   r   r!   r8   t   s    
zLexer.add_filterc                 C   s   dS )a~  
        Has to return a float between ``0`` and ``1`` that indicates
        if a lexer wants to highlight this text. Used by ``guess_lexer``.
        If this method returns ``0`` it won't highlight it in any case, if
        it returns ``1`` highlighting with this lexer is guaranteed.

        The `LexerMeta` metaclass automatically wraps this function so
        that it works like a static method (no ``self`` or ``cls``
        parameter) and the return value is automatically converted to
        `float`. If the return value is an object that is boolean `False`
        it's the same as if the return values was ``0.0``.
        Nr   )Útextr   r   r!   r%   |   s    zLexer.analyse_textFc              
      s¶  t ˆtƒsþˆ jdkr"tˆƒ\‰}nÚˆ jdkrÖzddl}W n. tyf } ztdƒ|‚W Y d}~n
d}~0 0 d}tD ].\}}ˆ |¡rpˆt|ƒd…  	|d¡} q qp|du rÐ| 
ˆdd… ¡}	ˆ 	|	 d¡pÊd	d¡}|‰n&ˆ 	ˆ j¡‰ˆ d
¡rüˆtd
ƒd… ‰nˆ d
¡�rˆtd
ƒd… ‰ˆ dd¡‰ˆ dd¡‰ˆ j�rDˆ ¡ ‰nˆ j�rVˆ d¡‰ˆ jdk�rnˆ ˆ j¡‰ˆ j�rŠˆ d¡�sŠˆd7 ‰‡ ‡fdd„}
|
ƒ }|�s²t|ˆ jˆ ƒ}|S )a=  
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
        r4   Úchardetr   NzkTo enable chardet encoding guessing, please install the chardet library from http://chardet.feedparser.org/Úreplacei   r3   r   u   ï»¿z
Ú
úc                  3   s$   ˆ   ˆ¡D ]\} }}||fV  q
d S ©N)Úget_tokens_unprocessed)Ú_ÚtÚv©r9   rA   r   r!   Ústreamer½   s    z"Lexer.get_tokens.<locals>.streamer)r?   Ústrr3   r   rB   ÚImportErrorÚ_encoding_mapÚ
startswithÚlenÚdecodeÚdetectr7   rC   r0   Ústripr/   r2   Ú
expandtabsr1   Úendswithr   r5   )r9   rA   Z
unfilteredrH   rB   ÚeÚdecodedÚbomr3   ÚencrL   Ústreamr   rK   r!   Ú
get_tokensŠ   sR    	


þ
ÿ


zLexer.get_tokensc                 C   s   t ‚dS )zú
        Return an iterable of (index, tokentype, value) pairs where "index"
        is the starting position of the token within the input text.

        In subclasses, implement this method as a generator to
        maximize effectiveness.
        N)ÚNotImplementedErrorrK   r   r   r!   rG   Å   s    zLexer.get_tokens_unprocessed)F)r+   r,   r-   r.   r(   ÚurlÚaliasesÚ	filenamesZalias_filenamesÚ	mimetypesÚpriorityr;   r>   r8   r%   r\   rG   r   r   r   r!   r   0   s   
;r   )Ú	metaclassc                   @   s$   e Zd ZdZefdd„Zdd„ ZdS )r   a   
    This lexer takes two lexer as arguments. A root lexer and
    a language lexer. First everything is scanned using the language
    lexer, afterwards all ``Other`` tokens are lexed using the root
    lexer.

    The lexers from the ``template`` lexer package use this base lexer.
    c                 K   s<   |f i |¤Ž| _ |f i |¤Ž| _|| _tj| fi |¤Ž d S rF   )Ú
root_lexerÚlanguage_lexerÚneedler   r;   )r9   Z_root_lexerZ_language_lexerZ_needler6   r   r   r!   r;   Ú   s    zDelegatingLexer.__init__c                 C   sŠ   d}g }g }| j  |¡D ]H\}}}|| ju rP|rF| t|ƒ|f¡ g }||7 }q| |||f¡ q|rx| t|ƒ|f¡ t|| j |¡ƒS )NÚ )re   rG   rf   r@   rQ   Údo_insertionsrd   )r9   rA   ZbufferedÚ
insertionsZ
lng_bufferÚirI   rJ   r   r   r!   rG   à   s    


ÿz&DelegatingLexer.get_tokens_unprocessedN)r+   r,   r-   r.   r   r;   rG   r   r   r   r!   r   Ð   s   	r   c                   @   s   e Zd ZdZdS )r   zI
    Indicates that a state should include rules from another state.
    N©r+   r,   r-   r.   r   r   r   r!   r   ÷   s   r   c                   @   s   e Zd ZdZdd„ ZdS )Ú_inheritzC
    Indicates the a state should inherit from its superclass.
    c                 C   s   dS )Nr   r   r=   r   r   r!   r>     s    z_inherit.__repr__N)r+   r,   r-   r.   r>   r   r   r   r!   rl   þ   s   rl   c                   @   s    e Zd ZdZdd„ Zdd„ ZdS )Úcombinedz:
    Indicates a state combined from multiple states.
    c                 G   s   t  | |¡S rF   )Útupler'   )ÚclsÚargsr   r   r!   r'     s    zcombined.__new__c                 G   s   d S rF   r   )r9   rp   r   r   r!   r;     s    zcombined.__init__N)r+   r,   r-   r.   r'   r;   r   r   r   r!   rm     s   rm   c                   @   sF   e Zd ZdZdd„ Zddd„Zddd„Zdd	d
„Zdd„ Zdd„ Z	dS )Ú_PseudoMatchz:
    A pseudo match object constructed from a string.
    c                 C   s   || _ || _d S rF   )Ú_textÚ_start)r9   ÚstartrA   r   r   r!   r;     s    z_PseudoMatch.__init__Nc                 C   s   | j S rF   )rs   ©r9   Úargr   r   r!   rt     s    z_PseudoMatch.startc                 C   s   | j t| jƒ S rF   )rs   rQ   rr   ru   r   r   r!   Úend!  s    z_PseudoMatch.endc                 C   s   |rt dƒ‚| jS )NzNo such group)Ú
IndexErrorrr   ru   r   r   r!   Úgroup$  s    z_PseudoMatch.groupc                 C   s   | j fS rF   )rr   r=   r   r   r!   Úgroups)  s    z_PseudoMatch.groupsc                 C   s   i S rF   r   r=   r   r   r!   Ú	groupdict,  s    z_PseudoMatch.groupdict)N)N)N)
r+   r,   r-   r.   r;   rt   rw   ry   rz   r{   r   r   r   r!   rq     s   


rq   c                     s   d‡ fdd„	}|S )zL
    Callback that yields multiple actions for each group in the match.
    Nc                 3   s¼   t ˆ ƒD ] \}}|d u rqqt|ƒtu rR| |d ¡}|r¨| |d ¡||fV  q| |d ¡}|d ur|r|| |d ¡|_|| t| |d ¡|ƒ|ƒD ]}|r˜|V  q˜q|r¸| ¡ |_d S )Né   )Ú	enumerater&   r   ry   rt   Úposrq   rw   )ÚlexerÚmatchÚctxrj   ÚactionÚdataÚitem©rp   r   r!   Úcallback4  s$    ÿ
zbygroups.<locals>.callback)Nr   )rp   r†   r   r…   r!   r   0  s    r   c                   @   s   e Zd ZdZdS )Ú_ThiszX
    Special singleton used for indicating the caller class.
    Used by ``using``.
    Nrk   r   r   r   r!   r‡   J  s   r‡   c                    sj   i ‰dˆv r:ˆ  d¡}t|ttfƒr.|ˆd< nd|fˆd< ˆ tu rTd‡‡fdd„	}nd	‡ ‡‡fdd„	}|S )
aÂ  
    Callback that processes the match with a different lexer.

    The keyword arguments are forwarded to the lexer, except `state` which
    is handled separately.

    `state` specifies the state that the new lexer will start in, and can
    be an enumerable such as ('root', 'inline', 'string') or a simple
    string which is assumed to be on top of the root state.

    Note: For that to work, `_other` must not be an `ExtendedRegexLexer`.
    ÚstateÚstackÚrootNc                 3   sr   ˆr"ˆ  | j¡ | jf i ˆ¤Ž}n| }| ¡ }|j| ¡ fi ˆ ¤ŽD ]\}}}|| ||fV  qD|rn| ¡ |_d S rF   )Úupdater6   r<   rt   rG   ry   rw   r~   ©r   r€   r�   ZlxÚsrj   rI   rJ   )Ú	gt_kwargsÚkwargsr   r!   r†   i  s     zusing.<locals>.callbackc                 3   sf   ˆ  | j¡ ˆ f i ˆ¤Ž}| ¡ }|j| ¡ fi ˆ¤ŽD ]\}}}|| ||fV  q8|rb| ¡ |_d S rF   )r‹   r6   rt   rG   ry   rw   r~   rŒ   ©Ú_otherrŽ   r�   r   r!   r†   x  s     )N)N)Úpopr?   Úlistrn   r   )r‘   r�   r�   r†   r   r�   r!   r   S  s    


r   c                   @   s   e Zd ZdZdd„ ZdS )r   zÑ
    Indicates a state or state action (e.g. #pop) to apply.
    For example default('#pop') is equivalent to ('', Token, '#pop')
    Note that state tuples may be used as well.

    .. versionadded:: 2.0
    c                 C   s
   || _ d S rF   )rˆ   )r9   rˆ   r   r   r!   r;   �  s    zdefault.__init__N)r+   r,   r-   r.   r;   r   r   r   r!   r   …  s   r   c                   @   s"   e Zd ZdZddd„Zdd„ ZdS )	r   z•
    Indicates a list of literal words that is transformed into an optimized
    regex that matches any of the words.

    .. versionadded:: 2.0
    rg   c                 C   s   || _ || _|| _d S rF   )r   ÚprefixÚsuffix)r9   r   r”   r•   r   r   r!   r;   ˜  s    zwords.__init__c                 C   s   t | j| j| jd�S )N©r”   r•   )r   r   r”   r•   r=   r   r   r!   r7   �  s    z	words.getN)rg   rg   )r+   r,   r-   r.   r;   r7   r   r   r   r!   r   ‘  s   
r   c                   @   sJ   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zddd„Zdd„ Z	dd„ Z
d
S )ÚRegexLexerMetazw
    Metaclass for RegexLexer, creates the self._tokens attribute from
    self.tokens on the first instantiation.
    c                 C   s    t |tƒr| ¡ }t ||¡jS )zBPreprocess the regular expression component of a token definition.)r?   r   r7   ÚreÚcompiler€   )ro   ÚregexÚrflagsrˆ   r   r   r!   Ú_process_regex§  s    
zRegexLexerMeta._process_regexc                 C   s&   t |ƒtu s"t|ƒs"J d|f ƒ‚|S )z5Preprocess the token component of a token definition.z2token type must be simple type or callable, not %r)r&   r   Úcallable)ro   Útokenr   r   r!   Ú_process_token­  s    ÿzRegexLexerMeta._process_tokenc                 C   s  t |tƒrd|dkrdS ||v r$|fS |dkr0|S |dd… dkrRt|dd… ƒ S dsbJ d| ƒ‚n®t |tƒrÊd	| j }|  jd
7  _g }|D ],}||ks¦J d| ƒ‚| |  |||¡¡ qŽ|||< |fS t |tƒ�r |D ] }||v sÚ|dv sÚJ d| ƒ‚qÚ|S d�sJ d| ƒ‚dS )z=Preprocess the state transition action of a token definition.ú#popéÿÿÿÿú#pushNé   z#pop:Fzunknown new state %rz_tmp_%dr|   zcircular state ref %r)r    r¢   zunknown new state zunknown new state def %r)r?   rM   Úintrm   Ú_tmpnameÚextendÚ_process_statern   )ro   Ú	new_stateÚunprocessedÚ	processedZ	tmp_stateÚitokensZistater   r   r!   Ú_process_new_state³  s<    



ÿÿþz!RegexLexerMeta._process_new_statec                 C   s’  t |ƒtu sJ d| ƒ‚|d dks0J d| ƒ‚||v r@|| S g  }||< | j}|| D �]0}t|tƒr˜||ks~J d| ƒ‚| |  ||t|ƒ¡¡ qZt|tƒr¤qZt|tƒrØ|  	|j
||¡}| t d¡jd|f¡ qZt |ƒtu sðJ d| ƒ‚z|  |d ||¡}W n@ t�yF }	 z&td	|d || |	f ƒ|	‚W Y d}	~	n
d}	~	0 0 |  |d
 ¡}
t|ƒdk�rjd}n|  	|d ||¡}| ||
|f¡ qZ|S )z%Preprocess a single state definition.zwrong state name %rr   ú#zinvalid state name %rzcircular state reference %rrg   Nzwrong rule def %rz+uncompilable regex %r in state %r of %r: %sr|   é   )r&   rM   Úflagsr?   r   r¦   r§   rl   r   r¬   rˆ   r@   r˜   r™   r€   rn   rœ   Ú	ExceptionÚ
ValueErrorrŸ   rQ   )ro   r©   rª   rˆ   Útokensr›   Ztdefr¨   ÚrexÚerrrž   r   r   r!   r§   Ö  sH    
ÿ

ÿÿ
ÿzRegexLexerMeta._process_stateNc                 C   s<   i  }| j |< |p| j| }t|ƒD ]}|  |||¡ q$|S )z-Preprocess a dictionary of token definitions.)Ú_all_tokensr²   r“   r§   )ro   r(   Ú	tokendefsrª   rˆ   r   r   r!   Úprocess_tokendef  s
    zRegexLexerMeta.process_tokendefc           
   
   C   sÖ   i }i }| j D ]Â}|j di ¡}| ¡ D ]¦\}}| |¡}|du rz|||< z| t¡}W n tyn   Y q(Y n0 |||< q(| |d¡}|du r�q(||||d …< z| t¡}	W n tyÀ   Y q(0 ||	 ||< q(q|S )a  
        Merge tokens from superclasses in MRO order, returning a single tokendef
        dictionary.

        Any state that is not defined by a subclass will be inherited
        automatically.  States that *are* defined by subclasses will, by
        default, override that state in the superclass.  If a subclass wishes to
        inherit definitions from a superclass, it can use the special value
        "inherit", which will cause the superclass' state definition to be
        included at that point in the state.
        r²   Nr|   )Ú__mro__Ú__dict__r7   ÚitemsÚindexr   r±   r’   )
ro   r²   ÚinheritableÚcÚtoksrˆ   rº   ZcuritemsZinherit_ndxZnew_inh_ndxr   r   r!   Úget_tokendefs
  s0    


zRegexLexerMeta.get_tokendefsc                 O   sR   d| j vr:i | _d| _t| dƒr(| jr(n|  d|  ¡ ¡| _tj	| g|¢R i |¤ŽS )z:Instantiate cls after preprocessing its token definitions.Ú_tokensr   Útoken_variantsrg   )
r¹   rµ   r¥   ÚhasattrrÁ   r·   r¿   rÀ   r&   Ú__call__)ro   rp   Úkwdsr   r   r!   rÃ   ;  s    
zRegexLexerMeta.__call__)N)r+   r,   r-   r.   rœ   rŸ   r¬   r§   r·   r¿   rÃ   r   r   r   r!   r—   ¡  s   #,
1r—   c                   @   s$   e Zd ZdZejZi Zddd„ZdS )r   z±
    Base for simple stateful regular expression-based lexers.
    Simplifies the lexing process so that you need only
    provide a list of states and regular expressions.
    ©rŠ   c                 c   sÀ  d}| j }t|ƒ}||d  }|D �](\}}}	|||ƒ}
|
r"|durrt|ƒtu rb|||
 ¡ fV  n|| |
ƒE dH  |
 ¡ }|	du�rHt|	tƒrÚ|	D ]D}|dkr´t|ƒdkrÖ| 	¡  q’|dkrÌ| 
|d ¡ q’| 
|¡ q’nbt|	tƒ�rt|	ƒt|ƒk�r|dd…= n
||	d…= n,|	dk�r*| 
|d ¡ nd�s<J d|	 ƒ‚||d  } qq"zP|| d	k�r„d
g}|d
 }|td	fV  |d7 }W q|t|| fV  |d7 }W q t�y¸   Y �q¼Y q0 qdS )z~
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the initial stack (default: ``['root']``)
        r   r¡   Nr    r|   r¢   Fúwrong state def: %rrD   rŠ   )rÀ   r“   r&   r   ry   rw   r?   rn   rQ   r’   r@   r¤   Úabsr   r   rx   )r9   rA   r‰   r~   r¶   Z
statestackÚstatetokensÚrexmatchr‚   r¨   Úmrˆ   r   r   r!   rG   l  sR    




z!RegexLexer.get_tokens_unprocessedN)rÅ   )	r+   r,   r-   r.   r˜   Ú	MULTILINEr¯   r²   rG   r   r   r   r!   r   I  s   r   c                   @   s"   e Zd ZdZddd„Zdd„ ZdS )r   z9
    A helper object that holds lexer position data.
    Nc                 C   s*   || _ || _|pt|ƒ| _|p"dg| _d S )NrŠ   )rA   r~   rQ   rw   r‰   )r9   rA   r~   r‰   rw   r   r   r!   r;   ¯  s    zLexerContext.__init__c                 C   s   d| j | j| jf S )NzLexerContext(%r, %r, %r))rA   r~   r‰   r=   r   r   r!   r>   µ  s    ÿzLexerContext.__repr__)NN)r+   r,   r-   r.   r;   r>   r   r   r   r!   r   ª  s   
r   c                   @   s   e Zd ZdZddd„ZdS )r   zE
    A RegexLexer that uses a context object to store its state.
    Nc                 c   s:  | j }|st|dƒ}|d }n|}||jd  }|j}|D �]`\}}}|||j|jƒ}	|	r:|dur°t|ƒtu rŒ|j||	 ¡ fV  |	 ¡ |_n$|| |	|ƒE dH  |s°||jd  }|du�r˜t	|t
ƒ�r|D ]P}
|
dkrðt|jƒdkrî|j ¡  qÊ|
dk�r|j |jd ¡ qÊ|j |
¡ qÊnlt	|tƒ�rZt|ƒt|jƒk�rL|jdd…= n|j|d…= n0|dk�rx|j |jd ¡ nd�sŠJ d	| ƒ‚||jd  } q6q:zz|j|jk�r´W �q6||j d
k�rôdg|_|d }|jtd
fV  | jd7  _W q6|jt||j fV  | jd7  _W q6 t�y2   Y �q6Y q60 q6dS )z
        Split ``text`` into (tokentype, text) pairs.
        If ``context`` is given, use this lexer context instead.
        r   rŠ   r¡   Nr    r|   r¢   FrÆ   rD   )rÀ   r   r‰   rA   r~   rw   r&   r   ry   r?   rn   rQ   r’   r@   r¤   rÇ   r   r   rx   )r9   rA   Úcontextr¶   r�   rÈ   rÉ   r‚   r¨   rÊ   rˆ   r   r   r!   rG   ¿  s`    




z)ExtendedRegexLexer.get_tokens_unprocessed)NN)r+   r,   r-   r.   rG   r   r   r   r!   r   º  s   r   c              	   c   s¨  t | ƒ} zt| ƒ\}}W n ty6   |E dH  Y dS 0 d}d}|D ]ð\}}}|du rZ|}d}	|�r|t|ƒ |k�r||	|| … }
|
r¢|||
fV  |t|
ƒ7 }|D ]"\}}}|||fV  |t|ƒ7 }q¦|| }	zt| ƒ\}}W q^ t�y    d}Y �qY q^0 q^|	t|ƒk rD||||	d… fV  |t|ƒ|	 7 }qD|�r¤|�pDd}|D ]$\}}}|||fV  |t|ƒ7 }�qJzt| ƒ\}}W n t�yž   d}Y �q¤Y n0 �q6dS )ag  
    Helper for lexers which must combine the results of several
    sublexers.

    ``insertions`` is a list of ``(index, itokens)`` pairs.
    Each ``itokens`` iterable should be inserted at position
    ``index`` into the token stream given by the ``tokens``
    argument.

    The result is a combined token stream.

    TODO: clean up the code here.
    NTr   F)ÚiterÚnextÚStopIterationrQ   )ri   r²   r»   r«   ÚrealposZinsleftrj   rI   rJ   ZoldiZtmpvalZit_indexZit_tokenZit_valueÚpr   r   r!   rh     sN    

rh   c                   @   s   e Zd ZdZdd„ ZdS )ÚProfilingRegexLexerMetaz>Metaclass for ProfilingRegexLexer, collects regex timing info.c                    sL   t |tƒr t|j|j|jd�‰n|‰t ˆ|¡‰tjf‡ ‡‡‡fdd„	}|S )Nr–   c                    s`   ˆ j d  ˆˆfddg¡}t ¡ }ˆ | ||¡}t ¡ }|d  d7  < |d  || 7  < |S )Nr¡   r   r   r|   )Ú
_prof_dataÚ
setdefaultÚtimer€   )rA   r~   ÚendposÚinfoÚt0ÚresÚt1©ro   Zcompiledr³   rˆ   r   r!   Ú
match_funcM  s    z:ProfilingRegexLexerMeta._process_regex.<locals>.match_func)	r?   r   r   r”   r•   r˜   r™   ÚsysÚmaxsize)ro   rš   r›   rˆ   rÜ   r   rÛ   r!   rœ   E  s    

ÿz&ProfilingRegexLexerMeta._process_regexN)r+   r,   r-   r.   rœ   r   r   r   r!   rÒ   B  s   rÒ   c                   @   s"   e Zd ZdZg ZdZddd„ZdS )ÚProfilingRegexLexerzFDrop-in replacement for RegexLexer that does profiling of its regexes.é   rÅ   c                 #   sÂ   ˆ j j i ¡ t ˆ ||¡E d H  ˆ j j ¡ }tdd„ | ¡ D ƒ‡ fdd„dd�}tdd„ |D ƒƒ}t	ƒ  t	dˆ j j
t|ƒ|f ƒ t	d	ƒ t	d
d ƒ t	dƒ |D ]}t	d| ƒ q¤t	d	ƒ d S )Nc                 s   sN   | ]F\\}}\}}|t |ƒ d ¡ dd¡dd… |d| d| | fV  qdS )zu'z\\ú\NéA   iè  )ÚreprrT   rC   )Ú.0r�   ÚrÚnrI   r   r   r!   Ú	<genexpr>c  s   þÿz=ProfilingRegexLexer.get_tokens_unprocessed.<locals>.<genexpr>c                    s
   | ˆ j  S rF   )Ú_prof_sort_indexr   r=   r   r!   r"   f  r#   z<ProfilingRegexLexer.get_tokens_unprocessed.<locals>.<lambda>T)ÚkeyÚreversec                 s   s   | ]}|d  V  qdS )é   Nr   )rä   r    r   r   r!   rç   h  r#   z2Profiling result for %s lexing %d chars in %.3f mszn==============================================================================================================z$%-20s %-64s ncalls  tottime  percall)rˆ   rš   zn--------------------------------------------------------------------------------------------------------------z%-20s %-65s %5d %8.4f %8.4f)r<   rÓ   r@   r   rG   r’   Úsortedrº   ÚsumÚprintr+   rQ   )r9   rA   r‰   Úrawdatarƒ   Z	sum_totalr*   r   r=   r!   rG   ^  s(    þ
üÿz*ProfilingRegexLexer.get_tokens_unprocessedN)rÅ   )r+   r,   r-   r.   rÓ   rè   rG   r   r   r   r!   rß   X  s   rß   )3r.   r˜   rÝ   rÕ   Zpygments.filterr   r   Zpygments.filtersr   Zpygments.tokenr   r   r   r   Zpygments.utilr	   r
   r   r   r   r   Zpygments.regexoptr   Ú__all__rO   ÚstaticmethodZ_default_analyser&   r$   r   r   rM   r   rl   r   rn   rm   rq   r   r‡   r   r   r   r   r—   r   r   r   rh   rÒ   rß   r   r   r   r!   Ú<module>   sD   
  !'2 )aH@