3
W­E^Ó	  ã               @   sV   d Z ddlmZ ddlmZ ddlmZmZ ddlm	Z	 G dd„ de
ƒZdd	d
„ZdS )z	SQL Lexeré    )Útokens)Ú	SQL_REGEX)Ú	text_typeÚ
file_types)Úconsumec               @   s   e Zd ZdZeddd„ƒZdS )ÚLexerz?Lexer
    Empty class. Leaving for backwards-compatibility
    Nc             c   s  t | tƒr| jƒ } t | tƒrn\t | tƒrh|r8| j|ƒ} qzy| jdƒ} W qz tk
rd   | jdƒ} Y qzX ntdjt	| ƒƒƒ‚t
| ƒ}xŠ|D ]‚\}}xxtD ]d\}}|| |ƒ}|s°q–n2t |tjƒrÌ||jƒ fV  nt|ƒrâ||jƒ ƒV  t||jƒ | d ƒ P q–W tj|fV  qˆW dS )a³  
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.

        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the initial stack (default: ``['root']``)
        zutf-8zunicode-escapez+Expected text or file-like object, got {!r}é   N)Ú
isinstancer   Úreadr   ÚbytesÚdecodeÚUnicodeDecodeErrorÚ	TypeErrorÚformatÚtypeÚ	enumerater   r   Z
_TokenTypeÚgroupÚcallabler   ÚendÚError)ÚtextÚencodingÚiterableÚposÚcharZrexmatchÚactionÚm© r   ú2/tmp/pip-build-78s3knkw/sqlparse/sqlparse/lexer.pyÚ
get_tokens   s4    



zLexer.get_tokens)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodr   r   r   r   r   r      s   r   Nc             C   s   t ƒ j| |ƒS )z�Tokenize sql.

    Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream
    of ``(token type, value)`` items.
    )r   r   )Zsqlr   r   r   r   ÚtokenizeL   s    r%   )N)r#   Zsqlparser   Zsqlparse.keywordsr   Zsqlparse.compatr   r   Zsqlparse.utilsr   Úobjectr   r%   r   r   r   r   Ú<module>	   s   6