3
Êb_U'  ã               @   sÊ   d Z ddlmZmZmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZmZ ddlmZmZmZ edd	œd
d„ZG dd„ dƒZG dd„ deƒZG dd„ deƒZdeeee dœdd„Zdedœdd„ZdS )z parquet compat é    )ÚAnyÚDictÚOptional)Úcatch_warnings)Úimport_optional_dependency)ÚAbstractMethodError)Ú	DataFrameÚ
get_option)Úget_filepath_or_bufferÚ
is_gcs_urlÚ	is_s3_urlÚBaseImpl)ÚengineÚreturnc             C   s€   | dkrt dƒ} | dkrXytƒ S  tk
r2   Y nX ytƒ S  tk
rN   Y nX tdƒ‚| dkrftƒ S | dkrttƒ S tdƒ‚dS )z return our implementation Úautozio.parquet.enginez}Unable to find a usable engine; tried using: 'pyarrow', 'fastparquet'.
pyarrow or fastparquet is required for parquet supportÚpyarrowÚfastparquetz.engine must be one of 'pyarrow', 'fastparquet'N)r	   ÚPyArrowImplÚImportErrorÚFastParquetImplÚ
ValueError)r   © r   ú3/tmp/pip-build-jkwsxyiy/pandas/pandas/io/parquet.pyÚ
get_engine   s$    r   c               @   s6   e Zd Zeedœdd„ƒZedœdd„Zd	dd„ZdS )
r   )Údfc             C   sL   t | tƒstdƒ‚| jjd	kr&tdƒ‚tdd„ | jjD ƒƒ}|sHtdƒ‚d S )
Nz+to_parquet only supports IO with DataFramesÚstringÚunicodeÚemptyz%parquet must have string column namesc             s   s    | ]}|d k	rt |tƒV  qd S )N)Ú
isinstanceÚstr)Ú.0Únamer   r   r   ú	<genexpr><   s    z.BaseImpl.validate_dataframe.<locals>.<genexpr>z!Index level names must be strings>   r   r   r   )r   r   r   ÚcolumnsZinferred_typeÚallÚindexÚnames)r   Zvalid_namesr   r   r   Úvalidate_dataframe0   s    
zBaseImpl.validate_dataframec             K   s   t | ƒ‚d S )N)r   )Úselfr   ÚpathÚcompressionÚkwargsr   r   r   ÚwriteA   s    zBaseImpl.writeNc             K   s   t | ƒ‚d S )N)r   )r(   r)   r#   r+   r   r   r   ÚreadD   s    zBaseImpl.read)N)Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr   r'   r,   r-   r   r   r   r   r   /   s   c               @   s4   e Zd Zdd„ Zdeee dœdd„Zdd	d
„ZdS )r   c             C   s&   t ddd� dd l}dd l}|| _d S )Nr   z(pyarrow is required for parquet support.)Úextrar   )r   Zpyarrow.parquetZpandas.core.arrays._arrow_utilsÚapi)r(   r   Úpandasr   r   r   Ú__init__I   s
    
zPyArrowImpl.__init__ÚsnappyÚmsN)r   r%   c             K   s¨   | j |ƒ t|dd�\}}}}	d|jdd ƒi}
|d k	r>||
d< | jjj|f|
Ž}|d k	rz| jjj||f|||dœ|—Ž n| jjj||f||dœ|—Ž |	r¤|j	ƒ  d S )NÚwb)ÚmodeZschemaZpreserve_index)r*   Úcoerce_timestampsÚpartition_cols)r*   r:   )
r'   r
   Úpopr3   ZTableZfrom_pandasÚparquetZwrite_to_datasetZwrite_tableÚclose)r(   r   r)   r*   r:   r%   r;   r+   Ú_Úshould_closeZfrom_pandas_kwargsÚtabler   r   r   r,   T   s,    


zPyArrowImpl.writec             K   sF   t |ƒ\}}}}d|d< | jjj|fd|i|—Žjƒ }|rB|jƒ  |S )NTZuse_pandas_metadatar#   )r
   r3   r=   Z
read_tableÚ	to_pandasr>   )r(   r)   r#   r+   r?   r@   Úresultr   r   r   r-   z   s    zPyArrowImpl.read)r6   r7   NN)N)	r.   r/   r0   r5   r   r   Úboolr,   r-   r   r   r   r   r   H   s      r   c               @   s.   e Zd Zdd„ Zd
edœdd„Zddd	„ZdS )r   c             C   s   t ddd�}|| _d S )Nr   z,fastparquet is required for parquet support.)r2   )r   r3   )r(   r   r   r   r   r5   ˆ   s    
zFastParquetImpl.__init__r6   N)r   c             K   sÀ   | j |ƒ d|kr$|d k	r$tdƒ‚nd|kr6|jdƒ}|d k	rFd|d< t|ƒsVt|ƒrxt|dd�\}}}}dd„ |d	< nt|ƒ\}}}}td
d��$ | jj||f|||dœ|—Ž W d Q R X d S )NÚpartition_onzYCannot use both partition_on and partition_cols. Use partition_cols for partitioning dataZhiveZfile_schemer8   )r9   c             S   s   | S )Nr   )r)   r?   r   r   r   Ú<lambda>°   s    z'FastParquetImpl.write.<locals>.<lambda>Ú	open_withT)Úrecord)r*   Zwrite_indexrE   )	r'   r   r<   r   r   r
   r   r3   r,   )r(   r   r)   r*   r%   r;   r+   r?   r   r   r   r,   �   s(    	

zFastParquetImpl.writec       	      K   st   t |ƒrDddlm} ||ƒ\}}z| jj||jd�}W d |jƒ  X nt|ƒ\}}}}| jj|ƒ}|jf d|i|—ŽS )Nr   )Úget_file_and_filesystem)rG   r#   )	r   Zpandas.io.s3rI   r3   ZParquetFileÚopenr>   r
   rB   )	r(   r)   r#   r+   rI   Zs3Ú
filesystemZparquet_filer?   r   r   r   r-   ¾   s    zFastParquetImpl.read)r6   NN)N)r.   r/   r0   r5   r   r,   r-   r   r   r   r   r   ‡   s
     (r   r   r6   N)r   r   r%   c             K   s4   t |tƒr|g}t|ƒ}|j| |f|||dœ|—ŽS )añ  
    Write a DataFrame to the parquet format.

    Parameters
    ----------
    df : DataFrame
    path : str
        File path or Root Directory path. Will be used as Root Directory path
        while writing a partitioned dataset.

        .. versionchanged:: 0.24.0

    engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
        Parquet library to use. If 'auto', then the option
        ``io.parquet.engine`` is used. The default ``io.parquet.engine``
        behavior is to try 'pyarrow', falling back to 'fastparquet' if
        'pyarrow' is unavailable.
    compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy'
        Name of the compression to use. Use ``None`` for no compression.
    index : bool, default None
        If ``True``, include the dataframe's index(es) in the file output. If
        ``False``, they will not be written to the file.
        If ``None``, similar to ``True`` the dataframe's index(es)
        will be saved. However, instead of being saved as values,
        the RangeIndex will be stored as a range in the metadata so it
        doesn't require much space and is faster. Other indexes will
        be included as columns in the file output.

        .. versionadded:: 0.24.0

    partition_cols : str or list, optional, default None
        Column names by which to partition the dataset
        Columns are partitioned in the order they are given

        .. versionadded:: 0.24.0

    kwargs
        Additional keyword arguments passed to the engine
    )r*   r%   r;   )r   r   r   r,   )r   r)   r   r*   r%   r;   r+   Úimplr   r   r   Ú
to_parquetÑ   s    0
rM   )r   c             K   s   t |ƒ}|j| fd|i|—ŽS )aâ  
    Load a parquet object from the file path, returning a DataFrame.

    .. versionadded:: 0.21.0

    Parameters
    ----------
    path : str, path object or file-like object
        Any valid string path is acceptable. The string could be a URL. Valid
        URL schemes include http, ftp, s3, and file. For file URLs, a host is
        expected. A local file could be:
        ``file://localhost/path/to/table.parquet``.
        A file URL can also be a path to a directory that contains multiple
        partitioned parquet files. Both pyarrow and fastparquet support
        paths to directories as well as file URLs. A directory path could be:
        ``file://localhost/path/to/tables``

        If you want to pass in a path object, pandas accepts any
        ``os.PathLike``.

        By file-like object, we refer to objects with a ``read()`` method,
        such as a file handler (e.g. via builtin ``open`` function)
        or ``StringIO``.
    engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
        Parquet library to use. If 'auto', then the option
        ``io.parquet.engine`` is used. The default ``io.parquet.engine``
        behavior is to try 'pyarrow', falling back to 'fastparquet' if
        'pyarrow' is unavailable.
    columns : list, default=None
        If not None, only these columns will be read from the file.

        .. versionadded:: 0.21.1
    **kwargs
        Any additional kwargs are passed to the engine.

    Returns
    -------
    DataFrame
    r#   )r   r-   )r)   r   r#   r+   rL   r   r   r   Úread_parquet  s    )rN   )r   r6   NN)r   N)Ú__doc__Útypingr   r   r   Úwarningsr   Zpandas.compat._optionalr   Zpandas.errorsr   r4   r   r	   Zpandas.io.commonr
   r   r   r   r   r   r   r   rD   rM   rN   r   r   r   r   Ú<module>   s   !?M   7