3
®ybd*  ã            	   @   sô   d Z ddlmZmZmZmZmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZmZ ddlmZmZmZ ed	d
œdd„ZG dd	„ d	ƒZG dd„ deƒZG dd„ deƒZdee
e eee ee eee  dœdd„Zdedœdd„ZdS )z parquet compat é    )ÚAnyÚAnyStrÚDictÚListÚOptional)Úcatch_warnings)ÚFilePathOrBuffer)Úimport_optional_dependency)ÚAbstractMethodError)Ú	DataFrameÚ
get_option)Ú_expand_userÚget_filepath_or_bufferÚis_fsspec_urlÚBaseImpl)ÚengineÚreturnc             C   s¢   | dkrt dƒ} | dkrzttg}d}xF|D ]>}y|ƒ S  tk
rf } z|dt|ƒ 7 }W Y dd}~X q*X q*W td|› �ƒ‚| dkrˆtƒ S | dkr–tƒ S td	ƒ‚dS )
z return our implementation Úautozio.parquet.engineÚ z
 - NzÉUnable to find a usable engine; tried using: 'pyarrow', 'fastparquet'.
A suitable version of pyarrow or fastparquet is required for parquet support.
Trying to import the above resulted in these errors:ÚpyarrowÚfastparquetz.engine must be one of 'pyarrow', 'fastparquet')r   ÚPyArrowImplÚFastParquetImplÚImportErrorÚstrÚ
ValueError)r   Zengine_classesZ
error_msgsZengine_classÚerr© r   ú5/tmp/pip-install-4cy230dp/pandas/pandas/io/parquet.pyÚ
get_engine   s"    
&	r   c               @   s6   e Zd Zeedœdd„ƒZedœdd„Zd	dd„ZdS )
r   )Údfc             C   sL   t | tƒstdƒ‚| jjdkr&tdƒ‚tdd„ | jjD ƒƒ}|sHtdƒ‚d S )	Nz+to_parquet only supports IO with DataFramesÚstringÚemptyz%parquet must have string column namesc             s   s    | ]}|d k	rt |tƒV  qd S )N)Ú
isinstancer   )Ú.0Únamer   r   r   ú	<genexpr>>   s    z.BaseImpl.validate_dataframe.<locals>.<genexpr>z!Index level names must be strings>   r"   r!   )r#   r   r   ÚcolumnsZinferred_typeÚallÚindexÚnames)r    Zvalid_namesr   r   r   Úvalidate_dataframe2   s    
zBaseImpl.validate_dataframec             K   s   t | ƒ‚d S )N)r
   )Úselfr    ÚpathÚcompressionÚkwargsr   r   r   ÚwriteC   s    zBaseImpl.writeNc             K   s   t | ƒ‚d S )N)r
   )r,   r-   r'   r/   r   r   r   ÚreadF   s    zBaseImpl.read)N)Ú__name__Ú
__module__Ú__qualname__Ústaticmethodr   r+   r0   r1   r   r   r   r   r   1   s   c               @   sJ   e Zd Zdd„ Zd
eee ee ee	 ee
e  dœdd„Zddd	„ZdS )r   c             C   s&   t ddd� dd l}dd l}|| _d S )Nr   z(pyarrow is required for parquet support.)Úextrar   )r	   Zpyarrow.parquetZpandas.core.arrays._arrow_utilsÚapi)r,   r   Úpandasr   r   r   Ú__init__K   s
    
zPyArrowImpl.__init__ÚsnappyN)r    r-   r.   r)   Úpartition_colsc             K   sÆ   | j |ƒ d|jdd ƒi}|d k	r*||d< | jjj|f|Ž}t|ƒrvd|krvtdƒ dd l}	|	jj	|ƒ\}
}|
|d< nt
|ƒ}|d k	r¦| jjj||f||dœ|—Ž n| jjj||fd|i|—Ž d S )NZschemaZpreserve_indexÚ
filesystemÚfsspecr   )r.   r;   r.   )r+   Úpopr7   ZTableZfrom_pandasr   r	   Úfsspec.coreÚcoreÚ	url_to_fsr   ÚparquetZwrite_to_datasetZwrite_table)r,   r    r-   r.   r)   r;   r/   Zfrom_pandas_kwargsÚtabler=   Úfsr   r   r   r0   V   s&    	


zPyArrowImpl.writec       	      K   sš   t |ƒr6d|kr6tdƒ dd l}|jj|ƒ\}}d}n|jdd ƒ}d}t|ƒ}|sbt|ƒ\}}}}d|d< | jj	j
|f||dœ|—Žjƒ }|r–|jƒ  |S )Nr<   r=   r   FTZuse_pandas_metadata)r'   r<   )r   r	   r?   r@   rA   r>   r   r   r7   rB   Z
read_tableÚ	to_pandasÚclose)	r,   r-   r'   r/   r=   rD   Zshould_closeÚ_Úresultr   r   r   r1   }   s     zPyArrowImpl.read)r:   NN)N)r2   r3   r4   r9   r   r   r   r   r   Úboolr   r0   r1   r   r   r   r   r   J   s
     ,!r   c               @   s.   e Zd Zdd„ Zd
edœdd„Zddd	„ZdS )r   c             C   s   t ddd�}|| _d S )Nr   z,fastparquet is required for parquet support.)r6   )r	   r7   )r,   r   r   r   r   r9   —   s    
zFastParquetImpl.__init__r:   N)r    c                s°   | j |ƒ d|kr$|d k	r$tdƒ‚nd|kr6|jdƒ}|d k	rFd|d< t|ƒrhtdƒ‰ ‡ fdd„|d< nt|ƒ\}}}}td	d
��$ | jj||f|||dœ|—Ž W d Q R X d S )NÚpartition_onzYCannot use both partition_on and partition_cols. Use partition_cols for partitioning dataZhiveZfile_schemer=   c                s   ˆ j | dƒj ƒ S )NÚwb)Úopen)r-   rG   )r=   r   r   Ú<lambda>¼   s    z'FastParquetImpl.write.<locals>.<lambda>Ú	open_withT)Úrecord)r.   Zwrite_indexrJ   )	r+   r   r>   r   r	   r   r   r7   r0   )r,   r    r-   r.   r)   r;   r/   rG   r   )r=   r   r0   Ÿ   s(    	

zFastParquetImpl.writec                s^   t |ƒr.tdƒ‰ ‡ fdd„}| jj||d�}nt|ƒ\}}}}| jj|ƒ}|jf d|i|—ŽS )Nr=   c                s   ˆ j | dƒj ƒ S )NÚrb)rL   )r-   rG   )r=   r   r   rM   Î   s    z&FastParquetImpl.read.<locals>.<lambda>)rN   r'   )r   r	   r7   ZParquetFiler   rE   )r,   r-   r'   r/   rN   Zparquet_filerG   r   )r=   r   r1   Ê   s    zFastParquetImpl.read)r:   NN)N)r2   r3   r4   r9   r   r0   r1   r   r   r   r   r   –   s
     %r   r   r:   N)r    r-   r   r.   r)   r;   c             K   s4   t |tƒr|g}t|ƒ}|j| |f|||dœ|—ŽS )aû  
    Write a DataFrame to the parquet format.

    Parameters
    ----------
    df : DataFrame
    path : str or file-like object
        If a string, it will be used as Root Directory path
        when writing a partitioned dataset. By file-like object,
        we refer to objects with a write() method, such as a file handler
        (e.g. via builtin open function) or io.BytesIO. The engine
        fastparquet does not accept file-like objects.

        .. versionchanged:: 0.24.0

    engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
        Parquet library to use. If 'auto', then the option
        ``io.parquet.engine`` is used. The default ``io.parquet.engine``
        behavior is to try 'pyarrow', falling back to 'fastparquet' if
        'pyarrow' is unavailable.
    compression : {'snappy', 'gzip', 'brotli', None}, default 'snappy'
        Name of the compression to use. Use ``None`` for no compression.
    index : bool, default None
        If ``True``, include the dataframe's index(es) in the file output. If
        ``False``, they will not be written to the file.
        If ``None``, similar to ``True`` the dataframe's index(es)
        will be saved. However, instead of being saved as values,
        the RangeIndex will be stored as a range in the metadata so it
        doesn't require much space and is faster. Other indexes will
        be included as columns in the file output.

        .. versionadded:: 0.24.0

    partition_cols : str or list, optional, default None
        Column names by which to partition the dataset.
        Columns are partitioned in the order they are given.
        Must be None if path is not a string.

        .. versionadded:: 0.24.0

    kwargs
        Additional keyword arguments passed to the engine
    )r.   r)   r;   )r#   r   r   r0   )r    r-   r   r.   r)   r;   r/   Úimplr   r   r   Ú
to_parquet×   s    4
rR   )r   c             K   s   t |ƒ}|j| fd|i|—ŽS )aÃ  
    Load a parquet object from the file path, returning a DataFrame.

    Parameters
    ----------
    path : str, path object or file-like object
        Any valid string path is acceptable. The string could be a URL. Valid
        URL schemes include http, ftp, s3, and file. For file URLs, a host is
        expected. A local file could be:
        ``file://localhost/path/to/table.parquet``.
        A file URL can also be a path to a directory that contains multiple
        partitioned parquet files. Both pyarrow and fastparquet support
        paths to directories as well as file URLs. A directory path could be:
        ``file://localhost/path/to/tables`` or ``s3://bucket/partition_dir``

        If you want to pass in a path object, pandas accepts any
        ``os.PathLike``.

        By file-like object, we refer to objects with a ``read()`` method,
        such as a file handler (e.g. via builtin ``open`` function)
        or ``StringIO``.
    engine : {'auto', 'pyarrow', 'fastparquet'}, default 'auto'
        Parquet library to use. If 'auto', then the option
        ``io.parquet.engine`` is used. The default ``io.parquet.engine``
        behavior is to try 'pyarrow', falling back to 'fastparquet' if
        'pyarrow' is unavailable.
    columns : list, default=None
        If not None, only these columns will be read from the file.
    **kwargs
        Any additional kwargs are passed to the engine.

    Returns
    -------
    DataFrame
    r'   )r   r1   )r-   r   r'   r/   rQ   r   r   r   Úread_parquet  s    $rS   )r   r:   NN)r   N)Ú__doc__Útypingr   r   r   r   r   Úwarningsr   Zpandas._typingr   Zpandas.compat._optionalr	   Zpandas.errorsr
   r8   r   r   Zpandas.io.commonr   r   r   r   r   r   r   r   rI   rR   rS   r   r   r   r   Ú<module>   s    "LD   .;