a
    yn„fb-  ã                   @  sº   d dl mZ d dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ dZeƒ dddddd	d
ddddœ
dd„ƒZeƒ ddd„ƒZddd„Zdd„ Zddd„ZG dd„ dƒZddd„ZdS )é    )ÚannotationsN)Úhigh_level_function)Úwrap_layout©Ú
is_integer)Úfrom_parqueti ú  i @Bi@B FT)
ÚcolumnsÚ
row_groupsÚstorage_optionsÚmax_gapÚ	max_blockÚfooter_sample_sizeÚgenerate_bitmasksÚ	highlevelÚbehaviorÚattrsc       
         C  sF   t | |||ƒ\}}}}}}}t||dur,|nd||||||||	||
ƒS )aÇ  
    Args:
        path (str): Local filename or remote URL, passed to fsspec for resolution.
            May contain glob patterns.
        columns (None, str, or list of str): Glob pattern(s) with bash-like curly
            brackets for matching column names. Nested records are separated by dots.
            If a list of patterns, the logical-or is matched. If None, all columns
            are read.
        row_groups (None or set of int): Row groups to read; must be non-negative.
            Order is ignored: the output array is presented in the order specified by
            Parquet metadata. If None, all row groups/all rows are read.
        storage_options: Passed to `fsspec.parquet.open_parquet_file`.
        max_gap (int): Passed to `fsspec.parquet.open_parquet_file`.
        max_block (int): Passed to `fsspec.parquet.open_parquet_file`.
        footer_sample_size (int): Passed to `fsspec.parquet.open_parquet_file`.
        generate_bitmasks (bool): If enabled and Arrow/Parquet does not have Awkward
            metadata, `generate_bitmasks=True` creates empty bitmasks for nullable
            types that don't have bitmasks in the Arrow/Parquet data, so that the
            Form (BitMaskedForm vs UnmaskedForm) is predictable.
        highlevel (bool): If True, return an #ak.Array; otherwise, return
            a low-level #ak.contents.Content subclass.
        behavior (None or dict): Custom #ak.behavior for the output array, if
            high-level.
        attrs (None or dict): Custom attributes for the output array, if
            high-level.

    Reads data from a local or remote Parquet file or collection of files.

    The data are eagerly (not lazily) read and must fit into memory. Use `columns`
    and/or `row_groups` to select and filter manageable subsets of the data, and
    use #ak.metadata_from_parquet to find column names and the range of row groups
    that a dataset has.

    See also #ak.to_parquet, #ak.metadata_from_parquet.
    N)ÚmetadataÚ_load)Úpathr   r	   r
   r   r   r   r   r   r   r   Úparquet_columnsÚsubformÚactual_pathsÚfsÚsubrgZ
row_countsÚmeta© r   úe/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/awkward/operations/ak_from_parquet.pyr      s(    3üôr   c           !   	     s"  t jj d¡}|d urLtdd„ |D ƒƒs0tdƒ‚tt|ƒƒt|ƒk rLtdƒ‚tj	ˆd|d�\}}}	t
ˆ||	||ƒ\}
}}d gt|
ƒ }|
}| |¡�}| |¡}W d   ƒ n1 s´0    Y  d}|jD ] }|jd	krÈd
|jv rÈd} qêqÈtjjj|jdd�}|d u�r| |¡}|jjdgk�r(d}nd}|j‰ |�rÆ| d¡�sÆ||
v �r\|
dd … }n|
}|D ]`}| |d¡�<}| |¡j}| | dd¡d ¡ ˆ  |¡ W d   ƒ n1 �s¸0    Y  �qd|d u�rÞt‡ fdd„|D ƒƒ�rütdˆ jd › �ƒ‚|�s
tdƒ‚i }i }g }g }tˆ jƒD ]2}ˆ  |¡ d	¡j }| !|g ¡ "|¡ |||< �q$g }|D ]z}|| ‰t#‡fdd„|
D ƒƒ}||v�r®| "|¡ | "|ˆ  $|¡g¡ n|d  "|ˆ  $|¡¡ | "ˆ  |¡j%¡ �q`n$|�rþ‡ fdd„tˆ jƒD ƒ}nd }|j&||d�} | |||||ˆ fS )Núak.from_parquetc                 s  s   | ]}t |ƒo|d kV  qdS )r   Nr   )Ú.0Úxr   r   r   Ú	<genexpr>f   ó    zmetadata.<locals>.<genexpr>z1row_groups must be a set of non-negative integersz!row group indices must not repeatÚrb)Úmoder
   z	list.itemr   z.list.elementzlist.elementT)Zpass_empty_fieldÚ )r$   r   z
/_metadataé   ú/éÿÿÿÿc                 3  s   | ]}|ˆ j kV  qd S ©N)Únum_row_groups©r   Ú_©r   r   r   r    ›   r!   z%Row group selection out of bounds 0..z<Requested selection of row-groups, but not scanning metadatac                 3  s   | ]}|  ˆ ¡r|V  qd S r(   )Úendswithr*   )r   r   r   r    °   r!   c                   s   g | ]}ˆ   |¡j‘qS r   )Ú	row_groupÚnum_rows)r   Úir,   r   r   Ú
<listcomp>¹   s   zmetadata.<locals>.<listcomp>)Úlist_indicatorÚcolumn_prefix)'ÚawkwardÚ_connectÚpyarrowÚimport_pyarrow_parquetÚallÚ
ValueErrorÚlenÚsetÚfsspecZget_fs_token_pathsÚ_all_and_metadata_pathsÚopenÚParquetFileZschemaZmax_repetition_levelr   ÚakZform_handle_arrowZschema_arrowZselect_columnsÚnamesr   r-   Zset_file_pathÚrsplitZappend_row_groupsÚanyr)   Ú	TypeErrorÚranger.   ÚcolumnÚ	file_pathÚ
setdefaultÚappendÚnextÚindexr/   r   )!r   r
   r	   r   Úignore_metadataÚ
scan_filesÚpyarrow_parquetr   r+   ÚpathsÚ	all_pathsZpath_for_schemaÚcan_subr   r   Zfile_for_metadataZparquetfile_for_metadatar2   Zcolumn_metadatar   r3   Z
scan_pathsZapathÚfÚmdZpath_rgsZrgs_pathZ
col_countsr0   ÚfpÚselectZpath2r   r   )r   r   r   r   Y   s     
ÿ
ÿ
ÿ(
ÿþÿ


.
ÿÿ


ÿÿr   c                 C  sˆ   g }t | ƒD ],\}}| t||
||| |||||d�	¡ qt|ƒdksJJ ‚t|ƒdkrjt|d |||	d�S tjjj|dd||	|d�S d S )N)r   r   r	   r   r   r   r   r   r   r%   )r   r   r   T)ZaxisZ	mergeboolr   r   r   )	Ú	enumeraterI   Ú_read_parquet_filer:   r   r@   Ú
operationsZak_concatenateÚ_impl)r   r   r   r   r   r   r   r   r   r   r   r   r   Zarraysr0   Úpr   r   r   r   Æ   s8    ÷ÿÿúr   c                 C  sB   |s|r2t |ddƒr2tjj| |d||||||d�	S | | d¡S dS )z/Picks between fsspec.parquet and normal fs.openZ
async_implFr6   )r   Zenginer   r	   r   r   r   r   r"   N)Úgetattrr<   ZparquetZopen_parquet_filer>   )r   r   r   r	   r   r   r   r   r   r   r   Ú
_open_fileö   s    ÷r\   c	              	   C  s’   t jj d¡}	t| |||||||ƒ�:}
|	 |
¡}|d u rD| |¡}n| ||¡}W d   ƒ n1 sd0    Y  tjj 	|¡}tj
j ||dd d ¡S )Nr   F)r4   r5   r6   r7   r\   r?   ÚreadZread_row_groupsr@   Z%convert_native_arrow_table_to_awkwardrX   Zak_from_arrowrY   )r   r   r   r	   r   r   r   r   r   rN   ÚfileZparquetfileZarrow_tabler   r   r   rW     s0    ø	
*úrW   c                   @  s   e Zd Zdd„ ZdS )Ú_DictOfEmptyBuffersc                 C  s   dS )Ns           r   )ÚselfÚwherer   r   r   Ú__getitem__5  s    z_DictOfEmptyBuffers.__getitem__N)Ú__name__Ú
__module__Ú__qualname__rb   r   r   r   r   r_   4  s   r_   c                 C  s’  g }|D ]Ø}|  |¡rd| dd¡d dk}|r4|r4q| dd¡d dk}|rR|rRq| |||f¡ q| |¡r|j|dd� ¡ D ]^\}	}
|	 d¡}|rœ|rœq€|	 d¡}|r°|r°q€|	 d¡sÂ|sÂ|r€|
d	 d
kr€| |	||f¡ q€qdd„ |D ƒ}t|ƒdk�r|d }d}nRdd„ |D ƒ}t|ƒdk�r2|d }nt|ƒdk�rL|d d }|�p\t|ƒdk}dd„ |D ƒ}t|ƒdk�rˆtd| ›�ƒ‚|||fS )Nr&   r%   r'   Ú	_metadataZ_common_metadataT)Údetail)z.parqz.parquetÚtyper^   c                 S  s   g | ]\}}}|r|‘qS r   r   ©r   r   Úis_metaÚis_commr   r   r   r1   P  r!   z+_all_and_metadata_paths.<locals>.<listcomp>r   c                 S  s   g | ]\}}}|r|‘qS r   r   ri   r   r   r   r1   U  r!   c                 S  s   g | ]\}}}|s|s|‘qS r   r   ri   r   r   r   r1   ^  r!   z(no *.parquet or *.parq matches for path )	ÚisfilerB   rI   ÚisdirÚfindÚitemsr-   r:   r9   )r   r   rO   rL   rM   rP   r   rj   rk   rR   ÚfdataZpath_for_metadatarQ   r   r   r   r=   9  sF    




r=   )NNNFT)N)N)FT)Ú
__future__r   Zfsspec.parquetr<   r4   r@   Zawkward._connect.pyarrowZawkward._dispatchr   Zawkward._layoutr   Zawkward._regularizer   Ú__all__r   r   r   r\   rW   r_   r=   r   r   r   r   Ú<module>   sB   ôH     úy ó
0 ÷
)