a
    yn„fPS  ã                   @  sÈ   d dl mZ d dlmZmZ d dlmZ d dlZd dlZ	d dl
Zd dl
mZ d dlmZ d dlmZ dZe ¡ Zeƒ d	d
d
dd	d
d
dddddddd
d	d	ddd	dddœdd„ƒZdd„ Zd
dœdd„ZdS )é    )Úannotations)ÚMappingÚSequence)ÚfsdecodeN)Ú%convert_awkward_arrow_table_to_native)Úhigh_level_function)ÚNumpyMetadata)Ú
to_parquetFTÚzstdi   z2.4z1.0)Ú	list_to32Ústring_to32Úbytestring_to32Úemptyarray_toÚcategorical_as_dictionaryÚextensionarrayÚcount_nullsÚcompressionÚcompression_levelÚrow_group_sizeÚdata_page_sizeÚparquet_flavorÚparquet_versionÚparquet_page_versionÚparquet_metadata_statisticsÚparquet_dictionary_encodingÚparquet_byte_stream_splitÚparquet_coerce_timestampsÚparquet_old_int96_timestampsÚparquet_compliant_nestedÚparquet_extra_optionsÚstorage_optionsc                c  sB   | fV  t | |||||||||	|
|||||||||||||dd�S )u$&  
    Args:
        array: Array-like data (anything #ak.to_layout recognizes).
        destination (path-like): Name of the output file, file path, or
            remote URL passed to [fsspec.core.url_to_fs](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.core.url_to_fs)
            for remote writing.
        list_to32 (bool): If True, convert Awkward lists into 32-bit Arrow lists
            if they're small enough, even if it means an extra conversion. Otherwise,
            signed 32-bit #ak.types.ListType maps to Arrow `ListType`,
            signed 64-bit #ak.types.ListType maps to Arrow `LargeListType`,
            and unsigned 32-bit #ak.types.ListType picks whichever Arrow type its
            values fit into.
        string_to32 (bool): Same as the above for Arrow `string` and `large_string`.
        bytestring_to32 (bool): Same as the above for Arrow `binary` and `large_binary`.
        emptyarray_to (None or dtype): If None, #ak.types.UnknownType maps to Arrow's
            null type; otherwise, it is converted a given numeric dtype.
        categorical_as_dictionary (bool): If True, #ak.contents.IndexedArray and
            #ak.contents.IndexedOptionArray labeled with `__array__ = "categorical"`
            are mapped to Arrow `DictionaryArray`; otherwise, the projection is
            evaluated before conversion (always the case without
            `__array__ = "categorical"`).
        extensionarray (bool): If True, this function returns extended Arrow arrays
            (at all levels of nesting), which preserve metadata so that Awkward â†’
            Arrow â†’ Awkward preserves the array's #ak.types.Type (though not
            the #ak.forms.Form). If False, this function returns generic Arrow arrays
            that might be needed for third-party tools that don't recognize Arrow's
            extensions. Even with `extensionarray=False`, the values produced by
            Arrow's `to_pylist` method are the same as the values produced by Awkward's
            #ak.to_list.
        count_nulls (bool): If True, count the number of missing values at each level
            and include these in the resulting Arrow array, which makes some downstream
            applications faster. If False, skip the up-front cost of counting them.
        compression (None, str, or dict): Compression algorithm name, passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            Parquet supports `{"NONE", "SNAPPY", "GZIP", "BROTLI", "LZ4", "ZSTD"}`
            (where `"GZIP"` is also known as "zlib" or "deflate"). If a dict, the keys
            are column names (the same column names that #ak.forms.Form.columns returns
            and #ak.forms.Form.select_columns accepts) and the values are compression
            algorithm names, to compress each column differently.
        compression_level (None, int, or dict None): Compression level, passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            Compression levels have different meanings for different compression
            algorithms: GZIP ranges from 1 to 9, but ZSTD ranges from -7 to 22, for
            example. Generally, higher numbers provide slower but smaller compression.
        row_group_size (int or None): Number of entries in each row group (except the last),
            passed to [pyarrow.parquet.ParquetWriter.write_table](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html#pyarrow.parquet.ParquetWriter.write_table).
            If None, the Parquet default of 64 MiB is used.
        data_page_size (None or int): Number of bytes in each data page, passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            If None, the Parquet default of 1 MiB is used.
        parquet_flavor (None or `"spark"`): If None, the output Parquet file will follow
            Arrow conventions; if `"spark"`, it will follow Spark conventions. Some
            systems, such as Spark and Google BigQuery, might need Spark conventions,
            while others might need Arrow conventions. Passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `flavor`.
        parquet_version (`"1.0"`, `"2.4"`, or `"2.6"`): Parquet file format version.
            Passed to [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `version`.
        parquet_page_version (`"1.0"` or `"2.0"`): Parquet page format version.
            Passed to [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `data_page_version`.
        parquet_metadata_statistics (bool or dict): If True, include summary
            statistics for each data page in the Parquet metadata, which lets some
            applications search for data more quickly (by skipping pages). If a dict
            mapping column names to bool, include summary statistics on only the
            specified columns. Passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `write_statistics`.
        parquet_dictionary_encoding (bool or dict): If True, allow Parquet to pre-compress
            with dictionary encoding. If a dict mapping column names to bool, only
            use dictionary encoding on the specified columns. Passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `use_dictionary`.
        parquet_byte_stream_split (bool or dict): If True, pre-compress floating
            point fields (`float32` or `float64`) with byte stream splitting, which
            collects all mantissas in one part of the stream and exponents in another.
            Passed to [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `use_byte_stream_split`.
        parquet_coerce_timestamps (None, `"ms"`, or `"us"`): If None, any timestamps
            (`datetime64` data) are coerced to a given resolution depending on
            `parquet_version`: version `"1.0"` and `"2.4"` are coerced to microseconds,
            but later versions use the `datetime64`'s own units. If `"ms"` is explicitly
            specified, timestamps are coerced to milliseconds; if `"us"`, microseconds.
            Passed to [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `coerce_timestamps`.
        parquet_old_int96_timestamps (None or bool): If True, use Parquet's INT96 format
            for any timestamps (`datetime64` data), taking priority over `parquet_coerce_timestamps`.
            If None, let the `parquet_flavor` decide. Passed to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `use_deprecated_int96_timestamps`.
        parquet_compliant_nested (bool): If True, use the Spark/BigQuery/Parquet
            [convention for nested lists](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#nested-types),
            in which each list is a one-field record with field name "`element`";
            otherwise, use the Arrow convention, in which the field name is "`item`".
            Passed to [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
            as `use_compliant_nested_type`.
        parquet_extra_options (None or dict): Any additional options to pass to
            [pyarrow.parquet.ParquetWriter](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html).
        storage_options (None or dict): Any additional options to pass to
            [fsspec.core.url_to_fs](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.core.url_to_fs)
            to open a remote file for writing.

    Returns:
    `pyarrow._parquet.FileMetaData` instance

    Writes an Awkward Array to a Parquet file (through pyarrow).

        >>> array1 = ak.Array([[1, 2, 3], [], [4, 5], [], [], [6, 7, 8, 9]])
        >>> ak.to_parquet(array1, "array1.parquet")
        <pyarrow._parquet.FileMetaData object at 0x7f646c38ff40>
          created_by: parquet-cpp-arrow version 9.0.0
          num_columns: 1
          num_rows: 6
          num_row_groups: 1
          format_version: 2.6
          serialized_size: 0

    If the `array` does not contain records at top-level, the Arrow table will consist
    of one field whose name is `""` iff. `extensionarray` is False.

    If `extensionarray` is True`, use a custom Arrow extension to store this array.
    Otherwise, generic Arrow arrays are used, and if the `array` does not
    contain records at top-level, the Arrow table will consist of one field whose
    name is `""`. See #ak.to_arrow_table for more details.

    Parquet files can maintain the distinction between "option-type but no elements are
    missing" and "not option-type" at all levels, including the top level. However,
    there is no distinction between `?union[X, Y, Z]]` type and `union[?X, ?Y, ?Z]` type.
    Be aware of these type distinctions when passing data through Arrow or Parquet.

    See also #ak.to_arrow, which is used as an intermediate step.
    F)Úwrite_iteratively)Ú_impl)ÚarrayÚdestinationr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    © r%   úc/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/awkward/operations/ak_to_parquet.pyr	      s:     #çr	   c           '        sâ  | }t jj d¡}‡ ‡‡‡‡‡‡	fdd„}|s<||ƒ\}}n‚t|tjjtjjtjj	tj
jtjjfƒsxt|dƒsxt|dƒr€tdƒ‚t|ƒ}zt|ƒ}W n ty°   tdƒd ‚Y n0 ||ƒ\}}|rÈd‰nd	‰|jd
gkrÞd‰nd‰t|tjƒrø|jj‰n|j‰d(‡‡‡fdd„	}|	du �r d}	n^|	du �s4|	d u �r:d}	nDt|	tƒ�r~i } |	 ¡ D ]&\}!‰
|  ‡
fdd„||!ƒD ƒ¡ �qR| }	t|
tƒ�rÂi } |
 ¡ D ]&\}!‰
|  ‡
fdd„||!ƒD ƒ¡ �q–| }
|du �rÒd}n |du �sæ|d u �rìd}n†t|tƒ�r@i } | ¡ D ]&\}!‰
|  ‡
fdd„||!ƒD ƒ¡ �qdd„ |  ¡ D ƒ}n2t|tƒ�rrg } |D ]}!|  t||!ƒƒ¡ �qT| }|du �rŠ|d dd�}np|du �sž|d u �r¤d}nVt|tƒ�rúi } | ¡ D ]*\}!‰
|  ‡
fdd„||!dd�D ƒ¡ �q¼dd„ |  ¡ D ƒ}|du �r|d dd�}np|du �s&|d u �r,d}nVt|tƒ�r‚i } | ¡ D ]*\}!‰
|  ‡
fdd„||!dd�D ƒ¡ �qDdd„ |  ¡ D ƒ}ˆ�r�t|ƒ}|d u �rži }zt|ƒ}W n, t�yÖ   td t|ƒj› d!�ƒd ‚Y n0 t j!j"|fi |�pìi ¤Ž\}"}g }#|j#||j$f|"||||	|||
||||||#d"œ|¤Ž�€}$zd|�sH|$j%||d#� nB|$j%||d#� |D ].}%||%ƒ\}}ˆ�rxt|ƒ}|$j%||d#� �qZW |$ &¡  n
|$ &¡  0 W d   ƒ n1 �s¶0    Y  |#d$ }&|& '| (d%d&¡d' ¡ |&S ))Nzak.to_parquetc              
     sB   t jjj| dddddddd�}t jj |ˆˆˆ ˆˆˆˆ¡}||fS )NTFÚerrorzas-characters)Zallow_recordZallow_unknownZnone_policyZregulararrayZuse_from_iterZprimitive_policyZstring_policy)ÚakÚ
operationsZak_to_layoutr"   Zak_to_arrow_table)ÚxÚlayoutÚtable)r   r   r   r   r   r   r   r%   r&   Úget_layout_and_table÷   s*    ø
ø
z#_impl.<locals>.get_layout_and_tableÚdtypeÚshapezÖThe first argument of ak.to_parquet_row_groups should be an iterable collection of arrays to put into separate row groups, not a single array. If this was intended, wrap the first argument with the `iter` function.zOThe first argument of ak.to_parquet_row_groups must contain at least one array.zlist.elementz	list.itemÚ )r0   r%   c                   sŒ   | d u rˆ}n
ˆ  | ¡}|jˆˆ d�}|d ur„| ¡ }t|ƒt|ƒksJJ ‚|dkrfdd„ t||ƒD ƒS |dkrˆdd„ t||ƒD ƒS n|S d S )N)Úlist_indicatorÚcolumn_prefixÚstringc                 S  s   g | ]\}}|d kr|‘qS )r3   r%   ©Ú.0r*   Úyr%   r%   r&   Ú
<listcomp>O  s   þz2_impl.<locals>.parquet_columns.<locals>.<listcomp>Úfloatingc                 S  s.   g | ]&\}}t |tjƒrt|jtjƒr|‘qS r%   )Ú
isinstanceÚmetadatar.   Ú
issubclassÚtyper8   r4   r%   r%   r&   r7   U  s   ý)Zselect_columnsÚcolumnsÚcolumn_typesÚlenÚzip)Ú	specifierÚonlyZselected_formZparquet_column_namesr>   )r2   Úformr1   r%   r&   Úparquet_columnsB  s$    
ÿþþz_impl.<locals>.parquet_columnsTr
   FÚnonec                   s   i | ]
}|ˆ “qS r%   r%   ©r5   r*   ©Úvaluer%   r&   Ú
<dictcomp>e  ó    z_impl.<locals>.<dictcomp>c                   s   i | ]
}|ˆ “qS r%   r%   rF   rG   r%   r&   rI   k  rJ   c                   s   i | ]
}|ˆ “qS r%   r%   rF   rG   r%   r&   rI   u  rJ   c                 S  s   g | ]\}}|r|‘qS r%   r%   ©r5   r*   rH   r%   r%   r&   r7   v  rJ   z_impl.<locals>.<listcomp>r3   )rB   c                   s   i | ]
}|ˆ “qS r%   r%   rF   rG   r%   r&   rI   …  rJ   c                 S  s   g | ]\}}|r|‘qS r%   r%   rK   r%   r%   r&   r7   ‡  rJ   r8   c                   s   i | ]
}|ˆ “qS r%   r%   rF   rG   r%   r&   rI   ‘  rJ   c                 S  s   g | ]\}}|r|‘qS r%   r%   rK   r%   r%   r&   r7   “  rJ   zb'destination' argument of 'ak.to_parquet' and 'ak.to_parquet_row_groups' must be a path-like, not z2 ('array' argument is first; 'destination' second))Ú
filesystemZflavorÚversionZuse_dictionaryr   Zwrite_statisticsZuse_deprecated_int96_timestampsr   Zuse_byte_stream_splitZdata_page_versionZuse_compliant_nested_typer   Zcoerce_timestampsZmetadata_collector)r   r   ú/é   éÿÿÿÿ)N))ÚawkwardÚ_connectZpyarrowZimport_pyarrow_parquetr9   r(   Z	highlevelZArrayZRecordZArrayBuilderÚcontentsZContentÚrecordÚhasattrÚ	TypeErrorÚiterÚnextÚStopIterationÚ
ValueErrorZcolumn_namesr#   rC   r   ÚitemsÚupdater   ÚextendÚlistr   r   r<   Ú__name__ÚfsspecÚcoreZ	url_to_fsZParquetWriterZschemaZwrite_tableÚcloseZset_file_pathÚrsplit)'r#   r$   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   ÚdataZpyarrow_parquetr-   r+   r,   Zfirst_arrayrD   ÚreplacementrA   ÚfsZmetalistÚwriterÚitemÚmetar%   )r   r   r2   r   r   r   rC   r1   r   r   rH   r&   r"   Ö   s   ûþÿõôÿÿþ

  
 
ÿ
ÿ
ÿþþðï4r"   )Úglobal_metadatac                G  s´   |sJ ‚|d }|  d | dg¡d¡�}| |¡ W d  ƒ n1 sF0    Y  |r°|dd… D ]}| |¡ q`|  d | dg¡d¡�}| |¡ W d  ƒ n1 s¦0    Y  dS )z?Generate metadata file(s) from list of arrow metadata instancesr   rN   Z_common_metadataÚwbNrO   Ú	_metadata)ÚopenÚjoinZwrite_metadata_fileZappend_row_groups)Zdir_pathrf   rj   ZmetasÚmdZfilri   r%   r%   r&   Úwrite_metadataÐ  s    (rp   )Ú
__future__r   Úcollections.abcr   r   Úosr   r`   rQ   r(   Zawkward._connect.pyarrowr   Zawkward._dispatchr   Zawkward._nplikes.numpy_liker   Ú__all__Úinstancer:   r	   r"   rp   r%   r%   r%   r&   Ú<module>   sL   ç A {