a
    yn„fÇp  ã                   @  s  d dl mZ d dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 d dlmZ d dlZd dlmZ d dlZd dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZ dZe ¡ Ze ¡ Z eƒ dddddddddddddœdd„ƒZ!dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd„ Z&dS ) é    )ÚannotationsN)ÚIterableÚSized)Únullcontext)ÚBytesIO)Úurlparse)Ú_ext)Úhigh_level_function)ÚHighLevelContextÚwrap_layout)ÚNumpy)ÚNumpyMetadata)Ú
is_integer)Ú	from_jsonFi   i   é   T)Úline_delimitedÚschemaÚ
nan_stringÚposinf_stringÚneginf_stringÚcomplex_record_fieldsÚ
buffersizeÚinitialÚresizeÚ	highlevelÚbehaviorÚattrsc                C  sJ   |du r&t | ||||||||	|
||ƒS t| |||||||||	|
||ƒS dS )u`5  
    Args:
        source (bytes/str, pathlib.Path, or file-like object): Data source of the
            JSON-formatted string(s). If bytes/str, the string is parsed. If a
            `pathlib.Path`, a file with that name is opened, parsed, and closed.
            If that path has a URI protocol (like `"https://"` or `"s3://"`), this
            function attempts to open the file with the fsspec library. If a
            file-like object with a `read` method, this function reads from the
            object, but does not close it.
        line_delimited (bool): If False, a single JSON document is read as an
            entire array or record. If True, this function reads line-delimited
            JSON into an array (regardless of how many there are). The line
            delimiter is not actually checked, so it may be `"\n"`, `"\r\n"`
            or anything else.
        schema (None, JSON str or equivalent lists/dicts): If None, the data type
            is discovered while parsing. If a JSONSchema
            ([json-schema.org](https://json-schema.org/)), that schema is used to
            parse the JSON more quickly by skipping type-discovery.
        nan_string (None or str): If not None, strings with this value will be
            interpreted as floating-point NaN values.
        posinf_string (None or str): If not None, strings with this value will
            be interpreted as floating-point positive infinity values.
        neginf_string (None or str): If not None, strings with this value
            will be interpreted as floating-point negative infinity values.
        complex_record_fields (None or (str, str)): If not None, defines a pair of
            field names to interpret 2-field records as complex numbers.
        buffersize (int): Number of bytes in each read from source: larger
            values use more memory but read less frequently. (Python GIL is
            released before and after read events.)
        initial (int): Initial size (in bytes) of buffers used by the `ak::ArrayBuilder`.
        resize (float): Resize multiplier for buffers used by the `ak::ArrayBuilder`;
            should be strictly greater than 1.
        highlevel (bool): If True, return an #ak.Array; otherwise, return
            a low-level #ak.contents.Content subclass.
        behavior (None or dict): Custom #ak.behavior for the output array, if
            high-level.
        attrs (None or dict): Custom attributes for the output array, if
            high-level.

    Converts a JSON string into an Awkward Array.

    There are a few different dichotomies in JSON-reading; all of the combinations
    are supported:

    * Reading from in-memory str/bytes, on-disk or over-network file, or an
      arbitrary Python object with a `read(num_bytes)` method.
    * Reading a single JSON document or a sequence of line-delimited documents.
    * Unknown schema (slow and general) or with a provided JSONSchema (fast, but
      not all possible cases are supported).
    * Conversion of strings representing not-a-number, plus and minus infinity
      into the appropriate floating-point numbers.
    * Conversion of records with a real and imaginary part into complex numbers.

    Non-JSON features not allowed, including literals for not-a-number or infinite
    numbers; they must be quoted strings for `nan_string`, `posinf_string`, and
    `neginf_string` to recognize. The document or line-delimited documents must
    adhere to the strict [JSON schema](https://www.json.org/).

    Sources
    =======

    In-memory strings or bytes are simply passed as the first argument:

        >>> ak.from_json("[[1.1, 2.2, 3.3], [], [4.4, 5.5]]")
        <Array [[1.1, 2.2, 3.3], [], [4.4, 5.5]] type='3 * var * float64'>

    File names/paths need to be wrapped in `pathlib.Path`, and remote files are
    recognized by URI protocol (like `"https://"` or `"s3://"`) and handled by fsspec
    (which must be installed).

        >>> import pathlib
        >>> with open("tmp.json", "w") as file:
        ...     file.write("[[1.1, 2.2, 3.3], [], [4.4, 5.5]]")
        ...
        33
        >>> ak.from_json(pathlib.Path("tmp.json"))
        <Array [[1.1, 2.2, 3.3], [], [4.4, 5.5]] type='3 * var * float64'>

    And any object with a `read(num_bytes)` method can be used as the `source`.

        >>> class HasReadMethod:
        ...     def __init__(self, data):
        ...         self.bytes = data.encode()
        ...         self.pos = 0
        ...     def read(self, num_bytes):
        ...         start = self.pos
        ...         self.pos += num_bytes
        ...         return self.bytes[start:self.pos]
        ...
        >>> filelike_obj = HasReadMethod("[[1.1, 2.2, 3.3], [], [4.4, 5.5]]")
        >>> ak.from_json(filelike_obj)
        <Array [[1.1, 2.2, 3.3], [], [4.4, 5.5]] type='3 * var * float64'>

    If this function opens a file or network connection (because it is passed as
    a `pathlib.Path`), then this function will also close that file or connection.

    If this function is provided a file-like object with a `read(num_bytes)` method,
    this function will not close it. (It might not even have a `close` method.)

    Data structures
    ===============

    This function interprets JSON arrays and JSON objects in the same way that
    #ak.from_iter interprets Python lists and Python dicts. It could be used as a
    synonym for Python's `json.loads` followed by #ak.from_iter, but the direct
    JSON-reading is faster (especially with a schema) and uses less memory.

    Consider

        >>> import json
        >>> json_data = "[[1.1, 2.2, 3.3], [], [4.4, 5.5]]"
        >>> ak.from_iter(json.loads(json_data))
        <Array [[1.1, 2.2, 3.3], [], [4.4, 5.5]] type='3 * var * float64'>
        >>> ak.from_json(json_data)
        <Array [[1.1, 2.2, 3.3], [], [4.4, 5.5]] type='3 * var * float64'>

    and

        >>> json_data = '{"x": 1.1, "y": [1, 2]}'
        >>> ak.from_iter(json.loads(json_data))
        <Record {x: 1.1, y: [1, 2]} type='{x: float64, y: var * int64}'>
        >>> ak.from_json(json_data)
        <Record {x: 1.1, y: [1, 2]} type='{x: float64, y: var * int64}'>

    As shown above, reading JSON may result in #ak.Array or #ak.Record, but line-delimited
    (`line_delimited=True`) only results in #ak.Array:

        >>> ak.from_json(
        ...     '{"x": 1.1, "y": [1]}\n{"x": 2.2, "y": [1, 2]}\n{"x": 3.3, "y": [1, 2, 3]}',
        ...     line_delimited=True,
        ... )
        <Array [{x: 1.1, y: [1]}, ..., {x: 3.3, ...}] type='3 * {x: float64, y: var...'>

    Even arrays of length zero:

        >>> ak.from_json("", line_delimited=True)
        <Array [] type='0 * unknown'>

    Note that JSON interpreted with `line_delimited` doesn't actually need delimiters
    between JSON documents or an absence of delimiters within each document. Parsing
    with `line_delimited=True` continues to the end of a JSON document and starts
    again with the next JSON document. It may be necessary to require actual delimiters
    between and never within JSON documents to split a large source for
    parallel-processing, but that consideration is beyond this function.

    If a JSONSchema is provided, the schema describes the structure of the JSON
    document, regardless of whether there's only one of them (may be an #ak.Record)
    or many of them (must be an #ak.Array).

        >>> schema = {
        ...     "type": "object",
        ...     "properties": {
        ...         "x": {"type": "number"},
        ...         "y": {"type": "array", "items": {"type": "integer"}},
        ...     },
        ...     "required": ["x", "y"],
        ... }

        >>> ak.from_json(
        ...     '{"x": 1.1, "y": [1, 2, 3]}',
        ...     schema=schema,
        ... )
        <Record {x: 1.1, y: [1, ..., 3]} type='{x: float64, y: var * int64}'>

        >>> ak.from_json(
        ...     '{"x": 1.1, "y": [1]}\n{"x": 2.2, "y": [1, 2]}\n{"x": 3.3, "y": [1, 2, 3]}',
        ...     schema=schema,
        ...     line_delimited=True,
        ... )
        <Array [{x: 1.1, y: [1]}, ..., {x: 3.3, ...}] type='3 * {x: float64, y: var...'>

    All numbers in the final array are signed 64-bit (integers and floating-point).

    JSONSchemas
    ===========

    This function supports a subset of JSONSchema (see the
    [JSONSchema specification](https://json-schema.org/)). The schemas may be passed
    as JSON text or as Python lists and dicts representing JSON, but the following
    conditions apply:

    * The root of the schema must be `"type": "array"` or `"type": "object"`.
    * Every level must have a `"type"`, which can only name one type (as a string
      or length-1 list) or one type and `"null"` (as a length-2 list).
    * `"type": "boolean"` â†’ 1-byte boolean values.
    * `"type": "integer"` â†’ 8-byte integer values. If a part of the schema
      is declared to have integer type but the JSON numbers are expressed as
      floating-point, such as `3.14`, `3.0`, or `3e0`, this function raises an
      error.
    * `"type": "number"` â†’ 8-byte floating-point values. If used with
      this function's `nan_string`, `posinf_string`, and/or `neginf_string`, the
      value in the JSON could be a string, as long as it matches one of these
      three.
    * `"type": "string"` â†’ UTF-8 encoded strings. All JSON escape sequences are
      supported. Remember that the `source` data are ASCII; Unicode is derived from
      "`\uXXXX`" escape sequences. If an `"enum"` is given, strings are represented
      as categorical values (#ak.contents.IndexedArray or #ak.contents.IndexedOptionArray).
    * `"type": "array"` â†’ nested lists. The `"items"` must be specified. If
      `"minItems"` and `"maxItems"` are specified and equal to each other, the
      list has regular-type (#ak.types.RegularType); otherwise, it has variable-length
      type (#ak.types.ListType).
    * `"type": "object"` â†’ nested records. The `"properties"` must be specified,
      and any properties in the data not described by `"properties"` will not
      appear in the output.

    Substitutions for non-finite and complex numbers
    ================================================

    JSON doesn't support not-a-number values, infinite values, or complex number
    types (as in numbers with a real and imaginary part). Some work-arounds use
    non-JSON syntax, but this function converts valid JSON into these numbers with
    user-specified rules.

    The `nan_string`, `posinf_string`, and `neginf_string` convert quoted strings
    into floating-point numbers. You can specify what these strings are.

        >>> ak.from_json(
        ...     '[1, 2, "nan", "inf", "-inf"]',
        ...     nan_string="nan",
        ...     posinf_string="inf",
        ...     neginf_string="-inf",
        ... )
        <Array [1, 2, nan, inf, -inf] type='5 * float64'>

    Without these rules, the array would be interpreted as a union of numbers and
    strings:

        >>> ak.from_json(
        ...     '[1, 2, "nan", "inf", "-inf"]',
        ... )
        <Array [1, 2, 'nan', 'inf', '-inf'] type='5 * union[int64, string]'>

    When combined with a JSONSchema, you need to say that these values have type
    `"number"`, not a union of strings and numbers (i.e. the conversion is performed
    *before* schema-validation). Note that they can't be `"integer"`, since
    not-a-number and infinite values are only possible for floating-point numbers.

        >>> ak.from_json(
        ...     '[1, 2, "nan", "inf", "-inf"]',
        ...     nan_string="nan",
        ...     posinf_string="inf",
        ...     neginf_string="-inf",
        ...     schema={"type": "array", "items": {"type": "number"}}
        ... )
        <Array [1, 2, nan, inf, -inf] type='5 * float64'>

    The `complex_record_fields` is a 2-tuple of field names (strings) of objects
    to identify as the real and imaginary parts of complex numbers. Complex number
    representations in JSON vary, though most are JSON objects with real and
    imaginary parts and possibly other fields. Any other fields will be excluded
    from the output array.

        >>> ak.from_json(
        ...     '[{"r": 1, "i": 1.1, "other": ""}, {"r": 2, "i": 2.2, "other": ""}]',
        ...     complex_record_fields=("r", "i"),
        ... )
        <Array [1+1.1j, 2+2.2j] type='2 * complex128'>

    Without this rule, the array would be interpreted as an array of records:

        >>> ak.from_json(
        ...     '[{"r": 1, "i": 1.1, "other": ""}, {"r": 2, "i": 2.2, "other": ""}]',
        ... )
        <Array [{r: 1, i: 1.1, other: ''}, {...}] type='2 * {r: int64, i: float64, ...'>

    When combined with a JSONSchema, you need to specify the object type (i.e. the
    conversion is performed *after* schema-validation). Note that even the fields
    that will be ignored by `complex_record_fields` need to be specified.

        >>> ak.from_json(
        ...     '[{"r": 1, "i": 1.1, "other": ""}, {"r": 2, "i": 2.2, "other": ""}]',
        ...     complex_record_fields=("r", "i"),
        ...     schema={
        ...         "type": "array",
        ...         "items": {
        ...             "type": "object",
        ...             "properties": {
        ...                 "r": {"type": "number"},
        ...                 "i": {"type": "number"},
        ...                 "other": {"type": "string"},
        ...             },
        ...             "required": ["r", "i"],
        ...         },
        ...     },
        ... )
        <Array [1+1.1j, 2+2.2j] type='2 * complex128'>

    See also #ak.to_json.
    N)Ú
_no_schemaÚ_yes_schema)Úsourcer   r   r   r   r   r   r   r   r   r   r   r   © r    úb/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/awkward/operations/ak_from_json.pyr      s@      4ôór   c                 C  s~   t | tƒr| jddd�} t | tƒr*t| ƒS t | tjƒrrtt| ƒƒ}|jdksV|j	dkr`t
| dƒS t 
| d¡ 
¡ S nt| ƒS d S )NÚutf8Úsurrogateescape©ÚerrorsÚ Úrb)Ú
isinstanceÚstrÚencodeÚbytesr   ÚpathlibÚPathr   ÚschemeÚnetlocÚopenÚfsspecr   )r   Ú
parsed_urlr    r    r!   Ú_get_readerp  s    


r3   c                   sn   ˆ d u r| S t ˆ tƒrbt ˆ tƒrbtˆ ƒdkrbt ˆ d tƒrbt ˆ d tƒrb‡ fdd„}tj | |¡S tdƒ‚d S )Né   r   é   c              
     sH  t | tjjƒ�rDtˆ ƒ | j¡�rD|  ˆ d ¡}|  ˆ d ¡}t |tjjƒ�rt	|j
ƒdk�rt|jjtjtjfƒ�rt |tjjƒ�rt	|j
ƒdk�rt|jjtjtjfƒ�rtjjdd��< tj | jj |j¡| jj |j¡d  ¡W  d   ƒ S 1 �s 0    Y  n8tdˆ d ›dˆ d ›dt|jjƒ›dt|jjƒ›�ƒ‚d S )	Nr   r5   Úignore)Úinvalidy              ð?zexpected record with fields z and z. to have integer or floating point types, not )r(   ÚakÚcontentsZRecordArrayÚsetÚissubsetÚfieldsZ_getitem_fieldZ
NumpyArrayÚlenÚshapeÚ
issubclassÚdtypeÚtypeÚnpÚintegerZfloatingÚnumpyÚ_moduleZerrstateÚbackendZnplikeZasarrayÚdataÚ
ValueErrorr)   Úform)ÚnodeÚkwargsÚrealÚimag©r   r    r!   ÚactionŽ  s2    ÿþýüûúÿÿ&2ÿz"_record_to_complex.<locals>.actionz7complex_record_fields must be None or a pair of strings)	r(   r   r   r=   r)   r8   Z_doZrecursively_applyÚ	TypeError)Úlayoutr   rO   r    rN   r!   Ú_record_to_complex‚  s    ÿþ
ýüûrR   c                 C  sô   t |
|d� ¡ }tj||d�}| }t| ƒ�^}zt |||||||¡ W n2 tyz } ztt|ƒƒd ‚W Y d }~n
d }~0 0 W d   ƒ n1 s�0    Y  | 	¡ \}}}t
j |¡}t
jj|||t
jjdd�}t||ƒ}|rä|d }|j||	dd�S )N)r   r   )r   r   F©Ú	byteorderr   r   T)r   Zallow_other)r
   Úfinalizer   ZArrayBuilderr3   ZfromjsonobjÚ	ExceptionrH   r)   Z
to_buffersr8   Úformsr   Ú
operationsÚfrom_buffersÚ_utilÚnative_byteorderrR   Úwrap)r   r   r   r   r   r   r   r   r   r   r   r   ÚctxÚbuilderÚread_oneÚobjÚerrZformstrÚlengthÚbuffersrI   rQ   r    r    r!   r   «  s2    
ù	Bÿ
r   c                 C  st  t |ttfƒrt |¡}t |tƒs0td|›�ƒ‚i }g }| d¡dkrxd|vrVtdƒ‚| dg¡ t	|d ||ƒ}d}n(| d¡dkr˜t	|||ƒ}d	}ntd
ƒ‚| }t
| ƒ�l}z&t |||||||t |¡||	¡
}W n4 t�y
 } ztt|ƒƒd ‚W Y d }~n
d }~0 0 W d   ƒ n1 �s"0    Y  tjj|||tjjdd�}t||ƒ}|�rd|�rd|d }t||
||d�S )Nú,unrecognized JSONSchema: expected dict, got rA   ÚarrayÚitemsz4JSONSchema type is not concrete: array without itemsZTopLevelArrayFÚobjectTz@only 'array' and 'object' types supported at the JSONSchema rootrS   r   )r   r   r   )r(   r+   r)   ÚjsonÚloadsÚdictrP   ÚgetÚappendÚ_build_assemblyr3   r   Zfromjsonobj_schemaÚdumpsrV   rH   r8   rX   rY   rZ   r[   rR   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   Ú	containerÚinstructionsrI   Z	is_recordr_   r`   rb   ra   rQ   r    r    r!   r   Ú  sT    

ÿ
öDÿ
r   c              	   C  s–  t | tƒstd| ›�ƒ‚d| vr.td| ›�ƒ‚| d }d}t |tƒrrd|v r^d}dd„ |D ƒ}t|ƒd	krr|d
 }|dv �r€|dkr’d}d}d}n*|dkr¨d}d }}n|dkr¼d}d }}|�r>dt|ƒ› �}d ||d < dt|ƒ› �}	d ||	d < | d|d dg¡ | ||	d |g¡ tjjdtjj	||	d�d|d�S dt|ƒ› �}	d ||	d < | ||	d |g¡ tjj	||	d�S �n|dk�rÀd| v �rð| d }
t |
tƒ�s¬J ‚t|
ƒd	k�s¾J ‚t
d d!„ |
D ƒƒ�sÖJ ‚d"d„ |
D ƒ}dt|ƒ› �}d ||d# < dt|ƒ› �}tjt|
ƒd	 tjd$�||d% < d
||d%  d
< t d&d„ |D ƒ¡||d%  d	d …< d't|ƒ› �}	d( |¡||	d < |�r”d)}tjj}nd*}tjj}| ||d# d|
g¡ |d+tjjd+tjj	dd,d-i|	d.�d,di|d.�d,d/i|d.�S |�r$dt|ƒ› �}d ||d < | d|d dg¡ dt|ƒ› �}d ||d% < dt|ƒ› �}	d ||	d < | d0|d% d|	d dg¡ tjjd+tjj	dd,d-i|	d.�d,di|d.�}|�r¸tjjd|d|d�S |S �nÒ|d1k�rJd2| v�rÜtd3ƒ‚|  d4¡|  d5¡  k�r d k�r¤n n t|  d4¡ƒ�sJ ‚|�rLdt|ƒ› �}d ||d# < | d6|d# dg¡ | d7|  d4¡g¡ t| d2 ||ƒ}tjj||  d4¡d8�}|�ržtjjd+||d�S |S n¢|�rØdt|ƒ› �}d ||d < | d|d dg¡ dt|ƒ› �}d ||d% < | d9|d% dg¡ t| d2 ||ƒ}tjjd+||d�}|�rBtjjd|d|d�S |S �nH|d:k�rnd;| v�rftd<ƒ‚g }g }| d;  ¡ D ]\}}| |¡ | |¡ �qz|�rÎdt|ƒ› �}d ||d# < | d6|d# dg¡ | d=t|ƒg¡ t|ƒ}|D ]}| d>|d g¡ �qìg }t|ƒD ]0\}}t|ƒ|||  d?< | t|||ƒ¡ �qtj ||¡}|�rhtjjd+||d�S |S n$t |tƒ�r„td@ƒ‚ntdA|›�ƒ‚d S )BNrd   rA   z&unrecognized JSONSchema: no 'type' in FÚnullTc                 S  s   g | ]}|d kr|‘qS )rq   r    ©Ú.0Úxr    r    r!   Ú
<listcomp>.  ó    z#_build_assembly.<locals>.<listcomp>r5   r   >   ÚbooleanrC   Únumberrw   ZFillBooleanZuint8ÚboolrC   ZFillIntegerÚint64rx   Z
FillNumberZfloat64rJ   z-maskz-dataZFillByteMaskedArrayZint8Úi8)Úform_key)Z
valid_whenr|   ÚstringÚenumc                 s  s   | ]}t |tƒV  qd S )N)r(   r)   rr   r    r    r!   Ú	<genexpr>[  rv   z"_build_assembly.<locals>.<genexpr>c                 S  s   g | ]}|j d dd�‘qS )zutf-8r#   r$   )r*   rr   r    r    r!   ru   \  rv   z-index)r@   z-offsetsc                 S  s   g | ]}t |ƒ‘qS r    )r=   rr   r    r    r!   ru   f  rv   ro   rv   ZFillNullEnumStringZFillEnumStringZi64Z	__array__Úchar)Ú
parametersr|   ZcategoricalZ
FillStringre   rf   z6JSONSchema type is not concrete: array without 'items'ZminItemsZmaxItemsZFillIndexedOptionArrayZFixedLengthList)ÚsizeZVarLengthListrg   Ú
propertiesz<JSONSchema type is not concrete: object without 'properties'ZKeyTableHeaderZKeyTableItemr4   z/arbitrary unions of types are not yet supportedzunrecognized JSONSchema: )r(   rj   rP   Úlistr=   rl   r8   rW   ZByteMaskedFormZ	NumpyFormÚallrD   ÚemptyrB   rz   ZcumsumÚjoinZIndexedOptionFormZIndexedFormZListOffsetFormrk   r   rm   ZRegularFormrf   Ú	enumerateZ
RecordFormÚNotImplementedError)r   ro   rp   ZtpeZis_optionalZinstructionr@   Z	primitiveÚmaskrJ   ÚstringsZbytestringsÚindexÚoffsetsZformtypeÚoutÚcontentÚnamesZ
subschemasÚnameZ	subschemaZ	startkeysr9   Zkeyindexr    r    r!   rm   !  sH   



ü

ÿÿ

ÿúõÿýø
ÿ

(ÿÿ

ÿ

rm   )'Ú
__future__r   rh   r,   Úcollections.abcr   r   Ú
contextlibr   Úior   Úurllib.parser   r1   Zawkward_cpp.libr   Zawkwardr8   Zawkward._dispatchr	   Zawkward._layoutr
   r   Zawkward._nplikes.numpyr   Zawkward._nplikes.numpy_liker   Zawkward._regularizer   Ú__all__ÚinstancerB   rD   r   r3   rR   r   r   rm   r    r    r    r!   Ú<module>   sL   ò  U)/G