a
    yn„fM�  ã                   @  s~  d dl mZ d dlZd dlmZmZ d dlmZ d dlZd dl	Z
d dlmZ d dlmZ d dlmZ d dlmZ d	d
lmZmZ e ¡ Ze ¡ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdDdd„Z dd„ Z!dd„ Z"d d!„ Z#dEd#d$„Z$ed%d&„ ƒZ%d'd(„ Z&d)d*„ Z'd+d,„ Z(d-d.„ Z)d/d0„ Z*dFd1d2„Z+dGd3d4„Z,dHd5d6„Z-e .¡ e /¡ e 0¡ e 1¡ fZ2e 3¡ de 4d7¡fe 5¡ d"e 4d8¡fe 6d9¡de 4d:¡fe 6d;¡de 4d8¡fe 7d<¡d"e 4d=¡fe 7d>¡d"e 4d?¡fe 8d9¡d"e 4d:¡fe 8d;¡d"e 4d8¡fe 8d<¡d"e 4d=¡fe 8d>¡d"e 4d?¡fe 9d9¡d"e 4d@¡fe 9d;¡d"e 4dA¡fe 9d<¡d"e 4dB¡fe 9d>¡d"e 4dC¡fiZ:dS )Ié    )ÚannotationsN)ÚIterableÚSized)Ú	lru_cache)ÚNumpyBackend)ÚNumpy)ÚNumpyMetadata)Úparameters_unioné   )ÚAwkwardArrowTypeÚto_awkwardarrow_storage_typesc                 C  s$   | d u r|S |d u r| S | |@ S d S ©N© )Zvalidbytes1Zvalidbytes2r   r   úg/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/awkward/_connect/pyarrow/conversions.pyÚand_validbytes   s
    r   c                 C  s$   | d u rd S t  tj| dd�¡S d S )NÚlittle©Zbitorder)ÚpyarrowÚ	py_bufferÚnumpyZpackbits)Ú
validbytesr   r   r   Úto_validbits    s    r   c                 C  s<   t | ƒ|k r.tj|| jd�}| |d t | ƒ…< n| }t |¡S )N©Údtype)Úlenr   Úemptyr   r   r   )ZnparrayÚlengthÚoutr   r   r   Ú	to_length'   s
    r   c                 C  s&   | d u s|sdS t | ƒt | ¡ S d S )Néÿÿÿÿ)r   r   Zcount_nonzero)r   Zcount_nullsr   r   r   Úto_null_count0   s    r    c                 C  s   t | tƒr| jS d S d S r   )Ú
isinstancer   Únode_parameters©Úawkwardarrow_typer   r   r   r"   7   s    
r"   c                 C  s   t | tƒr| jS d S d S r   )r!   r   Úmask_parametersr#   r   r   r   r%   >   s    
r%   c                 C  s
   || _ | S r   ©Ú__pyarrow_original)ÚmodifiedÚoriginalr   r   r   Ú
revertableE   s    r*   Tc              	   C  sØ  |r6|j dkst|ƒt| ƒkr6| |j |j t|ƒ … } t|tƒ�rN|jdkrœ|d u s‚t tj|tj	d�d t| ƒd … dk¡s‚J ‚t
tjjj| |jd�| ƒS |d u �r|röt
tjjjtj tjt| ƒd  t 	d¡tj	d�¡| dt| ƒd|jd	�| ƒS t
tjjj| |jd�| ƒS n:t
tjjjtj tj|tj	d�¡| dt| ƒd|jd	�| ƒS n†|d u �r€|�r€tjt| ƒd  t 	d¡tj	d�}|d u �ržt
tjj | ¡| ƒS t
tjjjtj tj|tj	d�¡| dt| ƒdd
�| ƒS d S )Nr   ÚUnmaskedArrayr   é   éÿ   ©Ú
parametersiøÿÿÿT)Ú
valid_whenr   Ú	lsb_orderr/   )r0   r   r1   )Úoffsetr   r!   r   Ú	mask_typer   ÚallÚ
frombufferÚnpÚuint8r*   ÚakÚcontentsr+   Ú
simplifiedr%   ÚBitMaskedArrayÚindexZIndexU8Úfull)r   ÚarrayÚ	validbitsr$   Úgenerate_bitmasksÚfix_offsetsr   r   r   Úpopbuffers_finalizeJ   sz    
"ÿÿü
ÿÿõòÿüú÷"
ûørB   c                 C  sT   t |tƒr.|jdkr.ttjjj| |jd�| ƒS ttjj	jd| ddt|ƒd�| ƒS d S )Nr+   r.   Úu8T)r0   r1   r/   )
r!   r   r3   r*   r8   ÚformsZUnmaskedFormr:   r%   ZBitMaskedForm)r   r$   r   r   r   Úform_popbuffers_finalize›   s(    ÿþÿüûørE   c           !      C  sT  |d ur| j } t|tjjƒr(tdƒ‚�n(t|tjjƒrVt|tƒrDJ ‚t| ||j	||ƒS t|tjj
ƒ�r0t| jd |j||ƒ}|jj}t|tjjƒsº|jdd�}| ¡ rºtj|dd�}d||< t| j|ƒ}tt|ƒt|ƒƒ}	|	d u rèddi}	ttjjjtj |¡||	d	�tjj tj |¡|j!�r$t"|ƒn||	d	�ƒS t|tjj#ƒ�r´|j$d
k�sPJ ‚| %d¡}
t&|j'ƒ\}}t| j(||||ƒ}|j)j*�sŒt"|ƒ}tjj+||j,t|ƒd	�}t-|| |
||ƒS t|tjj.tjj/fƒ�rŠ|j$dk�sÜJ ‚| %d¡}
| %d¡}t|tjj.ƒ�rtj 0tj1|t2j3d�¡}ntj 4tj1|t2j5d�¡}t&|j'ƒ\}}t| j(||||ƒ}|j)j*�sdt"|ƒ}tjj6||t|ƒd	�}t-|| |
||ƒS t|tjj7ƒ�r¢t8‚�n®t|tjj9tjj:fƒ�rÆtdƒ‚�nŠt|tjj;ƒ�r^|j$dk�sæJ ‚| %d¡}
| %d¡}t|ƒ}	|	d u �rddi}	ddi}tjj+tjj<tj1|t2j=d�|t> ?¡ d�|j@|	d	�}t-|| |
||ƒS |tAv �rl|j$dk�sxJ ‚| %d¡}
| %d¡}| %d¡}|tAd d d… v �rÄtj 4tj1|t2j5d�¡}ntj 0tj1|t2j3d�¡}t|ƒ}	|tAd d… v �r|	d u �rddi}	ddi}n|	d u �r$ddi}	ddi}tjj6|tjj<tj1|t2j=d�|t> ?¡ d�|	d	�}t-|| |
||ƒS t|tjjBƒ�rL|j$d
k�sŒJ ‚| %d¡}
g }g }tC|jDƒD ]\}|| }|jE}| F|¡ t&|jGƒ\}}t|  H|¡||||ƒ}|j*�søt"|ƒ}| F|¡ �q¨|d u�r|jI�rd }tjjJ||tK| ƒt|ƒd�}t-|| |
||dd�S t|tjjLƒ�rŽt|tjjMƒ�r²|j$dk�s|J ‚| %d¡}
tj1| %d¡t2jNd�}tjOtK|ƒt2j5d�}nF|j$dk�sÂJ ‚| %d¡}
tj1| %d¡t2jNd�}tj1| %d¡t2j5d�}g }tC|jDƒD ]L}|| }t&|jGƒ\}}t|  H|¡||||ƒ}|j*�sFt"|ƒ}| F|¡ �qtjjPjtj Q|¡tj 4|¡|t|ƒd	�}t-|| d ||ƒS |t R¡ k�r| %d¡}
|jDdk�s¶J ‚tjjSt|ƒd	�}|d u�rä|jT�rät||ƒS tjjtj 0tjUtK| ƒdt2j3d�¡|t|ƒd	�S |t V¡ k�r’|j$dk�s2J ‚| %d¡}
| %d¡}tjWtj1|t2j=d�dd�}tjj<| Xt2jV¡t|ƒt> ?¡ d�}t-|| |
||ƒS t|tjjYƒ�rB|j$dk�s²J ‚| %d¡}
| %d¡}tZ [t\|ƒd¡\}} |�rütj]tj1|t2j5d�t2j3d�}| d u �r| ^¡ } tjj<tj1|| d�t|ƒt> ?¡ d�}t-|| |
||ƒS t_d|›�ƒ‚d S )NúUArrow arrays containing pickled Python objects can't be converted into Awkward ArraysF©r0   T)Úcopyr   Ú	__array__Úcategoricalr.   r
   r   é   r   úmArrow arrays containing pyarrow.decimal128 or pyarrow.decimal256 types can't be converted into Awkward ArraysÚ
bytestringÚbyte)r/   Úbackendé   ÚstringÚchar©r   r/   )rA   r   r   ©FNúunrecognized Arrow array type: )`Zstorager!   r   ÚlibÚPyExtensionTypeÚ
ValueErrorÚExtensionTyper   Ú
popbuffersÚstorage_typeÚDictionaryTypeÚindicesÚ
index_typeÚcontentÚdatar8   r9   r+   Úmask_as_boolÚanyr   ZasarrayÚhandle_arrowÚ
dictionaryr	   r%   r"   r*   ÚIndexedOptionArrayr:   r<   ÚIndexZIndexedArrayÚ	is_optionÚremove_optiontypeÚFixedSizeListTypeZnum_buffersÚpopr   Ú
value_typeÚvaluesÚvalue_fieldÚnullableZRegularArrayÚ	list_sizerB   ÚLargeListTypeÚListTypeZIndex64r5   r6   Úint64ZIndex32Úint32ZListOffsetArrayÚMapTypeÚNotImplementedErrorÚDecimal128TypeÚDecimal256TypeÚFixedSizeBinaryTypeZ
NumpyArrayr7   r   ÚinstanceÚ
byte_widthÚ_string_likeÚ
StructTypeÚrangeÚ
num_fieldsÚnameÚappendÚtypeÚfieldÚrecord_is_tupleÚRecordArrayr   Ú	UnionTypeZSparseUnionTypeZint8ZarangeZ
UnionArrayÚIndex8ÚnullZ
EmptyArrayZ_is_nonnullable_nulltyper=   Úbool_Z
unpackbitsÚviewÚDataTypeÚ_pyarrow_to_numpy_dtypeÚgetÚstrZastypeÚto_pandas_dtypeÚ	TypeError)!Zpaarrayr$   r[   Úbuffersr@   Zmasked_indexr<   Úmaskr_   r/   r?   ÚaÚbÚ	akcontentr   Z	paoffsetsÚ	akoffsetsZ	pacontentÚsub_parametersÚkeysr9   Úir‚   Ú
field_nameZnptagsZnpindexÚ
akcontentsZempty_arrayZbitdataZbytedatar`   Zto64Údtr   r   r   rZ   ´   sà   ÿûûÿ
ý
ýú

ý
ÿ



ÿ
ÿÿÿ


ýù	
ÿ






ýù	
ÿ

ÿüú	



ü
ÿ
ÿ
ý

ÿ
ý
ÿ


ý
ÿrZ   c                 C  sê  t |tjjƒrtdƒ‚�nÌt |tjjƒrBt |tƒr6J ‚t| |jƒS t |tjj	ƒ�r|j
 ¡ }|tju rld}n.|tju r|d}n|tju rŒd}ntd|› �ƒ‚t|jƒ\}}t||ƒ}tt| ƒt| ƒƒ}|d u rÔddi}ttjjj|||d�tjj||j�r t|ƒn||d�ƒS t |tjjƒ�rht|jƒ\}}t||ƒ}|jj �sFt|ƒ}tjj!||j"t| ƒd�}	t#|	| ƒS t |tjj$tjj%fƒ�rät |tjj$ƒ�r–d}
nd}
t|jƒ\}}t||ƒ}|jj �sÄt|ƒ}tjj&|
|t| ƒd�}	t#|	| ƒS t |tjj'ƒ�rüt(‚�nêt |tjj)tjj*fƒ�r td	ƒ‚�nÆt |tjj+ƒ�r|t| ƒ}|d u �rJdd
i}ddi}tjj!tjj,d|d�|j-|d�}	t#|	| ƒS |t.v �r|t.d d d… v �r d}
nd}
t| ƒ}|t.d d… v �rÚ|d u �rÐddi}ddi}n|d u �rìdd
i}ddi}tjj&|
tjj,d|d�|d�}	t#|	| ƒS t |tjj/ƒ�rÆg }g }t0|j1ƒD ]P}|| }|j2}| 3|¡ t|j4ƒ\}}t||ƒ}|j �s‚t|ƒ}| 3|¡ �q>| d u�r¦| j5�r¦d }tjj6||t| ƒd�}	t#|	| ƒS t |tjj7ƒ�rJg }t0|j1ƒD ]@}|| }t|j4ƒ\}}t||ƒ}|j �st|ƒ}| 3|¡ �qätjj8jdd|t| ƒd�}	t#|	| ƒS |t 9¡ k�rttjjdtj :¡ t| ƒd�S t |tjj;ƒ�rØt< =t>|ƒd¡\}}|d u �r°t ?| ¡ ¡}tjj,tj@jA B|¡t| ƒd�}	t#|	| ƒS td|›�ƒ‚d S )NrF   Zi64Úu32Zi32z.unrecognized Arrow DictionaryType index type: rI   rJ   r.   rL   rM   rN   r7   rK   rQ   rR   Úi8rT   rU   )Cr!   r   rV   rW   rX   rY   r   Úform_popbuffersr[   r\   r^   rŽ   r6   rr   Zuint32rs   r�   r   rk   r	   r%   r"   r*   r8   rD   ZIndexedOptionFormr:   ZIndexedFormrg   Úform_remove_optiontyperi   rm   rn   ZRegularFormro   rE   rp   rq   ZListOffsetFormrt   ru   rv   rw   rx   Z	NumpyFormrz   r{   r|   r}   r~   r   r€   r�   rƒ   Ú
RecordFormr…   Z	UnionFormr‡   Z	EmptyFormrŠ   r‹   rŒ   r�   r   ÚtypesZ	numpytypeZdtype_to_primitive)r$   r[   r^   r<   r’   r“   r_   r/   r”   r   r•   r–   r—   r9   r˜   r‚   r™   rš   Ú_r›   r   r   r   rž   é  s$   ÿ



ÿ
ÿýýú

ý



ÿ
ÿÿ
ý




ý


ý

ÿ
ý
þ
rž   Fc              
   C  s\   |rTt | t|ƒt|ƒ|d u r d n|j|d u r0d n|jt|tjjƒrH|jnd ||d�S | S d S )N)r[   r3   Z	node_typer%   r"   rƒ   Úrecord_is_scalarÚis_nonnullable_nulltype)r   Údirect_Content_subclass_namer/   r!   r8   r9   r„   Úis_tuple)r[   Zuse_extensionarrayr£   r‘   Únoder¤   r   r   r   Úto_awkwardarrow_type®  s    ÿör¨   c                 C  s<   t |  ¡ ƒD ]&}|tjjurt|tjjƒr|  S qt‚d S r   )ÚreversedÚmror8   r9   ZContentÚ
issubclassr�   )ÚclsÚbase_clsr   r   r   Úfind_first_content_subclassÇ  s    ÿ
r®   c                 C  s   | d u rd S t t| ƒƒS d S r   )r®   r�   )r§   r   r   r   Údirect_Content_subclassÑ  s    r¯   c                 C  s   t | ƒ}|d u rd S |jS d S r   )r¯   Ú__name__)r§   r   r   r   r   r¥   Ø  s    r¥   c                 C  s
   t | dƒS )Nr'   )Úhasattr©Zakarrayr   r   r   Úis_revertableà  s    r³   c                 C  s   t | jƒr|  ¡ S | jS d S r   )Úcallabler'   r²   r   r   r   rh   ä  s    
rh   c                 C  s   | j S r   r&   )Úakformr   r   r   rŸ   ë  s    rŸ   c                   sÄ  t | tjjƒrH|  ¡ }t| jƒ\}}t| |||ˆƒ}t|ƒdksDJ ‚|S t | tjj	ƒrØ‡fdd„| j
D ƒ‰tˆƒdkr~ˆd S tdd„ ˆD ƒƒrÄtdd„ ˆD ƒƒs¦J ‚ttjjˆdd	�‡fd
d„ƒS tjjˆdd	�S �nèt | tjjƒ�rˆ�r2t| jjƒdgk�r2t|  d¡ˆƒ}| j d¡j�s*t|ƒS |S �qÀd}g }	d}
d }d }| jjd u�rÒd| jjv �rÒt | jjd ¡D ]\}| ¡ \}| ¡ \}|dk�rœ|}	n2|dk�r¬|}
n"|dv �rÀd}|}n|dk�rt|}�qtd }g }t| j ƒD ]„}| j |¡}t|  |¡ˆƒ}|�r2|d u �r"|j!dd�}n||j!dd�M }|�rD|j"|	v�sL|j�s\| #t|ƒ¡ n
| #|¡ �qätj$j%|| jjt| ƒ|d�}|
�r–| &d¡S |�rÀ|d u �rÀˆ�rÀt'j(t|ƒt)j*d�}|�râ|d u �râtj$j+j,||d�S |�rtj$j-j,tj. /|¡|d|d�S |S �n²t | tjj0ƒ�rØ|  1¡  2¡ }t|ƒdk�rJt3| jˆd� 4¡ S t|ƒdk�rht|d ˆˆƒS ‡‡fdd„|D ƒ‰ tdd„ ˆ D ƒƒ�rÆtdd„ ˆ D ƒƒ�s¨J ‚ttjjˆ dd	�‡ fdd„ƒS tjjˆ dd	�S nèt | t5ƒ�r‚t | t6ƒ�r‚t| ƒdk�r‚tdd„ | D ƒƒ�r‚tdd„ | D ƒƒ�r‚g }| D ]*}t|ˆˆƒ}t|ƒdk�r.| #|¡ �q.t|ƒdk�rp|d S tjj|dd	�S n>t | t5ƒ�r®t| ƒdk�r®tj$j%g g dd�S t7d t| ƒ› �ƒ‚d S )!Nr   c                   s"   g | ]}t |ƒd krt|ˆ ƒ‘qS ©r   ©r   rc   ©Ú.0Úx)r@   r   r   Ú
<listcomp>ü  ó    z handle_arrow.<locals>.<listcomp>r
   c                 s  s   | ]}t |ƒV  qd S r   ©r³   ©r¹   Zarrr   r   r   Ú	<genexpr>   r¼   zhandle_arrow.<locals>.<genexpr>c                 s  s   | ]}t |ƒV  qd S r   r½   r¾   r   r   r   r¿     r¼   F©Z	highlevelc                     s   t jjdd„ ˆ D ƒdd�S )Nc                 S  s   g | ]}t |ƒ‘qS r   ©rh   r¸   r   r   r   r»     r¼   ú2handle_arrow.<locals>.<lambda>.<locals>.<listcomp>FrÀ   ©r8   Ú
operationsÚconcatenater   )Úlayoutsr   r   Ú<lambda>  s   ÿzhandle_arrow.<locals>.<lambda>Ú ó   ak:parametersÚoptiontype_fieldsr£   ©r+   r;   ÚByteMaskedArrayre   Tr„   rG   rS   r   r.   ©r0   r/   )Úpass_empty_fieldc                   s$   g | ]}t |ƒd krt|ˆ ˆƒ‘qS r¶   r·   )r¹   Úbatch)r@   rÎ   r   r   r»   i  s   þc                 s  s   | ]}t |ƒV  qd S r   r½   r¾   r   r   r   r¿   n  r¼   c                 s  s   | ]}t |ƒV  qd S r   r½   r¾   r   r   r   r¿   o  r¼   c                     s   t jjdd„ ˆ D ƒdd�S )Nc                 S  s   g | ]}t |ƒ‘qS r   rÁ   r¸   r   r   r   r»   v  r¼   rÂ   FrÀ   rÃ   r   )Úarraysr   r   rÇ   u  s   ÿc                 s  s   | ]}t |tjjƒV  qd S r   )r!   r   rV   ÚRecordBatchr¸   r   r   r   r¿   €  r¼   c                 s  s   | ]}t |ƒd kV  qdS )r   N)r   r¸   r   r   r   r¿   �  r¼   )r   zunrecognized Arrow type: )8r!   r   rV   ZArrayr�   r   r�   rZ   r   ZChunkedArrayÚchunksrb   r4   r*   r8   rÄ   rÅ   rÑ   ÚlistÚschemaÚnamesrc   Úcolumnr‚   rn   rh   ÚmetadataÚjsonÚloadsr—   rl   r}   Znum_columnsra   r   r€   r9   r„   Z_getitem_atr   Zzerosr6   rˆ   r+   r:   rÌ   r<   r†   ÚTableZcombine_chunksZ
to_batchesÚform_handle_arrowZlength_zero_arrayr   r   r�   )Úobjr@   rÎ   r�   r$   r[   r   ÚlayoutÚrecord_is_optiontyperÊ   r£   Úoptiontype_parametersÚrecordtype_parametersrº   ÚkeyÚvalueZrecord_maskr9   r˜   r‚   ZbatchesrÒ   rÏ   Úchunkr   )rÐ   r@   rÆ   rÎ   r   rc   ï  sø    
ÿ
þ
ÿ
þ






ÿÿþü
ÿ
üÿ
þ
þÿþ
ýüûrc   c                 C  sn  |rLt | jƒdgkrLt| jd ƒ\}}t||ƒ}|  d¡jsDt|ƒS |S �nd}g }d }d }| jd urÆd| jv rÆt	 
| jd ¡D ]D}	|	 ¡ \}
|	 ¡ \}|
dkr¦|}q€|
dv r¸d}|}q€|
dkr€|}q€g }t| jƒD ]\\}}|  |¡}t|ƒ\}}t||ƒ}|�r|j|v�s|j�s&| t|ƒ¡ qÔ| |¡ qÔtjj|t | jƒ|d	�}|�rftjjjd
|d|d�S |S d S )NrÈ   r   FrÉ   rÊ   rË   Tr„   r.   r�   rÍ   )rÓ   rÕ   r   r¡   rž   r‚   rn   rŸ   r×   rØ   rÙ   r—   rl   Ú	enumerater   r€   r8   rD   r    ZByteMaskedFormr:   )rÔ   rÎ   r$   r[   rµ   rÞ   rÊ   rß   rà   rº   rá   râ   rD   r˜   Z	arrowtyper‚   r   r   r   r   rÛ   ”  sT    




ÿÿþÿÿrÛ   c                 C  s.   t jj| dd�}|d u r|S tj||d�S d S )NF)Zextensionarray)r�   )r8   rÄ   Zto_arrowr   r>   )rÝ   r�   r   r   r   r   Úconvert_to_arrayÍ  s    rå   zM8[D]zM8[ms]ÚszM8[s]ÚmsÚuszM8[us]ÚnszM8[ns]zm8[s]zm8[ms]zm8[us]zm8[ns])T)F)FF)F)N);Ú
__future__r   rØ   Úcollections.abcr   r   Ú	functoolsr   r   Zawkwardr8   Zawkward._backends.numpyr   Zawkward._nplikes.numpyr   Zawkward._nplikes.numpy_liker   Zawkward._parametersr	   Z
extn_typesr   r   ry   r6   r   r   r   r   r    r"   r%   r*   rB   rE   rZ   rž   r¨   r®   r¯   r¥   r³   rh   rŸ   rc   rÛ   rå   rQ   Zlarge_stringÚbinaryZlarge_binaryr{   Zdate32r   Zdate64Ztime32Ztime64Ú	timestampÚdurationr‹   r   r   r   r   Ú<module>   sx   		 ÿ
Q  7 L ú

	
 &
9

üò