a
    yn„f·!  ã                   @   sÄ   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZmZmZmZ dd„ ZdaG d	d
„ d
eƒZedg d¢ƒG dd„ dejƒƒZG dd„ deƒZdS )é    N)Úsuppress)Úcached_propertyÚwraps)Úparse_qs)ÚAbstractFileSystem)Ú"get_package_version_without_importÚinfer_storage_optionsÚmirror_fromÚtokenizec                    s   t ˆ ƒ‡ fdd„ƒ}|S )Nc               
      sp   zˆ | i |¤ŽW S  t yj } zB|js(‚ |j^}} t|tƒrTd|v rTttj|ƒ|‚n‚ W Y d }~n
d }~0 0 d S )Nzdoes not exist)ÚOSErrorÚargsÚ
isinstanceÚstrÚFileNotFoundErrorÚerrnoÚENOENT)r   ÚkwargsÚ	exceptionÚmessage©Úfunc© ú_/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/fsspec/implementations/arrow.pyÚwrapper   s    
z wrap_exceptions.<locals>.wrapper)r   )r   r   r   r   r   Úwrap_exceptions   s    r   c                       sø   e Zd ZdZdZ‡ fdd„Zedd„ ƒZedd„ ƒZ	e
d	d
„ ƒZd-dd„Zdd„ Zdd„ Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZed.dd„ƒZed/dd „ƒZed0d!d"„ƒZed1d#d$„ƒZed%d&„ ƒZed'd(„ ƒZd2‡ fd)d*„	Z‡ fd+d,„Z‡  ZS )3ÚArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem.

    Parameters
    ----------
    fs : pyarrow.fs.FileSystem

    ú/c                    s$   t dƒa|| _tƒ jf i |¤Ž d S )NZpyarrow)r   ÚPYARROW_VERSIONÚfsÚsuperÚ__init__)Úselfr   r   ©Ú	__class__r   r   r    3   s    zArrowFSWrapper.__init__c                 C   s   | j jS ©N)r   Ú	type_name©r!   r   r   r   Úprotocol9   s    zArrowFSWrapper.protocolc                 C   s   dt | jj| jjƒ S )NZhdfs_)r
   r   ÚhostÚportr&   r   r   r   Úfsid=   s    zArrowFSWrapper.fsidc                 C   s*   t |ƒ}|d }| d¡r&|dd … }|S )NÚpathz//é   )r   Ú
startswith)Úclsr+   Úopsr   r   r   Ú_strip_protocolA   s
    
zArrowFSWrapper._strip_protocolFc                    sN   ˆ   |¡}ddlm} ‡ fdd„ˆ j ||ƒ¡D ƒ}|r<|S dd„ |D ƒS d S )Nr   )ÚFileSelectorc                    s   g | ]}ˆ   |¡‘qS r   )Ú_make_entry©Ú.0Úentryr&   r   r   Ú
<listcomp>N   s   ÿz%ArrowFSWrapper.ls.<locals>.<listcomp>c                 S   s   g | ]}|d  ‘qS )Únamer   r3   r   r   r   r6   U   ó    )r0   Ú
pyarrow.fsr1   r   Úget_file_info)r!   r+   Údetailr   r1   Úentriesr   r&   r   ÚlsJ   s    

þzArrowFSWrapper.lsc                 K   s$   |   |¡}| j |g¡\}|  |¡S r$   )r0   r   r:   r2   )r!   r+   r   Úinfor   r   r   r>   W   s    
zArrowFSWrapper.infoc                 C   s6   |   |¡}z|  |¡ W n ty,   Y dS 0 dS d S )NFT)r0   r>   r   ©r!   r+   r   r   r   Úexists\   s    
zArrowFSWrapper.existsc                 C   sn   ddl m} |j|ju rd}n<|j|ju r0d}n*|j|ju rVttjt	 
tj¡|jƒ‚nd}|j|j||jdœS )Nr   )ÚFileTypeÚ	directoryÚfileÚother)r7   ÚsizeÚtypeÚmtime)r9   rA   rF   Z	DirectoryÚFileZNotFoundr   r   r   ÚosÚstrerrorr+   rE   rG   )r!   r>   rA   Úkindr   r   r   r2   e   s    üzArrowFSWrapper._make_entryc                 K   sø   |   |¡ d¡}|   |¡ d¡}|  |d¡�¸}|› dt d¡› �}zJ|  |d¡�}t ||¡ W d   ƒ n1 sr0    Y  | j 	||¡ W nH t
yÔ   ttƒ� | j |¡ W d   ƒ n1 sÄ0    Y  ‚ Y n0 W d   ƒ n1 sê0    Y  d S )Nr   Úrbz.tmp.é   Úwb)r0   ÚrstripÚ_openÚsecretsZ	token_hexÚopenÚshutilÚcopyfileobjr   ÚmoveÚBaseExceptionr   r   Údelete_file)r!   Úpath1Úpath2r   ZlstreamZ	tmp_fnameZrstreamr   r   r   Úcp_filex   s    *
*zArrowFSWrapper.cp_filec                 K   s2   |   |¡ d¡}|   |¡ d¡}| j ||¡ d S )Nr   )r0   rO   r   rU   )r!   rX   rY   r   r   r   r   Úmvˆ   s    zArrowFSWrapper.mvc                 C   s   |   |¡}| j |¡ d S r$   )r0   r   rW   r?   r   r   r   Úrm_fileŽ   s    
zArrowFSWrapper.rm_fileNc                 C   sF   |   |¡ d¡}|  |¡r6|r,| j |¡ qBtdƒ‚n| j |¡ d S )Nr   z0Can't delete directories without recursive=False)r0   rO   Úisdirr   Ú
delete_dirÚ
ValueErrorrW   )r!   r+   Ú	recursiveZmaxdepthr   r   r   Úrm“   s    

zArrowFSWrapper.rmrL   Tc           	      K   s¨   |dkr |r| j j}qR| j j}n2|dkr2| j j}n |dkrD| j j}ntd|›�ƒ‚i }|dksb|s€tt d¡d ƒdkr€d |d< ||fi |¤Ž}t	| ||||fi |¤ŽS )	NrL   rN   Úabz'unsupported mode for Arrow filesystem: Ú.r   é   Úcompression)
r   Zopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr_   Úintr   ÚsplitÚ	ArrowFile)	r!   r+   ÚmodeÚ
block_sizeÚseekabler   ÚmethodÚ_kwargsÚstreamr   r   r   rP   ž   s    



zArrowFSWrapper._openc                 K   s2   |   |¡}|r| j|dd� n| jj|dd� d S )NT)Úexist_okF©r`   )r0   Úmakedirsr   Ú
create_dir)r!   r+   Zcreate_parentsr   r   r   r   Úmkdirµ   s    
zArrowFSWrapper.mkdirc                 C   s   |   |¡}| jj|dd� d S )NTrp   )r0   r   rr   )r!   r+   ro   r   r   r   rq   ½   s    
zArrowFSWrapper.makedirsc                 C   s   |   |¡}| j |¡ d S r$   )r0   r   r^   r?   r   r   r   ÚrmdirÂ   s    
zArrowFSWrapper.rmdirc                 C   s   |   |¡}| j |¡jS r$   )r0   r   r:   rG   r?   r   r   r   ÚmodifiedÇ   s    
zArrowFSWrapper.modifiedc                    s&   |dv|d< t ƒ j|fd d dœ|¤ŽS )N)Nr   rk   )ÚstartÚend)r   Úcat_file)r!   r+   rv   rw   r   r"   r   r   rx   Ì   s    zArrowFSWrapper.cat_filec                    s"   d|d< t ƒ j||fi |¤Ž d S )NFrk   )r   Úget_file)r!   ZrpathZlpathr   r"   r   r   ry   Ð   s    zArrowFSWrapper.get_file)F)FN)rL   NT)T)F)NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Zroot_markerr    Úpropertyr'   r   r*   Úclassmethodr0   r=   r>   r@   r2   r   rZ   r[   r\   ra   rP   rs   rq   rt   ru   rx   ry   Ú__classcell__r   r   r"   r   r   (   sB   



	





r   rn   )	ÚreadÚseekÚtellÚwriteÚreadableÚwritableÚcloserE   rk   c                   @   s&   e Zd Zddd„Zdd„ Zdd„ ZdS )	rh   Nc                 K   s.   || _ || _|| _|| _| | _| _|| _d S r$   )r+   ri   r   rn   Ú	blocksizerj   r   )r!   r   rn   r+   ri   rj   r   r   r   r   r    ä   s    zArrowFile.__init__c                 C   s   | S r$   r   r&   r   r   r   Ú	__enter__î   s    zArrowFile.__enter__c                 G   s   |   ¡ S r$   )r‡   )r!   r   r   r   r   Ú__exit__ñ   s    zArrowFile.__exit__)N)rz   r{   r|   r    r‰   rŠ   r   r   r   r   rh   Õ   s   

rh   c                       s2   e Zd ZdZdZd‡ fdd„	Zed	d
„ ƒZ‡  ZS )ÚHadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem
    to connect it's interface with fsspecZhdfsÚdefaultr   Né   c           
         s:   ddl m} |||||||d�}	tƒ jf d|	i|¤Ž dS )a;  

        Parameters
        ----------
        host: str
            Hostname, IP or "default" to try to read from Hadoop config
        port: int
            Port to connect on, or default from Hadoop config if 0
        user: str or None
            If given, connect as this username
        kerb_ticket: str or None
            If given, use this ticket for authentication
        replication: int
            set replication factor of file for write operations. default value is 3.
        extra_conf: None or dict
            Passed on to HadoopFileSystem
        r   )r‹   )r(   r)   ÚuserÚkerb_ticketÚreplicationÚ
extra_confr   N)r9   r‹   r   r    )
r!   r(   r)   rŽ   r�   r�   r‘   r   r‹   r   r"   r   r   r    û   s    úzHadoopFileSystem.__init__c                 C   s�   t | ƒ}i }| dd ¡r$|d |d< | dd ¡r<|d |d< | dd ¡rT|d |d< | dd ¡rŒt|d ƒ}| dd ¡rŒt|d d ƒ|d< |S )Nr(   ÚusernamerŽ   r)   Z	url_queryr�   r   )r   Úgetr   rf   )r+   r/   ÚoutZqueriesr   r   r   Ú_get_kwargs_from_urls"  s    z&HadoopFileSystem._get_kwargs_from_urls)rŒ   r   NNr�   N)	rz   r{   r|   r}   r'   r    Ústaticmethodr•   r€   r   r   r"   r   r‹   õ   s         ù'r‹   )r   ÚiorI   rQ   rS   Ú
contextlibr   Ú	functoolsr   r   Úurllib.parser   Zfsspec.specr   Zfsspec.utilsr   r   r	   r
   r   r   r   ÚIOBaserh   r‹   r   r   r   r   Ú<module>   s&    .þ