a
    yn„fQA  ã                   @   sž   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
ddlmZmZ ddlmZmZ e  d¡ZG dd	„ d	eƒZG d
d„ deƒZdS )é    N)Úsuppress)Úquoteé   )ÚAbstractBufferedFileÚAbstractFileSystem)Úinfer_storage_optionsÚtokenizeÚwebhdfsc                       s  e Zd ZdZee ¡ ƒZdZd=‡ fdd„	Z	e
d	d
„ ƒZdd„ Zd>dd„Zd?dd„Zedd„ ƒZedd„ ƒZedd„ ƒZdd„ Zd@dd„Zdd„ Zdd „ Zd!d"„ ZdAd#d$„Zd%d&„ Zd'd(„ Zd)d*„ ZdBd+d,„Zd-d.„ Zd/d0„ ZdCd1d2„Z d3d4„ Z!dDd5d6„Z"d7d8„ Z#d9d:„ Z$d;d<„ Z%‡  Z&S )EÚWebHDFSaà  
    Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways.

    Four auth mechanisms are supported:

    insecure: no auth is done, and the user is assumed to be whoever they
        say they are (parameter ``user``), or a predefined value such as
        "dr.who" if not given
    spnego: when kerberos authentication is enabled, auth is negotiated by
        requests_kerberos https://github.com/requests/requests-kerberos .
        This establishes a session based on existing kinit login and/or
        specified principal/password; parameters are passed with ``kerb_kwargs``
    token: uses an existing Hadoop delegation token from another secured
        service. Indeed, this client can also generate such tokens when
        not insecure. Note that tokens expire, but can be renewed (by a
        previously specified user) and may allow for proxying.
    basic-auth: used when both parameter ``user`` and parameter ``password``
        are provided.

    )r	   ZwebHDFSé–Ã  FNTc                    s  | j r
dS tƒ jf i |¤Ž |
r$dnd› d|› d|› d�| _|| _|pHi | _i | _|	pXi | _|dur†|dust|dur|tdƒ‚|| jd< || _	|| _
|dur¬|du r¾td	ƒ‚n|dur¾|| jd
< |durÐ|| jd< |rä|durätdƒ‚|| _|| _|  ¡  dt||ƒ› �| _dS )a¢  
        Parameters
        ----------
        host: str
            Name-node address
        port: int
            Port for webHDFS
        kerberos: bool
            Whether to authenticate with kerberos for this connection
        token: str or None
            If given, use this token on every call to authenticate. A user
            and user-proxy may be encoded in the token and should not be also
            given
        user: str or None
            If given, assert the user name to connect with
        password: str or None
            If given, assert the password to use for basic auth. If password
            is provided, user must be provided also
        proxy_to: str or None
            If given, the user has the authority to proxy, and this value is
            the user in who's name actions are taken
        kerb_kwargs: dict
            Any extra arguments for HTTPKerberosAuth, see
            `<https://github.com/requests/requests-kerberos/blob/master/requests_kerberos/kerberos_.py>`_
        data_proxy: dict, callable or None
            If given, map data-node addresses. This can be necessary if the
            HDFS cluster is behind a proxy, running on Docker or otherwise has
            a mismatch between the host-names given by the name-node and the
            address by which to refer to them from the client. If a dict,
            maps host names ``host->data_proxy[host]``; if a callable, full
            URLs are passed, and function must conform to
            ``url->data_proxy(url)``.
        use_https: bool
            Whether to connect to the Name-node using HTTPS instead of HTTP
        session_cert: str or Tuple[str, str] or None
            Path to a certificate file, or tuple of (cert, key) files to use
            for the requests.Session
        session_verify: str, bool or None
            Path to a certificate file to use for verifying the requests.Session.
        kwargs
        NÚhttpsÚhttpz://ú:z/webhdfs/v1z_If passing a delegation token, must not set user or proxy_to, as these are encoded in the tokenZ
delegationzQIf passing a password, the user must also beset in order to set up the basic-authz	user.nameZdoaszJIf using Kerberos auth, do not specify the user, this is handled by kinit.Zwebhdfs_)Ú_cachedÚsuperÚ__init__ÚurlÚkerbÚkerb_kwargsÚparsÚproxyÚ
ValueErrorÚuserÚpasswordÚsession_certÚsession_verifyÚ_connectr   Ú_fsid)ÚselfÚhostÚportZkerberosÚtokenr   r   Zproxy_tor   Z
data_proxyZ	use_httpsr   r   Úkwargs©Ú	__class__© úa/home/yuhw/wc/wire-cell-python/wcpy/lib/python3.9/site-packages/fsspec/implementations/webhdfs.pyr   -   sB    9 

ÿ
ÿ

ÿzWebHDFS.__init__c                 C   s   | j S ©N)r   ©r   r%   r%   r&   Úfsid’   s    zWebHDFS.fsidc                 C   s€   t  ¡ | _| jr| j| j_| j| j_| jrJddlm	} |f i | j
¤Ž| j_| jd ur|| jd ur|ddlm} || j| jƒ| j_d S )Nr   )ÚHTTPKerberosAuth)ÚHTTPBasicAuth)ÚrequestsÚSessionÚsessionr   Úcertr   Úverifyr   Zrequests_kerberosr*   r   Úauthr   r   Zrequests.authr+   )r   r*   r+   r%   r%   r&   r   –   s    


zWebHDFS._connectÚgetc              	   K   sô   |   | jt|pddd� ¡}| ¡ }| | j¡ | ¡ |d< t d||¡ | j	j
| ¡ ||||d�}	|	jdv rèz$|	 ¡ }
|
d d	 }|
d d
 }W n ttfy¨   Y n@0 |dv r¼t|ƒ‚n,|dv rÎt|ƒ‚n|dv ràt|ƒ‚nt|ƒ‚|	 ¡  |	S )NÚ z/=)ÚsafeÚopzsending %s with %s)Úmethodr   ÚparamsÚdataÚallow_redirects)i�  i‘  i“  i”  iô  ZRemoteExceptionÚmessageÚ	exception)ZIllegalArgumentExceptionZUnsupportedOperationException)ZSecurityExceptionZAccessControlException)ZFileNotFoundException)Ú_apply_proxyr   r   ÚcopyÚupdater   ÚupperÚloggerÚdebugr.   ÚrequestÚstatus_codeÚjsonr   ÚKeyErrorÚPermissionErrorÚFileNotFoundErrorÚRuntimeErrorÚraise_for_status)r   r5   r6   Úpathr8   Úredirectr"   r   ÚargsÚoutÚerrÚmsgÚexpr%   r%   r&   Ú_call¨   s8    û



zWebHDFS._callÚrbc              
   K   s$   |p| j }t| |||| j|||d�S )a^  

        Parameters
        ----------
        path: str
            File location
        mode: str
            'rb', 'wb', etc.
        block_size: int
            Client buffer size for read-ahead or write buffer
        autocommit: bool
            If False, writes to temporary file that only gets put in final
            location upon commit
        replication: int
            Number of copies of file on the cluster, write mode only
        permissions: str or int
            posix permissions, write mode only
        kwargs

        Returns
        -------
        WebHDFile instance
        )ÚmodeÚ
block_sizeÚtempdirÚ
autocommitÚreplicationÚpermissions)Ú	blocksizeÚ	WebHDFilerU   )r   rJ   rS   rT   rV   rW   rX   r"   r%   r%   r&   Ú_openÈ   s    !
øzWebHDFS._openc                 C   s    | d   ¡ | d< | d | d< | S )NÚtypeÚlengthÚsize)Úlower)Úinfor%   r%   r&   Ú_process_infoõ   s    zWebHDFS._process_infoc                 C   s   t |ƒd S )NrJ   )r   )ÚclsrJ   r%   r%   r&   Ú_strip_protocolû   s    zWebHDFS._strip_protocolc                 C   s:   t | ƒ}| dd ¡ | dd ¡ d|v r6| d¡|d< |S )NrJ   ÚprotocolÚusernamer   )r   Úpop)ZurlpathrM   r%   r%   r&   Ú_get_kwargs_from_urlsÿ   s    zWebHDFS._get_kwargs_from_urlsc                 C   s,   | j d|d�}| ¡ d }||d< |  |¡S )NZGETFILESTATUS©rJ   Ú
FileStatusÚname)rQ   rD   ra   )r   rJ   rM   r`   r%   r%   r&   r`     s    zWebHDFS.infoc                 C   sv   | j d|d�}| ¡ d d }|D ](}|  |¡ | d¡d |d  |d< q"|r`t|dd	„ d
�S tdd„ |D ƒƒS d S )NZ
LISTSTATUSrh   ZFileStatusesri   ú/Z
pathSuffixrj   c                 S   s   | d S )Nrj   r%   )Úir%   r%   r&   Ú<lambda>  ó    zWebHDFS.ls.<locals>.<lambda>)Úkeyc                 s   s   | ]}|d  V  qdS )rj   Nr%   )Ú.0r`   r%   r%   r&   Ú	<genexpr>  rn   zWebHDFS.ls.<locals>.<genexpr>)rQ   rD   ra   ÚrstripÚsorted)r   rJ   ÚdetailrM   Úinfosr`   r%   r%   r&   Úls  s    
z
WebHDFS.lsc                 C   s   | j d|d�}| ¡ d S )z8Total numbers of files, directories and bytes under pathZGETCONTENTSUMMARYrh   ZContentSummary©rQ   rD   )r   rJ   rM   r%   r%   r&   Úcontent_summary  s    zWebHDFS.content_summaryc                 C   sb   | j d|dd�}d|jv rJ|  |jd ¡}| j |¡}| ¡  | ¡ d S | ¡  | ¡ d S dS )z/Checksum info of file, giving method and resultZGETFILECHECKSUMF)rJ   rK   ÚLocationZFileChecksumN)rQ   Úheadersr<   r.   r2   rI   rD   )r   rJ   rM   ÚlocationÚout2r%   r%   r&   Úukey  s    
zWebHDFS.ukeyc                 C   s   |   d¡}| ¡ d S )zGet user's home directoryZGETHOMEDIRECTORYÚPathrw   )r   rM   r%   r%   r&   Úhome_directory*  s    
zWebHDFS.home_directoryc                 C   sB   |r| j d|d�}n
|   d¡}| ¡ d }|du r:tdƒ‚|d S )zÒRetrieve token which can give the same authority to other uses

        Parameters
        ----------
        renewer: str or None
            User who may use this token; if None, will be current user
        ZGETDELEGATIONTOKEN)ÚrenewerÚTokenNz1No token available for this user/security contextZ	urlString)rQ   rD   r   )r   r€   rM   Útr%   r%   r&   Úget_delegation_token/  s    
zWebHDFS.get_delegation_tokenc                 C   s   | j dd|d�}| ¡ d S )z/Make token live longer. Returns new expiry timeZRENEWDELEGATIONTOKENÚput©r6   r!   Úlongrw   )r   r!   rM   r%   r%   r&   Úrenew_delegation_token@  s    zWebHDFS.renew_delegation_tokenc                 C   s   | j dd|d� dS )z Stop the token from being usefulZCANCELDELEGATIONTOKENr„   r…   N©rQ   )r   r!   r%   r%   r&   Úcancel_delegation_tokenE  s    zWebHDFS.cancel_delegation_tokenc                 C   s   | j dd||d� dS )a  Set the permission at path

        Parameters
        ----------
        path: str
            location to set (file or directory)
        mod: str or int
            posix epresentation or permission, give as oct string, e.g, '777'
            or 0o777
        ZSETPERMISSIONr„   )r6   rJ   Z
permissionNrˆ   )r   rJ   Úmodr%   r%   r&   ÚchmodI  s    zWebHDFS.chmodc                 C   s>   i }|dur||d< |dur$||d< | j dd|dœ|¤Ž dS )zChange owning user and/or groupNÚownerÚgroupÚSETOWNERr„   ©r6   rJ   )rŽ   rˆ   )r   rJ   rŒ   r�   r"   r%   r%   r&   ÚchownV  s    zWebHDFS.chownc                 C   s   | j d|d|d� dS )a9  
        Set file replication factor

        Parameters
        ----------
        path: str
            File location (not for directories)
        replication: int
            Number of copies of file on the cluster. Should be smaller than
            number of data nodes; normally 3 on most systems.
        ZSETREPLICATIONr„   )rJ   r6   rW   Nrˆ   )r   rJ   rW   r%   r%   r&   Úset_replication_  s    zWebHDFS.set_replicationc                 K   s   | j dd|d� d S )NZMKDIRSr„   r�   rˆ   ©r   rJ   r"   r%   r%   r&   Úmkdirm  s    zWebHDFS.mkdirc                 C   s(   |du r|   |¡rt|ƒ‚|  |¡ d S )NF)ÚexistsÚFileExistsErrorr“   )r   rJ   Úexist_okr%   r%   r&   Úmakedirsp  s    zWebHDFS.makedirsc                 K   s   | j dd||d� d S )NZRENAMEr„   )r6   rJ   Údestinationrˆ   )r   Zpath1Zpath2r"   r%   r%   r&   Úmvu  s    z
WebHDFS.mvc                 K   s   | j dd||rdndd� d S )NÚDELETEÚdeleteÚtrueÚfalse)r6   rJ   Ú	recursiverˆ   )r   rJ   rž   r"   r%   r%   r&   Úrmx  s    
üz
WebHDFS.rmc                 K   s   |   |¡ d S r'   )rŸ   r’   r%   r%   r&   Úrm_file€  s    zWebHDFS.rm_filec                 K   sÞ   |   |¡�À}d |  |¡dt d¡› �g¡}zH|   |d¡�}t ||¡ W d   ƒ n1 s\0    Y  |  ||¡ W nF tyº   t	t
ƒ� |  |¡ W d   ƒ n1 sª0    Y  ‚ Y n0 W d   ƒ n1 sÐ0    Y  d S )Nrk   z.tmp.é   Úwb)ÚopenÚjoinÚ_parentÚsecretsZ	token_hexÚshutilÚcopyfileobjr™   ÚBaseExceptionr   rG   rŸ   )r   ZlpathZrpathr"   ZlstreamZ	tmp_fnameZrstreamr%   r%   r&   Úcp_fileƒ  s     *
(zWebHDFS.cp_filec                 C   sH   | j rt| j ƒr|   |¡}n(| j rD| j  ¡ D ]\}}| ||d¡}q,|S )Né   )r   ÚcallableÚitemsÚreplace)r   r{   ÚkÚvr%   r%   r&   r<   ‘  s    zWebHDFS._apply_proxy)r   FNNNNNNFNT)r2   NNT)rR   NTNN)F)N)NN)F)F)'Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚtempfileÚ
gettempdirrU   rd   r   Úpropertyr)   r   rQ   r[   Ústaticmethodra   Úclassmethodrc   rg   r`   rv   rx   r}   r   rƒ   r‡   r‰   r‹   r�   r‘   r“   r—   r™   rŸ   r    rª   r<   Ú__classcell__r%   r%   r#   r&   r
      sb              óe

#     ù
-





	

r
   c                       sJ   e Zd ZdZ‡ fdd„Zddd„Zdd„ Zd	d
„ Zdd„ Zdd„ Z	‡  Z
S )rZ   z"A file living in HDFS over webHDFSc                    s¢   t ƒ j||fi |¤Ž | ¡ }| dd ¡d u r:| dd ¡ | dd ¡d u rV| dd ¡ | dd¡| _| d¡}| dd¡du rž| j| _tj 	|t
t ¡ ƒ¡| _d S )NrX   rW   iÿ  rU   rV   F)r   r   r=   r2   rf   rX   rJ   ÚtargetÚosr¤   rµ   ÚuuidÚuuid4)r   ÚfsrJ   r"   rU   r#   r%   r&   r   ž  s    
zWebHDFile.__init__Fc                 C   s,   | j jj| j| j ¡ ddid�}| ¡  dS )zÔWrite one part of a multi-block file upload

        Parameters
        ==========
        final: bool
            This is the last block, so should complete file, if
            self.autocommit is True.
        úcontent-typeúapplication/octet-stream)r8   rz   T)rÀ   r.   Úpostr{   ÚbufferÚgetvaluerI   )r   ÚfinalrM   r%   r%   r&   Ú_upload_chunk«  s    	ýzWebHDFile._upload_chunkc                 C   s¼   | j  ¡ }d| jv rd\}}nd\}}d|d< | jj||| jfddi|¤Ž}| j |jd ¡}d	| jv r¸| jjj	|d
did�}| 
¡  | jjdd| jfddi|¤Ž}| j |jd ¡| _dS )zCreate remote file/uploadÚa)ÚAPPENDÚPOST)ZCREATEÚPUTrœ   Ú	overwriterK   Fry   ÚwrÁ   rÂ   )rz   rÉ   rÊ   N)r"   r=   rS   rÀ   rQ   rJ   r<   rz   r.   r„   rI   r{   )r   r"   r5   r6   rM   r{   r|   r%   r%   r&   Ú_initiate_upload¼  s    



ÿzWebHDFile._initiate_uploadc                 C   sŠ   t |dƒ}t| j|ƒ}||ks(|| jkr,dS | jjd| j||| dd�}| ¡  d|jv r€|jd }| jj 	| j 
|¡¡}|jS |jS d S )Nr   rn   ZOPENF)rJ   Úoffsetr]   rK   ry   )ÚmaxÚminr^   rÀ   rQ   rJ   rI   rz   r.   r2   r<   Úcontent)r   ÚstartÚendrM   r{   r|   r%   r%   r&   Ú_fetch_rangeÐ  s    
ÿ

zWebHDFile._fetch_rangec                 C   s   | j  | j| j¡ d S r'   )rÀ   r™   rJ   r¼   r(   r%   r%   r&   Úcommità  s    zWebHDFile.commitc                 C   s   | j  | j¡ d S r'   )rÀ   rŸ   rJ   r(   r%   r%   r&   Údiscardã  s    zWebHDFile.discard)F)r±   r²   r³   r´   r   rÇ   rÎ   rÕ   rÖ   r×   r»   r%   r%   r#   r&   rZ   ›  s   
rZ   )Úloggingr½   r¦   r§   r¶   r¾   Ú
contextlibr   Úurllib.parser   r,   Úspecr   r   Úutilsr   r   Ú	getLoggerr@   r
   rZ   r%   r%   r%   r&   Ú<module>   s    
   
