3
Êb_�k  ã               @   s˜   d Z ddlmZ ddlmZ ddlZddlZddlmZ ddl	Z
ddlmZ ddlmZ ddljjjZG dd	„ d	ƒZG d
d„ dƒZG dd„ dejƒZdS )a�  
Read SAS7BDAT files

Based on code written by Jared Hobbs:
  https://bitbucket.org/jaredhobbs/sas7bdat

See also:
  https://github.com/BioStatMatt/sas7bdat

Partial documentation of the file format:
  https://cran.r-project.org/package=sas7bdat/vignettes/sas7bdat.pdf

Reference for binary data compression:
  http://collaboration.cmc.ec.gc.ca/science/rpn/biblio/ddj/Website/articles/CUJ/1992/9210/ross/ross.htm
é    )Úabc)ÚdatetimeN)ÚEmptyDataError)Úget_filepath_or_buffer)ÚParserc               @   s   e Zd ZdS )Ú_subheader_pointerN)Ú__name__Ú
__module__Ú__qualname__© r   r   ú8/tmp/pip-build-jkwsxyiy/pandas/pandas/io/sas/sas7bdat.pyr      s   r   c               @   s   e Zd ZdS )Ú_columnN)r   r	   r
   r   r   r   r   r   #   s   r   c               @   sü   e Zd ZdZd>dd„Zdd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*d+„ Zd,d-„ Zd.d/„ Zd0d1„ Zd2d3„ Zd4d5„ Zd6d7„ Zd?d8d9„Zd:d;„ Zd<d=„ Z dS )@ÚSAS7BDATReadera!  
    Read SAS files in SAS7BDAT format.

    Parameters
    ----------
    path_or_buf : path name or buffer
        Name of SAS file or file-like object pointing to SAS file
        contents.
    index : column identifier, defaults to None
        Column to use as index.
    convert_dates : boolean, defaults to True
        Attempt to convert dates to Pandas datetime values.  Note that
        some rarely used SAS date formats may be unsupported.
    blank_missing : boolean, defaults to True
        Convert empty strings to missing values (SAS uses blanks to
        indicate missing character variables).
    chunksize : int, defaults to None
        Return SAS7BDATReader object for iterations, returns chunks
        with given number of lines.
    encoding : string, defaults to None
        String encoding.
    convert_text : bool, defaults to True
        If False, text variables are left as raw bytes.
    convert_header_text : bool, defaults to True
        If False, header text, including column names, are left as raw
        bytes.
    NTc	       
      C   sÆ   || _ || _|| _|| _|| _|| _|| _d| _d| _g | _	g | _
g | _g | _g | _d | _g | _g | _g | _d| _d| _d| _t|ƒ\| _}	}	}	t| jtƒr²t| jdƒ| _| j| _| jƒ  | jƒ  d S )Nzlatin-1Ú r   Úrb)ÚindexÚconvert_datesÚblank_missingÚ	chunksizeÚencodingÚconvert_textÚconvert_header_textÚdefault_encodingÚcompressionÚcolumn_names_stringsÚcolumn_namesÚcolumn_formatsÚcolumnsÚ%_current_page_data_subheader_pointersÚ_cached_pageÚ_column_data_lengthsÚ_column_data_offsetsÚ_column_typesÚ_current_row_in_file_indexZ_current_row_on_page_indexr   Ú_path_or_bufÚ
isinstanceÚstrÚopenÚhandleÚ_get_propertiesÚ_parse_metadata)
ÚselfZpath_or_bufr   r   r   r   r   r   r   Ú_r   r   r   Ú__init__E   s6    zSAS7BDATReader.__init__c             C   s   t j| jt jd�S )z5Return a numpy int64 array of the column data lengths)Údtype)ÚnpÚasarrayr    Úint64)r+   r   r   r   Úcolumn_data_lengthsr   s    z"SAS7BDATReader.column_data_lengthsc             C   s   t j| jt jd�S )z0Return a numpy int64 array of the column offsets)r.   )r/   r0   r!   r1   )r+   r   r   r   Úcolumn_data_offsetsv   s    z"SAS7BDATReader.column_data_offsetsc             C   s   t j| jt jdƒd�S )zXReturns a numpy character array of the column types:
           s (string) or d (double)ZS1)r.   )r/   r0   r"   r.   )r+   r   r   r   Úcolumn_typesz   s    zSAS7BDATReader.column_typesc             C   s(   y| j jƒ  W n tk
r"   Y nX d S )N)r(   ÚcloseÚAttributeError)r+   r   r   r   r5      s    zSAS7BDATReader.closec             C   s
  | j jdƒ | j jdƒ| _| jdttjƒ… tjkrD| jƒ  tdƒ‚d\}}| j	tj
tjƒ}|tjkrŠtj}d| _d| _tj| _tj| _nd| _tj| _tj| _d| _| j	tjtjƒ}|tjkrÆtj}|| }| j	tjtjƒ}|dkrîd	| _nd
| _| j	tjtjƒd }|tjk�r"tj| | _nd|› d�| _| j	tj tj!ƒ}|dk�rRd| _"n|dk�rdd| _"nd| _"| j	tj#tj$ƒ}|j%dƒ| _&| j'�r¦| j&j(| j)�p | j*ƒ| _&| j	tj+tj,ƒ}|j%dƒ| _-| j'�râ| j-j(| j)�pÜ| j*ƒ| _-t.dddƒ}| j/tj0| tj1ƒ}|t2j3|dd� | _4| j/tj5| tj6ƒ}|t2j3|dd� | _7| j8tj9| tj:ƒ| _;| j j| j;d ƒ}|  j|7  _t| jƒ| j;k�r–| jƒ  tdƒ‚| j8tj<| tj=ƒ| _>| j8tj?| tj@ƒ| _A| j	tjB| tjCƒ}|j%dƒ| _D| j'�r| jDj(| j)�pü| j*ƒ| _D| j	tjE| tjFƒ}|j%dƒ| _G| j'�rB| jGj(| j)�p<| j*ƒ| _G| j	tjH| tjIƒ}|j%dƒ| _J| j'�r‚| jJj(| j)�p|| j*ƒ| _J| j	tjK| tjLƒ}|j%dƒ}t|ƒdk�rÆ|j(| j)�p¾| j*ƒ| _Mn@| j	tjN| tjOƒ}|j%dƒ| _M| j'�r| jMj(| j)�p | j*ƒ| _Md S )Nr   i   z'magic number mismatch (not a SAS file?)Té   Fé   ó   ú<ú>zunknown (code=ú)ó   1Úunixó   2ZwindowsÚunknowns     i¨  é   Ús)Úunitz*The SAS7BDAT file appears to be truncated.)r   r   )Pr$   ÚseekÚreadr   ÚlenÚconstÚmagicr5   Ú
ValueErrorÚ_read_bytesZalign_1_offsetZalign_1_lengthZu64_byte_checker_valueZalign_2_valueÚU64Ú_int_lengthZpage_bit_offset_x64Ú_page_bit_offsetZsubheader_pointer_length_x64Ú_subheader_pointer_lengthZpage_bit_offset_x86Zsubheader_pointer_length_x86Zalign_2_offsetZalign_2_lengthZalign_1_checker_valueZendianness_offsetZendianness_lengthÚ
byte_orderZencoding_offsetZencoding_lengthZencoding_namesÚfile_encodingZplatform_offsetZplatform_lengthÚplatformZdataset_offsetZdataset_lengthÚrstripÚnamer   Údecoder   r   Zfile_type_offsetZfile_type_lengthÚ	file_typer   Ú_read_floatZdate_created_offsetZdate_created_lengthÚpdZto_timedeltaZdate_createdZdate_modified_offsetZdate_modified_lengthZdate_modifiedÚ	_read_intZheader_size_offsetZheader_size_lengthÚheader_lengthZpage_size_offsetZpage_size_lengthÚ_page_lengthZpage_count_offsetZpage_count_lengthZ_page_countZsas_release_offsetZsas_release_lengthZsas_releaseZsas_server_type_offsetZsas_server_type_lengthZserver_typeZos_version_number_offsetZos_version_number_lengthÚ
os_versionZos_name_offsetZos_name_lengthÚos_nameZos_maker_offsetZos_maker_length)r+   Zalign1Zalign2ÚbufZtotal_alignÚepochÚxr   r   r   r)   …   s´    





zSAS7BDATReader._get_propertiesc             C   s"   | j | jpdd�}|d krt‚|S )NrA   )Únrows)rE   r   ÚStopIteration)r+   Údar   r   r   Ú__next__
  s    zSAS7BDATReader.__next__c             C   sJ   |dkr| j ƒ  tdƒ‚| j||ƒ}|dkr0dnd}tj| j| |ƒd S )Nr8   r7   zinvalid float widthÚfÚdr   )r8   r7   )r5   rI   rJ   ÚstructÚunpackrO   )r+   ÚoffsetÚwidthr]   Úfdr   r   r   rV     s    zSAS7BDATReader._read_floatc             C   sP   |dkr| j ƒ  tdƒ‚| j||ƒ}dddd	d
œ| }tj| j| |ƒd }|S )NrA   é   r8   r7   zinvalid int widthÚbÚhÚlÚq)rA   rk   r8   r7   r   )rA   rk   r8   r7   )r5   rI   rJ   rf   rg   rO   )r+   rh   ri   r]   ÚitZivr   r   r   rX     s    zSAS7BDATReader._read_intc             C   s�   | j d krX| jj|ƒ | jj|ƒ}t|ƒ|k rT| jƒ  d|d›d|d›d�}t|ƒ‚|S || t| j ƒkrz| jƒ  tdƒ‚| j ||| … S d S )NzUnable to read re   z bytes from file position Ú.zThe cached page is too small.)r   r$   rD   rE   rF   r5   rI   )r+   rh   Úlengthr]   Úmsgr   r   r   rJ   #  s    
zSAS7BDATReader._read_bytesc             C   sZ   d}xP|sT| j j| jƒ| _t| jƒdkr*P t| jƒ| jkrJ| jƒ  tdƒ‚| jƒ }qW d S )NFr   z2Failed to read a meta data page from the SAS file.)r$   rE   rZ   r   rF   r5   rI   Ú_process_page_meta)r+   Údoner   r   r   r*   2  s    zSAS7BDATReader._parse_metadatac             C   sV   | j ƒ  tjtjgtj }| j|kr,| jƒ  | jtj@ }| jtjk}|pT|pT| jg kS )N)	Ú_read_page_headerrG   Úpage_meta_typeZpage_amd_typeÚpage_mix_typesÚ_current_page_typeÚ_process_page_metadataÚpage_data_typer   )r+   ÚptÚis_data_pageZis_mix_pager   r   r   rt   =  s    
z!SAS7BDATReader._process_page_metac             C   sX   | j }tj| }| j|tjƒ| _tj| }| j|tjƒ| _tj	| }| j|tj
ƒ| _d S )N)rM   rG   Zpage_type_offsetrX   Zpage_type_lengthry   Zblock_count_offsetZblock_count_lengthZ_current_page_block_countZsubheader_count_offsetZsubheader_count_lengthÚ_current_page_subheaders_count)r+   Ú
bit_offsetZtxr   r   r   rv   J  s    


z SAS7BDATReader._read_page_headerc             C   st   | j }xht| jƒD ]Z}| jtj| |ƒ}|jdkr4q|jtjkrBq| j	|j
ƒ}| j||j|jƒ}| j||ƒ qW d S )Nr   )rM   Úranger~   Ú_process_subheader_pointersrG   Zsubheader_pointers_offsetrr   r   Ztruncated_subheader_idÚ_read_subheader_signaturerh   Ú_get_subheader_indexÚptypeÚ_process_subheader)r+   r   ÚiÚpointerÚsubheader_signatureÚsubheader_indexr   r   r   rz   U  s    
z%SAS7BDATReader._process_page_metadatac             C   s`   t jj|ƒ}|d kr\|t jkp$|dk}|t jk}| jdkrL|rL|rLt jj}n| jƒ  t	dƒ‚|S )Nr   r   zUnknown subheader signature)
rG   Zsubheader_signature_to_indexÚgetZcompressed_subheader_idZcompressed_subheader_typer   ÚSASIndexÚdata_subheader_indexr5   rI   )r+   Ú	signaturer   r„   r   Úf1Úf2r   r   r   rƒ   f  s    

z#SAS7BDATReader._get_subheader_indexc       
      C   s„   | j }|||  }| j|| jƒ}|| j7 }| j|| jƒ}|| j7 }| j|dƒ}|d7 }| j|dƒ}tƒ }	||	_||	_||	_||	_|	S )NrA   )rN   rX   rL   r   rh   rr   r   r„   )
r+   rh   Zsubheader_pointer_indexZsubheader_pointer_lengthZtotal_offsetZsubheader_offsetZsubheader_lengthZsubheader_compressionZsubheader_typer_   r   r   r   r�   r  s    

z*SAS7BDATReader._process_subheader_pointersc             C   s   | j || jƒ}|S )N)rJ   rL   )r+   rh   rˆ   r   r   r   r‚   Š  s    z(SAS7BDATReader._read_subheader_signaturec             C   sÞ   |j }|j}|tjjkr | j}n°|tjjkr4| j}nœ|tjjkrH| j	}nˆ|tjj
kr\| j}nt|tjjkrp| j}n`|tjjkr„| j}nL|tjjkr˜| j}n8|tjjkr¬| j}n$|tjjkrÈ| jj|ƒ d S tdƒ‚|||ƒ d S )Nzunknown subheader index)rh   rr   rG   r‹   Zrow_size_indexÚ_process_rowsize_subheaderZcolumn_size_indexÚ_process_columnsize_subheaderZcolumn_text_indexÚ_process_columntext_subheaderZcolumn_name_indexÚ_process_columnname_subheaderZcolumn_attributes_indexÚ#_process_columnattributes_subheaderZformat_and_label_indexÚ_process_format_subheaderZcolumn_list_indexÚ_process_columnlist_subheaderZsubheader_counts_indexÚ_process_subheader_countsrŒ   r   ÚappendrI   )r+   r‰   r‡   rh   rr   Ú	processorr   r   r   r…   Ž  s.    z!SAS7BDATReader._process_subheaderc             C   sÒ   | j }|}|}| jr&|d7 }|d7 }n|d7 }|d7 }| j|tj|  |ƒ| _| j|tj|  |ƒ| _| j|tj|  |ƒ| _	| j|tj
|  |ƒ| _tj| }| j|| |ƒ| _| j|dƒ| _| j|dƒ| _d S )Niª  iÂ  ib  iz  rk   )rL   rK   rX   rG   Zrow_length_offset_multiplierZ
row_lengthZrow_count_offset_multiplierÚ	row_countZcol_count_p1_multiplierÚcol_count_p1Zcol_count_p2_multiplierÚcol_count_p2Z'row_count_on_mix_page_offset_multiplierZ_mix_page_row_countÚ_lcsÚ_lcp)r+   rh   rr   Úint_lenZ
lcs_offsetZ
lcp_offsetZmxr   r   r   r�   ª  s(    

z)SAS7BDATReader._process_rowsize_subheaderc             C   sT   | j }||7 }| j||ƒ| _| j| j | jkrPtd| j› d| j› d| j› d�ƒ d S )Nz Warning: column count mismatch (z + z != z)
)rL   rX   Úcolumn_countr›   rœ   Úprint)r+   rh   rr   rŸ   r   r   r   r‘   Ç  s    z,SAS7BDATReader._process_columnsize_subheaderc             C   s   d S )Nr   )r+   rh   rr   r   r   r   r—   Ó  s    z(SAS7BDATReader._process_subheader_countsc       
      C   sÎ  || j 7 }| j|tjƒ}| j||ƒ}|d|… jdƒ}|}| jrR|j| jpN| j	ƒ}| j
j|ƒ t| j
ƒdk�rÊd}xtjD ]}||krz|}qzW || _|| j 8 }|d }	| jr´|	d7 }	| j|	| jƒ}|jdƒ}|dk�rd| _|d }	| jrò|	d7 }	| j|	| jƒ}|d| j… | _nŒ|tjk�rV|d	 }	| j�r6|	d7 }	| j|	| jƒ}|d| j… | _nH| jdk�ržd| _|d }	| j�r€|	d7 }	| j|	| jƒ}|d| j… | _| j�rÊt| d
ƒ�rÊ| jj| j�pÄ| j	ƒ| _d S )Nr   s     rA   r   é   r8   ó    é    é(   Úcreator_proc)rL   rX   rG   Ztext_block_size_lengthrJ   rR   r   rT   r   r   r   r˜   rF   Zcompression_literalsr   rK   rž   r�   r¦   Zrle_compressionÚhasattr)
r+   rh   rr   Ztext_block_sizer]   Z	cname_rawÚcnameZcompression_literalZclZoffset1r   r   r   r’   Ö  sX    



z,SAS7BDATReader._process_columntext_subheaderc             C   sÌ   | j }||7 }|d|  d d }x¤t|ƒD ]˜}|tj|d   tj }|tj|d   tj }|tj|d   tj }| j|tjƒ}	| j|tj	ƒ}
| j|tj
ƒ}| j|	 }| jj||
|
| … ƒ q,W d S )Nrk   é   r7   rA   )rL   r€   rG   Zcolumn_name_pointer_lengthZ!column_name_text_subheader_offsetZcolumn_name_offset_offsetZcolumn_name_length_offsetrX   Z!column_name_text_subheader_lengthZcolumn_name_offset_lengthZcolumn_name_length_lengthr   r   r˜   )r+   rh   rr   rŸ   Zcolumn_name_pointers_countr†   Ztext_subheaderZcol_name_offsetZcol_name_lengthÚidxÚ
col_offsetZcol_lenZname_strr   r   r   r“   
  s    


z,SAS7BDATReader._process_columnname_subheaderc       
      C   sâ   | j }|d|  d |d  }x¾t|ƒD ]²}|| tj ||d   }|d|  tj ||d   }|d|  tj ||d   }| j||ƒ}	| jj|	ƒ | j|tj	ƒ}	| j
j|	ƒ | j|tjƒ}	| jj|	dkrÔdndƒ q(W d S )Nrk   r©   r7   rA   ó   dó   s)rL   r€   rG   Zcolumn_data_offset_offsetZcolumn_data_length_offsetZcolumn_type_offsetrX   r!   r˜   Zcolumn_data_length_lengthr    Zcolumn_type_lengthr"   )
r+   rh   rr   rŸ   Zcolumn_attributes_vectors_countr†   Zcol_data_offsetZcol_data_lenZ	col_typesr_   r   r   r   r”   *  s    z2SAS7BDATReader._process_columnattributes_subheaderc             C   s   d S )Nr   )r+   rh   rr   r   r   r   r–   D  s    z,SAS7BDATReader._process_columnlist_subheaderc             C   s„  | j }|tj d|  }|tj d|  }|tj d|  }|tj d|  }|tj d|  }|tj d|  }	| j|tj	ƒ}
t
|
t| jƒd ƒ}| j|tjƒ}| j|tjƒ}| j|tjƒ}t
|t| jƒd ƒ}| j|tjƒ}| j|	tjƒ}| j| }|||| … }| j| }|||| … }t| jƒ}tƒ }||_| j| |_||_||_| j| |_| j| |_| jj|ƒ | jj|ƒ d S )Né   rA   )rL   rG   Z)column_format_text_subheader_index_offsetZcolumn_format_offset_offsetZcolumn_format_length_offsetZ(column_label_text_subheader_index_offsetZcolumn_label_offset_offsetZcolumn_label_length_offsetrX   Z)column_format_text_subheader_index_lengthÚminrF   r   Zcolumn_format_offset_lengthZcolumn_format_length_lengthZ(column_label_text_subheader_index_lengthZcolumn_label_offset_lengthZcolumn_label_length_lengthr   r   Zcol_idr   rS   ÚlabelÚformatr"   Úctyper    rr   r   r˜   )r+   rh   rr   rŸ   Ztext_subheader_formatZcol_format_offsetZcol_format_lenZtext_subheader_labelZcol_label_offsetZcol_label_lenr_   Z
format_idxZformat_startZ
format_lenZ	label_idxZlabel_startZ	label_lenZlabel_namesZcolumn_labelZformat_namesZcolumn_formatZcurrent_column_numberÚcolr   r   r   r•   H  s@    





z(SAS7BDATReader._process_format_subheaderc             C   sð   |d kr| j d k	r| j }n|d kr(| j}t| jƒdkrF| jƒ  tdƒ‚| j| jkrVd S | j| j }||krn|}| jjdƒ}| jjdƒ}tj	||ftj
d�| _tj|d| ftjd�| _d| _t| ƒ}|j|ƒ | jƒ }| jd k	rì|j| jƒ}|S )Nr   zNo columns to parse from filer¬   r­   )r.   r7   )r   rš   rF   r"   r5   r   r#   Úcountr/   ÚemptyÚobjectÚ_string_chunkÚzerosZuint8Ú_byte_chunkÚ_current_row_in_chunk_indexr   rE   Ú_chunk_to_dataframer   Z	set_index)r+   r`   ÚmZndÚnsÚpÚrsltr   r   r   rE   x  s.    

zSAS7BDATReader.readc             C   sº   g | _ | jj| jƒ| _t| jƒdkr(dS t| jƒ| jkrf| jƒ  dt| jƒd›d| jd›d�}t|ƒ‚| jƒ  | j	}|t
jkr†| jƒ  |t
j@ }t
jgt
j }| r¶| j	|kr¶| jƒ S dS )Nr   Tz-failed to read complete page from file (read re   z of z bytes)F)r   r$   rE   rZ   r   rF   r5   rI   rv   ry   rG   rw   rz   r{   rx   Ú_read_next_page)r+   rs   Z	page_typer}   r|   r   r   r   rÀ   š  s"    

zSAS7BDATReader._read_next_pagec             C   s   | j }| j}t|| |ƒ}tj|d�}d\}}�xjt| jƒD �]Z}| j| }| j| dkrð| j|d d …f j	| j
d d�||< tj|| tjd�||< | jræd }	| j| tjkr¶d}	n| j| tjkrÊd}	|	rætj|| |	dd�||< |d	7 }q<| j| d
k�r|| j|d d …f ||< | j�rH| jd k	�rH|| jj| j�p@| jƒ||< | j�rr|| jjƒ dk}
tj|j|
|f< |d	7 }q<| jƒ  td| j| › �ƒ‚q<W |S )N)r   r   r¬   re   )r.   rB   z
1960-01-01)rC   ÚoriginrA   r­   zunknown column type )r   r   ) rº   r#   r€   rW   Z	DataFramer    r   r"   r¹   ÚviewrO   r/   r0   Zfloat64r   r   rG   Zsas_date_formatsZsas_datetime_formatsZto_datetimer·   r   r   r&   rT   r   r   rF   ÚnanÚlocr5   rI   )r+   Únr¼   Úixr¿   ZjsZjbÚjrS   rC   Úiir   r   r   r»   ´  s@    
$


z"SAS7BDATReader._chunk_to_dataframe)NTTNNTT)N)!r   r	   r
   Ú__doc__r-   r2   r3   r4   r5   r)   rc   rV   rX   rJ   r*   rt   rv   rz   rƒ   r�   r‚   r…   r�   r‘   r—   r’   r“   r”   r–   r•   rE   rÀ   r»   r   r   r   r   r   (   sJ         
$ 		4 0
"r   )rÉ   Úcollectionsr   r   rf   Znumpyr/   Zpandas.errorsr   ZpandasrW   Zpandas.io.commonr   Zpandas.io.sas._sasr   Zpandas.io.sas.sas_constantsÚioZsasZsas_constantsrG   r   r   ÚIteratorr   r   r   r   r   Ú<module>   s   