3
®ybh8  ã            	   @   s<  d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	m
Z
 ddlZddlZddlmZmZmZ ddljjZddlZddlmZ ddljZddlmZmZ eƒ Zejdd�d	d
„ ƒZ dPdd„Z!dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd„ Z&dd„ Z'dd„ Z(dd„ Z)ejej*j+ej*j,e-ƒdddd ƒƒZ.eje.d!�d"d„ ƒZ/d#d$„ Z0d%d&„ Z1d'd(„ Z2d)d*„ Z3d+d,„ Z4ejd-d.„ ƒZ5G d/d0„ d0ƒZ6G d1d2„ d2ƒZ7ej8j9d3d4gd5ejd6d7d8gƒfd9ej:j;d6d7d8gd:d;d<ggƒfgƒd=d>„ ƒZ<d?d@„ Z=ej8j9dAdBdCdDgƒdEdF„ ƒZ>ej?dGƒdHdI„ ƒZ@G dJdK„ dKejAƒZBdLdM„ ZCdNdO„ ZDdS )Qa<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
é    N)Úcatch_warningsÚsimplefilter)Ú_get_lzma_fileÚ_import_lzmaÚis_platform_little_endian)ÚIndex)ÚDayÚMonthEndÚmodule)Úscopec              C   s   ddl m}  | ƒ S )Nr   )Úcreate_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr   )r   © r   ú?/tmp/pip-install-4cy230dp/pandas/pandas/tests/io/test_pickle.pyÚcurrent_pickle_data%   s    r   c             C   sš   t |tƒrtj|| ƒ d S |jdƒr6tj}|| |ƒ n`|dkrv|tjkrX| tjkstt‚q–| |ksdt‚| j	|j	ks–t‚n t
td|› d�tjƒ}|| |ƒ d S )NZsp_Ú	timestampZassert_Z_equal)Ú
isinstancer   ÚtmÚassert_index_equalÚ
startswithZassert_equalÚpdZNaTÚAssertionErrorÚfreqÚgetattrZassert_almost_equal)ÚresultÚexpectedÚtypÚversionÚ
comparatorr   r   r   Úcompare_element0   s    


r   c       
      C   sz   t j|ƒ} tƒ }xd| jƒ D ]X\}}xN|jƒ D ]B\}}| | | }d|› d|› �}	|j|	|d ƒ}	|	||||ƒ q,W qW | S )NZcompare_Ú_r   )r   Úread_pickleÚglobalsÚitemsÚget)
ÚdataZvfr   Úmr   ÚdvÚdtr   r   r   r   r   r   ÚcompareC   s    
r(   c             C   sÔ   t j| |ƒ | jj|jjks t‚| jjj s0t‚t j| dk|dkƒ | jj}|tdƒ tdƒksdt‚|tjdd� }t	|tjƒs„t‚|tjddd�ksšt‚|tjdd� }t	|tjƒsºt‚|tjddd�ksÐt‚d S )Nr   é   é   )Úhours)Údaysr+   )Únanoseconds)r,   r-   )
r   Úassert_series_equalÚindexr   r   Ú	normalizer   r   Z	Timedeltar   )r   r   r   r   r   Úresr   r   r   Úcompare_series_tsU   s    r2   c             C   s   t j| |ƒ d S )N)r   r.   )r   r   r   r   r   r   r   Úcompare_series_dt_tzi   s    r3   c             C   s   t j| |ƒ d S )N)r   r.   )r   r   r   r   r   r   r   Úcompare_series_catm   s    r4   c             C   s   t j| |ƒ d S )N)r   Úassert_frame_equal)r   r   r   r   r   r   r   Úcompare_frame_dt_mixed_tzsq   s    r6   c             C   s   t j| |ƒ d S )N)r   r5   )r   r   r   r   r   r   r   Úcompare_frame_cat_onecolu   s    r7   c             C   s   t | |||ƒ d S )N)r7   )r   r   r   r   r   r   r   Úcompare_frame_cat_and_floaty   s    r8   c             C   sV   t j| |ƒ t| jtƒst‚| jtƒ ks,t‚| jdks:t‚t j| jdƒ|jdƒƒ d S )NÚMr*   )r   r   r   r   r	   r   ZfreqstrÚshift)r   r   r   r   r   r   r   Úcompare_index_period}   s
    r;   r$   Úlegacy_pickleÚ*z*.pickle)Úparamsc             C   s
   || j ƒS )N)Úparam)ÚrequestÚdatapathr   r   r   r<   Š   s    c             C   sR   t ƒ stjdƒ tjjtjj|ƒƒ}tdd�� tdƒ t	| ||ƒ W d Q R X d S )Nz"known failure on non-little endianT)ÚrecordÚignore)
r   ÚpytestÚskipÚosÚpathÚbasenameÚdirnamer   r   r(   )r   r<   r   r   r   r   Útest_pickles’   s    
rJ   c             C   sª   dd„ }dd„ }| }x�|j ƒ D ]„\}}xz|j ƒ D ]n\}}xdtj|gD ]V}|d krRqDtjƒ �:}	|||	ƒ tj|	ƒ}
t|
||ƒ ||	ƒ}
t|
||ƒ W d Q R X qDW q0W qW d S )Nc             S   s*   t |dƒ�}tj| |dd� W d Q R X d S )NÚwbr)   )Úprotocoléÿÿÿÿ)ÚopenÚpickleÚdump)ÚobjrG   Úfhr   r   r   Úpython_pickler�   s    z/test_round_trip_current.<locals>.python_picklerc          	   S   s*   t | dƒ�}|jdƒ tj|ƒS Q R X d S )NÚrbr   )rN   ÚseekrO   Úload)rG   rR   r   r   r   Úpython_unpickler¡   s    
z1test_round_trip_current.<locals>.python_unpickler)r"   r   Ú	to_pickler   Úensure_cleanr    r   )r   rS   rW   r$   r   r&   r'   r   ÚwriterrG   r   r   r   r   Útest_round_trip_currentœ   s    


r[   c              C   s(   t jƒ } t j| jtjƒ}t j| |ƒ d S )N)r   ÚmakeDataFrameZround_trip_pathlibrX   r   r    r5   )Údfr   r   r   r   Útest_pickle_path_pathlib»   s    r^   c              C   s(   t jƒ } t j| jtjƒ}t j| |ƒ d S )N)r   r\   Zround_trip_localpathrX   r   r    r5   )r]   r   r   r   r   Útest_pickle_path_localpathÁ   s    r_   c             C   st   t jtƒ�( tdtƒ tj| ddddƒdd� W dQ R X t jtƒ�( tdtƒ tj| dddd	ƒdd� W dQ R X dS )
ap  

    Generated with

    >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [0, 0, 1, 1]}).to_sparse()
    >>> df.to_pickle("pandas/tests/io/data/pickle/sparseframe-0.20.3.pickle.gz",
    ...              compression="gzip")

    >>> s = df['B']
    >>> s.to_pickle("pandas/tests/io/data/pickle/sparseseries-0.20.3.pickle.gz",
    ...             compression="gzip")
    rC   Úior$   rO   zsparseseries-0.20.3.pickle.gzÚgzip)ÚcompressionNzsparseframe-0.20.3.pickle.gz)r   Zassert_produces_warningÚFutureWarningr   ÚDeprecationWarningr   r    )rA   r   r   r   Útest_legacy_sparse_warningÇ   s    

re   c               C   s   dt jdƒ› d�S )NÚ__é
   z	__.pickle)r   Zrandsr   r   r   r   Úget_random_pathè   s    rh   c               @   s”   e Zd ZddddddœZdd„ Zd	d
„ Zejjdddddgƒdd„ ƒZ	ejjddddddgƒdd„ ƒZ
dd„ Zejjdddddddgƒdd„ ƒZdS )ÚTestCompressionz.nonez.gzz.bz2z.zipz.xz)Nra   Úbz2ÚzipÚxzc             C   sê   |d krt j||ƒ d S |dkr.tj|dƒ}n||dkrDtj|dƒ}nf|dkr€tj|dtjd��}|j	|t
jj|ƒƒ W d Q R X n*|dkr˜ttƒ|dƒ}nd|› �}t|ƒ‚|dkræt|dƒ�$}|� |j	|jƒ ƒ W d Q R X W d Q R X d S )	Nra   Úwrj   rk   )rb   rl   zUnrecognized compression type: rT   )ÚshutilÚcopyfilera   rN   rj   ÚBZ2FileÚzipfileÚZipFileÚZIP_DEFLATEDÚwriterF   rG   rH   r   ÚlzmaÚ
ValueErrorÚread)ÚselfZsrc_pathÚ	dest_pathrb   ÚfÚmsgrR   r   r   r   Úcompress_file÷   s"     
zTestCompression.compress_filec              C   s²   |}|d }|d }t j|ƒ�Š}t j|ƒ�t}t jƒ }|j||d� t j||d��*}	t|dƒ�}
|
j|	jƒ ƒ W d Q R X W d Q R X tj	|d d�}t j
||ƒ W d Q R X W d Q R X d S )Nz.compressedz.raw)rb   rK   )r   rY   r\   rX   Údecompress_filerN   rt   rw   r   r    r5   )rx   rb   rh   ÚbaseÚpath1Úpath2Úp1Úp2r]   rz   rR   Údf2r   r   r   Útest_write_explicit  s    "z#TestCompression.test_write_explicitrb   Ú ÚNoneÚbadZ7zc             C   sJ   t jtdd��2 tj|ƒ�}tjƒ }|j||d� W d Q R X W d Q R X d S )NzUnrecognized compression type)Úmatch)rb   )rD   Zraisesrv   r   rY   r\   rX   )rx   rb   rh   rG   r]   r   r   r   Útest_write_explicit_bad"  s    z'TestCompression.test_write_explicit_badÚextz.no_compressc              C   sÖ   |}|| }|d }d }x"| j D ]}| j | |kr |}P q W tj|ƒ�†}tj|ƒ�p}	tjƒ }
|
j|ƒ tj||d��*}t|	dƒ�}|j|jƒ ƒ W d Q R X W d Q R X t	j
|	d d�}tj|
|ƒ W d Q R X W d Q R X d S )Nz.raw)rb   rK   )Ú_compression_to_extensionr   rY   r\   rX   r}   rN   rt   rw   r   r    r5   )rx   rŠ   rh   r~   r   r€   rb   Úcr�   r‚   r]   rz   rR   rƒ   r   r   r   Útest_write_infer)  s     
"z TestCompression.test_write_inferc       
      C   s„   |}|d }|d }t j|ƒ�\}t j|ƒ�F}t jƒ }|j|d d� | j|||d� tj||d�}	t j||	ƒ W d Q R X W d Q R X d S )Nz.rawz.compressed)rb   )r   rY   r\   rX   r|   r   r    r5   )
rx   rb   rh   r~   r   r€   r�   r‚   r]   rƒ   r   r   r   Útest_read_explicitD  s    z"TestCompression.test_read_explicitc             C   s¨   |}|d }|| }d }x"| j D ]}| j | |kr |}P q W tj|ƒ�X}tj|ƒ�B}	tjƒ }
|
j|d d� | j||	|d� tj|	ƒ}tj|
|ƒ W d Q R X W d Q R X d S )Nz.raw)rb   )	r‹   r   rY   r\   rX   r|   r   r    r5   )rx   rŠ   rh   r~   r   r€   rb   rŒ   r�   r‚   r]   rƒ   r   r   r   Útest_read_inferW  s    
zTestCompression.test_read_inferN)Ú__name__Ú
__module__Ú__qualname__r‹   r|   r„   rD   ÚmarkÚparametrizer‰   r�   rŽ   r�   r   r   r   r   ri   í   s    ri   c               @   s*   e Zd Zejjdddddgƒdd„ ƒZdS )	ÚTestProtocolrL   r)   r   r*   c             C   sF   t j|ƒ�2}t jƒ }|j||d� tj|ƒ}t j||ƒ W d Q R X d S )N)rL   )r   rY   r\   rX   r   r    r5   )rx   rL   rh   rG   r]   rƒ   r   r   r   Ú	test_readw  s
    
zTestProtocol.test_readNrM   )r�   r‘   r’   rD   r“   r”   r–   r   r   r   r   r•   v  s   r•   Úpickle_fileÚexcolsztest_py27.pklÚaÚbrŒ   ztest_mi_py27.pklÚAÚBÚCc             C   s*   | ddd|ƒ}t j|ƒ}tj|j|ƒ d S )Nr`   r$   rO   )r   r    r   r   Úcolumns)rA   r—   r˜   rG   r]   r   r   r   Útest_unicode_decode_error€  s    
rŸ   c              C   sl   t jƒ �Z} t jƒ }t| dƒ�}|j|ƒ W d Q R X t| dƒ�}tj|ƒ}W d Q R X t j||ƒ W d Q R X d S )NrK   rT   )r   rY   r\   rN   rX   r   r    r5   )rG   r]   rR   r   r   r   r   Útest_pickle_buffer_roundtrip™  s    
r    Úmockurlzhttp://url.comzftp://test.comzhttp://gzip.comc          
      sp   dd„ }G dd„ dƒ‰ t jƒ �H‰‡ ‡fdd„}t jƒ }||ˆƒ | jd|ƒ tj|ƒ}t j||ƒ W d Q R X d S )Nc             S   s*   t |dƒ�}tj| |dd� W d Q R X d S )NrK   r)   )rL   rM   )rN   rO   rP   )rQ   rG   rR   r   r   r   rS   ¬  s    z3test_pickle_generalurl_read.<locals>.python_picklerc               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsec             S   s.   t |dƒ| _d|kr ddi| _n
dd i| _d S )NrT   ra   zContent-Encoding)rN   ÚfileÚheaders)rx   rG   r   r   r   Ú__init__±  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c             S   s
   | j jƒ S )N)r¢   rw   )rx   r   r   r   rw   ¸  s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc             S   s
   | j jƒ S )N)r¢   Úclose)rx   r   r   r   r¥   »  s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeN)r�   r‘   r’   r¤   rw   r¥   r   r   r   r   ÚMockReadResponse°  s   r¦   c                 s   ˆ ˆƒS )Nr   )ÚargsÚkwargs)r¦   rG   r   r   Úmock_urlopen_readÀ  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rY   r\   Úsetattrr   r    r5   )Zmonkeypatchr¡   rS   r©   r]   r   r   )r¦   rG   r   Útest_pickle_generalurl_read¨  s    


r«   Zfsspecc           
   C   sD   t jƒ �2 d} t jƒ }|j| ƒ tj| ƒ}t j||ƒ W d Q R X d S )Nzmemory://afile)r   rY   r\   rX   r   r    r5   )r¡   r]   r   r   r   r   Útest_pickle_fsspec_roundtripÊ  s    


r¬   c               @   s   e Zd Zdd„ ZdS )ÚMyTzc             C   s   d S )Nr   )rx   r   r   r   r¤   Õ  s    zMyTz.__init__N)r�   r‘   r’   r¤   r   r   r   r   r­   Ô  s   r­   c              C   sF   t jtd�tƒ f} tj| ƒ}tj|d | d ƒ t|d tƒsBt‚d S )N)Zdtyper   r)   )	r   ÚSeriesÚobjectr­   r   Úround_trip_pickler.   r   r   )r   r   r   r   r   Útest_read_pickle_with_subclassÙ  s    
r±   c              C   sh   t jtdƒƒjdƒjdg } tj| ƒ}|jjd j	dks<t
‚|jjd jdksRt
‚tj|dg | ƒ d S )NÚabcÚcategoryr   r)   T)r)   )r   r®   ÚlistZastypeZilocr   r°   Z_mgrÚblocksÚndimr   Úshaper.   )Zserr1   r   r   r   Ú test_pickle_preserves_block_ndimâ  s
    
r¸   )N)EÚ__doc__rj   ÚdatetimeÚglobra   rF   rO   rn   Úwarningsr   r   rq   rD   Zpandas.compatr   r   r   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsÚtdZpandasr   r   Zpandas._testingZ_testingr   Zpandas.tseries.offsetsr   r	   ru   Zfixturer   r   r(   r2   r3   r4   r6   r7   r8   r;   rG   ÚjoinrI   Ú__file__Úfilesr<   rJ   r[   r^   r_   re   rh   ri   r•   r“   r”   Z
MultiIndexZfrom_arraysrŸ   r    r«   Z
skip_if_nor¬   Útzinfor­   r±   r¸   r   r   r   r   Ú<module>   sf   


! 

*!
	