3
Êb_:  ã            	   @   sJ  d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
Z
ddlZddlmZmZmZ ddljjZddlZddlmZ ddljZddlmZmZ eƒ Zejdd�d	d
„ ƒZdRdd„Z dd„ Z!dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd„ Z&dd„ Z'dd„ Z(ejej)j*ej)j+e,ƒdddd ƒƒZ-eje-d!�d"d„ ƒZ.d#d$„ Z/d%d&„ Z0d'd(„ Z1d)d*„ Z2d+d,„ Z3ejd-d.„ ƒZ4G d/d0„ d0ƒZ5G d1d2„ d2ƒZ6ej7j8d3d4gd5ejd6d7d8gƒfd9ej9j:d6d7d8gd:d;d<ggƒfgƒd=d>„ ƒZ;d?d@„ Z<ej7j8dAdBdCdDgƒdEdF„ ƒZ=ej>dGƒej7j8dAdHdIgƒdJdK„ ƒƒZ?ej>dLƒej7j8dAdMdNdOgƒdPdQ„ ƒƒZ@dS )Sa<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
é    N)Úcatch_warningsÚsimplefilter)Ú_get_lzma_fileÚ_import_lzmaÚis_platform_little_endian)ÚIndex)ÚDayÚMonthEndÚmodule)Zscopec              C   s   ddl m}  | ƒ S )Nr   )Úcreate_pickle_data)Z-pandas.tests.io.generate_legacy_storage_filesr   )r   © r   ú=/tmp/pip-build-jkwsxyiy/pandas/pandas/tests/io/test_pickle.pyÚcurrent_pickle_data$   s    r   c             C   sš   t |tƒrtj|| ƒ d S |jdƒr6tj}|| |ƒ n`|dkrv|tjkrX| tjkstt‚q–| |ksdt‚| j	|j	ks–t‚n t
tdj|d�tjƒ}|| |ƒ d S )NZsp_Ú	timestampzassert_{typ}_equal)Útyp)Ú
isinstancer   ÚtmÚassert_index_equalÚ
startswithZassert_equalÚpdZNaTÚAssertionErrorÚfreqÚgetattrÚformatZassert_almost_equal)ÚresultÚexpectedr   ÚversionÚ
comparatorr   r   r   Úcompare_element/   s    


r   c       
      C   sx   t j|ƒ} tƒ }xb| jƒ D ]V\}}xL|jƒ D ]@\}}| | | }dj||d�}	|j|	|d ƒ}	|	||||ƒ q,W qW | S )Nzcompare_{typ}_{dt})r   Údtr   )r   Úread_pickleÚglobalsÚitemsr   Úget)
ÚdataZvfr   Úmr   Údvr   r   r   r   r   r   r   ÚcompareD   s    
r'   c             C   sÔ   t j| |ƒ | jj|jjks t‚| jjj s0t‚t j| dk|dkƒ | jj}|tdƒ tdƒksdt‚|tjdd� }t	|tjƒs„t‚|tjddd�ksšt‚|tjdd� }t	|tjƒsºt‚|tjddd�ksÐt‚d S )Nr   é   é   )Úhours)Údaysr*   )Únanoseconds)r+   r,   )
r   Úassert_series_equalÚindexr   r   Ú	normalizer   r   Z	Timedeltar   )r   r   r   r   r   Úresr   r   r   Úcompare_series_tsV   s    r1   c             C   s   t j| |ƒ d S )N)r   r-   )r   r   r   r   r   r   r   Úcompare_series_dt_tzj   s    r2   c             C   s   t j| |ƒ d S )N)r   r-   )r   r   r   r   r   r   r   Úcompare_series_catn   s    r3   c             C   s   t j| |ƒ d S )N)r   Úassert_frame_equal)r   r   r   r   r   r   r   Úcompare_frame_dt_mixed_tzsr   s    r5   c             C   s   t j| |ƒ d S )N)r   r4   )r   r   r   r   r   r   r   Úcompare_frame_cat_onecolv   s    r6   c             C   s   t | |||ƒ d S )N)r6   )r   r   r   r   r   r   r   Úcompare_frame_cat_and_floatz   s    r7   c             C   sV   t j| |ƒ t| jtƒst‚| jtƒ ks,t‚| jdks:t‚t j| jdƒ|jdƒƒ d S )NÚMr)   )r   r   r   r   r	   r   ZfreqstrÚshift)r   r   r   r   r   r   r   Úcompare_index_period~   s
    r:   r$   Úlegacy_pickleÚ*z*.pickle)Úparamsc             C   s
   || j ƒS )N)Úparam)ÚrequestÚdatapathr   r   r   r;   ‹   s    c             C   sR   t ƒ stjdƒ tjjtjj|ƒƒ}tdd�� tdƒ t	| ||ƒ W d Q R X d S )Nz"known failure on non-little endianT)ÚrecordÚignore)
r   ÚpytestÚskipÚosÚpathÚbasenameÚdirnamer   r   r'   )r   r;   r   r   r   r   Útest_pickles“   s    
rI   c             C   sª   dd„ }dd„ }| }x�|j ƒ D ]„\}}xz|j ƒ D ]n\}}xdtj|gD ]V}|d krRqDtjƒ �:}	|||	ƒ tj|	ƒ}
t|
||ƒ ||	ƒ}
t|
||ƒ W d Q R X qDW q0W qW d S )Nc             S   s*   t |dƒ�}tj| |dd� W d Q R X d S )NÚwbr(   )Úprotocoléÿÿÿÿ)ÚopenÚpickleÚdump)ÚobjrF   Úfhr   r   r   Úpython_picklerž   s    z/test_round_trip_current.<locals>.python_picklerc          	   S   s*   t | dƒ�}|jdƒ tj|ƒS Q R X d S )NÚrbr   )rM   ÚseekrN   Úload)rF   rQ   r   r   r   Úpython_unpickler¢   s    
z1test_round_trip_current.<locals>.python_unpickler)r"   r   Ú	to_pickler   Úensure_cleanr    r   )r   rR   rV   r$   r   r&   r   r   ÚwriterrF   r   r   r   r   Útest_round_trip_current�   s    


rZ   c              C   s(   t jƒ } t j| jtjƒ}t j| |ƒ d S )N)r   ÚmakeDataFrameZround_trip_pathlibrW   r   r    r4   )Údfr   r   r   r   Útest_pickle_path_pathlib¼   s    r]   c              C   s(   t jƒ } t j| jtjƒ}t j| |ƒ d S )N)r   r[   Zround_trip_localpathrW   r   r    r4   )r\   r   r   r   r   Útest_pickle_path_localpathÂ   s    r^   c             C   st   t jtƒ�( tdtƒ tj| ddddƒdd� W dQ R X t jtƒ�( tdtƒ tj| dddd	ƒdd� W dQ R X dS )
ap  

    Generated with

    >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [0, 0, 1, 1]}).to_sparse()
    >>> df.to_pickle("pandas/tests/io/data/pickle/sparseframe-0.20.3.pickle.gz",
    ...              compression="gzip")

    >>> s = df['B']
    >>> s.to_pickle("pandas/tests/io/data/pickle/sparseseries-0.20.3.pickle.gz",
    ...             compression="gzip")
    rB   Úior$   rN   zsparseseries-0.20.3.pickle.gzÚgzip)ÚcompressionNzsparseframe-0.20.3.pickle.gz)r   Zassert_produces_warningÚFutureWarningr   ÚDeprecationWarningr   r    )r@   r   r   r   Útest_legacy_sparse_warningÈ   s    

rd   c               C   s   dj tjdƒƒS )Nz__{}__.pickleé
   )r   r   Zrandsr   r   r   r   Úget_random_pathé   s    rf   c               @   s”   e Zd ZddddddœZdd„ Zd	d
„ Zejjdddddgƒdd„ ƒZ	ejjddddddgƒdd„ ƒZ
dd„ Zejjdddddddgƒdd„ ƒZdS )ÚTestCompressionz.nonez.gzz.bz2z.zipz.xz)Nr`   Úbz2ÚzipÚxzc             C   sê   |d krt j||ƒ d S |dkr.tj|dƒ}n||dkrDtj|dƒ}nf|dkr€tj|dtjd��}|j	|t
jj|ƒƒ W d Q R X n*|dkr˜ttƒ|dƒ}ndj|ƒ}t|ƒ‚|dkræt|dƒ�$}|� |j	|jƒ ƒ W d Q R X W d Q R X d S )	Nr`   Úwrh   ri   )ra   rj   z!Unrecognized compression type: {}rS   )ÚshutilÚcopyfiler`   rM   rh   ÚBZ2FileÚzipfileÚZipFileÚZIP_DEFLATEDÚwriterE   rF   rG   r   Úlzmar   Ú
ValueErrorÚread)ÚselfZsrc_pathZ	dest_pathra   ÚfÚmsgrQ   r   r   r   Úcompress_fileø   s"     
zTestCompression.compress_filec              C   s²   |}|d }|d }t j|ƒ�Š}t j|ƒ�t}t jƒ }|j||d� t j||d��*}	t|dƒ�}
|
j|	jƒ ƒ W d Q R X W d Q R X tj	|d d�}t j
||ƒ W d Q R X W d Q R X d S )Nz.compressedz.raw)ra   rJ   )r   rX   r[   rW   Údecompress_filerM   rr   ru   r   r    r4   )rv   ra   rf   ÚbaseÚpath1Úpath2Úp1Úp2r\   rw   rQ   Údf2r   r   r   Útest_write_explicit  s    "z#TestCompression.test_write_explicitra   Ú ÚNoneÚbadZ7zc             C   sJ   t jtdd��2 tj|ƒ�}tjƒ }|j||d� W d Q R X W d Q R X d S )NzUnrecognized compression type)Úmatch)ra   )rC   Zraisesrt   r   rX   r[   rW   )rv   ra   rf   rF   r\   r   r   r   Útest_write_explicit_bad#  s    z'TestCompression.test_write_explicit_badÚextz.no_compressc              C   sÖ   |}|| }|d }d }x"| j D ]}| j | |kr |}P q W tj|ƒ�†}tj|ƒ�p}	tjƒ }
|
j|ƒ tj||d��*}t|	dƒ�}|j|jƒ ƒ W d Q R X W d Q R X t	j
|	d d�}tj|
|ƒ W d Q R X W d Q R X d S )Nz.raw)ra   rJ   )Ú_compression_to_extensionr   rX   r[   rW   rz   rM   rr   ru   r   r    r4   )rv   r‡   rf   r{   r|   r}   ra   Úcr~   r   r\   rw   rQ   r€   r   r   r   Útest_write_infer*  s     
"z TestCompression.test_write_inferc       
      C   s„   |}|d }|d }t j|ƒ�\}t j|ƒ�F}t jƒ }|j|d d� | j|||d� tj||d�}	t j||	ƒ W d Q R X W d Q R X d S )Nz.rawz.compressed)ra   )r   rX   r[   rW   ry   r   r    r4   )
rv   ra   rf   r{   r|   r}   r~   r   r\   r€   r   r   r   Útest_read_explicitE  s    z"TestCompression.test_read_explicitc             C   s¨   |}|d }|| }d }x"| j D ]}| j | |kr |}P q W tj|ƒ�X}tj|ƒ�B}	tjƒ }
|
j|d d� | j||	|d� tj|	ƒ}tj|
|ƒ W d Q R X W d Q R X d S )Nz.raw)ra   )	rˆ   r   rX   r[   rW   ry   r   r    r4   )rv   r‡   rf   r{   r|   r}   ra   r‰   r~   r   r\   r€   r   r   r   Útest_read_inferX  s    
zTestCompression.test_read_inferN)Ú__name__Ú
__module__Ú__qualname__rˆ   ry   r�   rC   ÚmarkÚparametrizer†   rŠ   r‹   rŒ   r   r   r   r   rg   î   s    rg   c               @   s*   e Zd Zejjdddddgƒdd„ ƒZdS )	ÚTestProtocolrK   r(   r   r)   c             C   sF   t j|ƒ�2}t jƒ }|j||d� tj|ƒ}t j||ƒ W d Q R X d S )N)rK   )r   rX   r[   rW   r   r    r4   )rv   rK   rf   rF   r\   r€   r   r   r   Ú	test_readx  s
    
zTestProtocol.test_readNrL   )r�   rŽ   r�   rC   r�   r‘   r“   r   r   r   r   r’   w  s   r’   Úpickle_fileÚexcolsztest_py27.pklÚaÚbr‰   ztest_mi_py27.pklÚAÚBÚCc             C   s*   | ddd|ƒ}t j|ƒ}tj|j|ƒ d S )Nr_   r$   rN   )r   r    r   r   Úcolumns)r@   r”   r•   rF   r\   r   r   r   Útest_unicode_decode_error�  s    
rœ   c              C   sl   t jƒ �Z} t jƒ }t| dƒ�}|j|ƒ W d Q R X t| dƒ�}tj|ƒ}W d Q R X t j||ƒ W d Q R X d S )NrJ   rS   )r   rX   r[   rM   rW   r   r    r4   )rF   r\   rQ   r   r   r   r   Útest_pickle_buffer_roundtripš  s    
r�   Úmockurlzhttp://url.comzftp://test.comzhttp://gzip.comc          
      sp   dd„ }G dd„ dƒ‰ t jƒ �H‰‡ ‡fdd„}t jƒ }||ˆƒ | jd|ƒ tj|ƒ}t j||ƒ W d Q R X d S )Nc             S   s*   t |dƒ�}tj| |dd� W d Q R X d S )NrJ   r(   )rK   rL   )rM   rN   rO   )rP   rF   rQ   r   r   r   rR   ­  s    z3test_pickle_generalurl_read.<locals>.python_picklerc               @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsec             S   s.   t |dƒ| _d|kr ddi| _n
dd i| _d S )NrS   r`   zContent-Encoding)rM   ÚfileÚheaders)rv   rF   r   r   r   Ú__init__²  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c             S   s
   | j jƒ S )N)rŸ   ru   )rv   r   r   r   ru   ¹  s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc             S   s
   | j jƒ S )N)rŸ   Úclose)rv   r   r   r   r¢   ¼  s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeN)r�   rŽ   r�   r¡   ru   r¢   r   r   r   r   ÚMockReadResponse±  s   r£   c                 s   ˆ ˆƒS )Nr   )ÚargsÚkwargs)r£   rF   r   r   Úmock_urlopen_readÁ  s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rX   r[   Úsetattrr   r    r4   )Úmonkeypatchrž   rR   r¦   r\   r   r   )r£   rF   r   Útest_pickle_generalurl_read©  s    


r©   Zgcsfszgs://gcs.comzgcs://gcs.comc                s^   t jƒ �L‰ G ‡ fdd„dƒ}| jd|ƒ t jƒ }|j|ƒ tj|ƒ}t j||ƒ W d Q R X d S )Nc                   s    e Zd Zdd„ Z‡ fdd„ZdS )z7test_pickle_gcsurl_roundtrip.<locals>.MockGCSFileSystemc             _   s   d S )Nr   )rv   r¤   r¥   r   r   r   r¡   Ñ  s    z@test_pickle_gcsurl_roundtrip.<locals>.MockGCSFileSystem.__init__c                s   |d p
d }t ˆ |ƒ}|S )Nr(   )rM   )rv   r¤   Úmoderw   )rF   r   r   rM   Ô  s    
z<test_pickle_gcsurl_roundtrip.<locals>.MockGCSFileSystem.openN)r�   rŽ   r�   r¡   rM   r   )rF   r   r   ÚMockGCSFileSystemÐ  s   r«   zgcsfs.GCSFileSystem)r   rX   r§   r[   rW   r   r    r4   )r¨   rž   r«   r\   r   r   )rF   r   Útest_pickle_gcsurl_roundtripË  s    
	

r¬   Zs3fszs3://s3.comzs3n://s3.comzs3a://s3.comc                s^   t jƒ �L‰ G ‡ fdd„dƒ}| jd|ƒ t jƒ }|j|ƒ tj|ƒ}t j||ƒ W d Q R X d S )Nc                   s    e Zd Zdd„ Z‡ fdd„ZdS )z5test_pickle_s3url_roundtrip.<locals>.MockS3FileSystemc             _   s   d S )Nr   )rv   r¤   r¥   r   r   r   r¡   æ  s    z>test_pickle_s3url_roundtrip.<locals>.MockS3FileSystem.__init__c                s   |d p
d }t ˆ |ƒ}|S )Nr(   )rM   )rv   r¤   rª   rw   )rF   r   r   rM   é  s    
z:test_pickle_s3url_roundtrip.<locals>.MockS3FileSystem.openN)r�   rŽ   r�   r¡   rM   r   )rF   r   r   ÚMockS3FileSystemå  s   r­   zs3fs.S3FileSystem)r   rX   r§   r[   rW   r   r    r4   )r¨   rž   r­   r\   r   r   )rF   r   Útest_pickle_s3url_roundtripà  s    
	

r®   )N)AÚ__doc__rh   Úglobr`   rE   rN   rl   Úwarningsr   r   ro   rC   Zpandas.compatr   r   r   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsÚtdZpandasr   r   Zpandas._testingZ_testingr   Zpandas.tseries.offsetsr   r	   rs   Zfixturer   r   r'   r1   r2   r3   r5   r6   r7   r:   rF   ÚjoinrH   Ú__file__Úfilesr;   rI   rZ   r]   r^   rd   rf   rg   r’   r�   r‘   Z
MultiIndexZfrom_arraysrœ   r�   r©   Z
skip_if_nor¬   r®   r   r   r   r   Ú<module>   sd   


! 

*!