3
Á3W^•J  ã               @   sî   d dl mZ ddlmZ d dlZd dlmZ d dlZd dlZd dl	Z
d dlZd dlZddlmZmZmZmZmZ G dd„ deƒZG d	d
„ d
eƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZdd„ Zdd„ Zddd„Zdd„ Zdd„ ZdS )é    )Úprint_functioné   )ÚVisionDatasetN)ÚImage)Údownload_urlÚdownload_and_extract_archiveÚextract_archiveÚmakedir_exist_okÚverify_str_argc            
       sÊ   e Zd ZdZddddgZdZdZdd	d
dddddddg
Zedd„ ƒZ	edd„ ƒZ
edd„ ƒZedd„ ƒZd/‡ fdd„	Zdd „ Zd!d"„ Zed#d$„ ƒZed%d&„ ƒZed'd(„ ƒZd)d*„ Zd+d,„ Zd-d.„ Z‡  ZS )0ÚMNISTa3  `MNIST <http://yann.lecun.com/exdb/mnist/>`_ Dataset.

    Args:
        root (string): Root directory of dataset where ``MNIST/processed/training.pt``
            and  ``MNIST/processed/test.pt`` exist.
        train (bool, optional): If True, creates dataset from ``training.pt``,
            otherwise from ``test.pt``.
        download (bool, optional): If true, downloads the dataset from the internet and
            puts it in root directory. If dataset is already downloaded, it is not
            downloaded again.
        transform (callable, optional): A function/transform that  takes in an PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
    z;http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gzz;http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gzz:http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gzz:http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gzztraining.ptztest.ptz0 - zeroz1 - onez2 - twoz	3 - threez4 - fourz5 - fivez6 - sixz	7 - sevenz	8 - eightz9 - ninec             C   s   t jdƒ | jS )Nz%train_labels has been renamed targets)ÚwarningsÚwarnÚtargets)Úself© r   úA/tmp/pip-build-7wfi2gtz/torchvision/torchvision/datasets/mnist.pyÚtrain_labels)   s    
zMNIST.train_labelsc             C   s   t jdƒ | jS )Nz$test_labels has been renamed targets)r   r   r   )r   r   r   r   Útest_labels.   s    
zMNIST.test_labelsc             C   s   t jdƒ | jS )Nz train_data has been renamed data)r   r   Údata)r   r   r   r   Ú
train_data3   s    
zMNIST.train_datac             C   s   t jdƒ | jS )Nztest_data has been renamed data)r   r   r   )r   r   r   r   Ú	test_data8   s    
zMNIST.test_dataTNFc                sn   t t| ƒj|||d� || _|r(| jƒ  | jƒ s8tdƒ‚| jrF| j}n| j}t	j
tjj| j|ƒƒ\| _| _d S )N)Ú	transformÚtarget_transformzDataset not found.z) You can use download=True to download itz;Dataset not found. You can use download=True to download it)Úsuperr   Ú__init__ÚtrainÚdownloadÚ_check_existsÚRuntimeErrorÚtraining_fileÚ	test_fileÚtorchÚloadÚosÚpathÚjoinÚprocessed_folderr   r   )r   Úrootr   r   r   r   Ú	data_file)Ú	__class__r   r   r   =   s    zMNIST.__init__c             C   s\   | j | t| j| ƒ }}tj|jƒ dd�}| jdk	r@| j|ƒ}| jdk	rT| j|ƒ}||fS )z–
        Args:
            index (int): Index

        Returns:
            tuple: (image, target) where target is index of the target class.
        ÚL)ÚmodeN)r   Úintr   r   Ú	fromarrayÚnumpyr   r   )r   ÚindexÚimgÚtargetr   r   r   Ú__getitem__P   s    



zMNIST.__getitem__c             C   s
   t | jƒS )N)Úlenr   )r   r   r   r   Ú__len__f   s    zMNIST.__len__c             C   s   t jj| j| jjdƒS )NÚraw)r#   r$   r%   r'   r)   Ú__name__)r   r   r   r   Ú
raw_folderi   s    zMNIST.raw_folderc             C   s   t jj| j| jjdƒS )NÚ	processed)r#   r$   r%   r'   r)   r6   )r   r   r   r   r&   m   s    zMNIST.processed_folderc             C   s   dd„ t | jƒD ƒS )Nc             S   s   i | ]\}}||“qS r   r   )Ú.0ÚiÚ_classr   r   r   ú
<dictcomp>s   s    z&MNIST.class_to_idx.<locals>.<dictcomp>)Ú	enumerateÚclasses)r   r   r   r   Úclass_to_idxq   s    zMNIST.class_to_idxc             C   s4   t jjt jj| j| jƒƒo2t jjt jj| j| jƒƒS )N)r#   r$   Úexistsr%   r&   r   r    )r   r   r   r   r   u   s    
zMNIST._check_existsc             C   s  | j ƒ rdS t| jƒ t| jƒ x,| jD ]"}|jdƒd }t|| j|d� q(W tdƒ tt	j
j| jdƒƒtt	j
j| jdƒƒf}tt	j
j| jdƒƒtt	j
j| jd	ƒƒf}tt	j
j| j| jƒd
ƒ�}tj||ƒ W dQ R X tt	j
j| j| jƒd
ƒ�}tj||ƒ W dQ R X tdƒ dS )zHDownload the MNIST data if it doesn't exist in processed_folder already.Nú/é   )Údownload_rootÚfilenamezProcessing...ztrain-images-idx3-ubyteztrain-labels-idx1-ubytezt10k-images-idx3-ubytezt10k-labels-idx1-ubyteÚwbzDone!)r   r	   r7   r&   ÚurlsÚ
rpartitionr   ÚprintÚread_image_filer#   r$   r%   Úread_label_fileÚopenr   r!   Úsaver    )r   ÚurlrD   Útraining_setÚtest_setÚfr   r   r   r   {   s"    

zMNIST.downloadc             C   s   dj | jdkrdndƒS )Nz	Split: {}TZTrainZTest)Úformatr   )r   r   r   r   Ú
extra_repr›   s    zMNIST.extra_repr)TNNF)r6   Ú
__module__Ú__qualname__Ú__doc__rF   r   r    r>   Úpropertyr   r   r   r   r   r2   r4   r7   r&   r?   r   r   rR   Ú__classcell__r   r   )r)   r   r      s.   
  r   c            
   @   s4   e Zd ZdZddddgZdddd	d
dddddg
ZdS )ÚFashionMNISTaZ  `Fashion-MNIST <https://github.com/zalandoresearch/fashion-mnist>`_ Dataset.

    Args:
        root (string): Root directory of dataset where ``Fashion-MNIST/processed/training.pt``
            and  ``Fashion-MNIST/processed/test.pt`` exist.
        train (bool, optional): If True, creates dataset from ``training.pt``,
            otherwise from ``test.pt``.
        download (bool, optional): If true, downloads the dataset from the internet and
            puts it in root directory. If dataset is already downloaded, it is not
            downloaded again.
        transform (callable, optional): A function/transform that  takes in an PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
    zUhttp://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gzzUhttp://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gzzThttp://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gzzThttp://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gzzT-shirt/topZTrouserZPulloverZDressZCoatZSandalZShirtZSneakerZBagz
Ankle bootN)r6   rS   rT   rU   rF   r>   r   r   r   r   rX   Ÿ   s   rX   c            
   @   s4   e Zd ZdZddddgZdddd	d
dddddg
ZdS )ÚKMNISTaA  `Kuzushiji-MNIST <https://github.com/rois-codh/kmnist>`_ Dataset.

    Args:
        root (string): Root directory of dataset where ``KMNIST/processed/training.pt``
            and  ``KMNIST/processed/test.pt`` exist.
        train (bool, optional): If True, creates dataset from ``training.pt``,
            otherwise from ``test.pt``.
        download (bool, optional): If true, downloads the dataset from the internet and
            puts it in root directory. If dataset is already downloaded, it is not
            downloaded again.
        transform (callable, optional): A function/transform that  takes in an PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
    zGhttp://codh.rois.ac.jp/kmnist/dataset/kmnist/train-images-idx3-ubyte.gzzGhttp://codh.rois.ac.jp/kmnist/dataset/kmnist/train-labels-idx1-ubyte.gzzFhttp://codh.rois.ac.jp/kmnist/dataset/kmnist/t10k-images-idx3-ubyte.gzzFhttp://codh.rois.ac.jp/kmnist/dataset/kmnist/t10k-labels-idx1-ubyte.gzÚoZkiZsuZtsuÚnaZhaÚmaZyaÚreZwoN)r6   rS   rT   rU   rF   r>   r   r   r   r   rY   ¹   s   rY   c                   sH   e Zd ZdZdZdZ‡ fd	d
„Zedd„ ƒZedd„ ƒZ	dd„ Z
‡  ZS )ÚEMNISTa)  `EMNIST <https://www.westernsydney.edu.au/bens/home/reproducible_research/emnist>`_ Dataset.

    Args:
        root (string): Root directory of dataset where ``EMNIST/processed/training.pt``
            and  ``EMNIST/processed/test.pt`` exist.
        split (string): The dataset has 6 different splits: ``byclass``, ``bymerge``,
            ``balanced``, ``letters``, ``digits`` and ``mnist``. This argument specifies
            which one to use.
        train (bool, optional): If True, creates dataset from ``training.pt``,
            otherwise from ``test.pt``.
        download (bool, optional): If true, downloads the dataset from the internet and
            puts it in root directory. If dataset is already downloaded, it is not
            downloaded again.
        transform (callable, optional): A function/transform that  takes in an PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
    zIhttps://cloudstor.aarnet.edu.au/plus/index.php/s/54h3OuGJhFLwAlQ/downloadÚbyclassÚbymergeÚbalancedÚlettersÚdigitsÚmnistc                s@   t |d| jƒ| _| j|ƒ| _| j|ƒ| _tt| ƒj	|f|Ž d S )NÚsplit)
r
   Úsplitsre   Ú_training_filer   Ú
_test_filer    r   r^   r   )r   r'   re   Úkwargs)r)   r   r   r   é   s    zEMNIST.__init__c             C   s
   dj | ƒS )Nztraining_{}.pt)rQ   )re   r   r   r   rg   ï   s    zEMNIST._training_filec             C   s
   dj | ƒS )Nz
test_{}.pt)rQ   )re   r   r   r   rh   ó   s    zEMNIST._test_filec             C   s€  ddl }| jƒ rdS t| jƒ t| jƒ tdƒ t| j| jddd� tj	j
| jdƒ}x0tj|ƒD ]"}|jdƒr`ttj	j
||ƒ|ƒ q`W xâ| jD ]Ø}td	| ƒ ttj	j
|d
j|ƒƒƒttj	j
|dj|ƒƒƒf}ttj	j
|dj|ƒƒƒttj	j
|dj|ƒƒƒf}ttj	j
| j| j|ƒƒdƒ�}tj||ƒ W dQ R X ttj	j
| j| j|ƒƒdƒ�}tj||ƒ W dQ R X qŽW |j|ƒ tdƒ dS )zIDownload the EMNIST data if it doesn't exist in processed_folder already.r   Nz&Downloading and extracting zip archivez
emnist.zipT)rC   rD   Zremove_finishedÚgzipz.gzzProcessing z!emnist-{}-train-images-idx3-ubytez!emnist-{}-train-labels-idx1-ubytez emnist-{}-test-images-idx3-ubytez emnist-{}-test-labels-idx1-ubyterE   zDone!)Úshutilr   r	   r7   r&   rH   r   rM   r#   r$   r%   ÚlistdirÚendswithr   rf   rI   rQ   rJ   rK   rg   r!   rL   rh   Úrmtree)r   rk   Zgzip_folderZ	gzip_filere   rN   rO   rP   r   r   r   r   ÷   s0    



zEMNIST.download)r_   r`   ra   rb   rc   rd   )r6   rS   rT   rU   rM   rf   r   Ústaticmethodrg   rh   r   rW   r   r   )r)   r   r^   Ò   s   r^   c            
       sz   e Zd ZdZddddddœZddgdd	gd
dgdœZddddddddddg
Zd!‡ fdd„	Zdd„ Zdd„ Z	dd „ Z
‡  ZS )"ÚQMNISTa_  `QMNIST <https://github.com/facebookresearch/qmnist>`_ Dataset.

    Args:
        root (string): Root directory of dataset whose ``processed''
            subdir contains torch binary files with the datasets.
        what (string,optional): Can be 'train', 'test', 'test10k',
            'test50k', or 'nist' for respectively the mnist compatible
            training set, the 60k qmnist testing set, the 10k qmnist
            examples that match the mnist testing set, the 50k
            remaining qmnist testing examples, or all the nist
            digits. The default is to select 'train' or 'test'
            according to the compatibility argument 'train'.
        compat (bool,optional): A boolean that says whether the target
            for each example is class number (for compatibility with
            the MNIST dataloader) or a torch vector containing the
            full qmnist information. Default=True.
        download (bool, optional): If true, downloads the dataset from
            the internet and puts it in root directory. If dataset is
            already downloaded, it is not downloaded again.
        transform (callable, optional): A function/transform that
            takes in an PIL image and returns a transformed
            version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform
            that takes in the target and transforms it.
        train (bool,optional,compatibility): When argument 'what' is
            not specified, this boolean decides whether to load the
            training set ot the testing set.  Default: True.

    r   ÚtestÚnist)r   rq   Útest10kÚtest50krr   zbhttps://raw.githubusercontent.com/facebookresearch/qmnist/master/qmnist-train-images-idx3-ubyte.gzz`https://raw.githubusercontent.com/facebookresearch/qmnist/master/qmnist-train-labels-idx2-int.gzzahttps://raw.githubusercontent.com/facebookresearch/qmnist/master/qmnist-test-images-idx3-ubyte.gzz_https://raw.githubusercontent.com/facebookresearch/qmnist/master/qmnist-test-labels-idx2-int.gzz[https://raw.githubusercontent.com/facebookresearch/qmnist/master/xnist-images-idx3-ubyte.xzzYhttps://raw.githubusercontent.com/facebookresearch/qmnist/master/xnist-labels-idx2-int.xz)r   rq   rr   z0 - zeroz1 - onez2 - twoz	3 - threez4 - fourz5 - fivez6 - sixz	7 - sevenz	8 - eightz9 - nineNTc                sf   |d kr|rdnd}t |dt| jjƒ ƒƒ| _|| _|d | _| j| _| j| _t	t
| ƒj||f|Ž d S )Nr   rq   Úwhatz.pt)r
   ÚtupleÚsubsetsÚkeysru   Úcompatr(   r   r    r   rp   r   )r   r'   ru   ry   r   ri   )r)   r   r   r   M  s    
zQMNIST.__init__c       	   
   C   sš  | j ƒ rdS t| jƒ t| jƒ | j| j| j  }g }xR|D ]J}|jdƒd }tj	j
| j|ƒ}tj	j|ƒs|t|| j|dd� |j|ƒ q<W tdƒ t|d ƒ}|jtjks®t‚|jƒ dks¾t‚t|d ƒjƒ }|jƒ dksÞt‚| jd	k�r |dd
…dd…dd…f jƒ }|dd
…dd…f jƒ }| jdk�rb|d
d…dd…dd…f jƒ }|d
d…dd…f jƒ }ttj	j
| j| jƒdƒ�}tj||f|ƒ W dQ R X dS )z£Download the QMNIST data if it doesn't exist in processed_folder already.
           Note that we only download what has been asked for (argument 'what').
        NrA   rB   )r'   rD   Úmd5zProcessing...r   é   r   rs   i'  rt   rE   )r   r	   r7   r&   rF   rw   ru   rG   r#   r$   r%   Úisfiler   ÚappendrH   Úread_sn3_pascalvincent_tensorÚdtyper!   Úuint8ÚAssertionErrorÚ
ndimensionÚlongÚclonerK   r(   rL   )	r   rF   ÚfilesrM   rD   Ú	file_pathr   r   rP   r   r   r   r   W  s4    


zQMNIST.downloadc             C   sj   | j | | j|  }}tj|jƒ dd�}| jd k	r<| j|ƒ}| jrNt|d ƒ}| jd k	rb| j|ƒ}||fS )Nr*   )r+   r   )	r   r   r   r-   r.   r   ry   r,   r   )r   r/   r0   r1   r   r   r   r2   z  s    



zQMNIST.__getitem__c             C   s   dj | jƒS )Nz	Split: {})rQ   ru   )r   r   r   r   rR   †  s    zQMNIST.extra_repr)NTT)r6   rS   rT   rU   rw   rF   r>   r   r   r2   rR   rW   r   r   )r)   r   rp     s    


#rp   c             C   s   t tj| dƒdƒS )NÚhexé   )r,   ÚcodecsÚencode)Úbr   r   r   Úget_intŠ  s    rŒ   c             C   sX   t | tjjƒs| S | jdƒr0ddl}|j| dƒS | jdƒrNddl}|j| dƒS t| dƒS )z¢Return a file object that possibly decompresses 'path' on the fly.
       Decompression occurs when argument `path` is a string and ends with '.gz' or '.xz'.
    z.gzr   NÚrbz.xz)Ú
isinstancer!   Z_sixZstring_classesrm   rj   rK   Úlzma)r$   rj   r�   r   r   r   Úopen_maybe_compressed_fileŽ  s    

r�   Tc       	         sN  t tdƒsntjtjtjftjtjtjftjtjdƒdftjtjdƒdftj	tjdƒdftj
tjdƒd	fd
œt_t| ƒ�}|jƒ ‰ W dQ R X tˆ dd… ƒ}|d }|d }|dkrº|dks¾t‚|dkrÎ|dksÒt‚tj| }‡ fdd„t|ƒD ƒ}tjˆ |d d|d  d�}|jd tj|ƒk�s0| �s0t‚tj|j|d dd�ƒj|Ž S )z˜Read a SN3 file in "Pascal Vincent" format (Lush file 'libidx/idx-io.lsh').
       Argument may be a filename, compressed filename, or file object.
    Útypemapz>i2Úi2z>i4Úi4z>f4Zf4z>f8Zf8)é   é	   é   é   é   é   Nr   é   é   r   r{   r”   r™   c                s,   g | ]$}t ˆ d |d  d |d  … ƒ‘qS )rš   r   rB   )rŒ   )r9   r:   )r   r   r   ú
<listcomp>´  s    z1read_sn3_pascalvincent_tensor.<locals>.<listcomp>)r   ÚoffsetrB   F)Úcopy)Úhasattrr~   r!   r€   ÚnpZint8Zint16r   Zint32Zfloat32Zfloat64r‘   r�   ÚreadrŒ   r�   ÚrangeZ
frombufferÚshapeÚprodZ
from_numpyZastypeÚview)	r$   ÚstrictrP   ÚmagicZndÚtyÚmÚsÚparsedr   )r   r   r~   �  s&    


"r~   c             C   sJ   t | dƒ�}t|dd�}W d Q R X |jtjks2t‚|jƒ dksBt‚|jƒ S )Nr�   F)r¦   r   )rK   r~   r   r!   r€   r�   r‚   rƒ   )r$   rP   Úxr   r   r   rJ   º  s
    rJ   c             C   sF   t | dƒ�}t|dd�}W d Q R X |jtjks2t‚|jƒ dksBt‚|S )Nr�   F)r¦   r{   )rK   r~   r   r!   r€   r�   r‚   )r$   rP   r¬   r   r   r   rI   Â  s
    rI   )T)Ú
__future__r   Zvisionr   r   ZPILr   r#   Úos.pathr.   r    r!   r‰   Úutilsr   r   r   r	   r
   r   rX   rY   r^   rp   rŒ   r�   r~   rJ   rI   r   r   r   r   Ú<module>   s(    Ll
