3
I4W^Z)  ã               @   s”   d dl Z d dlZd dlmZ d dlmZ G dd„ deƒZG dd„ deƒZG dd	„ d	eƒZ	G d
d„ deƒZ
G dd„ deƒZG dd„ deƒZdd„ ZdS )é    N)Ú_accumulate)Úrandpermc               @   s    e Zd ZdZdd„ Zdd„ ZdS )ÚDatasetaì  An abstract class representing a :class:`Dataset`.

    All datasets that represent a map from keys to data samples should subclass
    it. All subclasses should overrite :meth:`__getitem__`, supporting fetching a
    data sample for a given key. Subclasses could also optionally overwrite
    :meth:`__len__`, which is expected to return the size of the dataset by many
    :class:`~torch.utils.data.Sampler` implementations and the default options
    of :class:`~torch.utils.data.DataLoader`.

    .. note::
      :class:`~torch.utils.data.DataLoader` by default constructs a index
      sampler that yields integral indices.  To make it work with a map-style
      dataset with non-integral indices/keys, a custom sampler must be provided.
    c             C   s   t ‚d S )N)ÚNotImplementedError)ÚselfÚindex© r   ú9/tmp/pip-build-e66j97b2/torch/torch/utils/data/dataset.pyÚ__getitem__   s    zDataset.__getitem__c             C   s   t | |gƒS )N)ÚConcatDataset)r   Úotherr   r   r	   Ú__add__   s    zDataset.__add__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   r   r   r   r   r	   r      s   r   c               @   s    e Zd ZdZdd„ Zdd„ ZdS )ÚIterableDataseta€  An iterable Dataset.

    All datasets that represent an iterable of data samples should subclass it.
    Such form of datasets is particularly useful when data come from a stream.

    All subclasses should overrite :meth:`__iter__`, which would return an
    iterator of samples in this dataset.

    When a subclass is used with :class:`~torch.utils.data.DataLoader`, each
    item in the dataset will be yielded from the :class:`~torch.utils.data.DataLoader`
    iterator. When :attr:`num_workers > 0`, each worker process will have a
    different copy of the dataset object, so it is often desired to configure
    each copy independently to avoid having duplicate data returned from the
    workers. :func:`~torch.utils.data.get_worker_info`, when called in a worker
    process, returns information about the worker. It can be used in either the
    dataset's :meth:`__iter__` method or the :class:`~torch.utils.data.DataLoader` 's
    :attr:`worker_init_fn` option to modify each copy's behavior.

    Example 1: splitting workload across all workers in :meth:`__iter__`::

        >>> class MyIterableDataset(torch.utils.data.IterableDataset):
        ...     def __init__(self, start, end):
        ...         super(MyIterableDataset).__init__()
        ...         assert end > start, "this example code only works with end >= start"
        ...         self.start = start
        ...         self.end = end
        ...
        ...     def __iter__(self):
        ...         worker_info = torch.utils.data.get_worker_info()
        ...         if worker_info is None:  # single-process data loading, return the full iterator
        ...             iter_start = self.start
        ...             iter_end = self.end
        ...         else:  # in a worker process
        ...             # split workload
        ...             per_worker = int(math.ceil((self.end - self.start) / float(worker_info.num_workers)))
        ...             worker_id = worker_info.id
        ...             iter_start = self.start + worker_id * per_worker
        ...             iter_end = min(iter_start + per_worker, self.end)
        ...         return iter(range(iter_start, iter_end))
        ...
        >>> # should give same set of data as range(3, 7), i.e., [3, 4, 5, 6].
        >>> ds = MyIterableDataset(start=3, end=7)

        >>> # Single-process loading
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=0)))
        [3, 4, 5, 6]

        >>> # Mult-process loading with two worker processes
        >>> # Worker 0 fetched [3, 4].  Worker 1 fetched [5, 6].
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=2)))
        [3, 5, 4, 6]

        >>> # With even more workers
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=20)))
        [3, 4, 5, 6]

    Example 2: splitting workload across all workers using :attr:`worker_init_fn`::

        >>> class MyIterableDataset(torch.utils.data.IterableDataset):
        ...     def __init__(self, start, end):
        ...         super(MyIterableDataset).__init__()
        ...         assert end > start, "this example code only works with end >= start"
        ...         self.start = start
        ...         self.end = end
        ...
        ...     def __iter__(self):
        ...         return iter(range(self.start, self.end))
        ...
        >>> # should give same set of data as range(3, 7), i.e., [3, 4, 5, 6].
        >>> ds = MyIterableDataset(start=3, end=7)

        >>> # Single-process loading
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=0)))
        [3, 4, 5, 6]
        >>>
        >>> # Directly doing multi-process loading yields duplicate data
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=2)))
        [3, 3, 4, 4, 5, 5, 6, 6]

        >>> # Define a `worker_init_fn` that configures each dataset copy differently
        >>> def worker_init_fn(worker_id):
        ...     worker_info = torch.utils.data.get_worker_info()
        ...     dataset = worker_info.dataset  # the dataset copy in this worker process
        ...     overall_start = dataset.start
        ...     overall_end = dataset.end
        ...     # configure the dataset to only process the split workload
        ...     per_worker = int(math.ceil((overall_end - overall_start) / float(worker_info.num_workers)))
        ...     worker_id = worker_info.id
        ...     dataset.start = overall_start + worker_id * per_worker
        ...     dataset.end = min(dataset.start + per_worker, overall_end)
        ...

        >>> # Mult-process loading with the custom `worker_init_fn`
        >>> # Worker 0 fetched [3, 4].  Worker 1 fetched [5, 6].
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=2, worker_init_fn=worker_init_fn)))
        [3, 5, 4, 6]

        >>> # With even more workers
        >>> print(list(torch.utils.data.DataLoader(ds, num_workers=20, worker_init_fn=worker_init_fn)))
        [3, 4, 5, 6]
    c             C   s   t ‚d S )N)r   )r   r   r   r	   Ú__iter__Š   s    zIterableDataset.__iter__c             C   s   t | |gƒS )N)ÚChainDataset)r   r   r   r   r	   r   �   s    zIterableDataset.__add__N)r   r   r   r   r   r   r   r   r   r	   r   #   s   er   c               @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚTensorDatasetzÓDataset wrapping tensors.

    Each sample will be retrieved by indexing tensors along the first dimension.

    Arguments:
        *tensors (Tensor): tensors that have the same size of the first dimension.
    c                s$   t ‡ fdd„ˆ D ƒƒst‚ˆ | _d S )Nc             3   s&   | ]}ˆ d  j d ƒ|j d ƒkV  qdS )r   N)Úsize)Ú.0Útensor)Útensorsr   r	   ú	<genexpr>ž   s    z)TensorDataset.__init__.<locals>.<genexpr>)ÚallÚAssertionErrorr   )r   r   r   )r   r	   Ú__init__�   s    zTensorDataset.__init__c                s   t ‡ fdd„| jD ƒƒS )Nc             3   s   | ]}|ˆ  V  qd S )Nr   )r   r   )r   r   r	   r   ¢   s    z,TensorDataset.__getitem__.<locals>.<genexpr>)Útupler   )r   r   r   )r   r	   r
   ¡   s    zTensorDataset.__getitem__c             C   s   | j d jdƒS )Nr   )r   r   )r   r   r   r	   Ú__len__¤   s    zTensorDataset.__len__N)r   r   r   r   r   r
   r   r   r   r   r	   r   ”   s   r   c                   sH   e Zd ZdZedd„ ƒZ‡ fdd„Zdd„ Zdd	„ Ze	d
d„ ƒZ
‡  ZS )r   zÉDataset as a concatenation of multiple datasets.

    This class is useful to assemble different existing datasets.

    Arguments:
        datasets (sequence): List of datasets to be concatenated
    c             C   s:   g d }}x*| D ]"}t |ƒ}|j|| ƒ ||7 }qW |S )Nr   )ÚlenÚappend)ÚsequenceÚrÚsÚeÚlr   r   r	   Úcumsum±   s    

zConcatDataset.cumsumc                sb   t t| ƒjƒ  t|ƒdks"tdƒ‚t|ƒ| _x"| jD ]}t|tƒ s4tdƒ‚q4W | j	| jƒ| _
d S )Nr   z(datasets should not be an empty iterablez.ConcatDataset does not support IterableDataset)Úsuperr   r   r    r   ÚlistÚdatasetsÚ
isinstancer   r'   Úcumulative_sizes)r   r*   Úd)Ú	__class__r   r	   r   º   s    
zConcatDataset.__init__c             C   s
   | j d S )Né   éÿÿÿÿ)r,   )r   r   r   r	   r   Â   s    zConcatDataset.__len__c             C   sf   |dk r*| t | ƒkrtdƒ‚t | ƒ| }tj| j|ƒ}|dkrF|}n|| j|d   }| j| | S )Nr   z8absolute value of index should not exceed dataset lengthr/   )r    Ú
ValueErrorÚbisectÚbisect_rightr,   r*   )r   ÚidxZdataset_idxZ
sample_idxr   r   r	   r
   Å   s    zConcatDataset.__getitem__c             C   s   t jdtdd� | jS )Nz:cummulative_sizes attribute is renamed to cumulative_sizesé   )Ú
stacklevel)ÚwarningsÚwarnÚDeprecationWarningr,   )r   r   r   r	   Úcummulative_sizesÑ   s    
zConcatDataset.cummulative_sizes)r   r   r   r   Ústaticmethodr'   r   r   r
   Úpropertyr:   Ú__classcell__r   r   )r.   r	   r   ¨   s   	r   c                   s0   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Z‡  ZS )r   af  Dataset for chainning multiple :class:`IterableDataset` s.

    This class is useful to assemble different existing dataset streams. The
    chainning operation is done on-the-fly, so concatenating large-scale
    datasets with this class will be efficient.

    Arguments:
        datasets (iterable of IterableDataset): datasets to be chained together
    c                s   t t| ƒjƒ  || _d S )N)r(   r   r   r*   )r   r*   )r.   r   r	   r   â   s    zChainDataset.__init__c             c   s:   x4| j D ]*}t|tƒstdƒ‚x|D ]
}|V  q$W qW d S )Nz*ChainDataset only supports IterableDataset)r*   r+   r   r   )r   r-   Úxr   r   r	   r   æ   s    
zChainDataset.__iter__c             C   s6   d}x,| j D ]"}t|tƒs"tdƒ‚|t|ƒ7 }qW |S )Nr   z*ChainDataset only supports IterableDataset)r*   r+   r   r   r    )r   Útotalr-   r   r   r	   r   ì   s
    zChainDataset.__len__)r   r   r   r   r   r   r   r=   r   r   )r.   r	   r   Ø   s   	r   c               @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚSubsetz¹
    Subset of a dataset at specified indices.

    Arguments:
        dataset (Dataset): The whole Dataset
        indices (sequence): Indices in the whole set selected for subset
    c             C   s   || _ || _d S )N)ÚdatasetÚindices)r   rA   rB   r   r   r	   r   ü   s    zSubset.__init__c             C   s   | j | j|  S )N)rA   rB   )r   r4   r   r   r	   r
      s    zSubset.__getitem__c             C   s
   t | jƒS )N)r    rB   )r   r   r   r	   r     s    zSubset.__len__N)r   r   r   r   r   r
   r   r   r   r   r	   r@   ô   s   r@   c                sF   t |ƒtˆ ƒkrtdƒ‚tt |ƒƒjƒ ‰‡ ‡fdd„tt|ƒ|ƒD ƒS )zÒ
    Randomly split a dataset into non-overlapping new datasets of given lengths.

    Arguments:
        dataset (Dataset): Dataset to be split
        lengths (sequence): lengths of splits to be produced
    zDSum of input lengths does not equal the length of the input dataset!c                s&   g | ]\}}t ˆ ˆ|| |… ƒ‘qS r   )r@   )r   ÚoffsetÚlength)rA   rB   r   r	   ú
<listcomp>  s    z random_split.<locals>.<listcomp>)Úsumr    r1   r   ÚtolistÚzipr   )rA   Úlengthsr   )rA   rB   r	   Úrandom_split  s    rJ   )r2   r7   Ztorch._utilsr   Ztorchr   Úobjectr   r   r   r   r   r@   rJ   r   r   r   r	   Ú<module>   s   q0