<commit_before>"""TODO: module-level docstring."""
import numpy as np
N = np
import copy
from pylearn2.datasets.dataset import Dataset
from pylearn2.datasets import control
from theano import config


class DenseDesignMatrix(Dataset):
    """
    A class for representing datasets that can be stored as a dense design
    matrix, such as MNIST or CIFAR10.
    """
    def __init__(self, X=None, topo_view=None, y=None,
                 view_converter=None, rng=None):
        """
        Parameters
        ----------

        X : ndarray, 2-dimensional, optional
            Should be supplied if `topo_view` is not. A design
            matrix of shape (number examples, number features)
            that defines the dataset.
        topo_view : ndarray, optional
            Should be supplied if X is not.  An array whose first
            dimension is of length number examples. The remaining
            dimensions are xamples with topological significance,
            e.g. for images the remaining axes are rows, columns,
            and channels.
        y : ndarray, 1-dimensional(?), optional
            Labels or targets for each example. The semantics here
            are not quite nailed down for this yet.
        view_converter : object, optional
            An object for converting between design matrices and
            topological views. Currently DefaultViewConverter is
            the only type available but later we may want to add
            one that uses the retina encoding that the U of T group
            uses.
        rng : object, optional
            A random number generator used for picking random
            indices into the design matrix when choosing minibatches.
        """
        if X is not None:
            self.X = X
            self.view_converter = view_converter
        else:
            assert topo_view is not None
            self.set_topological_view(topo_view)
        self.y = y
        if rng is None:
            rng = N.random.RandomState([17, 2, 946])
        self.default_rng = copy.copy(rng)
        self.rng = rng
        self.compress = False
        self.design_loc = None

    def use_design_loc(self, path):
        """
        When pickling, save the design matrix to path as a .npy file rather
        than pickling the design matrix along with the rest of the dataset<commit_msg>New dataset iterator interface.<commit_after>"""TODO: module-level docstring."""
import numpy as np
from pylearn2.utils.iteration import (
    SequentialSubsetIterator,
    RandomSliceSubsetIterator,
    RandomUniformSubsetIterator
)
N = np
import copy

from pylearn2.datasets.dataset import Dataset
from pylearn2.datasets import control
from theano import config

_iteration_schemes = {
    'sequential': SequentialSubsetIterator,
    'random_slice': RandomSliceSubsetIterator,
    'random_uniform': RandomUniformSubsetIterator,
}


def _resolve_iterator_class(mode):
    if isinstance(mode, basestring) and mode not in _iteration_schemes:
        raise ValueError("unknown iteration mode string: %s" % mode)
    elif mode in _iteration_schemes:
        subset_iter_class = _iteration_schemes[mode]
    else:
        subset_iter_class = mode
    return subset_iter_class


class DatasetIterator(object):
    """A thin wrapper around one of the mode iterators."""
    def __init__(self, dataset, subset_iterator, topo=False):
        self._topo = topo
        self._dataset = dataset
        self._subset_iterator = subset_iterator
        if self._topo:
            self._raw_data = self._dataset.get_topological_view()
        else:
            self._raw_data = self._dataset.get_design_matrix()

    def __iter__(self):
        return self

    def next(self):
        # TODO: handle fancy-index copies by allocating a buffer and
        # using numpy.take()
        next_index = self._subset_iterator.next()
        return self._raw_data[next_index]


class DenseDesignMatrix(Dataset):
    """
    A class for representing datasets that can be stored as a dense design
    matrix, such as MNIST or CIFAR10.
    """
    _default_seed = (17, 2, 946)

    def __init__(self, X=None, topo_view=None, y=None,
                 view_converter=None, rng=_default_seed):
        """
        Parameters
        ----------

        X : ndarray, 2-dimensional, optional
            Should be supplied if `topo_view` is not. A design
            matrix of shape (number examples, number features)
            that defines the dataset.
        topo_view : ndarray, optional
            Should be supplied if X is not.  An array whose first
            dimension is of length number examples. The remaining
            dimensions are xamples with topological significance,
            e.g. for images the remaining axes are rows, columns,
            and channels.
        y : ndarray, 1-dimensional(?), optional
            Labels or targets for each example. The semantics here
            are not quite nailed down for this yet.
        view_converter : object, optional
            An object for converting between design matrices and
            topological views. Currently DefaultViewConverter is
            the only type available but later we may want to add
            one that uses the retina encoding that the U of T group
            uses.
        rng : object, optional
            A random number generator used for picking random
            indices into the design matrix when choosing minibatches.
        """
        if X is not None:
            self.X = X
            self.view_converter = view_converter
        else:
            assert topo_view is not None
            self.set_topological_view(topo_view)
        self.y = y
        self.compress = False
        self.design_loc = None
        if hasattr(rng, 'random_integers'):
            self.rng = rng
        else:
            self.rng = np.random.RandomState(rng)

    def set_iteration_scheme(self, mode=None, batch_size=None,
                             num_batches=None, topo=False):
        if mode is not None:
            self._iter_subset_class = _resolve_iterator_class(mode)
        else:
            mode = self._iter_subset_class
        # If this didn't raise an exception, we should be fine.
        self._iter_batch_size = batch_size
        self._iter_num_batches = num_batches
        self._iter_topo = topo
        # Try to create an iterator with these settings.
        rng = self.rng if self._iter_subset_class.stochastic else None
        print rng
        test = self.iterator(mode, batch_size, num_batches, topo, rng=rng)

    def iterator(self, mode=None, batch_size=None, num_batches=None,
                 topo=None, rng=None):
        if mode is None:
            mode = self._iter_subset_class
        else:
            mode = _resolve_iterator_class(mode)
        if batch_size is None:
            batch_size = self._iter_batch_size
        if num_batches is None:
            num_batches = self._iter_num_batches
        if topo is None:
            topo = self._iter_topo
        if rng is None and mode.stochastic:
            rng = self.rng
        return DatasetIterator(self,
                               mode(self.X.shape[0], batch_size,
                                    num_batches, rng),
                               topo)

    def __iter__(self):
        return self.iterator()

    def use_design_loc(self, path):
        """
        When pickling, save the design matrix to path as a .npy file rather
        than pickling the design matrix along with the rest of the dataset
{'fraction_non_alphanumeric': 0.06196213425129088, 'fraction_numerical': 0.0034423407917383822, 'mean_word_length': 2.087045361667348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1547077', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1982, 'n_words': 714}