<commit_before>"""
Cross-validation dataset iterators.
"""
import numpy as np
import warnings

try:
    from sklearn.cross_validation import (KFold, StratifiedKFold, ShuffleSplit,
                                          StratifiedShuffleSplit)
except ImportError:
    warnings.warn("Could not import from sklearn.")

try:
    from collections import OrderedDict
except ImportError:
    warnings.warn("Could not import from collections.")

from pylearn2.cross_validation.blocks import StackedBlocksCV
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.datasets.transformer_dataset import TransformerDataset


class DatasetCV(object):
    """
    Construct a new DenseDesignMatrix for each subset.

    Parameters
    ----------
    dataset : object
        Full dataset for use in cross validation.
    index_iterator : iterable
        Iterable that returns (train, test) or (train, valid, test) indices
        for slicing the dataset during cross validation.
    which_set : str, list or None
        If None, return all subset datasets. If one or more of 'train',
        'valid', or 'test', return only the dataset(s) corresponding to the
        given subset(s).
    return_dict : bool
        Whether to return subset datasets as a dictionary. If True,
        returns a dict with keys 'train', 'valid', and/or 'test' (if
        index_iterator returns two slices per partition, 'train' and 'test'
        are used, and if index_iterator returns three slices per partition,
        'train', 'valid', and 'test' are used). If False, returns a list of
        datasets matching the slice order given by index_iterator.
    """
    def __init__(self, dataset, index_iterator, which_set=None,
                 return_dict=True):
        self.dataset = dataset
        self.index_iterator = list(index_iterator)  # allow reuse of generators
        dataset_iterator = dataset.iterator(mode='sequential', num_batches=1,
                                            data_specs=dataset.data_specs)
        self._data = tuple(dataset_iterator.next())
        self.which_set = which_set
        if which_set is not None:
            which_set = np.atleast_1d(which_set)
            assert len(which_set)
            for label in which_set:
                if label not in ['train', 'valid', 'test']:
                    raise ValueError("Unrecognized subset '{}'".format(label))
            self.which_set = which_set
        self.return_dict = return_dict

    def get_data_subsets(self):
        """
        Partition the dataset according to cross-validation subsets and
        return the raw data in each subset.
        """
        for subsets in self.index_iterator:
            labels = None
            if len(subsets) == 3:
                labels = ['train', 'valid', 'test']
            elif len(subsets) == 2:
                labels = ['train', 'test']
            # data_subsets is an OrderedDict to maintain label order
            data_subsets = OrderedDict()
            for i, subset in enumerate(subsets):
                subset_data = tuple(data[subset] for data in self._data)
                if len(subset_data) == 2:
                    X, y = subset_data
                else:
                    X, = subset_data
                    y = None
                label = labels[i]
                print "DATASET", label, X.shape
                if self.which_set is None or label in self.which_set:
                    data_subsets[labels[i]] = (X, y)
            if not len(data_subsets):
                raise ValueError("No matching dataset(s) for " +
                                 "{}".format(self.which_set))<commit_msg>Remove debug comment<commit_after>"""
Cross-validation dataset iterators.
"""
import numpy as np
import warnings

try:
    from sklearn.cross_validation import (KFold, StratifiedKFold, ShuffleSplit,
                                          StratifiedShuffleSplit)
except ImportError:
    warnings.warn("Could not import from sklearn.")

try:
    from collections import OrderedDict
except ImportError:
    warnings.warn("Could not import from collections.")

from pylearn2.cross_validation.blocks import StackedBlocksCV
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
from pylearn2.datasets.transformer_dataset import TransformerDataset


class DatasetCV(object):
    """
    Construct a new DenseDesignMatrix for each subset.

    Parameters
    ----------
    dataset : object
        Full dataset for use in cross validation.
    index_iterator : iterable
        Iterable that returns (train, test) or (train, valid, test) indices
        for slicing the dataset during cross validation.
    which_set : str, list or None
        If None, return all subset datasets. If one or more of 'train',
        'valid', or 'test', return only the dataset(s) corresponding to the
        given subset(s).
    return_dict : bool
        Whether to return subset datasets as a dictionary. If True,
        returns a dict with keys 'train', 'valid', and/or 'test' (if
        index_iterator returns two slices per partition, 'train' and 'test'
        are used, and if index_iterator returns three slices per partition,
        'train', 'valid', and 'test' are used). If False, returns a list of
        datasets matching the slice order given by index_iterator.
    """
    def __init__(self, dataset, index_iterator, which_set=None,
                 return_dict=True):
        self.dataset = dataset
        self.index_iterator = list(index_iterator)  # allow reuse of generators
        dataset_iterator = dataset.iterator(mode='sequential', num_batches=1,
                                            data_specs=dataset.data_specs)
        self._data = tuple(dataset_iterator.next())
        self.which_set = which_set
        if which_set is not None:
            which_set = np.atleast_1d(which_set)
            assert len(which_set)
            for label in which_set:
                if label not in ['train', 'valid', 'test']:
                    raise ValueError("Unrecognized subset '{}'".format(label))
            self.which_set = which_set
        self.return_dict = return_dict

    def get_data_subsets(self):
        """
        Partition the dataset according to cross-validation subsets and
        return the raw data in each subset.
        """
        for subsets in self.index_iterator:
            labels = None
            if len(subsets) == 3:
                labels = ['train', 'valid', 'test']
            elif len(subsets) == 2:
                labels = ['train', 'test']
            # data_subsets is an OrderedDict to maintain label order
            data_subsets = OrderedDict()
            for i, subset in enumerate(subsets):
                subset_data = tuple(data[subset] for data in self._data)
                if len(subset_data) == 2:
                    X, y = subset_data
                else:
                    X, = subset_data
                    y = None
                label = labels[i]
                if self.which_set is None or label in self.which_set:
                    data_subsets[labels[i]] = (X, y)
            if not len(data_subsets):
                raise ValueError("No matching dataset(s) for " +
                                 "{}".format(self.which_set))
{'fraction_non_alphanumeric': 0.08031194417841019, 'fraction_numerical': 0.0021890819537556437, 'mean_word_length': 2.081787521079258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '926003', 'n_tokens_mistral': 1912, 'n_tokens_neox': 1857, 'n_words': 639}