<commit_before>import numpy as N
from pylearn2.datasets import dense_design_matrix
from pylearn2.utils import serial

class MNIST(dense_design_matrix.DenseDesignMatrix):
    def __init__(self, which_set, center = False):

        if which_set not in ['train','test']:
            if which_set == 'valid':
                raise ValueError("There is no such thing as the MNIST "
"validation set. MNIST consists of 60,000 train examples and 10,000 test"
" examples. If you wish to use a validation set you should divide the train "
"set yourself. The pylearn2 dataset implements and will only ever implement "
"the standard train / test split used in the literature.")
            raise ValueError('Unrecognized which_set value "%s".' %
                    (which_set,)+'". Valid values are ["train","test"].')


        path = "${PYLEARN2_DATA_PATH}/mnist/mnist-python/%s.pkl" % which_set

        obj = serial.load(path)
        X = obj['data']
        X = N.cast['float32'](X)
        y = N.asarray(obj['labels'])

        assert len(X.shape) == 2
        assert X.shape[1] == 784

        if which_set == 'train':
            assert X.shape[0] == 60000
        elif which_set == 'test':
            assert X.shape[0] == 10000
        else:
            assert False


        if center:
            X -= X.mean(axis=0)

        view_converter = dense_design_matrix.DefaultViewConverter((28,28,1))

        super(MNIST,self).__init__(X = X, y = y, view_converter = view_converter)

        assert not N.any(N.isnan(self.X))
<commit_msg>made mnist support shuffling<commit_after>import numpy as N
np = N
from pylearn2.datasets import dense_design_matrix
from pylearn2.utils import serial

class MNIST(dense_design_matrix.DenseDesignMatrix):
    def __init__(self, which_set, center = False, shuffle = False):

        if which_set not in ['train','test']:
            if which_set == 'valid':
                raise ValueError("There is no such thing as the MNIST "
"validation set. MNIST consists of 60,000 train examples and 10,000 test"
" examples. If you wish to use a validation set you should divide the train "
"set yourself. The pylearn2 dataset implements and will only ever implement "
"the standard train / test split used in the literature.")
            raise ValueError('Unrecognized which_set value "%s".' %
                    (which_set,)+'". Valid values are ["train","test"].')


        path = "${PYLEARN2_DATA_PATH}/mnist/mnist-python/%s.pkl" % which_set

        obj = serial.load(path)
        X = obj['data']
        X = N.cast['float32'](X)
        y = N.asarray(obj['labels'])

        assert len(X.shape) == 2
        assert X.shape[1] == 784

        if which_set == 'train':
            assert X.shape[0] == 60000
        elif which_set == 'test':
            assert X.shape[0] == 10000
        else:
            assert False


        if center:
            X -= X.mean(axis=0)

        if shuffle:
            self.shuffle_rng = np.random.RandomState([1,2,3])
            for i in xrange(X.shape[0]):
                j = self.shuffle_rng.randint(X.shape[0])
                tmp = X[i,:]
                X[i,:] = X[j,:]
                X[j,:] = tmp
                tmp = y[i]
                y[i] = y[j]
                y[j] = tmp


        view_converter = dense_design_matrix.DefaultViewConverter((28,28,1))

        super(MNIST,self).__init__(X = X, y = y, view_converter = view_converter)

        assert not N.any(N.isnan(self.X))

{'fraction_non_alphanumeric': 0.11723938781403408, 'fraction_numerical': 0.02396765809991337, 'mean_word_length': 2.246485473289597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4708676', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1041, 'n_words': 305}