<commit_before>import numpy as np
from pylearn2.datasets import dense_design_matrix
from pylearn2.utils.serial import load

class STL10(dense_design_matrix.DenseDesignMatrix):
    def __init__(self, which_set, center = False):

        if which_set == 'train':
            train = load('${PYLEARN2_DATA_PATH}/stl10_matlab/train.mat')

            #Load the class names
            self.class_names = [array[0].encode('utf-8') for array in train['class_names'][0] ]

            #Load the fold indices
            fold_indices = train['fold_indices']
            assert fold_indices.shape == (1,10)
            self.fold_indices = np.zeros((10,1000),dtype='uint16')
            for i in xrange(10):
                indices = fold_indices[0,i]
                assert indices.shape == (1000,1)
                assert indices.dtype == 'uint16'
                self.fold_indices[i,:] = indices[:,0]

            #The data is stored as uint8
            #If we leave it as uint8, it will cause the CAE to silently fail
            #since theano will treat derivatives wrt X as 0
            X = np.cast['float32'](train['X'])

            assert X.shape == (5000, 96*96*3)

            #this is uint8
            y = train['y'][:,0]
            assert y.shape == (5000,)
        elif which_set == 'test':
            test = load('${PYLEARN2_DATA_PATH}/stl10_matlab/test.mat')

            #Load the class names
            self.class_names = [array[0].encode('utf-8') for array in test['class_names'][0] ]

            #The data is stored as uint8
            #If we leave it as uint8, it will cause the CAE to silently fail
            #since theano will treat derivatives wrt X as 0
            X = np.cast['float32'](test['X'])

            assert X.shape == (8000, 96*96*3)

            #this is uint8
            y = test['y'][:,0]
            assert y.shape == (8000,)

        elif which_set == 'unlabeled':
            raise NotImplementedError()
        else:
            raise ValueError('"'+which_set+'" is not an STL10 dataset. '
                    'Recognized values are "train", "test", and "unlabeled".')
        if center:
            X -= 127.5

        view_converter = dense_design_matrix.DefaultViewConverter((32,32,3))

        super(STL10,self).__init__(X = X, y = y, view_converter = view_converter)

        assert not np.any(np.isnan(self.X))
    #

#<commit_msg>finished implementation of STL10 class<commit_after>import numpy as np
from pylearn2.datasets import dense_design_matrix
from pylearn2.utils.serial import load

class STL10(dense_design_matrix.DenseDesignMatrix):
    def __init__(self, which_set, center = False, example_range = None):

        if which_set == 'train':
            train = load('${PYLEARN2_DATA_PATH}/stl10_matlab/train.mat')

            #Load the class names
            self.class_names = [array[0].encode('utf-8') for array in train['class_names'][0] ]

            #Load the fold indices
            fold_indices = train['fold_indices']
            assert fold_indices.shape == (1,10)
            self.fold_indices = np.zeros((10,1000),dtype='uint16')
            for i in xrange(10):
                indices = fold_indices[0,i]
                assert indices.shape == (1000,1)
                assert indices.dtype == 'uint16'
                self.fold_indices[i,:] = indices[:,0]

            #The data is stored as uint8
            #If we leave it as uint8, it will cause the CAE to silently fail
            #since theano will treat derivatives wrt X as 0
            X = np.cast['float32'](train['X'])

            assert X.shape == (5000, 96*96*3)

            if example_range is not None:
                X = X[example_range[0]:example_range[1],:]

            #this is uint8
            y = train['y'][:,0]
            assert y.shape == (5000,)
        elif which_set == 'test':
            test = load('${PYLEARN2_DATA_PATH}/stl10_matlab/test.mat')

            #Load the class names
            self.class_names = [array[0].encode('utf-8') for array in test['class_names'][0] ]

            #The data is stored as uint8
            #If we leave it as uint8, it will cause the CAE to silently fail
            #since theano will treat derivatives wrt X as 0

            X = np.cast['float32'](test['X'])
            assert X.shape == (8000, 96*96*3)

            if example_range is not None:
                X = X[example_range[0]:example_range[1],:]

            #this is uint8
            y = test['y'][:,0]
            assert y.shape == (8000,)

        elif which_set == 'unlabeled':
            unlabeled = load('${PYLEARN2_DATA_PATH}/stl10_matlab/unlabeled.mat')

            X =  unlabeled['X']

            #this file is stored in HDF format, which transposes everything
            assert X.shape == (96*96*3, 100000)
            assert X.dtype == 'uint8'

            if example_range is None:
                X = X.value
            else:
                X = X.value[:,example_range[0]:example_range[1]]
            X = np.cast['float32'](X.T)

            unlabeled.close()

            y = None
        else:
            raise ValueError('"'+which_set+'" is not an STL10 dataset. '
                    'Recognized values are "train", "test", and "unlabeled".')
        if center:
            X -= 127.5

        view_converter = dense_design_matrix.DefaultViewConverter((96,96,3))

        super(STL10,self).__init__(X = X, y = y, view_converter = view_converter)


        for i in xrange(self.X.shape[0]):
            mat = X[i:i+1,:]
            topo = self.get_topological_view(mat)
            for j in xrange(topo.shape[3]):
                temp = topo[0,:,:,j].T.copy()
                topo[0,:,:,j] = temp
            mat = self.get_design_matrix(topo)
            X[i:i+1,:] = mat

        assert not np.any(np.isnan(self.X))
    #

#
{'fraction_non_alphanumeric': 0.11952601751674395, 'fraction_numerical': 0.03657908294693457, 'mean_word_length': 1.9032901296111664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5599656', 'n_tokens_mistral': 1916, 'n_tokens_neox': 1748, 'n_words': 463}