<commit_before>import numpy as N
from pylearn2.datasets import dense_design_matrix
from pylearn2.utils import serial

class CIFAR100(dense_design_matrix.DenseDesignMatrix):
    def __init__(self, which_set, center = False):

        assert which_set in ['train','test']

        path = "${PYLEARN2_DATA_PATH}/cifar100/cifar-100-python/"+which_set

        obj = serial.load(path)
        X = obj['data']

        assert X.max() == 255.
        assert X.min() == 0.

        X = N.cast['float32'](X)
        y = None #not implemented yet

        self.center = center

        if center:
            X -= 127.5

        view_converter = dense_design_matrix.DefaultViewConverter((32,32,3))

        super(CIFAR100,self).__init__(X = X, y =y, view_converter = view_converter)

        assert not N.any(N.isnan(self.X))

        self.y_fine = N.asarray(obj['fine_labels'])
        self.y_coarse = N.asarray(obj['coarse_labels'])


    def adjust_for_viewer(self, X):
<commit_msg>made adjust_for_viewer work for older saved cifar100 datasets<commit_after>import numpy as np
N = np
from pylearn2.datasets import dense_design_matrix
from pylearn2.utils import serial

class CIFAR100(dense_design_matrix.DenseDesignMatrix):
    def __init__(self, which_set, center = False):

        assert which_set in ['train','test']

        path = "${PYLEARN2_DATA_PATH}/cifar100/cifar-100-python/"+which_set

        obj = serial.load(path)
        X = obj['data']

        assert X.max() == 255.
        assert X.min() == 0.

        X = N.cast['float32'](X)
        y = None #not implemented yet

        self.center = center

        if center:
            X -= 127.5

        view_converter = dense_design_matrix.DefaultViewConverter((32,32,3))

        super(CIFAR100,self).__init__(X = X, y =y, view_converter = view_converter)

        assert not N.any(N.isnan(self.X))

        self.y_fine = N.asarray(obj['fine_labels'])
        self.y_coarse = N.asarray(obj['coarse_labels'])


    def adjust_for_viewer(self, X):

        #this is a bit of a hack
        #it detects older saved pkls, which are probably preprocessed somehow
        #really, the preprocessing needs to update the behavior of adjust_for_viewer
        if not hasattr(self, 'center'):
            rval = X.copy()
            rval = rval.T

            rval /= np.abs(rval).max(axis=0)
            return rval.T


{'fraction_non_alphanumeric': 0.11223628691983123, 'fraction_numerical': 0.0270042194092827, 'mean_word_length': 2.5707831325301207, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5083186', 'n_tokens_mistral': 845, 'n_tokens_neox': 762, 'n_words': 180}