<commit_before>"""TODO: module-level docstring."""
import numpy as np
N = np
import copy
from pylearn2.datasets.dataset import Dataset
from pylearn2.datasets import control
from theano import config

class DenseDesignMatrix(Dataset):
    """A class for representing datasets that can be stored
       as a dense design matrix, such as MNIST or CIFAR10.
       """
    def __init__(self, X=None, topo_view=None, y=None,
                 view_converter=None, rng=None):
        """
            Parameters
            ----------

            X:  Should be supplied if topo_view is not
                A design matrix of shape (number examples, number features)
                that defines the dataset

            topo_view:  Should be supplied if X is not.
                        A tensor whose first dimension is of length number
                        examples. The remaining tensor dimensions are examples
                        with topological significance, e.g. for images
                        the remaining axes are rows, columns, and channels.
            y:  Labels for the examples. Optional.
            view_converter: An object for converting between design matrices
                            and topological views. Currently DefaultViewConverter
                            is the only type available but later we may want
                            to add one that uses the retina encoding that the
                            U of T group uses.
            rng:    A random number generator used for picking random indices
                    into the design matrix when choosing minibatches
        """


        if X is not None:
            self.X = X
            self.view_converter = view_converter
        else:
            assert topo_view is not None
            self.set_topological_view(topo_view)
        #

        self.y = y
        if rng is None:
            rng = N.random.RandomState([17, 2, 946])
        #
        self.default_rng = copy.copy(rng)
        self.rng = rng
        self.compress = False
        self.design_loc = None

    def use_design_loc(self, path):
        """ When pickling, save the design matrix to path as a .npy file
            rather than pickling the design matrix along with the rest
            of the dataset object. This avoids pickle's unfortunate
            behavior of using 2X the RAM when unpickling. """
        self.design_loc = path

    def enable_compression(self):
        """ If called, when pickled the dataset will be saved using only
            8 bits per element """
        self.compress = True

    def __getstate__(self):
        rval = copy.copy(self.__dict__)

        if self.compress:
            rval['compress_min'] = rval['X'].min(axis=0)
            rval['X'] -= rval['compress_min']
            rval['compress_max'] = rval['X'].max(axis=0)
            rval['compress_max'][rval['compress_max'] == 0] = 1
            rval['X'] *= 255. / rval['compress_max']
            rval['X'] = N.cast['uint8'](rval['X'])

        if self.design_loc is not None:
            N.save(self.design_loc, rval['X'])
            del rval['X']

        return rval

    def __setstate__(self, d):

        if d['design_loc'] is not None:
            if control.get_load_data():
                d['X'] = N.load(d['design_loc'])
            else:
                d['X'] = None

        if d['compress']:
            X = d['X']
            mx = d['compress_max']
            mn = d['compress_min']
            del d['compress_max']
            del d['compress_min']<commit_msg>fixed bug where serializing a dense design matrix modified the object<commit_after>"""TODO: module-level docstring."""
import numpy as np
N = np
import copy
from pylearn2.datasets.dataset import Dataset
from pylearn2.datasets import control
from theano import config

class DenseDesignMatrix(Dataset):
    """A class for representing datasets that can be stored
       as a dense design matrix, such as MNIST or CIFAR10.
       """
    def __init__(self, X=None, topo_view=None, y=None,
                 view_converter=None, rng=None):
        """
            Parameters
            ----------

            X:  Should be supplied if topo_view is not
                A design matrix of shape (number examples, number features)
                that defines the dataset

            topo_view:  Should be supplied if X is not.
                        A tensor whose first dimension is of length number
                        examples. The remaining tensor dimensions are examples
                        with topological significance, e.g. for images
                        the remaining axes are rows, columns, and channels.
            y:  Labels for the examples. Optional.
            view_converter: An object for converting between design matrices
                            and topological views. Currently DefaultViewConverter
                            is the only type available but later we may want
                            to add one that uses the retina encoding that the
                            U of T group uses.
            rng:    A random number generator used for picking random indices
                    into the design matrix when choosing minibatches
        """


        if X is not None:
            self.X = X
            self.view_converter = view_converter
        else:
            assert topo_view is not None
            self.set_topological_view(topo_view)
        #

        self.y = y
        if rng is None:
            rng = N.random.RandomState([17, 2, 946])
        #
        self.default_rng = copy.copy(rng)
        self.rng = rng
        self.compress = False
        self.design_loc = None

    def use_design_loc(self, path):
        """ When pickling, save the design matrix to path as a .npy file
            rather than pickling the design matrix along with the rest
            of the dataset object. This avoids pickle's unfortunate
            behavior of using 2X the RAM when unpickling. """
        self.design_loc = path

    def enable_compression(self):
        """ If called, when pickled the dataset will be saved using only
            8 bits per element """
        self.compress = True

    def __getstate__(self):
        rval = copy.copy(self.__dict__)

        if self.compress:
            rval['compress_min'] = rval['X'].min(axis=0)
            #important not to do -= on this line, as that will modify the original object
            rval['X'] = rval['X'] - rval['compress_min']
            rval['compress_max'] = rval['X'].max(axis=0)
            rval['compress_max'][rval['compress_max'] == 0] = 1
            rval['X'] *= 255. / rval['compress_max']
            rval['X'] = N.cast['uint8'](rval['X'])

        if self.design_loc is not None:
            N.save(self.design_loc, rval['X'])
            del rval['X']

        return rval

    def __setstate__(self, d):

        if d['design_loc'] is not None:
            if control.get_load_data():
                d['X'] = N.load(d['design_loc'])
            else:
                d['X'] = None

        if d['compress']:
            X = d['X']
            mx = d['compress_max']
            mn = d['compress_min']
            del d['compress_max']
            del d['compress_min']
{'fraction_non_alphanumeric': 0.08361985971668272, 'fraction_numerical': 0.0055013065603080735, 'mean_word_length': 1.6903440621531631, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3734851', 'n_tokens_mistral': 1957, 'n_tokens_neox': 1870, 'n_words': 676}