<commit_before>"""Base class for the components in other modules."""
import cPickle
import os.path
from itertools import izip

import theano
from theano import tensor

theano.config.warn.sum_div_dimshuffle_bug = False
floatX = theano.config.floatX

if 0:
    print 'WARNING: using SLOW rng'
    RandomStreams = tensor.shared_randomstreams.RandomStreams
else:
    import theano.sandbox.rng_mrg
    RandomStreams = theano.sandbox.rng_mrg.MRG_RandomStreams

class Block(object):
    """
    Basic building block for deep architectures.
    """
    def params(self):
        """
        Returns a list of *shared* learnable parameters that
        are, in your judgment, typically learned in this
        model.
        """
        # NOTE: We return list(self._params) rather than self._params
        # in order to explicitly make a copy, so that the list isn't
        # absentmindedly modified. If a user really knows what they're
        # doing they can modify self._params.
        return list(self._params)

    def __call__(self, inputs):
        raise NotImplementedError('__call__')

    def save(self, save_dir, save_file):
        """
        Dumps the entire object to a pickle file.
        Individual classes should override __getstate__ and __setstate__
        to deal with object versioning in the case of API changes.
        """
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        if not os.path.isdir(save_dir):
            raise IOError('save_dir %s is not a directory' % save_dir)
        else:
            fhandle = open(os.path.join(save_dir, save_file), 'w')
            cPickle.dump(self, fhandle, -1)

    @classmethod
    def load(cls, save_file):
        """Load a serialized block."""
        obj = cPickle.load(open(save_file))
        if isinstance(obj, cls):
            return obj
        else:
            raise TypeError('unpickled object was of wrong class: %s' %
                            obj.__class__)

class Optimizer(object):
    """
    Basic abstract class for computing parameter updates of a model.
    """
    def updates(self):
        """Return symbolic updates to apply"""
        raise NotImplementedError()

    def function(self, inputs):
        """Return a compiled Theano function for training"""
        raise NotImplementedError()<commit_msg>Clean up and homogenize load/save code.  (You're right, David; I hadn't actually looked at the base.py load/save code.)<commit_after>"""Base class for the components in other modules."""
import cPickle
import os.path
from itertools import izip

import theano
from theano import tensor

theano.config.warn.sum_div_dimshuffle_bug = False
floatX = theano.config.floatX

if 0:
    print 'WARNING: using SLOW rng'
    RandomStreams = tensor.shared_randomstreams.RandomStreams
else:
    import theano.sandbox.rng_mrg
    RandomStreams = theano.sandbox.rng_mrg.MRG_RandomStreams

class Block(object):
    """
    Basic building block for deep architectures.
    """
    def params(self):
        """
        Returns a list of *shared* learnable parameters that
        are, in your judgment, typically learned in this
        model.
        """
        # NOTE: We return list(self._params) rather than self._params
        # in order to explicitly make a copy, so that the list isn't
        # absentmindedly modified. If a user really knows what they're
        # doing they can modify self._params.
        return list(self._params)

    def __call__(self, inputs):
        raise NotImplementedError('__call__')

    def save(self, save_dir, save_file):
        """
        Dumps the entire object to a pickle file.
        Individual classes should override __getstate__ and __setstate__
        to deal with object versioning in the case of API changes.
        """
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        if not os.path.isdir(save_dir):
            raise IOError('save_dir %s is not a directory' % save_dir)
        else:
            fhandle = open(os.path.join(save_dir, save_file), 'w')
            cPickle.dump(self, fhandle, -1)
            fhandle.close()

    @classmethod
    def load(cls, load_dir, load_file):
        """Load a serialized block."""
        filename = os.path.join(load_dir, load_file)
        if not os.path.isfile(filename):
            raise IOError('File %s does not exist' % filename)
        obj = cPickle.load(open(filename))
        if isinstance(obj, cls):
            return obj
        else:
            raise TypeError('unpickled object was of wrong class: %s' %
                            obj.__class__)

class Optimizer(object):
    """
    Basic abstract class for computing parameter updates of a model.
    """
    def updates(self):
        """Return symbolic updates to apply"""
        raise NotImplementedError()

    def function(self, inputs):
        """Return a compiled Theano function for training"""
        raise NotImplementedError()
{'fraction_non_alphanumeric': 0.08558740411788454, 'fraction_numerical': 0.0008074283407347598, 'mean_word_length': 2.48943661971831, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023179', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1372, 'n_words': 467}