<commit_before>import numpy as np

from theano import tensor as T

from theano.compat.python2x import OrderedDict
from pylearn2.space import NullSpace
from pylearn2.train_extensions import TrainExtension
from pylearn2.utils import sharedX

class LearningRule():
    """
    A pylearn2 learning rule is an object which computes new parameter values
    given (1) a learning rate (2) current parameter values and (3) the current
    estimated gradient.
    """

    def add_channels_to_monitor(self, monitor, monitoring_dataset):
        """
        Method called by the training algorithm, which allows LearningRules to
        add monitoring channels.

        monitor: a pylearn2.monitor.Monitor object, to which the rule
        should register additional monitoring channels.
        monitoring_dataset: a Dataset instance of dictionary whose values
        are Dataset objects.
        """
        raise NotImplementedError()

    def get_updates(self, learning_rate, grads, lr_scalers=None):
        """

        Parameters
        ----------
        learning_rate: float, learning rate coefficient.
        grads: a dictionary mapping from the model's parameters
        to their gradients.
        lr_scalers: dictionary mapping from the model's parameters
        to a learning rate multiplier.

        Returns
        -------
        A dictionary mapping from the old model parameters, to their new
        values after a single iteration of the learning rule.

        e.g. for standard SGD, one would return `sgd_rule_updates` defined
        below. Note that such a LearningRule object is not implemented, as these
        updates are implemented by default when the `learning_rule` parameter
        of sgd.SGD.__init__ is None.

            sgd_rule_updates = OrderedDict()
            for (param, grad) in grads.iteritems():
                sgd_rule_updates[k] = param - learning_rate * lr_scalers.get(param, 1.) * grad
        """
        raise NotImplementedError()


class Momentum(LearningRule):
    """
    Implements momentum as described in Section 9 of 
    "A Practical Guide to Training Restricted Boltzmann Machines", <NAME>.

    Parameters are updated by the formula:
    inc := momentum * inc - learning_rate * d cost / d param
    param := param + inc
    """

    def __init__(self, init_momentum):
        """
        init_momentum: initial value for the momentum coefficient. It remains
        fixed during training unless used with a
        training_algorithms.sgd.MomentumAdjustor extension.
        """
        assert init_momentum >= 0.
        assert init_momentum < 1.
        self.momentum = sharedX(init_momentum, 'momentum')

    def add_channels_to_monitor(self, monitor, monitoring_dataset):
        monitor.add_channel(
                name='momentum',
                ipt=None,
                val=self.momentum,
                data_specs=(NullSpace(), ''),
                dataset=monitoring_dataset)

    def get_updates(self, learning_rate, grads, lr_scalers=None):

        updates = OrderedDict()

        for (param, grad) in grads.iteritems():
            inc = sharedX(param.get_value() * 0.)
            if param.name is not None:
                inc.name = 'inc_'+param.name
            updated_inc = self.momentum * inc -\
                          learning_rate * lr_scalers.get(param, 1.) * grad
            updates[inc] = updated_inc
            updates[param] = param + updated_inc

        return updates


class MomentumAdjustor(TrainExtension):

    def __init__(self, final_momentum, start, saturate):
        """
            final_momentum: the momentum coefficient to use at the end
                            of learning.
            start: the epoch on which to start growing the momentum coefficient.
            saturate: the epoch on which the moment should reach its final value
        """

        if saturate < start:
            raise TypeError("Momentum can't saturate at its maximum value before it starts increasing.")

        self.__dict__.update(locals())
        del self.self
        self._initialized = False
        self._count = 0

    def on_monitor(self, model, dataset, algorithm):
        if hasattr(algorithm, 'learning_rule'):
            momentum = algorithm.learning_rule.momentum
        else:
            # TODO: remove once training_algorithm.sgd.SGD(init_momentum)
            # is officially deprecated.
            momentum = algorithm.momentum

        if not self._initialized:
            self._init_momentum = momentum.get_value()
            self._initialized = True
        self._count += 1
        momentum.set_value( np.cast[config.floatX](self.current_momentum()))

    def current_momentum(self):
        w = self.saturate - self.start

        if w == 0:
            # saturate=start, so just jump straight to final momentum
            if self._count >= self.start:
                return self.final_momentum
            return self._init_momentum

        alpha = float(self._count - self.start) / float(w)
        if alpha < 0.:
            alpha = 0.
        if alpha > 1.:
            alpha = 1.
        return self._init_momentum * (1.-alpha)+alpha*self.final_momentum



<commit_msg>Small bug fix to make MomemtumAdjustor work with bold old (init_momentum=) and new (LearninRule) interfaces to momentum.<commit_after>import numpy as np

from theano import config
from theano import tensor as T

from theano.compat.python2x import OrderedDict
from pylearn2.space import NullSpace
from pylearn2.train_extensions import TrainExtension
from pylearn2.utils import sharedX

class LearningRule():
    """
    A pylearn2 learning rule is an object which computes new parameter values
    given (1) a learning rate (2) current parameter values and (3) the current
    estimated gradient.
    """

    def add_channels_to_monitor(self, monitor, monitoring_dataset):
        """
        Method called by the training algorithm, which allows LearningRules to
        add monitoring channels.

        monitor: a pylearn2.monitor.Monitor object, to which the rule
        should register additional monitoring channels.
        monitoring_dataset: a Dataset instance of dictionary whose values
        are Dataset objects.
        """
        raise NotImplementedError()

    def get_updates(self, learning_rate, grads, lr_scalers=None):
        """

        Parameters
        ----------
        learning_rate: float, learning rate coefficient.
        grads: a dictionary mapping from the model's parameters
        to their gradients.
        lr_scalers: dictionary mapping from the model's parameters
        to a learning rate multiplier.

        Returns
        -------
        A dictionary mapping from the old model parameters, to their new
        values after a single iteration of the learning rule.

        e.g. for standard SGD, one would return `sgd_rule_updates` defined
        below. Note that such a LearningRule object is not implemented, as these
        updates are implemented by default when the `learning_rule` parameter
        of sgd.SGD.__init__ is None.

            sgd_rule_updates = OrderedDict()
            for (param, grad) in grads.iteritems():
                sgd_rule_updates[k] = param - learning_rate * lr_scalers.get(param, 1.) * grad
        """
        raise NotImplementedError()


class Momentum(LearningRule):
    """
    Implements momentum as described in Section 9 of 
    "A Practical Guide to Training Restricted Boltzmann Machines", <NAME>.

    Parameters are updated by the formula:
    inc := momentum * inc - learning_rate * d cost / d param
    param := param + inc
    """

    def __init__(self, init_momentum):
        """
        init_momentum: initial value for the momentum coefficient. It remains
        fixed during training unless used with a
        training_algorithms.sgd.MomentumAdjustor extension.
        """
        assert init_momentum >= 0.
        assert init_momentum < 1.
        self.momentum = sharedX(init_momentum, 'momentum')

    def add_channels_to_monitor(self, monitor, monitoring_dataset):
        monitor.add_channel(
                name='momentum',
                ipt=None,
                val=self.momentum,
                data_specs=(NullSpace(), ''),
                dataset=monitoring_dataset)

    def get_updates(self, learning_rate, grads, lr_scalers=None):

        updates = OrderedDict()

        for (param, grad) in grads.iteritems():
            inc = sharedX(param.get_value() * 0.)
            if param.name is not None:
                inc.name = 'inc_'+param.name
            updated_inc = self.momentum * inc -\
                          learning_rate * lr_scalers.get(param, 1.) * grad
            updates[inc] = updated_inc
            updates[param] = param + updated_inc

        return updates


class MomentumAdjustor(TrainExtension):

    def __init__(self, final_momentum, start, saturate):
        """
            final_momentum: the momentum coefficient to use at the end
                            of learning.
            start: the epoch on which to start growing the momentum coefficient.
            saturate: the epoch on which the moment should reach its final value
        """

        if saturate < start:
            raise TypeError("Momentum can't saturate at its maximum value before it starts increasing.")

        self.__dict__.update(locals())
        del self.self
        self._initialized = False
        self._count = 0

    def on_monitor(self, model, dataset, algorithm):
        if algorithm.learning_rule:
            momentum = algorithm.learning_rule.momentum
        else:
            # TODO: remove once training_algorithm.sgd.SGD(init_momentum)
            # is officially deprecated.
            momentum = algorithm.momentum

        if not self._initialized:
            self._init_momentum = momentum.get_value()
            self._initialized = True
        self._count += 1
        momentum.set_value( np.cast[config.floatX](self.current_momentum()))

    def current_momentum(self):
        w = self.saturate - self.start

        if w == 0:
            # saturate=start, so just jump straight to final momentum
            if self._count >= self.start:
                return self.final_momentum
            return self._init_momentum

        alpha = float(self._count - self.start) / float(w)
        if alpha < 0.:
            alpha = 0.
        if alpha > 1.:
            alpha = 1.
        return self._init_momentum * (1.-alpha)+alpha*self.final_momentum




{'fraction_non_alphanumeric': 0.07421467210781058, 'fraction_numerical': 0.004365568947518269, 'mean_word_length': 2.201093560145808, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4323642', 'n_tokens_mistral': 2813, 'n_tokens_neox': 2640, 'n_words': 915}