<commit_before>from __future__ import division
import numpy as np
import theano.sparse
from theano import function
import theano.tensor as T
from pylearn2.monitor import Monitor
from pylearn2.training_algorithms.training_algorithm import TrainingAlgorithm
import pylearn2.costs.cost
from theano.printing import Print


# TODO: This needs renaming based on specifics. Specifically it needs
# "unsupervised" in its name, and some sort of qualification based on
# its slightly unorthodox batch selection strategy.
class SGD(TrainingAlgorithm):
    """Stochastic Gradient Descent with an optional validation set
    for error monitoring.

    TODO: right now, assumes there is just one variable, X, i.e.
    is designed for unsupervised learning need to support other tasks.

    TODO: document parameters, especially monitoring_batches
    """

    def __init__(self, learning_rate, cost, batch_size=None,
                 batches_per_iter=1000, monitoring_batches=-1,
                 monitoring_dataset=None, termination_criterion=None,
                 update_callbacks=None):
        """
        Instantiates an SGD object.

        Parameters
        ----------
        learning_rate : float
            The stochastic gradient step size, relative to your cost
            function.
        cost : object
            An object implementing the pylearn2 cost interface.
        batch_size : int, optional
            Batch size per update. TODO: What if this is not provided?
        batches_per_iter : int, optional
            How many batch updates per epoch. Default is 1000.
            TODO: Is there any way to specify "as many as the dataset
            provides"?
        monitoring_batches : int, optional
            WRITEME
        monitoring_dataset : object, optional
            WRITEME
        termination_criterion : object, optional
            WRITEME
        update_callback : iterable or object, optional
            WRITEME

        Notes
        -----
        TODO: for now, learning_rate is just a float, but later it
        should support passing in a class that dynamically adjusts the
        learning rate if batch_size is None, reverts to the
        force_batch_size field of the model if monitoring_dataset is
        provided, uses monitoring_batches batches of data from
        monitoring_dataset to report monitoring errors
        """
        #Store parameters
        self.learning_rate = float(learning_rate)
        self.batch_size = batch_size
        self.batches_per_iter = batches_per_iter
        self.cost = cost
        if monitoring_dataset is None:
            assert monitoring_batches == -1, ("no monitoring dataset, but "
                                              "monitoring_batches > 0")
        self.monitoring_dataset = monitoring_dataset
        self.monitoring_batches = monitoring_batches
        self.termination_criterion = termination_criterion
        self._register_update_callbacks(update_callbacks)
        self.bSetup = False
        self.first = True

    def setup(self, model, dataset):
        """
        Initialize the training algorithm. Should be called
        once before calls to train.

        Parameters
        ----------
        model : object
            Model to be trained.  Object implementing the pylearn2 Model
            interface.
        dataset : object
            Dataset on which to train.  Object implementing the
            pylearn2 Dataset interface.
        """

        self.model = model

        self.monitor = Monitor.get_monitor(model)
        self.monitor.set_dataset(dataset=self.monitoring_dataset,
                                 mode="sequential",
                                 batch_size=self.batch_size,
                                 num_batches=self.monitoring_batches)


        #Make the right kind of theano variable for the type of space
        #the model acts on
        space = self.model.get_input_space()
        X = space.make_theano_batch(name='sgd_X')

        if isinstance(X, theano.sparse.basic.SparseVariable):
            self.topo = False
        else:
            self.topo = len(X.type.broadcastable) > 2

        try:
            J = sum(c(model, X) for c in self.cost)
        except TypeError:
            J = self.cost(model, X)

        if J.name is None:
            J.name = 'sgd_cost(' + X.name + ')'
        self.monitor.add_channel(name=J.name, ipt=X, val=J)
        params = model.get_params()

        for i, param in enumerate(params):
            if param.name is None:
                param.name = 'sgd_params[%d]' % i

        grads = dict(zip(params, T.grad(J, params)))

        for param in grads:

            if grads[param].name is None:
                grads[param].name = ('grad(%(costname)s, %(paramname)s)' %
                                     {'costname': J.name,
                                      'paramname': param.name})

        learning_rate = T.scalar('sgd_learning_rate')

        updates = dict(zip(params, [param - learning_rate * grads[param]
                                    for param in params]))

        for param in updates:
            if updates[param].name is None:
                updates[param].name = 'sgd_update(' + param.name + ')'<commit_msg>sgd bug fixes<commit_after>from __future__ import division
import numpy as np
import theano.sparse
from theano import function
import theano.tensor as T
from pylearn2.monitor import Monitor
from pylearn2.training_algorithms.training_algorithm import TrainingAlgorithm
import pylearn2.costs.cost
from theano.printing import Print


# TODO: This needs renaming based on specifics. Specifically it needs
# "unsupervised" in its name, and some sort of qualification based on
# its slightly unorthodox batch selection strategy.
class SGD(TrainingAlgorithm):
    """Stochastic Gradient Descent with an optional validation set
    for error monitoring.

    TODO: right now, assumes there is just one variable, X, i.e.
    is designed for unsupervised learning need to support other tasks.

    TODO: document parameters, especially monitoring_batches
    """

    def __init__(self, learning_rate, cost, batch_size=None,
                 batches_per_iter=1000, monitoring_batches=-1,
                 monitoring_dataset=None, termination_criterion=None,
                 update_callbacks=None):
        """
        Instantiates an SGD object.

        Parameters
        ----------
        learning_rate : float
            The stochastic gradient step size, relative to your cost
            function.
        cost : object
            An object implementing the pylearn2 cost interface.
        batch_size : int, optional
            Batch size per update.
            If None, uses model.force_batch_size
        batches_per_iter : int, optional
            How many batch updates per epoch. Default is 1000.
            TODO: Is there any way to specify "as many as the dataset
            provides"?
        monitoring_batches : int, optional
            WRITEME
        monitoring_dataset : object, optional
            WRITEME
        termination_criterion : object, optional
            WRITEME
        update_callback : iterable or object, optional
            WRITEME

        Notes
        -----
        TODO: for now, learning_rate is just a float, but later it
        should support passing in a class that dynamically adjusts the
        learning rate if batch_size is None, reverts to the
        force_batch_size field of the model if monitoring_dataset is
        provided, uses monitoring_batches batches of data from
        monitoring_dataset to report monitoring errors
        """
        #Store parameters
        self.learning_rate = float(learning_rate)
        self.batch_size = batch_size
        self.batches_per_iter = batches_per_iter
        self.cost = cost
        if monitoring_dataset is None:
            assert monitoring_batches == -1, ("no monitoring dataset, but "
                                              "monitoring_batches > 0")
        self.monitoring_dataset = monitoring_dataset
        self.monitoring_batches = monitoring_batches
        self.termination_criterion = termination_criterion
        self._register_update_callbacks(update_callbacks)
        self.bSetup = False
        self.first = True

    def setup(self, model, dataset):
        """
        Initialize the training algorithm. Should be called
        once before calls to train.

        Parameters
        ----------
        model : object
            Model to be trained.  Object implementing the pylearn2 Model
            interface.
        dataset : object
            Dataset on which to train.  Object implementing the
            pylearn2 Dataset interface.
        """

        self.model = model

        if self.batch_size is None:
            self.batch_size = model.force_batch_size

        self.monitor = Monitor.get_monitor(model)
        self.monitor.set_dataset(dataset=self.monitoring_dataset,
                                 mode="sequential",
                                 batch_size=self.batch_size,
                                 num_batches=self.monitoring_batches)


        #Make the right kind of theano variable for the type of space
        #the model acts on
        space = self.model.get_input_space()
        X = space.make_theano_batch(name='sgd_X')

        if isinstance(X, theano.sparse.basic.SparseVariable):
            self.topo = False
        else:
            self.topo = len(X.type.broadcastable) > 2

        try:
            J = sum(c(model, X) for c in self.cost)
        except TypeError:
            J = self.cost(model, X)

        if J.name is None:
            J.name = 'sgd_cost(' + X.name + ')'
        self.monitor.add_channel(name=J.name, ipt=X, val=J)
        params = list(model.get_params())

        for i, param in enumerate(params):
            if param.name is None:
                param.name = 'sgd_params[%d]' % i

        grads = dict(zip(params, T.grad(J, params)))

        for param in grads:

            if grads[param].name is None:
                grads[param].name = ('grad(%(costname)s, %(paramname)s)' %
                                     {'costname': J.name,
                                      'paramname': param.name})

        learning_rate = T.scalar('sgd_learning_rate')

        updates = dict(zip(params, [param - learning_rate * grads[param]
                                    for param in params]))

        for param in updates:
            if updates[param].name is None:
                updates[param].name = 'sgd_update(' + param.name + ')'
{'fraction_non_alphanumeric': 0.06948241727161308, 'fraction_numerical': 0.0033939851041764873, 'mean_word_length': 1.9401330376940134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6869833', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2675, 'n_words': 871}