<commit_before>"""
.. todo::

    WRITEME
"""
__authors__ = "<NAME>"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["<NAME>"]
__license__ = "3-clause BSD"
__maintainer__ = "LISA Lab"
__email__ = "<EMAIL>"

import logging
import time

import numpy as np

from theano import config
from theano.compat.python2x import OrderedDict
from theano.printing import var_descriptor
import theano.tensor as T

from pylearn2.utils import function
from pylearn2.utils import grad
from pylearn2.utils import safe_zip
from pylearn2.utils import sharedX


logger = logging.getLogger(__name__)


class BatchGradientDescent(object):
    """
    A class for minimizing a function via the method of steepest descent.

    Parameters
    ----------
    objective : tensor_like
        A theano expression to be minimized should be a function of params and,
        if provided, inputs
    params : list
        A list of theano shared variables. These are the optimization variables
    inputs : list, optional
        A list of theano variables to serve as inputs to the graph.
    param_constrainers : list
        A list of callables to be called on all updates dictionaries to be
        applied to params. This is how you implement constrained
        optimization.
    reset_alpha : bool
        If True, reverts to using init_alpha after each call. If False, the
        final set of alphas is used at the start of the next call to minimize.
    conjugate : bool
        If True, tries to pick conjugate gradient directions. For the
        directions to be truly conjugate, you must use line_search_mode =
        'exhaustive' and the objective function must be quadratic. Using
        line_search_mode = 'exhaustive' on a non-quadratic objective function
        implements nonlinear conjugate gradient descent.
    reset_conjugate : bool
        Has no effect unless conjugate == True. If reset_conjugate ==
        True, reverts to direction of steepest descent for the first
        step in each call to minimize. Otherwise, tries to make the new
        search direction conjugate to the last one (even though the
        objective function might be totally different on each call to
        minimize)
    gradients : WRITEME
        If None, compute the gradients of obj using T.grad otherwise, a
        dictionary mapping from params to expressions for their gradients
        (this allows you to use approximate gradients computed with
        something other than T.grad)
    gradient_updates : dict
        A dictionary of shared variable updates to run each time the
        gradient is computed

    Notes
    -----
    Calling the `minimize` method with values for for `inputs` will
    update `params` to minimize `objective`.
    """
    def __init__(self, objective, params, inputs=None,
                 param_constrainers=None, max_iter=-1,
                 lr_scalers=None, verbose=0, tol=None,
                 init_alpha=None, min_init_alpha=1e-3,
                 reset_alpha=True, conjugate=False,
                 reset_conjugate=True, gradients=None,
                 gradient_updates=None, line_search_mode=None,
                 accumulate=False, theano_function_mode=None):

        self.__dict__.update(locals())
        del self.self

        if line_search_mode is None:
            if init_alpha is None:
                init_alpha = (.001, .005, .01, .05, .1)
        else:
            assert line_search_mode == 'exhaustive'
            if init_alpha is None:
                init_alpha = (.5, 1.)

        self.init_alpha = tuple([float(elem) for elem in init_alpha])

        if inputs is None:
            inputs = []

        if param_constrainers is None:
            param_constrainers = []

        obj = objective

        self.verbose = verbose

        param_to_grad_sym = OrderedDict()
        param_to_grad_shared = OrderedDict()
        updates = OrderedDict()
        if self.gradient_updates is not None:
            updates.update(self.gradient_updates)

        self.params = [param for param in params]

        for param in params:
            if self.gradients is not None and param in self.gradients:
                g = self.gradients[param]
            else:
                g = grad(objective, param)
            param_to_grad_sym[param] = g
            if param.name is not None:
                param_name = param.name
            else:
                param_name = 'anon_param'
            grad_name = 'BatchGradientDescent.grad_' + param_name
            grad_shared = sharedX(param.get_value() * 0., name=grad_name)
            param_to_grad_shared[param] = grad_shared
            updates[grad_shared] = g

        self.param_to_grad_shared = param_to_grad_shared

        if self.verbose:
            logger.info('batch gradient class compiling gradient function')
        t1 = time.time()
        if self.accumulate:
            self._compute_grad = Accumulator(inputs, updates=updates)
        else:
            self._compute_grad = function(inputs, updates=updates,
                                          mode=self.theano_function_mode,
                                          name=
                                          'BatchGradientDescent._compute_grad')
        if self.verbose:
            t2 = time.time()
            logger.info('done. Took {0}'.format(t2-t1))

        if self.verbose:
            logger.info('batch gradient class compiling objective function')
        if self.accumulate:
            self.obj = Accumulator(inputs, obj)
        else:
            self.obj = function(inputs, obj, mode=self.theano_function_mode,
                                name='BatchGradientDescent.obj')

        if self.verbose:
            logger.info('done')

        self.param_to_cache = OrderedDict()
        alpha = T.scalar(name='alpha')
        alpha.tag.test_value = np.cast[alpha.dtype](.01)
        cache_updates = OrderedDict()
        goto_updates = OrderedDict()
        for param in params:
            if param.name is None:
                param_name = 'anon_param'
            else:
                param_name = param.name
            cache_name = 'BatchGradientDescent.param_to_cache[%s]' % param_name
            self.param_to_cache[param] = sharedX(param.get_value(borrow=
                                                 False), name=cache_name)
            cache_updates[self.param_to_cache[param]] = param
            cached = self.param_to_cache[param]
            g = self.param_to_grad_shared[param]
            if lr_scalers is not None and param in lr_scalers:
                scaled_alpha = alpha * lr_scalers[param]
            else:
                scaled_alpha = alpha
            mul = scaled_alpha * g
            diff = cached - mul
            goto_updates[param] = diff
        self._cache_values = function([], updates=cache_updates,
                                      mode=self.theano_function_mode, name=
                                      'BatchGradientDescent._cache_values')
        assert isinstance(param_constrainers, (list, tuple))
        for param_constrainer in param_constrainers:
            param_constrainer(goto_updates)
        self._goto_alpha = function([alpha], updates=goto_updates,
                                    mode=self.theano_function_mode, name=
                                    'BatchGradientDescent._goto_alpha')

        norm = T.sqrt(sum([T.sqr(elem).sum() for elem in
                           self.param_to_grad_shared.values()]))
        norm.name = 'BatchGradientDescent.norm'
        normalize_grad_updates = OrderedDict()
        for grad_shared in self.param_to_grad_shared.values():
            normalize_grad_updates[grad_shared] = grad_shared / norm
<commit_msg>make the undealt requested changes<commit_after>"""
.. todo::

    WRITEME
"""
__authors__ = "<NAME>"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["<NAME>"]
__license__ = "3-clause BSD"
__maintainer__ = "LISA Lab"
__email__ = "<EMAIL>"

import logging
import time

import numpy as np

from theano import config
from theano.compat.python2x import OrderedDict
from theano.printing import var_descriptor
import theano.tensor as T

from pylearn2.utils import function
from pylearn2.utils import grad
from pylearn2.utils import safe_zip
from pylearn2.utils import sharedX


logger = logging.getLogger(__name__)


class BatchGradientDescent(object):
    """
    A class for minimizing a function via the method of steepest descent.

    Parameters
    ----------
    objective : tensor_like
        A theano expression to be minimized should be a function of params and,
        if provided, inputs
    params : list
        A list of theano shared variables. These are the optimization variables
    inputs : list, optional
        A list of theano variables to serve as inputs to the graph.
    param_constrainers : list
        A list of callables to be called on all updates dictionaries to be
        applied to params. This is how you implement constrained
        optimization.
    reset_alpha : bool
        If True, reverts to using init_alpha after each call. If False, the
        final set of alphas is used at the start of the next call to minimize.
    conjugate : bool
        If True, tries to pick conjugate gradient directions. For the
        directions to be truly conjugate, you must use line_search_mode =
        'exhaustive' and the objective function must be quadratic. Using
        line_search_mode = 'exhaustive' on a non-quadratic objective function
        implements nonlinear conjugate gradient descent.
    reset_conjugate : bool
        Has no effect unless conjugate == True. If reset_conjugate ==
        True, reverts to direction of steepest descent for the first
        step in each call to minimize. Otherwise, tries to make the new
        search direction conjugate to the last one (even though the
        objective function might be totally different on each call to
        minimize)
    gradients : WRITEME
        If None, compute the gradients of obj using T.grad otherwise, a
        dictionary mapping from params to expressions for their gradients
        (this allows you to use approximate gradients computed with
        something other than T.grad)
    gradient_updates : dict
        A dictionary of shared variable updates to run each time the
        gradient is computed

    Notes
    -----
    Calling the `minimize` method with values for for `inputs` will
    update `params` to minimize `objective`.
    """
    def __init__(self, objective, params, inputs=None,
                 param_constrainers=None, max_iter=-1,
                 lr_scalers=None, verbose=0, tol=None,
                 init_alpha=None, min_init_alpha=1e-3,
                 reset_alpha=True, conjugate=False,
                 reset_conjugate=True, gradients=None,
                 gradient_updates=None, line_search_mode=None,
                 accumulate=False, theano_function_mode=None):

        self.__dict__.update(locals())
        del self.self

        if line_search_mode is None:
            if init_alpha is None:
                init_alpha = (.001, .005, .01, .05, .1)
        else:
            assert line_search_mode == 'exhaustive'
            if init_alpha is None:
                init_alpha = (.5, 1.)

        self.init_alpha = tuple([float(elem) for elem in init_alpha])

        if inputs is None:
            inputs = []

        if param_constrainers is None:
            param_constrainers = []

        obj = objective

        self.verbose = verbose

        param_to_grad_sym = OrderedDict()
        param_to_grad_shared = OrderedDict()
        updates = OrderedDict()
        if self.gradient_updates is not None:
            updates.update(self.gradient_updates)

        self.params = [param for param in params]

        for param in params:
            if self.gradients is not None and param in self.gradients:
                g = self.gradients[param]
            else:
                g = grad(objective, param)
            param_to_grad_sym[param] = g
            if param.name is not None:
                param_name = param.name
            else:
                param_name = 'anon_param'
            grad_name = 'BatchGradientDescent.grad_' + param_name
            grad_shared = sharedX(param.get_value() * 0., name=grad_name)
            param_to_grad_shared[param] = grad_shared
            updates[grad_shared] = g

        self.param_to_grad_shared = param_to_grad_shared

        if self.verbose:
            logger.info('batch gradient class compiling gradient function')
        t1 = time.time()
        if self.accumulate:
            self._compute_grad = Accumulator(inputs, updates=updates)
        else:
            self._compute_grad = function(
                inputs,
                updates=updates,
                mode=self.theano_function_mode,
                name='BatchGradientDescent._compute_grad')
        if self.verbose:
            t2 = time.time()
            logger.info('done. Took {0}'.format(t2-t1))

        if self.verbose:
            logger.info('batch gradient class compiling objective function')
        if self.accumulate:
            self.obj = Accumulator(inputs, obj)
        else:
            self.obj = function(inputs, obj, mode=self.theano_function_mode,
                                name='BatchGradientDescent.obj')

        if self.verbose:
            logger.info('done')

        self.param_to_cache = OrderedDict()
        alpha = T.scalar(name='alpha')
        alpha.tag.test_value = np.cast[alpha.dtype](.01)
        cache_updates = OrderedDict()
        goto_updates = OrderedDict()
        for param in params:
            if param.name is None:
                param_name = 'anon_param'
            else:
                param_name = param.name
            cache_name = 'BatchGradientDescent.param_to_cache[%s]' % param_name
            self.param_to_cache[param] = sharedX(param.get_value(borrow=False),
                                                 name=cache_name)
            cache_updates[self.param_to_cache[param]] = param
            cached = self.param_to_cache[param]
            g = self.param_to_grad_shared[param]
            if lr_scalers is not None and param in lr_scalers:
                scaled_alpha = alpha * lr_scalers[param]
            else:
                scaled_alpha = alpha
            mul = scaled_alpha * g
            diff = cached - mul
            goto_updates[param] = diff
        self._cache_values = function(
            [],
            updates=cache_updates,
            mode=self.theano_function_mode,
            name='BatchGradientDescent._cache_values')
        assert isinstance(param_constrainers, (list, tuple))
        for param_constrainer in param_constrainers:
            param_constrainer(goto_updates)
        self._goto_alpha = function(
            [alpha],
            updates=goto_updates,
            mode=self.theano_function_mode,
            name='BatchGradientDescent._goto_alpha')

        norm = T.sqrt(sum([T.sqr(elem).sum() for elem in
                           self.param_to_grad_shared.values()]))
        norm.name = 'BatchGradientDescent.norm'
        normalize_grad_updates = OrderedDict()
        for grad_shared in self.param_to_grad_shared.values():
            normalize_grad_updates[grad_shared] = grad_shared / norm

{'fraction_non_alphanumeric': 0.07007526602647288, 'fraction_numerical': 0.005060991435245263, 'mean_word_length': 2.059348947995236, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4815914', 'n_tokens_mistral': 4238, 'n_tokens_neox': 4050, 'n_words': 1238}