# Path: emolga/utils/theano_utils.py
# def shared_zeros(shape, dtype=theano.config.floatX, name=None):
#     return sharedX(np.zeros(shape), dtype=dtype, name=name)
# 
# def shared_scalar(val=0., dtype=theano.config.floatX, name=None):
#     return theano.shared(np.cast[dtype](val), name=name)
# 
# def floatX(X):
#     return np.asarray(X, dtype=theano.config.floatX)
# 
# Path: emolga/utils/generic_utils.py
# def get_from_module(identifier, module_params, module_name, instantiate=False, kwargs=None):
#     if isinstance(identifier, six.string_types):
#         res = module_params.get(identifier)
#         if not res:
#             raise Exception('Invalid ' + str(module_name) + ': ' + str(identifier))
#         if instantiate and not kwargs:
#             return res()
#         elif instantiate and kwargs:
#             return res(**kwargs)
#         else:
#             return res
#     return identifier


import theano
import sys
import theano.tensor as T
import logging
from theano.sandbox.rng_mrg import MRG_RandomStreams
from emolga.utils.theano_utils import shared_zeros, shared_scalar, floatX
from emolga.utils.generic_utils import get_from_module
from six.moves import zip
from copy import copy, deepcopy

    def get_gradients(self, loss, params):
        """
        Consider the situation that gradient is weighted.
        """
        if isinstance(loss, list):
            grads = T.grad(loss[0], params, consider_constant=loss[1:])  # gradient of loss
        else:
            grads = T.grad(loss, params)

        if hasattr(self, 'clipnorm') and self.clipnorm > 0:
            print('use gradient clipping!!')
            print('clipnorm = %f' % self.clipnorm)
            norm = T.sqrt(sum([T.sum(g ** 2) for g in grads]))
            grads = [clip_norm(g, self.clipnorm, norm) for g in grads]
        else:
            print('not use gradient clipping!!')

        return grads

    def get_config(self):
        return {"name": self.__class__.__name__}


class SGD(Optimizer):

    def __init__(self, lr=0.05, momentum=0.9, decay=0.01, nesterov=True, *args, **kwargs):
        super(SGD, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.iterations = shared_scalar(0)

# Path: emolga/utils/theano_utils.py
# def shared_zeros(shape, dtype=theano.config.floatX, name=None):
#     return sharedX(np.zeros(shape), dtype=dtype, name=name)
# 
# def shared_scalar(val=0., dtype=theano.config.floatX, name=None):
#     return theano.shared(np.cast[dtype](val), name=name)
# 
# def floatX(X):
#     return np.asarray(X, dtype=theano.config.floatX)
# 
# Path: emolga/utils/generic_utils.py
# def get_from_module(identifier, module_params, module_name, instantiate=False, kwargs=None):
#     if isinstance(identifier, six.string_types):
#         res = module_params.get(identifier)
#         if not res:
#             raise Exception('Invalid ' + str(module_name) + ': ' + str(identifier))
#         if instantiate and not kwargs:
#             return res()
#         elif instantiate and kwargs:
#             return res(**kwargs)
#         else:
#             return res
#     return identifier


# Path: emolga/basic/optimizers.py
import theano
import sys
import theano.tensor as T
import logging
from theano.sandbox.rng_mrg import MRG_RandomStreams
from emolga.utils.theano_utils import shared_zeros, shared_scalar, floatX
from emolga.utils.generic_utils import get_from_module
from six.moves import zip
from copy import copy, deepcopy


    def get_gradients(self, loss, params):
        """
        Consider the situation that gradient is weighted.
        """
        if isinstance(loss, list):
            grads = T.grad(loss[0], params, consider_constant=loss[1:])  # gradient of loss
        else:
            grads = T.grad(loss, params)

        if hasattr(self, 'clipnorm') and self.clipnorm > 0:
            print('use gradient clipping!!')
            print('clipnorm = %f' % self.clipnorm)
            norm = T.sqrt(sum([T.sum(g ** 2) for g in grads]))
            grads = [clip_norm(g, self.clipnorm, norm) for g in grads]
        else:
            print('not use gradient clipping!!')

        return grads

    def get_config(self):
        return {"name": self.__class__.__name__}


class SGD(Optimizer):

    def __init__(self, lr=0.05, momentum=0.9, decay=0.01, nesterov=True, *args, **kwargs):
        super(SGD, self).__init__(**kwargs)
        self.__dict__.update(locals())
        self.iterations = shared_scalar(0)

        self.lr = shared_scalar(lr)