# Path: lxmls/deep_learning/utils.py
# class Model(object):
#     def __init__(self, **config):
#         self.initialized = False
# 
#     def initialize_features(self, *args):
#         self.initialized = True
#         raise NotImplementedError(
#             "Need to implement initialize_features method"
#         )
# 
#     def get_features(self, input=None, output=None):
#         """
#         Default feature extraction is do nothing
#         """
#         return {'input': input, 'output': output}
# 
#     def predict(self, *args):
#         raise NotImplementedError("Need to implement predict method")
# 
#     def update(self, *args):
#         # This needs to return at least {'cost' : 0}
#         raise NotImplementedError("Need to implement update method")
#         return {'cost': None}
# 
#     def set(self, **kwargs):
#         raise NotImplementedError("Need to implement set method")
# 
#     def get(self, name):
#         raise NotImplementedError("Need to implement get method")
# 
#     def save(self):
#         raise NotImplementedError("Need to implement save method")
# 
#     def load(self, model_folder):
#         raise NotImplementedError("Need to implement load method")
# 
# def glorot_weight_init(shape, activation_function, random_seed=None):
#     """Layer weight initialization after Xavier Glorot et. al"""
# 
#     if random_seed is None:
#         random_seed = np.random.RandomState(1234)
# 
#     # Weights are uniform distributed with span depending on input and output
#     # sizes
#     num_inputs, num_outputs = shape
#     weight = random_seed.uniform(
#         low=-np.sqrt(6. / (num_inputs + num_outputs)),
#         high=np.sqrt(6. / (num_inputs + num_outputs)),
#         size=(num_outputs, num_inputs)
#     )
# 
#     # Scaling factor depending on non-linearity
#     if activation_function == 'sigmoid':
#         weight *= 4
#     elif activation_function == 'softmax':
#         weight *= 4
# 
#     return weight
# 
# def index2onehot(index, N):
#     """
#     Transforms index to one-hot representation, for example
# 
#     Input: e.g. index = [1, 2, 0], N = 4
#     Output:     [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0]]
#     """
#     L = index.shape[0]
#     onehot = np.zeros((L, N))
#     for l in np.arange(L):
#         onehot[l, index[l]] = 1
#     return onehot
# 
# def logsumexp(a, axis=None, keepdims=False):
#     """
#     This is an improvement over the original logsumexp of
#     scipy/maxentropy/maxentutils.py that allows specifying an axis to sum
#     It also allows keepdims=True.
#     """
#     if axis is None:
#         a = np.asarray(a)
#         a_max = a.max()
#         return a_max + np.log(np.exp(a-a_max).sum())
#     else:
#         a_max = np.amax(a, axis=axis, keepdims=keepdims)
#         return a_max + np.log((np.exp(a-a_max)).sum(axis, keepdims=keepdims))


import numpy as np
from lxmls.deep_learning.utils import (
    Model,
    glorot_weight_init,
    index2onehot,
    logsumexp
)
        # Initialize parameters
        weight_shape = (config['input_size'], config['num_classes'])
        # after Xavier Glorot et al
        self.weight = glorot_weight_init(weight_shape, 'softmax')
        self.bias = np.zeros((1, config['num_classes']))
        self.learning_rate = config['learning_rate']

    def log_forward(self, input=None):
        """Forward pass of the computation graph"""

        # Linear transformation
        z = np.dot(input, self.weight.T) + self.bias

        # Softmax implemented in log domain
        log_tilde_z = z - logsumexp(z, axis=1, keepdims=True)

        return log_tilde_z

    def predict(self, input=None):
        """Most probable class index"""
        return np.argmax(np.exp(self.log_forward(input)), axis=1)

    def update(self, input=None, output=None):
        """Stochastic Gradient Descent update"""

        # Probabilities of each class
        class_probabilities = np.exp(self.log_forward(input))
        batch_size, num_classes = class_probabilities.shape

        # Error derivative at softmax layer

# Path: lxmls/deep_learning/utils.py
# class Model(object):
#     def __init__(self, **config):
#         self.initialized = False
# 
#     def initialize_features(self, *args):
#         self.initialized = True
#         raise NotImplementedError(
#             "Need to implement initialize_features method"
#         )
# 
#     def get_features(self, input=None, output=None):
#         """
#         Default feature extraction is do nothing
#         """
#         return {'input': input, 'output': output}
# 
#     def predict(self, *args):
#         raise NotImplementedError("Need to implement predict method")
# 
#     def update(self, *args):
#         # This needs to return at least {'cost' : 0}
#         raise NotImplementedError("Need to implement update method")
#         return {'cost': None}
# 
#     def set(self, **kwargs):
#         raise NotImplementedError("Need to implement set method")
# 
#     def get(self, name):
#         raise NotImplementedError("Need to implement get method")
# 
#     def save(self):
#         raise NotImplementedError("Need to implement save method")
# 
#     def load(self, model_folder):
#         raise NotImplementedError("Need to implement load method")
# 
# def glorot_weight_init(shape, activation_function, random_seed=None):
#     """Layer weight initialization after Xavier Glorot et. al"""
# 
#     if random_seed is None:
#         random_seed = np.random.RandomState(1234)
# 
#     # Weights are uniform distributed with span depending on input and output
#     # sizes
#     num_inputs, num_outputs = shape
#     weight = random_seed.uniform(
#         low=-np.sqrt(6. / (num_inputs + num_outputs)),
#         high=np.sqrt(6. / (num_inputs + num_outputs)),
#         size=(num_outputs, num_inputs)
#     )
# 
#     # Scaling factor depending on non-linearity
#     if activation_function == 'sigmoid':
#         weight *= 4
#     elif activation_function == 'softmax':
#         weight *= 4
# 
#     return weight
# 
# def index2onehot(index, N):
#     """
#     Transforms index to one-hot representation, for example
# 
#     Input: e.g. index = [1, 2, 0], N = 4
#     Output:     [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0]]
#     """
#     L = index.shape[0]
#     onehot = np.zeros((L, N))
#     for l in np.arange(L):
#         onehot[l, index[l]] = 1
#     return onehot
# 
# def logsumexp(a, axis=None, keepdims=False):
#     """
#     This is an improvement over the original logsumexp of
#     scipy/maxentropy/maxentutils.py that allows specifying an axis to sum
#     It also allows keepdims=True.
#     """
#     if axis is None:
#         a = np.asarray(a)
#         a_max = a.max()
#         return a_max + np.log(np.exp(a-a_max).sum())
#     else:
#         a_max = np.amax(a, axis=axis, keepdims=keepdims)
#         return a_max + np.log((np.exp(a-a_max)).sum(axis, keepdims=keepdims))


# Path: lxmls/deep_learning/numpy_models/log_linear.py
import numpy as np
from lxmls.deep_learning.utils import (
    Model,
    glorot_weight_init,
    index2onehot,
    logsumexp
)

        # Initialize parameters
        weight_shape = (config['input_size'], config['num_classes'])
        # after Xavier Glorot et al
        self.weight = glorot_weight_init(weight_shape, 'softmax')
        self.bias = np.zeros((1, config['num_classes']))
        self.learning_rate = config['learning_rate']

    def log_forward(self, input=None):
        """Forward pass of the computation graph"""

        # Linear transformation
        z = np.dot(input, self.weight.T) + self.bias

        # Softmax implemented in log domain
        log_tilde_z = z - logsumexp(z, axis=1, keepdims=True)

        return log_tilde_z

    def predict(self, input=None):
        """Most probable class index"""
        return np.argmax(np.exp(self.log_forward(input)), axis=1)

    def update(self, input=None, output=None):
        """Stochastic Gradient Descent update"""

        # Probabilities of each class
        class_probabilities = np.exp(self.log_forward(input))
        batch_size, num_classes = class_probabilities.shape

        # Error derivative at softmax layer

        I = index2onehot(output, num_classes)