# Path: lxmls/deep_learning/utils.py
# class AmazonData(object):
#     """
#     Template
#     """
#     def __init__(self, **config):
# 
#         # Data-sets
#         self.datasets = {
#             'train': {
#                 'input': config['corpus'].train_X,
#                 'output': config['corpus'].train_y[:, 0]
#             },
#             #  'dev': (config['corpus'].dev_X, config['corpus'].dev_y[:, 0]),
#             'test': {
#                 'input': config['corpus'].test_X,
#                 'output': config['corpus'].test_y[:, 0]
#             }
#         }
#         # Config
#         self.config = config
#         # Number of samples
#         self.nr_samples = {
#            sset: content['output'].shape[0]
#            for sset, content in self.datasets.items()
#         }
# 
#     def size(self, set_name):
#         return self.nr_samples[set_name]
# 
#     def batches(self, set_name, batch_size=None):
# 
#         dset = self.datasets[set_name]
#         nr_examples = self.nr_samples[set_name]
#         if batch_size is None:
#             nr_batch = 1
#             batch_size = nr_examples
#         else:
#             nr_batch = int(np.ceil(nr_examples*1./batch_size))
# 
#         data = []
#         for batch_n in range(nr_batch):
#             # Colect data for this batch
#             data_batch = {}
#             for side in ['input', 'output']:
#                 data_batch[side] = dset[side][
#                    batch_n * batch_size:(batch_n + 1) * batch_size
#                 ]
#             data.append(data_batch)
# 
#         return DataIterator(data, nr_samples=self.nr_samples[set_name])
# 
# Path: lxmls/deep_learning/utils.py
# class Model(object):
#     def __init__(self, **config):
#         self.initialized = False
# 
#     def initialize_features(self, *args):
#         self.initialized = True
#         raise NotImplementedError(
#             "Need to implement initialize_features method"
#         )
# 
#     def get_features(self, input=None, output=None):
#         """
#         Default feature extraction is do nothing
#         """
#         return {'input': input, 'output': output}
# 
#     def predict(self, *args):
#         raise NotImplementedError("Need to implement predict method")
# 
#     def update(self, *args):
#         # This needs to return at least {'cost' : 0}
#         raise NotImplementedError("Need to implement update method")
#         return {'cost': None}
# 
#     def set(self, **kwargs):
#         raise NotImplementedError("Need to implement set method")
# 
#     def get(self, name):
#         raise NotImplementedError("Need to implement get method")
# 
#     def save(self):
#         raise NotImplementedError("Need to implement save method")
# 
#     def load(self, model_folder):
#         raise NotImplementedError("Need to implement load method")
# 
# def glorot_weight_init(shape, activation_function, random_seed=None):
#     """Layer weight initialization after Xavier Glorot et. al"""
# 
#     if random_seed is None:
#         random_seed = np.random.RandomState(1234)
# 
#     # Weights are uniform distributed with span depending on input and output
#     # sizes
#     num_inputs, num_outputs = shape
#     weight = random_seed.uniform(
#         low=-np.sqrt(6. / (num_inputs + num_outputs)),
#         high=np.sqrt(6. / (num_inputs + num_outputs)),
#         size=(num_outputs, num_inputs)
#     )
# 
#     # Scaling factor depending on non-linearity
#     if activation_function == 'sigmoid':
#         weight *= 4
#     elif activation_function == 'softmax':
#         weight *= 4
# 
#     return weight


import lxmls.readers.sentiment_reader as srs
import numpy as np
import torch
from lxmls.deep_learning.utils import AmazonData
from lxmls.deep_learning.utils import Model, glorot_weight_init
from torch.autograd import Variable


corpus = srs.SentimentCorpus("books")
data = AmazonData(corpus=corpus)


# ### Train Log Linear in Pytorch
# In order to learn the differences between a numpy and a Pytorch implementation, explore the reimplementation of Ex. 3.1 in Pytorch. Compare the content of each of the functions, in particular the `forward()` and `update()` methods. The comments indicated as IMPORTANT will highlight common sources of errors.

# In[ ]:



class PytorchLogLinear(Model):
    
    def __init__(self, **config):
        
        # Initialize parameters
        weight_shape = (config['input_size'], config['num_classes'])
        # after Xavier Glorot et al
        self.weight = glorot_weight_init(weight_shape, 'softmax')
        self.bias = np.zeros((1, config['num_classes']))
        self.learning_rate = config['learning_rate']
        
        # IMPORTANT: Cast to pytorch format
        self.weight = Variable(torch.from_numpy(self.weight).float(), requires_grad=True)
        self.bias = Variable(torch.from_numpy(self.bias).float(), requires_grad=True)
        
        # Instantiate softmax and negative logkelihood in log domain
        self.logsoftmax = torch.nn.LogSoftmax(dim=1)

# Path: lxmls/deep_learning/utils.py
# class AmazonData(object):
#     """
#     Template
#     """
#     def __init__(self, **config):
# 
#         # Data-sets
#         self.datasets = {
#             'train': {
#                 'input': config['corpus'].train_X,
#                 'output': config['corpus'].train_y[:, 0]
#             },
#             #  'dev': (config['corpus'].dev_X, config['corpus'].dev_y[:, 0]),
#             'test': {
#                 'input': config['corpus'].test_X,
#                 'output': config['corpus'].test_y[:, 0]
#             }
#         }
#         # Config
#         self.config = config
#         # Number of samples
#         self.nr_samples = {
#            sset: content['output'].shape[0]
#            for sset, content in self.datasets.items()
#         }
# 
#     def size(self, set_name):
#         return self.nr_samples[set_name]
# 
#     def batches(self, set_name, batch_size=None):
# 
#         dset = self.datasets[set_name]
#         nr_examples = self.nr_samples[set_name]
#         if batch_size is None:
#             nr_batch = 1
#             batch_size = nr_examples
#         else:
#             nr_batch = int(np.ceil(nr_examples*1./batch_size))
# 
#         data = []
#         for batch_n in range(nr_batch):
#             # Colect data for this batch
#             data_batch = {}
#             for side in ['input', 'output']:
#                 data_batch[side] = dset[side][
#                    batch_n * batch_size:(batch_n + 1) * batch_size
#                 ]
#             data.append(data_batch)
# 
#         return DataIterator(data, nr_samples=self.nr_samples[set_name])
# 
# Path: lxmls/deep_learning/utils.py
# class Model(object):
#     def __init__(self, **config):
#         self.initialized = False
# 
#     def initialize_features(self, *args):
#         self.initialized = True
#         raise NotImplementedError(
#             "Need to implement initialize_features method"
#         )
# 
#     def get_features(self, input=None, output=None):
#         """
#         Default feature extraction is do nothing
#         """
#         return {'input': input, 'output': output}
# 
#     def predict(self, *args):
#         raise NotImplementedError("Need to implement predict method")
# 
#     def update(self, *args):
#         # This needs to return at least {'cost' : 0}
#         raise NotImplementedError("Need to implement update method")
#         return {'cost': None}
# 
#     def set(self, **kwargs):
#         raise NotImplementedError("Need to implement set method")
# 
#     def get(self, name):
#         raise NotImplementedError("Need to implement get method")
# 
#     def save(self):
#         raise NotImplementedError("Need to implement save method")
# 
#     def load(self, model_folder):
#         raise NotImplementedError("Need to implement load method")
# 
# def glorot_weight_init(shape, activation_function, random_seed=None):
#     """Layer weight initialization after Xavier Glorot et. al"""
# 
#     if random_seed is None:
#         random_seed = np.random.RandomState(1234)
# 
#     # Weights are uniform distributed with span depending on input and output
#     # sizes
#     num_inputs, num_outputs = shape
#     weight = random_seed.uniform(
#         low=-np.sqrt(6. / (num_inputs + num_outputs)),
#         high=np.sqrt(6. / (num_inputs + num_outputs)),
#         size=(num_outputs, num_inputs)
#     )
# 
#     # Scaling factor depending on non-linearity
#     if activation_function == 'sigmoid':
#         weight *= 4
#     elif activation_function == 'softmax':
#         weight *= 4
# 
#     return weight


# Path: labs/scripts/non_linear_classifiers/exercise_3.py
import lxmls.readers.sentiment_reader as srs
import numpy as np
import torch
from lxmls.deep_learning.utils import AmazonData
from lxmls.deep_learning.utils import Model, glorot_weight_init
from torch.autograd import Variable



corpus = srs.SentimentCorpus("books")
data = AmazonData(corpus=corpus)


# ### Train Log Linear in Pytorch
# In order to learn the differences between a numpy and a Pytorch implementation, explore the reimplementation of Ex. 3.1 in Pytorch. Compare the content of each of the functions, in particular the `forward()` and `update()` methods. The comments indicated as IMPORTANT will highlight common sources of errors.

# In[ ]:



class PytorchLogLinear(Model):
    
    def __init__(self, **config):
        
        # Initialize parameters
        weight_shape = (config['input_size'], config['num_classes'])
        # after Xavier Glorot et al
        self.weight = glorot_weight_init(weight_shape, 'softmax')
        self.bias = np.zeros((1, config['num_classes']))
        self.learning_rate = config['learning_rate']
        
        # IMPORTANT: Cast to pytorch format
        self.weight = Variable(torch.from_numpy(self.weight).float(), requires_grad=True)
        self.bias = Variable(torch.from_numpy(self.bias).float(), requires_grad=True)
        
        # Instantiate softmax and negative logkelihood in log domain
        self.logsoftmax = torch.nn.LogSoftmax(dim=1)

        self.loss = torch.nn.NLLLoss()