# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():
# 
# Path: shorttext/utils/gensim_corpora.py
# def generate_gensim_corpora(classdict, preprocess_and_tokenize=tokenize):
# def save_corpus(dictionary, corpus, prefix):
# def load_corpus(prefix):
# def update_corpus_labels(dictionary, corpus, newclassdict, preprocess_and_tokenize=tokenize):
# def tokens_to_fracdict(tokens):
# 
# Path: shorttext/utils/classification_exceptions.py
# class ModelNotTrainedException(Exception):
# class AlgorithmNotExistException(Exception):
# class WordEmbeddingModelNotExistException(Exception):
# class UnequalArrayLengthsException(Exception):
# class NotImplementedException(Exception):
# class IncorrectClassificationModelFileException(Exception):
# class OperationNotDefinedException(Exception):
#     def __init__(self):
#     def __init__(self, algoname):
#     def __init__(self, path):
#     def __init__(self, arr1, arr2):
#     def __init__(self):
#     def __init__(self, expectedname, actualname):
#     def __init__(self, opname):
# 
# Path: shorttext/utils/compactmodel_io.py
# class CompactIOMachine:
#     """ Base class that implements compact model I/O.
# 
#     This is to replace the original :func:`compactio` decorator.
# 
#     """
#     def __init__(self, infodict, prefix, suffices):
#         """
# 
#         :param infodict: information about the model. Must contain the key 'classifier'.
#         :param prefix: prefix of names of the model file
#         :param suffices: suffices of the names of the model file
#         :type infodict: dict
#         :type prefix: str
#         :type suffices: list
#         """
#         self.infodict = infodict
#         self.prefix = prefix
#         self.suffices = suffices
# 
#     def savemodel(self, nameprefix):
#         """ Abstract method for `savemodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def loadmodel(self, nameprefix):
#         """ Abstract method for `loadmodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def save_compact_model(self, filename, *args, **kwargs):
#         """ Save the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         save_compact_model(filename, self.savemodel, self.prefix, self.suffices, self.infodict, *args, **kwargs)
# 
#     def load_compact_model(self, filename, *args, **kwargs):
#         """ Load the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         return load_compact_model(filename, self.loadmodel, self.prefix, self.infodict, *args, **kwargs)
# 
#     def get_info(self):
#         """ Getting information for the dressed machine.
# 
#         :return: dictionary of the information for the dressed machine.
#         :rtype: dict
#         """
#         return {'classifier': self.infodict['classifier'],
#                 'prefix': self.prefix,
#                 'suffices': self.suffices}


import pickle
import shorttext.utils.kerasmodel_io as kerasio
from scipy.sparse import dok_matrix
from gensim.corpora import Dictionary
from keras.models import Sequential
from keras.layers import Dense
from keras.regularizers import l2
from shorttext.utils import tokenize
from shorttext.utils import gensim_corpora as gc
from shorttext.utils import classification_exceptions as e
from shorttext.utils.compactmodel_io import CompactIOMachine
        rowid = 0
        for label in classdict:
            if label in self.labels2idx.keys():
                for shorttext in classdict[label]:
                    tokens = tokenize(self.preprocessor(shorttext))
                    for token in tokens:
                        X[rowid, self.dictionary.token2id[token]] += 1.0
                    y[rowid, self.labels2idx[label]] = 1.
                    rowid += 1

        return X, y

    def train(self, classdict, nb_epochs=500, l2reg=0.01, bias_l2reg=0.01, optimizer='adam'):
        """ Train the classifier.

        Given the training data, train the classifier.

        :param classdict: training data
        :param nb_epochs: number of epochs (Defauly: 500)
        :param l2reg: L2 regularization coefficient (Default: 0.01)
        :param bias_l2reg: L2 regularization coefficient for bias (Default: 0.01)
        :param optimizer: optimizer for gradient descent. Options: sgd, rmsprop, adagrad, adadelta, adam, adamax, nadam. (Default: adam)
        :return: None
        :type classdict: dict
        :type nb_epochs: int
        :type l2reg: float
        :type bias_l2reg: float
        :type optimizer: str
        """
        self.dictionary, self.corpus, self.classlabels = gc.generate_gensim_corpora(classdict,

# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():
# 
# Path: shorttext/utils/gensim_corpora.py
# def generate_gensim_corpora(classdict, preprocess_and_tokenize=tokenize):
# def save_corpus(dictionary, corpus, prefix):
# def load_corpus(prefix):
# def update_corpus_labels(dictionary, corpus, newclassdict, preprocess_and_tokenize=tokenize):
# def tokens_to_fracdict(tokens):
# 
# Path: shorttext/utils/classification_exceptions.py
# class ModelNotTrainedException(Exception):
# class AlgorithmNotExistException(Exception):
# class WordEmbeddingModelNotExistException(Exception):
# class UnequalArrayLengthsException(Exception):
# class NotImplementedException(Exception):
# class IncorrectClassificationModelFileException(Exception):
# class OperationNotDefinedException(Exception):
#     def __init__(self):
#     def __init__(self, algoname):
#     def __init__(self, path):
#     def __init__(self, arr1, arr2):
#     def __init__(self):
#     def __init__(self, expectedname, actualname):
#     def __init__(self, opname):
# 
# Path: shorttext/utils/compactmodel_io.py
# class CompactIOMachine:
#     """ Base class that implements compact model I/O.
# 
#     This is to replace the original :func:`compactio` decorator.
# 
#     """
#     def __init__(self, infodict, prefix, suffices):
#         """
# 
#         :param infodict: information about the model. Must contain the key 'classifier'.
#         :param prefix: prefix of names of the model file
#         :param suffices: suffices of the names of the model file
#         :type infodict: dict
#         :type prefix: str
#         :type suffices: list
#         """
#         self.infodict = infodict
#         self.prefix = prefix
#         self.suffices = suffices
# 
#     def savemodel(self, nameprefix):
#         """ Abstract method for `savemodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def loadmodel(self, nameprefix):
#         """ Abstract method for `loadmodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def save_compact_model(self, filename, *args, **kwargs):
#         """ Save the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         save_compact_model(filename, self.savemodel, self.prefix, self.suffices, self.infodict, *args, **kwargs)
# 
#     def load_compact_model(self, filename, *args, **kwargs):
#         """ Load the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         return load_compact_model(filename, self.loadmodel, self.prefix, self.infodict, *args, **kwargs)
# 
#     def get_info(self):
#         """ Getting information for the dressed machine.
# 
#         :return: dictionary of the information for the dressed machine.
#         :rtype: dict
#         """
#         return {'classifier': self.infodict['classifier'],
#                 'prefix': self.prefix,
#                 'suffices': self.suffices}


# Path: shorttext/classifiers/bow/maxent/MaxEntClassification.py
import pickle
import shorttext.utils.kerasmodel_io as kerasio
from scipy.sparse import dok_matrix
from gensim.corpora import Dictionary
from keras.models import Sequential
from keras.layers import Dense
from keras.regularizers import l2
from shorttext.utils import tokenize
from shorttext.utils import gensim_corpora as gc
from shorttext.utils import classification_exceptions as e
from shorttext.utils.compactmodel_io import CompactIOMachine

        rowid = 0
        for label in classdict:
            if label in self.labels2idx.keys():
                for shorttext in classdict[label]:
                    tokens = tokenize(self.preprocessor(shorttext))
                    for token in tokens:
                        X[rowid, self.dictionary.token2id[token]] += 1.0
                    y[rowid, self.labels2idx[label]] = 1.
                    rowid += 1

        return X, y

    def train(self, classdict, nb_epochs=500, l2reg=0.01, bias_l2reg=0.01, optimizer='adam'):
        """ Train the classifier.

        Given the training data, train the classifier.

        :param classdict: training data
        :param nb_epochs: number of epochs (Defauly: 500)
        :param l2reg: L2 regularization coefficient (Default: 0.01)
        :param bias_l2reg: L2 regularization coefficient for bias (Default: 0.01)
        :param optimizer: optimizer for gradient descent. Options: sgd, rmsprop, adagrad, adadelta, adam, adamax, nadam. (Default: adam)
        :return: None
        :type classdict: dict
        :type nb_epochs: int
        :type l2reg: float
        :type bias_l2reg: float
        :type optimizer: str
        """
        self.dictionary, self.corpus, self.classlabels = gc.generate_gensim_corpora(classdict,

                                                                                    preprocess_and_tokenize=lambda s: tokenize(self.preprocessor(s)))