# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():
# 
# Path: shorttext/utils/compactmodel_io.py
# class CompactIOMachine:
#     """ Base class that implements compact model I/O.
# 
#     This is to replace the original :func:`compactio` decorator.
# 
#     """
#     def __init__(self, infodict, prefix, suffices):
#         """
# 
#         :param infodict: information about the model. Must contain the key 'classifier'.
#         :param prefix: prefix of names of the model file
#         :param suffices: suffices of the names of the model file
#         :type infodict: dict
#         :type prefix: str
#         :type suffices: list
#         """
#         self.infodict = infodict
#         self.prefix = prefix
#         self.suffices = suffices
# 
#     def savemodel(self, nameprefix):
#         """ Abstract method for `savemodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def loadmodel(self, nameprefix):
#         """ Abstract method for `loadmodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def save_compact_model(self, filename, *args, **kwargs):
#         """ Save the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         save_compact_model(filename, self.savemodel, self.prefix, self.suffices, self.infodict, *args, **kwargs)
# 
#     def load_compact_model(self, filename, *args, **kwargs):
#         """ Load the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         return load_compact_model(filename, self.loadmodel, self.prefix, self.infodict, *args, **kwargs)
# 
#     def get_info(self):
#         """ Getting information for the dressed machine.
# 
#         :return: dictionary of the information for the dressed machine.
#         :rtype: dict
#         """
#         return {'classifier': self.infodict['classifier'],
#                 'prefix': self.prefix,
#                 'suffices': self.suffices}


import json
import os
import warnings
import numpy as np
import shorttext.utils.kerasmodel_io as kerasio
import shorttext.utils.classification_exceptions as e
from shorttext.utils import tokenize
from shorttext.utils.compactmodel_io import CompactIOMachine
        CompactIOMachine.__init__(self, {'classifier': 'nnlibvec'}, 'nnlibvec', ['_classlabels.txt', '.json', '.h5', '_config.json'])
        self.wvmodel = wvmodel
        self.vecsize = self.wvmodel.vector_size if vecsize == None else vecsize
        self.maxlen = maxlen
        self.with_gensim = False if not with_gensim else with_gensim
        self.trained = False

    def convert_trainingdata_matrix(self, classdict):
        """ Convert the training data into format put into the neural networks.

        Convert the training data into format put into the neural networks.
        This is called by :func:`~train`.

        :param classdict: training data
        :return: a tuple of three, containing a list of class labels, matrix of embedded word vectors, and corresponding outputs
        :type classdict: dict
        :rtype: (list, numpy.ndarray, list)
        """
        classlabels = classdict.keys()
        lblidx_dict = dict(zip(classlabels, range(len(classlabels))))

        # tokenize the words, and determine the word length
        phrases = []
        indices = []
        for label in classlabels:
            for shorttext in classdict[label]:
                shorttext = shorttext if type(shorttext)==str else ''
                category_bucket = [0]*len(classlabels)
                category_bucket[lblidx_dict[label]] = 1
                indices.append(category_bucket)

# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():
# 
# Path: shorttext/utils/compactmodel_io.py
# class CompactIOMachine:
#     """ Base class that implements compact model I/O.
# 
#     This is to replace the original :func:`compactio` decorator.
# 
#     """
#     def __init__(self, infodict, prefix, suffices):
#         """
# 
#         :param infodict: information about the model. Must contain the key 'classifier'.
#         :param prefix: prefix of names of the model file
#         :param suffices: suffices of the names of the model file
#         :type infodict: dict
#         :type prefix: str
#         :type suffices: list
#         """
#         self.infodict = infodict
#         self.prefix = prefix
#         self.suffices = suffices
# 
#     def savemodel(self, nameprefix):
#         """ Abstract method for `savemodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def loadmodel(self, nameprefix):
#         """ Abstract method for `loadmodel`.
# 
#         :param nameprefix: prefix of the model path
#         :type nameprefix: str
#         """
#         raise e.OperationNotDefinedException()
# 
#     def save_compact_model(self, filename, *args, **kwargs):
#         """ Save the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         save_compact_model(filename, self.savemodel, self.prefix, self.suffices, self.infodict, *args, **kwargs)
# 
#     def load_compact_model(self, filename, *args, **kwargs):
#         """ Load the model in a compressed binary format.
# 
#         :param filename: name of the model file
#         :param args: arguments
#         :param kwargs: arguments
#         :type filename: str
#         :type args: dict
#         :type kwargs: dict
#         """
#         return load_compact_model(filename, self.loadmodel, self.prefix, self.infodict, *args, **kwargs)
# 
#     def get_info(self):
#         """ Getting information for the dressed machine.
# 
#         :return: dictionary of the information for the dressed machine.
#         :rtype: dict
#         """
#         return {'classifier': self.infodict['classifier'],
#                 'prefix': self.prefix,
#                 'suffices': self.suffices}


# Path: shorttext/classifiers/embed/nnlib/VarNNEmbedVecClassification.py
import json
import os
import warnings
import numpy as np
import shorttext.utils.kerasmodel_io as kerasio
import shorttext.utils.classification_exceptions as e
from shorttext.utils import tokenize
from shorttext.utils.compactmodel_io import CompactIOMachine

        CompactIOMachine.__init__(self, {'classifier': 'nnlibvec'}, 'nnlibvec', ['_classlabels.txt', '.json', '.h5', '_config.json'])
        self.wvmodel = wvmodel
        self.vecsize = self.wvmodel.vector_size if vecsize == None else vecsize
        self.maxlen = maxlen
        self.with_gensim = False if not with_gensim else with_gensim
        self.trained = False

    def convert_trainingdata_matrix(self, classdict):
        """ Convert the training data into format put into the neural networks.

        Convert the training data into format put into the neural networks.
        This is called by :func:`~train`.

        :param classdict: training data
        :return: a tuple of three, containing a list of class labels, matrix of embedded word vectors, and corresponding outputs
        :type classdict: dict
        :rtype: (list, numpy.ndarray, list)
        """
        classlabels = classdict.keys()
        lblidx_dict = dict(zip(classlabels, range(len(classlabels))))

        # tokenize the words, and determine the word length
        phrases = []
        indices = []
        for label in classlabels:
            for shorttext in classdict[label]:
                shorttext = shorttext if type(shorttext)==str else ''
                category_bucket = [0]*len(classlabels)
                category_bucket[lblidx_dict[label]] = 1
                indices.append(category_bucket)

                phrases.append(tokenize(shorttext))