# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


from collections import defaultdict
from .textpreprocessing import tokenize
import gensim


def save_corpus(dictionary, corpus, prefix):
    """ Save gensim corpus and dictionary.

    :param dictionary: dictionary to save
    :param corpus: corpus to save
    :param prefix: prefix of the files to save
    :return: None
    :type dictionary: gensim.corpora.Dictionary
    :type corpus: list
    :type prefix: str
    """
    dictionary.save(prefix+'_dictionary.dict')
    gensim.corpora.MmCorpus.serialize(prefix+'_corpus.mm', corpus)


def load_corpus(prefix):
    """ Load gensim corpus and dictionary.

    :param prefix: prefix of the file to load
    :return: corpus and dictionary
    :type prefix: str
    :rtype: tuple
    """
    corpus = gensim.corpora.MmCorpus(prefix+'_corpus.mm')
    dictionary = gensim.corpora.Dictionary.load(prefix+'_dictionary.dict')
    return corpus, dictionary



# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


# Path: shorttext/utils/gensim_corpora.py
from collections import defaultdict
from .textpreprocessing import tokenize
import gensim



def save_corpus(dictionary, corpus, prefix):
    """ Save gensim corpus and dictionary.

    :param dictionary: dictionary to save
    :param corpus: corpus to save
    :param prefix: prefix of the files to save
    :return: None
    :type dictionary: gensim.corpora.Dictionary
    :type corpus: list
    :type prefix: str
    """
    dictionary.save(prefix+'_dictionary.dict')
    gensim.corpora.MmCorpus.serialize(prefix+'_corpus.mm', corpus)


def load_corpus(prefix):
    """ Load gensim corpus and dictionary.

    :param prefix: prefix of the file to load
    :return: corpus and dictionary
    :type prefix: str
    :rtype: tuple
    """
    corpus = gensim.corpora.MmCorpus(prefix+'_corpus.mm')
    dictionary = gensim.corpora.Dictionary.load(prefix+'_dictionary.dict')
    return corpus, dictionary



def update_corpus_labels(dictionary, corpus, newclassdict, preprocess_and_tokenize=tokenize):