# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


import numpy as np
import gensim
import requests
from gensim.models import KeyedVectors
from gensim.models.keyedvectors import KeyedVectors
from gensim.models.poincare import PoincareModel, PoincareKeyedVectors
from shorttext.utils import tokenize
    :param path: path of the file of the pre-trained Poincare embedding model
    :param word2vec_format: whether to load from word2vec format (default: True)
    :param binary: binary format (default: False)
    :return: a pre-trained Poincare embedding model
    :type path: str
    :type word2vec_format: bool
    :type binary: bool
    :rtype: gensim.models.poincare.PoincareKeyedVectors
    """
    if word2vec_format:
        return PoincareKeyedVectors.load_word2vec_format(path, binary=binary)
    else:
        return PoincareModel.load(path).kv


def shorttext_to_avgvec(shorttext, wvmodel):
    """ Convert the short text into an averaged embedded vector representation.

    Given a short sentence, it converts all the tokens into embedded vectors according to
    the given word-embedding model, sums
    them up, and normalize the resulting vector. It returns the resulting vector
    that represents this short sentence.

    :param shorttext: a short sentence
    :param wvmodel: word-embedding model
    :return: an embedded vector that represents the short sentence
    :type shorttext: str
    :type wvmodel: gensim.models.keyedvectors.KeyedVectors
    :rtype: numpy.ndarray
    """

# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


# Path: shorttext/utils/wordembed.py
import numpy as np
import gensim
import requests
from gensim.models import KeyedVectors
from gensim.models.keyedvectors import KeyedVectors
from gensim.models.poincare import PoincareModel, PoincareKeyedVectors
from shorttext.utils import tokenize

    :param path: path of the file of the pre-trained Poincare embedding model
    :param word2vec_format: whether to load from word2vec format (default: True)
    :param binary: binary format (default: False)
    :return: a pre-trained Poincare embedding model
    :type path: str
    :type word2vec_format: bool
    :type binary: bool
    :rtype: gensim.models.poincare.PoincareKeyedVectors
    """
    if word2vec_format:
        return PoincareKeyedVectors.load_word2vec_format(path, binary=binary)
    else:
        return PoincareModel.load(path).kv


def shorttext_to_avgvec(shorttext, wvmodel):
    """ Convert the short text into an averaged embedded vector representation.

    Given a short sentence, it converts all the tokens into embedded vectors according to
    the given word-embedding model, sums
    them up, and normalize the resulting vector. It returns the resulting vector
    that represents this short sentence.

    :param shorttext: a short sentence
    :param wvmodel: word-embedding model
    :return: an embedded vector that represents the short sentence
    :type shorttext: str
    :type wvmodel: gensim.models.keyedvectors.KeyedVectors
    :rtype: numpy.ndarray
    """

    vec = np.sum([wvmodel[token] for token in tokenize(shorttext) if token in wvmodel], axis=0)