# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


from itertools import product
from scipy.spatial.distance import cosine
from shorttext.utils import tokenize
import numpy as np





def jaccardscore_sents(sent1, sent2, wvmodel, sim_words=lambda vec1, vec2: 1-cosine(vec1, vec2)):
    """ Compute the Jaccard score between sentences based on their word similarities.

    :param sent1: first sentence
    :param sent2: second sentence
    :param wvmodel: word-embeding model
    :param sim_words: function for calculating the similarities between a pair of word vectors (default: cosine)
    :return: soft Jaccard score
    :type sent1: str
    :type sent2: str
    :type wvmodel: gensim.models.keyedvectors.KeyedVectors
    :type sim_words: function
    :rtype: float
    """
    tokens1 = tokenize(sent1)

# Path: shorttext/utils/textpreprocessing.py
# def preprocess_text(text, pipeline):
# def text_preprocessor(pipeline):
# def oldschool_standard_text_preprocessor(stopwordsfile):
# def standard_text_preprocessor_1():
# def standard_text_preprocessor_2():


# Path: shorttext/metrics/embedfuzzy/jaccard.py
from itertools import product
from scipy.spatial.distance import cosine
from shorttext.utils import tokenize
import numpy as np






def jaccardscore_sents(sent1, sent2, wvmodel, sim_words=lambda vec1, vec2: 1-cosine(vec1, vec2)):
    """ Compute the Jaccard score between sentences based on their word similarities.

    :param sent1: first sentence
    :param sent2: second sentence
    :param wvmodel: word-embeding model
    :param sim_words: function for calculating the similarities between a pair of word vectors (default: cosine)
    :return: soft Jaccard score
    :type sent1: str
    :type sent2: str
    :type wvmodel: gensim.models.keyedvectors.KeyedVectors
    :type sim_words: function
    :rtype: float
    """
    tokens1 = tokenize(sent1)

    tokens2 = tokenize(sent2)