<commit_before>import numpy
from gensim import corpora, models, similarities


def get_simlarity_scores_for_all_pairs(texts):
    '''Takes a list of texts as input and returns a matrix, where element (m,n) represents the cosine similarity of texts[m] and texts[n].'''
    n = len(texts)
    similarity_result_matrix = numpy.empty(shape=(n, n))
    similarity_matrix = __get_corpus_tfidf_similarity_matrix__(texts)
    # for every row in the matrix, find its similarity to every other row
    for i, vec in similarity_matrix:
        similarity_result_matrix[i] = list(similarity_matrix[vec]) #  the [vec] notation returns a vector where each element is the cosine similarity value
    return similarity_result_matrix


def __get_corpus_tfidf_similarity_matrix__(texts):
    '''Takes a list of strings as input. Returns a gensim.similarities.SparseMatrixSimilarity object for calculating cosine similarities.'''
    # gensim has us convert tokens to IDs using corpora.Dictionary
    dictionary = corpora.Dictionary([text.lower().split() for text in texts])
    corpus = [dictionary.doc2bow(text) for text in texts]
    corpus_tfidf = models.TfidfModel(corpus, normalize=True)[texts] #  Feed texts back into its own model to get the TF-IDF values for the texts
    return similarities.SparseMatrixSimilarity(corpus_tfidf)

# def add_tfidf_to(texts):
#     freq_dists = get_frequency_distributions(texts)
#     freq_dist_all = reduce(lambda x, y: x + y, freq_dists)

#     tf_idfs = []
#     for i, text in enumerate(texts):
#         idf = log(len(texts) / float())


#     tokens = {}
#     for id, doc in enumerate(documents):
#         get_frequency_distributions()

#         tf = {}
#         doc["tfidf"] = {}
#         doc_tokens = doc.get("tokens", [])
#         for token in doc_tokens:
#             tf[token] = tf.get(token, 0) + 1
#         num_tokens = len(doc_tokens)
#         if num_tokens > 0:
#             for token, freq in tf.iteritems():
#                 tokens.setdefault(token, []).append((id, float(freq) / num_tokens))

#     doc_count = float(len(documents))<commit_msg>Fixing stupid !@#!@# PEP 8 violations.<commit_after>import numpy
from gensim import corpora, models, similarities


def get_simlarity_scores_for_all_pairs(texts):
    '''Takes a list of texts as input and returns a matrix, where element (m,n) represents the cosine similarity of texts[m] and texts[n].'''
    n = len(texts)
    similarity_result_matrix = numpy.empty(shape=(n, n))
    similarity_matrix = __get_corpus_tfidf_similarity_matrix__(texts)
    # for every row in the matrix, find its similarity to every other row
    for i, vec in similarity_matrix:
        similarity_result_matrix[i] = list(similarity_matrix[vec])  # the [vec] notation returns a vector where each element is the cosine similarity value
    return similarity_result_matrix


def __get_corpus_tfidf_similarity_matrix__(texts):
    '''Takes a list of strings as input. Returns a gensim.similarities.SparseMatrixSimilarity object for calculating cosine similarities.'''
    # gensim has us convert tokens to IDs using corpora.Dictionary
    dictionary = corpora.Dictionary([text.lower().split() for text in texts])
    corpus = [dictionary.doc2bow(text) for text in texts]
    corpus_tfidf = models.TfidfModel(corpus, normalize=True)[texts]  # Feed texts back into its own model to get the TF-IDF values for the texts
    return similarities.SparseMatrixSimilarity(corpus_tfidf)

# def add_tfidf_to(texts):
#     freq_dists = get_frequency_distributions(texts)
#     freq_dist_all = reduce(lambda x, y: x + y, freq_dists)

#     tf_idfs = []
#     for i, text in enumerate(texts):
#         idf = log(len(texts) / float())


#     tokens = {}
#     for id, doc in enumerate(documents):
#         get_frequency_distributions()

#         tf = {}
#         doc["tfidf"] = {}
#         doc_tokens = doc.get("tokens", [])
#         for token in doc_tokens:
#             tf[token] = tf.get(token, 0) + 1
#         num_tokens = len(doc_tokens)
#         if num_tokens > 0:
#             for token, freq in tf.iteritems():
#                 tokens.setdefault(token, []).append((id, float(freq) / num_tokens))

#     doc_count = float(len(documents))
{'fraction_non_alphanumeric': 0.10173108845150582, 'fraction_numerical': 0.0021342186388427793, 'mean_word_length': 3.5209003215434085, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4313187', 'n_tokens_mistral': 1381, 'n_tokens_neox': 1314, 'n_words': 379}