<commit_before>import heapq,random
from compatibility import range, pickle
import numpy as np
from scipy.spatial.distance import cosine

def gen_embedding(word):
    '''
    Generates embedding of the word from the model trained.
    '''
    try:
        with open('./embeddings.pickle', 'rb') as f:
            embeddings = pickle.load(f)
        return embeddings[word]
    except Exception as e:
        print ("Exception: Model file not found, please train the model first by runing train")

def closest_words(word,topn=10):
    '''
    Returns top 10 closest words to word provided by user.
    '''
    try:
        embeddings = pickle.load(open("./embeddings.pickle"))
        words = embeddings.keys()
        closest = []
        # Embedding of word provided by user
        vec = embeddings[word]
        for word in words:
            heapq.heappush(closest, (1 - cosine(vec, embeddings[word]), word))
        closest_words = heapq.nlargest(topn, closest)
        return closest_words
    except Exception as e:
        print ("Exception: Model file not found, please train the model by running train function")
<commit_msg>Update utils.py<commit_after>import heapq,random
from compatibility import range, pickle
import numpy as np
from scipy.spatial.distance import cosine

def gen_embedding(word):
    '''
    Generates embedding of the word from the model trained.
    '''
    try:
        with open('./embeddings.pickle', 'rb') as f:
            embeddings = pickle.load(f)
        return embeddings[word]
    except Exception as e:
        print ("Exception: Model file not found, please train the model first by runing train")

def closest_words(word,topn=10):
    '''
    Returns top 10 closest words to word provided by user.
    '''
    try:
        with open("./embeddings.pickle", "rb") as f:
            embeddings = pickle.load(f)
        words = embeddings.keys()
        closest = []
        # Embedding of word provided by user
        vec = embeddings[word]
        for word in words:
            heapq.heappush(closest, (1 - cosine(vec, embeddings[word]), word))
        closest_words = heapq.nlargest(topn, closest)
        return closest_words
    except Exception as e:
        print ("Exception: Model file not found, please train the model by running train function")

{'fraction_non_alphanumeric': 0.09169926119078661, 'fraction_numerical': 0.004345936549326379, 'mean_word_length': 2.6773162939297124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1304341', 'n_tokens_mistral': 649, 'n_tokens_neox': 637, 'n_words': 230}