<jupyter_start><jupyter_text>Disambiguation<jupyter_code>%%time
import itertools
import math

def disamb_sentences(sentences):
    cor_count=0
    count=0
    for sentence in sentences:
        count=count+1
        if (disambiguate(sentence[0])[0]==sentence[2][2]):
            cor_count = cor_count+1
        print(str(count/len(sentences))+"pecent done")
    return cor_count/len(sentences)

disamb_sentences(copy)
%%time
import itertools
import math

def disamb_sentences(sentences):
    cor_count=0
    for sentence in sentences:
    
        if (disambiguate(sentence[0])[0]==sentence[2][2]):
            cor_count = cor_count+1
    return cor_count/len(sentences)

disamb_sentences(copy[0:10000])<jupyter_output>Wall time: 1h 1min 26s<jupyter_text>https://arxiv.org/abs/1802.01074http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.557.136https://www.kaggle.com/gabrielaltay/kdwd-subclass-path-ner<jupyter_code>%%time
import itertools
import math
from multiprocessing import Pool


def process_image(sentence):
    cor_count=0
    if (disambiguate(sentence[0])[0]==sentence[2][2]):
        cor_count = cor_count+1
    return cor_count

if __name__ == '__main__':
    pool = Pool(os.cpu_count()-1)
    pool.map(process_image, sentences)  
    
disambiguate(sentences[0][0])<jupyter_output>[[275102], [1675432, 56538769, 30272204], [18366588, 217, 209754], [-1], [-1], [7913, 2175775, 57477615], [3627558, 2497107], [13211, 26245], [4135847, 4135849, 4135859, 7128194, 16562681, 17536409, 31283669, 11692785], [809, 385059, 22061936, 36982750, 57477613], [-1]]
4320
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<class 'list'>
<cla[...]
{'fraction_non_alphanumeric': 0.1818576388888889, 'fraction_numerical': 0.10460069444444445, 'mean_word_length': 5.196236559139785, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 55, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '171981', 'license': 'MIT', 'n_tokens_mistral': 1057, 'n_tokens_neox': 900, 'n_words': 198, 'path': 'intermediate_model/graph_traversal_serial_05_04.ipynb', 'repo_name': 'iacs-capstone-kensho/named-entity-linking', 'size': '2304'}