<jupyter_start><jupyter_text>Test SAHAN<jupyter_code>PATH = "/home/benromdhane-w/Bureau/ls2n/projects/X5GON/hackaton/x5gon_hackathon_materials/x5gon_materials_catelogue.tsv"
PATHOUT = "/home/benromdhane-w/Bureau/ls2n/projects/X5GON/hackaton/x5gon_hackathon_materials/comparewiki.tsv"
import psycopg2

PGCred = dict(PGHOST='127.0.0.1',
              PGDATABASE='x5gon',
              PGUSER='developernantestest',
              PGPASSWORD='<PASSWORD>',
              PGPORT='5555')

def db_connect(PGHOST:str,PGDATABASE:str,PGUSER:str,PGPASSWORD:str,PGPORT:str):
    # Set up a connection to the postgres server.
    conn_string = "host="+ PGHOST +" port="+ PGPORT +" dbname="+ PGDATABASE +" user=" + PGUSER +" password="+ PGPASSWORD
    conn=psycopg2.connect(conn_string)
    conn.set_session(autocommit=True)
    cursor = conn.cursor()
    return {'connexion': conn, 'cursor': cursor}
connexion = db_connect(PGCred['PGHOST'],
                         PGCred['PGDATABASE'],
                         PGCred['PGUSER'],
                         PGCred['PGPASSWORD'],
                         PGCred['PGPORT'])
def get_data_generator(rid, *, verbose=True):
    query = "SELECT value FROM material_contents WHERE material_id=" + rid + " AND language='en' AND extension='plain';"  
    # query exec
    if verbose: print("Sending query...")
    try:
        cursor = connexion["cursor"]
        cursor.execute(query)
    except:
        print("Error when executing the query: verify it !")
    if verbose: print("Query processed!")
    return cursor
import csv 
import spacy
import time
nlp = spacy.load("en")
from wikification import LONG_TEXT_METHODS, wikification

# del LONG_TEXT_METHODS["sum_page_rank"]
nb = 0
with open(PATH, "r") as csvfile:
    with open(PATHOUT, "w") as csvout:
        csv_reader = csv.reader(csvfile, delimiter="\t", quotechar='"', quoting=csv.QUOTE_ALL)
        fields = next(csv_reader)
        print(fields)
        csv_writer = csv.writer(csvout, delimiter="\t", quotechar='"')
        csv_writer.writerow(fields + ["concepts sum_classic_page_rank cos",
                                      "concepts sum_classic_page_rank pk",
                                      "concepts sum_classic_page_rank cos top10",
                                      "concepts sum_classic_page_rank pk top10",
                                      "concepts recompute_on_anchor_text cos",
                                      "concepts recompute_on_anchor_text pk",
                                      "concepts recompute_on_anchor_text cos top10",
                                      "concepts recompute_on_anchor_text pk top10",
                                      "concepts sum_classic_page time",
                                      "concepts recompute_on_anchor_text time"])
        for row in csv_reader:
            nb += 1
            print(nb)
            rid = row[1]
            try:
                text = next(get_data_generator(rid, verbose=False))[0]["value"]
            except StopIteration:
                print(row, " has no en version in db!")
            wikifications = []
            runtime = []
            for m in LONG_TEXT_METHODS:
                print(f"Wikification {m}...")
                start = time.time()
                res = [(c['url'], c['title'], c['pageRank'], c['cosine']) for c in wikification(text, long_text_method_name=m)]
                end = time.time()
                ccos = sorted(res, key=lambda x: x[3], reverse=True)
                cpr = sorted(res, key=lambda x: x[2], reverse=True)
                top10cos = [c[1] for c in ccos[:10]]
                top10pr = [c[1] for c in cpr[:10]]
                wikifications.extend([ccos, cpr, top10cos, top10pr])
                runtime.append(end - start)
                print(f"{m} elapsed time: %f ms" % (1000*(end - start)))
            csv_writer.writerow(row + wikifications + runtime)
            if nb > 100: break
times
wikifications = {k: sorted(v, key=lambda x: x[2], reverse=True)[:20] for k, v in wikifications.items()}
wikifications["sum_classic_page_rank"]
wikifications["recompute_on_anchor_text"]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11530146528945472, 'fraction_numerical': 0.013932260389142446, 'mean_word_length': 2.1426415094339624, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '11650', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 1351, 'n_tokens_neox': 1266, 'n_words': 265, 'path': 'x5gonlamtools/tools/wikifier/test_wikification.ipynb', 'repo_name': 'X5GON/lamapi', 'size': '4170'}