<jupyter_start><jupyter_text>Deuxième essai BM25 pour requêter les DPEFs et chercher des sujets spécifiques.- Avec bag of words, courts fragemnts privilégiés- embeddings vraiment snécéssaire égalementsOn peut voir que certains sujets concernent certaines entreprises.<jupyter_code># ! pip install rank_bm25 
import pandas as pd
import numpy as np
from rank_bm25 import BM25Okapi
pd.options.display.max_colwidth=160

# text
from spacy.lang.fr import French
from spacy.cli.download import download
import spacy
# download('fr_core_news_sm')
nlp = French()
def tokenize(input_str):
    return [token.text for token in nlp(input_str)]
dpef_entreprises = "../../data/processed/DPEFs/dpef_paragraphs_sentences_long_format.csv"
df = pd.read_csv(dpef_entreprises, sep=";")
print(df.shape)
df = df[df.paragraph.notna()]
print(df.shape)
df.head()
df["sentence_"] = df["sentence"].apply(tokenize)
df= df.reset_index()

bm25_index = BM25Okapi(df["sentence_"].values.tolist())

def search(search_string, num_results=10):
    search_tokens = tokenize(search_string)
    scores = bm25_index.get_scores(search_tokens)
    top_indexes = np.argsort(scores)[::-1][:num_results]
    top_scores = scores[top_indexes]
    return top_indexes, top_scores
scores
pd.set_option('display.max_colwidth', -1)
indexes, scores = search('lutter pollution air')
df.loc[indexes, ["denomination","sentence"]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11404133998574484, 'fraction_numerical': 0.01496792587312901, 'mean_word_length': 6.389473684210526, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '84466', 'license': 'MIT', 'n_tokens_mistral': 517, 'n_tokens_neox': 490, 'n_words': 109, 'path': 'webapp/polls/rse_model/notebooks/Analysis - BM25 Querier V2 on sentences only.ipynb', 'repo_name': 'tristanrobert/batch7_rse', 'size': '1403'}