<jupyter_start><jupyter_text>Graphique exploratoire de la base *allocine*<jupyter_code>import pandas as pd
import spacy
import seaborn as sns
from datasets import load_dataset
dataset_allocine = load_dataset("allocine")
nlp = spacy.load('fr_core_news_sm', disable=["tagger", "parser","ner"])
def tokenizer(exemple):
    return [X.lemma_ for X in nlp(exemple) if X.is_alpha & (not(X.is_stop))]
allo_train = pd.DataFrame(dataset_allocine["train"]).apply(lambda x : tokenizer(x["review"]), axis = 1)<jupyter_output><empty_output><jupyter_text>Distribution des longueurs des phrases dans le training set<jupyter_code>mots_train = allo_train.apply(lambda x : len(x)).hist()
mots_train.set_xlabel("Number of words")
mots_train.set_ylabel("Number of sentences")

mots_train.get_figure().savefig('hist_train_word.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12738095238095237, 'fraction_numerical': 0.0011904761904761906, 'mean_word_length': 8.779069767441861, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '79653', 'license': 'MIT', 'n_tokens_mistral': 306, 'n_tokens_neox': 300, 'n_words': 60, 'path': 'docs/doc/embed_cnn/graphique/graphique presentation allocine.ipynb', 'repo_name': 'ENSAE-CKW/nlp_understanding', 'size': '840'}