<jupyter_start><jupyter_text>2020<jupyter_code>filtre1 = dfc['Groupe'] == 'Novice'
filtre2 = dfc['Annee'] == 2020
filtre3 = dfc['Groupe'] == 'Exp'

df_novice_2020 = dfc.where(filtre1 & filtre2).dropna().copy()
df_exp_2020 = dfc.where(filtre3 & filtre2).dropna().copy()

df_novice_2020_shuffled = df_novice_2020.sample(frac=1) #taille de la base : 871
phrases_novice_2020 = [df_novice_2020_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 * 400)]

WMD_novices_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_novice_2020[i], '_Novice'), 
            add_groupe(phrases_novice_2020[i + 1], '_Novice')) 
    for i in range(0, 800, 2)
                        ]
SI_N_2020 = SansInf(WMD_novices_allmodel)
means_novice_allmodel_2020 = [moyenne(SI_N_2020[:i]) for i in range(1, 400)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_novice_allmodel_2020)
a[0].set_title("WMD des experts, allmodel, 2020")

a[1].plot([i for i in range(300,399)],means_novice_allmodel_2020[300:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()
df_exp_2020_shuffled = df_exp_2020.sample(frac=1) #Taille de la base : 5062
phrases_exp_2020 = [df_exp_2020_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 *2500)]

WMD_exp_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_exp_2020[i], '_Exp'), 
            add_groupe(phrases_exp_2020[i + 1], '_Exp')) 
    for i in range(0, 5000, 2)
                        ]

SI_E_2020 = SansInf(WMD_exp_allmodel)
means_exp_allmodel_2020 = [moyenne(SI_E_2020[:i]) for i in range(1, 2500)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_exp_allmodel_2020)
a[0].set_title("WMD des experts, allmodel, 2020")

a[1].plot([i for i in range(1500,2499)],means_exp_allmodel_2020[1500:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()<jupyter_output><empty_output><jupyter_text>2019<jupyter_code>filtre1 = dfc['Groupe'] == 'Novice'
filtre2 = dfc['Annee'] == 2019
filtre3 = dfc['Groupe'] == 'Exp'

df_novice_2019 = dfc.where(filtre1 & filtre2).dropna().copy()
df_exp_2019 = dfc.where(filtre3 & filtre2).dropna().copy()

df_novice_2019_shuffled = df_novice_2019.sample(frac=1) #taille de la base : 1152
phrases_novice_2019 = [df_novice_2019_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 * 500)]

WMD_novices_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_novice_2019[i], '_Novice'), 
            add_groupe(phrases_novice_2019[i + 1], '_Novice')) 
    for i in range(0, 1000, 2)
                        ]

SI_N_2019 = SansInf(WMD_novices_allmodel)
means_novice_allmodel_2019 = [moyenne(SI_N_2019[:i]) for i in range(1, 500)]


fig,a =  plt.subplots(1,2)

a[0].plot(means_novice_allmodel_2019)
a[0].set_title("WMD des novices, allmodel, 2019")

a[1].plot([i for i in range(200,499)],means_novice_allmodel_2019[200:])
a[1].set_title("Zoom sur les valeurs")
plt.show()
df_exp_2019_shuffled = df_exp_2019.sample(frac=1) #Taille de la base : 8000
phrases_exp_2019 = [df_exp_2019_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 *4000)]

WMD_exp_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_exp_2019[i], '_Exp'), 
            add_groupe(phrases_exp_2019[i + 1], '_Exp')) 
    for i in range(0, 8000, 2)
                        ]

SI_E_2019 = SansInf(WMD_exp_allmodel)
means_exp_allmodel_2019 = [moyenne(SI_E_2019[:i]) for i in range(1, 4000)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_exp_allmodel_2019)
a[0].set_title("WMD des experts, allmodel, 2019")

a[1].plot([i for i in range(1500,3999)],means_exp_allmodel_2019[1500:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()<jupyter_output><empty_output><jupyter_text>2018<jupyter_code>filtre1 = dfc['Groupe'] == 'Novice'
filtre2 = dfc['Annee'] == 2018
filtre3 = dfc['Groupe'] == 'Exp'

df_novice_2018 = dfc.where(filtre1 & filtre2).dropna().copy()
df_exp_2018 = dfc.where(filtre3 & filtre2).dropna().copy()

df_novice_2018_shuffled = df_novice_2018.sample(frac=1) #taille de la base : 1500
phrases_novice_2018 = [df_novice_2018_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 * 700)]

WMD_novices_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_novice_2018[i], '_Novice'), 
            add_groupe(phrases_novice_2018[i + 1], '_Novice')) 
    for i in range(0, 1400, 2)
                        ]
SI_N_2018 = SansInf(WMD_novices_allmodel)
means_novice_allmodel_2018 = [moyenne(SI_N_2018[:i]) for i in range(1, 700)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_novice_allmodel_2018)
a[0].set_title("WMD des experts, allmodel, 2018")

a[1].plot([i for i in range(300,699)],means_novice_allmodel_2018[300:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()
df_exp_2018_shuffled = df_exp_2018.sample(frac=1) #Taille de la base : 11583
phrases_exp_2018 = [df_exp_2018_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 *5500)]

WMD_exp_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_exp_2018[i], '_Exp'), 
            add_groupe(phrases_exp_2018[i + 1], '_Exp')) 
    for i in range(0, 11000, 2)
                        ]

SI_E_2019 = SansInf(WMD_exp_allmodel)
means_exp_allmodel_2018 = [moyenne(SI_E_2019[:i]) for i in range(1, 5500)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_exp_allmodel_2018)
a[0].set_title("WMD des experts, allmodel, 2018")

a[1].plot([i for i in range(1500,5499)],means_exp_allmodel_2018[1500:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()<jupyter_output><empty_output><jupyter_text>2017<jupyter_code>filtre1 = dfc['Groupe'] == 'Novice'
filtre2 = dfc['Annee'] == 2017
filtre3 = dfc['Groupe'] == 'Exp'

df_novice_2017 = dfc.where(filtre1 & filtre2).dropna().copy()
df_exp_2017 = dfc.where(filtre3 & filtre2).dropna().copy()

df_novice_2017_shuffled = df_novice_2017.sample(frac=1) #taille de la base : 825
phrases_novice_2017 = [df_novice_2017_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 * 400)]

WMD_novices_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_novice_2017[i], '_Novice'), 
            add_groupe(phrases_novice_2017[i + 1], '_Novice')) 
    for i in range(0, 800, 2)
                        ]
SI_N_2017 = SansInf(WMD_novices_allmodel)
means_novice_allmodel_2017 = [moyenne(SI_N_2018[:i]) for i in range(1, 400)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_novice_allmodel_2017)
a[0].set_title("WMD des experts, allmodel, 2017")

a[1].plot([i for i in range(300,399)],means_novice_allmodel_2017[300:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()
df_exp_2017_shuffled = df_exp_2017.sample(frac=1) #Taille de la base : 5705
phrases_exp_2017 = [df_exp_2017_shuffled.iloc[i]['tokenized_replique'].split(' ') for i in range(2 *2800)]

WMD_exp_allmodel = [
    w2v_model_all.wv.wmdistance(
            add_groupe(phrases_exp_2017[i], '_Exp'), 
            add_groupe(phrases_exp_2017[i + 1], '_Exp')) 
    for i in range(0, 5600, 2)
                        ]

SI_E_2017 = SansInf(WMD_exp_allmodel)
means_exp_allmodel_2017 = [moyenne(SI_E_2017[:i]) for i in range(1, 2800)]

fig,a =  plt.subplots(1,2)

a[0].plot(means_exp_allmodel_2017)
a[0].set_title("WMD des experts, allmodel, 2017")

a[1].plot([i for i in range(1500,2799)],means_exp_allmodel_2017[1500:])
a[1].set_title("Zoom sur les dernières valeurs")

plt.show()
#On va comparer les distances intra de chaque modele selon le temps
fig, a = plt.subplots(1,4)

a[3].plot([i for i in range(300,399)],means_novice_allmodel_2020[300:])
a[3].set_title("novice allmodel 2020")

a[2].plot([i for i in range(200,499)],means_novice_allmodel_2019[200:])
a[2].set_title("2019")

a[1].plot([i for i in range(1500,5499)],means_exp_allmodel_2018[1500:])
a[1].set_title("2018")

a[0].plot([i for i in range(300,399)],means_novice_allmodel_2017[300:])
a[0].set_title("2017")

plt.show()
m1 = moyenne(means_novice_allmodel_2017[300:])
m2 = moyenne(means_novice_allmodel_2018[300:])
m3 = moyenne(means_novice_allmodel_2019[300:])
m4 = moyenne(means_novice_allmodel_2020[300:])



Moyennes = [m1,m2,m3,m4]
Annees = ["2017","2018","2019","2020"]
Donnees = pd.DataFrame({'Moyennes': Moyennes, 'Annees': Annees})

fig = px.bar(Donnees, x="Annees", y="Moyennes", title="Distances intra en fonction du temps, chez les novices")
fig.show()
fig, a = plt.subplots(1,4)

a[3].plot([i for i in range(1500,2499)],means_exp_allmodel_2020[1500:])
a[3].set_title("2020")

a[2].plot([i for i in range(1500,3999)],means_exp_allmodel_2019[1500:])
a[2].set_title("2019")

a[1].plot([i for i in range(300,699)],means_novice_allmodel_2018[300:])
a[1].set_title("2018")

a[0].plot([i for i in range(1500,2799)],means_exp_allmodel_2017[1500:])
a[0].set_title("2017")

plt.show()
m1 = moyenne(means_exp_allmodel_2017[300:])
m2 = moyenne(means_exp_allmodel_2018[300:])
m3 = moyenne(means_exp_allmodel_2019[300:])
m4 = moyenne(means_exp_allmodel_2020[300:])



Moyennes = [m1,m2,m3,m4]
Annees = ["2017","2018","2019","2020"]
Donnees = pd.DataFrame({'Moyennes': Moyennes, 'Annees': Annees})

fig = px.bar(Donnees, x="Annees", y="Moyennes", title="Distances intra en fonction du temps, chez les experts")
fig.show()<jupyter_output><empty_output><jupyter_text>On va regarder si, par exemple, la diversité des sujets en 2020 est plus petite qu'en 2019, ce qui expliquerait pourquoi la distance intra a baissé dans les deux groupes<jupyter_code>filtre1 = dfc['Annee'] == 2017
filtre2 = dfc['Annee'] == 2018
filtre3 = dfc['Annee'] == 2019
filtre4 = dfc['Annee'] == 2020

df_2017 = dfc.where(filtre1).dropna().copy()
df_2018 = dfc.where(filtre2).dropna().copy()
df_2019 = dfc.where(filtre3).dropna().copy()
df_2020 = dfc.where(filtre4).dropna().copy()
n1 = df_2017['Sujet.débattu'].nunique()
n2 = df_2018['Sujet.débattu'].nunique()
n3 = df_2019['Sujet.débattu'].nunique()
n4 = df_2020['Sujet.débattu'].nunique()

Sujets = [n1,n2,n3,n4]
Annees = ["2017","2018","2019","2020"]
Donnees = pd.DataFrame({'Nombre de sujets débattus': Sujets, 'Annees': Annees})

fig = px.bar(Donnees, x="Annees", y="Nombre de sujets débattus", title="Nombre de sujets débattus en fonction du temps")
fig.show()<jupyter_output><empty_output><jupyter_text>Le nombre de sujets débattus ne semble pas expliquer à lui seul les différences entre les distances<jupyter_code>n1 = df_2017['Orateur'].nunique()
n2 = df_2018['Orateur'].nunique()
n3 = df_2019['Orateur'].nunique()
n4 = df_2020['Orateur'].nunique()

Sujets = [n1,n2,n3,n4]
Annees = ["2017","2018","2019","2020"]
Donnees = pd.DataFrame({'Nombre dorateurs': Sujets, 'Annees': Annees})

fig = px.bar(Donnees, x="Annees", y="Nombre dorateurs", title="Nombre dorateurs en fonction du temps")
fig.show()
n1 = df_2017['Nature.de.séance'].nunique()
n2 = df_2018['Nature.de.séance'].nunique()
n3 = df_2019['Nature.de.séance'].nunique()
n4 = df_2020['Nature.de.séance'].nunique()

Sujets = [n1,n2,n3,n4]
Annees = ["2017","2018","2019","2020"]
Donnees = pd.DataFrame({'Nombre de séances': Sujets, 'Annees': Annees})

fig = px.bar(Donnees, x="Annees", y="Nombre de séances", title="Nombre de nature de séances différentes en fonction du temps")
fig.show()
dfc<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16026785714285716, 'fraction_numerical': 0.10910714285714286, 'mean_word_length': 5.538820782253357, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '131075', 'license': 'MIT', 'n_tokens_mistral': 5602, 'n_tokens_neox': 4678, 'n_words': 794, 'path': 'temporalite_WMD.ipynb', 'repo_name': 'lilianmarey/Statapp---Novices---NLP', 'size': '11200'}