<jupyter_start><jupyter_text>Explore training performances on conll2003<jupyter_code>import os 
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt 

import random
random.seed(12)
import sys
sys.path.insert(1, '../../utils_paper/')
from resultsUtils import build_jsons, get_results, json2pandas, plot_results
from visuClass import visualizer, yaset_pred

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Build jsons<jupyter_code># Fetch results from log files and store them in different paths
path2jsonsGG = '../jsons/jsons_ggnews/'
path2jsonsFT = '../jsons/jsons_fT/'
path2jsonsFTjoin = '../jsons/jsons_fT_train_dev/'
build_jsons('../results/bio/', path2jsonsGG, show_path=True)
build_jsons('../results/fastText_embs/', path2jsonsFT, show_path=False)
build_jsons('../results/fastText_embs_join/', path2jsonsFTjoin, show_path=False)<jupyter_output>100%|██████████| 3/3 [00:04<00:00,  1.44s/it]<jupyter_text>Analysis<jupyter_code>#!rm ../../jsons/.ipynb_checkpoints -rf<jupyter_output><empty_output><jupyter_text>ggnews embeddings<jupyter_code>df = json2pandas(path2jsonsGG, trajs = False, fixed_iter=None, verbose=False)
df['best_iter'].plot(kind = 'hist', bins = 30)
print(df.shape)
df.sort_values('f1', ascending=False).head(5)<jupyter_output>(30, 17)<jupyter_text>Box plot<jupyter_code>plt.figure(figsize=(10,4))
plt.boxplot(df['f1'], vert=False, showmeans=True)
plt.scatter(df['f1'], np.repeat(1.1, len(df['f1'])), alpha = 0.7)
plt.show()
print('average: ', np.mean(df['f1']))
print('sd: ', np.std(df['f1']))
print('max diff: ', np.max(df['f1']) - np.min(df['f1']))
print('max diff perc: ', (np.max(df['f1']) - np.min(df['f1']))/np.min(df['f1']))<jupyter_output><empty_output><jupyter_text>FastText embeddings trained on train set<jupyter_code>dfFT = json2pandas(path2jsonsFT, trajs = False, fixed_iter=None, verbose=False)
dfFT['best_iter'].plot(kind = 'hist', bins = 30)
print(dfFT.shape)
dfFT.sort_values('f1', ascending=False).head(5)
plt.figure(figsize=(10,4))
plt.boxplot(dfFT['f1'], vert=False, showmeans=True)
plt.scatter(dfFT['f1'], np.repeat(1.1, len(dfFT['f1'])), alpha = 0.7)
plt.show()
print('average: ', np.mean(dfFT['f1']))
print('sd: ', np.std(df['f1']))
print('max diff: ', np.max(dfFT['f1']) - np.min(dfFT['f1']))
print('max diff perc: ', (np.max(dfFT['f1']) - np.min(dfFT['f1']))/np.min(dfFT['f1']))<jupyter_output><empty_output><jupyter_text>FastText embeddings trained on train and dev set<jupyter_code>dfFTjoin = json2pandas(path2jsonsFTjoin, trajs = False, fixed_iter=None, verbose=False)
dfFTjoin['best_iter'].plot(kind = 'hist', bins = 30)
print(dfFTjoin.shape)
dfFTjoin.sort_values('f1', ascending=False).head(5)
plt.figure(figsize=(10,4))
plt.boxplot(dfFTjoin['f1'], vert=False, showmeans=True)
plt.scatter(dfFTjoin['f1'], np.repeat(1.1, len(dfFTjoin['f1'])), alpha = 0.7)
plt.show()<jupyter_output><empty_output><jupyter_text>Evaluation on test set  Decoding with yaset on the test set **Warning**: It takes an afternoon, you should directly fetch the results in `../results/conll_apply/30_runs/`<jupyter_code>path2test = '../data/bio_test.conll'
path2models = '../results/bio/'
path2outputs = '../results/conll_apply/30_runs/'

for model in os.listdir(path2models):
    path2model = path2models+model+'/'
    ## uncomment following line to decode (long time)
    #yaset_pred(path2model=path2model, pathgold=path2test, path2save=path2outputs)<jupyter_output><empty_output><jupyter_text>Building results from prediction<jupyter_code>from tqdm import tqdm
path2results = '../results/conll_apply/30_runs/'
path2golds = '../data/test.conll'
test_scores = []
for folder in tqdm(os.listdir(path2results)):
    path2pred = path2results + folder+'/'+'output-model-001.conll'
    join_visu = visualizer(path2golds, path2pred, verbose = False)
    test_scores.append(join_visu.score_predictions(average = 'micro', punct_ignore = True))

test_df = pd.DataFrame(test_scores, columns=['precision', 'recall', 'f1', 'count'])
#test_df.pivot(index=0, columns=1, values=2)
test_df.drop(['count'], axis = 1, inplace = True)
test_df.shape
plt.figure(figsize=(10,4))
plt.boxplot(test_df['f1'], vert=False, showmeans=True)
plt.scatter(test_df['f1'], np.repeat(1.1, len(test_df['f1'])), alpha = 0.7)
plt.show()
print('average: ', np.mean(test_df['f1']))
print('sd: ', np.std(test_df['f1']))
print('max diff: ', np.max(test_df['f1']) - np.min(test_df['f1']))
print('max diff perc: ', 100*(np.max(test_df['f1']) - np.min(test_df['f1']))/np.min(test_df['f1']))<jupyter_output><empty_output><jupyter_text>Mean time for one batch<jupyter_code>df.cpu_cores.value_counts()
nb_train = np.mean(df.nb_train_tokens )
print(nb_train)
nb_sents = 18451
df.mean_iter_time_s
valid_time = pd.DataFrame([t for t in df.mean_iter_time_s if t < 500])
print(len(valid_time))
sent_sec = nb_sents / valid_time
tok_sec = nb_train / valid_time
print('nb_sents /s {}'.format(np.mean(sent_sec)))
print('nb_toks /s {}'.format(np.mean(tok_sec)))
df.mean_iter_time_s
dfFT.mean_iter_time_s<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18252080856123662, 'fraction_numerical': 0.03111375346809354, 'mean_word_length': 7.355960264900662, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '123453', 'license': 'Apache-2.0', 'n_tokens_mistral': 2102, 'n_tokens_neox': 1992, 'n_words': 322, 'path': 'conll2003/code/results_conll2003.ipynb', 'repo_name': 'strayMat/bio-medical_ner', 'size': '5046'}