<jupyter_start><jupyter_text>Explore experiences result on NCBI<jupyter_code>import os 
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt 

import random
random.seed(12)

import sys
sys.path.insert(1, '../../utils_paper/')
from resultsUtils import build_jsons, json2pandas
from visuClass import visualizer, yaset_pred

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Build jsons<jupyter_code># fetch results and store in different paths
path2jsons = '../jsons/'
build_jsons('../results/ncbi/', path2jsons, show_path=False)<jupyter_output>94%|█████████▍| 34/36 [01:04<00:03,  1.89s/it]<jupyter_text>Analysis<jupyter_code>#!rm ../../jsons/.ipynb_checkpoints -rf
df = json2pandas(path2jsons, trajs = False, fixed_iter=None, verbose=False)<jupyter_output><empty_output><jupyter_text>Pubmed embeddings<jupyter_code>df['best_iter'].plot(kind = 'hist', bins = 30)
df = df.iloc[:30]
print(df.shape)
df.head(5)
plt.figure(figsize=(10,4))
plt.boxplot(df['f1'], vert=False, showmeans=True)
plt.scatter(df['f1'], np.repeat(1.1, len(df['f1'])), alpha = 0.5)
plt.show()
print('average: ', np.mean(df['f1']))
print('sd: ', np.std(df['f1']))
print('max diff: ', np.max(df['f1']) - np.min(df['f1']))
print('max diff perc: ', (np.max(df['f1']) - np.min(df['f1']))/np.min(df['f1']))<jupyter_output><empty_output><jupyter_text>Evaluation on the test setTake long time, prefer to exploit the preprocessed results in /results/ncbi_apply<jupyter_code>path2test = '../data/ncbi_test.conll'
path2models = '../results/ncbi/'
path2outputs = '../results/ncbi_apply/'

for model in os.listdir(path2models):
    path2model = path2models+model+'/'
    #yaset_pred(path2model=path2model, pathgold=path2test, path2save=path2outputs)
from tqdm import tqdm
path2results = '../results/ncbi_apply/'
path2golds = '../data/ncbi_test.conll'
test_scores = []
for folder in tqdm(os.listdir(path2results)):
    path2pred = path2results + folder+'/'+'output-model-001.conll'
    if not os.path.isfile(path2pred):
        print(path2results+folder, 'is uncomplete experiment')
    else:
        join_visu = visualizer(path2golds, path2pred, verbose = False)
        test_scores.append(join_visu.score_predictions(average = 'micro', punct_ignore = True))

test_df = pd.DataFrame(test_scores, columns=['precision', 'recall', 'f1', 'count'])
#test_df.pivot(index=0, columns=1, values=2)
test_df.drop(['count'], axis = 1, inplace = True)
test_df = test_df.iloc[:30]
test_df.shape
plt.figure(figsize=(10,4))
plt.boxplot(test_df['f1'], vert=False, showmeans=True)
plt.scatter(test_df['f1'], np.repeat(1.1, len(test_df['f1'])), alpha = 0.7)
plt.show()
print('average: ', np.mean(test_df['f1']))
print('sd: ', np.std(test_df['f1']))
print('max diff: ', np.max(test_df['f1']) - np.min(test_df['f1']))
print('max diff perc: ', 100*(np.max(test_df['f1']) - np.min(test_df['f1']))/np.min(test_df['f1']))<jupyter_output><empty_output><jupyter_text>Mean iter time on train<jupyter_code>df.cpu_cores.value_counts()
nb_train = np.mean(df.nb_train_tokens )
print(nb_train)
nb_sents = 7279
df.mean_iter_time_s
valid_time = pd.DataFrame([t for t in df.mean_iter_time_s if t < 150])
print(len(valid_time))
sent_sec = nb_sents / valid_time
tok_sec = nb_train / valid_time
print('nb_sents /s {}'.format(np.mean(sent_sec)))
print('nb_toks /s {}'.format(np.mean(tok_sec)))
df.mean_iter_time_s<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1792009400705053, 'fraction_numerical': 0.030846063454759105, 'mean_word_length': 6.467105263157895, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '375904', 'license': 'Apache-2.0', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1333, 'n_words': 228, 'path': 'ncbi/code/results_ncbi.ipynb', 'repo_name': 'strayMat/bio-medical_ner', 'size': '3404'}