<jupyter_start><jupyter_text>Load Data  Ground Truth<jupyter_code>gt_path = 'C:\\Users\\obarn\\Projects\\F-MT126-1\\vilio\\data\\features\\annotations\\gt'
# Ground truth
gt_dev = pd.read_json(os.path.join(gt_path, 'dev_all.jsonl'), lines=True)
gt_ts = pd.read_json(os.path.join(gt_path, 'test_seen.jsonl'), lines=True)
gt_tu = pd.read_json(os.path.join(gt_path, 'test_unseen.jsonl'), lines=True)
gt_test = gt_ts.append(gt_tu)
dev_or = {'Correct': gt_dev}<jupyter_output><empty_output><jupyter_text>Model Outputs<jupyter_code>paths = ['C:\\Users\\obarn\\Projects\\F-MT126-1\\vilio\\data\\outputs\\v3\\mean']
# Load data
dev, test = [], []
experiments = []
for path in tqdm(paths):
    for csv in sorted(os.listdir(path)):
        if ".csv" in csv and "test" in csv:
            exp = csv.split('_')[0]
            experiments.append(exp)            
            dev.append(pd.read_csv(os.path.join(path, f'{exp}_dev.csv')))
            test.append(pd.read_csv(os.path.join(path, f'{exp}_test.csv')))

#dev_or.update(dev)
dev = pd.concat(dev, axis=1)
test = pd.concat(test, axis=1)<jupyter_output>100%|██████████| 1/1 [00:01<00:00,  1.69s/it]<jupyter_text>Display  Dev<jupyter_code>scores = {}
for exp in dev.keys():
    scores[exp]=roc_auc_score(gt_dev.label, dev[exp])
    
out = '| 1 | 5 | 10 | 15 | 20 | 36 | 50 | 72 |\n'
out += '| --- | --- | --- | --- | --- | --- | --- | --- |\n'
for model in ['U', 'O', 'D', 'X']:   
    for flag in ['', 'a', 'c', 'ac']:
        row = [f'{scores.get(f"{model}{num}{flag}m")}' for num in [1, 5, 10, 15, 20, 36, 50, 72]]
        out += f'{" | ".join(row)} |\n'
md(out)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>scores = {}
for exp in test.keys():
    scores[exp]=roc_auc_score(gt_test.label, test[exp])
    
out = '| 1 | 5 | 10 | 15 | 20 | 36 | 50 | 72 |\n'
out += '| --- | --- | --- | --- | --- | --- | --- | --- |\n'
for model in ['U', 'O', 'D', 'X']:   
    for flag in ['', 'a', 'c', 'ac']:
        row = [f'{scores.get(f"{model}{num}{flag}m")}' for num in [1, 5, 10, 15, 20, 36, 50, 72]]
        out += f'{" | ".join(row)} |\n'
md(out)<jupyter_output><empty_output><jupyter_text>AUROC Curves<jupyter_code>exps = ['X20cm', 'D20cm', 'U20cm', 'O20cm']
for exp in exps:
    fpr, tpr, threshold = roc_curve(gt_test.label, test[exp])
    score = roc_auc_score(gt_test.label, test[exp])
    plt.plot(fpr, tpr, label = f'{exp[:-1]} AUC = {score:.3f}')
plt.title('Receiver Operating Characteristic')
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.23589164785553046, 'fraction_numerical': 0.03837471783295711, 'mean_word_length': 3.4614093959731544, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '247469', 'license': 'MIT', 'n_tokens_mistral': 1216, 'n_tokens_neox': 1110, 'n_words': 207, 'path': 'notebooks/display.ipynb', 'repo_name': 'obarnard99/vilio', 'size': '2658'}