<jupyter_start><jupyter_text>IntroductionThis notebook compares the test set performance of classifiers using term list scores based on the data-driven, RDoC, and DSM frameworks. Load the data<jupyter_code>import pandas as pd
import numpy as np
np.random.seed(42)

import sys
sys.path.append("..")
import utilities<jupyter_output><empty_output><jupyter_text>Input parameters<jupyter_code>metric = "arche" # Variable being compared between frameworks
n_iter = 1000 # Number of iterations for bootstrap distributions
frameworks = ["data-driven", "rdoc", "dsm"]
clfs = ["_nn", "", ""]
splits = ["discovery", "replication"]
suffix = "nn"<jupyter_output><empty_output><jupyter_text>Framework contents<jupyter_code>lists, circuits = {}, {}
for fw, clf in zip(frameworks, clfs):
    lists[fw], circuits[fw] = utilities.load_framework(fw, suffix=utilities.suffix[fw], clf=clf)<jupyter_output><empty_output><jupyter_text>Observed values<jupyter_code>obs = {}
for fw, clf in zip(frameworks, clfs):
    obs[fw] = pd.read_csv("data/{}_obs_{}{}.csv".format(metric, fw, clf), index_col=0, header=0)<jupyter_output><empty_output><jupyter_text>Bootstrap distributions<jupyter_code>boot, mean = {}, {}
for split in splits:
    boot[split], mean[split] = {}, {}
    for fw, clf in zip(frameworks, clfs):
        boot[split][fw] = pd.read_csv("data/{}_boot_{}{}_{}_{}iter.csv".format(metric, fw, clf, split, n_iter), 
                                      index_col=0, header=0)
        mean[split][fw] = pd.read_csv("data/{}_mean_{}{}_{}.csv".format(metric, fw, clf, split), 
                                      index_col=0, header=0)<jupyter_output><empty_output><jupyter_text>Perform comparison testsIs the mean of fwi greater than the mean of fwj?<jupyter_code>from statsmodels.stats.multitest import multipletests
p, fdr = {}, {}
for split in splits:
    p[split] = np.empty((len(frameworks), len(frameworks)))
    for i, fw_i in enumerate(frameworks):
        for j, fw_j in enumerate(frameworks):
            boot_i = np.mean(boot[split][fw_i], axis=0)
            boot_j = np.mean(boot[split][fw_j], axis=0)
            p[split][i,j] = np.sum((boot_i - boot_j) <= 0.0) / float(n_iter)

    fdr[split] = multipletests(p[split].ravel(), method="fdr_bh")[1].reshape(p[split].shape)
    fdr[split] = pd.DataFrame(fdr[split], index=frameworks, columns=frameworks)
    
    print("-" * 65 + "\n" + split.upper() + "\n" + "-" * 65)
    print(fdr[split])
    print("")<jupyter_output>-----------------------------------------------------------------
DISCOVERY
-----------------------------------------------------------------
             data-driven   rdoc  dsm
data-driven          1.0  0.039  0.0
rdoc                 1.0  1.000  0.0
dsm                  1.0  1.000  1.0

-----------------------------------------------------------------
REPLICATION
-----------------------------------------------------------------
             data-driven   rdoc  dsm
data-driven          1.0  0.036  0.0
rdoc                 1.0  1.000  0.0
dsm                  1.0  1.000  1.0<jupyter_text>Plot evaluation metrics<jupyter_code>%matplotlib inline
for split in splits:
    utilities.plot_framework_comparison(boot[split], obs, mean[split], metric=metric, 
                                        n_iter=n_iter, suffix=suffix + "_" + split, 
                                        ylim=[-0.25,1], yticks=[-0.25,0,0.25,0.5,0.75,1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20955989507432235, 'fraction_numerical': 0.025065578548528127, 'mean_word_length': 2.394658753709199, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '84116', 'license': 'MIT', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1107, 'n_words': 226, 'path': 'prototype/comp_frameworks_nn.ipynb', 'repo_name': 'ehbeam/neuro-knowledge-engine', 'size': '3431'}