<jupyter_start><jupyter_text>IntroductionThis notebook compares the generalizability of the data-driven, RDoC, and DSM frameworks. Load the data<jupyter_code>import pandas as pd
import numpy as np
np.random.seed(42)

import sys
sys.path.append("..")
import utilities<jupyter_output><empty_output><jupyter_text>Input parameters<jupyter_code>metric = "proto" # Variable being compared between frameworks
n_iter = 1000 # Number of iterations for bootstrap distributions
frameworks = ["data-driven", "rdoc", "dsm"]
clfs = ["_lr", "", ""]
splits = ["discovery", "replication"]
suffix = "lr"<jupyter_output><empty_output><jupyter_text>Framework contents<jupyter_code>lists, circuits = {}, {}
for fw, clf in zip(frameworks, clfs):
    lists[fw], circuits[fw] = utilities.load_framework(fw, suffix=utilities.suffix[fw], clf=clf)<jupyter_output><empty_output><jupyter_text>Observed values<jupyter_code>obs = {}
for fw, clf in zip(frameworks, clfs):
    obs[fw] = pd.read_csv("data/{}_obs_{}{}.csv".format(metric, fw, clf), index_col=0, header=0)
obs["data-driven"].loc[obs["data-driven"]["SPLIT"] == "discovery"]["OBSERVED"].mean()
obs["data-driven"].loc[obs["data-driven"]["SPLIT"] == "replication"]["OBSERVED"].mean()<jupyter_output><empty_output><jupyter_text>Bootstrap distributions<jupyter_code>boot, mean = {}, {}
for split in splits:
    boot[split], mean[split] = {}, {}
    for fw, clf in zip(frameworks, clfs):
        boot[split][fw] = pd.read_csv("data/{}_boot_{}{}_{}_{}iter.csv".format(metric, fw, clf, split, n_iter), 
                                      index_col=0, header=0)
        mean[split][fw] = pd.read_csv("data/{}_mean_{}{}_{}.csv".format(metric, fw, clf, split), 
                                      index_col=0, header=0)<jupyter_output><empty_output><jupyter_text>Perform comparison testsIs the mean of fwi greater than the mean of fwj?<jupyter_code>from statsmodels.stats.multitest import multipletests
interval = 0.999
tail = (1.0 - interval) / 2.0 # Two-sided test
idx_lower = int(n_iter * tail)
idx_upper = int(n_iter * (1.0 - tail))

CI = {}
for split in splits:
    CI[split] = {}
    CI[split] = pd.DataFrame("", index=frameworks, columns=frameworks)
    for i, fw_i in enumerate(frameworks):
        for j, fw_j in enumerate(frameworks):
            boot_i = np.mean(boot[split][fw_i], axis=0)
            boot_j = np.mean(boot[split][fw_j], axis=0)
            boot_dif = np.sort(boot_i - boot_j)
            CI_lower = boot_dif[idx_lower]
            CI_upper = boot_dif[idx_upper]
            CI_sig = ""
            if (CI_lower > 0 and CI_upper > 0) or (CI_lower < 0 and CI_upper < 0):
                CI_sig = "*"
            CI[split].loc[fw_i,fw_j] = "{:.3f}, {:.3f} {}".format(CI_lower, CI_upper, CI_sig)
    
    print("-" * 65 + "\n" + split.upper() + "\n" + "-" * 65)
    print(CI[split])
    print("")<jupyter_output>-----------------------------------------------------------------
DISCOVERY
-----------------------------------------------------------------
                  data-driven              rdoc             dsm
data-driven     0.000, 0.000     -0.023, 0.050   0.055, 0.176 *
rdoc           -0.050, 0.023      0.000, 0.000   0.043, 0.157 *
dsm          -0.176, -0.055 *  -0.157, -0.043 *   0.000, 0.000 

-----------------------------------------------------------------
REPLICATION
-----------------------------------------------------------------
                  data-driven              rdoc             dsm
data-driven     0.000, 0.000     -0.029, 0.050   0.051, 0.179 *
rdoc           -0.050, 0.029      0.000, 0.000   0.012, 0.157 *
dsm          -0.179, -0.051 *  -0.157, -0.012 *   0.000, 0.000<jupyter_text>Plot evaluation metrics<jupyter_code>%matplotlib inline
for split in splits:
    utilities.plot_framework_comparison(boot[split], obs, mean[split], metric=metric, 
                                        n_iter=n_iter, suffix=suffix + "_" + split, 
                                        ylim=[0,0.5], yticks=[0,0.25,0.5])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2103825136612022, 'fraction_numerical': 0.04644808743169399, 'mean_word_length': 2.268668831168831, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '474978', 'license': 'MIT', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1347, 'n_words': 270, 'path': 'prototype/comp_frameworks_lr.ipynb', 'repo_name': 'ehbeam/neuro-knowledge-engine', 'size': '4026'}