<jupyter_start><jupyter_text>Multiple Hypothesis Testing Correction<jupyter_code>import pandas as pd
import numpy as np

%load_ext autoreload
%autoreload 2

import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
sns.set_theme(
    context="paper", 
    style="whitegrid", 
    font_scale=1.2,
    rc={'figure.figsize': (10, 10), 'figure.dpi': 300}
)

from common.paths import PLS_RESULTS, RIDGE_RESULTS, CROSS_PRED_RESULTS
from common.results import load_results

from multipy.fwer import bonferroni
from multipy.fdr import lsu<jupyter_output><empty_output><jupyter_text>Get data  Manual Input<jupyter_code># bin_all = [0.0003, 0.0003, 0.0003, 0.0003, 0.0100, 0.0643]
# bin_1 = [0.0953, 0.0313, 0.0593, 0.6485, 0.6048, 0.8287]
# bin_2 = [0.0003, 0.0247, 0.0003, 0.0020, 0.0107, 0.4159]
# bin_3 = [0.1789, 0.0653, 0.3196, 0.4172, 0.2779, 0.3162]
# pvals = np.array(bin_all + bin_1 + bin_2 + bin_3)
pvals = np.array([0.3214, 0.0140, 0.0200, 0.5309, 0.1497, 0.4950])
df = pd.DataFrame({'Features': ['Feature {}'.format(i) for i in range(1, len(pvals) + 1)], 'P-value':pvals})
display(df)<jupyter_output><empty_output><jupyter_text>Within-Prediction<jupyter_code>fn = 'ridge_pts_adhd_two_bins'
df, results_path = load_results(fn, RIDGE_RESULTS)
pvals = df['P-value']
display(df)<jupyter_output><empty_output><jupyter_text>Cross-Prediction<jupyter_code># fn = 'ridge_pts_age_cross_prediction'
fn = 'ridge_pts_age_cross_prediction_bin_two_equal_samples'
df, results_path = load_results(fn, CROSS_PRED_RESULTS)
pvals = df['P-value']
display(df)<jupyter_output><empty_output><jupyter_text>Run correction<jupyter_code>a_threshold = 0.05
# df['BFR'] = bonferroni(pvals, alpha=a_threshold)
df['FDR'] = lsu(pvals, q=a_threshold)
display(df.round(4))<jupyter_output><empty_output><jupyter_text>Save and update results file<jupyter_code>df.to_csv(results_path)<jupyter_output><empty_output><jupyter_text>Get FDR q<jupyter_code>m = len(pvals)
sort_ind = np.argsort(pvals)
qs = [(i+1.)*0.05/m for i, p in enumerate(pvals[sort_ind])]
a = pd.DataFrame([(p, q) for p, q in zip(pvals[sort_ind], qs)])
display(a)<jupyter_output><empty_output><jupyter_text>Max Statistic Method<jupyter_code>from common.binning import BIN_LABELS
from common.paths import RIDGE_PSCORES, PLS_PSCORES
from common.results import save_perm_score, load_perm_score
from common.wisc import WISC_LEVEL

model = 'ridge'
population = 'adhd'

for bin_label in BIN_LABELS:
    bin_perm_scores = []
    
    for target in WISC_LEVEL[5]:
        perm_score = load_perm_score(f'{model}_{population}_{target}_{bin_label}_perm_scores', RIDGE_PSCORES)
        bin_perm_scores.append(perm_score)
    
    bin_perm_scores = np.concatenate(bin_perm_scores)
    print(bin_label, bin_perm_scores.shape)
    save_perm_score(bin_perm_scores, f'{model}_{population}_{bin_label}_perm_scores', RIDGE_PSCORES)
from common.binning import BIN_LABELS
from common.paths import RIDGE_RESULTS, RIDGE_PSCORES, PLS_RESULTS, PLS_PSCORES
from common.results import load_results, load_perm_score
from common.wisc import WISC_LEVEL

results, results_path = load_results(f'{model}_pts_{population}', RIDGE_RESULTS)

for bin_label in BIN_LABELS:
    bin_perm_scores = load_perm_score(f'{model}_{population}_{bin_label}_perm_scores', RIDGE_PSCORES)
    ninety_fifth_index = int(bin_perm_scores.shape[0] * 0.95)
    ninety_fifth_score = np.sort(bin_perm_scores)[ninety_fifth_index]
    
    g = sns.displot(bin_perm_scores, color='lightgrey', element="step")
    g.fig.subplots_adjust(top=0.9)
    g.fig.suptitle(f'Model: {model}, Population: {population}, {bin_label}, Num Perm: {bin_perm_scores.shape[0]}')
    g.ax.axvline(ninety_fifth_score, label='95th', color='black')
    
    for target_idx, target in enumerate(WISC_LEVEL[5]):
        target_score = results.loc[(results['Bin'] == bin_label) & (results['Target'] == target)]['Score']
        target_score = target_score.to_list()[0]
        g.ax.axvline(target_score, label=target, ls='--', color=sns.color_palette("husl", 6)[target_idx])
    g.fig.legend()
        
# Discrete variant
# g = sns.displot(bin_perm_scores, palette="Set2", element="step")
# g.fig.subplots_adjust(top=0.85)
# g.fig.suptitle(f'Model: ridge, Target: {selected_target}, Train: Self, Test: Self, Population: ADHD, Num Perm: {N_PERM}')
# for col_val, ax in g.axes_dict.items():
#     bin_idx = col_val - 1
#     ax.axvline(true_score['Score'].iloc[bin_idx], ls="--", color=sns.color_palette("Set2")[bin_idx])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1384820239680426, 'fraction_numerical': 0.043275632490013316, 'mean_word_length': 6.199680511182109, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '133359', 'license': 'MIT', 'n_tokens_mistral': 1949, 'n_tokens_neox': 1771, 'n_words': 305, 'path': 'src/Models/Analysis/Multiple Hypothesis Testing Correction.ipynb', 'repo_name': 'Brian-Pho/MSc_Research_Project', 'size': '4506'}