<jupyter_start><jupyter_text>Current recall generates a null distribution for each given gene, while the previous recall method uses a single null distribution. Reanalyze the data with a single null distribution.<jupyter_code>import pickle
import pandas as pd
import glob
import re
from scipy.stats import spearmanr
import numpy as np
# get results
dir_Lx = '../out/20.0518 Lx/'
Lx = 300
model_name = 'topfeat'
df_results = pd.read_csv('%s/L%sonly_reg_rf_boruta/model_results.csv' % (dir_Lx, Lx))
df_results = df_results.loc[df_results.model == model_name,:].copy()
# get y_actual and y_pred, list of pickle file names
dir_pkl = '../out/20.0518 Lx/L%sonly_reg_rf_boruta/model_perf' % Lx
pkl_files = glob.glob('%s/y_compr_*.pkl' % dir_pkl)
y_compr_list = []
gene_names = []

for pkl_file in pkl_files:
    gene = re.findall('.*/y_compr_(.*).pkl', pkl_file)[0]
    y_compr = pickle.load(open(pkl_file, 'rb'))

    y_compr_list.append(y_compr['te'])
    gene_names.append(gene)
cell_lines = []

#null rho distribution
y_actual = pd.DataFrame([i['y_actual'] for i in y_compr_list],
                        #columns=cell_lines,
                        index=gene_names)
y_actual = y_actual.T
y_inf = pd.DataFrame([i['y_pred'] for i in y_compr_list],
                     #columns=cell_lines,
                     index=gene_names)
y_inf = y_inf.T

rho,pval = spearmanr(y_actual, y_inf, axis=0)

rho_actual = rho[0:y_actual.shape[1], y_actual.shape[1]:]
rho_actual = rho_actual[np.eye(rho_actual.shape[0],dtype=bool)].reshape(rho_actual.shape[0])
rho_actual = pd.DataFrame(rho_actual, index=y_inf.columns, columns=['rho_actual'])
rho_null = rho[0:y_actual.shape[1], y_actual.shape[1]:]
rho_null = rho_null[~np.eye(rho_null.shape[0],dtype=bool)].reshape(rho_null.shape[0],-1) #remove diagonal
rho_null = np.squeeze(rho_null.reshape(1,np.prod(rho_null.shape))) #squeeze into 1d array

#calculate recall
def calcRecall(val):
    return np.sum(val > rho_null) / len(rho_null)

df_results['corr_test_recall_1null'] = df_results['corr_test'].apply(calcRecall)
df_results.to_csv('tmp/model_results_L%s_reg_rf_boruta_%s.csv' % (Lx,model_name), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13535911602209943, 'fraction_numerical': 0.01427255985267035, 'mean_word_length': 4.872972972972973, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '598263', 'license': 'MIT', 'n_tokens_mistral': 867, 'n_tokens_neox': 838, 'n_words': 154, 'path': 'notebooks/06-recall_one_null_distr.ipynb', 'repo_name': 'pritchardlabatpsu/cga', 'size': '2172'}