<jupyter_start><jupyter_text>Load the Prediction Model to find the omegas<jupyter_code>from gene_finding.models import load_model, EnsModel
import torch
models = []

for i in range(1, 11):
    mod = load_model(seed=i,drug=drug,n_genes=len(dataset.genes))
    mod.eval()
    models.append(mod)
    
ens = EnsModel(models)
from gene_finding.precalculate_omega import precalculate_omegas
loss = torch.nn.MSELoss(reduction='none')
omegas,_ = precalculate_omegas(ens, gdsc_expr, gdsc_dr, pathway_matrix, loss, mode='none', return_sum_one=False)
omega_dist = pd.DataFrame(omegas/omegas.sum(axis=-1, keepdims=True), gdsc_expr.index, columns=pathway_names)
i = 200
plt.hist([attr_gdsc.iloc[i].values, omega_dist.iloc[i].values], label=['out', 'omega'], bins=20)
plt.legend()
plt.show()
attr_gdsc.iloc[1].sum()
exp = explainers['cisplatin']
k = [exp._predict_single(exp.prediction_model[0], np.asarray(gdsc_expr.iloc[1].values).reshape((1,-1))),
     exp._predict_single(exp.prediction_model[1], np.asarray(gdsc_expr.iloc[1].values).reshape((1,-1))),
     exp._predict_single(exp.prediction_model[2], np.asarray(gdsc_expr.iloc[1].values).reshape((1,-1)))]
k = np.concatenate(k)
np.median(k, axis=0).sum()
y = omega_dist.mean(axis=0)
plt.scatter(path_size[y.index], y.values)
plt.ylabel(r'mean $\omega(P)$ across CLs')
plt.xlabel(r'$|P|$')
path_size[omega_dist.columns].values.shape
omegas/(path_size[omega_dist.columns].values)#.reshape(-1,1)
y = omegas/(path_size[omega_dist.columns].values)
plt.scatter(path_size[omegas.columns], y.mean(axis=0))
plt.ylabel(r'mean $\omega(P)/|P|$ across CLs')
plt.xlabel(r'$|P|$')
sorted_baseline = diff[drug].mean(axis=0).sort_values(ascending=False)
plt.scatter(path_size[sorted_baseline.index], sorted_baseline.values)
plt.ylabel(r'mean baseline across CLs')
plt.xlabel(r'$|P|$')
sorted_baseline = diff[drug].median(axis=0).sort_values(ascending=False)
plt.scatter(path_size[sorted_baseline.index], sorted_baseline.values)
plt.ylabel(r'median baseline across CLs')
plt.xlabel(r'$|P|$')
diff[drug].shape
omegas.shape
plt.scatter(diff[drug].iloc[0], omega_dist.iloc[0])
plt.xlabel('baseline')
plt.ylabel('unnormalized omega')
kl = torch.nn.KLDivLoss()
w = torch.FloatTensor(omega_dist.values)
a = torch.FloatTensor(attr_gdsc.values)
kl(a.log(), w)
k = [exp._predict_single(exp.prediction_model[0], np.asarray(gdsc_expr.values)),
     exp._predict_single(exp.prediction_model[1], np.asarray(gdsc_expr.values)),
     exp._predict_single(exp.prediction_model[2], np.asarray(gdsc_expr.values))]
a = torch.FloatTensor(k[0])
kl(a.log(), w)
a = torch.FloatTensor(k[1])
kl(a.log(),w)
a = torch.FloatTensor(k[2])
kl(a.log(), w)
omegas = pd.DataFrame(omegas, index=gdsc_expr.index, columns=pathway_names)
omegas.index = [int(x) for x in omegas.index]
diff_omega = np.maximum(omegas-diff[drug], np.ones(omegas.shape)*1e-7)
diff_omega = diff_omega/diff_omega.values.sum(axis=-1, keepdims=True)
y = diff_omega.mean(axis=0).sort_values(ascending=False)
plt.scatter(path_size[y.index], y.values)
plt.ylabel(r'mean difference score across CLs')
plt.xlabel(r'$|P|$')
unnorm_omega = omega_dist.mean(axis=0)
scaled_omega = (omega_dist/path_size[omega_dist.columns]).mean(axis=0)
scaled_omega = scaled_omega/scaled_omega.values.sum(axis=-1, keepdims=True)
y = diff_omega.mean(axis=0)

plt.scatter(path_size[unnorm_omega.index], unnorm_omega.values, label='unnormalized omega')
plt.scatter(path_size[y.index], y.values, label='difference_score')
plt.scatter(path_size[scaled_omega.index], scaled_omega.values, label='scaled omega')
# plt.ylabel(r'mean difference score across CLs')
plt.legend()
plt.xlabel(r'$|P|$')
plt.figure(figsize=(10,10))

sample = 100
unnorm_omega = omega_dist.iloc[sample]
scaled_omega = (omegas.iloc[sample]/path_size[omegas.columns])
scaled_omega = scaled_omega/scaled_omega.sum()
y = diff_omega.iloc[sample]
bl = diff[drug].iloc[sample]

# y = np.maximum(omegas.iloc[sample]-bl, np.ones(len(bl))*1e-7)
# y = y/y.sum()

plt.scatter(path_size[unnorm_omega.index], unnorm_omega.values, alpha=0.7, label='unnormalized omega')
plt.scatter(path_size[y.index], y.values, alpha=0.7, label='difference_score', marker='^')
plt.scatter(path_size[scaled_omega.index], scaled_omega.values, alpha=0.7, label='scaled omega')
plt.scatter(path_size[bl.index], bl.values, alpha=0.7, label='baseline', marker='s')
# plt.ylabel(r'mean difference score across CLs')
plt.legend()
plt.xlabel(r'$|P|$')
plt.figure(figsize=(10,10))

sample = 100
unnorm_omega = omegas.iloc[sample]
scaled_omega = (omegas.iloc[sample]/path_size[omegas.columns])*path_size.mean()

# y = diff_omega.iloc[sample]
bl = diff[drug].iloc[sample]

y = np.maximum(omegas.iloc[sample]-bl, np.ones(len(bl))*1e-7)

top = (omegas.iloc[sample]-bl).nlargest(4)
print(top, path_size[top.index])

plt.scatter(path_size[unnorm_omega.index], unnorm_omega.values, alpha=0.7, label=r'$\Delta \varepsilon(X,P)$')
plt.scatter(path_size[y.index], y.values, alpha=0.7, label='difference_score', marker='^')
plt.scatter(path_size[scaled_omega.index], scaled_omega.values, alpha=0.7, label='scaled omega', marker='*')
plt.scatter(path_size[bl.index], bl.values, alpha=0.7, label='baseline', marker='s')
# plt.ylabel(r'mean difference score across CLs')
plt.legend(loc='upper left')
plt.xlabel(r'$|P|$')
omega_dist.iloc[0].sum()
attr_gdsc.shape
omega_dist.shape
def get_pathway_rankings(folder):
    paths = {}
    for drug in drugs:
        x = pd.read_csv(root_dir + folder + '/' + drug + '/pathways.csv')
        paths[drug] = x
        
    return paths
rank_diff = get_pathway_rankings('CX_ens_enrichr3_diff')
fig, axes = plt.subplots(3, 2, figsize=(14, 15))

drug = 'bleomycin'
top6 = rank_diff[drug].loc[range(6)]['mean'].values
for i in range(6):
    ax = axes[i%3][i//3]
    ax.hist(diff[drug][top6[i]], bins=20)
    ax.set_title(top6[i][:50])
    ax.axvline(diff[drug][top6[i]].mean(), color='r', label='mean')
    ax.axvline(diff[drug][top6[i]].median(), color='orange', label='median')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18183333333333335, 'fraction_numerical': 0.016833333333333332, 'mean_word_length': 6.980053191489362, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '543806', 'license': 'MIT', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2368, 'n_words': 331, 'path': 'Baselines.ipynb', 'repo_name': 'ddhostallero/cxplain', 'size': '6000'}