<jupyter_start><jupyter_text>Significance<jupyter_code>comp_mut_df = pd.DataFrame(columns=["observed mutation count"])
for i, r in assoc_muts_df.iterrows():
    for f, links in r["pathway links"].items():
        if f not in comp_mut_df.index:
            df = pd.DataFrame({"observed mutation count": len(links)}, index=[f])
            comp_mut_df = comp_mut_df.append(df)
        else:
            comp_mut_df.loc[f, "observed mutation count"] += len(links)

comp_mut_df
for _, r in assoc_muts_df.iterrows():
    for feat_d in r["pathways"]:
        feat_d["mutation set count"] = comp_mut_df.loc[feat_d["name"]]["observed mutation count"]
pathway_len_df = pd.read_pickle("./data/pathway_len_df.pkl")
pathway_len_df.head()
import numpy as np

if len(comp_mut_df):
    comp_mut_df["length"] = comp_mut_df.apply(lambda r: pathway_len_df.loc[r.index, "pathway length"])
    total_seq_len = np.sum(comp_mut_df["length"])
    comp_mut_df["proportion"] = comp_mut_df["length"].apply(lambda comp_len: comp_len/total_seq_len)
    comp_mut_df
from collections import Counter

if len(comp_mut_df):
    comp_mut_df['equal or larger count'] = 0
    component_l = comp_mut_df.index.tolist()
    proportion_l = comp_mut_df["proportion"].tolist()
    num_muts = np.sum(comp_mut_df["observed mutation count"])

    # slides from UW speak about the relationship of permutations and precision.
    # should read through them to get a sense of what is the least iterations that should execute.
    num_itr = 10000
    for i in range(0, num_itr):
        random_mut_comp_a = np.random.choice(
            a=component_l,
            size=int(num_muts),
            p=proportion_l,
            replace=True,
        )
        random_mut_comp_count_d = Counter(random_mut_comp_a)
        for component in random_mut_comp_count_d.keys():
            if random_mut_comp_count_d[component] >= comp_mut_df.loc[component, 'observed mutation count']:
                comp_mut_df.loc[component, 'equal or larger count'] += 1

    comp_mut_df["p value"] = comp_mut_df["equal or larger count"].apply(
        lambda count: count/num_itr)

    comp_mut_df["significant"] = comp_mut_df["p value"].apply(lambda p_val: True if p_val < ENRICH_ALPHA else False)

    display(comp_mut_df.head())
from statsmodels.stats import multitest


if len(comp_mut_df):
    pvals = comp_mut_df["p value"]
    rejects, pvals_corrected, alphacSidak, alphacBonf = multitest.multipletests(
        pvals=pvals,
        alpha=ENRICH_ALPHA,
        method=MULTI_HYP_CORR_METHOD)

    pd.options.display.max_rows = 500
    comp_mut_df["corrected p value"] = pvals_corrected
    comp_mut_df["corrected significance"] = rejects
    comp_mut_df.head()
# # not wanting to consider mutation events with only 1 features as being significant
if len(comp_mut_df):
    comp_mut_df["final significance"] = comp_mut_df.apply(lambda r: r["corrected significance"] if r["observed mutation count"] > 1 else False, axis=1)
# comp_mut_df.head()
if len(comp_mut_df):
    for i, r in assoc_muts_df.iterrows():
        for d in r["pathways"]:
            if d["name"] in comp_mut_df.index:
                d["significantly enriched"] = comp_mut_df.loc[d["name"]]["final significance"]
assert(len(assoc_muts_df.exp.unique())==1)
exp = assoc_muts_df.exp.unique()[0]
comp_mut_df.to_csv("./data/supp/permutation_test_results_per_experiment/"+exp+"_mut_pwy_signif.csv")
assoc_muts_df.to_pickle("./data/12_df.pkl")
# import seaborn as sns
# import matplotlib
# import matplotlib.pyplot as plt
# %matplotlib inline
# plt.rcParams["figure.dpi"] = 100
# sns.set_context("talk")
# sns.set_style("ticks")


# def _get_name_signif_from_name(name, json):
#     is_signif = False
#     for d in json:
#         if d["name"] == name:
#             is_signif = d["significant"]
#             break
#     return is_signif


# # The reason we're using the links dictionary is because we want to count how genomic features were mutated for the genetic target.
# targ_cnt_d = dict()
# for _, r in assoc_muts_df.iterrows():
#     link_d = r["pathway links"]
#     for targ_name, hits in link_d.items():
#         is_signif = _get_name_signif_from_name(targ_name, r["pathways"])
#         t = str(targ_name)
#         if is_signif:
#             t = '*' + t
#         if t not in targ_cnt_d.keys():
#             targ_cnt_d[t] = 0
#         targ_cnt_d[t] += len(hits)
        
# df = pd.DataFrame.from_dict(targ_cnt_d, orient='index', columns=["mutated feature count"])
# df = df.sort_values("mutated feature count")
# df = df[-20:]
# ax = df.plot.barh(y='mutated feature count',
# #                   figsize=(10,25),
#                   figsize=(5,10),
#                   color="#4C72B0",
#                   width=1
#                  )
# for i, v in enumerate(df["mutated feature count"]):
#     ax.text(v, i - 0.2, str(v))
# sns.despine(ax=ax, top=True, right=True)
# ax.set_title("Top 20 mutated pathways")
# ax.get_legend().remove()
# import seaborn as sns
# import matplotlib
# import matplotlib.pyplot as plt
# %matplotlib inline
# plt.rcParams["figure.dpi"] = 100
# sns.set_context("talk")
# sns.set_style("ticks")


# def _get_name_signif_from_name(name, json):
#     is_signif = False
#     for d in json:
#         if d["name"] == name:
#             is_signif = d["significant"]
#             break
#     return is_signif


# # The reason we're using the links dictionary is because we want to count how genomic features were mutated for the genetic target.
# targ_cnt_d = dict()
# for _, r in assoc_muts_df.iterrows():
#     link_d = r["pathway links"]
#     for targ_name, hits in link_d.items():
#         is_signif = _get_name_signif_from_name(targ_name, r["pathways"])
#         t = str(targ_name)
#         if is_signif:
#             t = '*' + t
#             if t not in targ_cnt_d.keys():
#                 targ_cnt_d[t] = 0
#             targ_cnt_d[t] += len(hits)
            
# if len(targ_cnt_d) > 0:
#     df = pd.DataFrame.from_dict(targ_cnt_d, orient='index', columns=["mutated feature count"])
#     df = df.sort_values("mutated feature count")
#     df = df[-20:]
#     display(df)
#     ax = df.plot.barh(y='mutated feature count',
#     #                   figsize=(10,25),
#                       figsize=(5,10),
#                       color="#4C72B0",
#                       width=1
#                      )
#     for i, v in enumerate(df["mutated feature count"]):
#         ax.text(v, i - 0.2, str(v))
#     sns.despine(ax=ax, top=True, right=True)
#     ax.set_title("Significant mutated pathways")
#     ax.get_legend().remove()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1271571298819255, 'fraction_numerical': 0.009234029669996972, 'mean_word_length': 3.023751522533496, 'pattern_counts': {'":': 1, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '288743', 'license': 'MIT', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2288, 'n_words': 505, 'path': '12_pathway_convergence.ipynb', 'repo_name': 'Aletechdev/ava', 'size': '6606'}