<jupyter_start><jupyter_text>Section 3.2.2 Meta-analysis on consistently differentially expressed genes across diseases<jupyter_code>import getpass
import json
import os
import sys
import time

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pyobo
import seaborn as sns
from tqdm import tqdm
getpass.getuser()
sys.version
time.asctime()<jupyter_output><empty_output><jupyter_text>Increase figure quality<jupyter_code>mpl.rcParams['figure.dpi'] = 320<jupyter_output><empty_output><jupyter_text>Load data download data from https://doi.org/10.5281/zenodo.4700652 and place the folder 'data/' in the root of the repository, or if placed elsewhere, write the full path to 'data/' as the variable data_dir below optional: select your desired location of the output figures<jupyter_code># replace here the location of the data dir if not structured as instructed
data_dir = "../data"

# optional, replace here the desired location of the output figures
figures_dir = os.path.join(os.path.expanduser("~"), "coexpath_figures")
os.makedirs(figures_dir, exist_ok=True)<jupyter_output><empty_output><jupyter_text>Load cluster assignments<jupyter_code>with open(os.path.join(data_dir, "misc_data", 'doid_group_clusters_dict.json'), 'r') as f:
    doids_per_cluster = json.load(f)

doid_cluster_mapping = {doid : cluster for cluster, doid_list in doids_per_cluster.items() if cluster != "Normal" for doid in doid_list}<jupyter_output><empty_output><jupyter_text>Load disease name mappings<jupyter_code>doid_name_mapping = pyobo.get_id_name_mapping('doid')
doid_id_mapping = pyobo.get_name_id_mapping('doid')<jupyter_output><empty_output><jupyter_text>Load differentially expressed genes<jupyter_code>DEgenes_dir = os.path.join(data_dir, "DEgenes")

DEgene_dict = {
    doid : pd.read_table(os.path.join(DEgenes_dir, doid, "DEgenes.tsv"),index_col="HGNC",usecols=["HGNC","adj.P.Val","logFC"])
    for doid in tqdm(os.listdir(DEgenes_dir),desc="loading fold gene data")
}<jupyter_output>loading fold gene data: 100%|██████████| 46/46 [00:00<00:00, 64.69it/s]<jupyter_text>Prepare data  Create consensus for up- and down-regulated diseases<jupyter_code>DE_up_down = {doid: {"up": None, "down": None} for doid in DEgene_dict}
for doid, df in DEgene_dict.items():
    DE_up_down[doid]["up"] = df.loc[df["logFC"]>0]
    DE_up_down[doid]["down"] = df.loc[df["logFC"]<0]
avg_up_df = pd.concat([up_down["up"] for up_down in DE_up_down.values()]).groupby(level=0).mean()
avg_down_df = pd.concat([up_down["down"] for up_down in DE_up_down.values()]).groupby(level=0).mean()
combined_avg_df = pd.concat([avg_up_df,avg_down_df])
combined_avg_df = combined_avg_df.reset_index()<jupyter_output><empty_output><jupyter_text>analyze cutoffs, label each row with final groupings<jupyter_code>def prep_df_for_consensus(df, fc_cutoff):
    """prepare combined df for plotting, adds new column needed for color-labelling
    
    paramaters
    ----------
    df : pd.DataFrame
        combined dataframe with averaged values
    fc_cutoff: float
        absoluted value of chosen fold change cutoff for those most significantly differentially expressed to be highlighted
    """
    df["-log10(adj. p-value)"] = (-1) * np.log10(df["adj.P.Val"].values)
    for row, fc, p in zip(df.index, df["logFC"], df["adj.P.Val"]):
        if p < 0.01:
            if fc < 0:
                if fc < -fc_cutoff:
                    df.at[row, "Legend"] = f"FC<-{fc_cutoff}"
                else:
                    df.at[row, "Legend"] = "Down"
            else:
                if fc > fc_cutoff:
                    df.at[row, "Legend"] = f"FC>{fc_cutoff}"
                else:
                    df.at[row, "Legend"] = "Up"
        else:
            df.at[row, "Legend"] = "adj. p-val > 0.01"
    df = df.drop("adj.P.Val", axis=1)
    df = df.rename(columns={'logFC': 'log2(fold change)'})
    return df
consensus_df = prep_df_for_consensus(combined_avg_df, 1.75)<jupyter_output><empty_output><jupyter_text>Figure: Consensus for consistently differentially expressed genes<jupyter_code>sns.set_theme(style="whitegrid")

def volcano(df, fc_cutoff: float, text_location: float):
    """plots consensus for consistently differentially expressed genes
    
    paramaters
    ----------
    df : pd.DataFrame
        dataframe created from prep_df_for_consensus function
    fc_cutoff: float
        absoluted value of chosen fold change cutoff for those most significantly differentially expressed to be highlighted
    text_location: float
        x-axis value of where the text for adj. p-value cutoff line to be shown, depends on df values."""
    f, ax = plt.subplots(figsize=(15, 15))
    plt.rc('axes', labelsize=18)
    p = sns.scatterplot(x="log2(fold change)", y="-log10(adj. p-value)", 
                    hue="Legend", hue_order=[f"FC<-{fc_cutoff}","Down","adj. p-val > 0.01","Up",f"FC>{fc_cutoff}"], palette="coolwarm",
                    linewidth=0,
                    data=df, ax=ax)
    plt.title("Consistently differentially expressed genes in diseases vs normal", fontsize=25, weight="bold")
    p.axhline((-1) * np.log10(.01), color = "red", ls= "--")
    p.text(text_location,(-1) * np.log10(.01), "adj. p-value = 0.01", color="red",
            ha="left", va="center", fontsize = 15)
    plt.legend(fontsize=18,title_fontsize=25)
    
    return p
consensus_plot = volcano(consensus_df, 1.75, 3.35)
plt.savefig(os.path.join(figures_dir, "consistently_DEgenes.png"), bbox_inches='tight',dpi=320)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1320381363554596, 'fraction_numerical': 0.01834862385321101, 'mean_word_length': 4.110294117647059, 'pattern_counts': {'":': 2, '<': 43, '<?xml version=': 0, '>': 42, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '283143', 'license': 'Apache-2.0', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1896, 'n_words': 441, 'path': 'analysis/3.2.2_analysis.ipynb', 'repo_name': 'CoXPath/CoXPath', 'size': '5559'}