<jupyter_start><jupyter_text>Enrichment Analysis VisualizerThis appyter creates a variety of visualizations for enrichment analysis data for one selected Enrichr library. The scatter plot and hexagonal canvas may not be available for every library.Here are the visualizations created:  * **Scatter Plot** The scatterplot is organized so that simliar gene sets are clustered together. Enriched gene sets will appear blue instead of gray; the darker the blue the smaller the p-value. The name of the gene set and the associated p-value are displayed when a point is hovered over.Plots can be downloaded as an svg using the save function on the toolbar next to the plot.* **Bar Chart**The bar chart contains the top 10 enriched terms and their corresponding p-values for the chosen library. Colored bars correspond to terms with significant p-values (<0.05). An asterisk (*) next to a p-value indicates the term also has a significant adjusted p-value (<0.05).* **Hexagonal Canvas**The hexagonal canvas is made up of hexagons, each of which represents one gene set from the library selected. The hexagons are colored based on the Jaccard similarity index between the inputted gene list and the gene set the hexagon represents (the brighter, the more similar). The hexagons representing the most similar gene sets are grouped together. The name of the gene set and the associated similarity index are displayed when a hexagon is hovered over.* **Manhattan Plot**A manhattan plot is created displaying all the gene sets from the select library and their p-value. The x-axis of the plot is made up of gene sets from the library. The y-axis of the plot has the -log(p value) for each gene set. The name of the gene set and the associated p-value are displayed when a point is hovered over. You can also zoom, pan, and save the plot as an svg using the toolbar on the right.Additionally, the names and p-values of significant terms in the library are provided in a table output below the plot which can be downloaded. There is a link at the bottom to the full analysis results on the Enrichr website.<jupyter_code># Scatter Plot Imports
from maayanlab_bioinformatics.enrichment import enrich_crisp
import matplotlib as mpl
import matplotlib.colors as colors
import base64

# Bar Chart Imports
import pandas as pd 
import numpy as np
import json
import requests
import matplotlib.pyplot as plt
import seaborn as sns
import time
from matplotlib.ticker import MaxNLocator
from IPython.display import display, FileLink, Markdown, HTML

# Hexagonal Canvas Imports
import json
import math
import uuid
import urllib
from textwrap import dedent
from string import Template
from operator import itemgetter

# Manhattan Plot Imports
import matplotlib.patches as mpatches
import matplotlib.cm as cm

# Bokeh
from bokeh.io import output_notebook
from bokeh.plotting import figure, show
from bokeh.models import HoverTool, CustomJS, ColumnDataSource, Span
from bokeh.layouts import layout, row, column, gridplot
from bokeh.palettes import all_palettes
output_notebook()
%%appyter hide_code

{% do SectionField(name='section1', title = '1. Submit Your Gene List', subtitle = 'Upload a text file containing your gene list -OR- copy and paste your gene list into the text box below (One gene per row). You can also try the default gene list provided.', img = 'enrichr-icon.png')%}
{% do SectionField(name='section2', title = '2. Choose Enrichr Library', subtitle = 'Select one Enrichr library.', img = 'enrichr-icon.png')%}
%%appyter code_eval

gene_list_filename = {{ FileField(name='gene_list_filename', label='Gene List File', default='', description='Upload your gene list as a text file (One gene per row).',section = 'section1') }}

gene_list_input = {{ TextField(name='gene_list_input', label='Gene List', default='NSUN3\nPOLRMT\nNLRX1\nSFXN5\nZC3H12C\nSLC25A39\nARSG\nDEFB29\nNDUFB6\nZFAND1\nTMEM77\n5730403B10RIK\nRP23-195K8.6\nTLCD1\nPSMC6\nSLC30A6\nLOC100047292\nLRRC40\nORC5L\nMPP7\nUNC119B\nPRKACA\nTCN2\nPSMC3IP\nPCMTD2\nACAA1A\nLRRC1\n2810432D09RIK\nSEPHS2\nSAC3D1\nTMLHE\nLOC623451\nTSR2\nPLEKHA7\nGYS2\nARHGEF12\nHIBCH\nLYRM2\nZBTB44\nENTPD5\nRAB11FIP2\nLIPT1\nINTU\nANXA13\nKLF12\nSAT2\nGAL3ST2\nVAMP8\nFKBPL\nAQP11\nTRAP1\<KEY>TM7SF<KEY>3\nKDR\nZFP748\nNAP1L1\nDHRS1\nLRRC56\nWDR20A\nSTXBP2\nKLF1\nUFC1\nCCDC16\n9230114K14RIK\nRWDD3\n2610528K11RIK\nACO1\nCABLES1\nLOC100047214\nYARS2\nLYPLA1\nKALRN\nGYK\nZFP787\nZFP655\nRABEPK\nZFP650\n4732466D17RIK\nEXOSC4\nWDR42A\nGPHN\n2610528J11RIK\n1110003E01RIK\nMDH1\n1200014M14RIK\nAW209491\nMUT\n1700123L14RIK\n2610036D13RIK\nCOX15\nTMEM30A\nNSMCE4A\nTM2D2\nRHBDD3\nATXN2\nNFS1\n3110001I20RIK\nBC038156\nLOC100047782\n2410012H22RIK\nRILP\nA230062G08RIK\nPTTG1IP\nRAB1\nAFAP1L1\nLYRM5\n2310026E23RIK\nC330002I19RIK\nZFYVE20\nPOLI\nTOMM70A\nSLC7A6OS\nMAT2B\n4932438A13RIK\nLRRC8A\nSMO\nNUPL2\nTRPC2\nARSK\nD630023B12RIK\nMTFR1\n5730414N17RIK\nSCP2\nZRSR1\nNOL7\nC330018D20RIK\nIFT122\nLOC100046168\nD730039F16RIK\nSCYL1\n1700023B02RIK\n1700034H14RIK\nFBXO8\nPAIP1\nTMEM186\nATPAF1\nLOC100046254\nLOC100047604\nCOQ10A\nFN3K\nSIPA1L1\nSLC25A16\nSLC25A40\nRPS6KA5\nTRIM37\nLRRC61\nABHD3\nGBE1\nPARP16\nHSD3B2\nESM1\nDNAJC18\nDOLPP1\nLASS2\nWDR34\nRFESD\nCACNB4\n2310042D19RIK\nSRR\nBPNT1\n6530415H11RIK\nCLCC1\nTFB1M\n4632404H12RIK\nD4BWG0951E\nMED14\nADHFE1\nTHTPA\nCAT\nELL3\nAKR7A5\nMTMR14\nTIMM44\nSF1\nIPP\nIAH1\nTRIM23\nWDR89\nGSTZ1\nCRADD\n2510006D16RIK\nFBXL6\nLOC100044400\nZFP106\nCD55\n0610013E23RIK\nAFMID\nTMEM86A\nALDH6A1\nDALRD3\nSMYD4\nNME7\nFARS2\nTASP1\nCLDN10\nA930005H10RIK\nSLC9A6\nADK\nRBKS\n2210016F16RIK\nVWCE\n4732435N03RIK\nZFP11\nVLDLR\n9630013D21RIK\n4933407N01RIK\nFAHD1\nMIPOL1\n1810019D21RIK\n1810049H13RIK\nTFAM\nPAICS\n1110032A03RIK\nLOC100044139\nDNAJC19\nBC016495\nA930041I02RIK\nRQCD1\nUSP34\nZCCHC3\nH2AFJ\nPHF7\n4921508D12RIK\nKMO\nPRPF18\nMCAT\nTXNDC4\n4921530L18RIK\nVPS13B\nSCRN3\nTOR1A\nAI316807\nACBD4\nFAH\nAPOOL\nCOL4A4\nLRRC19\nGNMT\nNR3C1\nSIP1\nASCC1\nFECH\nABHD14A\nARHGAP18\n2700046G09RIK\nYME1L1\nGK5\nGLO1\nSBK1\nCISD1\n2210011C24RIK\nNXT2\nNOTUM\nANKRD42\nUBE2E1\nNDUFV1\nSLC33A1\nCEP68\nRPS6KB1\nHYI\nALDH1A3\nMYNN\n3110048L19RIK\nRDH14\nPROZ\nGORASP1\nLOC674449\nZFP775\n5430437P03RIK\nNPY\nADH5\nSYBL1\n4930432O21RIK\nNAT9\nLOC100048387\nMETTL8\nENY2\n2410018G20RIK\nPGM2\nFGFR4\nMOBKL2B\nATAD3A\n4932432K03RIK\nDHTKD1\nUBOX5\nA530050D06RIK\nZDHHC5\nMGAT1\nNUDT6\nTPMT\nWBSCR18\nLOC100041586\nCDK5RAP1\n4833426J09RIK\nMYO6\nCPT1A\nGADD45GIP1\nTMBIM4\n2010309E21RIK\nASB9\n2610019F03RIK\n7530414M10RIK\nATP6V1B2\n2310068J16RIK\nDDT\nKLHDC4\nHPN\nLIFR\nOVOL1\nNUDT12\nCDAN1\nFBXO9\nFBXL3\nHOXA7\nALDH8A1\n3110057O12RIK\nABHD11\nPSMB1\nENSMUSG00000074286\nCHPT1\nOXSM\n2310009A05RIK\n1700001L05RIK\nZFP148\n39509\nMRPL9\nTMEM80\n9030420J04RIK\nNAGLU\nPLSCR2\nAGBL3\nPEX1\nCNO\nNEO1\nASF1A\nTNFSF5IP1\nPKIG\nAI931714\nD130020L05RIK\nCNTD1\nCLEC2H\nZKSCAN1\n1810044D09RIK\nMETTL7A\nSIAE\nFBXO3\nFZD5\nTMEM166\nTMED4\nGPR155\nRNF167\nSPTLC1\nRIOK2\nTGDS\nPMS1\nPITPNC1\nPCSK7\n4933403G14RIK\nEI24\nCREBL2\nTLN1\nMRPL35\n2700038C09RIK\nUBIE\nOSGEPL1\n2410166I05RIK\nWDR24\nAP4S1\nLRRC44\nB3BP\nITFG1\nDMXL1\nC1D', description='Paste your gene list (One gene per row).', section = 'section1') }}

enrichr_library = '{{ ChoiceField(name='enrichr_library', description='Select one Enrichr library for which to create visualizations.', label='Enrichr Library', default='WikiPathways_2019_Human', section = 'section2',choices=[
    'ARCHS4_TFs_Coexp',
    'ChEA_2016',
    'ENCODE_and_ChEA_Consensus_TFs_from_ChIP-X',
    'ENCODE_Histone_Modifications_2015',
    'ENCODE_TF_ChIP-seq_2015',
    'Epigenomics_Roadmap_HM_ChIP-seq',
    'Enrichr_Submissions_TF-Gene_Coocurrence',
    'Genome_Browser_PWMs',
    'lncHUB_lncRNA_Co-Expression',
    'miRTarBase_2017',
    'TargetScan_microRNA_2017',
    'TF-LOF_Expression_from_GEO',
    'TF_Perturbations_Followed_by_Expression',
    'Transcription_Factor_PPIs',
    'TRANSFAC_and_JASPAR_PWMs',
    'TRRUST_Transcription_Factors_2019',
    'ARCHS4_Kinases_Coexp',
    'BioCarta_2016',
    'BioPlanet_2019',
    'BioPlex_2017',
    'CORUM',
    'Elsevier_Pathway_Collection',
    'HMS_LINCS_KinomeScan',
    'HumanCyc_2016',
    'huMAP',
    'KEA_2015',
    'KEGG_2019_Human',
    'KEGG_2019_Mouse',
    'Kinase_Perturbations_from_GEO_down',
    'Kinase_Perturbations_from_GEO_up',
    'L1000_Kinase_and_GPCR_Perturbations_down',
    'L1000_Kinase_and_GPCR_Perturbations_up',
    'NCI-Nature_2016',
    'NURSA_Human_Endogenous_Complexome',
    'Panther_2016',
    'Phosphatase_Substrates_from_DEPOD',
    'PPI_Hub_Proteins',
    'Reactome_2016',
    'SILAC_Phosphoproteomics',
    'SubCell_BarCode',
    'Virus-Host_PPI_P-HIPSTer_2020',
    'WikiPathways_2019_Human',
    'WikiPathways_2019_Mouse',
    'GO_Biological_Process_2018',
    'GO_Cellular_Component_2018',
    'GO_Molecular_Function_2018',
    'Human_Phenotype_Ontology',
    'Jensen_COMPARTMENTS',
    'Jensen_DISEASES',
    'Jensen_TISSUES',
    'MGI_Mammalian_Phenotype_Level_4_2019',  
    'Achilles_fitness_decrease',
    'Achilles_fitness_increase',
    'ARCHS4_IDG_Coexp',
    'ClinVar_2019',
    'dbGaP',
    'DepMap_WG_CRISPR_Screens_Broad_CellLines_2019',
    'DepMap_WG_CRISPR_Screens_Sanger_CellLines_2019',
    'DisGeNET',
    'DrugMatrix',
    'DSigDB',
    'GeneSigDB',
    'GWAS_Catalog_2019',
    'LINCS_L1000_Chem_Pert_down',
    'LINCS_L1000_Chem_Pert_up',
    'LINCS_L1000_Ligand_Perturbations_down',
    'LINCS_L1000_Ligand_Perturbations_up',
    'MSigDB_Computational',
    'MSigDB_Oncogenic_Signatures',
    'Old_CMAP_down',
    'Old_CMAP_up',
    'OMIM_Disease',
    'OMIM_Expanded',
    'PheWeb_2019',
    'Rare_Diseases_AutoRIF_ARCHS4_Predictions',
    'Rare_Diseases_AutoRIF_Gene_Lists',
    'Rare_Diseases_GeneRIF_ARCHS4_Predictions',
    'Rare_Diseases_GeneRIF_Gene_Lists',
    'UK_Biobank_GWAS_v1',
    'Virus_Perturbations_from_GEO_down',
    'Virus_Perturbations_from_GEO_up',
    'VirusMINT',       
    'Allen_Brain_Atlas_down',
    'Allen_Brain_Atlas_up',
    'ARCHS4_Cell-lines',
    'ARCHS4_Tissues',
    'Cancer_Cell_Line_Encyclopedia',
    'CCLE_Proteomics_2020',
    'ESCAPE',
    'GTEx_Tissue_Sample_Gene_Expression_Profiles_down',
    'GTEx_Tissue_Sample_Gene_Expression_Profiles_up',
    'Human_Gene_Atlas',
    'Mouse_Gene_Atlas',
    'NCI-60_Cancer_Cell_Lines',
    'ProteomicsDB_2020',
    'Tissue_Protein_Expression_from_Human_Proteome_Map',           
    'Chromosome_Location_hg19',
    'Data_Acquisition_Method_Most_Popular_Genes',
    'Enrichr_Libraries_Most_Popular_Genes',
    'Genes_Associated_with_NIH_Grants',
    'HMDB_Metabolites',
    'HomoloGene',
    'InterPro_Domains_2019',
    'NIH_Funded_PIs_2017_AutoRIF_ARCHS4_Predictions',
    'NIH_Funded_PIs_2017_GeneRIF_ARCHS4_Predictions',
    'NIH_Funded_PIs_2017_Human_AutoRIF',
    'NIH_Funded_PIs_2017_Human_GeneRIF',
    'Pfam_Domains_2019',
    'Pfam_InterPro_Domains',
    'Table_Mining_of_CRISPR_Studies',               
    'BioCarta_2013',
    'BioCarta_2015',
    'ChEA_2013',
    'ChEA_2015',
    'Chromosome_Location',
    'Disease_Signatures_from_GEO_down_2014',
    'Disease_Signatures_from_GEO_up_2014',
    'Drug_Perturbations_from_GEO_2014',
    'ENCODE_Histone_Modifications_2013',
    'ENCODE_TF_ChIP-seq_2014',
    'GO_Biological_Process_2013',
    'GO_Biological_Process_2015',
    'GO_Biological_Process_2017',
    'GO_Biological_Process_2017b',
    'GO_Cellular_Component_2013',
    'GO_Cellular_Component_2015',
    'GO_Cellular_Component_2017',
    'GO_Cellular_Component_2017b',
    'GO_Molecular_Function_2013',
    'GO_Molecular_Function_2015',
    'GO_Molecular_Function_2017',
    'GO_Molecular_Function_2017b',
    'HumanCyc_2015',
    'KEA_2013',
    'KEGG_2013',
    'KEGG_2015',
    'KEGG_2016',
    'MGI_Mammalian_Phenotype_2013',
    'MGI_Mammalian_Phenotype_2017',
    'MGI_Mammalian_Phenotype_Level_3',
    'MGI_Mammalian_Phenotype_Level_4',
    'NCI-Nature_2015',
    'Panther_2015',
    'Reactome_2013',
    'Reactome_2015',
    'TargetScan_microRNA',
    'Tissue_Protein_Expression_from_ProteomicsDB',
    'WikiPathways_2013',
    'WikiPathways_2015',
    'WikiPathways_2016',               
    'Aging_Perturbations_from_GEO_down',
    'Aging_Perturbations_from_GEO_up',
    'Disease_Perturbations_from_GEO_down',
    'Disease_Perturbations_from_GEO_up',
    'Drug_Perturbations_from_GEO_down',
    'Drug_Perturbations_from_GEO_up',
    'Gene_Perturbations_from_GEO_down',
    'Gene_Perturbations_from_GEO_up',
    'Ligand_Perturbations_from_GEO_down',
    'Ligand_Perturbations_from_GEO_up',
    'MCF7_Perturbations_from_GEO_down',
    'MCF7_Perturbations_from_GEO_up',
    'Microbe_Perturbations_from_GEO_down',
    'Microbe_Perturbations_from_GEO_up',
    'RNA-Seq_Disease_Gene_and_Drug_Signatures_from_GEO',
    'SysMyo_Muscle_Gene_Sets']) }}'<jupyter_output><empty_output><jupyter_text>Set Other ParametersFor simplicity, the only inputs for this appyter are a gene list and one library. Other parameters are set to default values in the cell below. You can download the notebook, change these parameters, and rerun it if you wish.<jupyter_code># Scatter Plot Parameters
significance_value = 0.05

# Bar Chart Parameters
figure_file_format = ['png', 'svg']
output_file_name = 'Enrichr_results'
color = 'lightskyblue'
final_output_file_names = [str(output_file_name+'.'+file_type) for file_type in figure_file_format]

# Hexagonal Canvas Parameters
canvas_color = 'Blue'
num_hex_colored = 10

# Manhattan Plot Parameters
manhattan_colors = ['#003f5c', '#7a5195', '#ef5675', '#ffa600']<jupyter_output><empty_output><jupyter_text>Import gene list<jupyter_code># Import gene list as file or from text box file
# Will choose file upload over textbox if a file is given 
if gene_list_filename != '':
    open_gene_list_file = open(gene_list_filename,'r')
    lines = open_gene_list_file.readlines()
    genes = [x.strip() for x in lines]
    open_gene_list_file.close()
else:
    genes = gene_list_input.split('\n')
    genes = [x.strip() for x in genes]<jupyter_output><empty_output><jupyter_text>Scatter Plot Functions<jupyter_code>def download_library(library_name):
    # download pre-processed library data
    try:
        df = pd.read_csv('https://raw.githubusercontent.com/skylar73/Enrichr-Processed-Library-Storage/master/Scatterplot/Libraries/' + library_name + '.csv')
    except:
        return -1, -1, -1

    name = df['Name'].tolist()
    gene_list = df['Genes'].tolist()
    library_data = [list(a) for a in zip(name, gene_list)]
    return genes, library_data, df
# enrichment analysis
def get_library_iter(library_data):
    for member in library_data:
        term = member[0]
        gene_set = member[1].split(' ')
        yield term, gene_set

def get_enrichment_results(genes, library_data):
    return sorted(enrich_crisp(genes, get_library_iter(library_data), 20000, True), key=lambda r: r[1].pvalue)

def get_pvalue(row, unzipped_results, all_results):
    if row['Name'] in list(unzipped_results[0]):
        index = list(unzipped_results[0]).index(row['Name'])
        return all_results[index][1].pvalue
    else:
        return 1

# call enrichment results and return a plot and dataframe for Scatter Plot
def get_plot(library_name):
    genes, library_data, df = download_library(library_name)

    # library not supported
    if genes == -1:
        return -1 ,-1

    all_results = get_enrichment_results(genes, library_data)
    unzipped_results = list(zip(*all_results))

    if len(all_results) == 0:
        print("There are no enriched terms with your inputted gene set and the ", library_name, " library.")
        my_colors = ['#808080'] * len(df.index)

        source = ColumnDataSource(
            data=dict(
                x = df['x'],
                y = df['y'],
                gene_set = df['Name'],
                colors = my_colors,
                sizes = [6] * len(df.index)
            )
        )

        hover_emb = HoverTool(names=["df"], tooltips="""
            <div style="margin: 10">
                <div style="margin: 0 auto; width:200px;">
                    <span style="font-size: 12px; font-weight: bold;">Gene Set:</span>
                    <span style="font-size: 12px">@gene_set</span>
                </div>
            </div>
            """)
    else:
        # add p value to the dataframe
        df['p value'] = df.apply (lambda row: get_pvalue(row, unzipped_results, all_results), axis=1)

        # normalize p values for color scaling
        cmap = mpl.cm.get_cmap('Blues_r')
        norm = colors.Normalize(vmin = df['p value'].min(), vmax=significance_value*2)

        my_colors = []
        my_sizes = []
        for index, row in df.iterrows():
            if row['p value'] < significance_value:
                my_colors += [mpl.colors.to_hex(cmap(norm(row['p value'])))]
                my_sizes += [12]
            else:
                my_colors += ['#808080']
                my_sizes += [6]

        source = ColumnDataSource(
                data=dict(
                    x = df['x'],
                    y = df['y'],
                    gene_set = df['Name'],
                    p_value = df['p value'],
                    colors = my_colors,
                    sizes = my_sizes
                )
            )

        hover_emb = HoverTool(names=["df"], tooltips="""
            <div style="margin: 10">
                <div style="margin: 0 auto; width:200px;">
                    <span style="font-size: 12px; font-weight: bold;">Gene Set:</span>
                    <span style="font-size: 12px">@gene_set</span>
                    <span style="font-size: 12px; font-weight: bold;">p-value:</span>
                    <span style="font-size: 12px">@p_value</span>
                </div>
            </div>
            """)

    tools_emb = [hover_emb, 'pan', 'wheel_zoom', 'reset', 'save']

    plot_emb = figure(plot_width=700, plot_height=800, tools=tools_emb)

    # hide axis labels and grid lines
    plot_emb.xaxis.major_tick_line_color = None
    plot_emb.xaxis.minor_tick_line_color = None
    plot_emb.yaxis.major_tick_line_color = None
    plot_emb.yaxis.minor_tick_line_color = None
    plot_emb.xaxis.major_label_text_font_size = '0pt'
    plot_emb.yaxis.major_label_text_font_size = '0pt' 

    plot_emb.circle('x', 'y', size = 'sizes', alpha = 0.7, line_alpha = 0, 
                    line_width = 0.01, source = source, fill_color = 'colors', name = "df")

    plot_emb.output_backend = "svg"
    
    return plot_emb, df<jupyter_output><empty_output><jupyter_text>Enrichr API Function for Manhattan Plot and Bar Chart<jupyter_code># Takes a gene list and Enrichr libraries as input
def Enrichr_API(enrichr_gene_list, all_libraries):

    all_terms = []
    all_pvalues =[] 
    all_adjusted_pvalues = []

    for library_name in all_libraries : 
        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'
        genes_str = '\n'.join(enrichr_gene_list)
        description = 'Example gene list'
        payload = {
            'list': (None, genes_str),
            'description': (None, description)
        }

        response = requests.post(ENRICHR_URL, files=payload)
        if not response.ok:
            raise Exception('Error analyzing gene list')

        data = json.loads(response.text)
        time.sleep(0.5)
        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/enrich'
        query_string = '?userListId=%s&backgroundType=%s'
        user_list_id = data['userListId']
        short_id = data["shortId"]
        gene_set_library = library_name
        response = requests.get(
            ENRICHR_URL + query_string % (user_list_id, gene_set_library)
         )
        if not response.ok:
            raise Exception('Error fetching enrichment results')

        data = json.loads(response.text)

        short_results_df  = pd.DataFrame(data[library_name][0:10])
        all_terms.append(list(short_results_df[1]))
        all_pvalues.append(list(short_results_df[2]))
        all_adjusted_pvalues.append(list(short_results_df[6]))
        
        results_df  = pd.DataFrame(data[library_name])
        # adds library name to the data frame so the libraries can be distinguished
        results_df['library'] = library_name.replace('_', '')

    return([results_df, short_results_df, all_terms, all_pvalues, all_adjusted_pvalues, str(short_id)])<jupyter_output><empty_output><jupyter_text>Bar Chart Functions<jupyter_code># Function plots results in bar chart
# Takes all terms, all p-values, all adjusted p-values, plot title, Enrichr libraries, and specified figure format
def enrichr_figure(all_terms, all_pvalues, all_adjusted_pvalues, plot_names, all_libraries, fig_format, bar_color): 
    
    # rows and columns depend on number of Enrichr libraries submitted 
    rows = []
    cols = []
    
    # Bar colors
    if bar_color != 'lightgrey':
        bar_color_not_sig = 'lightgrey'
        edgecolor=None
        linewidth=0
    else:
        bar_color_not_sig = 'white'
        edgecolor='black'
        linewidth=1
    

    plt.figure(figsize=(12, 12))
    rows = [0]
    cols = [0]
    i = 0 
    bar_colors = [bar_color if (x < 0.05) else bar_color_not_sig for x in all_pvalues[i]]
    fig = sns.barplot(x=np.log10(all_pvalues[i])*-1, y=all_terms[i], palette=bar_colors, edgecolor=edgecolor, linewidth=linewidth)
    fig.axes.get_yaxis().set_visible(False)
    fig.set_title(all_libraries[i].replace('_', ' '), fontsize=26)
    fig.set_xlabel('-Log10(p-value)', fontsize=25)
    fig.xaxis.set_major_locator(MaxNLocator(integer=True))
    fig.tick_params(axis='x', which='major', labelsize=20)
    if max(np.log10(all_pvalues[i])*-1)<1:
        fig.xaxis.set_ticks(np.arange(0, max(np.log10(all_pvalues[i])*-1), 0.1))
    for ii,annot in enumerate(all_terms[i]):
        if all_adjusted_pvalues[i][ii] < 0.05:
            annot = '  *'.join([annot, str(str(np.format_float_scientific(all_pvalues[i][ii], precision=2)))]) 
        else:
            annot = '  '.join([annot, str(str(np.format_float_scientific(all_pvalues[i][ii], precision=2)))])

        title_start= max(fig.axes.get_xlim())/200
        fig.text(title_start, ii, annot, ha='left', wrap = True, fontsize = 26)
        fig.patch.set_edgecolor('black')  
        fig.patch.set_linewidth('2')
    
    # Save results 
    for plot_name in plot_names:
        plt.savefig(plot_name, bbox_inches = 'tight')
    
    # Show plot 
    plt.show()<jupyter_output><empty_output><jupyter_text>Hexagonal Canvas Functions<jupyter_code>def library_processing():
    # downloads library data for the hexagonal canvas
    # library data is pre-annealed so the canvas will have the most similar gene sets closest together

    raw_library_data = []
    library_data = []

    try:
        library_name = enrichr_library
        with urllib.request.urlopen('https://raw.githubusercontent.com/skylar73/Enrichr-Processed-Library-Storage/master/Annealing/Annealed-Libraries/' + enrichr_library + '.txt') as f:
            for line in f.readlines():
                raw_library_data.append(line.decode('utf-8').split("\t\t"))
        name = []
        gene_list = []
    except:
        return [], -1, -1

    for i in range(len(raw_library_data)):
        name += [raw_library_data[i][0]]
        raw_genes = raw_library_data[i][1].split('\t')
        gene_list += [raw_genes[:-1]]

    library_data = [list(a) for a in zip(name, gene_list)]

    # raw_library_data: a 2D list where the first element is the name and the second element is a list of genes associated with that name

    jaccard_indices = []
    indices = []

    for gene_set in library_data:
        intersection = [value for value in gene_set[1] if value in genes]
        index = len(intersection)/(len(gene_set[1]) + len(genes))
        jaccard_indices += [[gene_set[0], index]]
        indices += [round(index, 5)]

    # determine the dimensions of the canvas
    x_dimension = math.ceil(math.sqrt(len(indices)))
    y_dimension = math.ceil(math.sqrt(len(indices)))

    # zip name, gene_list, indices, and blank list for neighbor score then add dummy entries to the zipped list
    num_hex = x_dimension*y_dimension
    anneal_list = list(zip(name, gene_list, indices))

    return anneal_list, x_dimension, y_dimension

def unzip_list(anneal_list):
    unzipped_list = zip(*anneal_list)
    return list(unzipped_list)
# define a list of colors for the hexagonal canvas
def get_color(anneal_list, cut_off_value, x_dimension, y_dimension):

    # Deal with cut_off_value (only color the most significant 10/20 hexagons)
    if cut_off_value == 2.0:
        sort_list = sorted(anneal_list, key=itemgetter(2), reverse=True)
        cut_off_value = sort_list[int(num_hex_colored)-1][2]

    r_value = 0
    g_value = 0
    b_value = 0

    if canvas_color == 'Red':
        r_value = 0.0
        g_value = 0.8
        b_value = 0.8
    if canvas_color == 'Yellow':
        r_value = 0.0
        g_value = 0.3
        b_value = 1.0
    if canvas_color == 'Purple':
        r_value = 0.5
        g_value = 1.0
        b_value = 0.0
    if canvas_color == 'Pink':
        r_value = 0.0
        g_value = 1.0
        b_value = 0.2
    if canvas_color == 'Orange':
        r_value = 0.0
        g_value = 0.45
        b_value = 1.0
    if canvas_color == 'Green':
        r_value = 1.0
        g_value = 0.0
        b_value = 1.0
    if canvas_color == 'Blue':
        r_value = 1.0
        g_value = 0.9
        b_value = 0.0

    color_list = []

    unzipped_anneal_list = unzip_list(anneal_list)

    max_index = max(unzipped_anneal_list[2])

    if max_index != 0:
        scaled_list = [i/max_index for i in unzipped_anneal_list[2]]
    else:
        scaled_list = unzipped_anneal_list[2]

    for i in range(x_dimension*y_dimension):
        if i < len(unzipped_anneal_list[2]) and float(unzipped_anneal_list[2][i]) >= cut_off_value:
            color_list += [mpl.colors.to_hex((1-scaled_list[i]*r_value, 
            1-scaled_list[i]*g_value, 1-scaled_list[i]*b_value))]
        elif i < len(unzipped_anneal_list[2]):
            color_list += [mpl.colors.to_hex((1-scaled_list[i], 
            1-scaled_list[i], 1-scaled_list[i]))]
        else:
            color_list += ["#FFFFFF"]
    return color_list, max_index, cut_off_value
def init_chart():
  chart_id = 'mychart-' + str(uuid.uuid4())
  display(HTML('<script src="/static/components/requirejs/require.js"></script>'))
  display(HTML(Template(dedent('''
  <script>
  require.config({
    paths: {
      'd3': 'https://cdnjs.cloudflare.com/ajax/libs/d3/5.16.0/d3.min',
      'd3-hexbin': 'https://d3js.org/d3-hexbin.v0.2.min',
    },
    shim: {
      'd3-hexbin': ['d3']
    }
  })

  // If we configure mychart via url, we can eliminate this define here
  define($chart_id, ['d3', 'd3-hexbin'], function(d3, d3_hexbin) {
    return function (figure_id, numA, numB, colorList, libraryList, indices) {
      var margin = {top: 50, right: 20, bottom: 20, left: 50},
        width = 850 - margin.left - margin.right,
        height = 350 - margin.top - margin.bottom;

      // append the svg object to the body of the page
      var svG = d3.select('#' + figure_id)
                  .attr("width", width + margin.left + margin.right)
                  .attr("height", height + margin.top + margin.bottom)
                  .append("g")
                  .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
      
      //The number of columns and rows of the heatmap
      var MapColumns = numA,
          MapRows = numB;

      //The maximum radius the hexagons can have to still fit the screen
      var hexRadius = d3.min([width/((MapColumns + 0.5) * Math.sqrt(3)), height/((MapRows + 1/3) * 1.5)]);

      //Calculate the center position of each hexagon
      var points = [];
      for (var i = 0; i < MapRows; i++) {
          for (var j = 0; j < MapColumns; j++) {
              var x = hexRadius * j * Math.sqrt(3)
              //Offset each uneven row by half of a "hex-width" to the right
              if(i%2 === 1) x += (hexRadius * Math.sqrt(3))/2
              var y = hexRadius * i * 1.5
              points.push([x,y])
          }
      }

      //Set the hexagon radius
      var hexbin = d3_hexbin.hexbin().radius(hexRadius);

      svG.append("g")
        .selectAll(".hexagon")
        .data(hexbin(points))
        .enter().append("path")
        .attr("class", "hexagon")
        .attr("d", function (d) {
            return "M" + d.x + "," + d.y + hexbin.hexagon();
        })
        .attr("stroke", "black")
        .attr("stroke-width", "1px")
        .style("fill", function (d,i) { return colorList[i]; })
        .on("mouseover", mover)
        .on("mouseout", mout)
        .append("svg:title")
        .text(function(d,i) { return libraryList[i].concat(" ").concat(indices[i]); });

      // Mouseover function
      function mover(d) {
      d3.select(this)
        .transition().duration(10)  
        .style("fill-opacity", 0.3)
      };

      // Mouseout function
      function mout(d) { 
      d3.select(this)
        .transition().duration(10)
        .style("fill-opacity", 1)
      };

  }

  })
  </script>
  ''')).substitute({ 'chart_id': repr(chart_id) })))
  return chart_id

def Canvas(numA, numB, colorList, libraryList, indices):
  chart_id = init_chart()
  display(HTML(Template(dedent('''
  <svg id=$figure_id></svg>
  <script>
  require([$chart_id], function(mychart) {
    mychart($figure_id, $numA, $numB, $colorList, $libraryList, $indices)
  })
  </script>
  ''')).substitute({
      'chart_id': repr(chart_id),
      'figure_id': repr('fig-' + str(uuid.uuid4())),
      'numA': repr(numA),
      'numB': repr(numB),
      'colorList': repr(colorList),
      'libraryList': repr(libraryList),
      'indices': repr(indices)
  })))<jupyter_output><empty_output><jupyter_text>Manhattan Plot Functions<jupyter_code># Processes Enrichr data for Manhattan plots

def get_data(genes):
    # Process Enrichr data
    sorted_data = pd.DataFrame({"Gene Set": [], "-log(p value)": [], "Library": []})

    # get enrichr results from the library selected
    results_df = Enrichr_API(genes, [enrichr_library])[0]

    all_terms = []
    all_pvalues = []
    library_names = []

    all_terms.append(list(results_df[1]))
    all_pvalues.append(list(results_df[2]))
    library_names.append(list(results_df['library']))

    x=np.log10(all_pvalues[0])*-1
    sorted_terms = list(zip(all_terms[0], x, library_names[0]))
    sorted_terms = sorted(sorted_terms, key = itemgetter(0))
    unzipped_sorted_list = list(zip(*sorted_terms))

    data = pd.DataFrame({"Gene Set": unzipped_sorted_list[0], "-log(p value)": unzipped_sorted_list[1], "Library": unzipped_sorted_list[2]})

    sorted_data = pd.concat([sorted_data, data])

    # group data by library
    groups = sorted_data.groupby("Library")
    return sorted_data, groups
# Create Manhattan Plots
def manhattan(sorted_data, groups):
    # split data frame into smaller data frames by library
    list_of_df = []
    for library_name in [enrichr_library]:
        library_name = library_name.replace('_', '')
        df_new = sorted_data[sorted_data['Library'] == library_name]
        list_of_df += [df_new]

    list_of_xaxis_values = []
    for df in list_of_df:  
        list_of_xaxis_values += df["Gene Set"].values.tolist()

    # define the output figure and the features we want
    p = figure(x_range = list_of_xaxis_values, plot_height=300, plot_width=750, tools='pan, box_zoom, hover, reset, save')

    # loop over all libraries
    r = []
    categories = []
    color_index = 0
    for df in list_of_df:
        if color_index >= len(manhattan_colors):
            color_index = 0 

        # calculate actual p value from -log(p value)
        actual_pvalues = []
        for log_value in df["-log(p value)"].values.tolist():
            actual_pvalues += ["{:.5e}".format(10**(-1*log_value))]

        # define ColumnDataSource with our data for this library
        source = ColumnDataSource(data=dict(
            x = df["Gene Set"].values.tolist(),
            y = df["-log(p value)"].values.tolist(),
            pvalue = actual_pvalues,
        ))
    
        # plot data from this library
        r += [p.circle(x = 'x', y = 'y', size=5, fill_color=manhattan_colors[color_index], line_color = manhattan_colors[color_index], line_width=1, source = source)]
        color_index += 1

    p.background_fill_color = 'white'
    p.xaxis.major_tick_line_color = None 
    p.xaxis.major_label_text_font_size = '0pt'
    p.y_range.start = 0
    p.yaxis.axis_label = '-log(p value)'

    p.hover.tooltips = [
        ("Gene Set", "@x"),
        ("p value", "@pvalue"),
    ]
    p.output_backend = "svg"
    
    # returns the plot
    return p<jupyter_output><empty_output><jupyter_text>Plot Outputs Scatter Plot<jupyter_code># Display Scatter Plot
plot, df = get_plot(enrichr_library)
if plot == -1:
    display(Markdown("The Scatter Plot visualization is not available for the selected library at this time."))
else:
    show(plot)
    display(Markdown('You may have to zoom in to see the details in densely-populated portions of the plots.'))
    display(Markdown('The larger blue points represent significantly enriched terms (the darker the blue, the more significant).'))
    display(Markdown('The gray points are not significant.'))
    display(Markdown('Hovering over points will display the associated gene set name and the p-value.'))
    display(Markdown('Plots can be downloaded as an svg using the save function on the toolbar next to the plot.'))
    display(Markdown('[Access the complete Scatter Plot Visualization Appyter here to create and compare up to 9 scatter plots at once](https://appyters.maayanlab.cloud/#Enrichr_Scatterplot_Appyter)'))<jupyter_output><empty_output><jupyter_text>Bar Chart<jupyter_code># Display Bar Chart
results = Enrichr_API(genes, [enrichr_library])
enrichr_figure(results[2], results[3], results[4], final_output_file_names, [enrichr_library], figure_file_format, color)
# Download Bar Chart
for i, file in enumerate(final_output_file_names):
    display(FileLink(file, result_html_prefix=str('Download ' + figure_file_format[i] + ': ')))
display(Markdown('The bar chart contains the top 10 enriched terms and their corresponding p-values for the chosen library. Colored bars correspond to terms with significant p-values (<0.05). An asterisk (*) next to a p-value indicates the term also has a significant adjusted p-value (<0.05).'))
display(Markdown('[Access the complete Bar Chart Appyter here to create customized bar charts for multiple libraries at once.](https://appyters.maayanlab.cloud/Enrichr_compressed_bar_chart_figure/)'))<jupyter_output><empty_output><jupyter_text>Hexagonal Canvas<jupyter_code># Display Hexagonal Canvas
anneal_list, x_dimension, y_dimension = library_processing()
if x_dimension < 0:
    display(Markdown("The Hexagonal Canvas visualization is not available for the selected library at this time."))
else:
    color_list, scaling_factor, cut_off_value = get_color(anneal_list, 2.0, x_dimension, y_dimension)
    unzipped_anneal_list = unzip_list(anneal_list)
    Canvas(x_dimension, y_dimension, color_list, list(unzipped_anneal_list[0]), list(unzipped_anneal_list[2]))
    print("The color scaling factor for this canvas is: " + str(scaling_factor))
    print("Gene sets with a similarity index less than " + str(cut_off_value) + " are not colored")
    display(Markdown('Each of the hexagons on the canvas represent one gene set from the library selected. The hexagons are colored based on the Jaccard similarity index between the inputted gene list and the gene set the hexagon represents (the brighter, the more similar). The name of the gene set and the associated similarity index are displayed when a hexagon is hovered over.'))
    display(Markdown('[Access the complete Hexagonal Canvas Appyter here to create customized canvases for 1 or 2 libraries at once.](https://appyters.maayanlab.cloud/#Enrichr_Canvas_Appyter)'))<jupyter_output><empty_output><jupyter_text>Manhattan Plot<jupyter_code># Display Manhattan Plot
sorted_data, groups = get_data(genes)
show(manhattan(sorted_data, groups))
display(Markdown('The x-axis of the plot is made up of gene sets from the library. The y-axis of the plot has the -log(p value) for each gene set. The name of the gene set and the associated p-value are displayed when a point is hovered over. You can also zoom, pan, and save the plot as an svg using the toolbar on the right.'))
display(Markdown('[Access the complete Manhattan Plot Appyter here to create customized static and dynamic Manhattan plots to compare multiple libraries at once.](https://appyters.maayanlab.cloud/#Enrichr_Manhattan_Plot)'))<jupyter_output><empty_output><jupyter_text>Table of significant p-values<jupyter_code># Output a table of significant p-values
def create_download_link(df, title = "Download CSV file of this table", filename = "data.csv"):  
    csv = df.to_csv(index = False)
    b64 = base64.b64encode(csv.encode())
    payload = b64.decode()
    html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
    html = html.format(payload=payload, title=title, filename=filename)
    return HTML(html)

if plot != -1 and 'p value' in df.columns:
    sorted_df = df.sort_values(by = ['p value'])
    filtered_df = sorted_df[sorted_df['p value'] <= significance_value]
    if len(filtered_df) != 0:
        display(HTML(f"<strong>Table of significant p-values for {enrichr_library.replace('_', ' ')}</strong>"))
        display(HTML(filtered_df[['Name', 'p value']].to_html(index = False)))
        display(create_download_link(filtered_df[['Name', 'p value']]))<jupyter_output><empty_output><jupyter_text>Link to Enrichr<jupyter_code># Get complete enrichment analysis results from Enrichr 
url = 'https://amp.pharm.mssm.edu/Enrichr/enrich?dataset=' + results[5]
display(HTML(f'<span style="font-size:1.5em;"><a href = "https://amp.pharm.mssm.edu/Enrichr/enrich?dataset={results[5]}"> Access your complete Enrichment results on Enrichr here. </a></span>'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1128871652429118, 'fraction_numerical': 0.046040563911744666, 'mean_word_length': 3.699298115995567, 'pattern_counts': {'":': 6, '<': 109, '<?xml version=': 0, '>': 97, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '169094', 'license': 'Apache-2.0', 'n_tokens_mistral': 15115, 'n_tokens_neox': 13701, 'n_words': 2731, 'path': 'appyters/Enrichment_Analysis_Visualizer/Enrichment-Appyter.ipynb', 'repo_name': 'cyoon23/appyter-catalog', 'size': '38173'}