<jupyter_start><jupyter_text>Modify this line to briefly discribe the functionality of new_script.ipynbCopyright (C) 2017  <NAME>This program is free software: you can redistribute it and/or modifyit under the terms of the GNU General Public License as published bythe Free Software Foundation, either version 3 of the License, or(at your option) any later version.This program is distributed in the hope that it will be useful,but WITHOUT ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See theGNU General Public License for more details.You should have received a copy of the GNU General Public Licensealong with this program.  If not, see .<jupyter_code>import os
from dotenv import load_dotenv, find_dotenv
from os.path import join, dirname, basename, exists, isdir

### Load environmental variables from the project root directory ###
# find .env automagically by walking up directories until it's found
dotenv_path = find_dotenv()

# load up the entries as environment variables
load_dotenv(dotenv_path)

# now you can get the variables using their names

# Check whether a network drive has been specified
DATABASE = os.environ.get("NETWORK_URL")
if DATABASE == 'None':
    pass
else:
    pass
    #mount network drive here

# set up directory paths
CURRENT_DIR = os.getcwd()
PROJ = dirname(dotenv_path) # project root directory

DATA = join(PROJ, 'data') #data directory
RAW_EXTERNAL = join(DATA, 'raw_external') # external data raw directory
RAW_INTERNAL = join(DATA, 'raw_internal') # internal data raw directory
INTERMEDIATE = join(DATA, 'intermediate') # intermediate data directory
FINAL = join(DATA, 'final') # final data directory

RESULTS = join(PROJ, 'results') # output directory
FIGURES = join(RESULTS, 'figures') # figure output directory
PICTURES = join(RESULTS, 'pictures') # picture output directory


# make folders specific for certain data
folder_name = ''
if folder_name != '':
    #make folders if they don't exist
    if not exists(join(RAW_EXTERNAL, folder_name)):
        os.makedirs(join(RAW_EXTERNAL, folder_name))

    if not exists(join(INTERMEDIATE, folder_name)):
        os.makedirs(join(INTERMEDIATE, folder_name))

    if not exists(join(FINAL, folder_name)):
        os.makedirs(join(FINAL, folder_name))


print('Standard variables loaded, you are good to go!')<jupyter_output>Standard variables loaded, you are good to go!<jupyter_text>Read in datahere, data is read in as well as translations from uniprot ID to NCBI Entrez (mapper obtained from: https://www.uniprot.org/uploadlists/)<jupyter_code>## Your code here ##
import pandas as pd

# de results loading
de_results = pd.read_csv("~/Documents/masters/thesis/thesis/data/final/de_edger_results_stress.csv")

# protein to gene translation file read in 
uniprot_to_entrez = pd.read_csv("~/Documents/masters/thesis/thesis/data/intermediate/uniprot_to_entrez.tsv", sep="\t")
uniprot_to_entrez = uniprot_to_entrez.drop_duplicates(subset ="From", keep = False)\
.set_index("From").to_dict('index')

uniprot_to_entrez
# de results translated to gene IDs
de_results_entrez = de_results
de_results_entrez["Unnamed: 0"] = [uniprot_to_entrez[i]['To'] if i in uniprot_to_entrez else "nan" for i in de_results_entrez["Unnamed: 0"]]
de_results_entrez = de_results_entrez[de_results_entrez["Unnamed: 0"] != "nan"]
de_results_ids = [list(de_results[de_results[i] != 0]["Unnamed: 0"]) for i in de_results.columns]

# Get ontologies (from http://geneontology.org/ontology/go-basic.obo)
from goatools.base import download_go_basic_obo
from goatools.obo_parser import GODag
import os
obo_fname = download_go_basic_obo()  # Download data
obodag = GODag(obo_fname)  # Load data

# Get associations genes <-> ontologies (from ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/gene2go.gz)
from goatools.base import download_ncbi_associations
from goatools.anno.genetogo_reader import Gene2GoReader
gene2go_fname = download_ncbi_associations()  # Download data
objanno = Gene2GoReader(gene2go_fname, taxids=[511145])  # Read data (only for e. coli)
ns2assoc = objanno.get_ns2assc()

#Remove downloaded data:
os.remove(obo_fname)
os.remove(gene2go_fname)<jupyter_output>requests.get(http://purl.obolibrary.org/obo/go/go-basic.obo, stream=True)
  WROTE: go-basic.obo

go-basic.obo: fmt(1.2) rel(2020-02-21) 47,245 GO Terms
FTP RETR ftp.ncbi.nlm.nih.gov gene/DATA gene2go.gz -> gene2go.gz
  gunzip gene2go.gz
HMS:0:00:03.399185  43,485 annotations,  3,893 genes,  4,109 GOs, 1 taxids READ: gene2go<jupyter_text>Read in translation from gene to go chosen taxid E. coli corresponds to Escherichia coli str. K-12 substr. MG1655, as found here: https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=511145&lvl=3&lin=f&keep=1&srchmode=1&unlock<jupyter_code>from __future__ import print_function
from goatools.anno.genetogo_reader import Gene2GoReader

for nspc, id2gos in ns2assoc.items():
    print("{NS} {N:,} annotated e. coli genes".format(NS=nspc, N=len(id2gos)))
# Over-representation object:
from goatools.goea.go_enrichment_ns import GOEnrichmentStudyNS
population = de_results_entrez["Unnamed: 0"]

goeaobj = GOEnrichmentStudyNS(
    population,  # Population (all mapped yeast genes in Entrez format)
    ns2assoc,  # geneid/GO associations
    obodag,  # Ontologies
    propagate_counts = False,  # ???
    alpha = 0.05,  # default significance cut-off
    methods = ['fdr_bh'])  # default multipletest correction method<jupyter_output><empty_output><jupyter_text>Run Gene ontologyrun gene ontology enrichment on the top genes contained within the<jupyter_code>%%capture
# 'p_' means "pvalue". 'fdr_bh' is the multipletest method we are currently using.
# genes to be analysed
geneids_study = list(de_results_entrez["Unnamed: 0"][0:400])

goea_results_sig = []
for i in de_results_ids:
    goea_results_all = goeaobj.run_study(i)
    goea_results_sig.append([r for r in goea_results_all if r.p_fdr_bh < 0.05])
from goatools.godag_plot import plot_gos, plot_results, plot_goid2goobj

# goeaobj.wr_tsv(join(FINAL, "p_values_" + group_name + ".tsv"), goea_results_sig)

# save results of goe analysis
j = 0
for i in de_results.columns[1:]:
    goeaobj.wr_tsv("/Users/jonas/Documents/masters/thesis/thesis/data/final/go_analysis/p_values_go_{}.tsv".format(i), goea_results_sig[j])
    j += 1
# plot_results("/Users/jonas/Documents/masters/thesis/thesis/results/figures/nbt3102_{NS}.png", goea_results_sig)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10081954538425854, 'fraction_numerical': 0.016700170094325034, 'mean_word_length': 5.559837728194726, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '229452', 'license': 'CC0-1.0', 'n_tokens_mistral': 2303, 'n_tokens_neox': 2145, 'n_words': 600, 'path': 'data_science/code/go_term_analysis/.ipynb_checkpoints/go_term_analysis-checkpoint.ipynb', 'repo_name': 'blablabananarama/thesis', 'size': '6480'}