<jupyter_start><jupyter_text>Harmonizome ETL: Gene Ontology (GO) Created by: <NAME> Credit to: <NAME>Data Source: http://geneontology.org/docs/download-ontology/<jupyter_code># appyter init
from appyter import magic
magic.init(lambda _=globals: _())
import sys
import os
from datetime import date

import numpy as np
import pandas as pd
import itertools
import xml.etree.ElementTree as ET
import matplotlib.pyplot as plt
%matplotlib inline

import harmonizome.utility_functions as uf
import harmonizome.lookup as lookup
%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Notebook Information<jupyter_code>print('This notebook was run on:', date.today(), '\nPython version:', sys.version)<jupyter_output><empty_output><jupyter_text>Initialization<jupyter_code>%%appyter hide_code

{% do SectionField(
    name='data',
    title='Upload Data',
    img='load_icon.png'
) %}

{% do SectionField(
    name='settings',
    title='Settings',
    img='setting_icon.png'
) %}
%%appyter code_eval

{% do DescriptionField(
    name='description',
    text='The examples below were sourced from <a href="http://geneontology.org/" target="_blank">geneontology.org</a>. If clicking on the examples does not work, they should be downloaded directly from the source website.',
    section='data'
) %}

{% set df_file = FileField(
    constraint='.*\.gaf.gz$',
    name='gaf', 
    label='Gene Assocation File (gaf.gz)', 
    default='goa_human.gaf.gz',
    examples={
        'goa_human.gaf.gz': 'http://geneontology.org/gene-associations/goa_human.gaf.gz'
    },
    section='data'
) %}

{% set ontology = FileField(
    constraint='.*\.owl$',
    name='ontology', 
    label='Ontology (owl)', 
    default='go.owl',
    examples={
        'go.owl': 'http://purl.obolibrary.org/obo/go.owl'
    },
    section='data'
) %}
%%appyter code_eval

{% set attribute = ChoiceField(
    name='attribute',
    label='Attribute',
    choices={
        'Biological Process': 'Process',
        'Cellular Component': 'Component',
        'Molecular Function': 'Function'
    },
    default='Cellular Component',
    section='settings'
) %}<jupyter_output><empty_output><jupyter_text>Load Mapping Dictionaries<jupyter_code>symbol_lookup, geneid_lookup = lookup.get_lookups()<jupyter_output><empty_output><jupyter_text>Output Path<jupyter_code>%%appyter code_exec

output_name = 'go_{{attribute}}'.lower()

path = 'Output/GO-{{attribute}}' 
if not os.path.exists(path):
    os.makedirs(path)<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>%%appyter code_exec

df = pd.read_csv(
    {{df_file}}, 
    skiprows=31, header=None, usecols=[2, 3, 4, 6, 8], sep='\t',
    names=['DB Object Symbol', 'Qualifier', 'GO ID', 'Evidence Code', 'Aspect'])
df.head()
df.shape<jupyter_output><empty_output><jupyter_text>Load Ontology<jupyter_code>%%appyter code_exec

tree = ET.parse({{ontology}})
root = tree.getroot()<jupyter_output><empty_output><jupyter_text>Pre-process Data  Get Relevant Data<jupyter_code>%%appyter code_exec

# Get only desired attribute:
# P for Biological Process, C for Cellular Component, F for Molecular Function
df = df[df['Aspect'] == '{{attribute}}'[0]]
# Drop data inferred from electronic annotation
df = df[df['Evidence Code'] != 'IEA']
# Drop NOT in qualifier
df = df[df['Qualifier'] != 'NOT']
df.shape
df = df[['DB Object Symbol', 'GO ID']]
df.head()<jupyter_output><empty_output><jupyter_text>Build GO Ontology Map<jupyter_code>ns = {
    'owl': 'http://www.w3.org/2002/07/owl#',
    'obo': 'http://www.geneontology.org/formats/oboInOwl#',
    'rdfs': 'http://www.w3.org/2000/01/rdf-schema#'
}
mapping = {'GO ID': [], 'Label': []}

for child in root.findall('owl:Class', ns):
    id = child.find('obo:id', ns)
    label = child.find('rdfs:label', ns)
    if id is not None and label is not None:
        mapping['GO ID'].append(id.text)
        mapping['Label'].append(label.text)

onto_meta = pd.DataFrame(mapping).set_index('GO ID')
onto_meta.head()<jupyter_output><empty_output><jupyter_text>Map GO IDs<jupyter_code>df['GO ID'] = onto_meta['Label'].reindex(df['GO ID']).reset_index(drop=True)
df = df.set_index('DB Object Symbol')
df.index.name = 'Gene Symbol'
df.columns = ['GO Term']
df.head()<jupyter_output><empty_output><jupyter_text>Filter Data  Map Gene Symbols to Up-to-date Approved Gene Symbols<jupyter_code>df = uf.map_symbols(df, symbol_lookup, remove_duplicates=True)
df.shape<jupyter_output><empty_output><jupyter_text>Analyze Data  Create Binary Matrix<jupyter_code>binary_matrix = uf.binary_matrix(df)
binary_matrix.head()
binary_matrix.shape
uf.save_data(binary_matrix, path, output_name + '_binary_matrix', 
            compression='npz', dtype=np.uint8)<jupyter_output><empty_output><jupyter_text>Create Gene List<jupyter_code>gene_list = uf.gene_list(binary_matrix, geneid_lookup)
gene_list.head()
gene_list.shape
uf.save_data(gene_list, path, output_name + '_gene_list',
            ext='tsv', compression='gzip', index=False)<jupyter_output><empty_output><jupyter_text>Create Attribute List<jupyter_code>attribute_list = uf.attribute_list(binary_matrix)
attribute_list.head()
attribute_list.shape
uf.save_data(attribute_list, path, output_name + '_attribute_list',
            ext='tsv', compression='gzip')<jupyter_output><empty_output><jupyter_text>Create Gene and Attribute Set Libraries<jupyter_code>uf.save_setlib(binary_matrix, 'gene', 'up', path, output_name + '_gene_up_set')
uf.save_setlib(binary_matrix, 'attribute', 'up', path, 
                           output_name + '_attribute_up_set')<jupyter_output><empty_output><jupyter_text>Create Attribute Similarity Matrix<jupyter_code>attribute_similarity_matrix = uf.similarity_matrix(binary_matrix.T, 'jaccard', sparse=True)
attribute_similarity_matrix.head()
uf.save_data(attribute_similarity_matrix, path,
            output_name + '_attribute_similarity_matrix', 
            compression='npz', symmetric=True, dtype=np.float32)<jupyter_output><empty_output><jupyter_text>Create Gene Similarity Matrix<jupyter_code>gene_similarity_matrix = uf.similarity_matrix(binary_matrix, 'jaccard', sparse=True)
gene_similarity_matrix.head()
uf.save_data(gene_similarity_matrix, path, 
            output_name + '_gene_similarity_matrix',
            compression='npz', symmetric=True, dtype=np.float32)<jupyter_output><empty_output><jupyter_text>Create Gene-Attribute Edge List<jupyter_code>edge_list = uf.edge_list(binary_matrix)
uf.save_data(edge_list, path, output_name + '_edge_list', 
        ext='tsv', compression='gzip')<jupyter_output><empty_output><jupyter_text>Create Downloadable Save File<jupyter_code>uf.archive(path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14140203714799282, 'fraction_numerical': 0.004194128220491312, 'mean_word_length': 5.758097165991903, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '251491', 'license': 'CC0-1.0', 'n_tokens_mistral': 2415, 'n_tokens_neox': 2305, 'n_words': 418, 'path': 'appyters/GO_Harmonizome_ETL/GO.ipynb', 'repo_name': 'shui02/appyter-catalog', 'size': '6692'}