<commit_before>import sys
from rdflib import Graph, Namespace, Literal
import csv
import urllib2

if __name__ == '__main__':
    indra_ns = 'http://sorger.med.harvard.edu/indra/'
    if len(sys.argv) > 1:
        proteins_file = sys.argv[1]
    else:
        proteins_file = '../../data/ras_pathway_proteins.csv'
    rn = Namespace(indra_ns + 'relations/')
    en = Namespace(indra_ns + 'entities/')
    g = Graph()

    has_name = rn.term('hasName')
    has_long_name = rn.term('hasLongName')
    has_synonym = rn.term('hasSynonym')
    isa = rn.term('isa')

    # Read BEL family names
    res = urllib2.urlopen('http://resource.belframework.org/belframework/'+\
        'latest-release/namespace/selventa-protein-families.belns')
    belns_text = res.read()
    start = belns_text.find('[Values]')
    lines = belns_text[start:].split('\n')
    bel_family_names = []
    for l in lines:
        if l.endswith(' Family|P'):
            family_name = l[:-2].replace(' ', '_').replace('/', '_')
            bel_family_names.append(family_name)

    family_names = set([])
    with open(proteins_file) as tsv:
        for line in csv.reader(tsv, dialect="excel-tab"):
            print line
            symbol, name, synonyms_str, family = line
            g.add((en.term(symbol), has_long_name, Literal(name.strip())))
            g.add((en.term(symbol), has_name, Literal(symbol)))
            synonyms = synonyms_str.split(', ')
            for s in synonyms:
                if s != '':
                    g.add((en.term(symbol), has_synonym, Literal(s)))
            family_name = family.strip()
            if family_name != '':
                print family_name
                g.add((en.term(symbol), isa, en.term(family_name)))
                g.add((en.term(family), has_name, Literal(family_name)))
                family_names.add(family_name)
    for fn in family_names:
        bel_synonyms = [bn for bn in bel_family_names 
                        if bn.find(fn) != -1]
        for b in bel_synonyms:
            g.add((en.term(fn), has_synonym, Literal(b.upper())))

    # Further BEL family names added manually
    g.add((en.term('ERK'), has_synonym, Literal('MAPK_ERK1_2_FAMILY')))

    with open('entity_hierarchy.rdf', 'wt') as out_file:
        out_file.write(g.serialize(format='xml'))
<commit_msg>Experimenting with adding to hierarchy<commit_after>import sys
from rdflib import Graph, Namespace, Literal
import csv
import urllib2

if __name__ == '__main__':
    indra_ns = 'http://sorger.med.harvard.edu/indra/'
    if len(sys.argv) > 1:
        proteins_file = sys.argv[1]
    else:
        proteins_file = '../../data/ras_pathway_proteins.csv'
    rn = Namespace(indra_ns + 'relations/')
    en = Namespace(indra_ns + 'entities/')
    g = Graph()

    has_name = rn.term('hasName')
    has_long_name = rn.term('hasLongName')
    has_synonym = rn.term('hasSynonym')
    isa = rn.term('isa')

    # Read BEL family names
    res = urllib2.urlopen('http://resource.belframework.org/belframework/'+\
        'latest-release/namespace/selventa-protein-families.belns')
    belns_text = res.read()
    start = belns_text.find('[Values]')
    lines = belns_text[start:].split('\n')
    bel_family_names = []
    for l in lines:
        if l.endswith(' Family|P'):
            family_name = l[:-2].replace(' ', '_').replace('/', '_')
            bel_family_names.append(family_name)

    family_names = set([])
    with open(proteins_file) as tsv:
        for line in csv.reader(tsv, dialect="excel-tab"):
            print line
            symbol, name, synonyms_str, family = line
            g.add((en.term(symbol), has_long_name, Literal(name.strip())))
            g.add((en.term(symbol), has_name, Literal(symbol)))
            synonyms = synonyms_str.split(', ')
            for s in synonyms:
                if s != '':
                    g.add((en.term(symbol), has_synonym, Literal(s)))
            family_name = family.strip()
            if family_name != '':
                print family_name
                g.add((en.term(symbol), isa, en.term(family_name)))
                g.add((en.term(family), has_name, Literal(family_name)))
                family_names.add(family_name)
    for fn in family_names:
        bel_synonyms = [bn for bn in bel_family_names 
                        if bn.find(fn) != -1]
        for b in bel_synonyms:
            g.add((en.term(fn), has_synonym, Literal(b.upper())))

    # Further BEL family names added manually
    g.add((en.term('ERK'), has_synonym, Literal('MAPK_ERK1_2_FAMILY')))

    # Add in makeshift hierarchies for families and complexes
    g.add((en.term('p85'), isa, en.term('PI3K')))
    g.add((en.term('p110'), isa, en.term('PI3K')))
    g.add((en.term('4EBP1'), isa, en.term('4EBP')))
    # MAP3K
    g.add((en.term('MAP3K1'), isa, en.term('MAP3K')))
    g.add((en.term('MAP3K2'), isa, en.term('MAP3K')))
    # MAP2K
    g.add((en.term('MAP2K1'), isa, en.term('MAP2K')))
    g.add((en.term('MAP2K2'), isa, en.term('MAP2K')))
    # MAPK
    g.add((en.term('MAPK1'), isa, en.term('ERK')))
    g.add((en.term('MAPK3'), isa, en.term('ERK')))
    g.add((en.term('ERK'), isa, en.term('MAPK')))

    with open('entity_hierarchy.rdf', 'wt') as out_file:
        out_file.write(g.serialize(format='xml'))

{'fraction_non_alphanumeric': 0.16004540295119182, 'fraction_numerical': 0.007945516458569807, 'mean_word_length': 2.5411922304085732, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2356335', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1853, 'n_words': 343}