<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import os
import glob
import json
from indra import sparser
from indra.statements import stmts_from_json, get_valid_location, \
                             get_valid_residue

base_folder = os.environ['HOME'] + \
    '/data/darpa/phase3_eval/sources/sparser-20170530'

def get_file_names(base_dir):
    fnames = glob.glob(os.path.join(base_dir, '*.json'))
    return fnames

def get_file_stmts(fname):
    with open(fname, 'rb') as fh:
        print(fname)
        try:
            jd = json.load(fh)
        except ValueError as e:
            print(e)
            return []
    for st in jd:
        if st.get('type') == 'Translocation':
            for loc in ['from_location', 'to_location']:
                val = st.get(loc)
                try:
                    loc_valid = get_valid_location(val)
                    st[loc] = loc_valid
                except:
                    st[loc] = None
        try:
            res = st['residue']
            if res is False:
                st['residue'] = None
        except:
            pass

        try:
            res = st.get('residue')
            if res:
                get_valid_residue(res)
        except:
            st['residue'] = None

        try:
            res = st['position']
            if res is False:
                st['position'] = None
        except:
            pass

    stmts = stmts_from_json(jd)
    return stmts

def read_stmts(folder):
    fnames = get_file_names(folder)
    all_stmts = []
    for fname in fnames:
        st = get_file_stmts(fname)
        all_stmts += st
    return all_stmts


if __name__ == '__main__':
    stmts = read_stmts(base_folder)<commit_msg>Fix statements from Sparser<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import os
import glob
import json
from indra import sparser
from indra.statements import *
from indra.databases import uniprot_client, hgnc_client

base_folder = os.environ['HOME'] + \
    '/data/darpa/phase3_eval/sources/sparser-20170530'

def get_file_names(base_dir):
    fnames = glob.glob(os.path.join(base_dir, '*.json'))
    return fnames

def get_file_stmts(fname):
    with open(fname, 'rb') as fh:
        print(fname)
        try:
            jd = json.load(fh)
        except ValueError as e:
            print(e)
            return []
    for st in jd:
        if st.get('type') == 'Translocation':
            for loc in ['from_location', 'to_location']:
                val = st.get(loc)
                try:
                    loc_valid = get_valid_location(val)
                    st[loc] = loc_valid
                except:
                    st[loc] = None
        try:
            res = st['residue']
            if res is False:
                st['residue'] = None
        except:
            pass

        try:
            res = st.get('residue')
            if res:
                get_valid_residue(res)
        except:
            st['residue'] = None

        try:
            res = st['position']
            if res is False:
                st['position'] = None
        except:
            pass

    stmts = stmts_from_json(jd)
    stmts = fix_stmts(stmts)
    return stmts

def read_stmts(folder):
    fnames = get_file_names(folder)
    all_stmts = []
    for fname in fnames:
        st = get_file_stmts(fname)
        all_stmts += st
    return all_stmts

def fix_stmts(stmts):
    new_stmts = []
    for stmt in stmts:
        # Skip if no subject
        if isinstance(stmt, RegulateActivity):
            if stmt.subj is None:
                continue
        # Skip if no locations
        if isinstance(stmt, Translocation):
            if not (stmt.from_location or stmt.to_location):
                continue
        for agent in stmt.agent_list():
            if agent is not None:
                upid = agent.db_refs.get('UP')
                if upid:
                    gene_name = uniprot_client.get_gene_name(upid)
                    if gene_name:
                        agent.name = gene_name
                        if uniprot_client.is_human(upid):
                            hgnc_id = hgnc_client.get_hgnc_id(gene_name)
                            if hgnc_id:
                                agent.db_refs['hgnc'] = hgnc_id

        new_stmts.append(stmt)
    return new_stmts

if __name__ == '__main__':
    stmts = read_stmts(base_folder)
{'fraction_non_alphanumeric': 0.0872157209096931, 'fraction_numerical': 0.003974387281960698, 'mean_word_length': 1.603448275862069, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '958968', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1388, 'n_words': 330}