<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import os
import csv
import sys
import pickle
from copy import deepcopy
from indra.databases import uniprot_client, hgnc_client
from itertools import groupby, chain
from collections import Counter
import logging
from indra.util import read_unicode_csv, write_unicode_csv

logger = logging.getLogger('grounding_mapper')

class GroundingMapper(object):
    def __init__(self, gm):
        self.gm = gm

    def map_agents(self, stmts, do_rename=True):
        # Make a copy of the stmts
        mapped_stmts = []
        num_skipped = 0
        # Iterate over the statements
        for stmt in stmts:
            mapped_stmt = deepcopy(stmt)
            # Iterate over the agents
            skip_stmt = False
            for agent in mapped_stmt.agent_list():
                if agent is None or agent.db_refs.get('TEXT') is None:
                    continue
                agent_text = agent.db_refs.get('TEXT')
                # Look this string up in the grounding map
                # If not in the map, leave agent alone and continue
                try:
                    map_db_refs = self.gm[agent_text]
                except KeyError:
                    continue
                # If it's in the map but it maps to None, then filter out
                # this statement by skipping it
                if map_db_refs is None:
                    # Increase counter if this statement has not already
                    # been skipped via another agent
                    if not skip_stmt:
                        num_skipped += 1
                    logger.debug("Skipping %s" % agent_text)
                    skip_stmt = True
                # If it has a value that's not None, map it and add it
                else:
                    # Otherwise, update the agent's db_refs field
                    gene_name = None
                    map_db_refs = deepcopy(self.gm.get(agent_text))
                    up_id = map_db_refs.get('UP')
                    hgnc_sym = map_db_refs.get('HGNC')
                    if up_id and not hgnc_sym:
                        gene_name = uniprot_client.get_gene_name(up_id)
                        if gene_name:
                            hgnc_id = hgnc_client.get_hgnc_id(gene_name)
                            if hgnc_id:
                                map_db_refs['HGNC'] = hgnc_id
                    elif hgnc_sym and not up_id:
                        # Override the HGNC symbol entry from the grounding
                        # map with an HGNC ID<commit_msg>Don't force web fallback for gene name lookup<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import os
import csv
import sys
import pickle
from copy import deepcopy
from indra.databases import uniprot_client, hgnc_client
from itertools import groupby, chain
from collections import Counter
import logging
from indra.util import read_unicode_csv, write_unicode_csv

logger = logging.getLogger('grounding_mapper')

class GroundingMapper(object):
    def __init__(self, gm):
        self.gm = gm

    def map_agents(self, stmts, do_rename=True):
        # Make a copy of the stmts
        mapped_stmts = []
        num_skipped = 0
        # Iterate over the statements
        for stmt in stmts:
            mapped_stmt = deepcopy(stmt)
            # Iterate over the agents
            skip_stmt = False
            for agent in mapped_stmt.agent_list():
                if agent is None or agent.db_refs.get('TEXT') is None:
                    continue
                agent_text = agent.db_refs.get('TEXT')
                # Look this string up in the grounding map
                # If not in the map, leave agent alone and continue
                try:
                    map_db_refs = self.gm[agent_text]
                except KeyError:
                    continue
                # If it's in the map but it maps to None, then filter out
                # this statement by skipping it
                if map_db_refs is None:
                    # Increase counter if this statement has not already
                    # been skipped via another agent
                    if not skip_stmt:
                        num_skipped += 1
                    logger.debug("Skipping %s" % agent_text)
                    skip_stmt = True
                # If it has a value that's not None, map it and add it
                else:
                    # Otherwise, update the agent's db_refs field
                    gene_name = None
                    map_db_refs = deepcopy(self.gm.get(agent_text))
                    up_id = map_db_refs.get('UP')
                    hgnc_sym = map_db_refs.get('HGNC')
                    if up_id and not hgnc_sym:
                        gene_name = uniprot_client.get_gene_name(up_id, False)
                        if gene_name:
                            hgnc_id = hgnc_client.get_hgnc_id(gene_name)
                            if hgnc_id:
                                map_db_refs['HGNC'] = hgnc_id
                    elif hgnc_sym and not up_id:
                        # Override the HGNC symbol entry from the grounding
                        # map with an HGNC ID
{'fraction_non_alphanumeric': 0.05102232226599137, 'fraction_numerical': 0.0007503282686175201, 'mean_word_length': 1.4904250350303596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3209312', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1371, 'n_words': 461}