<commit_before>import re
import json
import uuid
import pickle
import logging

from collections import defaultdict

from bioagents import group_and_sort_statements, make_string_from_sort_key
from indra import get_config
from indra.statements import stmts_to_json, Agent, get_all_descendants
from indra.sources import indra_db_rest as idbr

from indra.assemblers.html import HtmlAssembler
from indra.assemblers.graph import GraphAssembler
from indra.assemblers.english.assembler import _join_list

logger = logging.getLogger('MSA')

mod_map = {'demethylate': 'Demethylation',
           'methylate': 'Methylation',
           'phosphorylate': 'Phosphorylation',
           'dephosphorylate': 'Dephosphorylation',
           'ubiquitinate': 'Ubiquitination',
           'deubiquitinate': 'Deubiquitination',
           'inhibit': 'Inhibition',
           'activate': 'Activation'}

DB_REST_URL = get_config('INDRA_DB_REST_URL')


class EntityError(ValueError):
    pass


class StatementQuery(object):
    """This is an object that encapsulates the information used to make a query.

    Parameters
    ----------
    subj, obj : Agent or None
        The subject and object of the causal mechanism to be found. If an
        Agent, the db_refs will be used as grounding. If there is no subject or
        object, subj or obj may be None respectively.
    agents : list[Agent]
        A list of Agents like subj and obj, but without the implication of an
        order. Each element will be treated as above.
    verb : str or None
        A string describing a type of interaction between the subject, object,
        and/or agents. Must be mappable to a subclass of Statement.
    settings : dict
        A dictionary containing other parameters used by the
        IndraDbRestProcessor.
    valid_name_spaces : list[str] or None
        A list of name spaces that are allowed as grounding, in order of
        preference (most preferable first). If None, the default list will be
        used: ['HGNC', 'FPLX', 'CHEBI', 'TEXT'].
    """
    def __init__(self, subj, obj, agents, verb, settings,
                 valid_name_spaces=None):
        self.entities = {}
        self._ns_keys = valid_name_spaces if valid_name_spaces is not None \
            else ['HGNC', 'FPLX', 'CHEBI', 'TEXT']
        self.subj = subj
        self.subj_key = self.get_key(subj)
        self.obj = obj
        self.obj_key = self.get_key(obj)
        self.agents = agents
        self.agent_keys = [self.get_key(e) for e in agents]

        self.verb = verb
        if verb in mod_map.keys():
            self.stmt_type = mod_map[verb]
        else:
            self.stmt_type = verb

        self.settings = settings
        if not self.subj_key and not self.obj_key and not self.agent_keys:
            raise EntityError("Did not get any usable entity constraints!")
        return

    def get_key(self, agent):
        """If the entity is not already an agent, form an agent."""
        if agent is None:
            return None

        # Get the key
        for key in self._ns_keys:
            if key in agent.db_refs.keys():
                dbn = key
                dbi = agent.db_refs[key]
                break
        else:
            raise EntityError("Could not get valid grounding (%s) for %s."
                              % (', '.join(self._ns_keys), agent))
        self.entities[agent.name] = (dbn, dbi)

        return '%s@%s' % (dbi, dbn)


class StatementFinder(object):
    def __init__(self, *args, **kwargs):
        self._block_default = kwargs.pop('block_default', True)
        self.query = self._regularize_input(*args, **kwargs)
        self._processor = self._make_processor()
        self._statements = None
        self._sample = []
        return

    def _regularize_input(self, *args, **kwargs):
        """Convert arbitrary input in subject, object, agents, and verb."""
        raise NotImplementedError

    def _make_processor(self):<commit_msg>Relax valid keys, implement prioritization<commit_after>import re
import json
import uuid
import pickle
import logging

from collections import defaultdict

from bioagents import group_and_sort_statements, make_string_from_sort_key
from indra import get_config
from indra.statements import stmts_to_json, Agent, get_all_descendants
from indra.sources import indra_db_rest as idbr

from indra.assemblers.html import HtmlAssembler
from indra.assemblers.graph import GraphAssembler
from indra.assemblers.english.assembler import _join_list

logger = logging.getLogger('MSA')

mod_map = {'demethylate': 'Demethylation',
           'methylate': 'Methylation',
           'phosphorylate': 'Phosphorylation',
           'dephosphorylate': 'Dephosphorylation',
           'ubiquitinate': 'Ubiquitination',
           'deubiquitinate': 'Deubiquitination',
           'inhibit': 'Inhibition',
           'activate': 'Activation'}

DB_REST_URL = get_config('INDRA_DB_REST_URL')


class EntityError(ValueError):
    pass


class StatementQuery(object):
    """This is an object that encapsulates the information used to make a query.

    Parameters
    ----------
    subj, obj : Agent or None
        The subject and object of the causal mechanism to be found. If an
        Agent, the db_refs will be used as grounding. If there is no subject or
        object, subj or obj may be None respectively.
    agents : list[Agent]
        A list of Agents like subj and obj, but without the implication of an
        order. Each element will be treated as above.
    verb : str or None
        A string describing a type of interaction between the subject, object,
        and/or agents. Must be mappable to a subclass of Statement.
    settings : dict
        A dictionary containing other parameters used by the
        IndraDbRestProcessor.
    valid_name_spaces : list[str] or None
        A list of name spaces in order of preference (most preferable first).
        The special key !OTHER! can be used as a place holder for any other
        grounding not explicitly mentioned. If None, the default list will be
        used: ['HGNC', 'FPLX', 'CHEBI', '!OTHER!', 'TEXT'].
    """
    def __init__(self, subj, obj, agents, verb, settings,
                 valid_name_spaces=None):
        self.entities = {}
        self._ns_keys = valid_name_spaces if valid_name_spaces is not None \
            else ['HGNC', 'FPLX', 'CHEBI', '!OTHER!', 'TEXT']
        self.subj = subj
        self.subj_key = self.get_key(subj)
        self.obj = obj
        self.obj_key = self.get_key(obj)
        self.agents = agents
        self.agent_keys = [self.get_key(e) for e in agents]

        self.verb = verb
        if verb in mod_map.keys():
            self.stmt_type = mod_map[verb]
        else:
            self.stmt_type = verb

        self.settings = settings
        if not self.subj_key and not self.obj_key and not self.agent_keys:
            raise EntityError("Did not get any usable entity constraints!")
        return

    def get_key(self, agent):
        """If the entity is not already an agent, form an agent."""
        if agent is None:
            return None

        dbn, dbi = None, None
        # Iterate over all the keys in order
        for idx, key in enumerate(self._ns_keys):
            # If we hit OTHER, we need to make sure we only return on keys
            # that don't appear in the tail part of the list
            if key == '!OTHER!':
                low_priority_keys = self._ns_keys[idx+1:]
                for key, value in agent.db_refs.items():
                    if key not in low_priority_keys:
                        dbn, dbi = key, value
                        break
            # Otherwise, this is a regular key, and we just look for it in
            # the Agent's db_refs
            elif key in agent.db_refs:
                dbn, dbi = key, agent.db_refs[key]
                break

        if dbn is None:
            raise EntityError(("Could not get valid grounding (%s) for %s "
                               "with db_refs=%s.")
                               % (', '.join(self._ns_keys), agent,
                                  agent.db_refs))
        self.entities[agent.name] = (dbn, dbi)

        return '%s@%s' % (dbi, dbn)


class StatementFinder(object):
    def __init__(self, *args, **kwargs):
        self._block_default = kwargs.pop('block_default', True)
        self.query = self._regularize_input(*args, **kwargs)
        self._processor = self._make_processor()
        self._statements = None
        self._sample = []
        return

    def _regularize_input(self, *args, **kwargs):
        """Convert arbitrary input in subject, object, agents, and verb."""
        raise NotImplementedError

    def _make_processor(self):
{'fraction_non_alphanumeric': 0.08509423186750428, 'fraction_numerical': 0.0001142204454597373, 'mean_word_length': 2.3267477203647418, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1416366', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2439, 'n_words': 849}