<commit_before>import re
from indra.statements import BioContext, RefContext
from indra.preassembler.grounding_mapper.standardize import \
    standardize_db_refs, name_from_grounding


class HypothesisProcessor:
    def __init__(self, annotations, reader=None, grounder=None):
        self.annotations = annotations
        self.statements = []
        if reader is None:
            from indra.sources import reach
            self.reader = reach.process_text
        if grounder is None:
            from gilda import ground
            self.grounder = ground

    def extract_statements(self):
        for annotation in self.annotations:
            stmts = self.stmts_from_annotation(annotation)
            if stmt:
                self.statements.append(stmt)

    def stmts_from_annotation(self, annotation):
        text = annotation.get('text')
        if not text:
            return []
        parts = [t for t in text.split('\n') if t]
        rp = self.reader(parts[0])
        for part in parts[1:]:
            match = re.match(r'Context: (.*)', part)
            if not match:
                continue
            context_txt = match.groups()[0]
            terms = self.grounder(context_txt)
            db_refs = standardize_db_refs({terms[0].db: terms[0].id}) \
                if terms else {}
            db_refs['TEXT'] = context_txt
            standard_name = name_from_grounding(terms[0].db, terms[0].id)
            name = standard_name if standard_name else context_txt
            # TODO: how can we tell what kind of BioContext this is exactly?
            # Disease, organ, cell type, etc?
            context = RefContext(name=name, db_refs=db_refs)
        for stmt in rp.statements:
            stmt.evidence[0].annotations.extend(annotation)
<commit_msg>Extract full context and add to evidence<commit_after>import re
import logging
from indra.statements import BioContext, RefContext
from indra.preassembler.grounding_mapper.standardize import \
    standardize_db_refs, name_from_grounding

logger = logging.getLogger(__name__)


class HypothesisProcessor:
    def __init__(self, annotations, reader=None, grounder=None):
        self.annotations = annotations
        self.statements = []
        if reader is None:
            from indra.sources import reach
            self.reader = reach.process_text
        if grounder is None:
            from gilda import ground
            self.grounder = ground

    def extract_statements(self):
        for annotation in self.annotations:
            stmts = self.stmts_from_annotation(annotation)
            if stmt:
                self.statements.append(stmt)

    def stmts_from_annotation(self, annotation):
        text = annotation.get('text')
        if not text:
            return []
        parts = [t for t in text.split('\n') if t]
        rp = self.reader(parts[0])
        contexts = {}
        for part in parts[1:]:
            match = re.match(r'(.*): (.*)', part)
            if not match:
                continue
            context_type, context_txt = match.groups()[0]
            if context_type not in allowed_contexts:
                logger.warning('Unknown context type %s' % context_type)

            terms = self.grounder(context_txt)
            db_refs = standardize_db_refs({terms[0].db: terms[0].id}) \
                if terms else {}
            db_refs['TEXT'] = context_txt
            standard_name = name_from_grounding(terms[0].db, terms[0].id)
            name = standard_name if standard_name else context_txt
            # TODO: how can we tell what kind of BioContext this is exactly?
            # Disease, organ, cell type, etc?
            context = RefContext(name=name, db_refs=db_refs)
            contexts[allowed_contexts[context_type]] = context
        bio_context = BioContext(**contexts) if contexts else None
        for stmt in rp.statements:
            stmt.evidence[0].annotations.extend(annotation)
            stmt.evidence[0].context = bio_context


allowed_contexts = {
    'Location': 'location',
    'Cell line': 'cell_line',
    'Cell type': 'cell_type',
    'Organ': 'organ',
    'Disease': 'disease',
    'Species': 'species'
}
{'fraction_non_alphanumeric': 0.09201720841300191, 'fraction_numerical': 0.0040630975143403445, 'mean_word_length': 2.2291666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '350756', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1144, 'n_words': 305}