<commit_before>import re
import warnings
import pickle

import xml.etree.ElementTree as ET

from indra.statements import *
import indra.databases.hgnc_client as hgnc_client
import indra.databases.uniprot_client as up_client


residue_names = {
    'S': 'Serine',
    'T': 'Threonine',
    'Y': 'Tyrosine',
    'SER': 'Serine',
    'THR': 'Threonine',
    'TYR': 'Tyrosine',
    'SERINE': 'Serine',
    'THREONINE': 'Threonine',
    'TYROSINE': 'Tyrosine'
    }


mod_names = {
    'PHOSPHORYLATION': 'Phosphorylation'
    }


class TripsProcessor(object):
    def __init__(self, xml_string):
        try:
            self.tree = ET.fromstring(xml_string)
        except ET.ParseError:
            print 'Could not parse XML string'
            self.tree = None
            return
        # Get the document ID from the EKB tag. This is the PMC ID when available.
        self.doc_id = self.tree.attrib.get('id')
        # Store all paragraphs and store all sentences in a data structure
        paragraph_tags = self.tree.findall('input/paragraphs/paragraph')
        sentence_tags = self.tree.findall('input/sentences/sentence')
        self.paragraphs = {p.attrib['id']: p.text for p in paragraph_tags}
        self.sentences = {s.attrib['id']: s.text for s in sentence_tags}

        self.statements = []
        self._static_events = self._find_static_events()
        self.get_all_events()
        self.extracted_events = {k:[] for k in self.all_events.keys()}
        print 'All events by type'
        print '------------------'
        for k, v in self.all_events.iteritems():
            print k, len(v)
        print '------------------'

    def get_all_events(self):
        self.all_events = {}
        events = self.tree.findall('EVENT')
        for e in events:
            event_id = e.attrib['id']
            if event_id in self._static_events:
                continue
            event_type = e.find('type').text
            try:
                self.all_events[event_type].append(event_id)
            except KeyError:
                self.all_events[event_type] = [event_id]

    def get_evidence_text(self, event_tag):
        '''
        Pieces of text linked to an EVENT are fragments of a sentence. The
        EVENT refers to the paragraph ID and the "uttnum", which corresponds 
        to a sentence ID. Here we find and return the full sentence from which 
        the event was taken.
        '''
        par_id = event_tag.attrib['paragraph']
        uttnum = event_tag.attrib['uttnum']
        text = event_tag.find('text').text
        sentence = self.sentences[uttnum]
        return sentence

    def get_activations(self):
        act_events = self.tree.findall("EVENT/[type='ONT::ACTIVATE']")
        inact_events = self.tree.findall("EVENT/[type='ONT::DEACTIVATE']")
        inact_events += self.tree.findall("EVENT/[type='ONT::INHIBIT']")
        for event in (act_events + inact_events):
            sentence = self.get_evidence_text(event)
            ev = Evidence(source_api='trips', text=sentence,
                          pmid=self.doc_id)

            # Get the activating agent in the event
            agent = event.find(".//*[@role=':AGENT']")
            if agent is None:
                continue
            agent_id = agent.attrib['id']
            agent_name = self._get_name_by_id(agent_id)
            if agent_name is None:
                warnings.warn(
                    'Skipping activation with missing activator agent')
                continue
            activator_agent = Agent(agent_name)

            # Get the activated agent in the event
            affected = event.find(".//*[@role=':AFFECTED']")
            if affected is None:
                warnings.warn(<commit_msg>Handle missing text/sentence/paragraph in TRIPS processor<commit_after>import re
import warnings
import pickle

import xml.etree.ElementTree as ET

from indra.statements import *
import indra.databases.hgnc_client as hgnc_client
import indra.databases.uniprot_client as up_client


residue_names = {
    'S': 'Serine',
    'T': 'Threonine',
    'Y': 'Tyrosine',
    'SER': 'Serine',
    'THR': 'Threonine',
    'TYR': 'Tyrosine',
    'SERINE': 'Serine',
    'THREONINE': 'Threonine',
    'TYROSINE': 'Tyrosine'
    }


mod_names = {
    'PHOSPHORYLATION': 'Phosphorylation'
    }


class TripsProcessor(object):
    def __init__(self, xml_string):
        try:
            self.tree = ET.fromstring(xml_string)
        except ET.ParseError:
            print 'Could not parse XML string'
            self.tree = None
            return
        # Get the document ID from the EKB tag. This is the PMC ID when available.
        self.doc_id = self.tree.attrib.get('id')
        # Store all paragraphs and store all sentences in a data structure
        paragraph_tags = self.tree.findall('input/paragraphs/paragraph')
        sentence_tags = self.tree.findall('input/sentences/sentence')
        self.paragraphs = {p.attrib['id']: p.text for p in paragraph_tags}
        self.sentences = {s.attrib['id']: s.text for s in sentence_tags}

        self.statements = []
        self._static_events = self._find_static_events()
        self.get_all_events()
        self.extracted_events = {k:[] for k in self.all_events.keys()}
        print 'All events by type'
        print '------------------'
        for k, v in self.all_events.iteritems():
            print k, len(v)
        print '------------------'

    def get_all_events(self):
        self.all_events = {}
        events = self.tree.findall('EVENT')
        for e in events:
            event_id = e.attrib['id']
            if event_id in self._static_events:
                continue
            event_type = e.find('type').text
            try:
                self.all_events[event_type].append(event_id)
            except KeyError:
                self.all_events[event_type] = [event_id]

    def get_evidence_text(self, event_tag):
        '''
        Pieces of text linked to an EVENT are fragments of a sentence. The
        EVENT refers to the paragraph ID and the "uttnum", which corresponds 
        to a sentence ID. Here we find and return the full sentence from which 
        the event was taken.
        '''
        par_id = event_tag.attrib.get('paragraph')
        uttnum = event_tag.attrib.get('uttnum')
        event_text = event_tag.find('text')
        if self.sentences is not None and uttnum is not None:
            sentence = self.sentences[uttnum]
        elif event_text is not None:
            sentence = event_text.text
        else:
            sentence = None
        return sentence

    def get_activations(self):
        act_events = self.tree.findall("EVENT/[type='ONT::ACTIVATE']")
        inact_events = self.tree.findall("EVENT/[type='ONT::DEACTIVATE']")
        inact_events += self.tree.findall("EVENT/[type='ONT::INHIBIT']")
        for event in (act_events + inact_events):
            sentence = self.get_evidence_text(event)
            ev = Evidence(source_api='trips', text=sentence,
                          pmid=self.doc_id)

            # Get the activating agent in the event
            agent = event.find(".//*[@role=':AGENT']")
            if agent is None:
                continue
            agent_id = agent.attrib['id']
            agent_name = self._get_name_by_id(agent_id)
            if agent_name is None:
                warnings.warn(
                    'Skipping activation with missing activator agent')
                continue
            activator_agent = Agent(agent_name)

            # Get the activated agent in the event
            affected = event.find(".//*[@role=':AFFECTED']")
            if affected is None:
                warnings.warn(
{'fraction_non_alphanumeric': 0.11279265295563316, 'fraction_numerical': 0.0, 'mean_word_length': 2.0573349149861606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2065654', 'n_tokens_mistral': 2319, 'n_tokens_neox': 2240, 'n_words': 578}