<commit_before>#!/usr/bin/env python3

from collections import namedtuple
import re
from lxml import etree as ET
import skll
from collections import Counter
from pprint import pprint
from functools import reduce
import itertools

class InputError(Exception):
    pass

def reverse_find_from_index(iterable, match_function, index):
    for x in iterable[index::-1]:
        if match_function(x):
            return x

class AnnotationFile:
    """Given a GATE XML annotation file, returns an AnnotationFile object.
    """
    def __init__(self, filename):
        self.filename = filename
        self.tree = ET.parse(self.filename)
        self.root = self.tree.getroot()
        self._text_with_nodes = self.root.find(".//TextWithNodes")
        self.annotation_set_names = [
            annotation_set.get("Name")
            for annotation_set
            in self.root.findall(".//AnnotationSet")
        ]

    def get_text(self):
        return ''.join(
            ( x for x in self._text_with_nodes.itertext() )
        )

    def iter_annotations(self):
        annotations = self.root.findall(
            ".//Annotation"
        )
        for x in annotations:
            yield Annotation(x)

class Annotation:
    def __init__(self, annotation):
        self._annotation = annotation

        annotation_set_name = annotation.getparent().get("Name")
        if annotation_set_name:
            self._annotation_set = annotation_set_name
        else: self._annotation_set = ""

        self._type = annotation.get("Type")
        self._id = int(annotation.get("Id"))
        self._start_node = int(annotation.get("StartNode"))
        self._end_node = int(annotation.get("EndNode"))
        self._continuations = []

        if self._type == "Attribution":
            self._caused_event_id = None
            for feature in self.get_features():
                if feature.get_name().lower() == "caused_event":
                    self._caused_event_id = int(feature.get_value().split()[0])
                    break

    def add_continuation(self, annotation):
        self._continuations.append(annotation)

    def iter_spans(self):
        return itertools.chain( [self], ( x for x in self._continuations ) )

    def get_features(self):
        return [ Feature(x) for x in self._annotation if x.tag == "Feature" ]

    def get_text(self, text_with_nodes):
        return "".join(
            x.tail for x in text_with_nodes
            if int(x.get("id")) in range(self._start_node, self._end_node)
        )

    def get_concatenated_text(self, text_with_nodes, separator):
        return separator.join(
            x.get_text(text_with_nodes) for x in self.iter_spans()
        )

    def get_char_set(self):
        return frozenset(
            range(
                self._start_node,
                self._end_node<commit_msg>Use strings to store ID's, not numbers<commit_after>#!/usr/bin/env python3

from collections import namedtuple
import re
from lxml import etree as ET
import skll
from collections import Counter
from pprint import pprint
from functools import reduce
import itertools

class InputError(Exception):
    pass

def reverse_find_from_index(iterable, match_function, index):
    for x in iterable[index::-1]:
        if match_function(x):
            return x

class AnnotationFile:
    """Given a GATE XML annotation file, returns an AnnotationFile object.
    """
    def __init__(self, filename):
        self.filename = filename
        self.tree = ET.parse(self.filename)
        self.root = self.tree.getroot()
        self._text_with_nodes = self.root.find(".//TextWithNodes")
        self.annotation_set_names = [
            annotation_set.get("Name")
            for annotation_set
            in self.root.findall(".//AnnotationSet")
        ]

    def get_text(self):
        return ''.join(
            ( x for x in self._text_with_nodes.itertext() )
        )

    def iter_annotations(self):
        annotations = self.root.findall(
            ".//Annotation"
        )
        for x in annotations:
            yield Annotation(x)

class Annotation:
    def __init__(self, annotation):
        self._annotation = annotation

        annotation_set_name = annotation.getparent().get("Name")
        if annotation_set_name:
            self._annotation_set = annotation_set_name
        else: self._annotation_set = ""

        self._type = annotation.get("Type")
        self._id = annotation.get("Id")
        self._start_node = int(annotation.get("StartNode"))
        self._end_node = int(annotation.get("EndNode"))
        self._continuations = []

        if self._type == "Attribution":
            self._caused_event_id = None
            for feature in self.get_features():
                if feature.get_name().lower() == "caused_event":
                    self._caused_event_id = feature.get_value().split()[0]
                    break

    def add_continuation(self, annotation):
        self._continuations.append(annotation)

    def iter_spans(self):
        return itertools.chain( [self], ( x for x in self._continuations ) )

    def get_features(self):
        return [ Feature(x) for x in self._annotation if x.tag == "Feature" ]

    def get_text(self, text_with_nodes):
        return "".join(
            x.tail for x in text_with_nodes
            if int(x.get("id")) in range(self._start_node, self._end_node)
        )

    def get_concatenated_text(self, text_with_nodes, separator):
        return separator.join(
            x.get_text(text_with_nodes) for x in self.iter_spans()
        )

    def get_char_set(self):
        return frozenset(
            range(
                self._start_node,
                self._end_node
{'fraction_non_alphanumeric': 0.09931806259835635, 'fraction_numerical': 0.0010491344640671447, 'mean_word_length': 2.293033966609096, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '427213', 'n_tokens_mistral': 1685, 'n_tokens_neox': 1610, 'n_words': 387}