<commit_before>from elixir import *

class Annotation(Entity):
    id = Field(Integer, primary_key=True)
    text = Field(UnicodeText)
    ranges = OneToMany('Range')

    def from_dict(self, data):
        if u'ranges' in data:
            ranges = data[u'ranges']
            del data[u'ranges']

        super(Annotation, self).from_dict(data)

        for range_data in ranges:
            if u'id' in range_data:
                range = Range.get(range_data[u'id'])
            else:
                range = Range()

            range.from_dict(range_data)
            self.ranges.append(range)

    def to_dict(self, deep={}, exclude=[]):
        deep.update({'ranges': {}})

        return super(Annotation, self).to_dict(deep, exclude)

    def __repr__(self):
        return '<Annotation %s>' % (self.id)

class Range(Entity):
    id          = Field(Integer, primary_key=True)
    start       = Field(Unicode(255))
    end         = Field(Unicode(255))
    startOffset = Field(Integer)
    endOffset   = Field(Integer)

    annotation  = ManyToOne('Annotation')

    def __repr__(self):
        return '<Range %s>' % (self.id)<commit_msg>Tweaks to __repr__, setup_in_memory helper<commit_after>from elixir import *

def setup_in_memory():
    metadata.bind = "sqlite:///:memory:"
    setup_all(True)

class Annotation(Entity):
    id     = Field(Integer, primary_key=True)
    text   = Field(UnicodeText)
    ranges = OneToMany('Range')

    def from_dict(self, data):
        if u'ranges' in data:
            ranges = data[u'ranges']
            del data[u'ranges']

        super(Annotation, self).from_dict(data)

        for range_data in ranges:
            if u'id' in range_data:
                range = Range.get(range_data[u'id'])
            else:
                range = Range()

            range.from_dict(range_data)
            self.ranges.append(range)

    def to_dict(self, deep={}, exclude=[]):
        deep.update({'ranges': {}})

        return super(Annotation, self).to_dict(deep, exclude)

    def __repr__(self):
        return '<Annotation %s "%s">' % (self.id, self.text)

class Range(Entity):
    id          = Field(Integer, primary_key=True)
    start       = Field(Unicode(255))
    end         = Field(Unicode(255))
    startOffset = Field(Integer)
    endOffset   = Field(Integer)

    annotation  = ManyToOne('Annotation')

    def __repr__(self):
        return '<Range %s@%s %s@%s>' % (self.id, self.start, self.startOffset, self.end, self.endOffset)
{'fraction_non_alphanumeric': 0.13055666800160193, 'fraction_numerical': 0.004805766920304365, 'mean_word_length': 2.3756756756756756, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2438987', 'n_tokens_mistral': 805, 'n_tokens_neox': 782, 'n_words': 167}