<commit_before>from collections import Hashable, OrderedDict
import csv
from io import BytesIO
from StringIO import StringIO

from tri.declarative.declarative import declarative_member, declarative
from tri.struct import FrozenStruct, Struct


__version__ = '0.1.0'


class PRESENT(object):
    def __init__(self, attribute_name):
        self.attribute_name = attribute_name

MISSING = object()


@declarative_member
class TokenAttribute(FrozenStruct):
    def __init__(self, **kwargs):
        kwargs.setdefault('description')
        kwargs.setdefault('default', MISSING)
        kwargs.setdefault('value')
        kwargs.setdefault('optional_value')
        super(TokenAttribute, self).__init__(**kwargs)


@declarative_member
@declarative(TokenAttribute, 'token_attributes', add_init_kwargs=False)
class Token(FrozenStruct):
<commit_msg>Fix imports<commit_after>from collections import Hashable, OrderedDict
import csv
from io import BytesIO
from StringIO import StringIO

from tri.declarative import creation_ordered, declarative
from tri.struct import FrozenStruct, Struct


__version__ = '0.1.0'


class PRESENT(object):
    def __init__(self, attribute_name):
        self.attribute_name = attribute_name

MISSING = object()


@creation_ordered
class TokenAttribute(FrozenStruct):
    def __init__(self, **kwargs):
        kwargs.setdefault('description')
        kwargs.setdefault('default', MISSING)
        kwargs.setdefault('value')
        kwargs.setdefault('optional_value')
        super(TokenAttribute, self).__init__(**kwargs)


@creation_ordered
@declarative(TokenAttribute, 'token_attributes', add_init_kwargs=False)
class Token(FrozenStruct):

{'fraction_non_alphanumeric': 0.09194711538461539, 'fraction_numerical': 0.003605769230769231, 'mean_word_length': 5.143911439114391, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455402', 'n_tokens_mistral': 493, 'n_tokens_neox': 476, 'n_words': 111}