<commit_before>import re
from collections import deque, defaultdict
from itertools import chain
from delphin._exceptions import TdlError, TdlParsingError

_list_head = 'FIRST'
_list_tail = 'REST'
_diff_list_list = 'LIST'
_diff_list_last = 'LAST'

class TdlDefinition(object):

    __slots__ = ['supertypes', '_avm']

    def __init__(self, supertypes=None, features=None):
        self.supertypes = list(supertypes or [])
        self._avm = {}
        if isinstance(features, dict):
            features = features.items()
        for feat, val in list(features or []):
            self[feat] = val

    def __repr__(self):
        stlist = ' & '.join(self.supertypes)
        return '<TdlDefinition object{} at {}>'.format(
            '' if not self.supertypes else ' ({})'.format(stlist),
            id(self)
        )

    def __setitem__(self, key, val):
        try:
            first, rest = key.split('.', 1)
        except ValueError:
            self._avm[key.upper()] = val
        else:
            first = first.upper()  # features are case-insensitive
            try:
                subdef = self._avm[first]
            except KeyError:
                subdef = self._avm.setdefault(first, TdlDefinition())
            subdef[rest] = val

    def __getitem__(self, key):
        try:
            first, rest = key.split('.', 1)
        except ValueError:
            return self._avm[key.upper()]
        else:
            return self._avm[first.upper()][rest]

    def features(self):
        for feat, val in self._avm.items():
            try:
                if val.supertypes or not val._avm:
                    yield (feat, val)
                else:
                    for subfeat, subval in val.features():
                        yield ('{}.{}'.format(feat, subfeat), subval)
            except AttributeError:
                yield (feat, val)

    def local_constraints(self):
        for feat, val in self._avm.items():
            try:
                if val.supertypes and not val._avm:
                    yield (feat, val)
                else:
                    for subfeat, subval in val.features():
                        yield ('{}.{}'.format(feat, subfeat), subval)
            except AttributeError:
                yield (feat, val)



class TdlType(TdlDefinition):
    def __init__(self, identifier, supertypes=None, features=None,
                 coreferences=None, comment=None):
        TdlDefinition.__init__(self, supertypes, features)
        self.identifier = identifier
        self.coreferences = list(coreferences or [])
        self.comment = comment

    @classmethod
    def from_tdldef(cls, identifier, tdldef, coreferences=None, comment=None):
        self = cls(identifier, coreferences=coreferences, comment=comment)
        self._avm = tdldef._avm
        self.supertypes = tdldef.supertypes
        return self

    def __repr__(self):
        info = []
        if self.supertypes:
            info.append('supertypes={}'.format(self.supertypes))
        if self.features:
            info.append('features={}'
                        .format(sorted(self.features.items())))
        if self.comment:
            info.append('comment={}'.format(self.comment))
        post = '' if not info else ', '.join([''] + info)
        return "<TdlType object ('{}'{}) at {}>".format(
            self.identifier, post, id(self)
        )

<commit_msg>Fix repr string for TdlType.<commit_after>import re
from collections import deque, defaultdict
from itertools import chain
from delphin._exceptions import TdlError, TdlParsingError

_list_head = 'FIRST'
_list_tail = 'REST'
_diff_list_list = 'LIST'
_diff_list_last = 'LAST'

class TdlDefinition(object):

    __slots__ = ['supertypes', '_avm']

    def __init__(self, supertypes=None, features=None):
        self.supertypes = list(supertypes or [])
        self._avm = {}
        if isinstance(features, dict):
            features = features.items()
        for feat, val in list(features or []):
            self[feat] = val

    def __repr__(self):
        stlist = ' & '.join(self.supertypes)
        return '<TdlDefinition object{} at {}>'.format(
            '' if not self.supertypes else ' ({})'.format(stlist),
            id(self)
        )

    def __setitem__(self, key, val):
        try:
            first, rest = key.split('.', 1)
        except ValueError:
            self._avm[key.upper()] = val
        else:
            first = first.upper()  # features are case-insensitive
            try:
                subdef = self._avm[first]
            except KeyError:
                subdef = self._avm.setdefault(first, TdlDefinition())
            subdef[rest] = val

    def __getitem__(self, key):
        try:
            first, rest = key.split('.', 1)
        except ValueError:
            return self._avm[key.upper()]
        else:
            return self._avm[first.upper()][rest]

    def features(self):
        for feat, val in self._avm.items():
            try:
                if val.supertypes or not val._avm:
                    yield (feat, val)
                else:
                    for subfeat, subval in val.features():
                        yield ('{}.{}'.format(feat, subfeat), subval)
            except AttributeError:
                yield (feat, val)

    def local_constraints(self):
        for feat, val in self._avm.items():
            try:
                if val.supertypes and not val._avm:
                    yield (feat, val)
                else:
                    for subfeat, subval in val.features():
                        yield ('{}.{}'.format(feat, subfeat), subval)
            except AttributeError:
                yield (feat, val)



class TdlType(TdlDefinition):
    def __init__(self, identifier, supertypes=None, features=None,
                 coreferences=None, comment=None):
        TdlDefinition.__init__(self, supertypes, features)
        self.identifier = identifier
        self.coreferences = list(coreferences or [])
        self.comment = comment

    @classmethod
    def from_tdldef(cls, identifier, tdldef, coreferences=None, comment=None):
        self = cls(identifier, coreferences=coreferences, comment=comment)
        self._avm = tdldef._avm
        self.supertypes = tdldef.supertypes
        return self

    def __repr__(self):
        stlist = ' & '.join(self.supertypes)
        return "<TdlType object '{}'{} at {}>".format(
            self.identifier,
            '' if not self.supertypes else ' ({})'.format(stlist),
            id(self)
        )


{'fraction_non_alphanumeric': 0.11464677222898904, 'fraction_numerical': 0.0006090133982947625, 'mean_word_length': 1.8698121450415028, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3656613', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1848, 'n_words': 462}