<commit_before>from collections import OrderedDict as odict


class Field:
    def __init__(self, tag_name, default=None):
        self.tag_name = tag_name
        self.value = None
        self.default = default

        # This value will be set by metaclass
        self.name = None

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        else:
            return obj.__dict__.get(self.name, self.default)

    def __set__(self, obj, value):
        self.value = value
        obj.__dict__[self.name] = value

    def as_dict(self, obj):
        return {self.tag_name: self.__get__(obj)}

    def __repr__(self):
        return '<Field name={!r}>'.format(self.name)


class Nested:
    def __init__(self, tag_name, model, unwrap=False, many=False):
        assert (
            not (many and unwrap),
            'If `many` is `True `unwrap` must be set to `False`'
        )
        self.tag_name = tag_name
        self.model = model
        self.unwrap = unwrap
        self.many = many

        self.value = None
        self.default = [model()] if many else model()

        # This value will be set by metaclass
        self.name = None

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        else:
            return obj.__dict__.get(self.name, self.default)

    def __set__(self, obj, value):
        self.value = value
        obj.__dict__[self.name] = value

    def as_dict(self, obj):
        data = self.__get__(obj)
        if not self.many and self.unwrap:
            return {self.tag_name: data.as_dict()[self.model.root_tag]}
        elif not self.many and not self.unwrap:
            return {self.tag_name: data.as_dict()}
        elif self.many:
            return {self.tag_name: [item.as_dict() for item in data]}

    def __repr__(self):
        return '<Nested name={!r} type={!r}>'.format(self.name, self.model)


class ModelMeta(type):
    def __new__(mcs, clsname, bases, clsdict):
        fields = []
        for name, value in clsdict.items():
            if isinstance(value, (Field, Nested)):
                value.name = name  # Init `Field.name`
                fields.append(value)
        clsdict['fields'] = fields
        return type.__new__(mcs, clsname, bases, clsdict)

    @classmethod
    def __prepare__(mcs, clsname, bases):
        return odict()


class Model(metaclass=ModelMeta):
    root_tag = 'root'

    def __init__(self, **kwargs):
        for name, value in kwargs.items():
            setattr(self, name, value)

    def as_dict(self):
        tags = []
        for k in self.fields:
            value = k.as_dict(self)
            tags.append(value)

        return {self.root_tag: tags}
<commit_msg>Extract a BaseField<commit_after>from collections import OrderedDict as odict


class BaseField:
    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        else:
            return obj.__dict__.get(self.name, self.default)

    def __set__(self, obj, value):
        self.value = value
        obj.__dict__[self.name] = value


class Field(BaseField):
    def __init__(self, tag_name, default=None):
        self.tag_name = tag_name
        self.value = None
        self.default = default

        # This value will be set by metaclass
        self.name = None

    def as_dict(self, obj):
        return {self.tag_name: self.__get__(obj)}

    def __repr__(self):
        return '<Field name={!r}>'.format(self.name)


class Nested(BaseField):
    def __init__(self, tag_name, model, unwrap=False, many=False):
        assert (
            not (many and unwrap),
            'If `many` is `True `unwrap` must be set to `False`'
        )
        self.tag_name = tag_name
        self.model = model
        self.unwrap = unwrap
        self.many = many

        self.value = None
        self.default = [model()] if many else model()

        # This value will be set by metaclass
        self.name = None

    def as_dict(self, obj):
        data = self.__get__(obj)
        if not self.many and self.unwrap:
            return {self.tag_name: data.as_dict()[self.model.root_tag]}
        elif not self.many and not self.unwrap:
            return {self.tag_name: data.as_dict()}
        elif self.many:
            return {self.tag_name: [item.as_dict() for item in data]}

    def __repr__(self):
        return '<Nested name={!r} type={!r}>'.format(self.name, self.model)


class ModelMeta(type):
    def __new__(mcs, clsname, bases, clsdict):
        fields = []
        for name, value in clsdict.items():
            if isinstance(value, (Field, Nested)):
                value.name = name  # Init `Field.name`
                fields.append(value)
        clsdict['fields'] = fields
        return type.__new__(mcs, clsname, bases, clsdict)

    @classmethod
    def __prepare__(mcs, clsname, bases):
        return odict()


class Model(metaclass=ModelMeta):
    root_tag = 'root'

    def __init__(self, **kwargs):
        for name, value in kwargs.items():
            setattr(self, name, value)

    def as_dict(self):
        tags = []
        for k in self.fields:
            value = k.as_dict(self)
            tags.append(value)

        return {self.root_tag: tags}

{'fraction_non_alphanumeric': 0.11007781362687417, 'fraction_numerical': 0.0, 'mean_word_length': 2.065735892961024, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '179450', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1609, 'n_words': 446}