<commit_before>class Serializer(object):
    datetime_format = '%s'
    date_format = '%Y-%m%d'

def serialize_datetime(dt, opts):
    v = dt.strftime(opts.get('datetime_format', '%s'))
    try:
        return int(v)

    except ValueError:
        return v

def serialize_date(d, opts):
    return d.strftime(opts.get('date_format', '%Y-%m-%d'))

def serialize_Decimal(d, opts):
    return float(d)

def serialize_set(s, opts):
    return list(s)

class Serializable(object):
    __serializable_args__ = {
        'opts': {
            'date_format': '%Y-%m-%d',
        },
        'serializers': {
            'datetime': serialize_datetime,
            'date': serialize_date,
            'Decimal': serialize_Decimal,
            'set': serialize_set
        }
    }

    @classmethod
    def define_serializer(cls, target_cls, serializer):
        if isinstance(target_cls, type):
            target_cls == target_cls.__name__
        self.__serializable_args__['serializers'][target] = serializer

    @classmethod
    def set_serializer_option(cls, name, value):
        self.__serializable_args__['opts'][name] = value

    def serialize(self, *props):
        obj = {}

        for prop in self.__class__.serializable + props:
            v = getattr(self, prop)
            serializer_name = 'serialize_' + prop
            clsname = type(v).__name__

            if hasattr(v, 'serialize'):
                obj[prop] = v.serialize()

            elif hasattr(self, serializer_name):
                obj[prop] = getattr(self, serializer_name)(v)

            elif clsname in self.__serializable_args__.get('serializers'):
                serializer = self.__serializable_args__.get('serializers').get(clsname)
                obj[prop] = serializer(v, self.__serializable_args__.get('opts'))

            else:
                obj[prop] = getattr(self, prop)

        return obj<commit_msg>Minor improvements to `Serializable`<commit_after>import collections
import numbers

class Serializer(object):
    datetime_format = '%s'
    date_format = '%Y-%m%d'

def serialize_datetime(dt, opts):
    v = dt.strftime(opts.get('datetime_format', '%s'))
    try:
        return int(v)

    except ValueError:
        return v

def serialize_date(d, opts):
    return d.strftime(opts.get('date_format', '%Y-%m-%d'))

def serialize_Decimal(d, opts):
    return float(d)

def serialize_set(s, opts):
    return list(s)

def serialize(v, serializers, opts):
    if isinstance(v, basestring) or isinstance(v, numbers.Number):
        return v

    if hasattr(v, 'serialize'):
        return v.serialize()

    elif isinstance(v, (collections.Sequence, collections.Set)):
        return [serialize(w, serializers, opts) for w in v]

    elif isinstance(v, collections.Mapping):
        return {k: serialize(w, serializers, opts) for k, w in v.items()}

    elif type(v).__name__ in serializers:
        return serializers[type(v).__name__](v, opts)

class Serializable(object):
    __serializable_args__ = {
        'opts': {
            'date_format': '%Y-%m-%d',
        },
        'serializers': {
            'datetime': serialize_datetime,
            'date': serialize_date,
            'Decimal': serialize_Decimal,
            'set': serialize_set
        }
    }

    @classmethod
    def define_serializer(cls, target_cls, serializer):
        if isinstance(target_cls, type):
            target_cls == target_cls.__name__
        self.__serializable_args__['serializers'][target] = serializer

    @classmethod
    def set_serializer_option(cls, name, value):
        self.__serializable_args__['opts'][name] = value

    def serialize(self, fields=None, include=None, exclude=None):
        obj = {}

        if fields is None:
            fields = set(self.__class__.serializable)

            if include is not None:
                fields += set(include)

            if exclude is not None:
                fields -= set(exclude)

        for prop in fields:
            v = getattr(self, prop)
            serializer_name = 'serialize_' + prop
            clsname = type(v).__name__

            if hasattr(self, serializer_name):
                obj[prop] = getattr(self, serializer_name)(v)

            else:
                serializers = self.__serializable_args__['serializers']
                opts = self.__serializable_args__['opts']
                obj[prop] = serialize(v, serializers, opts)

        return obj
{'fraction_non_alphanumeric': 0.11727559765448804, 'fraction_numerical': 0.0, 'mean_word_length': 2.390672782874618, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1661729', 'n_tokens_mistral': 1334, 'n_tokens_neox': 1302, 'n_words': 291}