<commit_before>
import json
try:
    import msgpack
except ImportError:
    pass

from decimal import Decimal
from datetime import date, datetime, time

class JsonEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Decimal):
            return float(obj)
        if isinstance(obj, (datetime, time,)):
            return obj.replace(microsecond=0).isoformat()
        if isinstance(obj, date):
            return obj.isoformat()
        return super(JsonEncoder, self).default(obj)

class Engine(object):
    # The list of content types we match
    CONTENT_TYPES = []
    def dumps(self, data):
        '''How to serialiser an object'''
        raise NotImplementedError
    
    def loads(self, data):
        '''How to deserialise a string'''
        raise NotImplementedError

class JsonEngine(Engine):
    CONTENT_TYPES = ['application/json',]
    def dumps(self, data):
        return json.dumps(data, cls=JsonEncoder)
    def loads(self, data):
        return json.loads(data)

class MsgPackEngine(Engine):
    CONTENT_TYPES = ['application/x-msgpack',]
    def dumps(self, data):
        return msgpack.dumps(data)
    def loads(self, data):
        return msgpack.loads(data)<commit_msg>Removed auto-casting as we now have type-casting fields<commit_after>
import json
try:
    import msgpack
except ImportError:
    pass

from decimal import Decimal
from datetime import date, datetime, time

class Engine(object):
    # The list of content types we match
    CONTENT_TYPES = []
    def dumps(self, data):
        '''How to serialiser an object'''
        raise NotImplementedError
    
    def loads(self, data):
        '''How to deserialise a string'''
        raise NotImplementedError

class JsonEngine(Engine):
    CONTENT_TYPES = ['application/json',]
    def dumps(self, data):
        return json.dumps(data)
    def loads(self, data):
        return json.loads(data)

class MsgPackEngine(Engine):
    CONTENT_TYPES = ['application/x-msgpack',]
    def dumps(self, data):
        return msgpack.dumps(data)
    def loads(self, data):
        return msgpack.loads(data)
{'fraction_non_alphanumeric': 0.10014170996693435, 'fraction_numerical': 0.0004723665564478035, 'mean_word_length': 2.9887005649717513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3655459', 'n_tokens_mistral': 627, 'n_tokens_neox': 627, 'n_words': 185}