<commit_before>import collections
import datetime
import types

try:
    import json
except ImportError:
    import simplejson as json
import re


class MarathonJsonEncoder(json.JSONEncoder):
    """Custom JSON encoder for Marathon object serialization."""

    def default(self, obj):
        if hasattr(obj, 'json_repr'):
            return self.default(obj.json_repr())

        if isinstance(obj, datetime.datetime):
            return obj.isoformat()

        if isinstance(obj, collections.Iterable) and not isinstance(obj, str):
            try:
                return {k: self.default(v) for k,v in obj.items()}
            except AttributeError:
                return [self.default(e) for e in obj]

        return obj


class MarathonMinimalJsonEncoder(json.JSONEncoder):
    """Custom JSON encoder for Marathon object serialization."""

    def default(self, obj):
        if hasattr(obj, 'json_repr'):
            return self.default(obj.json_repr(minimal=True))

        if isinstance(obj, datetime.datetime):
            return obj.isoformat()

        if isinstance(obj, collections.Iterable) and not isinstance(obj, str):
            try:
                return {k: self.default(v) for k,v in obj.items() if (v or v == False)}
            except AttributeError:
                return [self.default(e) for e in obj if (e or e == False)]

        return obj


def to_camel_case(snake_str):
    words = snake_str.split('_')
    return words[0] + ''.join(w.title() for w in words[1:])


def to_snake_case(camel_str):
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel_str)<commit_msg>Make the json decoder handle unicode string properly in both python 3 and python 2. Fixes #50<commit_after>import collections
import datetime
import types

try:
    import json
except ImportError:
    import simplejson as json
import re


def is_stringy(obj):
    return isinstance(obj, str) or isinstance(obj, unicode)


class MarathonJsonEncoder(json.JSONEncoder):
    """Custom JSON encoder for Marathon object serialization."""

    def default(self, obj):
        if hasattr(obj, 'json_repr'):
            return self.default(obj.json_repr())

        if isinstance(obj, datetime.datetime):
            return obj.isoformat()

        if isinstance(obj, collections.Iterable) and not is_stringy(obj):
            try:
                return {k: self.default(v) for k,v in obj.items()}
            except AttributeError:
                return [self.default(e) for e in obj]

        return obj


class MarathonMinimalJsonEncoder(json.JSONEncoder):
    """Custom JSON encoder for Marathon object serialization."""

    def default(self, obj):
        if hasattr(obj, 'json_repr'):
            return self.default(obj.json_repr(minimal=True))

        if isinstance(obj, datetime.datetime):
            return obj.isoformat()

        if isinstance(obj, collections.Iterable) and not is_stringy(obj):
            try:
                return {k: self.default(v) for k,v in obj.items() if (v or v == False)}
            except AttributeError:
                return [self.default(e) for e in obj if (e or e == False)]

        return obj


def to_camel_case(snake_str):
    words = snake_str.split('_')
    return words[0] + ''.join(w.title() for w in words[1:])


def to_snake_case(camel_str):
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', camel_str)
{'fraction_non_alphanumeric': 0.11250373022978215, 'fraction_numerical': 0.0041778573560131305, 'mean_word_length': 2.6198704103671706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2591753', 'n_tokens_mistral': 1017, 'n_tokens_neox': 964, 'n_words': 297}