<commit_before>import datetime


# Find a json module.
# ===================
# The standard library includes simplejson as json since 2.6, but without the 
# C speedups. So we prefer simplejson if it is available.

try:
    import simplejson as _json
except ImportError:
    try:
        import json as _json
    except ImportError:
        _json = None


# Support pymongo binary #redhotbaby.
# ===================================

try:
    from pymongo.binary import Binary
except ImportError:
    class Binary:
        pass

if _json is not None:
    class FriendlyEncoder(_json.JSONEncoder):
        """Add support for additional types to the default JSON encoder.
        """

        def default(self, obj):
            if isinstance(obj, complex):
                # http://docs.python.org/library/json.html
                out = [obj.real, obj.imag]
            elif isinstance(obj, datetime.datetime):
                # http://stackoverflow.com/questions/455580/
                out = obj.isoformat()
            elif isinstance(obj, Binary):
                out = str(obj).encode('base64')
            else:
                out = super(FriendlyEncoder, self).default(obj)
            return out

def lazy_check(): 
    if _json is None:
        raise LoadError("Neither json nor simplejson was found. Try "
                        "installing simplejson to use dynamic JSON "
                        "resources. See "
                        "http://aspen.io/resources/json/#libraries for "
                        "more information.")

def loads(*a, **kw):
    lazy_check()
    return _json.loads(*a, **kw)

def dumps(*a, **kw):
    lazy_check()
    if 'cls' not in kw:
        kw['cls'] = FriendlyEncoder
    return _json.dumps(*a, **kw)
<commit_msg>Back out ill-advised Binary support.<commit_after>import datetime


# Find a json module.
# ===================
# The standard library includes simplejson as json since 2.6, but without the 
# C speedups. So we prefer simplejson if it is available.

try:
    import simplejson as _json
except ImportError:
    try:
        import json as _json
    except ImportError:
        _json = None


if _json is not None:
    class FriendlyEncoder(_json.JSONEncoder):
        """Add support for additional types to the default JSON encoder.
        """

        def default(self, obj):
            if isinstance(obj, complex):
                # http://docs.python.org/library/json.html
                out = [obj.real, obj.imag]
            elif isinstance(obj, datetime.datetime):
                # http://stackoverflow.com/questions/455580/
                out = obj.isoformat()
            else:
                out = super(FriendlyEncoder, self).default(obj)
            return out

def lazy_check(): 
    if _json is None:
        raise LoadError("Neither json nor simplejson was found. Try "
                        "installing simplejson to use dynamic JSON "
                        "resources. See "
                        "http://aspen.io/resources/json/#libraries for "
                        "more information.")

def loads(*a, **kw):
    lazy_check()
    return _json.loads(*a, **kw)

def dumps(*a, **kw):
    lazy_check()
    if 'cls' not in kw:
        kw['cls'] = FriendlyEncoder
    return _json.dumps(*a, **kw)

{'fraction_non_alphanumeric': 0.11666159000913798, 'fraction_numerical': 0.005482790130977765, 'mean_word_length': 1.9719457013574662, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3045094', 'n_tokens_mistral': 919, 'n_tokens_neox': 875, 'n_words': 283}