<commit_before>import base64
import binascii
from datetime import datetime, timedelta, tzinfo
import json
import re


__all__ = ['EncodeError', 'ParseError', 'loads', 'dumps']


INT_MAX = 2 ** 63 - 1
INT_MIN = -(2 ** 63)
UINT_MAX = 2 ** 64 - 1

text_type = type(u'')


class EncodeError(ValueError):
    """Exception class when trying to encode bad data."""


class ParseError(ValueError):
    """Exception class for tjson-specific parse errors."""


def raise_on_duplicate_keys(pairs):
    d = {}
    for k, v in pairs:
        if k in d:
            raise ParseError("Duplicate key: '%s'" % k)
        d[k] = v
    return d


def loads(s, **kwargs):
    kwargs['object_pairs_hook'] = raise_on_duplicate_keys
    obj = json.loads(s, **kwargs)
    if not isinstance(obj, (list, dict)):
        raise ParseError('Toplevel elements other than object or array are '
                         'disallowed')
    return unpack(obj)


def dumps(obj):
    if not isinstance(obj, (list, dict)):
        raise ParseError('Toplevel elements other than object or array are '
                         'disallowed')
    return json.dumps(pack(obj))


def pack(obj):
    if isinstance(obj, list):
        return [pack(e) for e in obj]
    elif isinstance(obj, dict):
        rv = {}
        for k in obj:
            if not isinstance(k, (text_type, bytes)):
                raise EncodeError('Object member names must be text or binary')
            rv[pack(k)] = pack(obj[k])
        return rv
    elif isinstance(obj, text_type):
        return u's:' + obj
    elif isinstance(obj, bytes):
        return u'b64:' + (base64.urlsafe_b64encode(obj).rstrip(b'=')
                          .decode('ascii'))
    elif isinstance(obj, float):
        return obj
    elif isinstance(obj, int):
        if obj < 0:
            return u'i:%d' % obj
        else:
            return u'u:%d' % obj


def unpack(obj):<commit_msg>Raise EncodeError, not ParseError, when encoding<commit_after>import base64
import binascii
from datetime import datetime, timedelta, tzinfo
import json
import re


__all__ = ['EncodeError', 'ParseError', 'loads', 'dumps']


INT_MAX = 2 ** 63 - 1
INT_MIN = -(2 ** 63)
UINT_MAX = 2 ** 64 - 1

text_type = type(u'')


class EncodeError(ValueError):
    """Exception class when trying to encode bad data."""


class ParseError(ValueError):
    """Exception class for tjson-specific parse errors."""


def raise_on_duplicate_keys(pairs):
    d = {}
    for k, v in pairs:
        if k in d:
            raise ParseError("Duplicate key: '%s'" % k)
        d[k] = v
    return d


def loads(s, **kwargs):
    kwargs['object_pairs_hook'] = raise_on_duplicate_keys
    obj = json.loads(s, **kwargs)
    if not isinstance(obj, (list, dict)):
        raise ParseError('Toplevel elements other than object or array are '
                         'disallowed')
    return unpack(obj)


def dumps(obj):
    if not isinstance(obj, (list, dict)):
        raise EncodeError('Toplevel elements other than object or array are '
                          'disallowed')
    return json.dumps(pack(obj))


def pack(obj):
    if isinstance(obj, list):
        return [pack(e) for e in obj]
    elif isinstance(obj, dict):
        rv = {}
        for k in obj:
            if not isinstance(k, (text_type, bytes)):
                raise EncodeError('Object member names must be text or binary')
            rv[pack(k)] = pack(obj[k])
        return rv
    elif isinstance(obj, text_type):
        return u's:' + obj
    elif isinstance(obj, bytes):
        return u'b64:' + (base64.urlsafe_b64encode(obj).rstrip(b'=')
                          .decode('ascii'))
    elif isinstance(obj, float):
        return obj
    elif isinstance(obj, int):
        if obj < 0:
            return u'i:%d' % obj
        else:
            return u'u:%d' % obj


def unpack(obj):
{'fraction_non_alphanumeric': 0.11969815248503773, 'fraction_numerical': 0.010408534998698933, 'mean_word_length': 2.386784140969163, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7329907', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1161, 'n_words': 374}