<commit_before>#!/usr/bin/python -OOOO
# vim: set fileencoding=utf8 shiftwidth=4 tabstop=4 textwidth=80 foldmethod=marker :
# Copyright (c) 2010, <NAME>. All rights reserved.
# Copyright (c) 2015, <NAME>. All rights reserved.
# For licensing, see LICENSE file included in the package.
"""
Base codec functions for bson.
"""
import struct
import warnings
from datetime import datetime
from abc import ABCMeta, abstractmethod
try:
    from io import BytesIO as StringIO
except ImportError:
    from cStringIO import StringIO

import calendar
import pytz
from binascii import b2a_hex

from six import integer_types, iterkeys, text_type, PY3
from six.moves import xrange


class MissingClassDefinition(ValueError):
    def __init__(self, class_name):
        super(MissingClassDefinition, self).__init__("No class definition for class %s" % (class_name,))


class UnknownSerializerError(ValueError):
    pass


class MissingTimezoneWarning(RuntimeWarning):
    def __init__(self, *args):
        args = list(args)
        if len(args) < 1:
            args.append("Input datetime object has no tzinfo, assuming UTC.")
        super(MissingTimezoneWarning, self).__init__(*args)


class TraversalStep(object):
    def __init__(self, parent, key):
        self.parent = parent
        self.key = key


class BSONCoding(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def bson_encode(self):
       pass

    @abstractmethod
    def bson_init(self, raw_values):
        pass


classes = {}


def import_class(cls):
    if not issubclass(cls, BSONCoding):
        return

    global classes
    classes[cls.__name__] = cls


def import_classes(*args):
    for cls in args:
       import_class(cls)


def import_classes_from_modules(*args):
    for module in args:
        for item in module.__dict__:
            if hasattr(item, "__new__") and hasattr(item, "__name__"):
                import_class(item)


def encode_object(obj, traversal_stack, generator_func, on_unknown=None):
    values = obj.bson_encode()
    class_name = obj.__class__.__name__
    values["$$__CLASS_NAME__$$"] = class_name
    return encode_document(values, traversal_stack, obj, generator_func, on_unknown)


def encode_object_element(name, value, traversal_stack, generator_func, on_unknown):
    return b"\x03" + encode_cstring(name) + encode_object(value, traversal_stack, generator_func=generator_func, on_unknown=on_unknown)


class _EmptyClass(object):
    pass


def decode_object(raw_values):
    global classes
    class_name = raw_values["$$__CLASS_NAME__$$"]
    cls = None
    try:
        cls = classes[class_name]
    except KeyError:
        raise MissingClassDefinition(class_name)

    retval = _EmptyClass()
    retval.__class__ = cls
    alt_retval = retval.bson_init(raw_values)
    return alt_retval or retval


def encode_string(value):
    value = value.encode("utf-8")
    length = len(value)
    return struct.pack("<i%dsb" % (length,), length + 1, value, 0)


def decode_string(data, base):
    length = struct.unpack("<i", data[base:base + 4])[0]
    value = data[base + 4: base + 4 + length - 1]
    value = value.decode("utf-8")
    return base + 4 + length, value


def encode_cstring(value):
    if "\x00" in value:
        raise ValueError("Element names may not include NUL bytes.")
        # A NUL byte is used to delimit our string, accepting one would cause
        # our string to terminate early.
    if not isinstance(value, bytes):
        value = value.encode("utf-8")
    return value + b"\x00"


def encode_binary(value):
    length = len(value)
    return struct.pack("<ib", length, 0) + value


def decode_binary(data, base):
    length, binary_type = struct.unpack("<ib", data[base:base + 5])
    return base + 5 + length, data[base + 5:base + 5 + length]


def encode_double(value):
    return struct.pack("<d", value)


def decode_double(data, base):
    return base + 8, struct.unpack("<d", data[base: base + 8])[0]


ELEMENT_TYPES = {
    0x01: "double",
    0x02: "string",
    0x03: "document",
    0x04: "array",
    0x05: "binary",
    0x07: "object_id",
    0x08: "boolean",
    0x09: "UTCdatetime",
    0x0A: "none",
    0x10: "int32",
    0x12: "int64"
}


def encode_double_element(name, value):
    return b"\x01" + encode_cstring(name) + encode_double(value)


def encode_string_element(name, value):
    return b"\x02" + encode_cstring(name) + encode_string(value)


def encode_value(name, value, buf, traversal_stack, generator_func, on_unknown=None):
    if isinstance(value, integer_types):
        if not PY3 and isinstance(value, long):
            buf.write(encode_int64_element(name, value))
        else:
            if value < -0x80000000 or value > 0x7fffffff:
                buf.write(encode_int64_element(name, value))
            else:
                buf.write(encode_int32_element(name, value))
    elif isinstance(value, float):
        buf.write(encode_double_element(name, value))
    elif isinstance(value, text_type):
        buf.write(encode_string_element(name, value))
    elif isinstance(value, str) or isinstance(value, bytes):
        buf.write(encode_binary_element(name, value))
    elif isinstance(value, bool):
        buf.write(encode_boolean_element(name, value))
    elif isinstance(value, datetime):
        buf.write(encode_UTCdatetime_element(name, value))<commit_msg>Fix bug where strings were encoded as binary.<commit_after>#!/usr/bin/python -OOOO
# vim: set fileencoding=utf8 shiftwidth=4 tabstop=4 textwidth=80 foldmethod=marker :
# Copyright (c) 2010, <NAME>. All rights reserved.
# Copyright (c) 2015, <NAME>. All rights reserved.
# For licensing, see LICENSE file included in the package.
"""
Base codec functions for bson.
"""
import struct
import warnings
from datetime import datetime
from abc import ABCMeta, abstractmethod
try:
    from io import BytesIO as StringIO
except ImportError:
    from cStringIO import StringIO

import calendar
import pytz
from binascii import b2a_hex

from six import integer_types, iterkeys, text_type, PY3
from six.moves import xrange


class MissingClassDefinition(ValueError):
    def __init__(self, class_name):
        super(MissingClassDefinition, self).__init__("No class definition for class %s" % (class_name,))


class UnknownSerializerError(ValueError):
    pass


class MissingTimezoneWarning(RuntimeWarning):
    def __init__(self, *args):
        args = list(args)
        if len(args) < 1:
            args.append("Input datetime object has no tzinfo, assuming UTC.")
        super(MissingTimezoneWarning, self).__init__(*args)


class TraversalStep(object):
    def __init__(self, parent, key):
        self.parent = parent
        self.key = key


class BSONCoding(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def bson_encode(self):
       pass

    @abstractmethod
    def bson_init(self, raw_values):
        pass


classes = {}


def import_class(cls):
    if not issubclass(cls, BSONCoding):
        return

    global classes
    classes[cls.__name__] = cls


def import_classes(*args):
    for cls in args:
       import_class(cls)


def import_classes_from_modules(*args):
    for module in args:
        for item in module.__dict__:
            if hasattr(item, "__new__") and hasattr(item, "__name__"):
                import_class(item)


def encode_object(obj, traversal_stack, generator_func, on_unknown=None):
    values = obj.bson_encode()
    class_name = obj.__class__.__name__
    values["$$__CLASS_NAME__$$"] = class_name
    return encode_document(values, traversal_stack, obj, generator_func, on_unknown)


def encode_object_element(name, value, traversal_stack, generator_func, on_unknown):
    return b"\x03" + encode_cstring(name) + encode_object(value, traversal_stack, generator_func=generator_func, on_unknown=on_unknown)


class _EmptyClass(object):
    pass


def decode_object(raw_values):
    global classes
    class_name = raw_values["$$__CLASS_NAME__$$"]
    cls = None
    try:
        cls = classes[class_name]
    except KeyError:
        raise MissingClassDefinition(class_name)

    retval = _EmptyClass()
    retval.__class__ = cls
    alt_retval = retval.bson_init(raw_values)
    return alt_retval or retval


def encode_string(value):
    value = value.encode("utf-8")
    length = len(value)
    return struct.pack("<i%dsb" % (length,), length + 1, value, 0)


def decode_string(data, base):
    length = struct.unpack("<i", data[base:base + 4])[0]
    value = data[base + 4: base + 4 + length - 1]
    value = value.decode("utf-8")
    return base + 4 + length, value


def encode_cstring(value):
    if "\x00" in value:
        raise ValueError("Element names may not include NUL bytes.")
        # A NUL byte is used to delimit our string, accepting one would cause
        # our string to terminate early.
    if not isinstance(value, bytes):
        value = value.encode("utf-8")
    return value + b"\x00"


def encode_binary(value):
    length = len(value)
    return struct.pack("<ib", length, 0) + value


def decode_binary(data, base):
    length, binary_type = struct.unpack("<ib", data[base:base + 5])
    return base + 5 + length, data[base + 5:base + 5 + length]


def encode_double(value):
    return struct.pack("<d", value)


def decode_double(data, base):
    return base + 8, struct.unpack("<d", data[base: base + 8])[0]


ELEMENT_TYPES = {
    0x01: "double",
    0x02: "string",
    0x03: "document",
    0x04: "array",
    0x05: "binary",
    0x07: "object_id",
    0x08: "boolean",
    0x09: "UTCdatetime",
    0x0A: "none",
    0x10: "int32",
    0x12: "int64"
}


def encode_double_element(name, value):
    return b"\x01" + encode_cstring(name) + encode_double(value)


def encode_string_element(name, value):
    return b"\x02" + encode_cstring(name) + encode_string(value)

def _is_string(value):
    if isinstance(value, text_type):
        return True
    elif isinstance(value, str) or isinstance(value, bytes):
        try:
            unicode(value, errors='strict')
            return True
        except:
            pass
    return False

def encode_value(name, value, buf, traversal_stack, generator_func, on_unknown=None):
    if isinstance(value, integer_types):
        if not PY3 and isinstance(value, long):
            buf.write(encode_int64_element(name, value))
        else:
            if value < -0x80000000 or value > 0x7fffffff:
                buf.write(encode_int64_element(name, value))
            else:
                buf.write(encode_int32_element(name, value))
    elif isinstance(value, float):
        buf.write(encode_double_element(name, value))
    elif _is_string(value):
        buf.write(encode_string_element(name, value))
    elif isinstance(value, str) or isinstance(value, bytes):
        buf.write(encode_binary_element(name, value))
    elif isinstance(value, bool):
        buf.write(encode_boolean_element(name, value))
    elif isinstance(value, datetime):
        buf.write(encode_UTCdatetime_element(name, value))
{'fraction_non_alphanumeric': 0.1077245018651624, 'fraction_numerical': 0.018014739332180875, 'mean_word_length': 3.4719283970707893, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1490546', 'n_tokens_mistral': 3579, 'n_tokens_neox': 3414, 'n_words': 917}