<commit_before>import re
import functools
import datetime
import calendar
import operator
from struct import pack

import six
from six.moves import reduce

from pyhessian.protocol import Call, Object, Remote, Binary

from .utils import toposort_flatten
from .data_types import long

# Implementation of Hessian 1.0.2 serialization
#   see: http://hessian.caucho.com/doc/hessian-1.0-spec.xtp

RETURN_TYPES = {
    type(None): 'null',
    bool: 'bool',
    int: 'int',
    long: 'long',
    float: 'double',
    datetime.datetime: 'date',
    Binary: 'binary',
    Remote: 'remote',
    Call: 'call',
    str: 'string',
    six.text_type: 'string',
    list: 'list',
    tuple: 'list',
    dict: 'map',
}


class bound_function_wrapper(object):

    def __init__(self, wrapped):
        self.wrapped = wrapped

    def __call__(self, *args, **kwargs):
        return self.wrapped(*args, **kwargs)


class encoder_method_wrapper(object):

    def __init__(self, wrapped, data_type):
        self.wrapped = wrapped
        self.data_type = data_type
        functools.update_wrapper(self, wrapped)

    def __call__(self, *args, **kwargs):
        return self.wrapped(*args, **kwargs)

    def __get__(self, instance, owner):
        wrapped = self.wrapped.__get__(instance, owner)
        return bound_function_wrapper(wrapped)


def encoder_for(data_type, version=1, return_type=None):
    return_type = RETURN_TYPES.get(data_type)

    def wrap(f):
        @functools.wraps(f)
        def wrapper(*args):
            if return_type:
                return return_type, f(*args)
            else:
                return f(*args)
        return encoder_method_wrapper(wrapper, data_type)

    return wrap


def sort_mro(encoders):
    """
    Sort encoders so that subclasses precede the types they extend when
    checking isinstance(value, encoder_data_type). This way, the encoder
    will (for instance) check whether isinstance(value, bool) before it
    checks isinstance(value, int), which is necessary because bool is a
    subclass of int.
    """
    type_encoders = dict([[e.data_type, e] for e in encoders])
    mro_dict = dict([[k, set(k.mro()[1:])] for k in type_encoders.keys()])
    sorted_classes = reversed(toposort_flatten(mro_dict, sort=False))
    return [type_encoders[cls] for cls in sorted_classes if cls in type_encoders]


class EncoderBase(type):

    def __new__(cls, name, bases, attrs):
        encoders = []
        for base in bases:
            if hasattr(base, '_hessian_encoders'):
                encoders.extend(base._hessian_encoders)
        for k, v in six.iteritems(attrs):
            if isinstance(v, encoder_method_wrapper):
                encoders.append(v)
        attrs['_hessian_encoders'] = sort_mro(encoders)
        return super(EncoderBase, cls).__new__(cls, name, bases, attrs)


@six.add_metaclass(EncoderBase)
class Encoder(object):

    def __init__(self):
        self._refs = []

    def _encode(self, obj):
        encoder = None
        for e in self._hessian_encoders:
            if isinstance(obj, e.data_type):
                encoder = e
                break
        if not encoder:
            raise TypeError("pyhessian.encoder cannot serialize %s" % (type(obj),))
        return encoder(self, obj)

    def add_ref(self, obj):
        """
        Add an object to the reference list. Returns None if the object
        has not yet been referenced, or the encoding of the reference
        if it has.
        """
        if id(obj) in self._refs:
            return pack('>cl', b'R', self._refs.index(id(obj)))

        self._refs.append(id(obj))

    def encode(self, obj):
        return self._encode(obj)[1]

    def encode_arg(self, obj):
        return self._encode(obj)

    @encoder_for(type(None))
    def encode_null(self, _):
        return 'N'

    @encoder_for(bool)
    def encode_boolean(self, value):
        if value:
            return 'T'
        else:
            return 'F'

    @encoder_for(int)
    def encode_int(self, value):
        return pack('>cl', b'I', value)

    @encoder_for(long)
    def encode_long(self, value):
        return pack('>cq', b'L', value)

    @encoder_for(float)
    def encode_double(self, value):
        return pack('>cd', b'D', value)<commit_msg>Python 3.x: encoder used plain strings instead of bytes<commit_after>import re
import functools
import datetime
import calendar
import operator
from struct import pack

import six
from six.moves import reduce

from pyhessian.protocol import Call, Object, Remote, Binary

from .utils import toposort_flatten
from .data_types import long

# Implementation of Hessian 1.0.2 serialization
#   see: http://hessian.caucho.com/doc/hessian-1.0-spec.xtp

RETURN_TYPES = {
    type(None): 'null',
    bool: 'bool',
    int: 'int',
    long: 'long',
    float: 'double',
    datetime.datetime: 'date',
    Binary: 'binary',
    Remote: 'remote',
    Call: 'call',
    str: 'string',
    six.text_type: 'string',
    list: 'list',
    tuple: 'list',
    dict: 'map',
}


class bound_function_wrapper(object):

    def __init__(self, wrapped):
        self.wrapped = wrapped

    def __call__(self, *args, **kwargs):
        return self.wrapped(*args, **kwargs)


class encoder_method_wrapper(object):

    def __init__(self, wrapped, data_type):
        self.wrapped = wrapped
        self.data_type = data_type
        functools.update_wrapper(self, wrapped)

    def __call__(self, *args, **kwargs):
        return self.wrapped(*args, **kwargs)

    def __get__(self, instance, owner):
        wrapped = self.wrapped.__get__(instance, owner)
        return bound_function_wrapper(wrapped)


def encoder_for(data_type, version=1, return_type=None):
    return_type = RETURN_TYPES.get(data_type)

    def wrap(f):
        @functools.wraps(f)
        def wrapper(*args):
            if return_type:
                return return_type, f(*args)
            else:
                return f(*args)
        return encoder_method_wrapper(wrapper, data_type)

    return wrap


def sort_mro(encoders):
    """
    Sort encoders so that subclasses precede the types they extend when
    checking isinstance(value, encoder_data_type). This way, the encoder
    will (for instance) check whether isinstance(value, bool) before it
    checks isinstance(value, int), which is necessary because bool is a
    subclass of int.
    """
    type_encoders = dict([[e.data_type, e] for e in encoders])
    mro_dict = dict([[k, set(k.mro()[1:])] for k in type_encoders.keys()])
    sorted_classes = reversed(toposort_flatten(mro_dict, sort=False))
    return [type_encoders[cls] for cls in sorted_classes if cls in type_encoders]


class EncoderBase(type):

    def __new__(cls, name, bases, attrs):
        encoders = []
        for base in bases:
            if hasattr(base, '_hessian_encoders'):
                encoders.extend(base._hessian_encoders)
        for k, v in six.iteritems(attrs):
            if isinstance(v, encoder_method_wrapper):
                encoders.append(v)
        attrs['_hessian_encoders'] = sort_mro(encoders)
        return super(EncoderBase, cls).__new__(cls, name, bases, attrs)


@six.add_metaclass(EncoderBase)
class Encoder(object):

    def __init__(self):
        self._refs = []

    def _encode(self, obj):
        encoder = None
        for e in self._hessian_encoders:
            if isinstance(obj, e.data_type):
                encoder = e
                break
        if not encoder:
            raise TypeError("pyhessian.encoder cannot serialize %s" % (type(obj),))
        return encoder(self, obj)

    def add_ref(self, obj):
        """
        Add an object to the reference list. Returns None if the object
        has not yet been referenced, or the encoding of the reference
        if it has.
        """
        if id(obj) in self._refs:
            return pack('>cl', b'R', self._refs.index(id(obj)))

        self._refs.append(id(obj))

    def encode(self, obj):
        return self._encode(obj)[1]

    def encode_arg(self, obj):
        return self._encode(obj)

    @encoder_for(type(None))
    def encode_null(self, _):
        return b'N'

    @encoder_for(bool)
    def encode_boolean(self, value):
        if value:
            return b'T'
        else:
            return b'F'

    @encoder_for(int)
    def encode_int(self, value):
        return pack('>cl', b'I', value)

    @encoder_for(long)
    def encode_long(self, value):
        return pack('>cq', b'L', value)

    @encoder_for(float)
    def encode_double(self, value):
        return pack('>cd', b'D', value)
{'fraction_non_alphanumeric': 0.11076133846333217, 'fraction_numerical': 0.001982045004080681, 'mean_word_length': 2.6720890410958904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852243', 'n_tokens_mistral': 2774, 'n_tokens_neox': 2671, 'n_words': 729}