<commit_before>import struct
from .exceptions import AMQPError
from datetime import datetime, timezone


def rethrow_as(expected_cls, to_throw):
    def decorator(f):
        def wrapper(*args, **kwargs):
            try:
                return f(*args, **kwargs)
            except expected_cls as e:
                raise to_throw from e
        return wrapper
    return decorator


###########################################################
#  Deserialisation
###########################################################


@rethrow_as(struct.error, AMQPError('failed to read an octet'))
def read_octet(stream):
    return _read_octet(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a short'))
def read_short(stream):
    return _read_short(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read an unsigned short'))
def read_unsigned_short(stream):
    return _read_unsigned_short(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a long'))
def read_long(stream):
    return _read_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read an unsigned long'))
def read_unsigned_long(stream):
    return _read_unsigned_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a long long'))
def read_long_long(stream):
    return _read_long_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read an unsigned long long'))
def read_unsigned_long_long(stream):
    return _read_unsigned_long_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a short string'))
def read_short_string(stream):
    return _read_short_string(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a long string'))
def read_long_string(stream):
    return _read_long_string(stream)[0]


@rethrow_as(KeyError, AMQPError('failed to read a table'))
@rethrow_as(struct.error, AMQPError('failed to read a table'))
def read_table(stream):
    return _read_table(stream)[0]


@rethrow_as(KeyError, AMQPError('failed to read an array'))
@rethrow_as(struct.error, AMQPError('failed to read an array'))
def read_array(stream):
    return _read_array(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a boolean'))
def read_bool(stream):
    return _read_bool(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a boolean'))
def read_bools(byte, number_of_bools):
    bits = "{0:b}".format(byte)
    bits = "0" * (number_of_bools - len(bits)) + bits
    return (b == "1" for b in reversed(bits))


@rethrow_as(struct.error, AMQPError('failed to read a boolean'))
def read_timestamp(stream):
    return _read_timestamp(stream)[0]


def qpid_rabbit_mq_table():
    # TODO: fix amqp 0.9.1 compatibility
    # TODO: Add missing types
    TABLE_VALUE_PARSERS = {
        b't': _read_bool,
        b'b': _read_signed_byte,
        b's': _read_short,
        b'I': _read_long,
        b'l': _read_long_long,
        b'S': _read_long_string,
        b'A': _read_array,
        b'V': _read_void,
        b'x': _read_byte_array,
        b'F': _read_table,
        b'T': _read_timestamp
    }
    return TABLE_VALUE_PARSERS
<commit_msg>Changed serialisation to Qpid/RabbitMq codes, added more types<commit_after>import struct
from .exceptions import AMQPError
from datetime import datetime, timezone


def rethrow_as(expected_cls, to_throw):
    def decorator(f):
        def wrapper(*args, **kwargs):
            try:
                return f(*args, **kwargs)
            except expected_cls as e:
                raise to_throw from e
        return wrapper
    return decorator


###########################################################
#  Deserialisation
###########################################################


@rethrow_as(struct.error, AMQPError('failed to read an octet'))
def read_octet(stream):
    return _read_octet(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a short'))
def read_short(stream):
    return _read_short(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read an unsigned short'))
def read_unsigned_short(stream):
    return _read_unsigned_short(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a long'))
def read_long(stream):
    return _read_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read an unsigned long'))
def read_unsigned_long(stream):
    return _read_unsigned_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a long long'))
def read_long_long(stream):
    return _read_long_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read an unsigned long long'))
def read_unsigned_long_long(stream):
    return _read_unsigned_long_long(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a short string'))
def read_short_string(stream):
    return _read_short_string(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a long string'))
def read_long_string(stream):
    return _read_long_string(stream)[0]


@rethrow_as(KeyError, AMQPError('failed to read a table'))
@rethrow_as(struct.error, AMQPError('failed to read a table'))
def read_table(stream):
    return _read_table(stream)[0]


@rethrow_as(KeyError, AMQPError('failed to read an array'))
@rethrow_as(struct.error, AMQPError('failed to read an array'))
def read_array(stream):
    return _read_array(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a boolean'))
def read_bool(stream):
    return _read_bool(stream)[0]


@rethrow_as(struct.error, AMQPError('failed to read a boolean'))
def read_bools(byte, number_of_bools):
    bits = "{0:b}".format(byte)
    bits = "0" * (number_of_bools - len(bits)) + bits
    return (b == "1" for b in reversed(bits))


@rethrow_as(struct.error, AMQPError('failed to read a boolean'))
def read_timestamp(stream):
    return _read_timestamp(stream)[0]

def qpid_rabbit_mq_table():
    # TODO: fix amqp 0.9.1 compatibility
    # TODO: Add missing types
    TABLE_VALUE_PARSERS = {
        b't': _read_bool,
        b'b': _read_signed_byte,
        b's': _read_short,
        b'I': _read_long,
        b'l': _read_long_long,
        b'S': _read_long_string,
        b'A': _read_array,
        b'V': _read_void,
        b'x': _read_byte_array,
        b'F': _read_table,
        b'T': _read_timestamp
    }
    return TABLE_VALUE_PARSERS

{'fraction_non_alphanumeric': 0.14774889029803423, 'fraction_numerical': 0.006024096385542169, 'mean_word_length': 4.553697183098592, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654920', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2083, 'n_words': 495}