<commit_before>import struct
import kyoto.conf


class MaxBERPSizeError(Exception):
    pass


def pack(message):
    if not isinstance(message, bytes):
        message = message.encode("utf-8")
    return struct.pack(">I", len(message)) + message


def unpack(message):
    head, tail = message[:4], message[4:]
    if len(head) == 4:
        length, = struct.unpack(">I", head)
    else:
        message = "Incomplete BERP head: received {0} of 4 bytes"
        raise ValueError(message.format(len(head)))
    if length > kyoto.conf.settings.MAX_BERP_SIZE:
        message = "Invalid BERP length: {0}/{1}"
        raise MaxBERPSizeError(
            message.format(kyoto.conf.settings.MAX_BERP_SIZE, length)
        )
    body, tail = tail[:length], tail[length:]
    if len(body) < length:
        message = "Incomplete BERP body: received {0} of {1} bytes"
        raise ValueError(message.format(len(body), length))
    return length, body, tail<commit_msg>Raise MaxBERPSizeError in berp.pack function<commit_after>import struct
import kyoto.conf


class MaxBERPSizeError(Exception):
    pass


def pack(message):
    length = len(message)
    if length > kyoto.conf.settings.MAX_BERP_SIZE:
        message = "Invalid BERP length: {0}/{1}"
        raise MaxBERPSizeError(
            message.format(kyoto.conf.settings.MAX_BERP_SIZE, length)
        )
    if not isinstance(message, bytes):
        message = message.encode("utf-8")
    return struct.pack(">I", length) + message

def unpack(message):
    head, tail = message[:4], message[4:]
    if len(head) == 4:
        length, = struct.unpack(">I", head)
    else:
        message = "Incomplete BERP head: received {0} of 4 bytes"
        raise ValueError(message.format(len(head)))
    if length > kyoto.conf.settings.MAX_BERP_SIZE:
        message = "Invalid BERP length: {0}/{1}"
        raise MaxBERPSizeError(
            message.format(kyoto.conf.settings.MAX_BERP_SIZE, length)
        )
    body, tail = tail[:length], tail[length:]
    if len(body) < length:
        message = "Incomplete BERP body: received {0} of {1} bytes"
        raise ValueError(message.format(len(body), length))
    return length, body, tail
{'fraction_non_alphanumeric': 0.12580054894784995, 'fraction_numerical': 0.010064043915827997, 'mean_word_length': 3.027624309392265, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2805310', 'n_tokens_mistral': 703, 'n_tokens_neox': 686, 'n_words': 173}