<commit_before>#!/usr/bin/env python
# coding=utf-8
"""
Balanced ternary binary encoding
============

Tools for encoding balanced ternary data into binary formats and back again.

The encoded scheme used here uses 8-bit segments to represent 5-trit segments.
Each 5-trit segment is mapped to an 8-bit binary value which corresponds to its
value as an unsigned integer.

So, for example, consider the trit sequence:

    0-00+

This sequence interpreted as a signed number is decimal -70, and as an unsigned
number decimal 51.  We encode it using the byte value 51, or hex 0x33, which
gives:

    0011 0011

If a trit sequence is not evenly divisible into 5-trit segments, the final
segment is padded to 5 trits by adding '-' trits to the left.
"""
from . import trit, integer


def encode(source):
    result = b''
    for i in xrange(0, len(source), 5):
        value = integer.UInt(source[i:i+5], 5)
        result += chr(value)
    return result


def decode(binary):
    trits = trit.Trits('')
    for i in xrange(len(binary)):
        value = ord(binary[i])
        if value > 242:
            raise ValueError(
                    "Invalid byte at position {}: {:#02x}".format(i, value))
        trits += integer.UInt(value, 5)
    return trits<commit_msg>Use 'range' rather than 'xrange' for Python 3 compatibility.<commit_after>#!/usr/bin/env python
# coding=utf-8
"""
Balanced ternary binary encoding
============

Tools for encoding balanced ternary data into binary formats and back again.

The encoded scheme used here uses 8-bit segments to represent 5-trit segments.
Each 5-trit segment is mapped to an 8-bit binary value which corresponds to its
value as an unsigned integer.

So, for example, consider the trit sequence:

    0-00+

This sequence interpreted as a signed number is decimal -70, and as an unsigned
number decimal 51.  We encode it using the byte value 51, or hex 0x33, which
gives:

    0011 0011

If a trit sequence is not evenly divisible into 5-trit segments, the final
segment is padded to 5 trits by adding '-' trits to the left.
"""
from . import trit, integer


def encode(source):
    result = b''
    for i in range(0, len(source), 5):
        value = integer.UInt(source[i:i+5], 5)
        result += chr(value)
    return result


def decode(binary):
    trits = trit.Trits('')
    for i in range(len(binary)):
        value = ord(binary[i])
        if value > 242:
            raise ValueError(
                    "Invalid byte at position {}: {:#02x}".format(i, value))
        trits += integer.UInt(value, 5)
    return trits
{'fraction_non_alphanumeric': 0.09762738234150137, 'fraction_numerical': 0.029171528588098017, 'mean_word_length': 3.0440251572327046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4453576', 'n_tokens_mistral': 821, 'n_tokens_neox': 751, 'n_words': 331}