<commit_before>#cython: language_level=3

# The contents of this file are subject to the BitTorrent Open Source License
# Version 1.1 (the License).  You may not copy or use this file, in either
# source code or executable form, except in compliance with the License.  You
# may obtain a copy of the License at http://www.bittorrent.com/license/.
#
# Software distributed under the License is distributed on an AS IS basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied.  See the License
# for the specific language governing rights and limitations under the
# License.

# Based on https://github.com/karamanolev/bencode3/blob/master/bencode.py

class BTFailure(Exception):
    pass


def decode_int(bytes x, int f):
    f += 1
    new_f = x.index(b'e', f)
    n = int(x[f:new_f])
    if x[f] == ord('-'):
        if x[f + 1] == ord('0'):
            raise ValueError()
    elif x[f] == ord('0') and new_f != f + 1:
        raise ValueError()
    return n, new_f + 1


def decode_string(bytes x, int f):
    colon = x.index(b':', f)
    n = int(x[f:colon])
    if x[f] == ord('0') and colon != f + 1:
        raise ValueError()
    colon += 1
    return x[colon:colon + n], colon + n


def decode_list(bytes x, int f):
    r, f = [], f + 1
    while x[f] != ord('e'):
        v, f = decode_func[x[f]](x, f)
        r.append(v)
    return r, f + 1


def decode_dict(bytes x, int f):
    r, f = {}, f + 1
    while x[f] != ord('e'):
        k, f = decode_string(x, f)
        r[k], f = decode_func[x[f]](x, f)
    return r, f + 1


decode_func = {
    ord('l'): decode_list,
    ord('d'): decode_dict,
    ord('i'): decode_int,
    ord('1'): decode_string,
    ord('2'): decode_string,
    ord('0'): decode_string,
    ord('3'): decode_string,
    ord('4'): decode_string,
    ord('5'): decode_string,
    ord('6'): decode_string,
    ord('7'): decode_string,
    ord('8'): decode_string,
    ord('9'): decode_string,
}


def bdecode(bytes x):
    try:
        r, l = decode_func[x[0]](x, 0)
    except (IndexError, KeyError, ValueError):
        raise BTFailure("not a valid bencoded string")
    if l != len(x):
        raise BTFailure("invalid bencoded value (data after valid prefix)")
    return r


def encode_int(int x, list r):
    r.extend((b'i', str(x).encode(), b'e'))


def encode_bool(x, list r):
    if x:
        encode_int(1, r)
    else:
        encode_int(0, r)


def encode_string(x, list r):
    if isinstance(x, str):<commit_msg>Fix bugs for Python 2<commit_after>#cython: language_level=3

# The contents of this file are subject to the BitTorrent Open Source License
# Version 1.1 (the License).  You may not copy or use this file, in either
# source code or executable form, except in compliance with the License.  You
# may obtain a copy of the License at http://www.bittorrent.com/license/.
#
# Software distributed under the License is distributed on an AS IS basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied.  See the License
# for the specific language governing rights and limitations under the
# License.

# Based on https://github.com/karamanolev/bencode3/blob/master/bencode.py

import sys
IS_PY2 = sys.version[0] == '2'

if IS_PY2:
    END_CHAR = 'e'
else:
    END_CHAR = ord('e')


class BTFailure(Exception):
    pass


def decode_int(bytes x, int f):
    f += 1
    new_f = x.index(b'e', f)
    n = int(x[f:new_f])
    if x[f] == ord('-'):
        if x[f + 1] == ord('0'):
            raise ValueError()
    elif x[f] == ord('0') and new_f != f + 1:
        raise ValueError()
    return n, new_f + 1


def decode_string(bytes x, int f):
    colon = x.index(b':', f)
    n = int(x[f:colon])
    if x[f] == ord('0') and colon != f + 1:
        raise ValueError()
    colon += 1
    return x[colon:colon + n], colon + n


def decode_list(bytes x, int f):
    r, f = [], f + 1
    while x[f] != END_CHAR:
        v, f = decode_func[x[f]](x, f)
        r.append(v)
    return r, f + 1


def decode_dict(bytes x, int f):
    r, f = {}, f + 1
    while x[f] != END_CHAR:
        k, f = decode_string(x, f)
        r[k], f = decode_func[x[f]](x, f)
    return r, f + 1


decode_func = dict()

for func, keys in [
    (decode_list, 'l'),
    (decode_dict, 'd'),
    (decode_int, 'i'),
    (decode_string, [str(x) for x in range(10)])
]:
    for key in keys:
        if IS_PY2:
            decode_func[key] = func
        else:
            decode_func[ord(key)] = func



def bdecode(bytes x):
    try:
        r, l = decode_func[x[0]](x, 0)
    except (IndexError, KeyError, ValueError) as e:
        raise e
#        raise BTFailure("not a valid bencoded string")
    if l != len(x):
        raise BTFailure("invalid bencoded value (data after valid prefix)")
    return r


def encode_int(int x, list r):
    r.extend((b'i', str(x).encode(), b'e'))


def encode_bool(x, list r):
    if x:
        encode_int(1, r)
    else:
        encode_int(0, r)


def encode_string(x, list r):
    if isinstance(x, str):
{'fraction_non_alphanumeric': 0.14363416599516518, 'fraction_numerical': 0.012087026591458501, 'mean_word_length': 2.7079910380881254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1008153', 'n_tokens_mistral': 1785, 'n_tokens_neox': 1746, 'n_words': 581}