<commit_before>#!/usr/bin/env python

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

import socket, sys

import sys

# Some code so we can use different features without worrying about versions.
PY2 = sys.version_info[0] == 2
if not PY2:
    text_type = str
    string_types = (str,bytes)
    unichr = chr
else:
    text_type = unicode
    string_types = (str, unicode)
    unichr = unichr


def _read_byte (s):
    return s.read(1)

def _read_int (s, terminator=None, init_data=None):
    int_chrs = init_data or []
    while True:
        c = _read_byte(s)
        if not c.isdigit() or c == terminator or not c:
            break
        else:
            int_chrs.append(c)
    return int(''.join(int_chrs))

def _read_bytes (s, n):
    data = StringIO()
    cnt = 0
    while cnt < n:
        m = s.read(n - cnt)
        if not m: raise Exception("Invalid bytestring, unexpected end of input.")
        data.write(m)
        cnt += len(m)
    data.flush()
    # Taking into account that Python3 can't decode strings
    try:
        ret = data.getvalue().decode("UTF-8")
    except AttributeError:
        ret = data.getvalue()
    return ret

def _read_delimiter (s):
    d = _read_byte(s)
    if d.isdigit(): d = _read_int(s, ":", [d])
    return d

def _read_list (s):
    data = []
    while True:
        datum = _read_datum(s)
        if not datum: break
        data.append(datum)
    return data

def _read_map (s):
    i = iter(_read_list(s))
    return dict(zip(i, i))

_read_fns = {"i": _read_int,
             "l": _read_list,
             "d": _read_map,
             "e": lambda _: None,
             # EOF
             None: lambda _: None}

def _read_datum (s):
    delim = _read_delimiter(s)
    if delim:
        return _read_fns.get(delim, lambda s: _read_bytes(s, delim))(s)

def _write_datum (x, out):
    if isinstance(x, string_types):
        # I don't see the need, all tests works fine.
        #x = x.encode("UTF-8")
        out.write(str(len(x)))
        out.write(":")
        out.write(x)
    elif isinstance(x, int):
        out.write("i")
        out.write(str(x))
        out.write("e")
    elif isinstance(x, (list, tuple)):
        out.write("l")
        for v in x: _write_datum(v, out)
        out.write("e")
    elif isinstance(x, dict):
        out.write("d")
        for k, v in x.items():
            _write_datum(k, out)
            _write_datum(v, out)
        out.write("e")
    out.flush()

def encode (v):
    "bencodes the given value, may be a string, integer, list, or dict."
    s = StringIO()
    _write_datum(v, s)
    return s.getvalue()

def decode_file (file):
    while True:
        x = _read_datum(file)
        if not x: break
        yield x<commit_msg>restore UTF-8 encoding of string values<commit_after>#!/usr/bin/env python

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

import socket, sys

import sys

# Some code so we can use different features without worrying about versions.
PY2 = sys.version_info[0] == 2
if not PY2:
    text_type = str
    string_types = (str,bytes)
    unichr = chr
else:
    text_type = unicode
    string_types = (str, unicode)
    unichr = unichr


def _read_byte (s):
    return s.read(1)

def _read_int (s, terminator=None, init_data=None):
    int_chrs = init_data or []
    while True:
        c = _read_byte(s)
        if not c.isdigit() or c == terminator or not c:
            break
        else:
            int_chrs.append(c)
    return int(''.join(int_chrs))

def _read_bytes (s, n):
    data = StringIO()
    cnt = 0
    while cnt < n:
        m = s.read(n - cnt)
        if not m: raise Exception("Invalid bytestring, unexpected end of input.")
        data.write(m)
        cnt += len(m)
    data.flush()
    # Taking into account that Python3 can't decode strings
    try:
        ret = data.getvalue().decode("UTF-8")
    except AttributeError:
        ret = data.getvalue()
    return ret

def _read_delimiter (s):
    d = _read_byte(s)
    if d.isdigit(): d = _read_int(s, ":", [d])
    return d

def _read_list (s):
    data = []
    while True:
        datum = _read_datum(s)
        if not datum: break
        data.append(datum)
    return data

def _read_map (s):
    i = iter(_read_list(s))
    return dict(zip(i, i))

_read_fns = {"i": _read_int,
             "l": _read_list,
             "d": _read_map,
             "e": lambda _: None,
             # EOF
             None: lambda _: None}

def _read_datum (s):
    delim = _read_delimiter(s)
    if delim:
        return _read_fns.get(delim, lambda s: _read_bytes(s, delim))(s)

def _write_datum (x, out):
    if isinstance(x, string_types):
        x = x.encode("UTF-8")
        out.write(str(len(x)))
        out.write(":")
        out.write(x)
    elif isinstance(x, int):
        out.write("i")
        out.write(str(x))
        out.write("e")
    elif isinstance(x, (list, tuple)):
        out.write("l")
        for v in x: _write_datum(v, out)
        out.write("e")
    elif isinstance(x, dict):
        out.write("d")
        for k, v in x.items():
            _write_datum(k, out)
            _write_datum(v, out)
        out.write("e")
    out.flush()

def encode (v):
    "bencodes the given value, may be a string, integer, list, or dict."
    s = StringIO()
    _write_datum(v, s)
    return s.getvalue()

def decode_file (file):
    while True:
        x = _read_datum(file)
        if not x: break
        yield x
{'fraction_non_alphanumeric': 0.12238207976689128, 'fraction_numerical': 0.0034602076124567475, 'mean_word_length': 2.14187643020595, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6564594', 'n_tokens_mistral': 1950, 'n_tokens_neox': 1929, 'n_words': 547}