<commit_before>"""Python 2/3 compat layer leftovers."""

import decimal as _decimal
import math as _math
import warnings
from contextlib import redirect_stderr, redirect_stdout
from io import BytesIO
from io import StringIO as UnicodeIO
from types import SimpleNamespace

warnings.warn(
    "The py23 module has been deprecated and will be removed in a future release. "
    "Please update your code.",
    DeprecationWarning,
)

__all__ = [
    "basestring",
    "bytechr",
    "byteord",
    "BytesIO",
    "bytesjoin",
    "open",
    "Py23Error",
    "range",
    "RecursionError",
    "round",
    "SimpleNamespace",
    "StringIO",
    "strjoin",
    "Tag",
    "tobytes",
    "tostr",
    "tounicode",
    "unichr",
    "unicode",
    "UnicodeIO",
    "xrange",
    "zip",
]


class Py23Error(NotImplementedError):
    pass


RecursionError = RecursionError
StringIO = UnicodeIO

basestring = str
isclose = _math.isclose
isfinite = _math.isfinite
open = open
range = range
round = round3 = round
unichr = chr
unicode = str
zip = zip


def bytechr(n):
    return bytes([n])


def byteord(c):
    return c if isinstance(c, int) else ord(c)


def strjoin(iterable, joiner=""):
    return tostr(joiner).join(iterable)


def tobytes(s, encoding="ascii", errors="strict"):
    if not isinstance(s, bytes):
        return s.encode(encoding, errors)
    else:
        return s


def tounicode(s, encoding="ascii", errors="strict"):
    if not isinstance(s, unicode):
        return s.decode(encoding, errors)
    else:
        return s

<commit_msg>[py23] Change tobytes() to accept bytearray<commit_after>"""Python 2/3 compat layer leftovers."""

import decimal as _decimal
import math as _math
import warnings
from contextlib import redirect_stderr, redirect_stdout
from io import BytesIO
from io import StringIO as UnicodeIO
from types import SimpleNamespace

warnings.warn(
    "The py23 module has been deprecated and will be removed in a future release. "
    "Please update your code.",
    DeprecationWarning,
)

__all__ = [
    "basestring",
    "bytechr",
    "byteord",
    "BytesIO",
    "bytesjoin",
    "open",
    "Py23Error",
    "range",
    "RecursionError",
    "round",
    "SimpleNamespace",
    "StringIO",
    "strjoin",
    "Tag",
    "tobytes",
    "tostr",
    "tounicode",
    "unichr",
    "unicode",
    "UnicodeIO",
    "xrange",
    "zip",
]


class Py23Error(NotImplementedError):
    pass


RecursionError = RecursionError
StringIO = UnicodeIO

basestring = str
isclose = _math.isclose
isfinite = _math.isfinite
open = open
range = range
round = round3 = round
unichr = chr
unicode = str
zip = zip


def bytechr(n):
    return bytes([n])


def byteord(c):
    return c if isinstance(c, int) else ord(c)


def strjoin(iterable, joiner=""):
    return tostr(joiner).join(iterable)


def tobytes(s, encoding="ascii", errors="strict"):
    if isinstance(s, str):
        return s.encode(encoding, errors)
    else:
        return s


def tounicode(s, encoding="ascii", errors="strict"):
    if not isinstance(s, unicode):
        return s.decode(encoding, errors)
    else:
        return s


{'fraction_non_alphanumeric': 0.1141025641025641, 'fraction_numerical': 0.00641025641025641, 'mean_word_length': 3.3167358229598896, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1521389', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1016, 'n_words': 302}