<commit_before>import re


def once(func):
    called = False
    value = None
    def f():
        nonlocal called, value
        if not called:
            called = True
            value = func()
        return value
    return f


class ParseError(ValueError):
    pass


_FORMAT_BYTE_SUFFIXES = (" bytes", "K", "M", "G", "T")
def format_byte_count(byte_count, float_precision=1):
    for suffix in _FORMAT_BYTE_SUFFIXES:
        if byte_count < 1024:
            break
        byte_count /= 1024

    if byte_count == int(byte_count):
        float_precision = 0

    return "{value:.{prec}f}{suffix}".format(
        value=byte_count, prec=float_precision, suffix=suffix
    )


_PARSE_BYTE_SUFFIXES = {
    "b": 1,
    "k": 1024,
    "m": 1024**2,<commit_msg>pylint control<commit_after>import re


def once(func):
    called = False
    value = None
    def f():
        nonlocal called, value
        if not called:
            called = True
            value = func()
        return value
    return f


class ParseError(ValueError):
    pass


_FORMAT_BYTE_SUFFIXES = (" bytes", "K", "M", "G", "T")
def format_byte_count(byte_count, float_precision=1):
    # shut pylint up: given that the iterated var never changes, the loop
    # always iterates through at least one entry
    #pylint: disable=undefined-loop-variable
    for suffix in _FORMAT_BYTE_SUFFIXES:
        if byte_count < 1024:
            break
        byte_count /= 1024

    if byte_count == int(byte_count):
        float_precision = 0

    return "{value:.{prec}f}{suffix}".format(
        value=byte_count, prec=float_precision, suffix=suffix
    )


_PARSE_BYTE_SUFFIXES = {
    "b": 1,
    "k": 1024,
    "m": 1024**2,
{'fraction_non_alphanumeric': 0.10758377425044091, 'fraction_numerical': 0.023515579071134628, 'mean_word_length': 2.4383838383838383, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2048847', 'n_tokens_mistral': 605, 'n_tokens_neox': 545, 'n_words': 147}