<commit_before>import sys
import codecs

from nimp.utilities.logging import *

class Logger:
    #---------------------------------------------------------------------------
    def __init__(self, format, verbose):
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        self._format = format
        self._verbose = verbose

    #---------------------------------------------------------------------------
    def log_message(self, log_level, message_format, *args):
        formatted_message = self._format.format_message(log_level, message_format, *args)
        if log_level != LOG_LEVEL_VERBOSE or self._verbose:
            if log_level == LOG_LEVEL_ERROR or log_level == LOG_LEVEL_WARNING:
                self._stderr.buffer.write(formatted_message.encode('cp850', errors='ignore')) #FIXME : I imagine it's gonna break on Linux
                self._stderr.flush()
            else:
                self._stdout.buffer.write(formatted_message.encode('cp850', errors='ignore')) #FIXME : I imagine it's gonna break on Linux
                self._stdout.flush()

    #---------------------------------------------------------------------------
    def start_progress(self,
                       total,
                       position_formatter,
                       speed_formatter,
                       total_formatter,
                       step_name_formatter,
                       template,
                       width):<commit_msg>Encode logs in UTF-8 because this is $CURRENT_YEAR.<commit_after>import sys
import codecs

from nimp.utilities.logging import *

class Logger:

    def __init__(self, format, verbose):
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        self._format = format
        self._verbose = verbose


    def log_message(self, log_level, message_format, *args):
        line = self._format.format_message(log_level, message_format, *args)
        if line[-1:] != '\n':
            line += '\n'

        if log_level != LOG_LEVEL_VERBOSE or self._verbose:
            stream = self._stdout
            if log_level in [LOG_LEVEL_ERROR, LOG_LEVEL_WARNING]:
                stream = self._stderr

            stream.buffer.write(line.encode('utf-8', errors='ignore'))
            stream.flush()


    def start_progress(self,
                       total,
                       position_formatter,
                       speed_formatter,
                       total_formatter,
                       step_name_formatter,
                       template,
                       width):
{'fraction_non_alphanumeric': 0.16387699493966523, 'fraction_numerical': 0.0035033086804203972, 'mean_word_length': 1.3906976744186046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4701973', 'n_tokens_mistral': 638, 'n_tokens_neox': 589, 'n_words': 142}