<commit_before>from __future__ import absolute_import
import sys
import time
import datetime
import six


class GeneralLogger():
    def __init__(self, logFD=None, job_backend=None, error=False):
        self.error = error
        self.job_backend = job_backend

        self.terminal = sys.stdout if error is False else sys.stderr
        self.logFD = logFD

    def get_time(self):
        ts = time.time()
        st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')

        return st + '.' + str(ts % 1)[2:6]

    def get_line(self, line):
        if line:
            return "[%s] %s\n" % (self.get_time(), line)

        return line

    def flush(self):
        if self.logFD:
            self.logFD.flush()

        self.terminal.flush()

    def write(self, message):

        # if message == '\n':
        #     return

        message = six.text_type(message)

        # if not self.error:
        #     message = self.get_line(message)

        self.terminal.write(message)

        # check for \b ASCII Backspace (BS) in message for a period of time, so a progress bar for example does not shred
        # our connection to job_backend or blow up local loggin file.

        if self.logFD:
            self.logFD.write(message)

        if self.job_backend:
            self.job_backend.write_log(message)<commit_msg>Throttle loggin message that are being sent to local file and server (e.g. progress bars)<commit_after>from __future__ import absolute_import
import sys
import time
import datetime
import six
from threading import Timer

class GeneralLogger():
    def __init__(self, logFD=None, job_backend=None, error=False):
        self.error = error
        self.job_backend = job_backend
        self.buffer = ''
        self.last_timer = None


        self.terminal = sys.stdout if error is False else sys.stderr
        self.logFD = logFD

    def get_time(self):
        ts = time.time()
        st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')

        return st + '.' + str(ts % 1)[2:6]

    def get_line(self, line):
        if line:
            return "[%s] %s\n" % (self.get_time(), line)

        return line

    def flush(self):
        if self.logFD:
            self.logFD.flush()

        self.terminal.flush()

    def send_to_buffer(self):
        self.last_timer = None

        if self.buffer:
            if self.logFD:
                self.logFD.write(self.buffer)

            if self.job_backend:
                self.job_backend.write_log(self.buffer)

        self.buffer = ''

    def write(self, message):

        # if message == '\n':
        #     return

        message = six.text_type(message)

        # if not self.error:
        #     message = self.get_line(message)

        self.terminal.write(message)

        for char in message:
            if '\b' == char:
                self.buffer = self.buffer[:-1]
            else:
                self.buffer += char

        if not self.last_timer:
            self.last_timer = Timer(1.0, self.send_to_buffer)
            self.last_timer.start()
{'fraction_non_alphanumeric': 0.10705045278137128, 'fraction_numerical': 0.0029107373868046574, 'mean_word_length': 1.9654841802492808, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3777311', 'n_tokens_mistral': 964, 'n_tokens_neox': 908, 'n_words': 244}