<commit_before>

import time
import multiprocessing


try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        pass


class StatsCollector(multiprocessing.Process):
    def __init__(self, queue):
        super(StatsCollector, self).__init__()
        self.queue = queue

    def close(self):
        pass

    def run(self):
        setproctitle("bucky: %s" % self.__class__.__name__)
        err = 0
        while True:
            start_timestamp = time.time()
            if not self.collect():
                err = min(err + 1, 2)
            else:
                err = 0
            stop_timestamp = time.time()
            sleep_time = (err + 1) * self.interval - (stop_timestamp - start_timestamp)
            if sleep_time > 0.1:
                time.sleep(sleep_time)

    def collect(self):
        raise NotImplementedError()

    def add_stat(self, name, value, timestamp, **metadata):
        if metadata:
            if self.metadata:
                metadata.update(self.metadata)
        else:
            metadata = self.metadata
        if metadata:
            self.queue.put((None, name, value, timestamp, metadata))
        else:
            self.queue.put((None, name, value, timestamp))
<commit_msg>Change the back-off algo for failures<commit_after>

import time
import multiprocessing


try:
    from setproctitle import setproctitle
except ImportError:
    def setproctitle(title):
        pass


class StatsCollector(multiprocessing.Process):
    def __init__(self, queue):
        super(StatsCollector, self).__init__()
        self.queue = queue

    def close(self):
        pass

    def run(self):
        setproctitle("bucky: %s" % self.__class__.__name__)
        interval = self.interval
        while True:
            start_timestamp = time.time()
            interval = self.interval if self.collect() else interval+interval
            stop_timestamp = time.time()
            interval = min(interval, 300)
            interval = interval - (stop_timestamp - start_timestamp)
            if interval > 0.1:
                time.sleep(interval)

    def collect(self):
        raise NotImplementedError()

    def add_stat(self, name, value, timestamp, **metadata):
        if metadata:
            if self.metadata:
                metadata.update(self.metadata)
        else:
            metadata = self.metadata
        if metadata:
            self.queue.put((None, name, value, timestamp, metadata))
        else:
            self.queue.put((None, name, value, timestamp))

{'fraction_non_alphanumeric': 0.09278752436647174, 'fraction_numerical': 0.004678362573099415, 'mean_word_length': 1.9976635514018692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2711805', 'n_tokens_mistral': 723, 'n_tokens_neox': 692, 'n_words': 182}