<commit_before>from collections import defaultdict
from Queue import Queue
from threading import Thread

import traceback

_results = defaultdict(list)

class Worker(Thread):
    """Thread executing tasks from a given tasks queue"""
    def __init__(self, tasks):
        Thread.__init__(self)
        self.tasks = tasks
        self.daemon = True
        self.start()
    
    def run(self):
        interrupt = False
        while True:
            func, args, kwargs, ident = self.tasks.get()

            if interrupt:
                self.tasks.task_done()
                continue
            
            try:
                _results[ident].append({
                    'func': func,
                    'args': args,
                    'kwargs': kwargs,
                    'result': func(*args, **kwargs),
                })
            except KeyboardInterrupt, e:
                # We don't record results if we hit an interrupt
                interrupt = True
            except Exception, e:
                _results[ident].append({
                    'func': func,
                    'args': args,
                    'kwargs': kwargs,
                    'result': traceback.format_exc(),
                })
            finally:
                self.tasks.task_done()

class ThreadPool:
    """Pool of threads consuming tasks from a queue"""
    def __init__(self, num_threads):
        self.tasks = Queue()
        self.workers = []
        for _ in xrange(num_threads):
            self.workers.append(Worker(self.tasks))<commit_msg>Revert "Dont store results of interrupted tests, as they would be incorrect"<commit_after>from collections import defaultdict
from Queue import Queue
from threading import Thread

import traceback

_results = defaultdict(list)

class Worker(Thread):
    """Thread executing tasks from a given tasks queue"""
    def __init__(self, tasks):
        Thread.__init__(self)
        self.tasks = tasks
        self.daemon = True
        self.start()
    
    def run(self):
        interrupt = False
        while True:
            func, args, kwargs, ident = self.tasks.get()

            if interrupt:
                self.tasks.task_done()
                continue
            
            try:
                _results[ident].append({
                    'func': func,
                    'args': args,
                    'kwargs': kwargs,
                    'result': func(*args, **kwargs),
                })
            except KeyboardInterrupt, e:
                _results[ident].append({
                    'func': func,
                    'args': args,
                    'kwargs': kwargs,
                    'result': e.args[0],
                })
                interrupt = True
            except Exception, e:
                _results[ident].append({
                    'func': func,
                    'args': args,
                    'kwargs': kwargs,
                    'result': traceback.format_exc(),
                })
            finally:
                self.tasks.task_done()

class ThreadPool:
    """Pool of threads consuming tasks from a queue"""
    def __init__(self, num_threads):
        self.tasks = Queue()
        self.workers = []
        for _ in xrange(num_threads):
            self.workers.append(Worker(self.tasks))
{'fraction_non_alphanumeric': 0.09502262443438914, 'fraction_numerical': 0.00030165912518853697, 'mean_word_length': 1.3787661406025824, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4795831', 'n_tokens_mistral': 835, 'n_tokens_neox': 802, 'n_words': 228}