<commit_before>import datetime

import operator

__version__ = '0.1.3'

try:
    import Queue
except ImportError:
    import queue as Queue
import threading
import sys


class ThreadPool:
    """
    Customized thread pool
    """

    class TreadPoolException(Exception):
        pass

    class NULLKEY:
        pass

    def __init__(self, worker_limit):
        self.task_queue = Queue.Queue(maxsize=worker_limit)
        self.result_dict = {}
        self.is_join = False

    def setup_func(self, key, func, *args, **kws):
        """ wrap the function, redirect the output to the `result_dict`.
        """

        def func_wrap():
            self.result_dict[key] = func(*args, **kws)
            # mark one position in queue is available.
            self.task_queue.get()
            self.task_queue.task_done()

        def func_origin():
            func(*args, **kws)
            self.task_queue.get()
            self.task_queue.task_done()

        if key is not self.NULLKEY:
            return func_wrap
        else:
            return func_origin

    def putting_task(self, func, *args, **kws):
        """ put task to the queue
        """
        if self.is_join:
            raise self.TreadPoolException('Thread pool is closed.')
        result_id = kws.pop('_result_id', self.NULLKEY)<commit_msg>[FIX] FIX pool stuck error<commit_after>import datetime

import operator

__version__ = '0.1.3'

try:
    import Queue
except ImportError:
    import queue as Queue
import threading
import sys


class ThreadPool:
    """
    Customized thread pool
    """

    class TreadPoolException(Exception):
        pass

    class NULLKEY:
        pass

    def __init__(self, worker_limit):
        self.task_queue = Queue(maxsize=worker_limit)
        self.result_dict = {}
        self.is_join = False

    def setup_func(self, key, func, *args, **kws):
        """ wrap the function, redirect the output to the `result_dict`.
        """

        def func_wrap():
            try:
                self.result_dict[key] = func(*args, **kws)
            except:
                raise
            finally:
                # mark one position in queue is available.
                self.task_queue.get()
                self.task_queue.task_done()

        def func_origin():
            try:
                func(*args, **kws)
            except:
                raise
            finally:
                self.task_queue.get()
                self.task_queue.task_done()

        if key is not self.NULLKEY:
            return func_wrap
        else:
            return func_origin

    def putting_task(self, func, *args, **kws):
        """ put task to the queue
        """
        if self.is_join:
            raise self.TreadPoolException('Thread pool is closed.')
        result_id = kws.pop('_result_id', self.NULLKEY)
{'fraction_non_alphanumeric': 0.09468497008095741, 'fraction_numerical': 0.0021119324181626186, 'mean_word_length': 1.8194444444444444, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3591539', 'n_tokens_mistral': 832, 'n_tokens_neox': 804, 'n_words': 217}