<commit_before>import sys
import time
from functools import wraps
from collections import UserDict
from datetime import datetime
import signal

from flask import current_app
from werkzeug.utils import cached_property
import greenlet
import gevent
import gevent.monkey
from gevent.hub import get_hub
from gevent.pool import Pool
from gevent.greenlet import Greenlet
from gevent.lock import Semaphore, BoundedSemaphore
from gevent.pywsgi import WSGIServer

from .app import Flask
from .utils import get_argv_opt, monkey_patch_meth


class LockedDict(UserDict):
    def __init__(self, timeout=None):
        self.timeout = timeout
        super().__init__()

    def __getitem__(self, key):
        self.get(key)  # For semaphore wait
        return self.data[key]  # For key error

    def get(self, key):
        if key in self.data:
            if isinstance(self.data[key], BoundedSemaphore):
                self.data[key].wait(self.timeout)
                assert not isinstance(self.data[key], BoundedSemaphore)
        return None

    def __setitem__(self, key, value):
        if not callable(value):
            raise ValueError('Value %s should be callable %s' % value)
        lock = self.data[key] = BoundedSemaphore()
        lock.acquire()
        self.data[key] = value()
        assert not isinstance(self.data[key], BoundedSemaphore)
        lock.release()

    def factory(self, key):
        raise NotImplementedError()

    def set(self, key):
        self[key] = lambda: self.factory(key)


class CachedBulkProcessor:
    def __init__(self, pool, cache, cache_key, cache_timeout, cache_fail_timeout=None,
                 update_timeout=10, join_timeout=30, join_timeout_raise=False,
                 worker=None, logger=None):
        self.pool = pool
        self.cache = cache
        assert '{}' in cache_key, 'Cache key should have format placeholder'
        self.cache_key = cache_key
        self.cache_timeout = cache_timeout
        self.cache_fail_timeout = cache_fail_timeout or cache_timeout
        self.update_timeout = update_timeout
        self.join_timeout = join_timeout
        self.join_timeout_raise = join_timeout_raise
        if worker:
            self._worker = worker<commit_msg>LockedDict renamed to LockedFactoryDict and now works as defaultdict<commit_after>import sys
import time
from functools import wraps
from collections import UserDict
from datetime import datetime
import signal

from flask import current_app
from werkzeug.utils import cached_property
import greenlet
import gevent
import gevent.monkey
from gevent.hub import get_hub
from gevent.pool import Pool
from gevent.greenlet import Greenlet
from gevent.lock import Semaphore, BoundedSemaphore
from gevent.pywsgi import WSGIServer

from .app import Flask
from .utils import get_argv_opt, monkey_patch_meth


class LockedFactoryDict(UserDict):
    def __init__(self, factory=None, timeout=None):
        self._factory = factory
        self.timeout = timeout
        super().__init__()

    def __getitem__(self, key):
        if key not in self.data:
            self[key] = lambda: self.factory(key)
        return self.get(key)

    def get(self, key):
        if key in self.data:
            if isinstance(self.data[key], BoundedSemaphore):
                self.data[key].wait(self.timeout)
                assert not isinstance(self.data[key], BoundedSemaphore)

    def __setitem__(self, key, factory):
        if not callable(factory):
            raise ValueError('value %s should be callable' % factory)
        lock = self.data[key] = BoundedSemaphore()
        lock.acquire()
        self.data[key] = factory()
        assert not isinstance(self.data[key], BoundedSemaphore)
        lock.release()

    def factory(self, key):
        if self._factory is not None:
            return self._factory(key)
        raise NotImplementedError()


class CachedBulkProcessor:
    def __init__(self, pool, cache, cache_key, cache_timeout, cache_fail_timeout=None,
                 update_timeout=10, join_timeout=30, join_timeout_raise=False,
                 worker=None, logger=None):
        self.pool = pool
        self.cache = cache
        assert '{}' in cache_key, 'Cache key should have format placeholder'
        self.cache_key = cache_key
        self.cache_timeout = cache_timeout
        self.cache_fail_timeout = cache_fail_timeout or cache_timeout
        self.update_timeout = update_timeout
        self.join_timeout = join_timeout
        self.join_timeout_raise = join_timeout_raise
        if worker:
            self._worker = worker
{'fraction_non_alphanumeric': 0.07453551912568306, 'fraction_numerical': 0.0017486338797814208, 'mean_word_length': 2.93127147766323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2333797', 'n_tokens_mistral': 1321, 'n_tokens_neox': 1280, 'n_words': 371}