<commit_before>import threading
import time
import pickle
import json
# noinspection PyUnresolvedReferences
import __main__


class RedisDict(dict):
    def __init__(self, key, redis, pubsub_namespace='amethyst'):
        super().__init__()
        self.key = key
        self.redis = redis
        self.die = False
        self._ready = threading.Event()
        self._modified = {}
        db = str(self.redis.connection_pool.connection_kwargs['db'])
        self.id = '{}.{}.data.{}'.format(pubsub_namespace, db, key)
        threading.Thread(target=self._initialize, name='dataIO init thread for {}'.format(key), daemon=True).start()
        self.uuid = hex(int(time.time() * 10 ** 7))[2:]

    def _initialize(self):
        self._pull()
        self._ready.set()
        threading.Thread(target=self._loop, name='dataIO loop thread for {}'.format(self.key), daemon=True).start()

    def _set(self, key):
        _key = pickle.dumps(key)
        value = pickle.dumps(super().__getitem__(key))
        self.redis.hset(self.key, _key, value)
        self.redis.publish(self.id, json.dumps({
            'origin': self.uuid,
            'action': 'get',
            'key': repr(_key)<commit_msg>Remove obsolete import<commit_after>import threading
import time
import pickle
import json


class RedisDict(dict):
    def __init__(self, key, redis, pubsub_namespace='amethyst'):
        super().__init__()
        self.key = key
        self.redis = redis
        self.die = False
        self._ready = threading.Event()
        self._modified = {}
        db = str(self.redis.connection_pool.connection_kwargs['db'])
        self.id = '{}.{}.data.{}'.format(pubsub_namespace, db, key)
        threading.Thread(target=self._initialize, name='dataIO init thread for {}'.format(key), daemon=True).start()
        self.uuid = hex(int(time.time() * 10 ** 7))[2:]

    def _initialize(self):
        self._pull()
        self._ready.set()
        threading.Thread(target=self._loop, name='dataIO loop thread for {}'.format(self.key), daemon=True).start()

    def _set(self, key):
        _key = pickle.dumps(key)
        value = pickle.dumps(super().__getitem__(key))
        self.redis.hset(self.key, _key, value)
        self.redis.publish(self.id, json.dumps({
            'origin': self.uuid,
            'action': 'get',
            'key': repr(_key)
{'fraction_non_alphanumeric': 0.14929817099106762, 'fraction_numerical': 0.0034028073160357296, 'mean_word_length': 2.7333333333333334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3574119', 'n_tokens_mistral': 761, 'n_tokens_neox': 739, 'n_words': 147}