<commit_before>import json
import stat

class CoalescingMachine(object):
    """
    Generic Coalescer object contains logic to build lists of tasks based on
    defined commonality eg. ProvisionerId, WorkerType, TaskDef env
    These 'defined commonalities' will be the index key used to quickly
    retrieve lists via the wsgi REST api multiple objects may be defined
    accommodate multiple 'defined commonalities'
    """

    pf = "default."

    pending_tasks = None

    def __init__(self, pending_tasks, redis_prefix, datastore, stats):
        self.pending_tasks = pending_tasks
        self.pf = redis_prefix
        self.rds = datastore
        self.stats = stats

    def insert_task(self, taskId):
        for coalesce_key in self._get_coalesce_key(taskId):
            self.rds.sadd(self.pf + "list_keys", coalesce_key)
            self.rds.rpush(self.pf + "lists." + coalesce_key, taskId)

    def remove_task(self, taskId):
        for coalesce_key in self._get_coalesce_key(taskId):
            self.rds.lrem(self.pf + 'lists.' + coalesce_key, taskId, num=0)
            if self.rds.llen(self.pf + 'lists.' + coalesce_key) == 0:
                self.rds.srem(self.pf + "list_keys", coalesce_key)

    def _get_coalesce_key(self, taskId):
        for route in self.pending_tasks[taskId]['task_optional_routes']:
            # strip 'route.'<commit_msg>Reverse side tasks are appended to coalesce lists<commit_after>import json
import stat

class CoalescingMachine(object):
    """
    Generic Coalescer object contains logic to build lists of tasks based on
    defined commonality eg. ProvisionerId, WorkerType, TaskDef env
    These 'defined commonalities' will be the index key used to quickly
    retrieve lists via the wsgi REST api multiple objects may be defined
    accommodate multiple 'defined commonalities'
    """

    pf = "default."

    pending_tasks = None

    def __init__(self, pending_tasks, redis_prefix, datastore, stats):
        self.pending_tasks = pending_tasks
        self.pf = redis_prefix
        self.rds = datastore
        self.stats = stats

    def insert_task(self, taskId):
        for coalesce_key in self._get_coalesce_key(taskId):
            self.rds.sadd(self.pf + "list_keys", coalesce_key)
            self.rds.lpush(self.pf + "lists." + coalesce_key, taskId)

    def remove_task(self, taskId):
        for coalesce_key in self._get_coalesce_key(taskId):
            self.rds.lrem(self.pf + 'lists.' + coalesce_key, taskId, num=0)
            if self.rds.llen(self.pf + 'lists.' + coalesce_key) == 0:
                self.rds.srem(self.pf + "list_keys", coalesce_key)

    def _get_coalesce_key(self, taskId):
        for route in self.pending_tasks[taskId]['task_optional_routes']:
            # strip 'route.'
{'fraction_non_alphanumeric': 0.09084354722422494, 'fraction_numerical': 0.0014419610670511895, 'mean_word_length': 2.9305949008498584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1845495', 'n_tokens_mistral': 895, 'n_tokens_neox': 861, 'n_words': 232}