<commit_before># coding=utf-8

import datetime
import logging
import traceback


def parse_frequency(s):
    if s == "never" or s == None:
        return None, None
    kind, num, unit = s.split()
    return int(num), unit


class DefaultScheduler(object):
    thread = None
    running = False
    registry = None

    def __init__(self):
        self.thread = None
        self.running = False
        self.registry = []

        self.tasks = {}
        self.init_storage()

    def init_storage(self):
        if "tasks" not in Dict:
            Dict["tasks"] = {}
            Dict.Save()

    def get_task_data(self, name):
        if "data" in Dict["tasks"][name]:
            return Dict["tasks"][name]["data"]

    def register(self, task):
        self.registry.append(task)

    def setup_tasks(self):
        # discover tasks;
        self.tasks = {}
        for cls in self.registry:
            task = cls(self)
            try:
                task_frequency = Prefs["scheduler.tasks.%s" % task.name]
            except KeyError:
                task_frequency = None

            self.tasks[task.name] = {"task": task, "frequency": parse_frequency(task_frequency)}
            Dict["tasks"][task.name] = {"data": {}}

    def run(self):
        self.running = True
        self.thread = Thread.Create(self.worker)

    def stop(self):
        self.running = False

    def task(self, name):
        if name not in self.tasks:
            return None
        return self.tasks[name]["task"]

    def is_task_running(self, name):
        task = self.task(name)
        return task.running

    def last_run(self, task):
        if task not in self.tasks:
            return None
        return self.tasks[task]["task"].last_run

    def next_run(self, task):
        if task not in self.tasks or not self.tasks[task]["task"].periodic:
            return None
        frequency_num, frequency_key = self.tasks[task]["frequency"]
        if not frequency_num:
            return None<commit_msg>don't kill task data in Dict by default<commit_after># coding=utf-8

import datetime
import logging
import traceback


def parse_frequency(s):
    if s == "never" or s == None:
        return None, None
    kind, num, unit = s.split()
    return int(num), unit


class DefaultScheduler(object):
    thread = None
    running = False
    registry = None

    def __init__(self):
        self.thread = None
        self.running = False
        self.registry = []

        self.tasks = {}
        self.init_storage()

    def init_storage(self):
        if "tasks" not in Dict:
            Dict["tasks"] = {}
            Dict.Save()

    def get_task_data(self, name):
        if "data" in Dict["tasks"][name]:
            return Dict["tasks"][name]["data"]

    def register(self, task):
        self.registry.append(task)

    def setup_tasks(self):
        # discover tasks;
        self.tasks = {}
        for cls in self.registry:
            task = cls(self)
            try:
                task_frequency = Prefs["scheduler.tasks.%s" % task.name]
            except KeyError:
                task_frequency = None

            self.tasks[task.name] = {"task": task, "frequency": parse_frequency(task_frequency)}
            if task.name not in Dict["tasks"]:
                Dict["tasks"][task.name] = {"data": {}}

    def run(self):
        self.running = True
        self.thread = Thread.Create(self.worker)

    def stop(self):
        self.running = False

    def task(self, name):
        if name not in self.tasks:
            return None
        return self.tasks[name]["task"]

    def is_task_running(self, name):
        task = self.task(name)
        return task.running

    def last_run(self, task):
        if task not in self.tasks:
            return None
        return self.tasks[task]["task"].last_run

    def next_run(self, task):
        if task not in self.tasks or not self.tasks[task]["task"].periodic:
            return None
        frequency_num, frequency_key = self.tasks[task]["frequency"]
        if not frequency_num:
            return None
{'fraction_non_alphanumeric': 0.11475409836065574, 'fraction_numerical': 0.0004893564962074871, 'mean_word_length': 1.9883040935672514, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2345557', 'n_tokens_mistral': 1214, 'n_tokens_neox': 1169, 'n_words': 317}