<commit_before># coding: utf-8
import atexit
import glob
import importlib
import inspect
import os.path
import queue
import threading
import time

import modder


class ModManager(object):

    def __init__(self, mod_directory=None):
        if not mod_directory:
            mod_directory = os.path.join(os.path.dirname(__file__), 'mods')
        self.mod_directory = mod_directory
        self.mods = []
        self.mod_definitions = []

        self.check_mod_directory()
        self.load_mods()
        self.init_mods()

    def check_mod_directory(self):
        '''Make sure `mod_directory` is a valid Python package'''
        if not os.path.isdir(self.mod_directory):
            os.makedirs(self.mod_directory)

        package_init_file = os.path.join(self.mod_directory, '__init__.py')
        if not os.path.isfile(package_init_file):
            with open(package_init_file, 'w') as wf:
                wf.write('# coding: utf-8')

    def load_mods(self):
        '''
        Import all modules in `mod_directory`,
        and load all ModBase subclasses into memory.
        '''
        base_package = os.path.basename(self.mod_directory)
        for pyfile in glob.glob('{}/*.py'.format(base_package)):
            # Get module name out of file path
            pymodule_name = os.path.splitext(os.path.basename(pyfile))[0]
            # Import target module
            pymodule = importlib.import_module('{}.{}'.format(base_package, pymodule_name))

            # Store all ModBase subclass definitions
            for name in dir(pymodule):
                reference = getattr(pymodule, name)
                if not name.startswith('_'):
                    if inspect.isclass(reference) and issubclass(reference, modder.ModBase):
                        self.mod_definitions.append(reference)

    def init_mods(self):
        '''Instanciate all ModBase subclasses'''
        for cls in self.mod_definitions:
            if issubclass(cls, modder.ModBase):
                self.mods.append(cls())

    def execute(self, mod_func, event):
        '''Trigger a registered mod method'''
        # TODO Use a threadpool or something to async execute funtions
        if callable(mod_func):
            mod_func(event)

    def trigger(self, name):
        '''Trigger an event and broadcast it to all subscribed mods'''
        if name in modder.EVENTS:
            if name in modder.MOD_REGISTRY:
                event = self.__generate_event(name)
                [self.execute(func, event) for func in modder.MOD_REGISTRY[name]]

    def __generate_event(self, name):
        '''Generate Event object with data'''
        if name.startswith('Modder.'):
            return modder.Event(name)
        elif name.startswith('Timer.'):
            return modder.Event(name, data={'timestamp': time.time()})
        else:
            return modder.Event(name)


def test():
    event_queue = queue.Queue()<commit_msg>Use thread in pool to execute mod functions; Remove class-based mods<commit_after># coding: utf-8
import atexit
import glob
import importlib
import os.path
import queue
import threading
import time

import modder


class ModManager(object):

    def __init__(self, mod_directory=None):
        if not mod_directory:
            mod_directory = os.path.join(os.path.dirname(__file__), 'mods')
        self.mod_directory = mod_directory
        self.__pool = None

        self.check_mod_directory()
        self.load_mods()
        self.init_pool()

    def init_pool(self):
        '''Init pool container for mod executors'''
        if self.__pool is None:
            pool_size = 0
            for event, mods in modder.MOD_REGISTRY.items():
                pool_size += len(mods)

            self.__pool = modder.ExecutorPool(pool_size)

    def check_mod_directory(self):
        '''Make sure `mod_directory` is a valid Python package'''
        if not os.path.isdir(self.mod_directory):
            os.makedirs(self.mod_directory)

        package_init_file = os.path.join(self.mod_directory, '__init__.py')
        if not os.path.isfile(package_init_file):
            with open(package_init_file, 'w') as wf:
                wf.write('# coding: utf-8')

    def load_mods(self):
        '''Import all modules in `mod_directory`'''
        base_package = os.path.basename(self.mod_directory)
        for pyfile in glob.glob('{}/*.py'.format(base_package)):
            # Get module name out of file path
            pymodule_name = os.path.splitext(os.path.basename(pyfile))[0]
            # Import target module
            importlib.import_module('{}.{}'.format(base_package, pymodule_name))

    def execute(self, mod_func, event):
        '''Trigger a registered mod callable'''
        if callable(mod_func):
            mod_worker = threading.Thread(
                target=mod_func,
                name='{} on {} event'.format(mod_func.__name__, event.name),
                args=(event, )
            )
            mod_worker.daemon = True

            try:
                self.__pool.add(mod_worker)
            except modder.exceptions.Full:
                raise UserWarning(
                    'ExecutorPool currently full (maxsize {:d})'.format(self.__pool.maxsize)
                )
            else:
                mod_worker.start()

    def trigger(self, name):
        '''Trigger an event and broadcast it to all subscribed mods'''
        self.__pool.clean_up()

        if name in modder.EVENTS:
            if name in modder.MOD_REGISTRY:
                event = self.__generate_event(name)
                [self.execute(func, event) for func in modder.MOD_REGISTRY[name]]

    def __generate_event(self, name):
        '''Generate Event object with data'''
        if name.startswith('Modder.'):
            return modder.Event(name)
        elif name.startswith('Timer.'):
            return modder.Event(name, data={'timestamp': time.time()})
        else:
            return modder.Event(name)


def test():
    event_queue = queue.Queue()
{'fraction_non_alphanumeric': 0.1043956043956044, 'fraction_numerical': 0.0011655011655011655, 'mean_word_length': 2.2157387580299788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7092899', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1712, 'n_words': 426}