<commit_before>from functools import lru_cache
import importlib.util
import os
import re
from prosconductor.providers import DepotProvider, DepotConfig, TemplateTypes, Identifier, TemplateDescriptor
from prosconfig.cliconfig import CliConfig
# from typing import Dict, List


@lru_cache()
def get_all_provider_types(pros_cfg=None):
    if pros_cfg is None:
        pros_cfg = CliConfig()

    for provider_file in pros_cfg.providers:
        spec = importlib.util.spec_from_file_location('module.name', provider_file)
        mod = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(mod)

    return {x.registrar: x for x in DepotProvider.__subclasses__()}


@lru_cache()
def get_depot(depot_cfg, pros_cfg=None):
    providers = get_all_provider_types(pros_cfg)
    if depot_cfg.registrar in providers:
        return providers[depot_cfg.registrar](depot_cfg)
    else:
        return None


@lru_cache()
def get_depot_config(name, pros_cfg=None):
    if pros_cfg is None:
        pros_cfg = CliConfig()

    return DepotConfig(os.path.join(pros_cfg.directory, name, 'depot.pros'))


def get_depot_configs(pros_cfg=None, filters=None):
    if pros_cfg is None:
        pros_cfg = CliConfig()
    if filters is None or not filters:
        filters = ['.*']
    return [depot for depot in [get_depot_config(d, pros_cfg=pros_cfg) for d in os.listdir(pros_cfg.directory)
                                if os.path.isdir(os.path.join(pros_cfg.directory, d))]
            if depot.name and not all(m is None for m in [re.match(string=depot.name, pattern=f) for f in filters])]
<commit_msg>Modify dynamic loading of depot providers<commit_after>from functools import lru_cache
import importlib.machinery
import importlib.util
import os
import re
from prosconductor.providers import DepotProvider, DepotConfig, TemplateTypes, Identifier, TemplateDescriptor
from prosconfig.cliconfig import CliConfig
# from typing import Dict, List


@lru_cache()
def get_all_provider_types(pros_cfg=None):
    if pros_cfg is None:
        pros_cfg = CliConfig()

    for provider_file in pros_cfg.providers:
        importlib.machinery.SourceFileLoader('modulename', provider_file).load_module()
        # spec = importlib.util.spec_from_file_location('module.name', provider_file)
        # mod = importlib.util.module_from_spec(spec)
        # spec.loader.exec_module(mod)

    return {x.registrar: x for x in DepotProvider.__subclasses__()}


@lru_cache()
def get_depot(depot_cfg, pros_cfg=None):
    providers = get_all_provider_types(pros_cfg)
    if depot_cfg.registrar in providers:
        return providers[depot_cfg.registrar](depot_cfg)
    else:
        return None


@lru_cache()
def get_depot_config(name, pros_cfg=None):
    if pros_cfg is None:
        pros_cfg = CliConfig()

    return DepotConfig(os.path.join(pros_cfg.directory, name, 'depot.pros'))


def get_depot_configs(pros_cfg=None, filters=None):
    if pros_cfg is None:
        pros_cfg = CliConfig()
    if filters is None or not filters:
        filters = ['.*']
    return [depot for depot in [get_depot_config(d, pros_cfg=pros_cfg) for d in os.listdir(pros_cfg.directory)
                                if os.path.isdir(os.path.join(pros_cfg.directory, d))]
            if depot.name and not all(m is None for m in [re.match(string=depot.name, pattern=f) for f in filters])]

{'fraction_non_alphanumeric': 0.09066507605129735, 'fraction_numerical': 0.0, 'mean_word_length': 3.7709815078236133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '906275', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1113, 'n_words': 267}