<commit_before>import os
from importlib import import_module
import pkgutil


def list_module(module):
    path = os.path.dirname(module.__file__)
    modules = [name for finder, name,
               is_pkg in pkgutil.iter_modules([path]) if is_pkg]
    return modules


def find_available_locales(providers):
    available_locales = set()

    for provider_path in providers:

        provider_module = import_module(provider_path)
        if getattr(provider_module, 'localized', False):
            langs = list_module(provider_module)
            available_locales.update(langs)
    return available_locales


def find_available_providers(modules):
    available_providers = set()
    for providers_mod in modules:
        providers = [
            '.'.join([providers_mod.__package__, mod])
            for mod in list_module(providers_mod) if mod != '__pycache__'
        ]
        available_providers.update(providers)
    return sorted(available_providers)
<commit_msg>fix path issue with cx_freeze<commit_after>import os
import sys
from importlib import import_module
import pkgutil


def get_path(module):
    if getattr(sys, 'frozen', False):
        # frozen
        path = os.path.dirname(sys.executable)
    else:
        # unfrozen
        path = os.path.dirname(os.path.realpath(module.__file__))
    return path


def list_module(module):
    path = get_path(module)
    modules = [name for finder, name,
               is_pkg in pkgutil.iter_modules([path]) if is_pkg]
    return modules


def find_available_locales(providers):
    available_locales = set()

    for provider_path in providers:

        provider_module = import_module(provider_path)
        if getattr(provider_module, 'localized', False):
            langs = list_module(provider_module)
            available_locales.update(langs)
    return available_locales


def find_available_providers(modules):
    available_providers = set()
    for providers_mod in modules:
        providers = [
            '.'.join([providers_mod.__package__, mod])
            for mod in list_module(providers_mod) if mod != '__pycache__'
        ]
        available_providers.update(providers)
    return sorted(available_providers)

{'fraction_non_alphanumeric': 0.08083560399636694, 'fraction_numerical': 0.0, 'mean_word_length': 3.102420856610801, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6306674', 'n_tokens_mistral': 649, 'n_tokens_neox': 632, 'n_words': 149}