<commit_before>from __future__ import absolute_import

import anyjson
import os
import sys

from kombu.exceptions import VersionMismatch

# avoid json implementation inconsistencies.
try:
    import json  # noqa
    anyjson.force_implementation('json')
except ImportError:
    anyjson.force_implementation('simplejson')


def find_distribution_modules(name=__name__, file=__file__):
    current_dist_depth = len(name.split('.')) - 1
    current_dist = os.path.join(os.path.dirname(file),
                                *([os.pardir] * current_dist_depth))
    abs = os.path.abspath(current_dist)
    dist_name = os.path.basename(abs)

    for dirpath, dirnames, filenames in os.walk(abs):
        package = (dist_name + dirpath[len(abs):]).replace('/', '.')
        if '__init__.py' in filenames:
            yield package
            for filename in filenames:
                if filename.endswith('.py') and filename != '__init__.py':
                    yield '.'.join([package, filename])[:-3]<commit_msg>testsuite: Remove multiprocessing _exit_function from atexit handlers<commit_after>from __future__ import absolute_import

import anyjson
import atexit
import os
import sys

from kombu.exceptions import VersionMismatch

# avoid json implementation inconsistencies.
try:
    import json  # noqa
    anyjson.force_implementation('json')
except ImportError:
    anyjson.force_implementation('simplejson')


def teardown():
    # Workaround for multiprocessing bug where logging
    # is attempted after global already collected at shutdown.
    cancelled = set()
    try:
        import multiprocessing.util
        cancelled.add(multiprocessing.util._exit_function)
    except (AttributeError, ImportError):
        pass

    atexit._exithandlers[:] = [
        e for e in atexit._exithandlers if e[0] not in cancelled
    ]


def find_distribution_modules(name=__name__, file=__file__):
    current_dist_depth = len(name.split('.')) - 1
    current_dist = os.path.join(os.path.dirname(file),
                                *([os.pardir] * current_dist_depth))
    abs = os.path.abspath(current_dist)
    dist_name = os.path.basename(abs)

    for dirpath, dirnames, filenames in os.walk(abs):
        package = (dist_name + dirpath[len(abs):]).replace('/', '.')
        if '__init__.py' in filenames:
            yield package
            for filename in filenames:
                if filename.endswith('.py') and filename != '__init__.py':
                    yield '.'.join([package, filename])[:-3]
{'fraction_non_alphanumeric': 0.10553564317005178, 'fraction_numerical': 0.001991238550378335, 'mean_word_length': 3.012779552715655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645303', 'n_tokens_mistral': 765, 'n_tokens_neox': 736, 'n_words': 178}