<commit_before>"""Utilities to support packages."""

# NOTE: This module must remain compatible with Python 2.3, as it is shared
# by setuptools for distribution with Python 2.3 and up.

import os
import sys
import imp
import os.path
from types import ModuleType

__all__ = [
    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
    'walk_packages', 'iter_modules',
    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
]

def read_code(stream):
    # This helper is needed in order for the PEP 302 emulation to
    # correctly handle compiled files
    import marshal

    magic = stream.read(4)
    if magic != imp.get_magic():
        return None

    stream.read(4) # Skip timestamp
    return marshal.load(stream)


def simplegeneric(func):
    """Make a trivial single-dispatch generic function"""
    registry = {}
    def wrapper(*args, **kw):
        ob = args[0]
        try:
            cls = ob.__class__
        except AttributeError:
            cls = type(ob)
        try:
            mro = cls.__mro__
        except AttributeError:
            try:
                class cls(cls, object):
                    pass
                mro = cls.__mro__[1:]
            except TypeError:
                mro = object,   # must be an ExtensionClass or some such  :(
        for t in mro:
            if t in registry:
                return registry[t](*args, **kw)
        else:
            return func(*args, **kw)
    try:
        wrapper.__name__ = func.__name__
    except (TypeError, AttributeError):
        pass    # Python 2.3 doesn't allow functions to be renamed

    def register(typ, func=None):
        if func is None:
            return lambda f: register(typ, f)
        registry[typ] = func
        return func

    wrapper.__dict__ = func.__dict__
    wrapper.__doc__ = func.__doc__
    wrapper.register = register
    return wrapper


def walk_packages(path=None, prefix='', onerror=None):
    """Yield submodule names+loaders recursively, for path or sys.path"""

    def seen(p, m={}):
        if p in m:
            return True
        m[p] = True

    for importer, name, ispkg in iter_modules(path, prefix):
        yield importer, name, ispkg

        if ispkg:
            try:
                __import__(name)
            except ImportError:
                if onerror is not None:
                    onerror()
            else:
                path = getattr(sys.modules[name], '__path__', None) or []

                # don't traverse path items we've seen before
                path = [p for p in path if not seen(p)]

                for item in walk_packages(path, name+'.'):
                    yield item


def iter_modules(path=None, prefix=''):
    """Yield submodule names+loaders for path or sys.path"""
    if path is None:
        importers = iter_importers()
    else:
        importers = map(get_importer, path)

    yielded = {}<commit_msg>Patch #1525766: correctly pass onerror arg to recursive calls of pkg.walk_packages. Also improve the docstrings.<commit_after>"""Utilities to support packages."""

# NOTE: This module must remain compatible with Python 2.3, as it is shared
# by setuptools for distribution with Python 2.3 and up.

import os
import sys
import imp
import os.path
from types import ModuleType

__all__ = [
    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
    'walk_packages', 'iter_modules',
    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
]

def read_code(stream):
    # This helper is needed in order for the PEP 302 emulation to
    # correctly handle compiled files
    import marshal

    magic = stream.read(4)
    if magic != imp.get_magic():
        return None

    stream.read(4) # Skip timestamp
    return marshal.load(stream)


def simplegeneric(func):
    """Make a trivial single-dispatch generic function"""
    registry = {}
    def wrapper(*args, **kw):
        ob = args[0]
        try:
            cls = ob.__class__
        except AttributeError:
            cls = type(ob)
        try:
            mro = cls.__mro__
        except AttributeError:
            try:
                class cls(cls, object):
                    pass
                mro = cls.__mro__[1:]
            except TypeError:
                mro = object,   # must be an ExtensionClass or some such  :(
        for t in mro:
            if t in registry:
                return registry[t](*args, **kw)
        else:
            return func(*args, **kw)
    try:
        wrapper.__name__ = func.__name__
    except (TypeError, AttributeError):
        pass    # Python 2.3 doesn't allow functions to be renamed

    def register(typ, func=None):
        if func is None:
            return lambda f: register(typ, f)
        registry[typ] = func
        return func

    wrapper.__dict__ = func.__dict__
    wrapper.__doc__ = func.__doc__
    wrapper.register = register
    return wrapper


def walk_packages(path=None, prefix='', onerror=None):
    """Yields (module_loader, name, ispkg) for all modules recursively
    on path, or, if path is None, all accessible modules.
    
    'path' should be either None or a list of paths to look for
    modules in.

    'prefix' is a string to output on the front of every module name
    on output.

    Note that this function must import all *packages* (NOT all
    modules!) on the given path, in order to access the __path__
    attribute to find submodules.
    
    'onerror' is a function which gets called with one argument (the
    name of the package which was being imported) if an ImportError
    occurs trying to import a package. By default the ImportError is
    caught and ignored.

    Examples:
    walk_packages() : list all modules python can access
    walk_packages(ctypes.__path__, ctypes.__name__+'.') : list all submodules of ctypes
    """

    def seen(p, m={}):
        if p in m:
            return True
        m[p] = True

    for importer, name, ispkg in iter_modules(path, prefix):
        yield importer, name, ispkg

        if ispkg:
            try:
                __import__(name)
            except ImportError:
                if onerror is not None:
                    onerror(name)
            else:
                path = getattr(sys.modules[name], '__path__', None) or []

                # don't traverse path items we've seen before
                path = [p for p in path if not seen(p)]

                for item in walk_packages(path, name+'.', onerror):
                    yield item


def iter_modules(path=None, prefix=''):
    """Yields (module_loader, name, ispkg) for all submodules on path,
    or, if path is None, all top-level modules on sys.path.

    'path' should be either None or a list of paths to look for
    modules in.

    'prefix' is a string to output on the front of every module name
    on output.
    """
    
    if path is None:
        importers = iter_importers()
    else:
        importers = map(get_importer, path)

    yielded = {}
{'fraction_non_alphanumeric': 0.0870436649964209, 'fraction_numerical': 0.004724409448818898, 'mean_word_length': 2.0788893785808726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1622797', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1928, 'n_words': 723}