<commit_before>"""
Import utilities

Exported classes:
    ImportManager   Manage the import process

    Importer        Base class for replacing standard import functions
    BuiltinImporter Emulate the import mechanism for builtin and frozen modules

    DynLoadSuffixImporter
"""

# note: avoid importing non-builtin modules
import imp                      ### not available in JPython?
import sys
import __builtin__

# for the DirectoryImporter
import struct
import marshal

__all__ = ["ImportManager","Importer","BuiltinImporter"]

_StringType = type('')
_ModuleType = type(sys)         ### doesn't work in JPython...

class ImportManager:
    "Manage the import process."

    def install(self, namespace=vars(__builtin__)):
        "Install this ImportManager into the specified namespace."

        if isinstance(namespace, _ModuleType):
            namespace = vars(namespace)

        # Note: we have no notion of "chaining"

        # Record the previous import hook, then install our own.
        self.previous_importer = namespace['__import__']
        self.namespace = namespace
        namespace['__import__'] = self._import_hook

        ### fix this
        #namespace['reload'] = self._reload_hook

    def uninstall(self):
        "Restore the previous import mechanism."
        self.namespace['__import__'] = self.previous_importer

    def add_suffix(self, suffix, importFunc):
        assert callable(importFunc)
        self.fs_imp.add_suffix(suffix, importFunc)

    ######################################################################
    #
    # PRIVATE METHODS
    #

    clsFilesystemImporter = None

    def __init__(self, fs_imp=None):
        # we're definitely going to be importing something in the future,
        # so let's just load the OS-related facilities.
        if not _os_stat:
            _os_bootstrap()

        # This is the Importer that we use for grabbing stuff from the
        # filesystem. It defines one more method (import_from_dir) for our use.
        if fs_imp is None:
            cls = self.clsFilesystemImporter or _FilesystemImporter
            fs_imp = cls()
        self.fs_imp = fs_imp

        # Initialize the set of suffixes that we recognize and import.
        # The default will import dynamic-load modules first, followed by
        # .py files (or a .py file's cached bytecode)
        for desc in imp.get_suffixes():
            if desc[2] == imp.C_EXTENSION:
                self.add_suffix(desc[0],
                                DynLoadSuffixImporter(desc).import_file)
        self.add_suffix('.py', py_suffix_importer)

    def _import_hook(self, fqname, globals=None, locals=None, fromlist=None):
        """Python calls this hook to locate and import a module."""

        parts = fqname.split('.')

        # determine the context of this import
        parent = self._determine_import_context(globals)

        # if there is a parent, then its importer should manage this import
        if parent:
            module = parent.__importer__._do_import(parent, parts, fromlist)
            if module:
                return module

        # has the top module already been imported?
        try:
            top_module = sys.modules[parts[0]]
        except KeyError:

            # look for the topmost module
            top_module = self._import_top_module(parts[0])
            if not top_module:
                # the topmost module wasn't found at all.
                raise ImportError, 'No module named ' + fqname

        # fast-path simple imports
        if len(parts) == 1:
            if not fromlist:
                return top_module

            if not top_module.__dict__.get('__ispkg__'):
                # __ispkg__ isn't defined (the module was not imported by us),
                # or it is zero.
                #
                # In the former case, there is no way that we could import
                # sub-modules that occur in the fromlist (but we can't raise an
                # error because it may just be names) because we don't know how
                # to deal with packages that were imported by other systems.
                #
                # In the latter case (__ispkg__ == 0), there can't be any sub-
                # modules present, so we can just return.
                #
                # In both cases, since len(parts) == 1, the top_module is also
                # the "bottom" which is the defined return when a fromlist
                # exists.
                return top_module

        importer = top_module.__dict__.get('__importer__')
        if importer:
            return importer._finish_import(top_module, parts[1:], fromlist)

        # Grrr, some people "import os.path"
        if len(parts) == 2 and hasattr(top_module, parts[1]):
            return top_module

        # If the importer does not exist, then we have to bail. A missing
        # importer means that something else imported the module, and we have
        # no knowledge of how to get sub-modules out of the thing.
        raise ImportError, 'No module named ' + fqname

    def _determine_import_context(self, globals):
        """Returns the context in which a module should be imported.

        The context could be a loaded (package) module and the imported module
        will be looked for within that package. The context could also be None,
        meaning there is no context -- the module should be looked for as a
        "top-level" module.
        """<commit_msg>Bug #1481530: allow "from os.path import ..." with imputil<commit_after>"""
Import utilities

Exported classes:
    ImportManager   Manage the import process

    Importer        Base class for replacing standard import functions
    BuiltinImporter Emulate the import mechanism for builtin and frozen modules

    DynLoadSuffixImporter
"""

# note: avoid importing non-builtin modules
import imp                      ### not available in JPython?
import sys
import __builtin__

# for the DirectoryImporter
import struct
import marshal

__all__ = ["ImportManager","Importer","BuiltinImporter"]

_StringType = type('')
_ModuleType = type(sys)         ### doesn't work in JPython...

class ImportManager:
    "Manage the import process."

    def install(self, namespace=vars(__builtin__)):
        "Install this ImportManager into the specified namespace."

        if isinstance(namespace, _ModuleType):
            namespace = vars(namespace)

        # Note: we have no notion of "chaining"

        # Record the previous import hook, then install our own.
        self.previous_importer = namespace['__import__']
        self.namespace = namespace
        namespace['__import__'] = self._import_hook

        ### fix this
        #namespace['reload'] = self._reload_hook

    def uninstall(self):
        "Restore the previous import mechanism."
        self.namespace['__import__'] = self.previous_importer

    def add_suffix(self, suffix, importFunc):
        assert callable(importFunc)
        self.fs_imp.add_suffix(suffix, importFunc)

    ######################################################################
    #
    # PRIVATE METHODS
    #

    clsFilesystemImporter = None

    def __init__(self, fs_imp=None):
        # we're definitely going to be importing something in the future,
        # so let's just load the OS-related facilities.
        if not _os_stat:
            _os_bootstrap()

        # This is the Importer that we use for grabbing stuff from the
        # filesystem. It defines one more method (import_from_dir) for our use.
        if fs_imp is None:
            cls = self.clsFilesystemImporter or _FilesystemImporter
            fs_imp = cls()
        self.fs_imp = fs_imp

        # Initialize the set of suffixes that we recognize and import.
        # The default will import dynamic-load modules first, followed by
        # .py files (or a .py file's cached bytecode)
        for desc in imp.get_suffixes():
            if desc[2] == imp.C_EXTENSION:
                self.add_suffix(desc[0],
                                DynLoadSuffixImporter(desc).import_file)
        self.add_suffix('.py', py_suffix_importer)

    def _import_hook(self, fqname, globals=None, locals=None, fromlist=None):
        """Python calls this hook to locate and import a module."""

        parts = fqname.split('.')

        # determine the context of this import
        parent = self._determine_import_context(globals)

        # if there is a parent, then its importer should manage this import
        if parent:
            module = parent.__importer__._do_import(parent, parts, fromlist)
            if module:
                return module

        # has the top module already been imported?
        try:
            top_module = sys.modules[parts[0]]
        except KeyError:

            # look for the topmost module
            top_module = self._import_top_module(parts[0])
            if not top_module:
                # the topmost module wasn't found at all.
                raise ImportError, 'No module named ' + fqname

        # fast-path simple imports
        if len(parts) == 1:
            if not fromlist:
                return top_module

            if not top_module.__dict__.get('__ispkg__'):
                # __ispkg__ isn't defined (the module was not imported by us),
                # or it is zero.
                #
                # In the former case, there is no way that we could import
                # sub-modules that occur in the fromlist (but we can't raise an
                # error because it may just be names) because we don't know how
                # to deal with packages that were imported by other systems.
                #
                # In the latter case (__ispkg__ == 0), there can't be any sub-
                # modules present, so we can just return.
                #
                # In both cases, since len(parts) == 1, the top_module is also
                # the "bottom" which is the defined return when a fromlist
                # exists.
                return top_module

        importer = top_module.__dict__.get('__importer__')
        if importer:
            return importer._finish_import(top_module, parts[1:], fromlist)

        # Grrr, some people "import os.path" or do "from os.path import ..."
        if len(parts) == 2 and hasattr(top_module, parts[1]):
            if fromlist:
                return getattr(top_module, parts[1])
            else:
                return top_module

        # If the importer does not exist, then we have to bail. A missing
        # importer means that something else imported the module, and we have
        # no knowledge of how to get sub-modules out of the thing.
        raise ImportError, 'No module named ' + fqname

    def _determine_import_context(self, globals):
        """Returns the context in which a module should be imported.

        The context could be a loaded (package) module and the imported module
        will be looked for within that package. The context could also be None,
        meaning there is no context -- the module should be looked for as a
        "top-level" module.
        """
{'fraction_non_alphanumeric': 0.08222242104321374, 'fraction_numerical': 0.0025051444931555875, 'mean_word_length': 2.1478456772740073, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4688264', 'n_tokens_mistral': 2968, 'n_tokens_neox': 2794, 'n_words': 1092}