<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Support for breakpoints on modules that haven't been loaded yet.

This is the new module import hook which:
  1. Takes a partial path of the module file excluding the file extension as
     input (can be as short as 'foo' or longer such as 'sys/path/pkg/foo').
  2. At each (top-level-only) import statement:
    a. Generates an estimate of the modules that might be loaded as a result
       of this import (and all chained imports) using the arguments of the
       import hook. The estimate is best-effort, it may contain extra entries
       that are not of interest to us (e.g., outer packages that were already
       loaded before this import), or may be missing some module names (not
       all intricacies of Python module importer are handled).
    b. Checks sys.modules if any of these modules have a file that matches the
       given path, using suffix match.

For the old module import hook, see imphook.py file.
"""

import importlib
import itertools
import os
import sys  # Must be imported, otherwise import hooks don't work.
import threading

import six
from six.moves import builtins  # pylint: disable=redefined-builtin

from . import module_utils2

# Callbacks to invoke when a module is imported.
_import_callbacks = {}
_import_callbacks_lock = threading.Lock()

# Per thread data holding information about the import call nest level.
_import_local = threading.local()

# Original __import__ function if import hook is installed or None otherwise.
_real_import = None

# Original importlib.import_module function if import hook is installed or None
# otherwise.
_real_import_module = None


def AddImportCallbackBySuffix(path, callback):
  """Register import hook.

  This function overrides the default import process. Then whenever a module
  whose suffix matches path is imported, the callback will be invoked.

  A module may be imported multiple times. Import event only means that the
  Python code contained an "import" statement. The actual loading and
  initialization of a new module normally happens only once, at which time
  the callback will be invoked. This function does not validates the existence
  of such a module and it's the responsibility of the caller.

  TODO(erezh): handle module reload.

  Args:
    path: python module file path. It may be missing the directories for the
          outer packages, and therefore, requires suffix comparison to match
          against loaded modules. If it contains all outer packages, it may
          contain the sys.path as well.
          It might contain an incorrect file extension (e.g., py vs. pyc).
    callback: callable to invoke upon module load.

  Returns:
    Function object to invoke to remove the installed callback.
  """

  def RemoveCallback():
    # This is a read-if-del operation on _import_callbacks. Lock to prevent
    # callbacks from being inserted just before the key is deleted. Thus, it
    # must be locked also when inserting a new entry below. On the other hand
    # read only access, in the import hook, does not require a lock.
    with _import_callbacks_lock:
      callbacks = _import_callbacks.get(path)
      if callbacks:
        callbacks.remove(callback)
        if not callbacks:
          del _import_callbacks[path]

  with _import_callbacks_lock:
    _import_callbacks.setdefault(path, set()).add(callback)
  _InstallImportHookBySuffix()

  return RemoveCallback


def _InstallImportHookBySuffix():
  """Lazily installs import hook."""
  global _real_import

  if _real_import:
    return  # Import hook already installed

  _real_import = getattr(builtins, '__import__')
  assert _real_import
  builtins.__import__ = _ImportHookBySuffix

  if six.PY3:
    # In Python 2, importlib.import_module calls __import__ internally so
    # overriding __import__ is enough. In Python 3, they are separate so it also
    # needs to be overwritten.
    global _real_import_module
    _real_import_module = importlib.import_module
    assert _real_import_module
    importlib.import_module = _ImportModuleHookBySuffix


def _IncrementNestLevel():
  """Increments the per thread nest level of imports."""
  # This is the top call to import (no nesting), init the per-thread nest level
  # and names set.
  if getattr(_import_local, 'nest_level', None) is None:
    _import_local.nest_level = 0

  if _import_local.nest_level == 0:
    # Re-initialize names set at each top-level import to prevent any
    # accidental unforeseen memory leak.
    _import_local.names = set()

  _import_local.nest_level += 1


# pylint: disable=redefined-builtin
def _ProcessImportBySuffix(name, fromlist, globals):
  """Processes an import.

  Calculates the possible names generated from an import and invokes
  registered callbacks if needed.

  Args:
    name: Argument as passed to the importer.
    fromlist: Argument as passed to the importer.
    globals: Argument as passed to the importer.
  """
  _import_local.nest_level -= 1

  # To improve common code path performance, compute the loaded modules only
  # if there are any import callbacks.
  if _import_callbacks:
    # Collect the names of all modules that might be newly loaded as a result
    # of this import. Add them in a thread-local list.
    _import_local.names |= _GenerateNames(name, fromlist, globals)

    # Invoke the callbacks only on the top-level import call.
    if _import_local.nest_level == 0:
      _InvokeImportCallbackBySuffix(_import_local.names)

  # To be safe, we clear the names set every time we exit a top level import.
  if _import_local.nest_level == 0:
    _import_local.names.clear()


# pylint: disable=redefined-builtin, g-doc-args, g-doc-return-or-yield
def _ImportHookBySuffix(
    name, globals=None, locals=None, fromlist=None, level=-1):
  """Callback when an import statement is executed by the Python interpreter.

  Argument names have to exactly match those of __import__. Otherwise calls
  to __import__ that use keyword syntax will fail: __import('a', fromlist=[]).
  """
  _IncrementNestLevel()

  try:
    # Really import modules.
    module = _real_import(name, globals, locals, fromlist, level)
  finally:
    # This _real_import call may raise an exception (e.g., ImportError).<commit_msg>Set default of the level argument of __import__ to 0 in Python 3. https://github.com/GoogleCloudPlatform/cloud-debug-python/issues/5<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Support for breakpoints on modules that haven't been loaded yet.

This is the new module import hook which:
  1. Takes a partial path of the module file excluding the file extension as
     input (can be as short as 'foo' or longer such as 'sys/path/pkg/foo').
  2. At each (top-level-only) import statement:
    a. Generates an estimate of the modules that might be loaded as a result
       of this import (and all chained imports) using the arguments of the
       import hook. The estimate is best-effort, it may contain extra entries
       that are not of interest to us (e.g., outer packages that were already
       loaded before this import), or may be missing some module names (not
       all intricacies of Python module importer are handled).
    b. Checks sys.modules if any of these modules have a file that matches the
       given path, using suffix match.

For the old module import hook, see imphook.py file.
"""

import importlib
import itertools
import os
import sys  # Must be imported, otherwise import hooks don't work.
import threading

import six
from six.moves import builtins  # pylint: disable=redefined-builtin

from . import module_utils2

# Callbacks to invoke when a module is imported.
_import_callbacks = {}
_import_callbacks_lock = threading.Lock()

# Per thread data holding information about the import call nest level.
_import_local = threading.local()

# Original __import__ function if import hook is installed or None otherwise.
_real_import = None

# Original importlib.import_module function if import hook is installed or None
# otherwise.
_real_import_module = None


def AddImportCallbackBySuffix(path, callback):
  """Register import hook.

  This function overrides the default import process. Then whenever a module
  whose suffix matches path is imported, the callback will be invoked.

  A module may be imported multiple times. Import event only means that the
  Python code contained an "import" statement. The actual loading and
  initialization of a new module normally happens only once, at which time
  the callback will be invoked. This function does not validates the existence
  of such a module and it's the responsibility of the caller.

  TODO(erezh): handle module reload.

  Args:
    path: python module file path. It may be missing the directories for the
          outer packages, and therefore, requires suffix comparison to match
          against loaded modules. If it contains all outer packages, it may
          contain the sys.path as well.
          It might contain an incorrect file extension (e.g., py vs. pyc).
    callback: callable to invoke upon module load.

  Returns:
    Function object to invoke to remove the installed callback.
  """

  def RemoveCallback():
    # This is a read-if-del operation on _import_callbacks. Lock to prevent
    # callbacks from being inserted just before the key is deleted. Thus, it
    # must be locked also when inserting a new entry below. On the other hand
    # read only access, in the import hook, does not require a lock.
    with _import_callbacks_lock:
      callbacks = _import_callbacks.get(path)
      if callbacks:
        callbacks.remove(callback)
        if not callbacks:
          del _import_callbacks[path]

  with _import_callbacks_lock:
    _import_callbacks.setdefault(path, set()).add(callback)
  _InstallImportHookBySuffix()

  return RemoveCallback


def _InstallImportHookBySuffix():
  """Lazily installs import hook."""
  global _real_import

  if _real_import:
    return  # Import hook already installed

  _real_import = getattr(builtins, '__import__')
  assert _real_import
  builtins.__import__ = _ImportHookBySuffix

  if six.PY3:
    # In Python 2, importlib.import_module calls __import__ internally so
    # overriding __import__ is enough. In Python 3, they are separate so it also
    # needs to be overwritten.
    global _real_import_module
    _real_import_module = importlib.import_module
    assert _real_import_module
    importlib.import_module = _ImportModuleHookBySuffix


def _IncrementNestLevel():
  """Increments the per thread nest level of imports."""
  # This is the top call to import (no nesting), init the per-thread nest level
  # and names set.
  if getattr(_import_local, 'nest_level', None) is None:
    _import_local.nest_level = 0

  if _import_local.nest_level == 0:
    # Re-initialize names set at each top-level import to prevent any
    # accidental unforeseen memory leak.
    _import_local.names = set()

  _import_local.nest_level += 1


# pylint: disable=redefined-builtin
def _ProcessImportBySuffix(name, fromlist, globals):
  """Processes an import.

  Calculates the possible names generated from an import and invokes
  registered callbacks if needed.

  Args:
    name: Argument as passed to the importer.
    fromlist: Argument as passed to the importer.
    globals: Argument as passed to the importer.
  """
  _import_local.nest_level -= 1

  # To improve common code path performance, compute the loaded modules only
  # if there are any import callbacks.
  if _import_callbacks:
    # Collect the names of all modules that might be newly loaded as a result
    # of this import. Add them in a thread-local list.
    _import_local.names |= _GenerateNames(name, fromlist, globals)

    # Invoke the callbacks only on the top-level import call.
    if _import_local.nest_level == 0:
      _InvokeImportCallbackBySuffix(_import_local.names)

  # To be safe, we clear the names set every time we exit a top level import.
  if _import_local.nest_level == 0:
    _import_local.names.clear()


# pylint: disable=redefined-builtin, g-doc-args, g-doc-return-or-yield
def _ImportHookBySuffix(
    name, globals=None, locals=None, fromlist=None, level=None):
  """Callback when an import statement is executed by the Python interpreter.

  Argument names have to exactly match those of __import__. Otherwise calls
  to __import__ that use keyword syntax will fail: __import('a', fromlist=[]).
  """
  _IncrementNestLevel()

  if level is None:
    # A level of 0 means absolute import, positive values means relative
    # imports, and -1 means to try both an absolute and relative import.
    # Since imports were disambiguated in Python 3, -1 is not a valid value.
    # The default values are 0 and -1 for Python 3 and 3 respectively.
    # https://docs.python.org/2/library/functions.html#__import__
    # https://docs.python.org/3/library/functions.html#__import__
    level = 0 if six.PY3 else -1

  try:
    # Really import modules.
    module = _real_import(name, globals, locals, fromlist, level)
  finally:
    # This _real_import call may raise an exception (e.g., ImportError).
{'fraction_non_alphanumeric': 0.06139058443836385, 'fraction_numerical': 0.003999158071984845, 'mean_word_length': 3.6642670157068062, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1063262', 'n_tokens_mistral': 3944, 'n_tokens_neox': 3820, 'n_words': 1755}