<commit_before># Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import hashlib
import logging
import os
import site

from pex import resolver
from pex.base import requirement_is_exact
from pex.interpreter import PythonInterpreter
from pkg_resources import Requirement
from pkg_resources import working_set as global_working_set
from wheel.install import WheelFile

from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.option.global_options import GlobalOptionsRegistrar
from pants.util.contextutil import temporary_dir
from pants.util.dirutil import safe_mkdir, safe_open
from pants.util.memo import memoized_property
from pants.util.strutil import ensure_text
from pants.version import PANTS_SEMVER


logger = logging.getLogger(__name__)


class PluginResolver:
  @staticmethod
  def _is_wheel(path):
    return os.path.isfile(path) and path.endswith('.whl')

  @classmethod
  def _activate_wheel(cls, wheel_path):
    install_dir = '{}-install'.format(wheel_path)
    if not os.path.isdir(install_dir):
      with temporary_dir(root_dir=os.path.dirname(install_dir)) as tmp:
        cls._install_wheel(wheel_path, tmp)
        os.rename(tmp, install_dir)
    # Activate any .pth files installed above.
    site.addsitedir(install_dir)
    return install_dir

  @classmethod
  def _install_wheel(cls, wheel_path, install_dir):
    safe_mkdir(install_dir, clean=True)
    WheelFile(wheel_path).install(force=True,
                                  overrides={
                                    'purelib': install_dir,
                                    'headers': os.path.join(install_dir, 'headers'),
                                    'scripts': os.path.join(install_dir, 'bin'),
                                    'platlib': install_dir,
                                    'data': install_dir
                                  })

  def __init__(self, options_bootstrapper, *, interpreter=None):
    self._options_bootstrapper = options_bootstrapper
    self._interpreter = interpreter or PythonInterpreter.get()

    bootstrap_options = self._options_bootstrapper.get_bootstrap_options().for_global_scope()
    self._plugin_requirements = bootstrap_options.plugins
    self._plugin_cache_dir = bootstrap_options.plugin_cache_dir

  def resolve(self, working_set=None):
    """Resolves any configured plugins and adds them to the global working set.

    :param working_set: The working set to add the resolved plugins to instead of the global
                        working set (for testing).
    :type: :class:`pkg_resources.WorkingSet`
    """
    working_set = working_set or global_working_set
    if self._plugin_requirements:
      for plugin_location in self._resolve_plugin_locations():
        if self._is_wheel(plugin_location):
          plugin_location = self._activate_wheel(plugin_location)
        working_set.add_entry(plugin_location)
    return working_set

  def _resolve_plugin_locations(self):
    # We jump through some hoops here to avoid a live resolve if possible for purposes of speed.
    # Even with a local resolve cache fully up to date, running a resolve to activate a plugin
    # takes ~250ms whereas loading from a pre-cached list takes ~50ms.
    if all(requirement_is_exact(Requirement.parse(req)) for req in self._plugin_requirements):
      return self._resolve_exact_plugin_locations()
    else:
      return (plugin.location for plugin in self._resolve_plugins())

  def _resolve_exact_plugin_locations(self):
    hasher = hashlib.sha1()

    # Assume we have platform-specific plugin requirements and pessimistically mix the ABI
    # identifier into the hash to ensure re-resolution of plugins for different interpreter ABIs.
    hasher.update(self._interpreter.identity.abi_tag.encode())  # EG: cp36m

    for req in sorted(self._plugin_requirements):<commit_msg>Don't choke on plugin requirements that are URLs. (#8708)<commit_after># Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import hashlib
import logging
import os
import site

from pex import resolver
from pex.base import requirement_is_exact
from pex.interpreter import PythonInterpreter
from pkg_resources import Requirement, RequirementParseError
from pkg_resources import working_set as global_working_set
from wheel.install import WheelFile

from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.option.global_options import GlobalOptionsRegistrar
from pants.util.contextutil import temporary_dir
from pants.util.dirutil import safe_mkdir, safe_open
from pants.util.memo import memoized_property
from pants.util.strutil import ensure_text
from pants.version import PANTS_SEMVER


logger = logging.getLogger(__name__)


class PluginResolver:
  @staticmethod
  def _is_wheel(path):
    return os.path.isfile(path) and path.endswith('.whl')

  @classmethod
  def _activate_wheel(cls, wheel_path):
    install_dir = '{}-install'.format(wheel_path)
    if not os.path.isdir(install_dir):
      with temporary_dir(root_dir=os.path.dirname(install_dir)) as tmp:
        cls._install_wheel(wheel_path, tmp)
        os.rename(tmp, install_dir)
    # Activate any .pth files installed above.
    site.addsitedir(install_dir)
    return install_dir

  @classmethod
  def _install_wheel(cls, wheel_path, install_dir):
    safe_mkdir(install_dir, clean=True)
    WheelFile(wheel_path).install(force=True,
                                  overrides={
                                    'purelib': install_dir,
                                    'headers': os.path.join(install_dir, 'headers'),
                                    'scripts': os.path.join(install_dir, 'bin'),
                                    'platlib': install_dir,
                                    'data': install_dir
                                  })

  def __init__(self, options_bootstrapper, *, interpreter=None):
    self._options_bootstrapper = options_bootstrapper
    self._interpreter = interpreter or PythonInterpreter.get()

    bootstrap_options = self._options_bootstrapper.get_bootstrap_options().for_global_scope()
    self._plugin_requirements = bootstrap_options.plugins
    self._plugin_cache_dir = bootstrap_options.plugin_cache_dir

  def resolve(self, working_set=None):
    """Resolves any configured plugins and adds them to the global working set.

    :param working_set: The working set to add the resolved plugins to instead of the global
                        working set (for testing).
    :type: :class:`pkg_resources.WorkingSet`
    """
    working_set = working_set or global_working_set
    if self._plugin_requirements:
      for plugin_location in self._resolve_plugin_locations():
        if self._is_wheel(plugin_location):
          plugin_location = self._activate_wheel(plugin_location)
        working_set.add_entry(plugin_location)
    return working_set

  def _resolve_plugin_locations(self):
    def _requirement_is_exact(req):
      try:
        return requirement_is_exact(Requirement.parse(req))
      except RequirementParseError:
        # Requirement string may be URL, which Requirement.parse does not handle.
        return False

    # We jump through some hoops here to avoid a live resolve if possible for purposes of speed.
    # Even with a local resolve cache fully up to date, running a resolve to activate a plugin
    # takes ~250ms whereas loading from a pre-cached list takes ~50ms.
    # TODO: Never live-resolve, even if we can't prove that requirements aren't exact.
    #  Instead, support some way of forcing a new resolve. This will allow requirement strings
    #  that are not of the foo==x.y.z form, e.g., URLs to .whl files.
    if all(_requirement_is_exact(req) for req in self._plugin_requirements):
      return self._resolve_exact_plugin_locations()
    else:
      return (plugin.location for plugin in self._resolve_plugins())

  def _resolve_exact_plugin_locations(self):
    hasher = hashlib.sha1()

    # Assume we have platform-specific plugin requirements and pessimistically mix the ABI
    # identifier into the hash to ensure re-resolution of plugins for different interpreter ABIs.
    hasher.update(self._interpreter.identity.abi_tag.encode())  # EG: cp36m

    for req in sorted(self._plugin_requirements):
{'fraction_non_alphanumeric': 0.0699833373006427, 'fraction_numerical': 0.0038086169959533444, 'mean_word_length': 3.3448810754912097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '660371', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2251, 'n_words': 671}