<commit_before>from __future__ import absolute_import

import io
import logging
import os
import re
import shutil
import sys
import sysconfig
import traceback
import zipfile
from distutils.util import change_root
from email.parser import FeedParser  # type: ignore

from pip._vendor import pkg_resources, pytoml, six
from pip._vendor.packaging import specifiers
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements

from pip._internal import wheel
from pip._internal.build_env import NoOpBuildEnvironment
from pip._internal.download import (
    is_archive_file, is_url, path_to_url, url_to_path,
)
from pip._internal.exceptions import InstallationError
from pip._internal.locations import (
    PIP_DELETE_MARKER_FILENAME, running_under_virtualenv,
)
from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.req.req_uninstall import UninstallPathSet
from pip._internal.utils.compat import native_str
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import (
    _make_build_dir, ask_path_exists, backup_dir, call_subprocess,
    display_path, dist_in_site_packages, dist_in_usersite, ensure_dir,
    get_installed_version, is_installable_dir, read_text_file, rmtree,
)
from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.ui import open_spinner
from pip._internal.vcs import vcs
from pip._internal.wheel import Wheel, move_wheel_files

logger = logging.getLogger(__name__)

operators = specifiers.Specifier._operators.keys()


def _strip_extras(path):
    m = re.match(r'^(.+)(\[[^\]]+\])$', path)
    extras = None
    if m:
        path_no_extras = m.group(1)
        extras = m.group(2)
    else:
        path_no_extras = path

    return path_no_extras, extras


class InstallRequirement(object):
    """
    Represents something that may be installed later on, may have information
    about where to fetch the relavant requirement and also contains logic for
    installing the said requirement.
    """

    def __init__(self, req, comes_from, source_dir=None, editable=False,
                 link=None, update=True, markers=None,
                 isolated=False, options=None, wheel_cache=None,
                 constraint=False, extras=()):
        assert req is None or isinstance(req, Requirement), req
        self.req = req
        self.comes_from = comes_from
        self.constraint = constraint
        if source_dir is not None:
            self.source_dir = os.path.normpath(os.path.abspath(source_dir))
        else:
            self.source_dir = None
        self.editable = editable

        self._wheel_cache = wheel_cache
        if link is not None:
            self.link = self.original_link = link
        else:
            from pip._internal.index import Link
            self.link = self.original_link = req and req.url and Link(req.url)

        if extras:
            self.extras = extras
        elif req:
            self.extras = {
                pkg_resources.safe_extra(extra) for extra in req.extras
            }
        else:
            self.extras = set()
        if markers is not None:
            self.markers = markers
        else:
            self.markers = req and req.marker
        self._egg_info_path = None
        # This holds the pkg_resources.Distribution object if this requirement
        # is already available:
        self.satisfied_by = None
        # This hold the pkg_resources.Distribution object if this requirement
        # conflicts with another installed distribution:
        self.conflicts_with = None
        # Temporary build location
        self._temp_build_dir = TempDirectory(kind="req-build")
        # Used to store the global directory where the _temp_build_dir should
        # have been created. Cf _correct_build_location method.
        self._ideal_build_dir = None
        # True if the editable should be updated:
        self.update = update
        # Set to True after successful installation
        self.install_succeeded = None
        # UninstallPathSet of uninstalled distribution (for possible rollback)
        self.uninstalled_pathset = None
        self.options = options if options else {}
        # Set to True after successful preparation of this requirement
        self.prepared = False
        self.is_direct = False

        self.isolated = isolated
        self.build_env = NoOpBuildEnvironment()

        # pyproject.toml handling
        self._pyproject_toml_loaded = False
        self._pyproject_requires = None
        self._pyproject_backend = None

        # Are we using PEP 517 for this requirement?
        # This can be None until load_pyproject_toml has been called,
        # after which it will have an explicit value.
        # TODO: Maybe make this a property, but if we do we'll need
        # to set the backing value based on the --use-pep517 flag.
        self.use_pep517 = None

    # Constructors
    #   TODO: Move these out of this class into custom methods.
    @classmethod
    def from_editable(cls, editable_req, comes_from=None, isolated=False,
                      options=None, wheel_cache=None, constraint=False):
        name, url, extras_override = parse_editable(editable_req)
        if url.startswith('file:'):
            source_dir = url_to_path(url)<commit_msg>Updated comment omitted from previous commit<commit_after>from __future__ import absolute_import

import io
import logging
import os
import re
import shutil
import sys
import sysconfig
import traceback
import zipfile
from distutils.util import change_root
from email.parser import FeedParser  # type: ignore

from pip._vendor import pkg_resources, pytoml, six
from pip._vendor.packaging import specifiers
from pip._vendor.packaging.markers import Marker
from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pip._vendor.packaging.utils import canonicalize_name
from pip._vendor.packaging.version import Version
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.pkg_resources import RequirementParseError, parse_requirements

from pip._internal import wheel
from pip._internal.build_env import NoOpBuildEnvironment
from pip._internal.download import (
    is_archive_file, is_url, path_to_url, url_to_path,
)
from pip._internal.exceptions import InstallationError
from pip._internal.locations import (
    PIP_DELETE_MARKER_FILENAME, running_under_virtualenv,
)
from pip._internal.models.index import PyPI, TestPyPI
from pip._internal.models.link import Link
from pip._internal.req.req_uninstall import UninstallPathSet
from pip._internal.utils.compat import native_str
from pip._internal.utils.hashes import Hashes
from pip._internal.utils.logging import indent_log
from pip._internal.utils.misc import (
    _make_build_dir, ask_path_exists, backup_dir, call_subprocess,
    display_path, dist_in_site_packages, dist_in_usersite, ensure_dir,
    get_installed_version, is_installable_dir, read_text_file, rmtree,
)
from pip._internal.utils.setuptools_build import SETUPTOOLS_SHIM
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.ui import open_spinner
from pip._internal.vcs import vcs
from pip._internal.wheel import Wheel, move_wheel_files

logger = logging.getLogger(__name__)

operators = specifiers.Specifier._operators.keys()


def _strip_extras(path):
    m = re.match(r'^(.+)(\[[^\]]+\])$', path)
    extras = None
    if m:
        path_no_extras = m.group(1)
        extras = m.group(2)
    else:
        path_no_extras = path

    return path_no_extras, extras


class InstallRequirement(object):
    """
    Represents something that may be installed later on, may have information
    about where to fetch the relavant requirement and also contains logic for
    installing the said requirement.
    """

    def __init__(self, req, comes_from, source_dir=None, editable=False,
                 link=None, update=True, markers=None,
                 isolated=False, options=None, wheel_cache=None,
                 constraint=False, extras=()):
        assert req is None or isinstance(req, Requirement), req
        self.req = req
        self.comes_from = comes_from
        self.constraint = constraint
        if source_dir is not None:
            self.source_dir = os.path.normpath(os.path.abspath(source_dir))
        else:
            self.source_dir = None
        self.editable = editable

        self._wheel_cache = wheel_cache
        if link is not None:
            self.link = self.original_link = link
        else:
            from pip._internal.index import Link
            self.link = self.original_link = req and req.url and Link(req.url)

        if extras:
            self.extras = extras
        elif req:
            self.extras = {
                pkg_resources.safe_extra(extra) for extra in req.extras
            }
        else:
            self.extras = set()
        if markers is not None:
            self.markers = markers
        else:
            self.markers = req and req.marker
        self._egg_info_path = None
        # This holds the pkg_resources.Distribution object if this requirement
        # is already available:
        self.satisfied_by = None
        # This hold the pkg_resources.Distribution object if this requirement
        # conflicts with another installed distribution:
        self.conflicts_with = None
        # Temporary build location
        self._temp_build_dir = TempDirectory(kind="req-build")
        # Used to store the global directory where the _temp_build_dir should
        # have been created. Cf _correct_build_location method.
        self._ideal_build_dir = None
        # True if the editable should be updated:
        self.update = update
        # Set to True after successful installation
        self.install_succeeded = None
        # UninstallPathSet of uninstalled distribution (for possible rollback)
        self.uninstalled_pathset = None
        self.options = options if options else {}
        # Set to True after successful preparation of this requirement
        self.prepared = False
        self.is_direct = False

        self.isolated = isolated
        self.build_env = NoOpBuildEnvironment()

        # pyproject.toml handling
        self._pyproject_toml_loaded = False
        self._pyproject_requires = None
        self._pyproject_backend = None

        # Are we using PEP 517 for this requirement?
        # After pyproject.toml has been loaded, the only valid values are True
        # and False. Before loading, None is valid (meaning "use the default").
        # Setting an explicit value before loading pyproject.toml is supported,
        # but after loading this flag should be treated as read only.
        self.use_pep517 = None

    # Constructors
    #   TODO: Move these out of this class into custom methods.
    @classmethod
    def from_editable(cls, editable_req, comes_from=None, isolated=False,
                      options=None, wheel_cache=None, constraint=False):
        name, url, extras_override = parse_editable(editable_req)
        if url.startswith('file:'):
            source_dir = url_to_path(url)
{'fraction_non_alphanumeric': 0.06432446396280031, 'fraction_numerical': 0.0016360974769654697, 'mean_word_length': 2.984219554030875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4135555', 'n_tokens_mistral': 3243, 'n_tokens_neox': 3138, 'n_words': 971}