<commit_before>from __future__ import absolute_import

import logging
import os
import re
import shutil
import sys
import tempfile
import warnings
import zipfile

from distutils.util import change_root
from distutils import sysconfig
from email.parser import FeedParser

from pip._vendor import pkg_resources, six
from pip._vendor.distlib.markers import interpret as markers_interpret
from pip._vendor.six.moves import configparser

import pip.wheel

from pip.compat import native_str, WINDOWS
from pip.download import is_url, url_to_path, path_to_url, is_archive_file
from pip.exceptions import (
    InstallationError, UninstallationError, UnsupportedWheel,
)
from pip.locations import (
    bin_py, running_under_virtualenv, PIP_DELETE_MARKER_FILENAME, bin_user,
)
from pip.utils import (
    display_path, rmtree, ask_path_exists, backup_dir, is_installable_dir,
    dist_in_usersite, dist_in_site_packages, egg_link_path, make_path_relative,
    call_subprocess, read_text_file, FakeFile, _make_build_dir, ensure_dir,
)
from pip.utils.deprecation import RemovedInPip8Warning
from pip.utils.logging import indent_log
from pip.req.req_uninstall import UninstallPathSet
from pip.vcs import vcs
from pip.wheel import move_wheel_files, Wheel
from pip._vendor.packaging.version import Version


_FILTER_INSTALL_OUTPUT_REGEX = re.compile(r"""
    (?:^running\s.*) |
    (?:^writing\s.*) |
    (?:creating\s.*) |
    (?:[Cc]opying\s.*) |
    (?:^reading\s.*') |
    (?:^removing\s.*\.egg-info'\s\(and\severything\sunder\sit\)$) |
    (?:^byte-compiling) |
    (?:^SyntaxError:) |
    (?:^SyntaxWarning:) |
    (?:^\s*Skipping\simplicit\sfixer:) |
    (?:^\s*(warning:\s)?no\spreviously-included\s(files|directories)) |
    (?:^\s*warning:\sno\sfiles\sfound matching\s\'.*\') |
    (?:^\s*changing\smode\sof) |
    # Not sure what this warning is, but it seems harmless:
    (?:^warning:\smanifest_maker:\sstandard\sfile\s'-c'\snot found$)
    """, re.VERBOSE)


logger = logging.getLogger(__name__)


def _filter_install(line):
    level = logging.INFO
    if _FILTER_INSTALL_OUTPUT_REGEX.search(line.strip()):
        level = logging.DEBUG
    return (level, line)


class InstallRequirement(object):

    def __init__(self, req, comes_from, source_dir=None, editable=False,
                 link=None, as_egg=False, update=True, editable_options=None,
                 pycompile=True, markers=None, isolated=False, options=None):
        self.extras = ()
        if isinstance(req, six.string_types):
            req = pkg_resources.Requirement.parse(req)
            self.extras = req.extras

        self.req = req
        self.comes_from = comes_from
        self.source_dir = source_dir
        self.editable = editable

        if editable_options is None:
            editable_options = {}

        self.editable_options = editable_options
        self.link = link
        self.as_egg = as_egg
        self.markers = markers
        self._egg_info_path = None
        # This holds the pkg_resources.Distribution object if this requirement
        # is already available:
        self.satisfied_by = None
        # This hold the pkg_resources.Distribution object if this requirement
        # conflicts with another installed distribution:
        self.conflicts_with = None
        # Temporary build location
        self._temp_build_dir = None
        # Used to store the global directory where the _temp_build_dir should
        # have been created. Cf _correct_build_location method.
        self._ideal_global_dir = None
        # True if the editable should be updated:
        self.update = update
        # Set to True after successful installation
        self.install_succeeded = None
        # UninstallPathSet of uninstalled distribution (for possible rollback)
        self.uninstalled = None
        self.use_user_site = False
        self.target_dir = None
        self.options = options if options else {}

        self.pycompile = pycompile

        self.isolated = isolated

    @classmethod
    def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
                      isolated=False):
        from pip.index import Link

        name, url, extras_override, editable_options = parse_editable(
            editable_req, default_vcs)
        if url.startswith('file:'):
            source_dir = url_to_path(url)
        else:
            source_dir = None

        res = cls(name, comes_from, source_dir=source_dir,
                  editable=True,
                  link=Link(url),
                  editable_options=editable_options,
                  isolated=isolated)

        if extras_override is not None:<commit_msg>Use options={} instead of options=None.<commit_after>from __future__ import absolute_import

import logging
import os
import re
import shutil
import sys
import tempfile
import warnings
import zipfile

from distutils.util import change_root
from distutils import sysconfig
from email.parser import FeedParser

from pip._vendor import pkg_resources, six
from pip._vendor.distlib.markers import interpret as markers_interpret
from pip._vendor.six.moves import configparser

import pip.wheel

from pip.compat import native_str, WINDOWS
from pip.download import is_url, url_to_path, path_to_url, is_archive_file
from pip.exceptions import (
    InstallationError, UninstallationError, UnsupportedWheel,
)
from pip.locations import (
    bin_py, running_under_virtualenv, PIP_DELETE_MARKER_FILENAME, bin_user,
)
from pip.utils import (
    display_path, rmtree, ask_path_exists, backup_dir, is_installable_dir,
    dist_in_usersite, dist_in_site_packages, egg_link_path, make_path_relative,
    call_subprocess, read_text_file, FakeFile, _make_build_dir, ensure_dir,
)
from pip.utils.deprecation import RemovedInPip8Warning
from pip.utils.logging import indent_log
from pip.req.req_uninstall import UninstallPathSet
from pip.vcs import vcs
from pip.wheel import move_wheel_files, Wheel
from pip._vendor.packaging.version import Version


_FILTER_INSTALL_OUTPUT_REGEX = re.compile(r"""
    (?:^running\s.*) |
    (?:^writing\s.*) |
    (?:creating\s.*) |
    (?:[Cc]opying\s.*) |
    (?:^reading\s.*') |
    (?:^removing\s.*\.egg-info'\s\(and\severything\sunder\sit\)$) |
    (?:^byte-compiling) |
    (?:^SyntaxError:) |
    (?:^SyntaxWarning:) |
    (?:^\s*Skipping\simplicit\sfixer:) |
    (?:^\s*(warning:\s)?no\spreviously-included\s(files|directories)) |
    (?:^\s*warning:\sno\sfiles\sfound matching\s\'.*\') |
    (?:^\s*changing\smode\sof) |
    # Not sure what this warning is, but it seems harmless:
    (?:^warning:\smanifest_maker:\sstandard\sfile\s'-c'\snot found$)
    """, re.VERBOSE)


logger = logging.getLogger(__name__)


def _filter_install(line):
    level = logging.INFO
    if _FILTER_INSTALL_OUTPUT_REGEX.search(line.strip()):
        level = logging.DEBUG
    return (level, line)


class InstallRequirement(object):

    def __init__(self, req, comes_from, source_dir=None, editable=False,
                 link=None, as_egg=False, update=True, editable_options=None,
                 pycompile=True, markers=None, isolated=False, options={}):
        self.extras = ()
        if isinstance(req, six.string_types):
            req = pkg_resources.Requirement.parse(req)
            self.extras = req.extras

        self.req = req
        self.comes_from = comes_from
        self.source_dir = source_dir
        self.editable = editable

        if editable_options is None:
            editable_options = {}

        self.editable_options = editable_options
        self.link = link
        self.as_egg = as_egg
        self.markers = markers
        self._egg_info_path = None
        # This holds the pkg_resources.Distribution object if this requirement
        # is already available:
        self.satisfied_by = None
        # This hold the pkg_resources.Distribution object if this requirement
        # conflicts with another installed distribution:
        self.conflicts_with = None
        # Temporary build location
        self._temp_build_dir = None
        # Used to store the global directory where the _temp_build_dir should
        # have been created. Cf _correct_build_location method.
        self._ideal_global_dir = None
        # True if the editable should be updated:
        self.update = update
        # Set to True after successful installation
        self.install_succeeded = None
        # UninstallPathSet of uninstalled distribution (for possible rollback)
        self.uninstalled = None
        self.use_user_site = False
        self.target_dir = None
        self.options = options

        self.pycompile = pycompile

        self.isolated = isolated

    @classmethod
    def from_editable(cls, editable_req, comes_from=None, default_vcs=None,
                      isolated=False):
        from pip.index import Link

        name, url, extras_override, editable_options = parse_editable(
            editable_req, default_vcs)
        if url.startswith('file:'):
            source_dir = url_to_path(url)
        else:
            source_dir = None

        res = cls(name, comes_from, source_dir=source_dir,
                  editable=True,
                  link=Link(url),
                  editable_options=editable_options,
                  isolated=isolated)

        if extras_override is not None:
{'fraction_non_alphanumeric': 0.09092856377851559, 'fraction_numerical': 0.0002142015636714148, 'mean_word_length': 2.983788395904437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6681566', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2736, 'n_words': 666}