<commit_before>import compileall
import fnmatch
import io
import os
import re
import shutil
import subprocess
import sys
import time
from contextlib import contextmanager

import pytest
import six
from mock import patch
from pip._vendor.contextlib2 import ExitStack, nullcontext
from setuptools.wheel import Wheel

from pip._internal.cli.main import main as pip_entry_point
from pip._internal.utils.temp_dir import global_tempdir_manager
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData
from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key
from tests.lib.path import Path
from tests.lib.server import make_mock_server, server_running
from tests.lib.venv import VirtualEnvironment

if MYPY_CHECK_RUNNING:
    from typing import Dict, Iterable

    from tests.lib.server import MockServer as _MockServer
    from tests.lib.server import Responder


def pytest_addoption(parser):
    parser.addoption(
        "--keep-tmpdir",
        action="store_true",
        default=False,
        help="keep temporary test directories",
    )
    parser.addoption(
        "--new-resolver",
        action="store_true",
        default=False,
        help="use new resolver in tests",
    )
    parser.addoption(
        "--new-resolver-runtests",
        action="store_true",
        default=False,
        help="run the skipped tests for the new resolver",
    )
    parser.addoption(
        "--use-venv",
        action="store_true",
        default=False,
        help="use venv for virtual environment creation",
    )


def pytest_collection_modifyitems(config, items):
    for item in items:
        if not hasattr(item, 'module'):  # e.g.: DoctestTextfile
            continue

        if "CI" in os.environ:
            # Mark network tests as flaky
            if item.get_closest_marker('network') is not None:
                item.add_marker(pytest.mark.flaky(reruns=3, reruns_delay=2))

        if six.PY3:
            if (item.get_closest_marker('incompatible_with_test_venv') and
                    config.getoption("--use-venv")):
                item.add_marker(pytest.mark.skip(
                    'Incompatible with test venv'))
            if (item.get_closest_marker('incompatible_with_venv') and
                    sys.prefix != sys.base_prefix):
                item.add_marker(pytest.mark.skip(
                    'Incompatible with venv'))

        module_path = os.path.relpath(
            item.module.__file__,
            os.path.commonprefix([__file__, item.module.__file__]),
        )

        module_root_dir = module_path.split(os.pathsep)[0]
        if (module_root_dir.startswith("functional") or
                module_root_dir.startswith("integration") or
                module_root_dir.startswith("lib")):
            item.add_marker(pytest.mark.integration)
        elif module_root_dir.startswith("unit"):
            item.add_marker(pytest.mark.unit)
        else:
            raise RuntimeError(
                "Unknown test type (filename = {})".format(module_path)
            )


@pytest.fixture(scope="session", autouse=True)
def resolver_variant(request):
    """Set environment variable to make pip default to the correct resolver.
    """
    new_resolver = request.config.getoption("--new-resolver")
    features = set(os.environ.get("PIP_USE_FEATURE", "").split())
    if new_resolver:
        retval = "2020-resolver"
        features.add("2020-resolver")
    else:
        retval = "legacy"
        features.discard("2020-resolver")

    with patch.dict(os.environ, {"PIP_USE_FEATURE": " ".join(features)}):
        yield retval

<commit_msg>Get the resolver name directly from test CLI<commit_after>import compileall
import fnmatch
import io
import os
import re
import shutil
import subprocess
import sys
import time
from contextlib import contextmanager

import pytest
import six
from mock import patch
from pip._vendor.contextlib2 import ExitStack, nullcontext
from setuptools.wheel import Wheel

from pip._internal.cli.main import main as pip_entry_point
from pip._internal.utils.temp_dir import global_tempdir_manager
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from tests.lib import DATA_DIR, SRC_DIR, PipTestEnvironment, TestData
from tests.lib.certs import make_tls_cert, serialize_cert, serialize_key
from tests.lib.path import Path
from tests.lib.server import make_mock_server, server_running
from tests.lib.venv import VirtualEnvironment

if MYPY_CHECK_RUNNING:
    from typing import Dict, Iterable

    from tests.lib.server import MockServer as _MockServer
    from tests.lib.server import Responder


def pytest_addoption(parser):
    parser.addoption(
        "--keep-tmpdir",
        action="store_true",
        default=False,
        help="keep temporary test directories",
    )
    parser.addoption(
        "--resolver",
        action="store",
        default="2020-resolver",
        choices=["2020-resolver", "legacy"],
        help="use given resolver in tests",
    )
    parser.addoption(
        "--use-venv",
        action="store_true",
        default=False,
        help="use venv for virtual environment creation",
    )


def pytest_collection_modifyitems(config, items):
    for item in items:
        if not hasattr(item, 'module'):  # e.g.: DoctestTextfile
            continue

        if "CI" in os.environ:
            # Mark network tests as flaky
            if item.get_closest_marker('network') is not None:
                item.add_marker(pytest.mark.flaky(reruns=3, reruns_delay=2))

        if six.PY3:
            if (item.get_closest_marker('incompatible_with_test_venv') and
                    config.getoption("--use-venv")):
                item.add_marker(pytest.mark.skip(
                    'Incompatible with test venv'))
            if (item.get_closest_marker('incompatible_with_venv') and
                    sys.prefix != sys.base_prefix):
                item.add_marker(pytest.mark.skip(
                    'Incompatible with venv'))

        module_path = os.path.relpath(
            item.module.__file__,
            os.path.commonprefix([__file__, item.module.__file__]),
        )

        module_root_dir = module_path.split(os.pathsep)[0]
        if (module_root_dir.startswith("functional") or
                module_root_dir.startswith("integration") or
                module_root_dir.startswith("lib")):
            item.add_marker(pytest.mark.integration)
        elif module_root_dir.startswith("unit"):
            item.add_marker(pytest.mark.unit)
        else:
            raise RuntimeError(
                "Unknown test type (filename = {})".format(module_path)
            )


@pytest.fixture(scope="session", autouse=True)
def resolver_variant(request):
    """Set environment variable to make pip default to the correct resolver.
    """
    resolver = request.config.getoption("--resolver")

    # Handle the environment variables for this test.
    features = set(os.environ.get("PIP_USE_FEATURE", "").split())
    if new_resolver:
        features.add("2020-resolver")
    else:
        features.discard("2020-resolver")

    with patch.dict(os.environ, {"PIP_USE_FEATURE": " ".join(features)}):
        yield resolver


{'fraction_non_alphanumeric': 0.09542142169668637, 'fraction_numerical': 0.005224804069847381, 'mean_word_length': 2.6942610462163534, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3578086', 'n_tokens_mistral': 2157, 'n_tokens_neox': 2122, 'n_words': 494}