This is on top of #2225. The goal is to be able to skip specific tests. Many core test sets have just one test that uses subprocess or threads. Others have a few tests that are failing due to upstream problems in emscripten. I went through and switched to skipping specific tests in as many cases as I could. In modules where most or all tests use threading or subprocess, I gave up and put a comment saying something like 35/50 tests fork and I didn't want to bother to salvage the remaining ones.

In order to accommodate using patterns to skip specific tests (and also extra comments) I switched the format of `python_tests` to a yaml file. This gives us significantly more flexibility in the structure of the file and allows us to use `ruamel.yaml` to update it. We gain a lot flexibility while reducing the length of `make_test_lists.py` modestly from 108 to 78 loc.

In the future, hopefully we should be able to make fixes by deleting one skip pattern and looking into why that particular test fails.

Rework core tests. This is on top of #2225. The goal is to be able to skip specific tests. Many core test sets have just one test that uses subprocess or threads. Others have a few tests that are failing due to upstream problems in emscripten. I went through and switched to skipping specific tests in as many cases as I could. In modules where most or all tests use threading or subprocess, I gave up and put a comment saying something like 35/50 tests fork and I didn't want to bother to salvage the remaining ones.

In order to accommodate using patterns to skip specific tests (and also extra comments) I switched the format of `python_tests` to a yaml file. This gives us significantly more flexibility in the structure of the file and allows us to use `ruamel.yaml` to update it. We gain a lot flexibility while reducing the length of `make_test_lists.py` modestly from 108 to 78 loc.

In the future, hopefully we should be able to make fixes by deleting one skip pattern and looking into why that particular test fails.

from pathlib import Path

import pytest

from pyodide_build.common import UNVENDORED_STDLIB_MODULES


def test_cpython_core(python_test, selenium, request):

    name, error_flags = python_test

    # keep only flags related to the current browser
    flags_to_remove = ["firefox", "chrome", "node"]
    flags_to_remove.remove(selenium.browser)
    for flag in flags_to_remove:
        if "crash-" + flag in error_flags:
            error_flags.remove("crash-" + flag)

    if any(flag.startswith("segfault") for flag in error_flags):
        pytest.skip('known segfault: "{}"'.format(" ".join(error_flags)))

    if error_flags:
        if request.config.option.run_xfail:
            request.applymarker(
                pytest.mark.xfail(
                    run=False,
                    reason='known failure: "{}"'.format(" ".join(error_flags)),
                )
            )
        else:
            pytest.xfail('known failure: "{}"'.format(" ".join(error_flags)))

    selenium.load_package(list(UNVENDORED_STDLIB_MODULES))
    try:
        selenium.run(
            """
            from test.libregrtest import main
            import unittest.case
            try:
                main(['{}'], verbose=True, verbose3=True)
            except SystemExit as e:
                if e.code != 0:
                    raise RuntimeError(f'Failed with code: {{e.code}}')
            except unittest.case.SkipTest:
                pass
            """.format(
                name
            )
        )
    except selenium.JavascriptException:
        print(selenium.logs)
        raise


def pytest_generate_tests(metafunc):
    if "python_test" in metafunc.fixturenames:
        test_modules = []
        test_modules_ids = []
        with open(Path(__file__).parent / "python_tests.txt") as fp:
            for line in fp:
                line = line.strip()
                if line.startswith("#") or not line:
                    continue
                error_flags = line.split()
                name = error_flags.pop(0)
                test_modules.append((name, error_flags))
                # explicitly define test ids to keep
                # a human readable test name in pytest
                test_modules_ids.append(name)
        metafunc.parametrize("python_test", test_modules, ids=test_modules_ids)

from functools import cache
from pathlib import Path
from textwrap import dedent
from typing import Any

import pytest
import yaml
from yaml import CLoader as Loader

from pyodide_build.common import UNVENDORED_STDLIB_MODULES


def filter_info(info: dict[str, Any], browser: str) -> dict[str, Any]:
    info = dict(info)
    # keep only flags related to the current browser
    flags_to_remove = ["firefox", "chrome", "node"]
    flags_to_remove.remove(browser)
    for browser in flags_to_remove:
        for key in list(info.keys()):
            if key.endswith(browser):
                del info[key]
    return info


def possibly_skip_test(request, info: dict[str, Any]) -> dict[str, Any]:
    for reason in (
        reason for (flag, reason) in info.items() if flag.startswith("segfault")
    ):
        pytest.skip(f"known segfault: {reason}")

    for reason in (
        reason for [flag, reason] in info.items() if flag.startswith("xfail")
    ):
        if request.config.option.run_xfail:
            request.applymarker(
                pytest.mark.xfail(
                    run=False,
                    reason=f"known failure: {reason}",
                )
            )
        else:
            pytest.xfail(f"known failure: {reason}")
    return info


def test_cpython_core(main_test, selenium, request):
    [name, info] = main_test
    info = filter_info(info, selenium.browser)
    possibly_skip_test(request, info)

    ignore_tests = info.get("skip", [])
    if not isinstance(ignore_tests, list):
        raise Exception("Invalid python_tests.yaml entry: 'skip' should be a list")

    selenium.load_package(list(UNVENDORED_STDLIB_MODULES))
    try:
        selenium.run(
            dedent(
                f"""
            import platform
            from test import libregrtest

            platform.platform(aliased=True)
            import _testcapi
            if hasattr(_testcapi, "raise_SIGINT_then_send_None"):
                # This uses raise() which doesn't work.
                del _testcapi.raise_SIGINT_then_send_None

            try:
                libregrtest.main(["{name}"], ignore_tests={ignore_tests}, verbose=True, verbose3=True)
            except SystemExit as e:
                if e.code != 0:
                    raise RuntimeError(f"Failed with code: {{e.code}}")
            """
            )
        )
    except selenium.JavascriptException:
        print(selenium.logs)
        raise


def get_test_info(test) -> tuple[str, dict[str, Any]]:
    if isinstance(test, dict):
        (name, info) = next(iter(test.items()))
    else:
        name = test
        info = {}
    return (name, info)


@cache
def get_tests() -> list[tuple[str, dict[str, Any]]]:
    with open(Path(__file__).parent / "python_tests.yaml") as file:
        data = yaml.load(file, Loader)

    return [get_test_info(test) for test in data]


def pytest_generate_tests(metafunc):
    if "main_test" in metafunc.fixturenames:
        tests = get_tests()
        metafunc.parametrize("main_test", tests, ids=[t[0] for t in tests])
