Our package build process currently has a significant flaw: we first run `setup.py`, recording all compilation commands, then we rewrite these compilation commands to invoke `emcc` and replay them, and then we pray that the cross compiled executables ended up in the right place to go into the wheel. This is not a good strategy because the build script is allowed to implement arbitrary logic, and if it moves, renames, etc any of the output files then we lose track of them. This has repeatedly caused difficulty for us.

However, we also make no particularly significant use of the two pass approach. We can just do the simpler thing: capture the compiler commands as they occur, modify them as needed, and then run the fixed command.

I also added a patch to fix the numpy feature detection for wasm so that we don't have to include `_npyconfig.h` and `config.h`, numpy can generate them in the way it would for a native build. I opened a numpy PR that would fix the detection for us upstream:
https://github.com/numpy/numpy/pull/21154

This clears the way for us to switch to using pypa/build (as @henryiii has suggested) by removing our dependence on specific setuptools behavior.

This is on top of #2238.
No replay. Our package build process currently has a significant flaw: we first run `setup.py`, recording all compilation commands, then we rewrite these compilation commands to invoke `emcc` and replay them, and then we pray that the cross compiled executables ended up in the right place to go into the wheel. This is not a good strategy because the build script is allowed to implement arbitrary logic, and if it moves, renames, etc any of the output files then we lose track of them. This has repeatedly caused difficulty for us.

However, we also make no particularly significant use of the two pass approach. We can just do the simpler thing: capture the compiler commands as they occur, modify them as needed, and then run the fixed command.

I also added a patch to fix the numpy feature detection for wasm so that we don't have to include `_npyconfig.h` and `config.h`, numpy can generate them in the way it would for a native build. I opened a numpy PR that would fix the detection for us upstream:
https://github.com/numpy/numpy/pull/21154

This clears the way for us to switch to using pypa/build (as @henryiii has suggested) by removing our dependence on specific setuptools behavior.

This is on top of #2238.
import functools
import os

import pytest

from conftest import ROOT_PATH, built_packages
from pyodide_build.io import parse_package_config
from pyodide_build.testing import PYVERSION

PKG_DIR = ROOT_PATH / "packages"


@functools.cache
def registered_packages() -> list[str]:
    """Returns a list of registered package names"""
    packages = []
    for name in os.listdir(PKG_DIR):
        if (PKG_DIR / name).is_dir() and (PKG_DIR / name / "meta.yaml").exists():
            packages.append(name)
    return packages


def registered_packages_meta():
    """Returns a dictionary with the contents of `meta.yaml`
    for each registered package
    """
    packages = registered_packages
    return {
        name: parse_package_config(PKG_DIR / name / "meta.yaml") for name in packages
    }


UNSUPPORTED_PACKAGES: dict[str, list[str]] = {
    "chrome": [],
    "firefox": [],
    "node": ["cmyt", "yt"],
}
if "CI" in os.environ:
    UNSUPPORTED_PACKAGES["chrome"].extend(["statsmodels"])


@pytest.mark.parametrize("name", registered_packages())
def test_parse_package(name):
    # check that we can parse the meta.yaml
    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    skip_host = meta.get("build", {}).get("skip_host", True)
    if name == "numpy":
        assert skip_host is False
    elif name == "pandas":
        assert skip_host is True


@pytest.mark.skip_refcount_check
@pytest.mark.driver_timeout(40)
@pytest.mark.parametrize("name", registered_packages())
def test_import(name, selenium_standalone):
    if name not in built_packages():
        raise AssertionError(
            "Implementation error. Test for an unbuilt package "
            "should have been skipped in selenium_standalone fixture"
        )

    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]:
        pytest.xfail(
            "{} fails to load and is not supported on {}.".format(
                name, selenium_standalone.browser
            )
        )

    selenium_standalone.run("import glob, os")

    baseline_pyc = selenium_standalone.run(
        f"""
        len(list(glob.glob(
            '/lib/{PYVERSION}/site-packages/**/*.pyc',
            recursive=True)
        ))
        """
    )
    for import_name in meta.get("test", {}).get("imports", []):
        selenium_standalone.run_async("import %s" % import_name)
        # Make sure that even after importing, there are no additional .pyc
        # files
        assert (
            selenium_standalone.run(
                f"""
                len(list(glob.glob(
                    '/lib/{PYVERSION}/site-packages/**/*.pyc',
                    recursive=True)
                ))
                """
            )
            == baseline_pyc
        )
        # Make sure no exe files were loaded!
        assert (
            selenium_standalone.run(
                f"""
                len(list(glob.glob(
                    '/lib/{PYVERSION}/site-packages/**/*.exe',
                    recursive=True)
                ))
                """
            )
            == 0
        )

import functools
import os

import pytest

from conftest import ROOT_PATH, built_packages
from pyodide_build.io import parse_package_config
from pyodide_build.testing import PYVERSION

PKG_DIR = ROOT_PATH / "packages"


@functools.cache
def registered_packages() -> list[str]:
    """Returns a list of registered package names"""
    packages = []
    for name in os.listdir(PKG_DIR):
        if (PKG_DIR / name).is_dir() and (PKG_DIR / name / "meta.yaml").exists():
            packages.append(name)
    return packages


def registered_packages_meta():
    """Returns a dictionary with the contents of `meta.yaml`
    for each registered package
    """
    packages = registered_packages
    return {
        name: parse_package_config(PKG_DIR / name / "meta.yaml") for name in packages
    }


UNSUPPORTED_PACKAGES: dict[str, list[str]] = {
    "chrome": [],
    "firefox": [],
    "node": ["cmyt", "yt"],
}
if "CI" in os.environ:
    UNSUPPORTED_PACKAGES["chrome"].extend(["statsmodels"])


@pytest.mark.parametrize("name", registered_packages())
def test_parse_package(name):
    # check that we can parse the meta.yaml
    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    sharedlibrary = meta.get("build", {}).get("sharedlibrary", False)
    if name == "sharedlib-test":
        assert sharedlibrary is True
    elif name == "sharedlib-test-py":
        assert sharedlibrary is False


@pytest.mark.skip_refcount_check
@pytest.mark.driver_timeout(40)
@pytest.mark.parametrize("name", registered_packages())
def test_import(name, selenium_standalone):
    if name not in built_packages():
        raise AssertionError(
            "Implementation error. Test for an unbuilt package "
            "should have been skipped in selenium_standalone fixture"
        )

    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]:
        pytest.xfail(
            "{} fails to load and is not supported on {}.".format(
                name, selenium_standalone.browser
            )
        )

    selenium_standalone.run("import glob, os")

    baseline_pyc = selenium_standalone.run(
        f"""
        len(list(glob.glob(
            '/lib/{PYVERSION}/site-packages/**/*.pyc',
            recursive=True)
        ))
        """
    )
    for import_name in meta.get("test", {}).get("imports", []):
        selenium_standalone.run_async("import %s" % import_name)
        # Make sure that even after importing, there are no additional .pyc
        # files
        assert (
            selenium_standalone.run(
                f"""
                len(list(glob.glob(
                    '/lib/{PYVERSION}/site-packages/**/*.pyc',
                    recursive=True)
                ))
                """
            )
            == baseline_pyc
        )
        # Make sure no exe files were loaded!
        assert (
            selenium_standalone.run(
                f"""
                len(list(glob.glob(
                    '/lib/{PYVERSION}/site-packages/**/*.exe',
                    recursive=True)
                ))
                """
            )
            == 0
        )
