Our package build process currently has a significant flaw: we first run `setup.py`, recording all compilation commands, then we rewrite these compilation commands to invoke `emcc` and replay them, and then we pray that the cross compiled executables ended up in the right place to go into the wheel. This is not a good strategy because the build script is allowed to implement arbitrary logic, and if it moves, renames, etc any of the output files then we lose track of them. This has repeatedly caused difficulty for us.

However, we also make no particularly significant use of the two pass approach. We can just do the simpler thing: capture the compiler commands as they occur, modify them as needed, and then run the fixed command.

I also added a patch to fix the numpy feature detection for wasm so that we don't have to include `_npyconfig.h` and `config.h`, numpy can generate them in the way it would for a native build. I opened a numpy PR that would fix the detection for us upstream:
https://github.com/numpy/numpy/pull/21154

This clears the way for us to switch to using pypa/build (as @henryiii has suggested) by removing our dependence on specific setuptools behavior.

This is on top of #2238.
No replay. Our package build process currently has a significant flaw: we first run `setup.py`, recording all compilation commands, then we rewrite these compilation commands to invoke `emcc` and replay them, and then we pray that the cross compiled executables ended up in the right place to go into the wheel. This is not a good strategy because the build script is allowed to implement arbitrary logic, and if it moves, renames, etc any of the output files then we lose track of them. This has repeatedly caused difficulty for us.

However, we also make no particularly significant use of the two pass approach. We can just do the simpler thing: capture the compiler commands as they occur, modify them as needed, and then run the fixed command.

I also added a patch to fix the numpy feature detection for wasm so that we don't have to include `_npyconfig.h` and `config.h`, numpy can generate them in the way it would for a native build. I opened a numpy PR that would fix the detection for us upstream:
https://github.com/numpy/numpy/pull/21154

This clears the way for us to switch to using pypa/build (as @henryiii has suggested) by removing our dependence on specific setuptools behavior.

This is on top of #2238.
import random

import pytest


def generate_largish_json(n_rows: int = 91746) -> dict:
    # with n_rows = 91746, the output JSON size will be ~15 MB/10k rows

    # Note: we don't fix the random seed here, but the actual values
    # shouldn't matter
    columns = [
        ("column0", lambda: "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"),
        (
            "column1",
            lambda: random.choice(
                [
                    "notification-interval-longer",
                    "notification-interval-short",
                    "control",
                ]
            ),
        ),
        ("column2", lambda: random.choice([True, False])),
        ("column3", lambda: random.randint(0, 4)),
        ("column4", lambda: random.randint(0, 4)),
        ("column5", lambda: random.randint(0, 4)),
        ("column6", lambda: random.randint(0, 4)),
        ("column7", lambda: random.randint(0, 4)),
    ]
    data = {}
    for name, generator in columns:
        data[name] = [generator() for _ in range(n_rows)]
    return data


@pytest.mark.driver_timeout(30)
def test_pandas(selenium, request):
    selenium.load_package("pandas")
    assert len(selenium.run("import pandas\ndir(pandas)")) == 142


@pytest.mark.driver_timeout(30)
def test_extra_import(selenium, request):

    selenium.load_package("pandas")
    selenium.run("from pandas import Series, DataFrame")


@pytest.mark.driver_timeout(40)
@pytest.mark.skip_refcount_check
def test_load_largish_file(selenium_standalone, request, httpserver):
    selenium = selenium_standalone
    if selenium.browser == "chrome":
        pytest.xfail(
            "test_load_largish_file triggers a fatal runtime error in Chrome 89 see #1495"
        )
    if selenium.browser == "node":
        pytest.xfail("open_url doesn't work in node")

    selenium.load_package("pandas")
    selenium.load_package("matplotlib")

    n_rows = 91746

    data = generate_largish_json(n_rows)

    httpserver.expect_request("/data").respond_with_json(
        data, headers={"Access-Control-Allow-Origin": "*"}
    )
    request_url = httpserver.url_for("/data")

    selenium.run(
        f"""
        import pyodide
        import matplotlib.pyplot as plt
        import pandas as pd

        df = pd.read_json(pyodide.open_url('{request_url}'))
        assert df.shape == ({n_rows}, 8)
        """
    )

import random

import pytest


def generate_largish_json(n_rows: int = 91746) -> dict:
    # with n_rows = 91746, the output JSON size will be ~15 MB/10k rows

    # Note: we don't fix the random seed here, but the actual values
    # shouldn't matter
    columns = [
        ("column0", lambda: "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"),
        (
            "column1",
            lambda: random.choice(
                [
                    "notification-interval-longer",
                    "notification-interval-short",
                    "control",
                ]
            ),
        ),
        ("column2", lambda: random.choice([True, False])),
        ("column3", lambda: random.randint(0, 4)),
        ("column4", lambda: random.randint(0, 4)),
        ("column5", lambda: random.randint(0, 4)),
        ("column6", lambda: random.randint(0, 4)),
        ("column7", lambda: random.randint(0, 4)),
    ]
    data = {}
    for name, generator in columns:
        data[name] = [generator() for _ in range(n_rows)]
    return data


@pytest.mark.driver_timeout(30)
def test_pandas(selenium, request):
    selenium.load_package("pandas")
    assert len(selenium.run("import pandas\ndir(pandas)")) == 142


@pytest.mark.driver_timeout(30)
def test_extra_import(selenium, request):

    selenium.load_package("pandas")
    selenium.run("from pandas import Series, DataFrame")


@pytest.mark.driver_timeout(40)
@pytest.mark.skip_refcount_check
def test_load_largish_file(selenium_standalone, request, httpserver):
    selenium = selenium_standalone
    if selenium.browser == "chrome":
        pytest.xfail(
            "test_load_largish_file triggers a fatal runtime error in Chrome 89 see #1495"
        )
    if selenium.browser == "node":
        pytest.xfail("open_url doesn't work in node")

    selenium.load_package("pandas")
    selenium.load_package("matplotlib")

    n_rows = 91746

    data = generate_largish_json(n_rows)

    httpserver.expect_request("/data").respond_with_json(
        data, headers={"Access-Control-Allow-Origin": "*"}
    )
    request_url = httpserver.url_for("/data")

    selenium.run(
        f"""
        import pyodide
        import matplotlib.pyplot as plt
        import pandas as pd

        df = pd.read_json(pyodide.open_url('{request_url}'))
        assert df.shape == ({n_rows}, 8)
        """
    )
