ENH Use dist-info + `importlib.metadata` to store and retrieve package metadata. This switches to using the file system to store the information about packages
and using `importlib.metadata` to retrieve it.
This has two related benefits:

1. We don't have to separately maintain our own separate state about what
we've installed.

2. We are guaranteed to agree with the Python stdlib about whether or not a
package is installed, and if so which version is installed. Since the state is 
maintained in only one place, there is no chance of it diverging.

According to the packaging specs, if the package is from a url we should put 
that into a file here:
https://packaging.python.org/en/latest/specifications/direct-url/#direct-url
Other than that, we should set the `INSTALLER` to `pyodide.loadPackage` 
if the package was loaded with `pyodide.loadPackage` either directly or 
indirectly via `micropip`. Otherwise set `INSTALLER` to `micropip`. That
way we can maintain the current behavior of `micropip.list`:
1. if `direct_url.json` is present, then `source` is the url in `direct_url.json`
2. otherwise, set `source` to `pyodide` if `INSTALLER` is `pyodide.loadPackage`
and set it to `micropip` if `INSTALLER` is `micropip`.

Oddly enough, the packaging specs suggest no place to put the source
index, so it seems that according to the specs if a package was installed
by name from some custom index, this info should not be recorded.

### Checklists

- [x] Add a [CHANGELOG](https://github.com/pyodide/pyodide/blob/main/docs/project/changelog.md) entry
- [x] Add / update tests

import pytest

from pyodide_build.common import (
    ALWAYS_PACKAGES,
    CORE_PACKAGES,
    CORE_SCIPY_PACKAGES,
    UNVENDORED_STDLIB_MODULES,
    _parse_package_subset,
    find_matching_wheels,
    get_make_environment_vars,
    get_make_flag,
    search_pyodide_root,
)


def test_parse_package_subset():
    assert (
        _parse_package_subset("numpy,pandas")
        == {
            "numpy",
            "pandas",
        }
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )

    # duplicates are removed
    assert (
        _parse_package_subset("numpy,numpy")
        == {
            "numpy",
        }
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )

    # no empty package name included, spaces are handled
    assert (
        _parse_package_subset("x,  a, b, c   ,,, d,,")
        == {
            "x",
            "a",
            "b",
            "c",
            "d",
        }
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )

    assert (
        _parse_package_subset("core")
        == CORE_PACKAGES | UNVENDORED_STDLIB_MODULES | ALWAYS_PACKAGES
    )
    # by default core packages are built
    assert _parse_package_subset(None) == _parse_package_subset("core")

    assert (
        _parse_package_subset("min-scipy-stack")
        == CORE_SCIPY_PACKAGES
        | CORE_PACKAGES
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )
    # reserved key words can be combined with other packages
    assert _parse_package_subset("core, unknown") == _parse_package_subset("core") | {
        "unknown"
    }


def test_get_make_flag():
    assert len(get_make_flag("SIDE_MODULE_LDFLAGS")) > 0
    assert len(get_make_flag("SIDE_MODULE_CFLAGS")) > 0
    # n.b. right now CXXFLAGS is empty so don't check length here, just check it returns
    get_make_flag("SIDE_MODULE_CXXFLAGS")


def test_get_make_environment_vars():
    vars = get_make_environment_vars()
    assert "SIDE_MODULE_LDFLAGS" in vars
    assert "SIDE_MODULE_CFLAGS" in vars
    assert "SIDE_MODULE_CXXFLAGS" in vars
    assert "TOOLSDIR" in vars


def test_wheel_paths():
    from pathlib import Path

    old_version = "cp38"
    PYMAJOR = int(get_make_flag("PYMAJOR"))
    PYMINOR = int(get_make_flag("PYMINOR"))
    current_version = f"cp{PYMAJOR}{PYMINOR}"
    future_version = f"cp{PYMAJOR}{PYMINOR + 1}"
    strings = []

    for interp in [
        old_version,
        current_version,
        future_version,
        "py3",
        "py2",
        "py2.py3",
    ]:
        for abi in [interp, "abi3", "none"]:
            for arch in ["emscripten_wasm32", "linux_x86_64", "any"]:
                strings.append(f"wrapt-1.13.3-{interp}-{abi}-{arch}.whl")

    paths = [Path(x) for x in strings]
    assert [x.stem.split("-", 2)[-1] for x in find_matching_wheels(paths)] == [
        f"{current_version}-{current_version}-emscripten_wasm32",
        f"{current_version}-abi3-emscripten_wasm32",
        f"{current_version}-none-emscripten_wasm32",
        f"{old_version}-abi3-emscripten_wasm32",
        "py3-none-emscripten_wasm32",
        "py2.py3-none-emscripten_wasm32",
        "py3-none-any",
        "py2.py3-none-any",
    ]


def test_search_pyodide_root(tmp_path):
    pyproject_file = tmp_path / "pyproject.toml"
    pyproject_file.write_text("[tool.pyodide]")
    assert search_pyodide_root(tmp_path) == tmp_path
    assert search_pyodide_root(tmp_path / "subdir") == tmp_path
    assert search_pyodide_root(tmp_path / "subdir" / "subdir") == tmp_path

    pyproject_file.unlink()
    with pytest.raises(FileNotFoundError):
        search_pyodide_root(tmp_path)

import pytest

from pyodide_build.common import (
    ALWAYS_PACKAGES,
    CORE_PACKAGES,
    CORE_SCIPY_PACKAGES,
    UNVENDORED_STDLIB_MODULES,
    _parse_package_subset,
    find_matching_wheels,
    get_make_environment_vars,
    get_make_flag,
    search_pyodide_root,
)


def test_parse_package_subset():
    assert (
        _parse_package_subset("numpy,pandas")
        == {
            "numpy",
            "pandas",
        }
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )

    # duplicates are removed
    assert (
        _parse_package_subset("numpy,numpy")
        == {
            "numpy",
        }
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )

    # no empty package name included, spaces are handled
    assert (
        _parse_package_subset("x,  a, b, c   ,,, d,,")
        == {
            "x",
            "a",
            "b",
            "c",
            "d",
        }
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )

    assert (
        _parse_package_subset("core")
        == CORE_PACKAGES | UNVENDORED_STDLIB_MODULES | ALWAYS_PACKAGES
    )
    # by default core packages are built
    assert _parse_package_subset(None) == _parse_package_subset("core")

    assert (
        _parse_package_subset("min-scipy-stack")
        == CORE_SCIPY_PACKAGES
        | CORE_PACKAGES
        | UNVENDORED_STDLIB_MODULES
        | ALWAYS_PACKAGES
    )
    # reserved key words can be combined with other packages
    assert _parse_package_subset("core, unknown") == _parse_package_subset("core") | {
        "unknown"
    }


def test_get_make_flag():
    assert len(get_make_flag("SIDE_MODULE_LDFLAGS")) > 0
    assert len(get_make_flag("SIDE_MODULE_CFLAGS")) > 0
    # n.b. right now CXXFLAGS is empty so don't check length here, just check it returns
    get_make_flag("SIDE_MODULE_CXXFLAGS")


def test_get_make_environment_vars():
    vars = get_make_environment_vars()
    assert "SIDE_MODULE_LDFLAGS" in vars
    assert "SIDE_MODULE_CFLAGS" in vars
    assert "SIDE_MODULE_CXXFLAGS" in vars
    assert "TOOLSDIR" in vars


def test_wheel_paths():
    from pathlib import Path

    old_version = "cp38"
    PYMAJOR = int(get_make_flag("PYMAJOR"))
    PYMINOR = int(get_make_flag("PYMINOR"))
    current_version = f"cp{PYMAJOR}{PYMINOR}"
    future_version = f"cp{PYMAJOR}{PYMINOR + 1}"
    strings = []

    for interp in [
        old_version,
        current_version,
        future_version,
        "py3",
        "py2",
        "py2.py3",
    ]:
        for abi in [interp, "abi3", "none"]:
            for arch in ["emscripten_wasm32", "linux_x86_64", "any"]:
                strings.append(f"wrapt-1.13.3-{interp}-{abi}-{arch}.whl")

    paths = [Path(x) for x in strings]
    assert [x.stem.split("-", 2)[-1] for x in find_matching_wheels(paths)] == [
        f"{current_version}-{current_version}-emscripten_wasm32",
        f"{current_version}-abi3-emscripten_wasm32",
        f"{current_version}-none-emscripten_wasm32",
        f"{old_version}-abi3-emscripten_wasm32",
        "py3-none-emscripten_wasm32",
        "py2.py3-none-emscripten_wasm32",
        "py3-none-any",
        "py2.py3-none-any",
    ]


def test_search_pyodide_root(tmp_path):
    pyproject_file = tmp_path / "pyproject.toml"
    pyproject_file.write_text("[tool.pyodide]")
    assert search_pyodide_root(tmp_path) == tmp_path
    assert search_pyodide_root(tmp_path / "subdir") == tmp_path
    assert search_pyodide_root(tmp_path / "subdir" / "subdir") == tmp_path

    pyproject_file.unlink()
    with pytest.raises(FileNotFoundError):
        search_pyodide_root(tmp_path)
