A while back we added tests to make sure we weren't leaking hiwire keys. As a special case of this, we also were testing that we weren't leaking JsProxies. In the future, I would like to add tests to show that we aren't leaking PyProxies either. Currently we leak quite a lot of them.

Before we actually add any tests to show that we aren't leaking proxies, I'd like to rework the model of the PyProxy lifetime. The planned changes should make it far easier to actually avoid these leaks while still using a reasonably natural coding style.

This change has no user-facing effect: the tracing is disabled by default and the API to enable it is private and always will be.
Add support for pyproxy allocation tracing. A while back we added tests to make sure we weren't leaking hiwire keys. As a special case of this, we also were testing that we weren't leaking JsProxies. In the future, I would like to add tests to show that we aren't leaking PyProxies either. Currently we leak quite a lot of them.

Before we actually add any tests to show that we aren't leaking proxies, I'd like to rework the model of the PyProxy lifetime. The planned changes should make it far easier to actually avoid these leaks while still using a reasonably natural coding style.

This change has no user-facing effect: the tracing is disabled by default and the API to enable it is private and always will be.
import pytest
import os
from pathlib import Path
from pyodide_build.common import _parse_package_subset
from pyodide_build.io import parse_package_config

PKG_DIR = Path(__file__).parent


def registered_packages():
    """Returns a list of registered package names"""
    packages = []
    for name in os.listdir(PKG_DIR):
        if (PKG_DIR / name).is_dir() and (PKG_DIR / name / "meta.yaml").exists():
            packages.append(name)
    return packages


def registered_packages_meta():
    """Returns a dictionary with the contents of `meta.yaml`
    for each registered package
    """
    packages = registered_packages
    return {
        name: parse_package_config(PKG_DIR / name / "meta.yaml") for name in packages
    }


UNSUPPORTED_PACKAGES: dict = {"chrome": ["scikit-image", "statsmodels"], "firefox": []}


@pytest.mark.parametrize("name", registered_packages())
def test_parse_package(name):
    # check that we can parse the meta.yaml
    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    skip_host = meta.get("build", {}).get("skip_host", True)
    if name == "numpy":
        assert skip_host is False
    elif name == "pandas":
        assert skip_host is True


@pytest.mark.skip_refcount_check
@pytest.mark.driver_timeout(40)
@pytest.mark.parametrize("name", registered_packages())
def test_import(name, selenium_standalone):
    # check that we can parse the meta.yaml
    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]:
        pytest.xfail(
            "{} fails to load and is not supported on {}.".format(
                name, selenium_standalone.browser
            )
        )

    built_packages = _parse_package_subset(os.environ.get("PYODIDE_PACKAGES"))
    # only a subset of packages were built
    if built_packages is not None and name not in built_packages:
        pytest.skip(f"{name} was skipped due to PYODIDE_PACKAGES")

    selenium_standalone.run("import glob, os")

    baseline_pyc = selenium_standalone.run(
        """
        len(list(glob.glob(
            '/lib/python3.8/site-packages/**/*.pyc',
            recursive=True)
        ))
        """
    )
    for import_name in meta.get("test", {}).get("imports", []):
        selenium_standalone.run_async("import %s" % import_name)
        # Make sure that even after importing, there are no additional .pyc
        # files
        assert (
            selenium_standalone.run(
                """
            len(list(glob.glob(
                '/lib/python3.8/site-packages/**/*.pyc',
                recursive=True)
            ))
            """
            )
            == baseline_pyc
        )

import pytest
import os
from pathlib import Path
from pyodide_build.common import _parse_package_subset
from pyodide_build.io import parse_package_config

PKG_DIR = Path(__file__).parent


def registered_packages():
    """Returns a list of registered package names"""
    packages = []
    for name in os.listdir(PKG_DIR):
        if (PKG_DIR / name).is_dir() and (PKG_DIR / name / "meta.yaml").exists():
            packages.append(name)
    return packages


def registered_packages_meta():
    """Returns a dictionary with the contents of `meta.yaml`
    for each registered package
    """
    packages = registered_packages
    return {
        name: parse_package_config(PKG_DIR / name / "meta.yaml") for name in packages
    }


UNSUPPORTED_PACKAGES: dict = {"chrome": ["scikit-image", "statsmodels"], "firefox": []}


@pytest.mark.parametrize("name", registered_packages())
def test_parse_package(name):
    # check that we can parse the meta.yaml
    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    skip_host = meta.get("build", {}).get("skip_host", True)
    if name == "numpy":
        assert skip_host is False
    elif name == "pandas":
        assert skip_host is True


@pytest.mark.skip_refcount_check
@pytest.mark.driver_timeout(40)
@pytest.mark.parametrize("name", registered_packages())
def test_import(name, selenium_standalone):
    # check that we can parse the meta.yaml
    meta = parse_package_config(PKG_DIR / name / "meta.yaml")

    if name in UNSUPPORTED_PACKAGES[selenium_standalone.browser]:
        pytest.xfail(
            "{} fails to load and is not supported on {}.".format(
                name, selenium_standalone.browser
            )
        )

    built_packages = _parse_package_subset(os.environ.get("PYODIDE_PACKAGES"))
    # only a subset of packages were built
    if built_packages is not None and name not in built_packages:
        pytest.skip(f"{name} was skipped due to PYODIDE_PACKAGES")

    selenium_standalone.run("import glob, os")

    baseline_pyc = selenium_standalone.run(
        """
        len(list(glob.glob(
            '/lib/python3.8/site-packages/**/*.pyc',
            recursive=True)
        ))
        """
    )
    for import_name in meta.get("test", {}).get("imports", []):
        selenium_standalone.run_async("import %s" % import_name)
        # Make sure that even after importing, there are no additional .pyc
        # files
        assert (
            selenium_standalone.run(
                """
            len(list(glob.glob(
                '/lib/python3.8/site-packages/**/*.pyc',
                recursive=True)
            ))
            """
            )
            == baseline_pyc
        )
