For reasons that are a bit beyond me, `--host` and `PLATFORM_TRIPLET`
seem to be independent, in particular we've had an empty
`PLATFORM_TRIPLET`. This is unfortunate because `PLATFORM_TRIPLET`
is used to generate the SOABI config variable which in turn is used
to decide whether a .so file is a good match for loading. We'd like
for linux Pythons not to try to import emscripten .so files (it
raises `ImportError: some_file.so: invalid ELF header`). Similarly,
we'd like to avoid attempting to load linux .so files in wasm. These
platform tags are our friends.

Anyways, this PR sets `PLATFORM_TRIPLET` and ensures that .so files
built by pywasmcross are tagged with our SOABI tag.

I moved the .so file renaming from pywasmcross to buildpkg just
before running the post script. That is a better place to put it in
case the package wants to look at the .so file after linking it. It
might be surprised that we moved it.

I also improved the error message if we try to `loadWebAssemblyModule`
something that is actually say a Linux .so file and updated get_dynlibs
to filter out .so files that have an incompatible abi tag.
Set PLATFORM_TRIPLET, include platform in so names, only load compatible so files. For reasons that are a bit beyond me, `--host` and `PLATFORM_TRIPLET`
seem to be independent, in particular we've had an empty
`PLATFORM_TRIPLET`. This is unfortunate because `PLATFORM_TRIPLET`
is used to generate the SOABI config variable which in turn is used
to decide whether a .so file is a good match for loading. We'd like
for linux Pythons not to try to import emscripten .so files (it
raises `ImportError: some_file.so: invalid ELF header`). Similarly,
we'd like to avoid attempting to load linux .so files in wasm. These
platform tags are our friends.

Anyways, this PR sets `PLATFORM_TRIPLET` and ensures that .so files
built by pywasmcross are tagged with our SOABI tag.

I moved the .so file renaming from pywasmcross to buildpkg just
before running the post script. That is a better place to put it in
case the package wants to look at the .so file after linking it. It
might be surprised that we moved it.

I also improved the error message if we try to `loadWebAssemblyModule`
something that is actually say a Linux .so file and updated get_dynlibs
to filter out .so files that have an incompatible abi tag.
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
from time import time

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import conftest  # noqa: E402

SKIP = {"fft", "hyantes"}


def print_entry(name, res):
    print(" - ", name)
    print(" " * 4, end="")
    for name, dt in res.items():
        print(f"{name}: {dt:.6f}  ", end="")
    print("")


def run_native(code):
    if "# non-native" in code:
        return float("NaN")

    root = Path(__file__).resolve().parents[1]
    output = subprocess.check_output(
        [sys.executable, "-c", code],
        cwd=Path(__file__).resolve().parent,
        env={
            "PYTHONPATH": str(root / "src/py/lib")
            + ":"
            + str(root / "packages" / ".artifacts" / "lib" / "python")
        },
    )
    return float(output.strip().split()[-1])


def run_wasm(code, selenium, interrupt_buffer):
    if interrupt_buffer:
        selenium.run_js(
            """
            let interrupt_buffer = new Int32Array(1);
            pyodide.setInterruptBuffer(interrupt_buffer)
            """
        )

    selenium.run(code)
    try:
        runtime = float(selenium.logs.split("\n")[-1])
    except ValueError:
        print(selenium.logs)
        raise
    return runtime


def run_all(selenium_backends, code):
    result = {"native": run_native(code)}

    for browser_name, selenium in selenium_backends.items():
        for interrupt_buffer in [False, True]:
            dt = run_wasm(code, selenium, interrupt_buffer)
            if interrupt_buffer:
                browser_name += "(w/ ib)"
            result[browser_name] = dt
    return result


def parse_benchmark(filename):
    lines = []
    with open(filename) as fp:
        for line in fp:
            m = re.match(r"^#\s*(setup|run): (.*)$", line)
            if m:
                line = f"{m.group(1)} = {m.group(2)!r}\n"
            lines.append(line)
    return "".join(lines)


def get_benchmark_scripts(scripts_dir, repeat=11, number=5):
    root = Path(__file__).resolve().parent / scripts_dir
    for filename in sorted(root.iterdir()):
        name = filename.stem

        if name in SKIP:
            continue

        content = parse_benchmark(filename)
        content += (
            "import numpy as np\n"
            "_ = np.empty(())\n"
            f"setup = setup + '\\nfrom __main__ import {name}'\n"
            "from timeit import Timer\n"
            "t = Timer(run, setup)\n"
            f"r = t.repeat({repeat}, {number})\n"
            "r.remove(min(r))\n"
            "r.remove(max(r))\n"
            "print(np.mean(r))\n"
        )

        yield name, content


def get_pystone_benchmarks():
    return get_benchmark_scripts("benchmarks/pystone_benchmarks", repeat=5, number=1)


def get_numpy_benchmarks():
    return get_benchmark_scripts("benchmarks/numpy_benchmarks")


def get_matplotlib_benchmarks():
    return get_benchmark_scripts("benchmarks/matplotlib_benchmarks")


def get_benchmarks(benchmarks, targets=("all",)):
    if "all" in targets:
        for benchmark in benchmarks.values():
            yield from benchmark()
    else:
        for target in targets:
            yield from benchmarks[target]()


def parse_args(benchmarks):
    benchmarks.append("all")

    parser = argparse.ArgumentParser("Run benchmarks on Pyodide's performance")
    parser.add_argument(
        "target",
        choices=benchmarks,
        nargs="+",
        help="Benchmarks to run ('all' to run all benchmarks)",
    )
    parser.add_argument(
        "-o",
        "--output",
        default="build/benchmarks.json",
        help="path to the json file where benchmark results will be saved",
    )
    parser.add_argument(
        "--timeout",
        default=1200,
        type=int,
        help="Browser timeout(sec) for each benchmark (default: %(default)s)",
    )

    return parser.parse_args()


def main():

    BENCHMARKS = {
        "pystone": get_pystone_benchmarks,
        "numpy": get_numpy_benchmarks,
        "matplotlib": get_matplotlib_benchmarks,
    }

    args = parse_args(list(BENCHMARKS.keys()))
    targets = [t.lower() for t in args.target]
    output = Path(args.output).resolve()
    timeout = args.timeout

    results = {}
    selenium_backends = {}
    browser_cls = [
        ("firefox", conftest.FirefoxWrapper),
        ("chrome", conftest.ChromeWrapper),
    ]

    with conftest.spawn_web_server() as (hostname, port, log_path):

        # selenium initialization time
        result = {"native": float("NaN")}
        for browser_name, cls in browser_cls:
            try:
                t0 = time()
                selenium = cls(port, script_timeout=timeout)
                result[browser_name] = time() - t0
            finally:
                selenium.driver.quit()

        results["selenium init"] = result
        print_entry("selenium init", result)

        # package loading time
        for package_name in ["numpy"]:
            result = {"native": float("NaN")}
            for browser_name, cls in browser_cls:
                selenium = cls(port, script_timeout=timeout)
                try:
                    t0 = time()
                    selenium.load_package(package_name)
                    result[browser_name] = time() - t0
                finally:
                    selenium.driver.quit()

            results[f"load {package_name}"] = result
            print_entry(f"load {package_name}", result)

        # run benchmarks
        for benchmark_name, content in get_benchmarks(BENCHMARKS, targets):
            try:
                # instantiate browsers for each benchmark to prevent side effects
                for browser_name, cls in browser_cls:
                    selenium_backends[browser_name] = cls(port, script_timeout=timeout)
                    # pre-load numpy and matplotlib for the selenium instance used in benchmarks
                    selenium_backends[browser_name].load_package(
                        ["numpy", "matplotlib"]
                    )

                results[benchmark_name] = run_all(selenium_backends, content)
                print_entry(benchmark_name, results[benchmark_name])
            finally:
                for selenium in selenium_backends.values():
                    selenium.driver.quit()

    output.parent.mkdir(exist_ok=True, parents=True)
    output.write_text(json.dumps(results))


if __name__ == "__main__":
    main()

import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
from time import time

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

import conftest  # noqa: E402

SKIP = {"fft", "hyantes"}


def print_entry(name, res):
    print(" - ", name)
    print(" " * 4, end="")
    for name, dt in res.items():
        print(f"{name}: {dt:.6f}  ", end="")
    print("")


def run_native(code):
    if "# non-native" in code:
        return float("NaN")

    root = Path(__file__).resolve().parents[1]
    output = subprocess.check_output(
        [sys.executable, "-c", code],
        cwd=Path(__file__).resolve().parent,
        env={
            "PYTHONPATH": str(root / "src/py/lib")
            + ":"
            + str(root / "packages" / ".artifacts" / "lib" / "python")
        },
    )
    return float(output.strip().split()[-1])


def run_wasm(code, selenium, interrupt_buffer):
    if interrupt_buffer:
        selenium.run_js(
            """
            let interrupt_buffer = new Int32Array(1);
            pyodide.setInterruptBuffer(interrupt_buffer)
            """
        )

    selenium.run(code)
    try:
        runtime = float(selenium.logs.split("\n")[-1])
    except ValueError:
        print(selenium.logs)
        raise
    return runtime


def run_all(selenium_backends, code):
    result = {"native": run_native(code)}

    for browser_name, selenium in selenium_backends.items():
        for interrupt_buffer in [False, True]:
            dt = run_wasm(code, selenium, interrupt_buffer)
            if interrupt_buffer:
                browser_name += "(w/ ib)"
            result[browser_name] = dt
    return result


def parse_benchmark(filename):
    lines = []
    with open(filename) as fp:
        for line in fp:
            m = re.match(r"^#\s*(setup|run): (.*)$", line)
            if m:
                line = f"{m.group(1)} = {m.group(2)!r}\n"
            lines.append(line)
    return "".join(lines)


def get_benchmark_scripts(scripts_dir, repeat=11, number=5):
    root = Path(__file__).resolve().parent / scripts_dir
    for filename in sorted(root.iterdir()):
        name = filename.stem

        if name in SKIP:
            continue

        content = parse_benchmark(filename)
        content += (
            "import numpy as np\n"
            "_ = np.empty(())\n"
            f"setup = setup + '\\nfrom __main__ import {name}'\n"
            "from timeit import Timer\n"
            "t = Timer(run, setup)\n"
            f"r = t.repeat({repeat}, {number})\n"
            "r.remove(min(r))\n"
            "r.remove(max(r))\n"
            "print(np.mean(r))\n"
        )

        yield name, content


def get_pystone_benchmarks():
    return get_benchmark_scripts("benchmarks/pystone_benchmarks", repeat=5, number=1)


def get_numpy_benchmarks():
    return get_benchmark_scripts("benchmarks/numpy_benchmarks")


def get_matplotlib_benchmarks():
    return get_benchmark_scripts("benchmarks/matplotlib_benchmarks")


def get_benchmarks(benchmarks, targets=("all",)):
    if "all" in targets:
        for benchmark in benchmarks.values():
            yield from benchmark()
    else:
        for target in targets:
            yield from benchmarks[target]()


def parse_args(benchmarks):
    benchmarks.append("all")

    parser = argparse.ArgumentParser("Run benchmarks on Pyodide's performance")
    parser.add_argument(
        "target",
        choices=benchmarks,
        nargs="+",
        help="Benchmarks to run ('all' to run all benchmarks)",
    )
    parser.add_argument(
        "-o",
        "--output",
        default="build/benchmarks.json",
        help="path to the json file where benchmark results will be saved",
    )
    parser.add_argument(
        "--timeout",
        default=1200,
        type=int,
        help="Browser timeout(sec) for each benchmark (default: %(default)s)",
    )

    return parser.parse_args()


def main():

    BENCHMARKS = {
        "pystone": get_pystone_benchmarks,
        "numpy": get_numpy_benchmarks,
        "matplotlib": get_matplotlib_benchmarks,
    }

    args = parse_args(list(BENCHMARKS.keys()))
    targets = [t.lower() for t in args.target]
    output = Path(args.output).resolve()
    timeout = args.timeout

    results = {}
    selenium_backends = {}
    browser_cls = [
        ("firefox", conftest.FirefoxWrapper),
        ("chrome", conftest.ChromeWrapper),
    ]

    with conftest.spawn_web_server() as (hostname, port, log_path):

        # selenium initialization time
        result = {"native": float("NaN")}
        for browser_name, cls in browser_cls:
            try:
                t0 = time()
                selenium = cls(port, script_timeout=timeout)
                result[browser_name] = time() - t0
            finally:
                selenium.driver.quit()

        results["selenium init"] = result
        print_entry("selenium init", result)

        # package loading time
        for package_name in ["numpy"]:
            result = {"native": float("NaN")}
            for browser_name, cls in browser_cls:
                selenium = cls(port, script_timeout=timeout)
                try:
                    t0 = time()
                    selenium.load_package(package_name)
                    result[browser_name] = time() - t0
                finally:
                    selenium.driver.quit()

            results[f"load {package_name}"] = result
            print_entry(f"load {package_name}", result)

        # run benchmarks
        for benchmark_name, content in get_benchmarks(BENCHMARKS, targets):
            try:
                # instantiate browsers for each benchmark to prevent side effects
                for browser_name, cls in browser_cls:
                    selenium_backends[browser_name] = cls(port, script_timeout=timeout)
                    # pre-load numpy and matplotlib for the selenium instance used in benchmarks
                    selenium_backends[browser_name].load_package(
                        ["numpy", "matplotlib"]
                    )

                results[benchmark_name] = run_all(selenium_backends, content)
                print_entry(benchmark_name, results[benchmark_name])
            finally:
                for selenium in selenium_backends.values():
                    selenium.driver.quit()

    output.parent.mkdir(exist_ok=True, parents=True)
    output.write_text(json.dumps(results))


if __name__ == "__main__":
    main()
