ENH Use dist-info + `importlib.metadata` to store and retrieve package metadata. This switches to using the file system to store the information about packages
and using `importlib.metadata` to retrieve it.
This has two related benefits:

1. We don't have to separately maintain our own separate state about what
we've installed.

2. We are guaranteed to agree with the Python stdlib about whether or not a
package is installed, and if so which version is installed. Since the state is 
maintained in only one place, there is no chance of it diverging.

According to the packaging specs, if the package is from a url we should put 
that into a file here:
https://packaging.python.org/en/latest/specifications/direct-url/#direct-url
Other than that, we should set the `INSTALLER` to `pyodide.loadPackage` 
if the package was loaded with `pyodide.loadPackage` either directly or 
indirectly via `micropip`. Otherwise set `INSTALLER` to `micropip`. That
way we can maintain the current behavior of `micropip.list`:
1. if `direct_url.json` is present, then `source` is the url in `direct_url.json`
2. otherwise, set `source` to `pyodide` if `INSTALLER` is `pyodide.loadPackage`
and set it to `micropip` if `INSTALLER` is `micropip`.

Oddly enough, the packaging specs suggest no place to put the source
index, so it seems that according to the specs if a package was installed
by name from some custom index, this info should not be recorded.

### Checklists

- [x] Add a [CHANGELOG](https://github.com/pyodide/pyodide/blob/main/docs/project/changelog.md) entry
- [x] Add / update tests

From 17c0cf37fa3a23bed2dbe3c365b24c20e7e862f0 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 28 Feb 2022 00:54:33 -0500
Subject: [PATCH 04/10] testing

---
 Lib/platform.py                   | 2 +-
 Lib/test/libregrtest/main.py      | 4 +++-
 Lib/test/support/script_helper.py | 5 +++++
 Lib/test/test_cgi.py              | 1 +
 Lib/test/test_fcntl.py            | 2 ++
 Lib/test/test_gzip.py             | 2 ++
 Lib/test/test_itertools.py        | 1 +
 7 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Lib/platform.py b/Lib/platform.py
index e32f9c11cd..69939e527f 100755
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -605,7 +605,7 @@ def _syscmd_file(target, default=''):
         default in case the command should fail.
 
     """
-    if sys.platform in ('dos', 'win32', 'win16'):
+    if sys.platform in ('dos', 'win32', 'win16', 'emscripten'):
         # XXX Others too ?
         return default
 
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 52cc065da1..bd217172fb 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -659,7 +659,9 @@ def main(self, tests=None, **kwargs):
         except SystemExit as exc:
             # bpo-38203: Python can hang at exit in Py_Finalize(), especially
             # on threading._shutdown() call: put a timeout
-            faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
+
+            # Skipping, pyodide doesn't support threading.
+            # faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
 
             sys.exit(exc.code)
 
diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
index 6d699c8486..b160d5fe30 100644
--- a/Lib/test/support/script_helper.py
+++ b/Lib/test/support/script_helper.py
@@ -9,6 +9,7 @@
 import subprocess
 import py_compile
 import zipfile
+import unittest
 
 from importlib.util import source_from_cache
 from test import support
@@ -36,6 +37,8 @@ def interpreter_requires_environment():
     situation.  PYTHONPATH or PYTHONUSERSITE are other common environment
     variables that might impact whether or not the interpreter can start.
     """
+    raise unittest.SkipTest('no subprocess')
+
     global __cached_interp_requires_environment
     if __cached_interp_requires_environment is None:
         # If PYTHONHOME is set, assume that we need it
@@ -177,6 +180,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
     kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
     object.
     """
+    raise unittest.SkipTest("no subprocess")
+
     cmd_line = [sys.executable]
     if not interpreter_requires_environment():
         cmd_line.append('-E')
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
index c1b893d3fe..cfd4d47fa3 100644
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -223,6 +223,7 @@ def test_separator(self):
 
     @warnings_helper.ignore_warnings(category=DeprecationWarning)
     def test_log(self):
+        raise unittest.SkipTest('known pyodide failure')
         cgi.log("Testing")
 
         cgi.logfp = StringIO()
diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py
index fc8c39365f..91b610542e 100644
--- a/Lib/test/test_fcntl.py
+++ b/Lib/test/test_fcntl.py
@@ -157,6 +157,7 @@ def test_flock(self):
 
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_exclusive(self):
+        raise unittest.SkipTest('threading not supported')
         self.f = open(TESTFN, 'wb+')
         cmd = fcntl.LOCK_EX | fcntl.LOCK_NB
         fcntl.lockf(self.f, cmd)
@@ -168,6 +169,7 @@ def test_lockf_exclusive(self):
 
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_share(self):
+        raise unittest.SkipTest('threading not supported')
         self.f = open(TESTFN, 'wb+')
         cmd = fcntl.LOCK_SH | fcntl.LOCK_NB
         fcntl.lockf(self.f, cmd)
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
index f86e767ac0..a763dbebe0 100644
--- a/Lib/test/test_gzip.py
+++ b/Lib/test/test_gzip.py
@@ -753,6 +753,7 @@ class TestCommandLine(unittest.TestCase):
     data = b'This is a simple test with gzip'
 
     def test_decompress_stdin_stdout(self):
+        raise unittest.SkipTest('no subprocess')
         with io.BytesIO() as bytes_io:
             with gzip.GzipFile(fileobj=bytes_io, mode='wb') as gzip_file:
                 gzip_file.write(self.data)
@@ -789,6 +790,7 @@ def test_decompress_infile_outfile_error(self):
 
     @create_and_remove_directory(TEMPDIR)
     def test_compress_stdin_outfile(self):
+        raise unittest.SkipTest('no subprocess')
         args = sys.executable, '-m', 'gzip'
         with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
             out, err = proc.communicate(self.data)
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index a12f6f0b97..acede437c3 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -1533,6 +1533,7 @@ def __next__(self):
             next(a)
 
     def test_tee_concurrent(self):
+        raise unittest.SkipTest('threading not supported')
         start = threading.Event()
         finish = threading.Event()
         class I:
-- 
2.25.1


From 17c0cf37fa3a23bed2dbe3c365b24c20e7e862f0 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 28 Feb 2022 00:54:33 -0500
Subject: [PATCH 04/10] testing

---
 Lib/platform.py                   | 2 +-
 Lib/test/libregrtest/main.py      | 4 +++-
 Lib/test/support/script_helper.py | 5 +++++
 Lib/test/test_cgi.py              | 1 +
 Lib/test/test_fcntl.py            | 2 ++
 Lib/test/test_gzip.py             | 2 ++
 Lib/test/test_itertools.py        | 1 +
 7 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Lib/platform.py b/Lib/platform.py
index e32f9c11cd..69939e527f 100755
--- a/Lib/platform.py
+++ b/Lib/platform.py
@@ -605,7 +605,7 @@ def _syscmd_file(target, default=''):
         default in case the command should fail.
 
     """
-    if sys.platform in ('dos', 'win32', 'win16'):
+    if sys.platform in ('dos', 'win32', 'win16', 'emscripten'):
         # XXX Others too ?
         return default
 
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 52cc065da1..bd217172fb 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -659,7 +659,9 @@ def main(self, tests=None, **kwargs):
         except SystemExit as exc:
             # bpo-38203: Python can hang at exit in Py_Finalize(), especially
             # on threading._shutdown() call: put a timeout
-            faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
+
+            # Skipping, pyodide doesn't support threading.
+            # faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
 
             sys.exit(exc.code)
 
diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
index 6d699c8486..b160d5fe30 100644
--- a/Lib/test/support/script_helper.py
+++ b/Lib/test/support/script_helper.py
@@ -9,6 +9,7 @@
 import subprocess
 import py_compile
 import zipfile
+import unittest
 
 from importlib.util import source_from_cache
 from test import support
@@ -36,6 +37,8 @@ def interpreter_requires_environment():
     situation.  PYTHONPATH or PYTHONUSERSITE are other common environment
     variables that might impact whether or not the interpreter can start.
     """
+    raise unittest.SkipTest('no subprocess')
+
     global __cached_interp_requires_environment
     if __cached_interp_requires_environment is None:
         # If PYTHONHOME is set, assume that we need it
@@ -177,6 +180,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
     kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
     object.
     """
+    raise unittest.SkipTest("no subprocess")
+
     cmd_line = [sys.executable]
     if not interpreter_requires_environment():
         cmd_line.append('-E')
diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py
index c1b893d3fe..cfd4d47fa3 100644
--- a/Lib/test/test_cgi.py
+++ b/Lib/test/test_cgi.py
@@ -223,6 +223,7 @@ def test_separator(self):
 
     @warnings_helper.ignore_warnings(category=DeprecationWarning)
     def test_log(self):
+        raise unittest.SkipTest('known pyodide failure')
         cgi.log("Testing")
 
         cgi.logfp = StringIO()
diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py
index fc8c39365f..91b610542e 100644
--- a/Lib/test/test_fcntl.py
+++ b/Lib/test/test_fcntl.py
@@ -157,6 +157,7 @@ def test_flock(self):
 
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_exclusive(self):
+        raise unittest.SkipTest('threading not supported')
         self.f = open(TESTFN, 'wb+')
         cmd = fcntl.LOCK_EX | fcntl.LOCK_NB
         fcntl.lockf(self.f, cmd)
@@ -168,6 +169,7 @@ def test_lockf_exclusive(self):
 
     @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError")
     def test_lockf_share(self):
+        raise unittest.SkipTest('threading not supported')
         self.f = open(TESTFN, 'wb+')
         cmd = fcntl.LOCK_SH | fcntl.LOCK_NB
         fcntl.lockf(self.f, cmd)
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
index f86e767ac0..a763dbebe0 100644
--- a/Lib/test/test_gzip.py
+++ b/Lib/test/test_gzip.py
@@ -753,6 +753,7 @@ class TestCommandLine(unittest.TestCase):
     data = b'This is a simple test with gzip'
 
     def test_decompress_stdin_stdout(self):
+        raise unittest.SkipTest('no subprocess')
         with io.BytesIO() as bytes_io:
             with gzip.GzipFile(fileobj=bytes_io, mode='wb') as gzip_file:
                 gzip_file.write(self.data)
@@ -789,6 +790,7 @@ def test_decompress_infile_outfile_error(self):
 
     @create_and_remove_directory(TEMPDIR)
     def test_compress_stdin_outfile(self):
+        raise unittest.SkipTest('no subprocess')
         args = sys.executable, '-m', 'gzip'
         with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
             out, err = proc.communicate(self.data)
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index a12f6f0b97..acede437c3 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -1533,6 +1533,7 @@ def __next__(self):
             next(a)
 
     def test_tee_concurrent(self):
+        raise unittest.SkipTest('threading not supported')
         start = threading.Event()
         finish = threading.Event()
         class I:
-- 
2.25.1

