Track completed platform dependency installations

Lib/test/test_tools/test_platform_deps.py

@@ -2,6 +2,7 @@
 
 import importlib.util
 import os
+import json
 from pathlib import Path
 import subprocess
 import tempfile
@@ -70,5 +71,86 @@ def test_completed_download_is_reused(self):
                     run.assert_not_called()
 
 
+class PrefixTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        path = ROOT / "Tools/build/dependency_cache.py"
+        spec = importlib.util.spec_from_file_location("dependency_cache", path)
+        cls.cache = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(cls.cache)
+
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.prefix = Path(self.tmp.name) / "prefix"
+
+    def install(self, staging):
+        (staging / "library").write_text("complete")
+
+    def test_reuse_and_update(self):
+        self.assertTrue(self.cache.install_prefix(
+            self.prefix, {"version": "1"}, self.install))
+        with patch.object(self, "install") as install:
+            self.assertFalse(self.cache.install_prefix(
+                self.prefix, {"version": "1"}, install))
+            install.assert_not_called()
+        self.assertTrue(self.cache.install_prefix(
+            self.prefix, {"version": "2"}, self.install))
+        receipt = json.loads((self.prefix / self.cache.RECEIPT).read_text())
+        self.assertEqual(receipt["inputs"], {"version": "2"})
+
+    def test_failed_install_can_retry(self):
+        def fail(staging):
+            (staging / "partial").touch()
+            raise RuntimeError("interrupted")
+        with self.assertRaisesRegex(RuntimeError, "interrupted"):
+            self.cache.install_prefix(self.prefix, {}, fail)
+        self.assertFalse(self.prefix.exists())
+        self.assertTrue(self.cache.install_prefix(
+            self.prefix, {}, self.install))
+        self.assertFalse((self.prefix / "partial").exists())
+
+    def test_failed_update_keeps_previous_install(self):
+        self.cache.install_prefix(self.prefix, {"version": "1"}, self.install)
+        def fail(staging):
+            raise RuntimeError("interrupted")
+        with self.assertRaisesRegex(RuntimeError, "interrupted"):
+            self.cache.install_prefix(self.prefix, {"version": "2"}, fail)
+        receipt = json.loads((self.prefix / self.cache.RECEIPT).read_text())
+        self.assertEqual(receipt["inputs"], {"version": "1"})
+        self.assertEqual((self.prefix / "library").read_text(), "complete")
+
+    def test_unmanaged_prefix_requires_override(self):
+        self.prefix.mkdir()
+        (self.prefix / "local").touch()
+        with self.assertRaisesRegex(ValueError, "the existing prefix"):
+            self.cache.install_prefix(self.prefix, {}, self.install)
+        self.assertFalse(self.cache.install_prefix(
+            self.prefix, {}, self.install, use_existing=True))
+        self.assertTrue((self.prefix / "local").exists())
+        self.assertFalse((self.prefix / self.cache.RECEIPT).exists())
+
+    def test_publish_failure_restores_previous_install(self):
+        self.cache.install_prefix(self.prefix, {"version": "1"}, self.install)
+        rename = Path.rename
+        def fail_publish(path, target):
+            if target == self.prefix and "previous" not in str(path):
+                raise OSError("publish failed")
+            return rename(path, target)
+        with patch.object(Path, "rename", fail_publish):
+            with self.assertRaisesRegex(OSError, "publish failed"):
+                self.cache.install_prefix(
+                    self.prefix, {"version": "2"}, self.install)
+        receipt = json.loads((self.prefix / self.cache.RECEIPT).read_text())
+        self.assertEqual(receipt["inputs"], {"version": "1"})
+
+    def test_overlapping_install_is_rejected(self):
+        def overlap(staging):
+            with self.assertRaisesRegex(RuntimeError, "in use"):
+                self.cache.install_prefix(self.prefix, {}, self.install)
+            self.install(staging)
+        self.assertTrue(self.cache.install_prefix(self.prefix, {}, overlap))
+
+
 if __name__ == "__main__":
     unittest.main()

Platforms/Android/README.md

@@ -172,3 +172,16 @@ For more details, run `android.py test --help`.
 ## Using in your own app
 
 See https://docs.python.org/3/using/android.html.
+
+### Dependency installations
+
+The build script records the target and dependency versions in the installed
+prefix. It reuses a completed matching installation and rebuilds dependencies
+when those inputs change. A failed download or extraction does not mark the
+replacement complete; the previous installation remains available.
+
+For a prefix created by an older build script, `--clean` removes the relevant
+host build directory and rebuilds it. Alternatively, pass `--use-existing-deps`
+to `configure-host`, `build`, or `ci` to use the existing target prefix without
+updating it. The override requires an existing prefix and does not record it as
+a managed installation.

Platforms/Android/__main__.py

@@ -218,20 +218,22 @@ def pythoninfo_build_python(context):
 #
 # If you're a member of the Python core team, and you'd like to be able to push
 # these tags yourself, please contact Malcolm Smith or Russell Keith-Magee.
+DEPENDENCIES = [
+    "bzip2-1.0.8-3",
+    "libffi-3.4.4-3",
+    "openssl-3.5.9-0",
+    "sqlite-3.53.4-0",
+    "xz-5.4.6-1",
+    "zstd-1.5.7-2",
+]
+
+
 def unpack_deps(host, prefix_dir, cache_dir):
-    os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
-    for name_ver in [
-        "bzip2-1.0.8-3",
-        "libffi-3.4.4-3",
-        "openssl-3.5.9-0",
-        "sqlite-3.53.4-0",
-        "xz-5.4.6-1",
-        "zstd-1.5.7-2"
-    ]:
+    for name_ver in DEPENDENCIES:
         filename = f"{name_ver}-{host}.tar.gz"
         out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir)
-        shutil.unpack_archive(out_path)
+        shutil.unpack_archive(out_path, prefix_dir)
 
 
 def download(url, cache_dir):
@@ -258,14 +260,23 @@ def configure_host_python(context, host=None):
 
     host_dir = subdir(host, create=True)
     prefix_dir = host_dir / "prefix"
-    if not prefix_dir.exists():
-        prefix_dir.mkdir()
-        cache_dir = (
-            Path(context.cache_dir).resolve()
-            if context.cache_dir
-            else CROSS_BUILD_DIR / "downloads"
-        )
-        unpack_deps(host, prefix_dir, cache_dir)
+    sys.path.insert(0, str(PYTHON_DIR / "Tools" / "build"))
+    try:
+        from dependency_cache import install_prefix
+    finally:
+        sys.path.pop(0)
+
+    cache_dir = (
+        Path(context.cache_dir).resolve()
+        if context.cache_dir
+        else CROSS_BUILD_DIR / "downloads"
+    )
+    install_prefix(
+        prefix_dir,
+        {"target": host, "packages": DEPENDENCIES, "recipe": 1},
+        lambda staging: unpack_deps(host, staging, cache_dir),
+        use_existing=context.use_existing_deps,
+    )
 
     os.chdir(host_dir)
     command = [
@@ -943,6 +954,12 @@ def add_parser(*args, **kwargs):
             help="The directory to store cached downloads.",
         )
 
+    for cmd in [configure_host, build, ci]:
+        cmd.add_argument(
+            "--use-existing-deps", action="store_true",
+            help="Use the existing dependency prefix without updating it",
+        )
+
     # --clean option
     for subcommand in [build, configure_build, configure_host, ci]:
         subcommand.add_argument(

Platforms/Apple/__main__.py

@@ -344,6 +344,16 @@ def apple_multiarch(host: str) -> str:
     raise KeyError(host)
 
 
+DEPENDENCIES = [
+    "BZip2-1.0.8-2",
+    "libFFI-3.4.7-2",
+    "OpenSSL-3.5.9-1",
+    "XZ-5.6.4-2",
+    "mpdecimal-4.0.0-2",
+    "zstd-1.5.7-1",
+]
+
+
 def unpack_deps(
     platform: str,
     host: str,
@@ -366,14 +376,7 @@ def unpack_deps(
     # push these tags yourself, please contact Malcolm Smith or Russell
     # Keith-Magee.
     deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download"
-    for name_ver in [
-        "BZip2-1.0.8-2",
-        "libFFI-3.4.7-2",
-        "OpenSSL-3.5.9-1",
-        "XZ-5.6.4-2",
-        "mpdecimal-4.0.0-2",
-        "zstd-1.5.7-1",
-    ]:
+    for name_ver in DEPENDENCIES:
         filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz"
         archive_path = download(
             f"{deps_url}/{name_ver}/{filename}",
@@ -431,16 +434,30 @@ def configure_host_python(
     prefix_dir = host_dir / "prefix"
 
     with group(f"Downloading dependencies ({host})"):
-        if not prefix_dir.exists():
-            prefix_dir.mkdir()
-            cache_dir = (
-                Path(context.cache_dir).resolve()
-                if context.cache_dir
-                else CROSS_BUILD_DIR / "downloads"
-            )
-            unpack_deps(context.platform, host, prefix_dir, cache_dir)
-        else:
-            print("Dependencies already installed")
+        sys.path.insert(0, str(PYTHON_DIR / "Tools" / "build"))
+        try:
+            from dependency_cache import install_prefix
+        finally:
+            sys.path.pop(0)
+
+        cache_dir = (
+            Path(context.cache_dir).resolve()
+            if context.cache_dir
+            else CROSS_BUILD_DIR / "downloads"
+        )
+        install_prefix(
+            prefix_dir,
+            {
+                "target": host,
+                "platform": context.platform,
+                "packages": DEPENDENCIES,
+                "recipe": 1,
+            },
+            lambda staging: unpack_deps(
+                context.platform, host, staging, cache_dir
+            ),
+            use_existing=context.use_existing_deps,
+        )
 
     with (
         group(f"Configuring host Python ({host})"),
@@ -1092,6 +1109,13 @@ def parse_args() -> argparse.Namespace:
             help="The directory to store cached downloads.",
         )
 
+    for cmd in [configure_host, build, ci]:
+        cmd.add_argument(
+            "--use-existing-deps",
+            action="store_true",
+            help="Use the existing dependency prefix without updating it",
+        )
+
     # --simulator option
     for cmd in [test, ci]:
         cmd.add_argument(

Platforms/Apple/iOS/README.md

@@ -454,3 +454,16 @@ select the root node of the project tree (labeled "iOSTestbed"), then the
 (this will likely be your own name), and plug in a physical device to your
 macOS machine with a USB cable. You should then be able to select your physical
 device from the list of targets in the pulldown in the Xcode titlebar.
+
+### Dependency installations
+
+The build script records the target and dependency versions in the installed
+prefix. It reuses a completed matching installation and rebuilds dependencies
+when those inputs change. A failed download or extraction does not mark the
+replacement complete; the previous installation remains available.
+
+For a prefix created by an older build script, `--clean` removes the relevant
+host build directory and rebuilds it. Alternatively, pass `--use-existing-deps`
+to `configure-host`, `build`, or `ci` to use the existing target prefix without
+updating it. The override requires an existing prefix and does not record it as
+a managed installation.

Tools/build/dependency_cache.py

@@ -0,0 +1,91 @@
+"""Install managed build dependencies with a completion receipt."""
+
+import contextlib
+import json
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+RECEIPT = ".cpython-dependencies.json"
+
+
+@contextlib.contextmanager
+def _prefix_lock(prefix):
+    # Keep the lock file: unlinking it could let two processes lock different
+    # files for the same prefix. Closing the handle releases the lock on exit.
+    with (prefix.parent / f".{prefix.name}.lock").open("a+b") as lock:
+        if os.name == "nt":
+            import msvcrt
+
+            if lock.tell() == 0:
+                lock.write(b"\0")
+                lock.flush()
+            lock.seek(0)
+            try:
+                msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1)
+            except OSError as exc:
+                raise RuntimeError(f"Dependency prefix is in use: {prefix}") from exc
+        else:
+            import fcntl
+
+            try:
+                fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError as exc:
+                raise RuntimeError(f"Dependency prefix is in use: {prefix}") from exc
+        yield
+
+
+def install_prefix(prefix, inputs, install, *, use_existing=False):
+    """Run install(staging_prefix), then publish its files and input receipt.
+
+    A matching receipt reuses a completed installation. Unmanaged prefixes
+    require an explicit override, so local dependency builds are not replaced.
+    """
+    prefix = Path(prefix).absolute()
+    if use_existing:
+        if not prefix.is_dir():
+            raise ValueError(f"Dependency prefix does not exist: {prefix}")
+        return False
+    if prefix.is_symlink():
+        raise ValueError(f"Cannot replace a symlinked dependency prefix: {prefix}")
+    prefix.parent.mkdir(parents=True, exist_ok=True)
+    expected = json.loads(json.dumps({"schema": 1, "inputs": inputs}))
+    with _prefix_lock(prefix):
+        if prefix.exists():
+            try:
+                receipt = json.loads((prefix / RECEIPT).read_text())
+            except (OSError, ValueError) as exc:
+                raise ValueError(
+                    f"No valid dependency receipt in {prefix}; refusing to replace "
+                    "the existing prefix"
+                ) from exc
+            if receipt == expected:
+                return False
+        with tempfile.TemporaryDirectory(
+            dir=prefix.parent, prefix=f".{prefix.name}-"
+        ) as tmp:
+            staging = Path(tmp) / "prefix"
+            staging.mkdir()
+            install(staging)
+            (staging / RECEIPT).write_text(
+                json.dumps(expected, indent=2, sort_keys=True) + "\n"
+            )
+            previous = None
+            if prefix.exists():
+                previous = Path(tempfile.mkdtemp(
+                    dir=prefix.parent, prefix=f".{prefix.name}-previous-"
+                )) / "prefix"
+                prefix.rename(previous)
+            try:
+                staging.rename(prefix)
+            except BaseException:
+                if previous is not None:
+                    # A failed restore leaves the old installation outside the
+                    # temporary directory, available for manual recovery.
+                    previous.rename(prefix)
+                    previous.parent.rmdir()
+                raise
+            if previous is not None:
+                shutil.rmtree(previous.parent)
+    return True