Select dependency consumers and check lock inputs

.github/workflows/check-ci-dependencies.yml

@@ -0,0 +1,68 @@
+name: Check CI dependencies
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+      - '.github/dependencies/**'
+      - 'Doc/requirements*.txt'
+      - 'Doc/constraints.txt'
+      - 'Doc/pylock*.toml'
+      - 'Tools/requirements*'
+      - 'Tools/pylock*.toml'
+      - 'Tools/build/check_ci_dependencies.py'
+      - 'Tools/build/ci-dependency-inputs.json'
+      - 'Lib/ensurepip/**'
+  push:
+    branches: [main, '3.*']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  locks:
+    name: Check lock inputs and installer pins
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.12.13'
+      - run: python Tools/build/check_ci_dependencies.py
+
+  notification:
+    name: Check notification dependencies
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        if: hashFiles('.github/dependencies/notification/package-lock.json') != ''
+        with:
+          node-version: '22.23.3'
+      - run: npm ci --ignore-scripts
+        if: hashFiles('.github/dependencies/notification/package-lock.json') != ''
+        working-directory: .github/dependencies/notification
+      - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        if: hashFiles('.github/dependencies/notification/package-lock.json') != ''
+        with:
+          script: |
+            const {createRequire} = require('node:module');
+            const {join} = require('node:path');
+            const notificationRequire = createRequire(join(
+              process.env.GITHUB_WORKSPACE,
+              '.github/dependencies/notification/package.json',
+            ));
+            const Mailgun = notificationRequire('mailgun.js');
+            const FormData = notificationRequire('form-data');
+            new Mailgun(FormData).client({username: 'api', key: 'dependency-check'});

.github/workflows/reusable-docs.yml

@@ -11,8 +11,14 @@ on:
       - main
       - '3.*'
     paths:
-      - 'Doc/pylock.toml'
-      - 'Doc/requirements.txt'
+      - 'Doc/pylock*.toml'
+      - 'Doc/requirements*.txt'
+      - 'Doc/constraints.txt'
+      - 'Doc/Makefile'
+      - 'Doc/tools/create_venv.py'
+      - 'Tools/requirements-bootstrap.txt'
+      - 'Tools/requirements-build.*'
+      - 'Lib/ensurepip/**'
       - '.github/workflows/reusable-docs.yml'
 
 permissions:

Lib/test/test_tools/test_ci_dependencies.py

@@ -0,0 +1,105 @@
+"""Tests for the offline CI dependency consistency checks."""
+
+import hashlib
+import importlib
+from pathlib import Path
+import tempfile
+import unittest
+
+from test.test_tools import imports_under_tool, skip_if_missing
+
+skip_if_missing("build")
+with imports_under_tool("build"):
+    dependencies = importlib.import_module("check_ci_dependencies")
+
+
+class DependencyTests(unittest.TestCase):
+    def setUp(self):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        self.root = Path(directory.name)
+        self.source = self.root / "requirements.txt"
+        self.source.write_text("example==1.0\n", newline="\n")
+        self.lock = self.root / "pylock.toml"
+        self.lock.write_text(
+            '[[packages]]\nname = "example"\nversion = "1.0"\n'
+            'wheels = [{url = "https://example.invalid/example.whl", '
+            'hashes = {sha256 = "' + "a" * 64 + '"}}]\n',
+            newline="\n",
+        )
+        self.inventory = {"pylock.toml": ("requirements.txt",)}
+
+    def snapshot(self):
+        return dependencies.snapshot(self.root, self.inventory)
+
+    def test_unchanged_inputs(self):
+        recorded = self.snapshot()
+        self.assertEqual(dependencies.check_record(self.snapshot(), recorded), [])
+
+    def test_checkout_line_endings(self):
+        recorded = self.snapshot()
+        for path in (self.source, self.lock):
+            path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n"))
+        self.assertEqual(dependencies.check_record(self.snapshot(), recorded), [])
+
+    def test_source_change_requires_refresh(self):
+        recorded = self.snapshot()
+        self.source.write_text("example>=2\n")
+        self.assertTrue(dependencies.check_record(self.snapshot(), recorded))
+
+    def test_lock_change_requires_refresh(self):
+        recorded = self.snapshot()
+        self.lock.write_text(self.lock.read_text().replace('"1.0"', '"2.0"'))
+        self.assertTrue(dependencies.check_record(self.snapshot(), recorded))
+
+    def test_constraints_are_recorded(self):
+        constraint = self.root / "constraints.txt"
+        constraint.write_text("dependency<2\n")
+        self.inventory["pylock.toml"] += ("constraints.txt",)
+        recorded = self.snapshot()
+        constraint.write_text("dependency<3\n")
+        self.assertTrue(dependencies.check_record(self.snapshot(), recorded))
+
+    def test_removed_lock_requires_record_update(self):
+        self.assertTrue(dependencies.check_record({}, self.snapshot()))
+
+    def test_matching_direct_pin(self):
+        self.assertEqual(
+            dependencies.check_python_locks(self.root, self.inventory), [],
+        )
+
+    def test_stale_direct_pin(self):
+        self.source.write_text("example==2.0\n")
+        errors = dependencies.check_python_locks(self.root, self.inventory)
+        self.assertTrue(any("missing direct pin example==2.0" in e for e in errors))
+
+    def test_missing_artifact_hash(self):
+        self.lock.write_text(self.lock.read_text().replace("a" * 64, "invalid"))
+        errors = dependencies.check_python_locks(self.root, self.inventory)
+        self.assertTrue(any("SHA-256" in e for e in errors))
+
+    def test_bootstrap_matches_wheel(self):
+        wheel = self.root / "Lib/ensurepip/_bundled/pip-1.0-py3-none-any.whl"
+        wheel.parent.mkdir(parents=True)
+        wheel.write_bytes(b"bundled\r\npip wheel")
+        bootstrap = self.root / "Tools/requirements-bootstrap.txt"
+        bootstrap.parent.mkdir()
+        sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest()
+        bootstrap.write_text(f"pip==1.0 --hash=sha256:{sha256}\n")
+        self.assertEqual(dependencies.check_bootstrap(self.root), [])
+        wheel.write_bytes(b"changed wheel")
+        self.assertTrue(dependencies.check_bootstrap(self.root))
+
+    def test_bootstrap_version_must_match(self):
+        wheel = self.root / "Lib/ensurepip/_bundled/pip-2.0-py3-none-any.whl"
+        wheel.parent.mkdir(parents=True)
+        wheel.write_bytes(b"bundled pip wheel")
+        bootstrap = self.root / "Tools/requirements-bootstrap.txt"
+        bootstrap.parent.mkdir()
+        sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest()
+        bootstrap.write_text(f"pip==1.0 --hash=sha256:{sha256}\n")
+        self.assertTrue(dependencies.check_bootstrap(self.root))
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_tools/test_compute_changes.py

@@ -60,6 +60,56 @@ def test_ci_fuzz_stdlib(self):
                     self.assertTrue(result.run_ci_fuzz_stdlib)
                     self.assertTrue(is_fuzzable_library_file(f))
 
+    def test_homebrew_dependencies(self):
+        result = process_changed_files({Path("Misc/Brewfile")})
+        self.assertEqual(result, Outputs(run_tests=True, run_macos=True))
+
+    def test_shared_python_dependencies(self):
+        for name in ("bootstrap.txt", "build.in", "build.txt"):
+            with self.subTest(name=name):
+                result = process_changed_files({Path(f"Tools/requirements-{name}")})
+                self.assertEqual(result, Outputs(
+                    run_tests=True, run_ubuntu=True, run_docs=True,
+                ))
+
+    def test_hypothesis_dependencies(self):
+        for name in ("requirements-hypothesis.txt", "pylock.hypothesis.toml"):
+            with self.subTest(name=name):
+                result = process_changed_files({Path("Tools") / name})
+                self.assertEqual(result, Outputs(run_tests=True, run_ubuntu=True))
+
+    def test_native_dependencies(self):
+        for name in ("dependency-hashes.json", "download_dependencies.py"):
+            with self.subTest(name=name):
+                result = process_changed_files({Path("Tools/build") / name})
+                self.assertEqual(result, Outputs(
+                    run_tests=True, run_android=True, run_ios=True,
+                    run_ubuntu=True, run_windows_tests=True,
+                ))
+
+    def test_ensurepip_also_runs_docs(self):
+        result = process_changed_files({Path("Lib/ensurepip/_bundled/pip.whl")})
+        self.assertTrue(result.run_docs)
+        self.assertTrue(result.run_tests)
+        self.assertTrue(result.run_windows_tests)
+
+    def test_dependency_and_source_changes(self):
+        result = process_changed_files({
+            Path("Tools/requirements-bootstrap.txt"), Path("Lib/ast.py"),
+        })
+        self.assertTrue(result.run_docs)
+        self.assertTrue(result.run_ubuntu)
+        self.assertTrue(result.run_windows_tests)
+        self.assertTrue(result.run_macos)
+
+    def test_dependency_consumers_are_combined(self):
+        result = process_changed_files({
+            Path("Tools/pylock.hypothesis.toml"), Path("Misc/Brewfile"),
+        })
+        self.assertEqual(result, Outputs(
+            run_tests=True, run_ubuntu=True, run_macos=True,
+        ))
+
     def test_android(self):
         for d in ANDROID_DIRS:
             with self.subTest(d=d):

Tools/build/check_ci_dependencies.py

@@ -0,0 +1,122 @@
+"""Check recorded CI lock inputs and selected metadata without network access."""
+
+import argparse
+import hashlib
+import json
+import re
+from pathlib import Path
+
+import tomllib
+
+ROOT = Path(__file__).resolve().parents[2]
+RECORD = Path("Tools/build/ci-dependency-inputs.json")
+LOCK_INPUTS = {
+    "Doc/pylock.toml": ("Doc/requirements.txt", "Doc/constraints.txt"),
+    "Doc/pylock.epubcheck.toml": ("Doc/requirements-epubcheck.txt",),
+    "Tools/pylock.dev.toml": ("Tools/requirements-dev.txt",),
+    "Tools/pylock.hypothesis.toml": ("Tools/requirements-hypothesis.txt",),
+    "Tools/requirements-build.txt": ("Tools/requirements-build.in",),
+}
+
+
+def digest(path, *, normalize_newlines=False):
+    content = path.read_bytes()
+    if normalize_newlines:
+        content = content.replace(b"\r\n", b"\n")
+    return hashlib.sha256(content).hexdigest()
+
+
+def snapshot(root, inventory=LOCK_INPUTS):
+    return {
+        lock: {
+            "sha256": digest(root / lock, normalize_newlines=True),
+            "inputs": {
+                source: digest(root / source, normalize_newlines=True)
+                for source in sources
+            },
+        }
+        for lock, sources in inventory.items()
+    }
+
+
+def normalize(name):
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def check_python_locks(root, inventory=LOCK_INPUTS):
+    errors = []
+    for filename, sources in inventory.items():
+        if not filename.endswith(".toml"):
+            continue
+        lock = tomllib.loads((root / filename).read_text(encoding="utf-8"))
+        packages = lock.get("packages", [])
+        if not packages:
+            errors.append(f"{filename}: no locked packages")
+        versions = {}
+        for package in packages:
+            name = normalize(package["name"])
+            versions.setdefault(name, set()).add(package.get("version"))
+            artifacts = [*package.get("wheels", [])]
+            if "sdist" in package:
+                artifacts.append(package["sdist"])
+            if not package.get("version") or not artifacts:
+                errors.append(f"{filename}: {name} needs a version and artifacts")
+            for artifact in artifacts:
+                sha256 = artifact.get("hashes", {}).get("sha256", "")
+                if not re.fullmatch(r"[0-9a-f]{64}", sha256):
+                    errors.append(f"{filename}: {name} has no valid SHA-256 hash")
+
+        # Check plain `name==version` pins after an input-record update.
+        source = root / sources[0]
+        for line in source.read_text(encoding="utf-8").splitlines():
+            match = re.fullmatch(r"([A-Za-z0-9_.-]+)==([^\s;*]+)", line.strip())
+            if match and match[2] not in versions.get(normalize(match[1]), set()):
+                errors.append(f"{filename}: missing direct pin {line.strip()}")
+    return errors
+
+
+def check_bootstrap(root):
+    wheels = list((root / "Lib/ensurepip/_bundled").glob("pip-*.whl"))
+    if len(wheels) != 1:
+        return ["Expected one bundled pip wheel"]
+    wheel = wheels[0]
+    version = wheel.name.split("-")[1]
+    bootstrap = (root / "Tools/requirements-bootstrap.txt").read_text()
+    if (not re.search(rf"(?m)^pip=={re.escape(version)}(?:\s|$)", bootstrap)
+            or f"--hash=sha256:{digest(wheel)}" not in bootstrap):
+        return ["Tools/requirements-bootstrap.txt must match the bundled pip wheel"]
+    return []
+
+
+def check_record(current, recorded):
+    return [
+        f"{lock}: regenerate the lock, then record its inputs with "
+        f"Tools/build/check_ci_dependencies.py --record {lock}"
+        for lock in sorted(current.keys() | recorded.keys())
+        if current.get(lock) != recorded.get(lock)
+    ]
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--record", nargs="+", choices=LOCK_INPUTS,
+        help="record selected locks after regenerating them",
+    )
+    args = parser.parse_args()
+    current = snapshot(ROOT)
+    record = ROOT / RECORD
+    recorded = json.loads(record.read_text()) if record.exists() else {}
+    errors = check_python_locks(ROOT) + check_bootstrap(ROOT)
+    if args.record and not errors:
+        for lock in args.record:
+            recorded[lock] = current[lock]
+        record.write_text(json.dumps(recorded, indent=2, sort_keys=True) + "\n")
+    errors.extend(check_record(current, recorded))
+    for error in errors:
+        print(error)
+    return bool(errors)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

Tools/build/ci-dependencies.md

@@ -0,0 +1,19 @@
+# Maintaining CI dependencies
+
+CI installs committed locks for the environments covered by this check.
+Dependency updates should change the source requirements and their generated lock
+together, without refreshing unrelated environments.
+
+Regenerate Python locks locally using `make -C Doc lock` or the command recorded
+in the generated file. Review the complete version and artifact-hash diff.
+
+After regenerating a lock, run
+`python3 Tools/build/check_ci_dependencies.py --record PATH_TO_LOCK` from the
+repository root, using a repository-relative lock path, and commit the updated
+input record. It associates each lock with its source requirements
+and constraints, so editing either side requires an input-record update. The record
+cannot prove that a lock was regenerated or that its transitive dependencies are
+complete. The check also compares plain `name==version` pins, verifies that `pylock`
+artifacts have SHA-256 metadata in the expected format, and compares the bootstrap
+`pip` version and hash with the bundled wheel. Run the consuming jobs to check
+compatibility.