From 7833cf9cd031339a23e8372a578a2152bafc0ca1 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 13:44:57 -0500 Subject: [PATCH 01/10] Support Cargo 1.100 workspace cache layout --- .../workflows/test-rust-workspace-cache.yml | 10 +- rust-workspace-cache/inputs/inputs.py | 35 +++++- rust-workspace-cache/inputs/test_inputs.py | 70 +++++++++++- rust-workspace-cache/prune/prune.py | 87 +++++++++++---- rust-workspace-cache/prune/test_prune.py | 100 ++++++++++++++++++ rust-workspace-cache/tests/smoke/check.cjs | 77 ++++++++++---- rust-workspace-cache/tests/smoke/setup.cjs | 56 +++++++--- 7 files changed, 378 insertions(+), 57 deletions(-) diff --git a/.github/workflows/test-rust-workspace-cache.yml b/.github/workflows/test-rust-workspace-cache.yml index 17df62f..06ff0b6 100644 --- a/.github/workflows/test-rust-workspace-cache.yml +++ b/.github/workflows/test-rust-workspace-cache.yml @@ -19,6 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + rust: [1.99.0, 1.100.0] runs-on: ${{ matrix.os }} timeout-minutes: 10 steps: @@ -28,6 +29,13 @@ jobs: with: persist-credentials: false + - name: Install Rust ${{ matrix.rust }} + run: rustup toolchain install ${{ matrix.rust }} --profile minimal + + - name: Select Rust ${{ matrix.rust }} + shell: bash + run: echo "RUSTUP_TOOLCHAIN=${{ matrix.rust }}" >> "$GITHUB_ENV" + - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: @@ -56,7 +64,7 @@ jobs: profiling cache-bin: ${{ runner.os != 'macOS' }} prefix-key: test-rust-workspace-cache - key: ${{ github.run_id }}-${{ github.run_attempt }} + key: ${{ matrix.rust }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Build workspace working-directory: ${{ steps.fixture.outputs.workspace }} diff --git a/rust-workspace-cache/inputs/inputs.py b/rust-workspace-cache/inputs/inputs.py index 6d3bd00..786665a 100644 --- a/rust-workspace-cache/inputs/inputs.py +++ b/rust-workspace-cache/inputs/inputs.py @@ -15,6 +15,7 @@ from pathlib import Path FINGERPRINT = re.compile(r"^(?P.+)-[0-9a-f]{16}$") +UNIT_HASH = re.compile(r"^[0-9a-f]{16}$") def workspace_packages(root: Path) -> list[dict]: @@ -149,9 +150,8 @@ def invalidate_units( removed = 0 for profile in profiles: fingerprints = profile / ".fingerprint" - if not fingerprints.is_dir(): - continue - for directory in fingerprints.iterdir(): + directories = fingerprints.iterdir() if fingerprints.is_dir() else () + for directory in directories: match = FINGERPRINT.fullmatch(directory.name) if ( match @@ -167,6 +167,30 @@ def invalidate_units( shutil.rmtree(output) shutil.rmtree(directory) removed += 1 + + # Cargo 1.100 stores each unit's fingerprint and output together. + build = profile / "build" + if not build.is_dir() or build.is_symlink(): + continue + for package in packages: + directory = build / package + if not directory.is_dir() or directory.is_symlink(): + continue + for unit in directory.iterdir(): + fingerprint = unit / "fingerprint" + if ( + UNIT_HASH.fullmatch(unit.name) + and unit.is_dir() + and not unit.is_symlink() + and fingerprint.is_dir() + and not fingerprint.is_symlink() + and ( + not build_scripts_only + or any(fingerprint.glob("run-build-script-*")) + ) + ): + shutil.rmtree(unit) + removed += 1 return removed @@ -286,7 +310,10 @@ def main() -> None: ) else: result = {"no_manifest": True} - if any((profile / ".fingerprint").is_dir() for profile in args.profiles): + if any( + (profile / ".fingerprint").is_dir() or (profile / "build").is_dir() + for profile in args.profiles + ): packages = {package["name"] for package in workspace_packages(Path.cwd())} result["invalidated_workspace_units"] = invalidate_units( args.profiles, packages, build_scripts_only=False diff --git a/rust-workspace-cache/inputs/test_inputs.py b/rust-workspace-cache/inputs/test_inputs.py index 199274e..f3406b4 100644 --- a/rust-workspace-cache/inputs/test_inputs.py +++ b/rust-workspace-cache/inputs/test_inputs.py @@ -2,8 +2,10 @@ from __future__ import annotations +import json import os import subprocess +import sys import tempfile import time import unittest @@ -33,6 +35,51 @@ def test_restore_replaces_stale_files_with_the_saved_directory(self) -> None: self.assertFalse((live / "stale").exists()) self.assertFalse(staged.exists()) + def test_new_layout_invalidation_removes_only_matching_units(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + profile = root / "debug" + package = profile / "build" / "fixture" + script = package / "0123456789abcdef" + binary = package / "fedcba9876543210" + other = profile / "build" / "other" / "0123456789abcdef" + for unit, marker in ( + (script, "run-build-script-build"), + (binary, "bin-fixture"), + (other, "run-build-script-build"), + ): + (unit / "fingerprint").mkdir(parents=True) + (unit / "fingerprint" / marker).touch() + (unit / "out").mkdir() + (unit / "out" / "generated").touch() + (script / "run").mkdir() + (script / "run" / "stdout").touch() + outside = root / "outside" + (outside / "fingerprint").mkdir(parents=True) + (outside / "fingerprint" / "run-build-script-build").touch() + try: + (package / "1111111111111111").symlink_to( + outside, target_is_directory=True + ) + except OSError as error: + self.skipTest(str(error)) + + self.assertEqual( + inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), + 1, + ) + self.assertFalse(script.exists()) + self.assertTrue(binary.exists()) + self.assertTrue(other.exists()) + self.assertTrue(outside.exists()) + self.assertEqual( + inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=False), + 1, + ) + self.assertFalse(binary.exists()) + self.assertTrue(other.exists()) + self.assertTrue(outside.exists()) + def test_source_layout_ignores_build_outputs_and_detects_retargeted_links(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary).resolve() @@ -190,7 +237,10 @@ def test_subsequent_checkout_keeps_matching_input_timestamps(self) -> None: def test_changed_bytes_rebuild_when_cached_output_is_in_the_future(self) -> None: future = time.time_ns() + 60_000_000_000 - for output in (self.root / "target/debug/build").glob("*/output"): + build = self.root / "target/debug/build" + outputs = [*build.glob("*/output"), *build.glob("*/*/run/stdout")] + self.assertTrue(outputs) + for output in outputs: os.utime(output, ns=(future, future)) old = self.value.stat().st_mtime_ns self.value.write_text("changed") @@ -231,6 +281,24 @@ def test_git_metadata_is_not_restored(self) -> None: self.restore() self.assertEqual(head.stat().st_mtime_ns, 1_000_000_000) + def test_missing_manifest_invalidates_cached_workspace_units(self) -> None: + result = subprocess.run( + [ + sys.executable, + str(Path(inputs.__file__).resolve()), + "restore", + str(self.root / "missing-manifest.json"), + str(self.root / "target/debug"), + ], + cwd=self.root, + env=self.environment, + text=True, + capture_output=True, + check=True, + ) + self.assertGreater(json.loads(result.stdout)["invalidated_workspace_units"], 0) + self.build("original", rebuilt=True) + def test_rejects_escaping_paths(self) -> None: with self.assertRaises(ValueError): inputs.safe_path(self.root, "../outside") diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index 7d61b46..c6bb01e 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -11,6 +11,7 @@ FINGERPRINT_DIRECTORY = re.compile(r"^(?P.+)-(?P[0-9a-f]{16})$") ARTIFACT_FINGERPRINT = re.compile(r"-([0-9a-f]{16})(?:\.|$)") +UNIT_FINGERPRINT = re.compile(r"^[0-9a-f]{16}$") def workspace_packages(workspace: Path) -> set[str]: @@ -31,7 +32,9 @@ def fingerprint_last_used(directory: Path) -> int | None: timestamps = [ fingerprint.stat().st_mtime_ns for metadata in directory.glob("*.json") - if (fingerprint := directory / metadata.stem).is_file() + if not metadata.is_symlink() + and not (fingerprint := directory / metadata.stem).is_symlink() + and fingerprint.is_file() ] return max(timestamps, default=None) @@ -40,24 +43,24 @@ def prune_workspace_fingerprints( profile: Path, marker_timestamp: int, packages: set[str] ) -> tuple[int, int, int]: fingerprint_root = profile / ".fingerprint" + build_root = profile / "build" stale_fingerprints_by_package: dict[str, set[str]] = {} active_fingerprints: set[str] = set() active_packages: set[str] = set() - - for directory in fingerprint_root.iterdir(): - if not directory.is_dir(): - continue - - match = FINGERPRINT_DIRECTORY.fullmatch(directory.name) - if match is None: - continue - + protected_fingerprints: set[str] = set() + units: list[tuple[str, str, Path]] = [] + legacy_units: set[tuple[str, str]] = set() + package_directories: set[Path] = set() + + def record(package: str, fingerprint: str, directory: Path) -> None: + if directory.is_symlink() or not directory.is_dir(): + protected_fingerprints.add(fingerprint) + return last_used = fingerprint_last_used(directory) + if last_used is None or package not in packages: + protected_fingerprints.add(fingerprint) if last_used is None: - continue - - package = match.group("package") - fingerprint = match.group("fingerprint") + return if last_used >= marker_timestamp: active_fingerprints.add(fingerprint) if package in packages: @@ -65,25 +68,69 @@ def prune_workspace_fingerprints( elif package in packages: stale_fingerprints_by_package.setdefault(package, set()).add(fingerprint) + # Cargo before 1.100 stores all fingerprints in one profile directory. + if not fingerprint_root.is_symlink() and fingerprint_root.is_dir(): + for directory in fingerprint_root.iterdir(): + match = FINGERPRINT_DIRECTORY.fullmatch(directory.name) + if match is not None: + package = match.group("package") + fingerprint = match.group("fingerprint") + record(package, fingerprint, directory) + legacy_units.add((package, fingerprint)) + + # Cargo 1.100 stores each compilation unit, including its outputs, under + # build//. Do not follow symlinks supplied by a cache. + if not build_root.is_symlink() and build_root.is_dir(): + for package_directory in build_root.iterdir(): + if package_directory.is_symlink() or not package_directory.is_dir(): + continue + for unit in package_directory.iterdir(): + if UNIT_FINGERPRINT.fullmatch(unit.name) is None: + continue + package_directories.add(package_directory) + if unit.is_symlink() or not unit.is_dir(): + protected_fingerprints.add(unit.name) + continue + record(package_directory.name, unit.name, unit / "fingerprint") + units.append((package_directory.name, unit.name, unit)) + if not active_fingerprints: raise RuntimeError("No active Cargo fingerprints found; refusing to prune") - stale_fingerprints = { - fingerprint + stale_units = { + (package, fingerprint) for package in active_packages for fingerprint in stale_fingerprints_by_package.get(package, set()) + if fingerprint not in active_fingerprints + and fingerprint not in protected_fingerprints } - stale_fingerprints.difference_update(active_fingerprints) + stale_fingerprints = {fingerprint for _, fingerprint in stale_units} + legacy_stale_units = stale_units & legacy_units + legacy_stale_fingerprints = {fingerprint for _, fingerprint in legacy_stale_units} removed_paths = 0 - for directory in (fingerprint_root, profile / "deps", profile / "build"): - if not directory.is_dir(): + for package, fingerprint, unit in units: + if (package, fingerprint) in stale_units: + shutil.rmtree(unit) + removed_paths += 1 + + for directory in (fingerprint_root, profile / "deps", build_root): + if directory.is_symlink() or not directory.is_dir(): continue for artifact in directory.iterdir(): match = ARTIFACT_FINGERPRINT.search(artifact.name) - if match is None or match.group(1) not in stale_fingerprints: + if match is None or match.group(1) not in legacy_stale_fingerprints: continue + if directory in (fingerprint_root, build_root): + fingerprint = FINGERPRINT_DIRECTORY.fullmatch(artifact.name) + if fingerprint is None or ( + fingerprint.group("package"), fingerprint.group("fingerprint") + ) not in legacy_stale_units: + continue + if directory == build_root and artifact in package_directories: + # This is a Cargo 1.100 package directory, not a legacy build unit. + continue if artifact.is_symlink() or artifact.is_file(): artifact.unlink() diff --git a/rust-workspace-cache/prune/test_prune.py b/rust-workspace-cache/prune/test_prune.py index 6b462b5..686f5ac 100644 --- a/rust-workspace-cache/prune/test_prune.py +++ b/rust-workspace-cache/prune/test_prune.py @@ -31,6 +31,106 @@ def fingerprint(self, package: str, fingerprint: str, timestamp: int) -> Path: os.utime(stamp, ns=(timestamp, timestamp)) return directory + def unit(self, package: str, fingerprint: str, timestamp: int) -> Path: + unit = self.profile / "build" / package / fingerprint + directory = unit / "fingerprint" + directory.mkdir(parents=True) + (directory / "lib-example.json").write_text("{}") + stamp = directory / "lib-example" + stamp.touch() + os.utime(stamp, ns=(timestamp, timestamp)) + (unit / "out").mkdir() + (unit / "out" / "artifact").write_text("artifact") + return unit + + def test_prunes_cargo_1_100_workspace_units(self) -> None: + (self.profile / ".fingerprint").rmdir() + old = self.unit("example", "1111111111111111", OLD) + current = self.unit("example", "2222222222222222", CURRENT) + dependency = self.unit("dependency", "3333333333333333", OLD) + unused = self.unit("unused", "4444444444444444", OLD) + unknown = self.unit("example", "5555555555555555", OLD) + (unknown / "fingerprint" / "lib-example").unlink() + malformed = self.profile / "build" / "example" / "not-a-hash" + malformed.mkdir() + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example", "unused"}), + (1, 1, 1), + ) + self.assertFalse(old.exists()) + for retained in (current, dependency, unused, unknown, malformed): + self.assertTrue(retained.is_dir()) + + def test_retains_colliding_dependency_and_active_hashes_across_layouts(self) -> None: + old = self.unit("example", "1111111111111111", OLD) + self.unit("example", "2222222222222222", CURRENT) + dependency = self.fingerprint("dependency", "1111111111111111", OLD) + other_old = self.fingerprint("example", "3333333333333333", OLD) + active = self.unit("dependency", "3333333333333333", CURRENT) + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example"}), + (0, 0, 2), + ) + for retained in (old, dependency, other_old, active): + self.assertTrue(retained.is_dir()) + + def test_retains_dependency_package_named_like_a_legacy_build_unit(self) -> None: + self.unit("example", "1111111111111111", OLD) + self.unit("example", "2222222222222222", CURRENT) + dependency = self.unit("example-1111111111111111", "3333333333333333", OLD) + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example"}), + (1, 1, 1), + ) + self.assertTrue(dependency.is_dir()) + + def test_does_not_apply_legacy_cleanup_to_cargo_1_100_hashes(self) -> None: + self.unit("example", "1111111111111111", OLD) + self.unit("example", "2222222222222222", CURRENT) + ambiguous = self.profile / "build" / "example-1111111111111111" + ambiguous.mkdir() + dependency = self.profile / "deps" / "libdependency-1111111111111111.rlib" + dependency.touch() + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example"}), + (1, 1, 1), + ) + self.assertTrue(ambiguous.is_dir()) + self.assertTrue(dependency.is_file()) + + def test_refuses_to_prune_cargo_1_100_without_active_units(self) -> None: + old = self.unit("example", "1111111111111111", OLD) + with self.assertRaisesRegex(RuntimeError, "No active Cargo fingerprints"): + prune_workspace_fingerprints(self.profile, MARKER, {"example"}) + self.assertTrue(old.is_dir()) + + @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") + def test_does_not_follow_cargo_1_100_symlink_nodes(self) -> None: + self.unit("example", "2222222222222222", CURRENT) + target = self.profile / "keep" + target.mkdir() + (target / "sentinel").write_text("keep") + package = self.profile / "build" / "linked-package" + package.symlink_to(target, target_is_directory=True) + unit = self.profile / "build" / "example" / "1111111111111111" + unit.symlink_to(target, target_is_directory=True) + fingerprint = self.profile / "build" / "example" / "3333333333333333" + fingerprint.mkdir() + (fingerprint / "fingerprint").symlink_to(target, target_is_directory=True) + + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, {"example", "linked-package"}), + (0, 0, 1), + ) + self.assertTrue(package.is_symlink()) + self.assertTrue(unit.is_symlink()) + self.assertTrue((fingerprint / "fingerprint").is_symlink()) + self.assertEqual((target / "sentinel").read_text(), "keep") + def test_removes_superseded_workspace_artifacts(self) -> None: old = self.fingerprint("example", "1111111111111111", OLD) current = self.fingerprint("example", "2222222222222222", CURRENT) diff --git a/rust-workspace-cache/tests/smoke/check.cjs b/rust-workspace-cache/tests/smoke/check.cjs index ccbf4bf..d9fed1e 100644 --- a/rust-workspace-cache/tests/smoke/check.cjs +++ b/rust-workspace-cache/tests/smoke/check.cjs @@ -3,23 +3,64 @@ const fs = require("node:fs"); const path = require("node:path"); for (const profile of ["debug", "profiling"]) { - const fingerprints = path.join( - process.env.STATE_workspace, - "target", - profile, - ".fingerprint", - ); - assert.ok( - !fs.existsSync( - path.join(fingerprints, "workspace-cache-fixture-1111111111111111"), - ), - "The stale workspace fingerprint must be pruned", - ); - assert.ok( - fs - .readdirSync(fingerprints) - .some((entry) => entry.startsWith("workspace-cache-fixture-")), - "The active workspace fingerprint must remain cached", - ); + const target = path.join(process.env.STATE_workspace, "target", profile); + if (process.env.STATE_layout === "unit") { + const units = path.join(target, "build", "workspace-cache-fixture"); + assert.ok( + !fs.existsSync(path.join(units, "1111111111111111")), + "The stale workspace unit must be pruned", + ); + assert.ok( + fs.readdirSync(units).some((entry) => + fs.existsSync( + path.join(units, entry, "fingerprint", "lib-workspace_cache_fixture"), + ), + ), + "The active workspace unit must remain cached", + ); + assert.ok( + fs.existsSync( + path.join( + target, + "build", + "cached-dependency", + "2222222222222222", + "out", + "libcached_dependency-2222222222222222.rlib", + ), + ), + "Cached dependency artifacts must remain", + ); + } else { + const fingerprints = path.join(target, ".fingerprint"); + assert.ok( + !fs.existsSync( + path.join(fingerprints, "workspace-cache-fixture-1111111111111111"), + ), + "The stale workspace fingerprint must be pruned", + ); + assert.ok( + !fs.existsSync( + path.join( + target, + "deps", + "libworkspace_cache_fixture-1111111111111111.rlib", + ), + ), + "The stale workspace artifact must be pruned", + ); + assert.ok( + fs + .readdirSync(fingerprints) + .some((entry) => entry.startsWith("workspace-cache-fixture-")), + "The active workspace fingerprint must remain cached", + ); + assert.ok( + fs.existsSync( + path.join(target, "deps", "libcached_dependency-2222222222222222.rlib"), + ), + "Cached dependency artifacts must remain", + ); + } } console.log("Superseded workspace artifacts were pruned."); diff --git a/rust-workspace-cache/tests/smoke/setup.cjs b/rust-workspace-cache/tests/smoke/setup.cjs index 9ec2ee6..7f732d1 100644 --- a/rust-workspace-cache/tests/smoke/setup.cjs +++ b/rust-workspace-cache/tests/smoke/setup.cjs @@ -2,16 +2,34 @@ const fs = require("node:fs"); const path = require("node:path"); const { spawnSync } = require("node:child_process"); +const cargoVersion = spawnSync("cargo", ["--version"], { encoding: "utf8" }); +if (cargoVersion.error) throw cargoVersion.error; +if (cargoVersion.status !== 0) process.exit(cargoVersion.status ?? 1); +const version = /^cargo (\d+)\.(\d+)\./.exec(cargoVersion.stdout); +if (!version) + throw new Error(`Unrecognized Cargo version: ${cargoVersion.stdout}`); +const layout = + Number(version[1]) > 1 || Number(version[2]) >= 100 ? "unit" : "legacy"; + const workspace = fs.mkdtempSync( path.join(process.env.RUNNER_TEMP, "cargo workspace "), ); fs.mkdirSync(path.join(workspace, "src")); fs.writeFileSync( path.join(workspace, "Cargo.toml"), - '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n\n[profile.profiling]\ninherits = "dev"\n', + '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n\n[dependencies]\ncached-dependency = { path = "dependency" }\n\n[profile.profiling]\ninherits = "dev"\n', ); fs.writeFileSync( path.join(workspace, "src", "lib.rs"), + "pub fn example() { cached_dependency::example(); }\n", +); +fs.mkdirSync(path.join(workspace, "dependency", "src"), { recursive: true }); +fs.writeFileSync( + path.join(workspace, "dependency", "Cargo.toml"), + '[package]\nname = "cached-dependency"\nversion = "0.1.0"\nedition = "2024"\n', +); +fs.writeFileSync( + path.join(workspace, "dependency", "src", "lib.rs"), "pub fn example() {}\n", ); // Pruning must skip discovery of the caller's Python project and uv configuration. @@ -27,19 +45,31 @@ const result = spawnSync("cargo", ["generate-lockfile", "--offline"], { if (result.error) throw result.error; if (result.status !== 0) process.exit(result.status ?? 1); -for (const profile of ["debug", "profiling"]) { - const stale = path.join( - workspace, - "target", - profile, - ".fingerprint", - "workspace-cache-fixture-1111111111111111", - ); - fs.mkdirSync(stale, { recursive: true }); - fs.writeFileSync(path.join(stale, "lib-workspace_cache_fixture.json"), "{}"); - const fingerprint = path.join(stale, "lib-workspace_cache_fixture"); +function createStaleArtifact(target, name, hash) { + const unit = path.join(target, "build", name, hash); + const directory = + layout === "unit" + ? path.join(unit, "fingerprint") + : path.join(target, ".fingerprint", `${name}-${hash}`); + const crate = name.replaceAll("-", "_"); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, `lib-${crate}.json`), "{}"); + const fingerprint = path.join(directory, `lib-${crate}`); fs.writeFileSync(fingerprint, "stale"); fs.utimesSync(fingerprint, 1, 1); + const artifacts = + layout === "unit" ? path.join(unit, "out") : path.join(target, "deps"); + fs.mkdirSync(artifacts, { recursive: true }); + fs.writeFileSync(path.join(artifacts, `lib${crate}-${hash}.rlib`), "stale"); +} + +for (const profile of ["debug", "profiling"]) { + const target = path.join(workspace, "target", profile); + createStaleArtifact(target, "workspace-cache-fixture", "1111111111111111"); + createStaleArtifact(target, "cached-dependency", "2222222222222222"); } fs.appendFileSync(process.env.GITHUB_OUTPUT, `workspace=${workspace}\n`); -fs.appendFileSync(process.env.GITHUB_STATE, `workspace=${workspace}\n`); +fs.appendFileSync( + process.env.GITHUB_STATE, + `workspace=${workspace}\nlayout=${layout}\n`, +); From 4c6687e472ab93efda2a52aa7bd58e82bbdd8ca7 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 13:46:39 -0500 Subject: [PATCH 02/10] Seed smoke artifacts after cache restoration --- .../workflows/test-rust-workspace-cache.yml | 3 ++ rust-workspace-cache/tests/smoke/action.yml | 2 ++ rust-workspace-cache/tests/smoke/seed.cjs | 28 +++++++++++++++++++ rust-workspace-cache/tests/smoke/setup.cjs | 28 +++---------------- 4 files changed, 37 insertions(+), 24 deletions(-) create mode 100644 rust-workspace-cache/tests/smoke/seed.cjs diff --git a/.github/workflows/test-rust-workspace-cache.yml b/.github/workflows/test-rust-workspace-cache.yml index 06ff0b6..49714c8 100644 --- a/.github/workflows/test-rust-workspace-cache.yml +++ b/.github/workflows/test-rust-workspace-cache.yml @@ -66,6 +66,9 @@ jobs: prefix-key: test-rust-workspace-cache key: ${{ matrix.rust }}-${{ github.run_id }}-${{ github.run_attempt }} + - name: Seed stale cached artifacts + run: node rust-workspace-cache/tests/smoke/seed.cjs "${{ steps.fixture.outputs.workspace }}" "${{ steps.fixture.outputs.layout }}" + - name: Build workspace working-directory: ${{ steps.fixture.outputs.workspace }} run: | diff --git a/rust-workspace-cache/tests/smoke/action.yml b/rust-workspace-cache/tests/smoke/action.yml index e852bd8..19a7b0f 100644 --- a/rust-workspace-cache/tests/smoke/action.yml +++ b/rust-workspace-cache/tests/smoke/action.yml @@ -3,6 +3,8 @@ description: Create a Cargo workspace and check its artifacts after the cache po outputs: workspace: description: Absolute path of the temporary Cargo workspace. + layout: + description: Cargo artifact layout used by the selected toolchain. runs: using: node24 main: setup.cjs diff --git a/rust-workspace-cache/tests/smoke/seed.cjs b/rust-workspace-cache/tests/smoke/seed.cjs new file mode 100644 index 0000000..4dfb2d0 --- /dev/null +++ b/rust-workspace-cache/tests/smoke/seed.cjs @@ -0,0 +1,28 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +const [workspace, layout] = process.argv.slice(2); + +function createStaleArtifact(target, name, hash) { + const unit = path.join(target, "build", name, hash); + const directory = + layout === "unit" + ? path.join(unit, "fingerprint") + : path.join(target, ".fingerprint", `${name}-${hash}`); + const crate = name.replaceAll("-", "_"); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, `lib-${crate}.json`), "{}"); + const fingerprint = path.join(directory, `lib-${crate}`); + fs.writeFileSync(fingerprint, "stale"); + fs.utimesSync(fingerprint, 1, 1); + const artifacts = + layout === "unit" ? path.join(unit, "out") : path.join(target, "deps"); + fs.mkdirSync(artifacts, { recursive: true }); + fs.writeFileSync(path.join(artifacts, `lib${crate}-${hash}.rlib`), "stale"); +} + +for (const profile of ["debug", "profiling"]) { + const target = path.join(workspace, "target", profile); + createStaleArtifact(target, "workspace-cache-fixture", "1111111111111111"); + createStaleArtifact(target, "cached-dependency", "2222222222222222"); +} diff --git a/rust-workspace-cache/tests/smoke/setup.cjs b/rust-workspace-cache/tests/smoke/setup.cjs index 7f732d1..f312da2 100644 --- a/rust-workspace-cache/tests/smoke/setup.cjs +++ b/rust-workspace-cache/tests/smoke/setup.cjs @@ -45,30 +45,10 @@ const result = spawnSync("cargo", ["generate-lockfile", "--offline"], { if (result.error) throw result.error; if (result.status !== 0) process.exit(result.status ?? 1); -function createStaleArtifact(target, name, hash) { - const unit = path.join(target, "build", name, hash); - const directory = - layout === "unit" - ? path.join(unit, "fingerprint") - : path.join(target, ".fingerprint", `${name}-${hash}`); - const crate = name.replaceAll("-", "_"); - fs.mkdirSync(directory, { recursive: true }); - fs.writeFileSync(path.join(directory, `lib-${crate}.json`), "{}"); - const fingerprint = path.join(directory, `lib-${crate}`); - fs.writeFileSync(fingerprint, "stale"); - fs.utimesSync(fingerprint, 1, 1); - const artifacts = - layout === "unit" ? path.join(unit, "out") : path.join(target, "deps"); - fs.mkdirSync(artifacts, { recursive: true }); - fs.writeFileSync(path.join(artifacts, `lib${crate}-${hash}.rlib`), "stale"); -} - -for (const profile of ["debug", "profiling"]) { - const target = path.join(workspace, "target", profile); - createStaleArtifact(target, "workspace-cache-fixture", "1111111111111111"); - createStaleArtifact(target, "cached-dependency", "2222222222222222"); -} -fs.appendFileSync(process.env.GITHUB_OUTPUT, `workspace=${workspace}\n`); +fs.appendFileSync( + process.env.GITHUB_OUTPUT, + `workspace=${workspace}\nlayout=${layout}\n`, +); fs.appendFileSync( process.env.GITHUB_STATE, `workspace=${workspace}\nlayout=${layout}\n`, From 31232b75387b1e10911029386557296a716174a1 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 13:48:53 -0500 Subject: [PATCH 03/10] Check built dependency artifacts in cache smoke test --- rust-workspace-cache/tests/smoke/check.cjs | 21 +++++++++++---------- rust-workspace-cache/tests/smoke/seed.cjs | 1 - 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/rust-workspace-cache/tests/smoke/check.cjs b/rust-workspace-cache/tests/smoke/check.cjs index d9fed1e..0097300 100644 --- a/rust-workspace-cache/tests/smoke/check.cjs +++ b/rust-workspace-cache/tests/smoke/check.cjs @@ -18,15 +18,16 @@ for (const profile of ["debug", "profiling"]) { ), "The active workspace unit must remain cached", ); + const dependency = path.join(target, "build", "cached-dependency"); assert.ok( - fs.existsSync( - path.join( - target, - "build", - "cached-dependency", - "2222222222222222", - "out", - "libcached_dependency-2222222222222222.rlib", + fs.readdirSync(dependency).some((entry) => + fs.existsSync( + path.join( + dependency, + entry, + "out", + `libcached_dependency-${entry}.rlib`, + ), ), ), "Cached dependency artifacts must remain", @@ -56,8 +57,8 @@ for (const profile of ["debug", "profiling"]) { "The active workspace fingerprint must remain cached", ); assert.ok( - fs.existsSync( - path.join(target, "deps", "libcached_dependency-2222222222222222.rlib"), + fs.readdirSync(path.join(target, "deps")).some((entry) => + /^libcached_dependency-[0-9a-f]{16}\.rlib$/.test(entry), ), "Cached dependency artifacts must remain", ); diff --git a/rust-workspace-cache/tests/smoke/seed.cjs b/rust-workspace-cache/tests/smoke/seed.cjs index 4dfb2d0..7453f3c 100644 --- a/rust-workspace-cache/tests/smoke/seed.cjs +++ b/rust-workspace-cache/tests/smoke/seed.cjs @@ -24,5 +24,4 @@ function createStaleArtifact(target, name, hash) { for (const profile of ["debug", "profiling"]) { const target = path.join(workspace, "target", profile); createStaleArtifact(target, "workspace-cache-fixture", "1111111111111111"); - createStaleArtifact(target, "cached-dependency", "2222222222222222"); } From f1e68b42d23cd46c4cb146ef530f3910d0bb85c1 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 13:50:50 -0500 Subject: [PATCH 04/10] Keep smoke dependency outside workspace root --- rust-workspace-cache/tests/smoke/setup.cjs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rust-workspace-cache/tests/smoke/setup.cjs b/rust-workspace-cache/tests/smoke/setup.cjs index f312da2..2d6a647 100644 --- a/rust-workspace-cache/tests/smoke/setup.cjs +++ b/rust-workspace-cache/tests/smoke/setup.cjs @@ -11,25 +11,28 @@ if (!version) const layout = Number(version[1]) > 1 || Number(version[2]) >= 100 ? "unit" : "legacy"; -const workspace = fs.mkdtempSync( +const directory = fs.mkdtempSync( path.join(process.env.RUNNER_TEMP, "cargo workspace "), ); +const workspace = path.join(directory, "workspace"); +const dependency = path.join(directory, "dependency"); +fs.mkdirSync(workspace); fs.mkdirSync(path.join(workspace, "src")); fs.writeFileSync( path.join(workspace, "Cargo.toml"), - '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n\n[dependencies]\ncached-dependency = { path = "dependency" }\n\n[profile.profiling]\ninherits = "dev"\n', + '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n\n[dependencies]\ncached-dependency = { path = "../dependency" }\n\n[profile.profiling]\ninherits = "dev"\n', ); fs.writeFileSync( path.join(workspace, "src", "lib.rs"), "pub fn example() { cached_dependency::example(); }\n", ); -fs.mkdirSync(path.join(workspace, "dependency", "src"), { recursive: true }); +fs.mkdirSync(path.join(dependency, "src"), { recursive: true }); fs.writeFileSync( - path.join(workspace, "dependency", "Cargo.toml"), + path.join(dependency, "Cargo.toml"), '[package]\nname = "cached-dependency"\nversion = "0.1.0"\nedition = "2024"\n', ); fs.writeFileSync( - path.join(workspace, "dependency", "src", "lib.rs"), + path.join(dependency, "src", "lib.rs"), "pub fn example() {}\n", ); // Pruning must skip discovery of the caller's Python project and uv configuration. From ff917ed44667bee85365b61660c52ee19e2971ff Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 13:54:16 -0500 Subject: [PATCH 05/10] Protect cache deletion boundaries and ambiguous package names --- rust-workspace-cache/inputs/inputs.py | 62 ++++++++++--- .../inputs/lifecycle.test.cjs | 7 ++ rust-workspace-cache/inputs/test_inputs.py | 86 +++++++++++++++++++ rust-workspace-cache/prune/lifecycle.test.cjs | 5 ++ rust-workspace-cache/prune/main.cjs | 2 +- rust-workspace-cache/prune/post.cjs | 6 +- rust-workspace-cache/prune/prune.py | 31 ++++++- rust-workspace-cache/prune/test_prune.py | 47 +++++++++- rust-workspace-cache/runtime.cjs | 15 +++- 9 files changed, 245 insertions(+), 16 deletions(-) diff --git a/rust-workspace-cache/inputs/inputs.py b/rust-workspace-cache/inputs/inputs.py index 786665a..11b36d1 100644 --- a/rust-workspace-cache/inputs/inputs.py +++ b/rust-workspace-cache/inputs/inputs.py @@ -145,12 +145,34 @@ def restorable_directories(snapshot: dict) -> set[str]: def invalidate_units( - profiles: list[Path], packages: set[str], *, build_scripts_only: bool + profiles: list[Path], + packages: set[str], + *, + build_scripts_only: bool, + target: Path | None = None, ) -> int: + # Cargo groups units by package name, including multiple versions and sources. + # Invalidate every matching unit so changed workspace inputs cannot reuse a stale + # build script, even if a same-name dependency must also be rebuilt. removed = 0 for profile in profiles: + boundary = target or profile + try: + relative = profile.relative_to(boundary) + except ValueError as error: + raise ValueError(f"Cargo profile is outside the target directory: {profile}") from error + component = boundary + for part in ("", *relative.parts): + component = component / part + if component.is_symlink(): + raise ValueError(f"Cargo profile has a symlinked path component: {profile}") fingerprints = profile / ".fingerprint" - directories = fingerprints.iterdir() if fingerprints.is_dir() else () + build = profile / "build" + directories = ( + fingerprints.iterdir() + if fingerprints.is_dir() and not fingerprints.is_symlink() + else () + ) for directory in directories: match = FINGERPRINT.fullmatch(directory.name) if ( @@ -160,16 +182,18 @@ def invalidate_units( and not directory.is_symlink() and (not build_scripts_only or any(directory.glob("run-build-script-*"))) ): - output = profile / "build" / directory.name - if output.is_symlink(): - output.unlink() - elif output.is_dir(): - shutil.rmtree(output) + output = build / directory.name + if build.is_dir() and not build.is_symlink(): + if output.is_symlink(): + output.unlink() + elif output.is_dir() and not any( + UNIT_HASH.fullmatch(entry.name) for entry in output.iterdir() + ): + shutil.rmtree(output) shutil.rmtree(directory) removed += 1 # Cargo 1.100 stores each unit's fingerprint and output together. - build = profile / "build" if not build.is_dir() or build.is_symlink(): continue for package in packages: @@ -219,14 +243,22 @@ def restore( } # Changed tracked inputs or directory listings must rerun their build scripts # even with preserved mtimes or a clock behind the cached build's timestamp. - invalidated = invalidate_units(profiles, changed_packages, build_scripts_only=True) + invalidated = invalidate_units( + profiles, + changed_packages, + build_scripts_only=True, + target=target or root / "target", + ) invalidated_workspace = 0 if ( current["source_layout"] is None or current["source_layout"] != saved.get("source_layout") ): invalidated_workspace = invalidate_units( - profiles, set(current["workspace"]), build_scripts_only=False + profiles, + set(current["workspace"]), + build_scripts_only=False, + target=target or root / "target", ) for group, names in matching.items(): for name in names: @@ -294,6 +326,11 @@ def main() -> None: parser.add_argument("--extra-target-directory", type=Path) args = parser.parse_args() start = time.monotonic() + if args.target_directory is not None: + target = args.target_directory.absolute() + for profile in args.profiles: + if not profile.absolute().is_relative_to(target): + parser.error(f"Cargo profile is outside the target directory: {profile}") if args.operation == "save": data = scan(Path.cwd(), args.target_directory) args.manifest.parent.mkdir(parents=True, exist_ok=True) @@ -316,7 +353,10 @@ def main() -> None: ): packages = {package["name"] for package in workspace_packages(Path.cwd())} result["invalidated_workspace_units"] = invalidate_units( - args.profiles, packages, build_scripts_only=False + args.profiles, + packages, + build_scripts_only=False, + target=args.target_directory or Path.cwd() / "target", ) if extra := args.extra_target_directory: staged = args.manifest.with_name(f"target-{extra.name}") diff --git a/rust-workspace-cache/inputs/lifecycle.test.cjs b/rust-workspace-cache/inputs/lifecycle.test.cjs index caac7d2..cb1aea3 100644 --- a/rust-workspace-cache/inputs/lifecycle.test.cjs +++ b/rust-workspace-cache/inputs/lifecycle.test.cjs @@ -165,6 +165,13 @@ test("rejects extra directories outside the target or used by Cargo", (context) } }); +test("rejects Cargo profiles outside the target directory", () => { + const target = path.resolve("target"); + for (const profile of [".", "../outside", path.resolve("outside")]) { + assert.throws(() => profileDirectories(target, profile), /within the target/); + } +}); + test("does not save metadata when restoration fails", (context) => { const result = lifecycle(context, true, 7); assert.equal(result.status, 7); diff --git a/rust-workspace-cache/inputs/test_inputs.py b/rust-workspace-cache/inputs/test_inputs.py index f3406b4..7846d43 100644 --- a/rust-workspace-cache/inputs/test_inputs.py +++ b/rust-workspace-cache/inputs/test_inputs.py @@ -35,6 +35,92 @@ def test_restore_replaces_stale_files_with_the_saved_directory(self) -> None: self.assertFalse((live / "stale").exists()) self.assertFalse(staged.exists()) + def test_same_name_build_scripts_are_conservatively_invalidated(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + profile = Path(temporary) / "debug" + package = profile / "build" / "example" + units = [package / name for name in ("0123456789abcdef", "fedcba9876543210")] + for unit in units: + (unit / "fingerprint").mkdir(parents=True) + (unit / "fingerprint" / "run-build-script-build").touch() + (unit / "out").mkdir() + (unit / "out" / "generated").touch() + + self.assertEqual( + inputs.invalidate_units([profile], {"example"}, build_scripts_only=True), + 2, + ) + self.assertTrue(all(not unit.exists() for unit in units)) + + def test_legacy_invalidation_preserves_a_modern_package_directory(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + profile = Path(temporary) / "debug" + name = "fixture-0123456789abcdef" + legacy = profile / ".fingerprint" / name + legacy.mkdir(parents=True) + (legacy / "run-build-script-build").touch() + modern = profile / "build" / name / "fedcba9876543210" + (modern / "fingerprint").mkdir(parents=True) + (modern / "out").mkdir() + (modern / "out" / "dependency.rlib").touch() + + self.assertEqual( + inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), + 1, + ) + self.assertFalse(legacy.exists()) + self.assertTrue((modern / "out" / "dependency.rlib").exists()) + + def test_invalidation_does_not_follow_a_symlinked_profile_ancestor(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + outside = root / "outside" + unit = outside / "debug" / "build" / "fixture" / "0123456789abcdef" + (unit / "fingerprint").mkdir(parents=True) + (unit / "fingerprint" / "run-build-script-build").touch() + target = root / "target" + try: + target.symlink_to(outside, target_is_directory=True) + except OSError as error: + self.skipTest(str(error)) + + with self.assertRaisesRegex(ValueError, "symlinked path component"): + inputs.invalidate_units( + [target / "debug"], {"fixture"}, build_scripts_only=True, target=target + ) + self.assertTrue(unit.exists()) + + def test_legacy_invalidation_does_not_follow_symlinked_roots(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + profile = root / "debug" + profile.mkdir() + outside = root / "outside" + unit = outside / "fixture-0123456789abcdef" + unit.mkdir(parents=True) + (unit / "run-build-script-build").touch() + try: + (profile / ".fingerprint").symlink_to(outside, target_is_directory=True) + except OSError as error: + self.skipTest(str(error)) + + self.assertEqual( + inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), + 0, + ) + self.assertTrue(unit.exists()) + + (profile / ".fingerprint").unlink() + fingerprint = profile / ".fingerprint" / unit.name + fingerprint.mkdir(parents=True) + (fingerprint / "run-build-script-build").touch() + (profile / "build").symlink_to(outside, target_is_directory=True) + self.assertEqual( + inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), + 1, + ) + self.assertTrue(unit.exists()) + def test_new_layout_invalidation_removes_only_matching_units(self) -> None: with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) diff --git a/rust-workspace-cache/prune/lifecycle.test.cjs b/rust-workspace-cache/prune/lifecycle.test.cjs index 1e65b51..a2018a7 100644 --- a/rust-workspace-cache/prune/lifecycle.test.cjs +++ b/rust-workspace-cache/prune/lifecycle.test.cjs @@ -48,6 +48,7 @@ test("records an isolated marker and the external workspace for pruning", (conte assert.equal(result.environment, "CARGO_UNSTABLE_MTIME_ON_USE=true\n"); const state = JSON.parse(result.state.slice("cache=".length)); assert.equal(state.workspace, result.workspace); + assert.equal(state.target, path.join(result.workspace, "build-output")); assert.deepEqual(state.profiles, [ path.join(result.workspace, "build-output", "fast-build-nightly"), ]); @@ -137,6 +138,8 @@ test("prefers uv without discovering the caller's Python project or configuratio path.join(__dirname, "prune.py"), ...result.state.profiles, result.state.marker, + "--target-directory", + result.state.target, ], options: { cwd: result.state.workspace, stdio: "inherit" }, }, @@ -162,6 +165,8 @@ for (const [platform, python] of [ path.join(__dirname, "prune.py"), ...result.state.profiles, result.state.marker, + "--target-directory", + result.state.target, ]); assert.deepEqual(result.calls[1].options, { cwd: result.state.workspace, diff --git a/rust-workspace-cache/prune/main.cjs b/rust-workspace-cache/prune/main.cjs index 021f711..6a2d920 100644 --- a/rust-workspace-cache/prune/main.cjs +++ b/rust-workspace-cache/prune/main.cjs @@ -19,6 +19,6 @@ if (enabled) { fs.writeFileSync(marker, ""); fs.appendFileSync( process.env.GITHUB_STATE, - `cache=${JSON.stringify({ workspace, profiles, marker })}\n`, + `cache=${JSON.stringify({ workspace, target, profiles, marker })}\n`, ); } diff --git a/rust-workspace-cache/prune/post.cjs b/rust-workspace-cache/prune/post.cjs index b369cc2..8b96aa0 100644 --- a/rust-workspace-cache/prune/post.cjs +++ b/rust-workspace-cache/prune/post.cjs @@ -3,13 +3,17 @@ const path = require("node:path"); const { runPython } = require("../runtime.cjs"); if (process.env.STATE_cache) { - const { workspace, profiles, marker } = JSON.parse(process.env.STATE_cache); + const { workspace, target, profiles, marker } = JSON.parse( + process.env.STATE_cache, + ); try { process.exitCode = runPython( workspace, "prune/prune.py", ...profiles, marker, + "--target-directory", + target, ); } finally { fs.rmSync(path.dirname(marker), { recursive: true, force: true }); diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index c6bb01e..192e8c0 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -7,6 +7,8 @@ import re import shutil import subprocess +import tomllib +from collections import Counter from pathlib import Path FINGERPRINT_DIRECTORY = re.compile(r"^(?P.+)-(?P[0-9a-f]{16})$") @@ -23,9 +25,16 @@ def workspace_packages(workspace: Path) -> set[str]: ) ) members = set(metadata["workspace_members"]) - return { + names = { package["name"] for package in metadata["packages"] if package["id"] in members } + # Cargo places different versions or sources of a package under the same + # package name. A unit hash alone cannot distinguish a workspace member from + # a dependency with that name, so retain both when the lockfile is ambiguous. + with (Path(metadata["workspace_root"]) / "Cargo.lock").open("rb") as lockfile: + locked = tomllib.load(lockfile) + counts = Counter(package["name"] for package in locked.get("package", ())) + return {name for name in names if counts[name] == 1} def fingerprint_last_used(directory: Path) -> int | None: @@ -39,6 +48,24 @@ def fingerprint_last_used(directory: Path) -> int | None: return max(timestamps, default=None) +def validate_profile(target: Path, profile: Path) -> None: + try: + relative = profile.relative_to(target) + except ValueError: + raise ValueError( + f"Cargo profile {profile} is outside target directory {target}" + ) from None + if not relative.parts: + raise ValueError("Cargo profile must be below the target directory") + current = target + for part in relative.parts: + if current.is_symlink(): + raise ValueError(f"Refusing to prune symlinked Cargo path: {current}") + current /= part + if current.is_symlink(): + raise ValueError(f"Refusing to prune symlinked Cargo path: {current}") + + def prune_workspace_fingerprints( profile: Path, marker_timestamp: int, packages: set[str] ) -> tuple[int, int, int]: @@ -150,11 +177,13 @@ def main() -> None: "profiles", type=Path, nargs="+", help="Cargo profile directories" ) parser.add_argument("marker", type=Path, help="Marker created before the build") + parser.add_argument("--target-directory", type=Path, required=True) args = parser.parse_args() marker_timestamp = args.marker.stat().st_mtime_ns packages = workspace_packages(Path.cwd()) for profile in args.profiles: + validate_profile(args.target_directory, profile) stale, removed, active = prune_workspace_fingerprints( profile, marker_timestamp, packages ) diff --git a/rust-workspace-cache/prune/test_prune.py b/rust-workspace-cache/prune/test_prune.py index 686f5ac..78e3c92 100644 --- a/rust-workspace-cache/prune/test_prune.py +++ b/rust-workspace-cache/prune/test_prune.py @@ -1,12 +1,14 @@ +import json import os import sys import tempfile import unittest from pathlib import Path +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).parent)) -from prune import prune_workspace_fingerprints +from prune import prune_workspace_fingerprints, validate_profile, workspace_packages OLD = 1_000_000_000 MARKER = 2_000_000_000 @@ -43,6 +45,22 @@ def unit(self, package: str, fingerprint: str, timestamp: int) -> Path: (unit / "out" / "artifact").write_text("artifact") return unit + @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") + def test_rejects_symlinked_target_and_nested_profile_components(self) -> None: + target = self.profile / "target" + target.mkdir() + outside = self.profile / "outside" + outside.mkdir() + (target / "triple").symlink_to(outside, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlinked Cargo path"): + validate_profile(target, target / "triple" / "debug") + linked_target = self.profile / "linked-target" + linked_target.symlink_to(target, target_is_directory=True) + with self.assertRaisesRegex(ValueError, "symlinked Cargo path"): + validate_profile(linked_target, linked_target / "debug") + with self.assertRaisesRegex(ValueError, "outside target directory"): + validate_profile(target, outside / "debug") + def test_prunes_cargo_1_100_workspace_units(self) -> None: (self.profile / ".fingerprint").rmdir() old = self.unit("example", "1111111111111111", OLD) @@ -76,6 +94,33 @@ def test_retains_colliding_dependency_and_active_hashes_across_layouts(self) -> for retained in (old, dependency, other_old, active): self.assertTrue(retained.is_dir()) + def test_retains_dependencies_with_the_same_name_as_a_workspace_package(self) -> None: + lockfile = self.profile / "Cargo.lock" + lockfile.write_text( + 'version = 4\n' + '[[package]]\nname = "example"\nversion = "0.1.0"\n' + '[[package]]\nname = "example"\nversion = "0.2.0"\n' + '[[package]]\nname = "unique"\nversion = "0.1.0"\n' + ) + metadata = { + "workspace_root": str(self.profile), + "workspace_members": ["example@0.2.0", "unique@0.1.0"], + "packages": [ + {"name": "example", "id": "example@0.2.0"}, + {"name": "unique", "id": "unique@0.1.0"}, + ], + } + with patch("prune.subprocess.check_output", return_value=json.dumps(metadata)): + packages = workspace_packages(self.profile) + self.assertEqual(packages, {"unique"}) + + dependency = self.unit("example", "1111111111111111", OLD) + self.unit("example", "2222222222222222", CURRENT) + self.assertEqual( + prune_workspace_fingerprints(self.profile, MARKER, packages), (0, 0, 1) + ) + self.assertTrue(dependency.is_dir()) + def test_retains_dependency_package_named_like_a_legacy_build_unit(self) -> None: self.unit("example", "1111111111111111", OLD) self.unit("example", "2222222222222222", CURRENT) diff --git a/rust-workspace-cache/runtime.cjs b/rust-workspace-cache/runtime.cjs index 86a9362..80adb9a 100644 --- a/rust-workspace-cache/runtime.cjs +++ b/rust-workspace-cache/runtime.cjs @@ -6,7 +6,20 @@ function profileDirectories(target, value) { .split(/\r?\n/) .map((profile) => profile.trim()) .filter(Boolean) - .map((profile) => path.resolve(target, profile)); + .map((profile) => { + const directory = path.resolve(target, profile); + const relative = path.relative(target, directory); + if ( + !relative || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) + throw new Error( + "Cargo profiles must be directories within the target directory", + ); + return directory; + }); if (profiles.length === 0) throw new Error("At least one build profile is required"); return profiles; From 1dde39563633931e7014f5c2571f1b1d5131bf8b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 14:01:41 -0500 Subject: [PATCH 06/10] Simplify stale-unit tracking and focus cache tests --- rust-workspace-cache/inputs/inputs.py | 5 +- rust-workspace-cache/inputs/test_inputs.py | 53 ++++++++++------------ rust-workspace-cache/prune/prune.py | 10 ++-- rust-workspace-cache/prune/test_prune.py | 8 +--- 4 files changed, 34 insertions(+), 42 deletions(-) diff --git a/rust-workspace-cache/inputs/inputs.py b/rust-workspace-cache/inputs/inputs.py index 11b36d1..e012524 100644 --- a/rust-workspace-cache/inputs/inputs.py +++ b/rust-workspace-cache/inputs/inputs.py @@ -243,11 +243,12 @@ def restore( } # Changed tracked inputs or directory listings must rerun their build scripts # even with preserved mtimes or a clock behind the cached build's timestamp. + target = target or root / "target" invalidated = invalidate_units( profiles, changed_packages, build_scripts_only=True, - target=target or root / "target", + target=target, ) invalidated_workspace = 0 if ( @@ -258,7 +259,7 @@ def restore( profiles, set(current["workspace"]), build_scripts_only=False, - target=target or root / "target", + target=target, ) for group, names in matching.items(): for name in names: diff --git a/rust-workspace-cache/inputs/test_inputs.py b/rust-workspace-cache/inputs/test_inputs.py index 7846d43..2ed3986 100644 --- a/rust-workspace-cache/inputs/test_inputs.py +++ b/rust-workspace-cache/inputs/test_inputs.py @@ -35,23 +35,6 @@ def test_restore_replaces_stale_files_with_the_saved_directory(self) -> None: self.assertFalse((live / "stale").exists()) self.assertFalse(staged.exists()) - def test_same_name_build_scripts_are_conservatively_invalidated(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - profile = Path(temporary) / "debug" - package = profile / "build" / "example" - units = [package / name for name in ("0123456789abcdef", "fedcba9876543210")] - for unit in units: - (unit / "fingerprint").mkdir(parents=True) - (unit / "fingerprint" / "run-build-script-build").touch() - (unit / "out").mkdir() - (unit / "out" / "generated").touch() - - self.assertEqual( - inputs.invalidate_units([profile], {"example"}, build_scripts_only=True), - 2, - ) - self.assertTrue(all(not unit.exists() for unit in units)) - def test_legacy_invalidation_preserves_a_modern_package_directory(self) -> None: with tempfile.TemporaryDirectory() as temporary: profile = Path(temporary) / "debug" @@ -127,10 +110,12 @@ def test_new_layout_invalidation_removes_only_matching_units(self) -> None: profile = root / "debug" package = profile / "build" / "fixture" script = package / "0123456789abcdef" + same_name_script = package / "1111111111111111" binary = package / "fedcba9876543210" other = profile / "build" / "other" / "0123456789abcdef" for unit, marker in ( (script, "run-build-script-build"), + (same_name_script, "run-build-script-build"), (binary, "bin-fixture"), (other, "run-build-script-build"), ): @@ -140,30 +125,40 @@ def test_new_layout_invalidation_removes_only_matching_units(self) -> None: (unit / "out" / "generated").touch() (script / "run").mkdir() (script / "run" / "stdout").touch() - outside = root / "outside" - (outside / "fingerprint").mkdir(parents=True) - (outside / "fingerprint" / "run-build-script-build").touch() - try: - (package / "1111111111111111").symlink_to( - outside, target_is_directory=True - ) - except OSError as error: - self.skipTest(str(error)) - self.assertEqual( inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), - 1, + 2, ) self.assertFalse(script.exists()) + self.assertFalse(same_name_script.exists()) self.assertTrue(binary.exists()) self.assertTrue(other.exists()) - self.assertTrue(outside.exists()) self.assertEqual( inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=False), 1, ) self.assertFalse(binary.exists()) self.assertTrue(other.exists()) + + def test_new_layout_invalidation_does_not_follow_symlinked_units(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + profile = root / "debug" + package = profile / "build" / "fixture" + package.mkdir(parents=True) + outside = root / "outside" + (outside / "fingerprint").mkdir(parents=True) + (outside / "fingerprint" / "run-build-script-build").touch() + try: + (package / "1111111111111111").symlink_to( + outside, target_is_directory=True + ) + except OSError as error: + self.skipTest(str(error)) + self.assertEqual( + inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), + 0, + ) self.assertTrue(outside.exists()) def test_source_layout_ignores_build_outputs_and_detects_retargeted_links(self) -> None: diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index 192e8c0..fdcc982 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -71,7 +71,7 @@ def prune_workspace_fingerprints( ) -> tuple[int, int, int]: fingerprint_root = profile / ".fingerprint" build_root = profile / "build" - stale_fingerprints_by_package: dict[str, set[str]] = {} + stale_candidates: set[tuple[str, str]] = set() active_fingerprints: set[str] = set() active_packages: set[str] = set() protected_fingerprints: set[str] = set() @@ -93,7 +93,7 @@ def record(package: str, fingerprint: str, directory: Path) -> None: if package in packages: active_packages.add(package) elif package in packages: - stale_fingerprints_by_package.setdefault(package, set()).add(fingerprint) + stale_candidates.add((package, fingerprint)) # Cargo before 1.100 stores all fingerprints in one profile directory. if not fingerprint_root.is_symlink() and fingerprint_root.is_dir(): @@ -126,9 +126,9 @@ def record(package: str, fingerprint: str, directory: Path) -> None: stale_units = { (package, fingerprint) - for package in active_packages - for fingerprint in stale_fingerprints_by_package.get(package, set()) - if fingerprint not in active_fingerprints + for package, fingerprint in stale_candidates + if package in active_packages + and fingerprint not in active_fingerprints and fingerprint not in protected_fingerprints } stale_fingerprints = {fingerprint for _, fingerprint in stale_units} diff --git a/rust-workspace-cache/prune/test_prune.py b/rust-workspace-cache/prune/test_prune.py index 78e3c92..71a6960 100644 --- a/rust-workspace-cache/prune/test_prune.py +++ b/rust-workspace-cache/prune/test_prune.py @@ -147,12 +147,6 @@ def test_does_not_apply_legacy_cleanup_to_cargo_1_100_hashes(self) -> None: self.assertTrue(ambiguous.is_dir()) self.assertTrue(dependency.is_file()) - def test_refuses_to_prune_cargo_1_100_without_active_units(self) -> None: - old = self.unit("example", "1111111111111111", OLD) - with self.assertRaisesRegex(RuntimeError, "No active Cargo fingerprints"): - prune_workspace_fingerprints(self.profile, MARKER, {"example"}) - self.assertTrue(old.is_dir()) - @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") def test_does_not_follow_cargo_1_100_symlink_nodes(self) -> None: self.unit("example", "2222222222222222", CURRENT) @@ -214,9 +208,11 @@ def test_retains_hashes_used_by_another_active_package(self) -> None: def test_refuses_to_prune_without_active_fingerprints(self) -> None: old = self.fingerprint("example", "1111111111111111", OLD) + modern = self.unit("example", "2222222222222222", OLD) with self.assertRaisesRegex(RuntimeError, "No active Cargo fingerprints"): prune_workspace_fingerprints(self.profile, MARKER, {"example"}) self.assertTrue(old.is_dir()) + self.assertTrue(modern.is_dir()) def test_retains_fingerprints_without_usage_metadata(self) -> None: unknown = self.fingerprint("example", "1111111111111111", OLD) From 5be17a5dd236647c842d80ab4b44415d88bb9149 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 16:37:42 -0500 Subject: [PATCH 07/10] Remove Python cache test suites --- .../workflows/test-rust-workspace-cache.yml | 4 +- rust-workspace-cache/inputs/test_inputs.py | 391 ------------------ rust-workspace-cache/prune/test_prune.py | 245 ----------- 3 files changed, 1 insertion(+), 639 deletions(-) delete mode 100644 rust-workspace-cache/inputs/test_inputs.py delete mode 100644 rust-workspace-cache/prune/test_prune.py diff --git a/.github/workflows/test-rust-workspace-cache.yml b/.github/workflows/test-rust-workspace-cache.yml index 49714c8..0b2e197 100644 --- a/.github/workflows/test-rust-workspace-cache.yml +++ b/.github/workflows/test-rust-workspace-cache.yml @@ -42,11 +42,9 @@ jobs: version: "0.13.5" enable-cache: false - - name: Test pruning and lifecycle + - name: Test lifecycle shell: bash run: | - python -m unittest discover -s rust-workspace-cache/prune -p 'test_*.py' - python -m unittest discover -s rust-workspace-cache/inputs -p 'test_*.py' node --test rust-workspace-cache/prune/lifecycle.test.cjs node --test rust-workspace-cache/inputs/lifecycle.test.cjs diff --git a/rust-workspace-cache/inputs/test_inputs.py b/rust-workspace-cache/inputs/test_inputs.py deleted file mode 100644 index 2ed3986..0000000 --- a/rust-workspace-cache/inputs/test_inputs.py +++ /dev/null @@ -1,391 +0,0 @@ -"""Exercise input restoration against Cargo's build-script freshness checks.""" - -from __future__ import annotations - -import json -import os -import subprocess -import sys -import tempfile -import time -import unittest -from pathlib import Path - -import inputs - - -class ToolDirectoryTest(unittest.TestCase): - def test_restore_replaces_stale_files_with_the_saved_directory(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - live = root / "target" / "tool-data" - staged = root / "cache" / "target-tool-data" - inputs.move_tool_directory(staged, live) - self.assertFalse(live.exists()) - - live.mkdir(parents=True) - (live / "graph").write_bytes(b"saved graph") - inputs.move_tool_directory(live, staged) - self.assertFalse(live.exists()) - - live.mkdir() - (live / "stale").touch() - inputs.move_tool_directory(staged, live) - self.assertEqual((live / "graph").read_bytes(), b"saved graph") - self.assertFalse((live / "stale").exists()) - self.assertFalse(staged.exists()) - - def test_legacy_invalidation_preserves_a_modern_package_directory(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - profile = Path(temporary) / "debug" - name = "fixture-0123456789abcdef" - legacy = profile / ".fingerprint" / name - legacy.mkdir(parents=True) - (legacy / "run-build-script-build").touch() - modern = profile / "build" / name / "fedcba9876543210" - (modern / "fingerprint").mkdir(parents=True) - (modern / "out").mkdir() - (modern / "out" / "dependency.rlib").touch() - - self.assertEqual( - inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), - 1, - ) - self.assertFalse(legacy.exists()) - self.assertTrue((modern / "out" / "dependency.rlib").exists()) - - def test_invalidation_does_not_follow_a_symlinked_profile_ancestor(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - outside = root / "outside" - unit = outside / "debug" / "build" / "fixture" / "0123456789abcdef" - (unit / "fingerprint").mkdir(parents=True) - (unit / "fingerprint" / "run-build-script-build").touch() - target = root / "target" - try: - target.symlink_to(outside, target_is_directory=True) - except OSError as error: - self.skipTest(str(error)) - - with self.assertRaisesRegex(ValueError, "symlinked path component"): - inputs.invalidate_units( - [target / "debug"], {"fixture"}, build_scripts_only=True, target=target - ) - self.assertTrue(unit.exists()) - - def test_legacy_invalidation_does_not_follow_symlinked_roots(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - profile = root / "debug" - profile.mkdir() - outside = root / "outside" - unit = outside / "fixture-0123456789abcdef" - unit.mkdir(parents=True) - (unit / "run-build-script-build").touch() - try: - (profile / ".fingerprint").symlink_to(outside, target_is_directory=True) - except OSError as error: - self.skipTest(str(error)) - - self.assertEqual( - inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), - 0, - ) - self.assertTrue(unit.exists()) - - (profile / ".fingerprint").unlink() - fingerprint = profile / ".fingerprint" / unit.name - fingerprint.mkdir(parents=True) - (fingerprint / "run-build-script-build").touch() - (profile / "build").symlink_to(outside, target_is_directory=True) - self.assertEqual( - inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), - 1, - ) - self.assertTrue(unit.exists()) - - def test_new_layout_invalidation_removes_only_matching_units(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - profile = root / "debug" - package = profile / "build" / "fixture" - script = package / "0123456789abcdef" - same_name_script = package / "1111111111111111" - binary = package / "fedcba9876543210" - other = profile / "build" / "other" / "0123456789abcdef" - for unit, marker in ( - (script, "run-build-script-build"), - (same_name_script, "run-build-script-build"), - (binary, "bin-fixture"), - (other, "run-build-script-build"), - ): - (unit / "fingerprint").mkdir(parents=True) - (unit / "fingerprint" / marker).touch() - (unit / "out").mkdir() - (unit / "out" / "generated").touch() - (script / "run").mkdir() - (script / "run" / "stdout").touch() - self.assertEqual( - inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), - 2, - ) - self.assertFalse(script.exists()) - self.assertFalse(same_name_script.exists()) - self.assertTrue(binary.exists()) - self.assertTrue(other.exists()) - self.assertEqual( - inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=False), - 1, - ) - self.assertFalse(binary.exists()) - self.assertTrue(other.exists()) - - def test_new_layout_invalidation_does_not_follow_symlinked_units(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - profile = root / "debug" - package = profile / "build" / "fixture" - package.mkdir(parents=True) - outside = root / "outside" - (outside / "fingerprint").mkdir(parents=True) - (outside / "fingerprint" / "run-build-script-build").touch() - try: - (package / "1111111111111111").symlink_to( - outside, target_is_directory=True - ) - except OSError as error: - self.skipTest(str(error)) - self.assertEqual( - inputs.invalidate_units([profile], {"fixture"}, build_scripts_only=True), - 0, - ) - self.assertTrue(outside.exists()) - - def test_source_layout_ignores_build_outputs_and_detects_retargeted_links(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - subprocess.run(["git", "init", "--quiet"], cwd=root, check=True) - target = root / "build [output]" - target.mkdir() - for name in ("a", "b"): - (root / name).write_text("same contents") - link = root / "source.rs" - try: - link.symlink_to("a") - except OSError as error: - self.skipTest(str(error)) - saved = inputs.source_layout(root, target) - (target / "artifact").touch() - (root / "a").write_text("new contents") - self.assertEqual(inputs.source_layout(root, target), saved) - link.unlink() - link.symlink_to("b") - self.assertNotEqual(inputs.source_layout(root, target), saved) - - -class CargoInputsTest(unittest.TestCase): - def setUp(self) -> None: - temporary = tempfile.TemporaryDirectory(prefix="cargo-inputs-") - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name).resolve() - (self.root / "src").mkdir() - (self.root / "inputs").mkdir() - (self.root / "Cargo.toml").write_text( - '[package]\nname = "cache-input-fixture"\nversion = "0.1.0"\nedition = "2021"\n' - ) - (self.root / "src/main.rs").write_text( - 'include!(concat!(env!("OUT_DIR"), "/generated.rs"));\n' - 'fn main() { println!("{}", VALUE); }\n' - ) - (self.root / "build.rs").write_text( - "use std::{env, fs, path::PathBuf};\n" - "fn main() {\n" - ' println!("cargo:rerun-if-changed=inputs");\n' - ' let mut files = fs::read_dir("inputs").unwrap()\n' - " .map(|p| p.unwrap().path()).collect::>();\n" - " files.sort();\n" - " let value = files.into_iter().map(|p| fs::read_to_string(p).unwrap())\n" - " .collect::();\n" - ' fs::write(PathBuf::from(env::var("OUT_DIR").unwrap()).join("generated.rs"),\n' - ' format!("const VALUE: &str = {:?};", value)).unwrap();\n' - "}\n" - ) - self.value = self.root / "inputs/value.txt" - self.value.write_text("original") - (self.root / ".gitignore").write_text("target/\n") - subprocess.run(["git", "init", "--quiet"], cwd=self.root, check=True) - subprocess.run(["git", "add", "."], cwd=self.root, check=True) - self.environment = os.environ | { - "CARGO_UNSTABLE_CHECKSUM_FRESHNESS": "true", - "RUSTC_BOOTSTRAP": "1", - "CARGO_INCREMENTAL": "0", - "CARGO_TERM_COLOR": "never", - "CARGO_TARGET_DIR": str(self.root / "target"), - } - self.build("original", rebuilt=True) - self.saved = inputs.scan(self.root) - - def build(self, expected: str, *, rebuilt: bool) -> None: - result = subprocess.run( - ["cargo", "run", "--offline", "-vv"], - cwd=self.root, - env=self.environment, - text=True, - capture_output=True, - check=True, - ) - self.assertEqual(result.stdout.splitlines()[-1], expected, result.stderr) - self.assertEqual( - "Compiling cache-input-fixture" in result.stderr, rebuilt, result.stderr - ) - - def restore(self) -> dict[str, int]: - return inputs.restore(self.root, self.saved, [self.root / "target/debug"]) - - def test_fresh_checkout_reuses_artifacts(self) -> None: - for group in ["files", "directories"]: - for name in self.saved[group]: - os.utime(self.root / name, None) - result = self.restore() - self.assertGreater(result["restored_files"], 0) - self.assertEqual(result["invalidated_build_scripts"], 0) - self.build("original", rebuilt=False) - - def test_new_module_path_invalidates_a_binary_without_a_build_script(self) -> None: - (self.root / "build.rs").unlink() - (self.root / "src/main.rs").write_text( - 'mod value; fn main() { println!("{}", value::get()); }\n' - ) - nested = self.root / "src/value/mod.rs" - nested.parent.mkdir() - nested.write_text("pub fn get() -> u32 { 42 }\n") - self.build("42", rebuilt=True) - self.saved = inputs.scan(self.root) - self.assertEqual(self.saved["packages"], {}) - self.assertEqual(self.restore()["invalidated_workspace_units"], 0) - self.build("42", rebuilt=False) - - (self.root / "src/value.rs").write_text("pub fn get() -> u32 { 43 }\n") - self.assertGreater(self.restore()["invalidated_workspace_units"], 0) - result = subprocess.run( - ["cargo", "run", "--offline"], - cwd=self.root, - env=self.environment, - text=True, - capture_output=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("E0761", result.stderr) - - def test_changed_bytes_with_preserved_timestamp_rebuild(self) -> None: - old = self.value.stat().st_mtime_ns - self.value.write_text("changed") - os.utime(self.value, ns=(old, old)) - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - staged = subprocess.check_output( - ["git", "show", ":inputs/value.txt"], cwd=self.root, text=True - ) - self.assertEqual(staged, "original") - self.build("changed", rebuilt=True) - - def test_subsequent_checkout_keeps_matching_input_timestamps(self) -> None: - subprocess.run( - [ - "git", - "-c", - "user.name=Fixture", - "-c", - "user.email=fixture@example.invalid", - "commit", - "-qm", - "seed", - ], - cwd=self.root, - check=True, - ) - for name in self.saved["files"]: - os.utime(self.root / name, None) - subprocess.run(["git", "update-index", "--refresh"], cwd=self.root, check=True) - self.restore() - subprocess.run( - ["git", "checkout", "--quiet", "--force", "HEAD"], cwd=self.root, check=True - ) - self.assertEqual( - self.value.stat().st_mtime_ns, - self.saved["files"]["inputs/value.txt"]["mtime_ns"], - ) - self.build("original", rebuilt=False) - - def test_changed_bytes_rebuild_when_cached_output_is_in_the_future(self) -> None: - future = time.time_ns() + 60_000_000_000 - build = self.root / "target/debug/build" - outputs = [*build.glob("*/output"), *build.glob("*/*/run/stdout")] - self.assertTrue(outputs) - for output in outputs: - os.utime(output, ns=(future, future)) - old = self.value.stat().st_mtime_ns - self.value.write_text("changed") - os.utime(self.value, ns=(old, old)) - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - self.build("changed", rebuilt=True) - - def test_new_untracked_input_rebuilds(self) -> None: - (self.root / "inputs/extra.txt").write_text("extra+") - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - self.build("extra+original", rebuilt=True) - - def test_replaced_untracked_input_keeps_directory_timestamp(self) -> None: - extra = self.root / "inputs/extra.txt" - extra.write_text("old+") - self.build("old+original", rebuilt=True) - self.saved = inputs.scan(self.root) - old = extra.stat().st_mtime_ns - extra.unlink() - extra.write_text("new+") - os.utime(extra, ns=(old, old)) - directory = extra.parent - modified = time.time_ns() + 1_000_000_000 - os.utime(directory, ns=(modified, modified)) - modified = directory.stat().st_mtime_ns - self.restore() - self.assertEqual(directory.stat().st_mtime_ns, modified) - self.build("new+original", rebuilt=True) - - def test_deleted_tracked_input_rebuilds(self) -> None: - self.value.unlink() - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - self.build("", rebuilt=True) - - def test_git_metadata_is_not_restored(self) -> None: - head = self.root / ".git/HEAD" - os.utime(head, ns=(1_000_000_000, 1_000_000_000)) - self.restore() - self.assertEqual(head.stat().st_mtime_ns, 1_000_000_000) - - def test_missing_manifest_invalidates_cached_workspace_units(self) -> None: - result = subprocess.run( - [ - sys.executable, - str(Path(inputs.__file__).resolve()), - "restore", - str(self.root / "missing-manifest.json"), - str(self.root / "target/debug"), - ], - cwd=self.root, - env=self.environment, - text=True, - capture_output=True, - check=True, - ) - self.assertGreater(json.loads(result.stdout)["invalidated_workspace_units"], 0) - self.build("original", rebuilt=True) - - def test_rejects_escaping_paths(self) -> None: - with self.assertRaises(ValueError): - inputs.safe_path(self.root, "../outside") - with self.assertRaises(ValueError): - inputs.safe_path(self.root, str(self.root.parent / "outside")) - - -if __name__ == "__main__": - unittest.main() diff --git a/rust-workspace-cache/prune/test_prune.py b/rust-workspace-cache/prune/test_prune.py deleted file mode 100644 index 71a6960..0000000 --- a/rust-workspace-cache/prune/test_prune.py +++ /dev/null @@ -1,245 +0,0 @@ -import json -import os -import sys -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -sys.path.insert(0, str(Path(__file__).parent)) - -from prune import prune_workspace_fingerprints, validate_profile, workspace_packages - -OLD = 1_000_000_000 -MARKER = 2_000_000_000 -CURRENT = 3_000_000_000 - - -class PruneWorkspaceCacheTest(unittest.TestCase): - def setUp(self) -> None: - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.profile = Path(self.temporary.name) - (self.profile / ".fingerprint").mkdir() - (self.profile / "deps").mkdir() - (self.profile / "build").mkdir() - - def fingerprint(self, package: str, fingerprint: str, timestamp: int) -> Path: - directory = self.profile / ".fingerprint" / f"{package}-{fingerprint}" - directory.mkdir() - (directory / "lib-example.json").write_text("{}") - stamp = directory / "lib-example" - stamp.touch() - os.utime(stamp, ns=(timestamp, timestamp)) - return directory - - def unit(self, package: str, fingerprint: str, timestamp: int) -> Path: - unit = self.profile / "build" / package / fingerprint - directory = unit / "fingerprint" - directory.mkdir(parents=True) - (directory / "lib-example.json").write_text("{}") - stamp = directory / "lib-example" - stamp.touch() - os.utime(stamp, ns=(timestamp, timestamp)) - (unit / "out").mkdir() - (unit / "out" / "artifact").write_text("artifact") - return unit - - @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") - def test_rejects_symlinked_target_and_nested_profile_components(self) -> None: - target = self.profile / "target" - target.mkdir() - outside = self.profile / "outside" - outside.mkdir() - (target / "triple").symlink_to(outside, target_is_directory=True) - with self.assertRaisesRegex(ValueError, "symlinked Cargo path"): - validate_profile(target, target / "triple" / "debug") - linked_target = self.profile / "linked-target" - linked_target.symlink_to(target, target_is_directory=True) - with self.assertRaisesRegex(ValueError, "symlinked Cargo path"): - validate_profile(linked_target, linked_target / "debug") - with self.assertRaisesRegex(ValueError, "outside target directory"): - validate_profile(target, outside / "debug") - - def test_prunes_cargo_1_100_workspace_units(self) -> None: - (self.profile / ".fingerprint").rmdir() - old = self.unit("example", "1111111111111111", OLD) - current = self.unit("example", "2222222222222222", CURRENT) - dependency = self.unit("dependency", "3333333333333333", OLD) - unused = self.unit("unused", "4444444444444444", OLD) - unknown = self.unit("example", "5555555555555555", OLD) - (unknown / "fingerprint" / "lib-example").unlink() - malformed = self.profile / "build" / "example" / "not-a-hash" - malformed.mkdir() - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example", "unused"}), - (1, 1, 1), - ) - self.assertFalse(old.exists()) - for retained in (current, dependency, unused, unknown, malformed): - self.assertTrue(retained.is_dir()) - - def test_retains_colliding_dependency_and_active_hashes_across_layouts(self) -> None: - old = self.unit("example", "1111111111111111", OLD) - self.unit("example", "2222222222222222", CURRENT) - dependency = self.fingerprint("dependency", "1111111111111111", OLD) - other_old = self.fingerprint("example", "3333333333333333", OLD) - active = self.unit("dependency", "3333333333333333", CURRENT) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (0, 0, 2), - ) - for retained in (old, dependency, other_old, active): - self.assertTrue(retained.is_dir()) - - def test_retains_dependencies_with_the_same_name_as_a_workspace_package(self) -> None: - lockfile = self.profile / "Cargo.lock" - lockfile.write_text( - 'version = 4\n' - '[[package]]\nname = "example"\nversion = "0.1.0"\n' - '[[package]]\nname = "example"\nversion = "0.2.0"\n' - '[[package]]\nname = "unique"\nversion = "0.1.0"\n' - ) - metadata = { - "workspace_root": str(self.profile), - "workspace_members": ["example@0.2.0", "unique@0.1.0"], - "packages": [ - {"name": "example", "id": "example@0.2.0"}, - {"name": "unique", "id": "unique@0.1.0"}, - ], - } - with patch("prune.subprocess.check_output", return_value=json.dumps(metadata)): - packages = workspace_packages(self.profile) - self.assertEqual(packages, {"unique"}) - - dependency = self.unit("example", "1111111111111111", OLD) - self.unit("example", "2222222222222222", CURRENT) - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, packages), (0, 0, 1) - ) - self.assertTrue(dependency.is_dir()) - - def test_retains_dependency_package_named_like_a_legacy_build_unit(self) -> None: - self.unit("example", "1111111111111111", OLD) - self.unit("example", "2222222222222222", CURRENT) - dependency = self.unit("example-1111111111111111", "3333333333333333", OLD) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (1, 1, 1), - ) - self.assertTrue(dependency.is_dir()) - - def test_does_not_apply_legacy_cleanup_to_cargo_1_100_hashes(self) -> None: - self.unit("example", "1111111111111111", OLD) - self.unit("example", "2222222222222222", CURRENT) - ambiguous = self.profile / "build" / "example-1111111111111111" - ambiguous.mkdir() - dependency = self.profile / "deps" / "libdependency-1111111111111111.rlib" - dependency.touch() - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (1, 1, 1), - ) - self.assertTrue(ambiguous.is_dir()) - self.assertTrue(dependency.is_file()) - - @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") - def test_does_not_follow_cargo_1_100_symlink_nodes(self) -> None: - self.unit("example", "2222222222222222", CURRENT) - target = self.profile / "keep" - target.mkdir() - (target / "sentinel").write_text("keep") - package = self.profile / "build" / "linked-package" - package.symlink_to(target, target_is_directory=True) - unit = self.profile / "build" / "example" / "1111111111111111" - unit.symlink_to(target, target_is_directory=True) - fingerprint = self.profile / "build" / "example" / "3333333333333333" - fingerprint.mkdir() - (fingerprint / "fingerprint").symlink_to(target, target_is_directory=True) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example", "linked-package"}), - (0, 0, 1), - ) - self.assertTrue(package.is_symlink()) - self.assertTrue(unit.is_symlink()) - self.assertTrue((fingerprint / "fingerprint").is_symlink()) - self.assertEqual((target / "sentinel").read_text(), "keep") - - def test_removes_superseded_workspace_artifacts(self) -> None: - old = self.fingerprint("example", "1111111111111111", OLD) - current = self.fingerprint("example", "2222222222222222", CURRENT) - dependency = self.fingerprint("dependency", "3333333333333333", OLD) - unused = self.fingerprint("unused", "4444444444444444", OLD) - library = self.profile / "deps" / "libexample-1111111111111111.rlib" - library.touch() - build = self.profile / "build" / "example-1111111111111111" - build.mkdir() - (build / "output").touch() - active_library = self.profile / "deps" / "libexample-2222222222222222.rlib" - active_library.touch() - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example", "unused"}), - (1, 3, 1), - ) - self.assertFalse(old.exists()) - self.assertFalse(library.exists()) - self.assertFalse(build.exists()) - self.assertTrue(current.is_dir()) - self.assertTrue(active_library.is_file()) - self.assertTrue(dependency.is_dir()) - self.assertTrue(unused.is_dir()) - - def test_retains_hashes_used_by_another_active_package(self) -> None: - old = self.fingerprint("example", "1111111111111111", OLD) - self.fingerprint("example", "2222222222222222", CURRENT) - self.fingerprint("dependency", "1111111111111111", CURRENT) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (0, 0, 2), - ) - self.assertTrue(old.is_dir()) - - def test_refuses_to_prune_without_active_fingerprints(self) -> None: - old = self.fingerprint("example", "1111111111111111", OLD) - modern = self.unit("example", "2222222222222222", OLD) - with self.assertRaisesRegex(RuntimeError, "No active Cargo fingerprints"): - prune_workspace_fingerprints(self.profile, MARKER, {"example"}) - self.assertTrue(old.is_dir()) - self.assertTrue(modern.is_dir()) - - def test_retains_fingerprints_without_usage_metadata(self) -> None: - unknown = self.fingerprint("example", "1111111111111111", OLD) - (unknown / "lib-example").unlink() - self.fingerprint("example", "2222222222222222", MARKER) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (0, 0, 1), - ) - self.assertTrue(unknown.is_dir()) - - @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") - def test_removes_stale_symlinks_without_removing_their_targets(self) -> None: - self.fingerprint("example", "1111111111111111", OLD) - self.fingerprint("example", "2222222222222222", CURRENT) - target = self.profile / "keep" - target.mkdir() - (target / "output").write_text("keep") - link = self.profile / "build" / "example-1111111111111111" - link.symlink_to(target, target_is_directory=True) - - prune_workspace_fingerprints(self.profile, MARKER, {"example"}) - - self.assertFalse(link.is_symlink()) - self.assertEqual((target / "output").read_text(), "keep") - - -if __name__ == "__main__": - unittest.main() From 957e8f91b247f27fbc56e829f3fb9f08c3940f7d Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 16:38:15 -0500 Subject: [PATCH 08/10] Reject Windows reparse points during cache cleanup --- rust-workspace-cache/inputs/inputs.py | 33 ++++++++++++++------- rust-workspace-cache/prune/prune.py | 42 ++++++++++++++++++--------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/rust-workspace-cache/inputs/inputs.py b/rust-workspace-cache/inputs/inputs.py index e012524..25ab473 100644 --- a/rust-workspace-cache/inputs/inputs.py +++ b/rust-workspace-cache/inputs/inputs.py @@ -18,6 +18,17 @@ UNIT_HASH = re.compile(r"^[0-9a-f]{16}$") +def is_redirect(path: Path) -> bool: + """Recognize symlinks and Windows reparse points without following them.""" + try: + info = path.lstat() + except (FileNotFoundError, NotADirectoryError): + return False + return stat.S_ISLNK(info.st_mode) or bool( + getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ) + + def workspace_packages(root: Path) -> list[dict]: metadata = json.loads( subprocess.check_output( @@ -164,13 +175,13 @@ def invalidate_units( component = boundary for part in ("", *relative.parts): component = component / part - if component.is_symlink(): - raise ValueError(f"Cargo profile has a symlinked path component: {profile}") + if is_redirect(component): + raise ValueError(f"Cargo profile has a symlinked path component or reparse point: {profile}") fingerprints = profile / ".fingerprint" build = profile / "build" directories = ( fingerprints.iterdir() - if fingerprints.is_dir() and not fingerprints.is_symlink() + if fingerprints.is_dir() and not is_redirect(fingerprints) else () ) for directory in directories: @@ -179,14 +190,14 @@ def invalidate_units( match and match.group("package") in packages and directory.is_dir() - and not directory.is_symlink() + and not is_redirect(directory) and (not build_scripts_only or any(directory.glob("run-build-script-*"))) ): output = build / directory.name - if build.is_dir() and not build.is_symlink(): + if build.is_dir() and not is_redirect(build): if output.is_symlink(): output.unlink() - elif output.is_dir() and not any( + elif not is_redirect(output) and output.is_dir() and not any( UNIT_HASH.fullmatch(entry.name) for entry in output.iterdir() ): shutil.rmtree(output) @@ -194,20 +205,20 @@ def invalidate_units( removed += 1 # Cargo 1.100 stores each unit's fingerprint and output together. - if not build.is_dir() or build.is_symlink(): + if not build.is_dir() or is_redirect(build): continue for package in packages: directory = build / package - if not directory.is_dir() or directory.is_symlink(): + if not directory.is_dir() or is_redirect(directory): continue for unit in directory.iterdir(): fingerprint = unit / "fingerprint" if ( UNIT_HASH.fullmatch(unit.name) and unit.is_dir() - and not unit.is_symlink() + and not is_redirect(unit) and fingerprint.is_dir() - and not fingerprint.is_symlink() + and not is_redirect(fingerprint) and ( not build_scripts_only or any(fingerprint.glob("run-build-script-*")) @@ -264,7 +275,7 @@ def restore( for group, names in matching.items(): for name in names: path = safe_path(root, name) - if path is None or path.is_symlink(): + if path is None or is_redirect(path): continue os.utime( path, diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index fdcc982..7036b2a 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -6,6 +6,7 @@ import json import re import shutil +import stat import subprocess import tomllib from collections import Counter @@ -16,6 +17,17 @@ UNIT_FINGERPRINT = re.compile(r"^[0-9a-f]{16}$") +def is_redirect(path: Path) -> bool: + """Recognize symlinks and Windows reparse points without following them.""" + try: + info = path.lstat() + except (FileNotFoundError, NotADirectoryError): + return False + return stat.S_ISLNK(info.st_mode) or bool( + getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ) + + def workspace_packages(workspace: Path) -> set[str]: metadata = json.loads( subprocess.check_output( @@ -41,8 +53,8 @@ def fingerprint_last_used(directory: Path) -> int | None: timestamps = [ fingerprint.stat().st_mtime_ns for metadata in directory.glob("*.json") - if not metadata.is_symlink() - and not (fingerprint := directory / metadata.stem).is_symlink() + if not is_redirect(metadata) + and not is_redirect(fingerprint := directory / metadata.stem) and fingerprint.is_file() ] return max(timestamps, default=None) @@ -59,11 +71,11 @@ def validate_profile(target: Path, profile: Path) -> None: raise ValueError("Cargo profile must be below the target directory") current = target for part in relative.parts: - if current.is_symlink(): - raise ValueError(f"Refusing to prune symlinked Cargo path: {current}") + if is_redirect(current): + raise ValueError(f"Refusing to prune symlinked Cargo path or reparse point: {current}") current /= part - if current.is_symlink(): - raise ValueError(f"Refusing to prune symlinked Cargo path: {current}") + if is_redirect(current): + raise ValueError(f"Refusing to prune symlinked Cargo path or reparse point: {current}") def prune_workspace_fingerprints( @@ -80,7 +92,7 @@ def prune_workspace_fingerprints( package_directories: set[Path] = set() def record(package: str, fingerprint: str, directory: Path) -> None: - if directory.is_symlink() or not directory.is_dir(): + if is_redirect(directory) or not directory.is_dir(): protected_fingerprints.add(fingerprint) return last_used = fingerprint_last_used(directory) @@ -96,7 +108,7 @@ def record(package: str, fingerprint: str, directory: Path) -> None: stale_candidates.add((package, fingerprint)) # Cargo before 1.100 stores all fingerprints in one profile directory. - if not fingerprint_root.is_symlink() and fingerprint_root.is_dir(): + if not is_redirect(fingerprint_root) and fingerprint_root.is_dir(): for directory in fingerprint_root.iterdir(): match = FINGERPRINT_DIRECTORY.fullmatch(directory.name) if match is not None: @@ -107,15 +119,15 @@ def record(package: str, fingerprint: str, directory: Path) -> None: # Cargo 1.100 stores each compilation unit, including its outputs, under # build//. Do not follow symlinks supplied by a cache. - if not build_root.is_symlink() and build_root.is_dir(): + if not is_redirect(build_root) and build_root.is_dir(): for package_directory in build_root.iterdir(): - if package_directory.is_symlink() or not package_directory.is_dir(): + if is_redirect(package_directory) or not package_directory.is_dir(): continue for unit in package_directory.iterdir(): if UNIT_FINGERPRINT.fullmatch(unit.name) is None: continue package_directories.add(package_directory) - if unit.is_symlink() or not unit.is_dir(): + if is_redirect(unit) or not unit.is_dir(): protected_fingerprints.add(unit.name) continue record(package_directory.name, unit.name, unit / "fingerprint") @@ -142,7 +154,7 @@ def record(package: str, fingerprint: str, directory: Path) -> None: removed_paths += 1 for directory in (fingerprint_root, profile / "deps", build_root): - if directory.is_symlink() or not directory.is_dir(): + if is_redirect(directory) or not directory.is_dir(): continue for artifact in directory.iterdir(): @@ -159,7 +171,11 @@ def record(package: str, fingerprint: str, directory: Path) -> None: # This is a Cargo 1.100 package directory, not a legacy build unit. continue - if artifact.is_symlink() or artifact.is_file(): + if artifact.is_symlink(): + artifact.unlink() + elif is_redirect(artifact): + continue + elif artifact.is_file(): artifact.unlink() elif artifact.is_dir(): shutil.rmtree(artifact) From f7959ae8fbf1285d371efc4218e43110ebe7d3a7 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 16:38:23 -0500 Subject: [PATCH 09/10] Retain Cargo units with case-ambiguous package names --- rust-workspace-cache/prune/prune.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index 7036b2a..b673789 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -45,8 +45,9 @@ def workspace_packages(workspace: Path) -> set[str]: # a dependency with that name, so retain both when the lockfile is ambiguous. with (Path(metadata["workspace_root"]) / "Cargo.lock").open("rb") as lockfile: locked = tomllib.load(lockfile) - counts = Counter(package["name"] for package in locked.get("package", ())) - return {name for name in names if counts[name] == 1} + # Case variants can share a build directory on case-insensitive filesystems. + counts = Counter(package["name"].casefold() for package in locked.get("package", ())) + return {name for name in names if counts[name.casefold()] == 1} def fingerprint_last_used(directory: Path) -> int | None: From 79da245fadf4d1ea41eeae50ff2fb8d2a4cecf57 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 16 Nov 2026 16:38:29 -0500 Subject: [PATCH 10/10] Count legacy Cargo lockfile root packages --- rust-workspace-cache/prune/prune.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index b673789..6f01e8f 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -47,6 +47,8 @@ def workspace_packages(workspace: Path) -> set[str]: locked = tomllib.load(lockfile) # Case variants can share a build directory on case-insensitive filesystems. counts = Counter(package["name"].casefold() for package in locked.get("package", ())) + if root := locked.get("root"): + counts[root["name"].casefold()] += 1 return {name for name in names if counts[name.casefold()] == 1}