diff --git a/.github/workflows/test-rust-workspace-cache.yml b/.github/workflows/test-rust-workspace-cache.yml index 17df62f..0b2e197 100644 --- a/.github/workflows/test-rust-workspace-cache.yml +++ b/.github/workflows/test-rust-workspace-cache.yml @@ -19,6 +19,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + rust: [1.99.0, 1.100.0] runs-on: ${{ matrix.os }} timeout-minutes: 10 steps: @@ -28,17 +29,22 @@ jobs: with: persist-credentials: false + - name: Install Rust ${{ matrix.rust }} + run: rustup toolchain install ${{ matrix.rust }} --profile minimal + + - name: Select Rust ${{ matrix.rust }} + shell: bash + run: echo "RUSTUP_TOOLCHAIN=${{ matrix.rust }}" >> "$GITHUB_ENV" + - name: Install uv uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: version: "0.13.5" enable-cache: false - - name: Test pruning and lifecycle + - name: Test lifecycle shell: bash run: | - python -m unittest discover -s rust-workspace-cache/prune -p 'test_*.py' - python -m unittest discover -s rust-workspace-cache/inputs -p 'test_*.py' node --test rust-workspace-cache/prune/lifecycle.test.cjs node --test rust-workspace-cache/inputs/lifecycle.test.cjs @@ -56,7 +62,10 @@ jobs: profiling cache-bin: ${{ runner.os != 'macOS' }} prefix-key: test-rust-workspace-cache - key: ${{ github.run_id }}-${{ github.run_attempt }} + key: ${{ matrix.rust }}-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Seed stale cached artifacts + run: node rust-workspace-cache/tests/smoke/seed.cjs "${{ steps.fixture.outputs.workspace }}" "${{ steps.fixture.outputs.layout }}" - name: Build workspace working-directory: ${{ steps.fixture.outputs.workspace }} diff --git a/rust-workspace-cache/inputs/inputs.py b/rust-workspace-cache/inputs/inputs.py index 6d3bd00..25ab473 100644 --- a/rust-workspace-cache/inputs/inputs.py +++ b/rust-workspace-cache/inputs/inputs.py @@ -15,6 +15,18 @@ from pathlib import Path FINGERPRINT = re.compile(r"^(?P.+)-[0-9a-f]{16}$") +UNIT_HASH = re.compile(r"^[0-9a-f]{16}$") + + +def is_redirect(path: Path) -> bool: + """Recognize symlinks and Windows reparse points without following them.""" + try: + info = path.lstat() + except (FileNotFoundError, NotADirectoryError): + return False + return stat.S_ISLNK(info.st_mode) or bool( + getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ) def workspace_packages(root: Path) -> list[dict]: @@ -144,29 +156,76 @@ def restorable_directories(snapshot: dict) -> set[str]: def invalidate_units( - profiles: list[Path], packages: set[str], *, build_scripts_only: bool + profiles: list[Path], + packages: set[str], + *, + build_scripts_only: bool, + target: Path | None = None, ) -> int: + # Cargo groups units by package name, including multiple versions and sources. + # Invalidate every matching unit so changed workspace inputs cannot reuse a stale + # build script, even if a same-name dependency must also be rebuilt. removed = 0 for profile in profiles: + boundary = target or profile + try: + relative = profile.relative_to(boundary) + except ValueError as error: + raise ValueError(f"Cargo profile is outside the target directory: {profile}") from error + component = boundary + for part in ("", *relative.parts): + component = component / part + if is_redirect(component): + raise ValueError(f"Cargo profile has a symlinked path component or reparse point: {profile}") fingerprints = profile / ".fingerprint" - if not fingerprints.is_dir(): - continue - for directory in fingerprints.iterdir(): + build = profile / "build" + directories = ( + fingerprints.iterdir() + if fingerprints.is_dir() and not is_redirect(fingerprints) + else () + ) + for directory in directories: match = FINGERPRINT.fullmatch(directory.name) if ( match and match.group("package") in packages and directory.is_dir() - and not directory.is_symlink() + and not is_redirect(directory) and (not build_scripts_only or any(directory.glob("run-build-script-*"))) ): - output = profile / "build" / directory.name - if output.is_symlink(): - output.unlink() - elif output.is_dir(): - shutil.rmtree(output) + output = build / directory.name + if build.is_dir() and not is_redirect(build): + if output.is_symlink(): + output.unlink() + elif not is_redirect(output) and output.is_dir() and not any( + UNIT_HASH.fullmatch(entry.name) for entry in output.iterdir() + ): + shutil.rmtree(output) shutil.rmtree(directory) removed += 1 + + # Cargo 1.100 stores each unit's fingerprint and output together. + if not build.is_dir() or is_redirect(build): + continue + for package in packages: + directory = build / package + if not directory.is_dir() or is_redirect(directory): + continue + for unit in directory.iterdir(): + fingerprint = unit / "fingerprint" + if ( + UNIT_HASH.fullmatch(unit.name) + and unit.is_dir() + and not is_redirect(unit) + and fingerprint.is_dir() + and not is_redirect(fingerprint) + and ( + not build_scripts_only + or any(fingerprint.glob("run-build-script-*")) + ) + ): + shutil.rmtree(unit) + removed += 1 return removed @@ -195,19 +254,28 @@ def restore( } # Changed tracked inputs or directory listings must rerun their build scripts # even with preserved mtimes or a clock behind the cached build's timestamp. - invalidated = invalidate_units(profiles, changed_packages, build_scripts_only=True) + target = target or root / "target" + invalidated = invalidate_units( + profiles, + changed_packages, + build_scripts_only=True, + target=target, + ) invalidated_workspace = 0 if ( current["source_layout"] is None or current["source_layout"] != saved.get("source_layout") ): invalidated_workspace = invalidate_units( - profiles, set(current["workspace"]), build_scripts_only=False + profiles, + set(current["workspace"]), + build_scripts_only=False, + target=target, ) for group, names in matching.items(): for name in names: path = safe_path(root, name) - if path is None or path.is_symlink(): + if path is None or is_redirect(path): continue os.utime( path, @@ -270,6 +338,11 @@ def main() -> None: parser.add_argument("--extra-target-directory", type=Path) args = parser.parse_args() start = time.monotonic() + if args.target_directory is not None: + target = args.target_directory.absolute() + for profile in args.profiles: + if not profile.absolute().is_relative_to(target): + parser.error(f"Cargo profile is outside the target directory: {profile}") if args.operation == "save": data = scan(Path.cwd(), args.target_directory) args.manifest.parent.mkdir(parents=True, exist_ok=True) @@ -286,10 +359,16 @@ def main() -> None: ) else: result = {"no_manifest": True} - if any((profile / ".fingerprint").is_dir() for profile in args.profiles): + if any( + (profile / ".fingerprint").is_dir() or (profile / "build").is_dir() + for profile in args.profiles + ): packages = {package["name"] for package in workspace_packages(Path.cwd())} result["invalidated_workspace_units"] = invalidate_units( - args.profiles, packages, build_scripts_only=False + args.profiles, + packages, + build_scripts_only=False, + target=args.target_directory or Path.cwd() / "target", ) if extra := args.extra_target_directory: staged = args.manifest.with_name(f"target-{extra.name}") diff --git a/rust-workspace-cache/inputs/lifecycle.test.cjs b/rust-workspace-cache/inputs/lifecycle.test.cjs index caac7d2..cb1aea3 100644 --- a/rust-workspace-cache/inputs/lifecycle.test.cjs +++ b/rust-workspace-cache/inputs/lifecycle.test.cjs @@ -165,6 +165,13 @@ test("rejects extra directories outside the target or used by Cargo", (context) } }); +test("rejects Cargo profiles outside the target directory", () => { + const target = path.resolve("target"); + for (const profile of [".", "../outside", path.resolve("outside")]) { + assert.throws(() => profileDirectories(target, profile), /within the target/); + } +}); + test("does not save metadata when restoration fails", (context) => { const result = lifecycle(context, true, 7); assert.equal(result.status, 7); diff --git a/rust-workspace-cache/inputs/test_inputs.py b/rust-workspace-cache/inputs/test_inputs.py deleted file mode 100644 index 199274e..0000000 --- a/rust-workspace-cache/inputs/test_inputs.py +++ /dev/null @@ -1,242 +0,0 @@ -"""Exercise input restoration against Cargo's build-script freshness checks.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import time -import unittest -from pathlib import Path - -import inputs - - -class ToolDirectoryTest(unittest.TestCase): - def test_restore_replaces_stale_files_with_the_saved_directory(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - live = root / "target" / "tool-data" - staged = root / "cache" / "target-tool-data" - inputs.move_tool_directory(staged, live) - self.assertFalse(live.exists()) - - live.mkdir(parents=True) - (live / "graph").write_bytes(b"saved graph") - inputs.move_tool_directory(live, staged) - self.assertFalse(live.exists()) - - live.mkdir() - (live / "stale").touch() - inputs.move_tool_directory(staged, live) - self.assertEqual((live / "graph").read_bytes(), b"saved graph") - self.assertFalse((live / "stale").exists()) - self.assertFalse(staged.exists()) - - def test_source_layout_ignores_build_outputs_and_detects_retargeted_links(self) -> None: - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary).resolve() - subprocess.run(["git", "init", "--quiet"], cwd=root, check=True) - target = root / "build [output]" - target.mkdir() - for name in ("a", "b"): - (root / name).write_text("same contents") - link = root / "source.rs" - try: - link.symlink_to("a") - except OSError as error: - self.skipTest(str(error)) - saved = inputs.source_layout(root, target) - (target / "artifact").touch() - (root / "a").write_text("new contents") - self.assertEqual(inputs.source_layout(root, target), saved) - link.unlink() - link.symlink_to("b") - self.assertNotEqual(inputs.source_layout(root, target), saved) - - -class CargoInputsTest(unittest.TestCase): - def setUp(self) -> None: - temporary = tempfile.TemporaryDirectory(prefix="cargo-inputs-") - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name).resolve() - (self.root / "src").mkdir() - (self.root / "inputs").mkdir() - (self.root / "Cargo.toml").write_text( - '[package]\nname = "cache-input-fixture"\nversion = "0.1.0"\nedition = "2021"\n' - ) - (self.root / "src/main.rs").write_text( - 'include!(concat!(env!("OUT_DIR"), "/generated.rs"));\n' - 'fn main() { println!("{}", VALUE); }\n' - ) - (self.root / "build.rs").write_text( - "use std::{env, fs, path::PathBuf};\n" - "fn main() {\n" - ' println!("cargo:rerun-if-changed=inputs");\n' - ' let mut files = fs::read_dir("inputs").unwrap()\n' - " .map(|p| p.unwrap().path()).collect::>();\n" - " files.sort();\n" - " let value = files.into_iter().map(|p| fs::read_to_string(p).unwrap())\n" - " .collect::();\n" - ' fs::write(PathBuf::from(env::var("OUT_DIR").unwrap()).join("generated.rs"),\n' - ' format!("const VALUE: &str = {:?};", value)).unwrap();\n' - "}\n" - ) - self.value = self.root / "inputs/value.txt" - self.value.write_text("original") - (self.root / ".gitignore").write_text("target/\n") - subprocess.run(["git", "init", "--quiet"], cwd=self.root, check=True) - subprocess.run(["git", "add", "."], cwd=self.root, check=True) - self.environment = os.environ | { - "CARGO_UNSTABLE_CHECKSUM_FRESHNESS": "true", - "RUSTC_BOOTSTRAP": "1", - "CARGO_INCREMENTAL": "0", - "CARGO_TERM_COLOR": "never", - "CARGO_TARGET_DIR": str(self.root / "target"), - } - self.build("original", rebuilt=True) - self.saved = inputs.scan(self.root) - - def build(self, expected: str, *, rebuilt: bool) -> None: - result = subprocess.run( - ["cargo", "run", "--offline", "-vv"], - cwd=self.root, - env=self.environment, - text=True, - capture_output=True, - check=True, - ) - self.assertEqual(result.stdout.splitlines()[-1], expected, result.stderr) - self.assertEqual( - "Compiling cache-input-fixture" in result.stderr, rebuilt, result.stderr - ) - - def restore(self) -> dict[str, int]: - return inputs.restore(self.root, self.saved, [self.root / "target/debug"]) - - def test_fresh_checkout_reuses_artifacts(self) -> None: - for group in ["files", "directories"]: - for name in self.saved[group]: - os.utime(self.root / name, None) - result = self.restore() - self.assertGreater(result["restored_files"], 0) - self.assertEqual(result["invalidated_build_scripts"], 0) - self.build("original", rebuilt=False) - - def test_new_module_path_invalidates_a_binary_without_a_build_script(self) -> None: - (self.root / "build.rs").unlink() - (self.root / "src/main.rs").write_text( - 'mod value; fn main() { println!("{}", value::get()); }\n' - ) - nested = self.root / "src/value/mod.rs" - nested.parent.mkdir() - nested.write_text("pub fn get() -> u32 { 42 }\n") - self.build("42", rebuilt=True) - self.saved = inputs.scan(self.root) - self.assertEqual(self.saved["packages"], {}) - self.assertEqual(self.restore()["invalidated_workspace_units"], 0) - self.build("42", rebuilt=False) - - (self.root / "src/value.rs").write_text("pub fn get() -> u32 { 43 }\n") - self.assertGreater(self.restore()["invalidated_workspace_units"], 0) - result = subprocess.run( - ["cargo", "run", "--offline"], - cwd=self.root, - env=self.environment, - text=True, - capture_output=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("E0761", result.stderr) - - def test_changed_bytes_with_preserved_timestamp_rebuild(self) -> None: - old = self.value.stat().st_mtime_ns - self.value.write_text("changed") - os.utime(self.value, ns=(old, old)) - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - staged = subprocess.check_output( - ["git", "show", ":inputs/value.txt"], cwd=self.root, text=True - ) - self.assertEqual(staged, "original") - self.build("changed", rebuilt=True) - - def test_subsequent_checkout_keeps_matching_input_timestamps(self) -> None: - subprocess.run( - [ - "git", - "-c", - "user.name=Fixture", - "-c", - "user.email=fixture@example.invalid", - "commit", - "-qm", - "seed", - ], - cwd=self.root, - check=True, - ) - for name in self.saved["files"]: - os.utime(self.root / name, None) - subprocess.run(["git", "update-index", "--refresh"], cwd=self.root, check=True) - self.restore() - subprocess.run( - ["git", "checkout", "--quiet", "--force", "HEAD"], cwd=self.root, check=True - ) - self.assertEqual( - self.value.stat().st_mtime_ns, - self.saved["files"]["inputs/value.txt"]["mtime_ns"], - ) - self.build("original", rebuilt=False) - - def test_changed_bytes_rebuild_when_cached_output_is_in_the_future(self) -> None: - future = time.time_ns() + 60_000_000_000 - for output in (self.root / "target/debug/build").glob("*/output"): - os.utime(output, ns=(future, future)) - old = self.value.stat().st_mtime_ns - self.value.write_text("changed") - os.utime(self.value, ns=(old, old)) - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - self.build("changed", rebuilt=True) - - def test_new_untracked_input_rebuilds(self) -> None: - (self.root / "inputs/extra.txt").write_text("extra+") - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - self.build("extra+original", rebuilt=True) - - def test_replaced_untracked_input_keeps_directory_timestamp(self) -> None: - extra = self.root / "inputs/extra.txt" - extra.write_text("old+") - self.build("old+original", rebuilt=True) - self.saved = inputs.scan(self.root) - old = extra.stat().st_mtime_ns - extra.unlink() - extra.write_text("new+") - os.utime(extra, ns=(old, old)) - directory = extra.parent - modified = time.time_ns() + 1_000_000_000 - os.utime(directory, ns=(modified, modified)) - modified = directory.stat().st_mtime_ns - self.restore() - self.assertEqual(directory.stat().st_mtime_ns, modified) - self.build("new+original", rebuilt=True) - - def test_deleted_tracked_input_rebuilds(self) -> None: - self.value.unlink() - self.assertEqual(self.restore()["invalidated_build_scripts"], 1) - self.build("", rebuilt=True) - - def test_git_metadata_is_not_restored(self) -> None: - head = self.root / ".git/HEAD" - os.utime(head, ns=(1_000_000_000, 1_000_000_000)) - self.restore() - self.assertEqual(head.stat().st_mtime_ns, 1_000_000_000) - - def test_rejects_escaping_paths(self) -> None: - with self.assertRaises(ValueError): - inputs.safe_path(self.root, "../outside") - with self.assertRaises(ValueError): - inputs.safe_path(self.root, str(self.root.parent / "outside")) - - -if __name__ == "__main__": - unittest.main() diff --git a/rust-workspace-cache/prune/lifecycle.test.cjs b/rust-workspace-cache/prune/lifecycle.test.cjs index 1e65b51..a2018a7 100644 --- a/rust-workspace-cache/prune/lifecycle.test.cjs +++ b/rust-workspace-cache/prune/lifecycle.test.cjs @@ -48,6 +48,7 @@ test("records an isolated marker and the external workspace for pruning", (conte assert.equal(result.environment, "CARGO_UNSTABLE_MTIME_ON_USE=true\n"); const state = JSON.parse(result.state.slice("cache=".length)); assert.equal(state.workspace, result.workspace); + assert.equal(state.target, path.join(result.workspace, "build-output")); assert.deepEqual(state.profiles, [ path.join(result.workspace, "build-output", "fast-build-nightly"), ]); @@ -137,6 +138,8 @@ test("prefers uv without discovering the caller's Python project or configuratio path.join(__dirname, "prune.py"), ...result.state.profiles, result.state.marker, + "--target-directory", + result.state.target, ], options: { cwd: result.state.workspace, stdio: "inherit" }, }, @@ -162,6 +165,8 @@ for (const [platform, python] of [ path.join(__dirname, "prune.py"), ...result.state.profiles, result.state.marker, + "--target-directory", + result.state.target, ]); assert.deepEqual(result.calls[1].options, { cwd: result.state.workspace, diff --git a/rust-workspace-cache/prune/main.cjs b/rust-workspace-cache/prune/main.cjs index 021f711..6a2d920 100644 --- a/rust-workspace-cache/prune/main.cjs +++ b/rust-workspace-cache/prune/main.cjs @@ -19,6 +19,6 @@ if (enabled) { fs.writeFileSync(marker, ""); fs.appendFileSync( process.env.GITHUB_STATE, - `cache=${JSON.stringify({ workspace, profiles, marker })}\n`, + `cache=${JSON.stringify({ workspace, target, profiles, marker })}\n`, ); } diff --git a/rust-workspace-cache/prune/post.cjs b/rust-workspace-cache/prune/post.cjs index b369cc2..8b96aa0 100644 --- a/rust-workspace-cache/prune/post.cjs +++ b/rust-workspace-cache/prune/post.cjs @@ -3,13 +3,17 @@ const path = require("node:path"); const { runPython } = require("../runtime.cjs"); if (process.env.STATE_cache) { - const { workspace, profiles, marker } = JSON.parse(process.env.STATE_cache); + const { workspace, target, profiles, marker } = JSON.parse( + process.env.STATE_cache, + ); try { process.exitCode = runPython( workspace, "prune/prune.py", ...profiles, marker, + "--target-directory", + target, ); } finally { fs.rmSync(path.dirname(marker), { recursive: true, force: true }); diff --git a/rust-workspace-cache/prune/prune.py b/rust-workspace-cache/prune/prune.py index 7d61b46..6f01e8f 100644 --- a/rust-workspace-cache/prune/prune.py +++ b/rust-workspace-cache/prune/prune.py @@ -6,11 +6,26 @@ import json import re import shutil +import stat import subprocess +import tomllib +from collections import Counter from pathlib import Path FINGERPRINT_DIRECTORY = re.compile(r"^(?P.+)-(?P[0-9a-f]{16})$") ARTIFACT_FINGERPRINT = re.compile(r"-([0-9a-f]{16})(?:\.|$)") +UNIT_FINGERPRINT = re.compile(r"^[0-9a-f]{16}$") + + +def is_redirect(path: Path) -> bool: + """Recognize symlinks and Windows reparse points without following them.""" + try: + info = path.lstat() + except (FileNotFoundError, NotADirectoryError): + return False + return stat.S_ISLNK(info.st_mode) or bool( + getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT + ) def workspace_packages(workspace: Path) -> set[str]: @@ -22,70 +37,148 @@ def workspace_packages(workspace: Path) -> set[str]: ) ) members = set(metadata["workspace_members"]) - return { + names = { package["name"] for package in metadata["packages"] if package["id"] in members } + # Cargo places different versions or sources of a package under the same + # package name. A unit hash alone cannot distinguish a workspace member from + # a dependency with that name, so retain both when the lockfile is ambiguous. + with (Path(metadata["workspace_root"]) / "Cargo.lock").open("rb") as lockfile: + locked = tomllib.load(lockfile) + # Case variants can share a build directory on case-insensitive filesystems. + counts = Counter(package["name"].casefold() for package in locked.get("package", ())) + if root := locked.get("root"): + counts[root["name"].casefold()] += 1 + return {name for name in names if counts[name.casefold()] == 1} def fingerprint_last_used(directory: Path) -> int | None: timestamps = [ fingerprint.stat().st_mtime_ns for metadata in directory.glob("*.json") - if (fingerprint := directory / metadata.stem).is_file() + if not is_redirect(metadata) + and not is_redirect(fingerprint := directory / metadata.stem) + and fingerprint.is_file() ] return max(timestamps, default=None) +def validate_profile(target: Path, profile: Path) -> None: + try: + relative = profile.relative_to(target) + except ValueError: + raise ValueError( + f"Cargo profile {profile} is outside target directory {target}" + ) from None + if not relative.parts: + raise ValueError("Cargo profile must be below the target directory") + current = target + for part in relative.parts: + if is_redirect(current): + raise ValueError(f"Refusing to prune symlinked Cargo path or reparse point: {current}") + current /= part + if is_redirect(current): + raise ValueError(f"Refusing to prune symlinked Cargo path or reparse point: {current}") + + def prune_workspace_fingerprints( profile: Path, marker_timestamp: int, packages: set[str] ) -> tuple[int, int, int]: fingerprint_root = profile / ".fingerprint" - stale_fingerprints_by_package: dict[str, set[str]] = {} + build_root = profile / "build" + stale_candidates: set[tuple[str, str]] = set() active_fingerprints: set[str] = set() active_packages: set[str] = set() - - for directory in fingerprint_root.iterdir(): - if not directory.is_dir(): - continue - - match = FINGERPRINT_DIRECTORY.fullmatch(directory.name) - if match is None: - continue - + protected_fingerprints: set[str] = set() + units: list[tuple[str, str, Path]] = [] + legacy_units: set[tuple[str, str]] = set() + package_directories: set[Path] = set() + + def record(package: str, fingerprint: str, directory: Path) -> None: + if is_redirect(directory) or not directory.is_dir(): + protected_fingerprints.add(fingerprint) + return last_used = fingerprint_last_used(directory) + if last_used is None or package not in packages: + protected_fingerprints.add(fingerprint) if last_used is None: - continue - - package = match.group("package") - fingerprint = match.group("fingerprint") + return if last_used >= marker_timestamp: active_fingerprints.add(fingerprint) if package in packages: active_packages.add(package) elif package in packages: - stale_fingerprints_by_package.setdefault(package, set()).add(fingerprint) + stale_candidates.add((package, fingerprint)) + + # Cargo before 1.100 stores all fingerprints in one profile directory. + if not is_redirect(fingerprint_root) and fingerprint_root.is_dir(): + for directory in fingerprint_root.iterdir(): + match = FINGERPRINT_DIRECTORY.fullmatch(directory.name) + if match is not None: + package = match.group("package") + fingerprint = match.group("fingerprint") + record(package, fingerprint, directory) + legacy_units.add((package, fingerprint)) + + # Cargo 1.100 stores each compilation unit, including its outputs, under + # build//. Do not follow symlinks supplied by a cache. + if not is_redirect(build_root) and build_root.is_dir(): + for package_directory in build_root.iterdir(): + if is_redirect(package_directory) or not package_directory.is_dir(): + continue + for unit in package_directory.iterdir(): + if UNIT_FINGERPRINT.fullmatch(unit.name) is None: + continue + package_directories.add(package_directory) + if is_redirect(unit) or not unit.is_dir(): + protected_fingerprints.add(unit.name) + continue + record(package_directory.name, unit.name, unit / "fingerprint") + units.append((package_directory.name, unit.name, unit)) if not active_fingerprints: raise RuntimeError("No active Cargo fingerprints found; refusing to prune") - stale_fingerprints = { - fingerprint - for package in active_packages - for fingerprint in stale_fingerprints_by_package.get(package, set()) + stale_units = { + (package, fingerprint) + for package, fingerprint in stale_candidates + if package in active_packages + and fingerprint not in active_fingerprints + and fingerprint not in protected_fingerprints } - stale_fingerprints.difference_update(active_fingerprints) + stale_fingerprints = {fingerprint for _, fingerprint in stale_units} + legacy_stale_units = stale_units & legacy_units + legacy_stale_fingerprints = {fingerprint for _, fingerprint in legacy_stale_units} removed_paths = 0 - for directory in (fingerprint_root, profile / "deps", profile / "build"): - if not directory.is_dir(): + for package, fingerprint, unit in units: + if (package, fingerprint) in stale_units: + shutil.rmtree(unit) + removed_paths += 1 + + for directory in (fingerprint_root, profile / "deps", build_root): + if is_redirect(directory) or not directory.is_dir(): continue for artifact in directory.iterdir(): match = ARTIFACT_FINGERPRINT.search(artifact.name) - if match is None or match.group(1) not in stale_fingerprints: + if match is None or match.group(1) not in legacy_stale_fingerprints: continue - - if artifact.is_symlink() or artifact.is_file(): + if directory in (fingerprint_root, build_root): + fingerprint = FINGERPRINT_DIRECTORY.fullmatch(artifact.name) + if fingerprint is None or ( + fingerprint.group("package"), fingerprint.group("fingerprint") + ) not in legacy_stale_units: + continue + if directory == build_root and artifact in package_directories: + # This is a Cargo 1.100 package directory, not a legacy build unit. + continue + + if artifact.is_symlink(): + artifact.unlink() + elif is_redirect(artifact): + continue + elif artifact.is_file(): artifact.unlink() elif artifact.is_dir(): shutil.rmtree(artifact) @@ -103,11 +196,13 @@ def main() -> None: "profiles", type=Path, nargs="+", help="Cargo profile directories" ) parser.add_argument("marker", type=Path, help="Marker created before the build") + parser.add_argument("--target-directory", type=Path, required=True) args = parser.parse_args() marker_timestamp = args.marker.stat().st_mtime_ns packages = workspace_packages(Path.cwd()) for profile in args.profiles: + validate_profile(args.target_directory, profile) stale, removed, active = prune_workspace_fingerprints( profile, marker_timestamp, packages ) diff --git a/rust-workspace-cache/prune/test_prune.py b/rust-workspace-cache/prune/test_prune.py deleted file mode 100644 index 6b462b5..0000000 --- a/rust-workspace-cache/prune/test_prune.py +++ /dev/null @@ -1,104 +0,0 @@ -import os -import sys -import tempfile -import unittest -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) - -from prune import prune_workspace_fingerprints - -OLD = 1_000_000_000 -MARKER = 2_000_000_000 -CURRENT = 3_000_000_000 - - -class PruneWorkspaceCacheTest(unittest.TestCase): - def setUp(self) -> None: - self.temporary = tempfile.TemporaryDirectory() - self.addCleanup(self.temporary.cleanup) - self.profile = Path(self.temporary.name) - (self.profile / ".fingerprint").mkdir() - (self.profile / "deps").mkdir() - (self.profile / "build").mkdir() - - def fingerprint(self, package: str, fingerprint: str, timestamp: int) -> Path: - directory = self.profile / ".fingerprint" / f"{package}-{fingerprint}" - directory.mkdir() - (directory / "lib-example.json").write_text("{}") - stamp = directory / "lib-example" - stamp.touch() - os.utime(stamp, ns=(timestamp, timestamp)) - return directory - - def test_removes_superseded_workspace_artifacts(self) -> None: - old = self.fingerprint("example", "1111111111111111", OLD) - current = self.fingerprint("example", "2222222222222222", CURRENT) - dependency = self.fingerprint("dependency", "3333333333333333", OLD) - unused = self.fingerprint("unused", "4444444444444444", OLD) - library = self.profile / "deps" / "libexample-1111111111111111.rlib" - library.touch() - build = self.profile / "build" / "example-1111111111111111" - build.mkdir() - (build / "output").touch() - active_library = self.profile / "deps" / "libexample-2222222222222222.rlib" - active_library.touch() - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example", "unused"}), - (1, 3, 1), - ) - self.assertFalse(old.exists()) - self.assertFalse(library.exists()) - self.assertFalse(build.exists()) - self.assertTrue(current.is_dir()) - self.assertTrue(active_library.is_file()) - self.assertTrue(dependency.is_dir()) - self.assertTrue(unused.is_dir()) - - def test_retains_hashes_used_by_another_active_package(self) -> None: - old = self.fingerprint("example", "1111111111111111", OLD) - self.fingerprint("example", "2222222222222222", CURRENT) - self.fingerprint("dependency", "1111111111111111", CURRENT) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (0, 0, 2), - ) - self.assertTrue(old.is_dir()) - - def test_refuses_to_prune_without_active_fingerprints(self) -> None: - old = self.fingerprint("example", "1111111111111111", OLD) - with self.assertRaisesRegex(RuntimeError, "No active Cargo fingerprints"): - prune_workspace_fingerprints(self.profile, MARKER, {"example"}) - self.assertTrue(old.is_dir()) - - def test_retains_fingerprints_without_usage_metadata(self) -> None: - unknown = self.fingerprint("example", "1111111111111111", OLD) - (unknown / "lib-example").unlink() - self.fingerprint("example", "2222222222222222", MARKER) - - self.assertEqual( - prune_workspace_fingerprints(self.profile, MARKER, {"example"}), - (0, 0, 1), - ) - self.assertTrue(unknown.is_dir()) - - @unittest.skipIf(sys.platform == "win32", "Creating symlinks requires privileges") - def test_removes_stale_symlinks_without_removing_their_targets(self) -> None: - self.fingerprint("example", "1111111111111111", OLD) - self.fingerprint("example", "2222222222222222", CURRENT) - target = self.profile / "keep" - target.mkdir() - (target / "output").write_text("keep") - link = self.profile / "build" / "example-1111111111111111" - link.symlink_to(target, target_is_directory=True) - - prune_workspace_fingerprints(self.profile, MARKER, {"example"}) - - self.assertFalse(link.is_symlink()) - self.assertEqual((target / "output").read_text(), "keep") - - -if __name__ == "__main__": - unittest.main() diff --git a/rust-workspace-cache/runtime.cjs b/rust-workspace-cache/runtime.cjs index 86a9362..80adb9a 100644 --- a/rust-workspace-cache/runtime.cjs +++ b/rust-workspace-cache/runtime.cjs @@ -6,7 +6,20 @@ function profileDirectories(target, value) { .split(/\r?\n/) .map((profile) => profile.trim()) .filter(Boolean) - .map((profile) => path.resolve(target, profile)); + .map((profile) => { + const directory = path.resolve(target, profile); + const relative = path.relative(target, directory); + if ( + !relative || + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) + throw new Error( + "Cargo profiles must be directories within the target directory", + ); + return directory; + }); if (profiles.length === 0) throw new Error("At least one build profile is required"); return profiles; diff --git a/rust-workspace-cache/tests/smoke/action.yml b/rust-workspace-cache/tests/smoke/action.yml index e852bd8..19a7b0f 100644 --- a/rust-workspace-cache/tests/smoke/action.yml +++ b/rust-workspace-cache/tests/smoke/action.yml @@ -3,6 +3,8 @@ description: Create a Cargo workspace and check its artifacts after the cache po outputs: workspace: description: Absolute path of the temporary Cargo workspace. + layout: + description: Cargo artifact layout used by the selected toolchain. runs: using: node24 main: setup.cjs diff --git a/rust-workspace-cache/tests/smoke/check.cjs b/rust-workspace-cache/tests/smoke/check.cjs index ccbf4bf..0097300 100644 --- a/rust-workspace-cache/tests/smoke/check.cjs +++ b/rust-workspace-cache/tests/smoke/check.cjs @@ -3,23 +3,65 @@ const fs = require("node:fs"); const path = require("node:path"); for (const profile of ["debug", "profiling"]) { - const fingerprints = path.join( - process.env.STATE_workspace, - "target", - profile, - ".fingerprint", - ); - assert.ok( - !fs.existsSync( - path.join(fingerprints, "workspace-cache-fixture-1111111111111111"), - ), - "The stale workspace fingerprint must be pruned", - ); - assert.ok( - fs - .readdirSync(fingerprints) - .some((entry) => entry.startsWith("workspace-cache-fixture-")), - "The active workspace fingerprint must remain cached", - ); + const target = path.join(process.env.STATE_workspace, "target", profile); + if (process.env.STATE_layout === "unit") { + const units = path.join(target, "build", "workspace-cache-fixture"); + assert.ok( + !fs.existsSync(path.join(units, "1111111111111111")), + "The stale workspace unit must be pruned", + ); + assert.ok( + fs.readdirSync(units).some((entry) => + fs.existsSync( + path.join(units, entry, "fingerprint", "lib-workspace_cache_fixture"), + ), + ), + "The active workspace unit must remain cached", + ); + const dependency = path.join(target, "build", "cached-dependency"); + assert.ok( + fs.readdirSync(dependency).some((entry) => + fs.existsSync( + path.join( + dependency, + entry, + "out", + `libcached_dependency-${entry}.rlib`, + ), + ), + ), + "Cached dependency artifacts must remain", + ); + } else { + const fingerprints = path.join(target, ".fingerprint"); + assert.ok( + !fs.existsSync( + path.join(fingerprints, "workspace-cache-fixture-1111111111111111"), + ), + "The stale workspace fingerprint must be pruned", + ); + assert.ok( + !fs.existsSync( + path.join( + target, + "deps", + "libworkspace_cache_fixture-1111111111111111.rlib", + ), + ), + "The stale workspace artifact must be pruned", + ); + assert.ok( + fs + .readdirSync(fingerprints) + .some((entry) => entry.startsWith("workspace-cache-fixture-")), + "The active workspace fingerprint must remain cached", + ); + assert.ok( + fs.readdirSync(path.join(target, "deps")).some((entry) => + /^libcached_dependency-[0-9a-f]{16}\.rlib$/.test(entry), + ), + "Cached dependency artifacts must remain", + ); + } } console.log("Superseded workspace artifacts were pruned."); diff --git a/rust-workspace-cache/tests/smoke/seed.cjs b/rust-workspace-cache/tests/smoke/seed.cjs new file mode 100644 index 0000000..7453f3c --- /dev/null +++ b/rust-workspace-cache/tests/smoke/seed.cjs @@ -0,0 +1,27 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +const [workspace, layout] = process.argv.slice(2); + +function createStaleArtifact(target, name, hash) { + const unit = path.join(target, "build", name, hash); + const directory = + layout === "unit" + ? path.join(unit, "fingerprint") + : path.join(target, ".fingerprint", `${name}-${hash}`); + const crate = name.replaceAll("-", "_"); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, `lib-${crate}.json`), "{}"); + const fingerprint = path.join(directory, `lib-${crate}`); + fs.writeFileSync(fingerprint, "stale"); + fs.utimesSync(fingerprint, 1, 1); + const artifacts = + layout === "unit" ? path.join(unit, "out") : path.join(target, "deps"); + fs.mkdirSync(artifacts, { recursive: true }); + fs.writeFileSync(path.join(artifacts, `lib${crate}-${hash}.rlib`), "stale"); +} + +for (const profile of ["debug", "profiling"]) { + const target = path.join(workspace, "target", profile); + createStaleArtifact(target, "workspace-cache-fixture", "1111111111111111"); +} diff --git a/rust-workspace-cache/tests/smoke/setup.cjs b/rust-workspace-cache/tests/smoke/setup.cjs index 9ec2ee6..2d6a647 100644 --- a/rust-workspace-cache/tests/smoke/setup.cjs +++ b/rust-workspace-cache/tests/smoke/setup.cjs @@ -2,16 +2,37 @@ const fs = require("node:fs"); const path = require("node:path"); const { spawnSync } = require("node:child_process"); -const workspace = fs.mkdtempSync( +const cargoVersion = spawnSync("cargo", ["--version"], { encoding: "utf8" }); +if (cargoVersion.error) throw cargoVersion.error; +if (cargoVersion.status !== 0) process.exit(cargoVersion.status ?? 1); +const version = /^cargo (\d+)\.(\d+)\./.exec(cargoVersion.stdout); +if (!version) + throw new Error(`Unrecognized Cargo version: ${cargoVersion.stdout}`); +const layout = + Number(version[1]) > 1 || Number(version[2]) >= 100 ? "unit" : "legacy"; + +const directory = fs.mkdtempSync( path.join(process.env.RUNNER_TEMP, "cargo workspace "), ); +const workspace = path.join(directory, "workspace"); +const dependency = path.join(directory, "dependency"); +fs.mkdirSync(workspace); fs.mkdirSync(path.join(workspace, "src")); fs.writeFileSync( path.join(workspace, "Cargo.toml"), - '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n\n[profile.profiling]\ninherits = "dev"\n', + '[package]\nname = "workspace-cache-fixture"\nversion = "0.1.0"\nedition = "2024"\n\n[dependencies]\ncached-dependency = { path = "../dependency" }\n\n[profile.profiling]\ninherits = "dev"\n', ); fs.writeFileSync( path.join(workspace, "src", "lib.rs"), + "pub fn example() { cached_dependency::example(); }\n", +); +fs.mkdirSync(path.join(dependency, "src"), { recursive: true }); +fs.writeFileSync( + path.join(dependency, "Cargo.toml"), + '[package]\nname = "cached-dependency"\nversion = "0.1.0"\nedition = "2024"\n', +); +fs.writeFileSync( + path.join(dependency, "src", "lib.rs"), "pub fn example() {}\n", ); // Pruning must skip discovery of the caller's Python project and uv configuration. @@ -27,19 +48,11 @@ const result = spawnSync("cargo", ["generate-lockfile", "--offline"], { if (result.error) throw result.error; if (result.status !== 0) process.exit(result.status ?? 1); -for (const profile of ["debug", "profiling"]) { - const stale = path.join( - workspace, - "target", - profile, - ".fingerprint", - "workspace-cache-fixture-1111111111111111", - ); - fs.mkdirSync(stale, { recursive: true }); - fs.writeFileSync(path.join(stale, "lib-workspace_cache_fixture.json"), "{}"); - const fingerprint = path.join(stale, "lib-workspace_cache_fixture"); - fs.writeFileSync(fingerprint, "stale"); - fs.utimesSync(fingerprint, 1, 1); -} -fs.appendFileSync(process.env.GITHUB_OUTPUT, `workspace=${workspace}\n`); -fs.appendFileSync(process.env.GITHUB_STATE, `workspace=${workspace}\n`); +fs.appendFileSync( + process.env.GITHUB_OUTPUT, + `workspace=${workspace}\nlayout=${layout}\n`, +); +fs.appendFileSync( + process.env.GITHUB_STATE, + `workspace=${workspace}\nlayout=${layout}\n`, +);