Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions .github/workflows/test-rust-workspace-cache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
rust: [1.99.0, 1.100.0]
runs-on: ${{ matrix.os }}
timeout-minutes: 10
steps:
Expand All @@ -28,17 +29,22 @@ jobs:
with:
persist-credentials: false

- name: Install Rust ${{ matrix.rust }}
run: rustup toolchain install ${{ matrix.rust }} --profile minimal

- name: Select Rust ${{ matrix.rust }}
shell: bash
run: echo "RUSTUP_TOOLCHAIN=${{ matrix.rust }}" >> "$GITHUB_ENV"

- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: "0.13.5"
enable-cache: false

- name: Test pruning and lifecycle
- name: Test lifecycle
shell: bash
run: |
python -m unittest discover -s rust-workspace-cache/prune -p 'test_*.py'
python -m unittest discover -s rust-workspace-cache/inputs -p 'test_*.py'
node --test rust-workspace-cache/prune/lifecycle.test.cjs
node --test rust-workspace-cache/inputs/lifecycle.test.cjs

Expand All @@ -56,7 +62,10 @@ jobs:
profiling
cache-bin: ${{ runner.os != 'macOS' }}
prefix-key: test-rust-workspace-cache
key: ${{ github.run_id }}-${{ github.run_attempt }}
key: ${{ matrix.rust }}-${{ github.run_id }}-${{ github.run_attempt }}

- name: Seed stale cached artifacts
run: node rust-workspace-cache/tests/smoke/seed.cjs "${{ steps.fixture.outputs.workspace }}" "${{ steps.fixture.outputs.layout }}"

- name: Build workspace
working-directory: ${{ steps.fixture.outputs.workspace }}
Expand Down
109 changes: 94 additions & 15 deletions rust-workspace-cache/inputs/inputs.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,18 @@
from pathlib import Path

FINGERPRINT = re.compile(r"^(?P<package>.+)-[0-9a-f]{16}$")
UNIT_HASH = re.compile(r"^[0-9a-f]{16}$")


def is_redirect(path: Path) -> bool:
"""Recognize symlinks and Windows reparse points without following them."""
try:
info = path.lstat()
except (FileNotFoundError, NotADirectoryError):
return False
return stat.S_ISLNK(info.st_mode) or bool(
getattr(info, "st_file_attributes", 0) & stat.FILE_ATTRIBUTE_REPARSE_POINT
)


def workspace_packages(root: Path) -> list[dict]:
Expand Down Expand Up @@ -144,29 +156,76 @@ def restorable_directories(snapshot: dict) -> set[str]:


def invalidate_units(
profiles: list[Path], packages: set[str], *, build_scripts_only: bool
profiles: list[Path],
packages: set[str],
*,
build_scripts_only: bool,
target: Path | None = None,
) -> int:
# Cargo groups units by package name, including multiple versions and sources.
# Invalidate every matching unit so changed workspace inputs cannot reuse a stale
# build script, even if a same-name dependency must also be rebuilt.
removed = 0
for profile in profiles:
boundary = target or profile
try:
relative = profile.relative_to(boundary)
except ValueError as error:
raise ValueError(f"Cargo profile is outside the target directory: {profile}") from error
component = boundary
for part in ("", *relative.parts):
component = component / part
if is_redirect(component):
raise ValueError(f"Cargo profile has a symlinked path component or reparse point: {profile}")
fingerprints = profile / ".fingerprint"
if not fingerprints.is_dir():
continue
for directory in fingerprints.iterdir():
build = profile / "build"
directories = (
fingerprints.iterdir()
if fingerprints.is_dir() and not is_redirect(fingerprints)
else ()
)
for directory in directories:
match = FINGERPRINT.fullmatch(directory.name)
if (
match
and match.group("package") in packages
and directory.is_dir()
and not directory.is_symlink()
and not is_redirect(directory)
and (not build_scripts_only or any(directory.glob("run-build-script-*")))
):
output = profile / "build" / directory.name
if output.is_symlink():
output.unlink()
elif output.is_dir():
shutil.rmtree(output)
output = build / directory.name
if build.is_dir() and not is_redirect(build):
if output.is_symlink():
output.unlink()
elif not is_redirect(output) and output.is_dir() and not any(
UNIT_HASH.fullmatch(entry.name) for entry in output.iterdir()
):
shutil.rmtree(output)
shutil.rmtree(directory)
removed += 1

# Cargo 1.100 stores each unit's fingerprint and output together.
if not build.is_dir() or is_redirect(build):
continue
for package in packages:
directory = build / package
if not directory.is_dir() or is_redirect(directory):
continue
for unit in directory.iterdir():
fingerprint = unit / "fingerprint"
if (
UNIT_HASH.fullmatch(unit.name)
and unit.is_dir()
and not is_redirect(unit)
and fingerprint.is_dir()
and not is_redirect(fingerprint)
and (
not build_scripts_only
or any(fingerprint.glob("run-build-script-*"))
)
):
shutil.rmtree(unit)
removed += 1
return removed


Expand Down Expand Up @@ -195,19 +254,28 @@ def restore(
}
# Changed tracked inputs or directory listings must rerun their build scripts
# even with preserved mtimes or a clock behind the cached build's timestamp.
invalidated = invalidate_units(profiles, changed_packages, build_scripts_only=True)
target = target or root / "target"
invalidated = invalidate_units(
profiles,
changed_packages,
build_scripts_only=True,
target=target,
)
invalidated_workspace = 0
if (
current["source_layout"] is None
or current["source_layout"] != saved.get("source_layout")
):
invalidated_workspace = invalidate_units(
profiles, set(current["workspace"]), build_scripts_only=False
profiles,
set(current["workspace"]),
build_scripts_only=False,
target=target,
)
for group, names in matching.items():
for name in names:
path = safe_path(root, name)
if path is None or path.is_symlink():
if path is None or is_redirect(path):
continue
os.utime(
path,
Expand Down Expand Up @@ -270,6 +338,11 @@ def main() -> None:
parser.add_argument("--extra-target-directory", type=Path)
args = parser.parse_args()
start = time.monotonic()
if args.target_directory is not None:
target = args.target_directory.absolute()
for profile in args.profiles:
if not profile.absolute().is_relative_to(target):
parser.error(f"Cargo profile is outside the target directory: {profile}")
if args.operation == "save":
data = scan(Path.cwd(), args.target_directory)
args.manifest.parent.mkdir(parents=True, exist_ok=True)
Expand All @@ -286,10 +359,16 @@ def main() -> None:
)
else:
result = {"no_manifest": True}
if any((profile / ".fingerprint").is_dir() for profile in args.profiles):
if any(
(profile / ".fingerprint").is_dir() or (profile / "build").is_dir()
for profile in args.profiles
):
packages = {package["name"] for package in workspace_packages(Path.cwd())}
result["invalidated_workspace_units"] = invalidate_units(
args.profiles, packages, build_scripts_only=False
args.profiles,
packages,
build_scripts_only=False,
target=args.target_directory or Path.cwd() / "target",
)
if extra := args.extra_target_directory:
staged = args.manifest.with_name(f"target-{extra.name}")
Expand Down
7 changes: 7 additions & 0 deletions rust-workspace-cache/inputs/lifecycle.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,13 @@ test("rejects extra directories outside the target or used by Cargo", (context)
}
});

test("rejects Cargo profiles outside the target directory", () => {
const target = path.resolve("target");
for (const profile of [".", "../outside", path.resolve("outside")]) {
assert.throws(() => profileDirectories(target, profile), /within the target/);
}
});

test("does not save metadata when restoration fails", (context) => {
const result = lifecycle(context, true, 7);
assert.equal(result.status, 7);
Expand Down
Loading
Loading