From ef18f05911aff29c2507aa1b66296ae163e2054a Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 23 Oct 2026 12:16:30 +0000 Subject: [PATCH 1/9] adopt-project.py: the mount removes a path a case twin is at or under with `git rm -f`, and writes each twin the root keeps back from the index, so a disk that ignores case no longer loses a set the plan splits across repositories or drops a path of (#1183) Measured with git 2.43 at 4d40ce9f, on a FUSE disk that ignores case: the split's clone checks out one file per twin set, holding the bytes of the twin git wrote last. Where the twin that goes sorts last (`A` root, `a` code), `git rm a` deleted the one shared file and `execute` stopped with `adopt-lost` after both legs and the split were pushed; where it sorts first (`A` code, `a` root), `git rm A` refused with "local modifications" and `execute` stopped mounting the legs. `_split_case_twins` lists the assembly's index once and keeps each `_case_twin_sets` set with a path at or under a removed path. Each removed path with such a twin is removed with `GIT_RM_FORCED`; every other path keeps `GIT_RM`. After the last `git rm`, the set's paths the root keeps are written back with `git checkout-index -f -z --stdin`. A source with no split set gets the same `git rm` commands and nothing written back. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 115 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 103 insertions(+), 12 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index 89cb86da..81f05479 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -9278,8 +9278,10 @@ def _unsafe_path_findings(plan: Plan) -> tuple[list[str], list[str]]: # path git refuses is `plan-path-git-refuses`'s alone, and a path no entry # answers once is another finding's. Twins the plan sends to DIFFERENT # repositories, and a twin it drops beside one it keeps, are not asked -# either, though `execute` on a disk that ignores case loses them too, as -# `adopt-lost` after both legs are pushed (#1183). +# either: no repository the split makes holds two of them, and `execute` +# mounts the legs so that a disk that ignores case loses none of them +# (`_split_case_twins`, #1183), where it lost them as `adopt-lost`, or +# stopped at `git rm`, after both legs were pushed. # # A set of ONE NAME whose paths hold one blob in one mode loses nothing on # such a disk: the one path it checks out holds what each path holds, and @@ -9296,10 +9298,11 @@ def _unsafe_path_findings(plan: Plan) -> tuple[list[str], list[str]]: #: as `plan-case-twins` names it, in the order it is asked. TWIN_REPOSITORIES = {"root": "the assembly root", "spec": "the spec leg", "code": "the code leg"} -#: What a plan holding such twins is told to do, once after every finding. -#: It offers no answer that sends the twins of a set into different -#: repositories, or drops one: on a disk that ignores case `execute` loses -#: those too, after both legs are pushed (#1183). +#: What a plan holding such twins is told to do, once after every finding: +#: a rename in the source. It was written while an answer that sends the +#: twins of a set into different repositories, or drops all of a set but +#: one, was lost by `execute` on a disk that ignores case too, after both +#: legs were pushed; `_mount_the_legs` keeps those since #1183. CASE_TWINS_REMEDIATION = ( "Remediation: rename paths in the source until no two that one " "repository would hold differ only in case -- each twin of a set but " @@ -14301,22 +14304,98 @@ def _refuse_an_unmountable_path(assembly: Path, path: str) -> None: #: still a pathspec, where `*` and `?` are wildcards and a leading `:` is #: magic -- `git rm -- ':!keep.md'` removes every other path in the tree. GIT_RM = ("git", "--literal-pathspecs", "rm", "-r", "-q", "--") +#: `GIT_RM` with `-f`, for a path at or above a case twin (#1183): see +#: `_split_case_twins`. `-f` skips only git's question whether the file +#: on the disk still holds what the index holds. +GIT_RM_FORCED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "-f", "--") -def _git_rm(assembly: Path, path: str) -> None: +def _git_rm(assembly: Path, path: str, forced: bool = False) -> None: """`git rm` of `path`, an entry's path as the plan holds it, in the assembly: what `run` does, with `path` handed to git as the bytes the - tree holds (`_path_argument`, #527). + tree holds (`_path_argument`, #527). `forced` is `GIT_RM_FORCED`, for a + path a case twin is at or under (#1183). `run` takes only text (`check_program`), so this asks `check_program` of the command as the person reads it, and a git that refuses raises `CommandFailed` with that command, as `run` raises it: what `execute` reports after both legs exist is unchanged. """ - command = [*GIT_RM, path] + removal = GIT_RM_FORCED if forced else GIT_RM + command = [*removal, path] check_program(command) - proc = subprocess.run([*GIT_RM, _path_argument(path)], cwd=str(assembly), - capture_output=True, check=False) + proc = subprocess.run([*removal, _path_argument(path)], + cwd=str(assembly), capture_output=True, check=False) + if proc.returncode != 0: + raise CommandFailed(command, assembly, proc.returncode, + (proc.stderr + proc.stdout).decode("utf-8", + "replace")) + + +#: `git checkout-index`, writing each path it reads from its input, each +#: ended by a NUL, from the index over whatever the disk holds there. +CHECKOUT_INDEX = ("git", "checkout-index", "-f", "-z", "--stdin") + + +def _at_or_under(path: str, top: str) -> bool: + """Whether `path` is `top` or a path under it, both as git spells them: + case counts, as it does in the index.""" + return path == top or path.startswith(top + "/") + + +def _split_case_twins(assembly: Path, + removed: list[str]) -> list[list[str]]: + """Each set of case twins the assembly's index holds (`_case_twin_sets`) + that has a path at or under one of `removed`, the paths `_mount_the_legs` + takes out of the root: a set the plan sends into different repositories, + or drops a path of (#1183). + + A DISK THAT IGNORES CASE HOLDS EACH SUCH SET AS ONE FILE. Measured for + #1183 with git 2.43, on a FUSE disk that ignores case and keeps it: the + split's clone checks out one file per set, holding the bytes of the twin + git wrote last, in the index's order (`a` after `A`, `d/x` after `D/x`), + git saying `the following paths have collided`. The mount's `git rm` of + a twin then meets that file: + - where the twin that goes sorts last (`A` kept in the root, `a` sent to + the code leg or dropped, and `F/x` kept beside `f`), `git rm` deletes + the one file both names share, `git add -A` stages the kept twin's + deletion, and `execute` stopped after both legs and the split were + pushed, as `adopt-lost`; + - where it sorts first (`A` sent to a leg or dropped, `a` kept), `git + rm` refused, `the following file has local modifications: A`, and + `execute` stopped mounting the legs after both were pushed: also with + BOTH twins dropped, or one sent to each leg. + So each path such a set has at or under it is removed with `-f`, and + the paths of these sets that the root keeps are written back from the + index after the last `git rm`. + + The list is asked of the index, which holds the source commit's tree, + once, and on every disk: on a disk that keeps case, the same commands + rewrite the bytes each kept twin already holds. A source with no such + set, as nearly every one is, gets the `git rm` of each path it always + got and nothing written back: this listing, which writes nothing, is + the one command added.""" + raw = git_out(["ls-files", "-z"], cwd=assembly, binary=True) + paths = [_git_path(record) for record in raw.split(b"\x00") if record] + return [named for named in _case_twin_sets(paths) + if any(_at_or_under(twin, top) + for twin in named for top in removed)] + + +def _write_from_the_index(assembly: Path, paths: list[str]) -> None: + """`git checkout-index` of `paths`, which the assembly's index holds, + over whatever the disk holds at each (#1183): handed to git on its + input as the bytes the tree holds, each `_git_path` undone, so that no + locale and no length of a command line is in the way. + + `CommandFailed` where git refuses, naming the command as `_git_rm` + names its own; what git said names the path.""" + command = list(CHECKOUT_INDEX) + check_program(command) + proc = subprocess.run( + command, cwd=str(assembly), capture_output=True, check=False, + input=b"".join(path.encode("utf-8", "surrogateescape") + b"\x00" + for path in paths)) if proc.returncode != 0: raise CommandFailed(command, assembly, proc.returncode, (proc.stderr + proc.stdout).decode("utf-8", @@ -14333,13 +14412,25 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, Each path is the source tree's, so `_git_rm` hands it to git as the bytes the tree holds (#527). The mount paths are `checked_value`s, ASCII. + + CASE TWINS THE PLAN SPLITS (#1183): a path with a twin at or under it + is removed with `-f`, and each twin the root keeps is written back from + the index after the last `git rm` (`_split_case_twins`), so that a disk + that ignores case loses none of them. """ removed = sorted((p.rstrip("/") for p in (paths_for["spec"] + paths_for["code"] + paths_for["drop"])), key=lambda p: (p != GITMODULES, p)) + split = [twin for named in _split_case_twins(assembly, removed) + for twin in named] for path in removed: - _git_rm(assembly, path) + _git_rm(assembly, path, + any(_at_or_under(twin, path) for twin in split)) + kept = [twin for twin in split + if not any(_at_or_under(twin, path) for path in removed)] + if kept: + _write_from_the_index(assembly, kept) # The plan moved (or dropped) the source's own .gitmodules, so its deletion # is staged, and `git submodule add` refuses to write into a file the index # says is going away. Give the assembly a fresh, EMPTY, staged one for its From a455c75928fcb490290915f596231a947ddc4d80 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 23 Oct 2026 12:20:53 +0000 Subject: [PATCH 2/9] tests/test_adopt_check_throwaway_repos.py: rows pin that the mount keeps each case twin the root keeps with its own blob where the plan sends its twin to a leg or drops it, removes only such a path with `-f`, writes back only the kept twins, and that `execute` adopts such a split, whole, on a disk that ignores case (#1183) The mount rows make each set one file on any disk (hard links to the twin git writes last), as #1183's trace measured on a disk that ignores case. At head 10 passed, 4 skipped (the FUSE-only rows, on a disk that keeps case); at base 4d40ce9f 8 failed: the six mount rows (a wrong blob, or `CommandFailed` at `git rm -- A`), the `-f` command row and the write-back row. The no-split command row and the control pass at both. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_check_throwaway_repos.py | 274 +++++++++++++++++++++- 1 file changed, 266 insertions(+), 8 deletions(-) diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index 168e5fc9..63133fd3 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -46,6 +46,17 @@ of 4 fresh sources. Now `plan-case-twins` names each set the plan keeps in one repository, the root or a leg, on every disk, and `execute` refuses it before any leg exists. + +#1183: twins the plan sends into different repositories, or drops one of, +are no finding, and `execute` lost them on such a disk. Measured with git +2.43 on 4d40ce9f: the split's clone checks out one file per set, holding the +bytes of the twin git wrote last, and the mount's `git rm` of the twin that +goes deleted that file (`adopt-lost` for the twin the root keeps) or refused +it as a local modification (`REFUSED mounting the legs`), after both legs +were pushed. Now the mount removes such a path with `-f` and writes each +twin the root keeps back from the index. The rows that drive the mount make +that one file on any disk, as hard links; the rows that run `execute` whole +skip on a disk that keeps case, but for one control. """ from __future__ import annotations @@ -62,8 +73,8 @@ import pytest from conftest import (ADOPT, WINDOWS_SKIP, appended_to_command_scope, - findings, git, planned, refused_before_any_leg, - run_script) + findings, git, make_source_repo, planned, + refused_before_any_leg, run_script) from test_adopt_nested_attributes import (KEEP_GITHUB, ROOT, ROOT_ATTRIBUTES, SOURCE, YML_CRLF, adoption_plan, committed_raw, github) @@ -500,9 +511,10 @@ def test_check_names_each_twin_git_refuses_and_each_set_it_holds(tmp_path): # --------------------------------------------------------------------------- #: The finding #1161 adds, the code of `execute`'s refusal, how its -#: remediation begins, and the whole of it: a rename in the source, and no -#: answer that sends the twins into different repositories, which a disk -#: that ignores case loses too (#1183, r1189's MUST 1). +#: remediation begins, and the whole of it: a rename in the source. It +#: offers no answer that sends the twins into different repositories +#: (r1189's MUST 1), written while a disk that ignores case lost those too; +#: `execute` keeps them since #1183 (the rows under #1183 below). PLAN_CASE_TWINS = "plan-case-twins" TWINS_REMEDY = "Remediation: rename paths in the source until no two" TWINS_REMEDIATION = ( @@ -798,9 +810,9 @@ def test_paths_no_one_repository_holds_as_one_name_are_no_finding( """Folders in another case merge into one whose names differ, and no path is lost. Twins the plan sends to different repositories, and a twin it drops beside one it keeps, are not this finding, which asks one - repository at a time; on a disk that ignores case `execute` loses them - too, as `adopt-lost` after both legs are pushed (r1189's SHOULD 3, - #1183).""" + repository at a time (r1189's SHOULD 3). On a disk that ignores case + `execute` lost them, as `adopt-lost` after both legs were pushed, until + #1183: it keeps them now (the rows under #1183 below).""" source = a_source(tmp_path) with_paths(source, paths, own_blobs=True) assert findings(planned(source, tmp_path, answers)) == [] @@ -851,6 +863,252 @@ def test_on_a_disk_that_ignores_case_check_names_the_twins_too(tmp_path): twins_kept("1 set", ROOT_TWINS, "2 paths", "`A` and `a`")] +# --------------------------------------------------------------------------- +# #1183: case twins the plan sends into different repositories, or drops +# --------------------------------------------------------------------------- + +#: The branch `_commit_the_split` checks the assembly out on. +SPLIT_BRANCH = "adopt/three-repo-shape" +#: The two legs as `_mount_the_legs` finds them in the work directory. +LEG_NAMES = {"spec": "Northwind-spec", "code": "Northwind-code"} +#: What the mount adds to the root's index: its two gitlinks and the file +#: that registers them. +MOUNTED = (".gitmodules", "spec", "code") +#: `git rm` of one path, as `_mount_the_legs` runs it, and with `-f`. +REMOVED_AS = ["git", "--literal-pathspecs", "rm", "-r", "-q", "--"] +FORCED_AS = ["git", "--literal-pathspecs", "rm", "-r", "-q", "-f", "--"] +#: How the mount writes the twins the root keeps back from the index. +WRITTEN_AS = ["git", "checkout-index", "-f", "-z", "--stdin"] + + +def at_or_under(path: str, top: str) -> bool: + return path == top or path.startswith(top + "/") + + +def an_assembly(base: Path, paths: tuple[str, ...]) -> tuple[Path, dict]: + """`(assembly, blobs)`: `a_source` whose commit gains `paths`, each with + a blob of its own, cloned into `base/work` as `_commit_the_split` clones + it, with no template and the split branch checked out at that commit, + beside the two legs; `blobs` holds each path of that commit's tree and + its blob.""" + source = a_source(base) + with_paths(source, paths, own_blobs=True) + commit = git("rev-parse", "HEAD", cwd=source).stdout.strip() + work = base / "work" + assembly = work / "Northwind" + git("clone", "-q", "--template=", str(source), str(assembly), cwd=base) + git("checkout", "-q", "-B", SPLIT_BRANCH, commit, cwd=assembly) + for role, name in LEG_NAMES.items(): + make_source_repo(work / name, tree={"file.txt": role}, edits=()) + rows = git("ls-tree", "-r", "-z", commit, cwd=source).stdout + return assembly, {row.split("\t", 1)[1]: row.split()[2] + for row in rows.split("\0") if row} + + +def as_one_file(assembly: Path, sets: tuple[tuple[str, ...], ...]) -> None: + """What a clone on a disk that ignores case checks out of each of + `sets`, made on a disk that keeps case: ONE FILE, holding the bytes of + the twin git wrote last, the last of the set in the index's order, as + #1183's trace measured. Each other twin is made a hard link to that + twin's file, so the twins share one file as they do there. Where the + clone's own disk ignores case, git did this itself, and nothing is + done.""" + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + return + for named in sets: + *others, last = named + for twin in others: + (assembly / twin).unlink() + os.link(assembly / last, assembly / twin) + + +def mounted(adopter, base: Path, assembly: Path, moved: dict) -> None: + """`_mount_the_legs` of `assembly`, `moved` its `paths_for`, then the + `git add -A -- .` that `_stage_the_split` begins with.""" + work = base / "work" + adopter._mount_the_legs( + assembly, work, LEG_NAMES, + {role: str(work / name) for role, name in LEG_NAMES.items()}, + {"spec": [], "code": [], "drop": [], **moved}, "spec", "code") + git("add", "-A", "--", ".", cwd=assembly) + + +def staged(assembly: Path) -> dict: + """Each path the assembly's index holds but the mount's own, and its + object.""" + rows = git("ls-files", "-s", "-z", cwd=assembly).stdout.split("\0") + held = {row.split("\t", 1)[1]: row.split()[1] for row in rows if row} + return {path: oid for path, oid in held.items() if path not in MOUNTED} + + +#: `(the paths a source commit gains, each with a blob of its own, the sets +#: of them a disk that ignores case holds as one, each in the index's +#: order, and the paths the plan sends to a leg or drops, by `leg:`)`. +#: Measured on 4d40ce9f on such a disk, `execute` stopped after both legs +#: were pushed: as `adopt-lost` for the twin the root keeps where the twin +#: that goes sorts last (`across`, `dropped`, `a-folder-kept-beside-a-file`), +#: and `REFUSED mounting the legs` at `git rm` where it sorts first. +SPLIT_TWINS = [ + pytest.param(("A", "a", "D/x", "d/x"), (("A", "a"), ("D/x", "d/x")), + {"code": ["a", "d/"]}, id="across"), + pytest.param(("A", "a", "D/x", "d/x"), (("A", "a"), ("D/x", "d/x")), + {"code": ["A", "D/"]}, id="across-the-other-way"), + pytest.param(("A", "a"), (("A", "a"),), {"drop": ["a"]}, id="dropped"), + pytest.param(("A", "a"), (("A", "a"),), {"drop": ["A", "a"]}, + id="both-dropped"), + pytest.param(("A", "a"), (("A", "a"),), {"spec": ["A"], "code": ["a"]}, + id="one-to-each-leg"), + pytest.param(("F/x", "f"), (("F/x", "f"),), {"code": ["f"]}, + id="a-folder-kept-beside-a-file"), +] + + +@pytest.mark.parametrize(("paths", "sets", "moved"), SPLIT_TWINS) +def test_the_mount_keeps_each_twin_the_root_keeps_with_its_own_blob( + adopter, tmp_path, paths, sets, moved): + """The twins share one file, as on a disk that ignores case. On + 4d40ce9f the mount's `git rm` refused that file as a local + modification where the twin that goes sorts first, and elsewhere the + root kept the other twin's bytes (on such a disk, none: the one file + was gone). Now the root's index holds each path the plan keeps there, + each with its own blob, and no path the plan moved or dropped.""" + assembly, blobs = an_assembly(tmp_path, paths) + as_one_file(assembly, sets) + mounted(adopter, tmp_path, assembly, moved) + removed = [path.rstrip("/") for leg in moved.values() for path in leg] + assert staged(assembly) == { + path: oid for path, oid in blobs.items() + if not any(at_or_under(path, top) for top in removed)} + + +def git_commands_of_the_mount(adopter, monkeypatch, base: Path, + assembly: Path, moved: dict) -> list: + """`(command, input)` of each `git rm` and `git checkout-index` that + `mounted` runs, in order, each command as text.""" + ran = [] + real = subprocess.run + + def recorded(command, *args, **named): + words = [os.fsdecode(word) for word in command] + if "rm" in words[:3] or "checkout-index" in words[:2]: + ran.append((words, named.get("input"))) + return real(command, *args, **named) + + monkeypatch.setattr(subprocess, "run", recorded) + mounted(adopter, base, assembly, moved) + monkeypatch.undo() + return ran + + +def test_a_source_with_no_twin_set_split_is_mounted_as_before( + adopter, monkeypatch, tmp_path): + """`AB` and `ab` are twins the root keeps both of, and `a` goes to the + code leg: no set has a path at or under a path that goes (`a` is no + folder above `ab`), so each path is removed as on 4d40ce9f, without + `-f`, and nothing is written back from the index.""" + assembly, _ = an_assembly(tmp_path, ("AB", "a", "ab", "b/x")) + assert git_commands_of_the_mount( + adopter, monkeypatch, tmp_path, assembly, + {"code": ["a"], "drop": ["b/"]}) == [ + ([*REMOVED_AS, "a"], None), ([*REMOVED_AS, "b"], None)] + + +def test_only_a_path_a_twin_is_at_or_under_is_removed_with_f( + adopter, monkeypatch, tmp_path): + """`a` and `b` go to the code leg, and `A`, the twin of `a`, stays in + the root: `a` alone is removed with `-f`, and `A` alone is written back + from the index, after the last `git rm`. On 4d40ce9f neither was.""" + assembly, _ = an_assembly(tmp_path, ("A", "a", "b")) + assert git_commands_of_the_mount( + adopter, monkeypatch, tmp_path, assembly, {"code": ["a", "b"]}) == [ + ([*FORCED_AS, "a"], None), ([*REMOVED_AS, "b"], None), + (WRITTEN_AS, b"A\0")] + + +def test_a_path_git_cannot_write_back_is_refused_naming_the_command( + adopter, tmp_path): + """`_write_from_the_index` of a path the index does not hold: git + refuses it, and the mount raises `CommandFailed`, one of + `LATE_FAILURES`, naming the command and carrying what git said, as + `_git_rm` does.""" + assembly, _ = an_assembly(tmp_path, ("A",)) + with pytest.raises(adopter.CommandFailed) as caught: + adopter._write_from_the_index(assembly, ["nowhere"]) + assert caught.value.args_list == WRITTEN_AS + assert caught.value.code != 0 + assert "nowhere" in caught.value.output + + +needs_filter_repo = pytest.mark.skipif( + shutil.which("git-filter-repo") is None, + reason="git filter-repo is not installed: `pip install git-filter-repo`") + + +def executed(base: Path, paths: tuple[str, ...], answers: dict): + """`execute` of a plan for `a_source` whose commit gains `paths`, each + with a blob of its own, `answers` given, with its legs under + `--local-remote-dir` and its work directory in `base`.""" + source = a_source(base) + with_paths(source, paths, own_blobs=True) + plan = planned(source, base, answers) + return run_script(ADOPT, "execute", "--plan", str(plan), "--yes", + "--local-remote-dir", str(base / "remotes"), + "--work-dir", str(base / "work")) + + +def assert_adopted(result) -> None: + assert result.returncode == 0, result.stderr + result.stdout + assert "adoption verified" in result.stdout, result.stdout + + +#: `(the paths, the plan's answers)`, run whole. On 4d40ce9f, on a disk +#: that ignores case, `execute` exited 1 with `adopt-lost` for `A` and +#: `D/x` (`across`) and `A` (`dropped`), and 2 with `REFUSED mounting the +#: legs` (`across-the-other-way`, `one-to-each-leg`), each after both legs +#: and, for `adopt-lost`, the split were pushed. +SPLIT_EXECUTED = [ + pytest.param(("A", "a", "D/x", "d/x"), + {"A": ROOT, "a": "code", "D/": ROOT, "d/": "code"}, + id="across"), + pytest.param(("A", "a", "D/x", "d/x"), + {"A": "code", "a": ROOT, "D/": "code", "d/": ROOT}, + id="across-the-other-way"), + pytest.param(("A", "a"), {"A": ROOT, "a": "drop"}, id="dropped"), + pytest.param(("A", "a"), {"A": "spec", "a": "code"}, + id="one-to-each-leg"), +] + + +@needs_filter_repo +@pytest.mark.parametrize(("paths", "answers"), SPLIT_EXECUTED) +def test_on_a_disk_that_ignores_case_execute_keeps_twins_it_splits( + tmp_path, paths, answers): + """`execute` whole, its work directory and its legs on the disk `git + init` is asked about; skipped where that disk keeps case, as every CI + disk does (CI is Linux only since #597), where the rows above make the + one file themselves.""" + probe = tmp_path / "probe" + git("init", "-q", str(probe), cwd=tmp_path) + if git("config", "--local", "--get", "core.ignorecase", cwd=probe, + check=False).stdout.strip() != "true": + pytest.skip("needs a disk that ignores case: `git init` wrote no " + "`core.ignorecase=true` in the work directory, so that " + "disk keeps case; the rows that drive the mount make " + "each set one file on every disk") + assert_adopted(executed(tmp_path, paths, answers)) + + +@needs_filter_repo +def test_execute_adopts_twins_it_sends_to_different_repositories(tmp_path): + """The control, on any disk: `A` and `D/x` kept in the root, `a` and + `d/x` sent to the code leg, `execute` whole. On a disk that keeps case + it was adopted on 4d40ce9f too, and it still is.""" + assert_adopted(executed(tmp_path, ("A", "a", "D/x", "d/x"), + {"A": ROOT, "a": "code", "D/": ROOT, + "d/": "code"})) + + # --------------------------------------------------------------------------- # L1-92: in the work directory # --------------------------------------------------------------------------- From c9f0cc185d4483c766f2ed6f112832ada389818a Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 23 Oct 2026 21:30:46 +0000 Subject: [PATCH 3/9] tests/test_adopt_check_throwaway_repos.py: rows pin that a forced `git rm` git refuses is named with `-f`, and that the mount keeps case twins whose names are not UTF-8 with their own blobs, reading the index and handing the kept twin back as bytes (#1183) Mutants of ef18f059's new lines left three that change what the tool does alive: `_git_rm` naming `GIT_RM` in its `CommandFailed` where it ran `GIT_RM_FORCED`, and `_split_case_twins` decoding the listing, or `_write_from_the_index` encoding the names, as strict UTF-8, which raises a `UnicodeError` mounting the legs, after both are pushed. `z_rows` reads git's NUL-ended rows as `_git_path` reads a path, so `an_assembly` and `staged` hold a name that is not UTF-8; `an_assembly` takes the function that adds the paths, and `with_names_not_utf8` adds them through the index, as `with_paths` hands `git mktree` text. The row skips where the disk takes no such name. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_check_throwaway_repos.py | 91 ++++++++++++++++++++--- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index 92ab141b..447273fb 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -966,14 +966,25 @@ def at_or_under(path: str, top: str) -> bool: return path == top or path.startswith(top + "/") -def an_assembly(base: Path, paths: tuple[str, ...]) -> tuple[Path, dict]: - """`(assembly, blobs)`: `a_source` whose commit gains `paths`, each with - a blob of its own, cloned into `base/work` as `_commit_the_split` clones - it, with no template and the split branch checked out at that commit, - beside the two legs; `blobs` holds each path of that commit's tree and - its blob.""" +def z_rows(repository: Path, *args: str) -> list[str]: + """The rows `git ` prints in `repository`, each ended by a NUL, + decoded as `_git_path` decodes a path: a byte that is not UTF-8 kept as + a surrogate, which `git`'s text would refuse.""" + raw = subprocess.run(["git", *args], cwd=str(repository), + capture_output=True, check=True).stdout + return [row.decode("utf-8", "surrogateescape") + for row in raw.split(b"\0") if row] + + +def an_assembly(base: Path, paths: tuple[str, ...], + add=with_paths) -> tuple[Path, dict]: + """`(assembly, blobs)`: `a_source` whose commit gains `paths` by `add`, + each with a blob of its own, cloned into `base/work` as + `_commit_the_split` clones it, with no template and the split branch + checked out at that commit, beside the two legs; `blobs` holds each + path of that commit's tree and its blob.""" source = a_source(base) - with_paths(source, paths, own_blobs=True) + add(source, paths, own_blobs=True) commit = git("rev-parse", "HEAD", cwd=source).stdout.strip() work = base / "work" assembly = work / "Northwind" @@ -981,9 +992,9 @@ def an_assembly(base: Path, paths: tuple[str, ...]) -> tuple[Path, dict]: git("checkout", "-q", "-B", SPLIT_BRANCH, commit, cwd=assembly) for role, name in LEG_NAMES.items(): make_source_repo(work / name, tree={"file.txt": role}, edits=()) - rows = git("ls-tree", "-r", "-z", commit, cwd=source).stdout return assembly, {row.split("\t", 1)[1]: row.split()[2] - for row in rows.split("\0") if row} + for row in z_rows(source, "ls-tree", "-r", "-z", + commit)} def as_one_file(assembly: Path, sets: tuple[tuple[str, ...], ...]) -> None: @@ -1018,8 +1029,8 @@ def mounted(adopter, base: Path, assembly: Path, moved: dict) -> None: def staged(assembly: Path) -> dict: """Each path the assembly's index holds but the mount's own, and its object.""" - rows = git("ls-files", "-s", "-z", cwd=assembly).stdout.split("\0") - held = {row.split("\t", 1)[1]: row.split()[1] for row in rows if row} + held = {row.split("\t", 1)[1]: row.split()[1] + for row in z_rows(assembly, "ls-files", "-s", "-z")} return {path: oid for path, oid in held.items() if path not in MOUNTED} @@ -1121,6 +1132,64 @@ def test_a_path_git_cannot_write_back_is_refused_naming_the_command( assert "nowhere" in caught.value.output +def test_a_forced_git_rm_git_refuses_is_refused_naming_it_with_f( + adopter, tmp_path): + """`_git_rm` with `forced` of a path the index does not hold: git + refuses it, and the mount raises `CommandFailed` naming the command it + ran, with `-f`, as `execute` prints it after both legs exist.""" + assembly, _ = an_assembly(tmp_path, ("A",)) + with pytest.raises(adopter.CommandFailed) as caught: + adopter._git_rm(assembly, "nowhere", True) + assert caught.value.args_list == [*FORCED_AS, "nowhere"] + assert caught.value.code != 0 + assert "nowhere" in caught.value.output + + +#: Case twins whose names hold the byte E9, which is not UTF-8, each spelled +#: as `_git_path` spells it: that byte as the surrogate U+DCE9. +NOT_UTF8_TWINS = ("caf\udce9A", "caf\udce9a") + + +def with_names_not_utf8(repository: Path, paths: tuple[str, ...], + own_blobs: bool = True) -> None: + """`with_paths` for names that are not UTF-8, which it hands `git + mktree` as text: each of `paths` goes into the index as the bytes it + spells, with a blob of its own holding those bytes and a newline, and + the index is committed on `main`.""" + assert own_blobs + for path in paths: + name = path.encode("utf-8", "surrogateescape") + blob = blob_of(repository, name + b"\n") + subprocess.run(["git", "update-index", "--add", "--cacheinfo", + b"100644," + blob.encode("ascii") + b"," + name], + cwd=str(repository), check=True) + git("commit", "-qm", "Add paths", cwd=repository) + + +def test_twins_whose_names_are_not_utf_8_keep_their_own_blobs( + adopter, tmp_path): + """`cafA` kept in the root and `cafa` sent to the code leg, the + byte E9 not UTF-8, sharing one file: the mount reads the index's names + as bytes, and hands the twin it writes back to git as the bytes the + tree holds, so the root's index holds that twin with its own blob. A + listing decoded, or a name encoded, as strict UTF-8 raises a + `UnicodeError` there, after both legs are pushed.""" + try: + os.close(os.open(os.fsencode(tmp_path) + b"/caf\xe9", + os.O_CREAT | os.O_WRONLY, 0o644)) + except (OSError, ValueError): + pytest.skip("this disk, or this platform, takes no file name that " + "is not UTF-8") + kept, moved = NOT_UTF8_TWINS + assembly, blobs = an_assembly(tmp_path, NOT_UTF8_TWINS, + with_names_not_utf8) + as_one_file(assembly, (NOT_UTF8_TWINS,)) + mounted(adopter, tmp_path, assembly, {"code": [moved]}) + assert kept in blobs + assert staged(assembly) == {path: oid for path, oid in blobs.items() + if path != moved} + + needs_filter_repo = pytest.mark.skipif( shutil.which("git-filter-repo") is None, reason="git filter-repo is not installed: `pip install git-filter-repo`") From 0b8252c35abb55bd15b01ab95dcf1ec01a45c8c4 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 8 Nov 2026 13:50:48 +0000 Subject: [PATCH 4/9] adopt-project.py: before the mount's `git rm -f`, each path the index holds beyond a symlink on the disk leaves the index alone (`git rm --cached`), so on a disk that ignores case no file is removed through a symlink a case twin left (#1183) #1237's review (r1237, MUST 1): on a disk that ignores case, a folder `D/` the plan sends to a leg (or drops) and a symlink `d` it keeps are one name, and the split's clone writes `d` last, so `D` is that symlink. The forced `git rm -r -q -f -- D` then unlinked `D/x` through it: the file `x` in the folder the symlink leads to, outside the work directory, and `execute` printed `adoption verified`. Without `-f`, git refused it. Now `_mount_the_legs` lists the index once (`_index_paths`), and before each removal with `-f` asks each path the index holds at or under it, folder by folder from the top, with `lstat` (`_beyond_a_symlink`, through `_is_a_symlink`, which asks as `_is_a` asks: nothing there is no symlink, and a refused `lstat` is raised, one of `LATE_FAILURES`). A path beyond a symlink is taken out of the index alone (`GIT_RM_CACHED`, `_taken_beyond_a_symlink`): the symlink's own path where it is at or under the removed path, else the path. The rest are removed with `-f` as before; where nothing is left, no `-f` runs. `--cached` rather than a refusal: it touches no file on the disk, git counts a path beyond a symlink as gone from the work tree, `git add -A` does not follow a symlink into a folder, and the symlink is the twin the root keeps, written back from the index, so the root's tree is the plan's and each leg's is the objects'. A refusal would come after both legs are pushed, for a plan that is good on every disk. Rows (section #1183): `test_no_file_is_removed_through_a_symlink_a_case_twin_left` (7 shapes: the probe's absolute link, `../../victim`, the folder dropped, a link to a folder the root keeps, both in a dropped folder beside other files, two files beyond the link, entries below the link), the control `test_a_folder_that_is_no_symlink_is_still_removed_with_f`, `test_the_first_symlink_on_the_way_is_named_and_nothing_else` (6) and `test_a_folder_on_the_way_nobody_may_search_is_raised_not_read_as_none`. At c9f0cc18 the 7 symlink shapes fail (the folder the link leads to lost `x`, or `E/x`); the control passes at both. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 140 ++++++++++++++-- tests/test_adopt_check_throwaway_repos.py | 193 ++++++++++++++++++++++ 2 files changed, 315 insertions(+), 18 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index 13579470..00621758 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -1884,6 +1884,25 @@ def _is_there(path: Path) -> bool: return _is_a(path, lambda _mode: True) +def _is_a_symlink(path: str | bytes) -> bool: + """Whether `path` is itself a symlink, asked with `os.lstat`, which + follows none, as `_is_a` asks `stat` (#430): nothing there is False, + and the `OSError` of an `lstat` the operating system REFUSED is raised, + never read as nothing there. NOT `Path.is_symlink()`, which from 3.14 + answers False to ANY `OSError`. `path` is a `str` or the bytes + `_path_argument` hands git, so that no locale is in the way (#527).""" + try: + mode = os.lstat(path).st_mode + except OSError as exc: + if _absent(exc): + return False + raise + except ValueError: + # a name the operating system cannot take at all (a NUL in it) + return False + return stat.S_ISLNK(mode) + + def _os_reason(exc: OSError) -> str: """The operating system's own words for `exc` (`Permission denied`), or the exception's name where it gave none, as `validate-pins.py`'s @@ -14333,23 +14352,35 @@ def _refuse_an_unmountable_path(assembly: Path, path: str) -> None: #: magic -- `git rm -- ':!keep.md'` removes every other path in the tree. GIT_RM = ("git", "--literal-pathspecs", "rm", "-r", "-q", "--") #: `GIT_RM` with `-f`, for a path at or above a case twin (#1183): see -#: `_split_case_twins`. `-f` skips only git's question whether the file -#: on the disk still holds what the index holds. +#: `_split_case_twins`. `-f` skips git's question whether the file on the +#: disk still holds what the index holds, and that question was also what +#: kept git from removing a file through a symlink: given a path beyond a +#: symlink the disk holds on the way to it, `git rm -f` unlinks the file +#: the symlink leads to, wherever that is (r1237). So it is never given +#: one: `_taken_beyond_a_symlink` takes each such path first. GIT_RM_FORCED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "-f", "--") +#: `GIT_RM` with `--cached`, for a path the index holds beyond a symlink +#: the disk holds on the way to it: git takes it out of the index and +#: touches no file on the disk (r1237). See `_taken_beyond_a_symlink`. +GIT_RM_CACHED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "--cached", + "--") -def _git_rm(assembly: Path, path: str, forced: bool = False) -> None: +def _git_rm(assembly: Path, path: str, forced: bool = False, + cached: bool = False) -> None: """`git rm` of `path`, an entry's path as the plan holds it, in the assembly: what `run` does, with `path` handed to git as the bytes the tree holds (`_path_argument`, #527). `forced` is `GIT_RM_FORCED`, for a - path a case twin is at or under (#1183). + path a case twin is at or under (#1183), and `cached` `GIT_RM_CACHED`, + for one the index holds beyond a symlink (r1237). `run` takes only text (`check_program`), so this asks `check_program` of the command as the person reads it, and a git that refuses raises `CommandFailed` with that command, as `run` raises it: what `execute` reports after both legs exist is unchanged. """ - removal = GIT_RM_FORCED if forced else GIT_RM + removal = GIT_RM_CACHED if cached else GIT_RM_FORCED if forced \ + else GIT_RM command = [*removal, path] check_program(command) proc = subprocess.run([*removal, _path_argument(path)], @@ -14371,12 +14402,20 @@ def _at_or_under(path: str, top: str) -> bool: return path == top or path.startswith(top + "/") -def _split_case_twins(assembly: Path, +def _index_paths(assembly: Path) -> list[str]: + """Each path the assembly's index holds, in its order, as `_git_path` + spells it: `git ls-files -z`, which writes nothing.""" + raw = git_out(["ls-files", "-z"], cwd=assembly, binary=True) + return [_git_path(record) for record in raw.split(b"\x00") if record] + + +def _split_case_twins(held: list[str], removed: list[str]) -> list[list[str]]: - """Each set of case twins the assembly's index holds (`_case_twin_sets`) - that has a path at or under one of `removed`, the paths `_mount_the_legs` - takes out of the root: a set the plan sends into different repositories, - or drops a path of (#1183). + """Each set of case twins of `held`, the paths the assembly's index + holds (`_index_paths`, `_case_twin_sets`), that has a path at or under + one of `removed`, the paths `_mount_the_legs` takes out of the root: a + set the plan sends into different repositories, or drops a path of + (#1183). A DISK THAT IGNORES CASE HOLDS EACH SUCH SET AS ONE FILE. Measured for #1183 with git 2.43, on a FUSE disk that ignores case and keeps it: the @@ -14401,15 +14440,75 @@ def _split_case_twins(assembly: Path, once, and on every disk: on a disk that keeps case, the same commands rewrite the bytes each kept twin already holds. A source with no such set, as nearly every one is, gets the `git rm` of each path it always - got and nothing written back: this listing, which writes nothing, is + got and nothing written back: that listing, which writes nothing, is the one command added.""" - raw = git_out(["ls-files", "-z"], cwd=assembly, binary=True) - paths = [_git_path(record) for record in raw.split(b"\x00") if record] - return [named for named in _case_twin_sets(paths) + return [named for named in _case_twin_sets(held) if any(_at_or_under(twin, top) for twin in named for top in removed)] +def _beyond_a_symlink(assembly: Path, path: str) -> str | None: + """The first folder on the way to `path`, a path the assembly's index + holds, from the top, that the disk holds as a symlink, as git spells + it; None where there is none (r1237). Each folder is asked with + `lstat` (`_is_a_symlink`), handed as the bytes the tree holds + (`_path_argument`). `path` itself is not asked: `git rm` of a symlink + unlinks the symlink, never what it leads to.""" + parts = path.split("/") + for end in range(1, len(parts)): + above = "/".join(parts[:end]) + here = _path_argument(above) + if _is_a_symlink(os.path.join(os.fsencode(assembly), here) + if isinstance(here, bytes) + else os.path.join(assembly, here)): + return above + return None + + +def _taken_beyond_a_symlink(assembly: Path, top: str, + held: list[str]) -> bool: + """`git rm --cached` (`GIT_RM_CACHED`) of each path of `held`, the + paths the assembly's index holds, at or under `top` that the disk holds + beyond a symlink (`_beyond_a_symlink`), asked before `top` is removed + with `-f`: of the symlink's own path where that is at or under `top`, + which takes each path beyond it at once, else of the path. Whether that + took every path `held` at or under `top`, leaving none for `-f`. + + WHERE A FOLDER ON THE WAY IS A SYMLINK (#1237's review, r1237). A disk + that ignores case holds a folder `D/` and a symlink `d` as one name, + and the split's clone writes `d` last, so `D` is that symlink. Measured + with git 2.43 on a FUSE disk that ignores case, for a plan that sends + `D/` to the code leg, or drops it, and keeps `d` in the root, `check` + printing `plan ok`: `git rm -f -- D` unlinked `D/x` through the + symlink, the file `x` in the folder it leads to, outside the work + directory for `d` leading to an absolute path or to `../../victim`, + and `execute` printed `adoption verified`; for `d` leading to `E`, a + folder the root keeps, `E/x` was gone, and `execute` stopped as + `adopt-lost` after both legs and the split were pushed. Without `-f`, + git refused `D/x` as a local modification. So such a path is taken out + of the index alone, and no file on the disk is touched: git counts a + path beyond a symlink as gone from the work tree, `git add -A` does not + follow a symlink into a folder, and the symlink is the twin the root + keeps, written back from the index. The root's tree is the plan's, as + is each leg's, which `git filter-repo` made from the objects. + + REFUSING THERE WAS REJECTED: it would come after both legs are pushed, + for a plan that is good on every disk, and a re-run then meets + `leg-remote-exists`.""" + under = [path for path in held if _at_or_under(path, top)] + taken: list[str] = [] + for path in under: + if any(_at_or_under(path, gone) for gone in taken): + continue + link = _beyond_a_symlink(assembly, path) + if link is not None: + gone = link if _at_or_under(link, top) else path + _git_rm(assembly, gone, cached=True) + taken.append(gone) + return bool(taken) and all(any(_at_or_under(path, gone) + for gone in taken) for path in under) + + def _write_from_the_index(assembly: Path, paths: list[str]) -> None: """`git checkout-index` of `paths`, which the assembly's index holds, over whatever the disk holds at each (#1183): handed to git on its @@ -14444,17 +14543,22 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, CASE TWINS THE PLAN SPLITS (#1183): a path with a twin at or under it is removed with `-f`, and each twin the root keeps is written back from the index after the last `git rm` (`_split_case_twins`), so that a disk - that ignores case loses none of them. + that ignores case loses none of them. Before that `-f`, each path the + index holds at or under it beyond a symlink the disk holds on the way + is taken out of the index alone (`_taken_beyond_a_symlink`, r1237), so + that `-f` removes no file through a symlink. """ removed = sorted((p.rstrip("/") for p in (paths_for["spec"] + paths_for["code"] + paths_for["drop"])), key=lambda p: (p != GITMODULES, p)) - split = [twin for named in _split_case_twins(assembly, removed) + held = _index_paths(assembly) + split = [twin for named in _split_case_twins(held, removed) for twin in named] for path in removed: - _git_rm(assembly, path, - any(_at_or_under(twin, path) for twin in split)) + forced = any(_at_or_under(twin, path) for twin in split) + if not (forced and _taken_beyond_a_symlink(assembly, path, held)): + _git_rm(assembly, path, forced) kept = [twin for twin in split if not any(_at_or_under(twin, path) for path in removed)] if kept: diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index 447273fb..821a4a24 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -83,6 +83,8 @@ from test_adopt_root_symlinks import links_as_links # noqa: F401 from test_adopt_root_symlinks import unwritable_work # noqa: F401 from test_adopt_root_symlinks import planned as planned_with_links +#: The one skip for a mode that root reads through and Windows ignores. +from test_validate_pins_unreadable_workflow import CANNOT_RUN_AS_ROOT #: The two findings #910 adds, and the codes of `execute`'s refusals. PATH_GIT_REFUSES = "plan-path-git-refuses" @@ -1190,6 +1192,197 @@ def test_twins_whose_names_are_not_utf_8_keep_their_own_blobs( if path != moved} +#: `git rm` of one path from the index alone, as the mount takes a path the +#: disk holds beyond a symlink (r1237). +CACHED_AS = ["git", "--literal-pathspecs", "rm", "-r", "-q", "--cached", + "--"] + + +def with_a_link(target: str): + """An `add` for `an_assembly`: each of `paths` goes into the source + commit with a blob of its own, as `with_paths` writes it, but a path + whose last name is `d`, which is a symlink leading to `target`.""" + def add(repository: Path, paths: tuple[str, ...], + own_blobs: bool = True) -> None: + assert own_blobs + for path in paths: + mode, data = (("120000", target.encode()) if path.split("/")[-1] + == "d" else ("100644", f"{path}\n".encode())) + git("update-index", "--add", "--cacheinfo", + f"{mode},{blob_of(repository, data)},{path}", cwd=repository) + git("commit", "-qm", "Add paths", cwd=repository) + return add + + +def as_the_link(assembly: Path, folder: str, link: str) -> None: + """What a clone on a disk that ignores case checks out of the folder + `folder` and its twin `link`, a symlink that sorts after it: the + symlink alone, which both names reach, as r1237 measured. Made on a + disk that keeps case: the folder gives way to a symlink leading where + `link` leads. Where the clone's own disk ignores case, git did this + itself, and nothing is done.""" + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + return + for child in (assembly / folder).iterdir(): + child.unlink() + (assembly / folder).rmdir() + os.symlink(os.readlink(assembly / link), assembly / folder) + + +def what_it_holds(folder: Path) -> dict: + return {child.name: child.read_bytes() for child in folder.iterdir()} + + +#: `(the paths a source commit gains, a path named `d` among them the +#: symlink, where it leads, `{victim}` being a folder outside the assembly, +#: the paths the plan sends to a leg or drops, the `git rm` and `git +#: checkout-index` the mount runs)`. On c9f0cc18 the mount ran `git rm -f` +#: of the folder `D` (or `src`) and unlinked `x` in the folder the symlink +#: leads to. +BEYOND_A_LINK = [ + pytest.param(("D/x", "d"), "{victim}", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-link-out-of-the-assembly"), + pytest.param(("D/x", "d"), "../../victim", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-relative-link-out-of-the-assembly"), + pytest.param(("D/x", "d"), "{victim}", {"drop": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="the-folder-dropped"), + pytest.param(("D/x", "E/x", "d"), "E", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-link-to-a-folder-the-root-keeps"), + pytest.param(("src/D/x", "src/d"), "{victim}", {"drop": ["src/"]}, + [([*CACHED_AS, "src/D"], None), ([*FORCED_AS, "src"], None)], + id="both-in-a-dropped-folder-beside-other-files"), + pytest.param(("D/x", "D/y", "d"), "{victim}", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="two-files-beyond-the-link-taken-at-once"), + pytest.param(("D/x", "D/y", "d"), "{victim}", + {"code": ["D/x"], "drop": ["D/y"]}, + [([*CACHED_AS, "D/x"], None), ([*CACHED_AS, "D/y"], None), + (WRITTEN_AS, b"d\0")], + id="entries-below-the-link-taken-path-by-path"), +] + + +@pytest.mark.parametrize(("paths", "target", "moved", "commands"), + BEYOND_A_LINK) +def test_no_file_is_removed_through_a_symlink_a_case_twin_left( + adopter, monkeypatch, tmp_path, paths, target, moved, commands): + """`D/x` goes, `d`, a symlink to a folder, stays in the root (or goes + with it), and the clone left `D` as that symlink, as on a disk that + ignores case. On c9f0cc18 the mount's `git rm -f -- D` unlinked `x` in + the folder the symlink leads to, outside the work directory where it + leads there, and `execute` printed `adoption verified` (r1237); without + `-f`, git refused it. Now each path beyond the symlink leaves the index + alone, `git rm --cached`, and the folder holds what it held; `git add + -A` stages nothing beyond the symlink, and the root's index holds each + path the plan keeps there with its own blob.""" + victim = tmp_path / "victim" + victim.mkdir() + (victim / "x").write_bytes(b"victim\n") + (victim / "keep").write_bytes(b"keep\n") + assembly, blobs = an_assembly( + tmp_path, paths, with_a_link(target.format(victim=victim))) + link = next(path for path in paths if path.split("/")[-1] == "d") + folder = link[:-1] + "D" + leads_to = assembly / target if target == "E" else victim + held = what_it_holds(leads_to) + as_the_link(assembly, folder, link) + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, moved) + assert what_it_holds(leads_to) == held + assert ran == commands + removed = [path.rstrip("/") for leg in moved.values() for path in leg] + left = staged(assembly) + # A disk that keeps case holds the symlink made in `folder`'s place + # beside `link`, and `git add -A` adds it there; on a disk that + # ignores case the two are one, `link`'s own. + assert left.pop(folder, blobs[link]) == blobs[link] + assert left == {path: oid for path, oid in blobs.items() + if not any(at_or_under(path, top) for top in removed)} + + +def test_a_folder_that_is_no_symlink_is_still_removed_with_f( + adopter, monkeypatch, tmp_path): + """The control: `D/x` goes and `d`, a symlink to a folder outside the + assembly, stays in the root, on a disk that keeps case, where `D` is + the folder the clone wrote. `D` is removed with `-f`, as on c9f0cc18: + `D/x` is gone from the disk and the index, the folder `d` leads to + holds what it held, and `d` is written back from the index.""" + victim = tmp_path / "victim" + victim.mkdir() + (victim / "x").write_bytes(b"victim\n") + assembly, blobs = an_assembly(tmp_path, ("D/x", "d"), + with_a_link(str(victim))) + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + pytest.skip("on a disk that ignores case the clone leaves `D` the " + "symlink `d`: the rows above") + assert (assembly / "D" / "x").read_bytes() == b"D/x\n" + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, {"code": ["D/"]}) + assert ran == [([*FORCED_AS, "D"], None), (WRITTEN_AS, b"d\0")] + assert not os.path.lexists(assembly / "D") + assert what_it_holds(victim) == {"x": b"victim\n"} + assert staged(assembly) == {path: oid for path, oid in blobs.items() + if path != "D/x"} + + +#: `(what the disk holds, each folder or symlink by its path, a path the +#: index holds, the folder `_beyond_a_symlink` names)`. A symlink leads to +#: `elsewhere`, a folder beside them. +ON_THE_WAY = [ + pytest.param({"D": "link"}, "D/x", "D", id="the-folder-a-symlink"), + pytest.param({"D": "folder", "D/E": "link"}, "D/E/x", "D/E", + id="a-folder-below-it-a-symlink"), + pytest.param({"D": "folder"}, "D/x", None, id="a-folder"), + pytest.param({"d": "link"}, "d", None, id="the-path-itself-a-symlink"), + pytest.param({}, "nowhere/D/x", None, id="nothing-there"), + pytest.param({"F": "file"}, "F/G/x", None, id="a-file-on-the-way"), +] + + +@pytest.mark.parametrize(("layout", "path", "named"), ON_THE_WAY) +def test_the_first_symlink_on_the_way_is_named_and_nothing_else( + adopter, tmp_path, layout, path, named): + """Each folder on the way to the path is asked with `lstat`, from the + top: the first that is a symlink is named; the path itself, which `git + rm` unlinks as the symlink it is, is not asked, and a folder that is + not there, or a file, is no symlink.""" + (tmp_path / "elsewhere").mkdir() + assembly = tmp_path / "assembly" + assembly.mkdir() + for where, what in layout.items(): + if what == "link": + os.symlink(tmp_path / "elsewhere", assembly / where) + elif what == "folder": + (assembly / where).mkdir() + else: + (assembly / where).write_bytes(b"F\n") + assert adopter._beyond_a_symlink(assembly, path) == named + + +@CANNOT_RUN_AS_ROOT +def test_a_folder_on_the_way_nobody_may_search_is_raised_not_read_as_none( + adopter, tmp_path): + """A folder on the way that this user may not search: its `lstat` is + refused, and the `PermissionError` is raised, one of `LATE_FAILURES`, + so `execute` names it, as `_is_a` raises a refused `stat` (#430). Read + as no symlink, the path would be removed with `-f` unasked.""" + assembly = tmp_path / "assembly" + (assembly / "D" / "E").mkdir(parents=True) + (assembly / "D").chmod(0) + try: + with pytest.raises(PermissionError): + adopter._beyond_a_symlink(assembly, "D/E/x") + finally: + (assembly / "D").chmod(0o755) + assert issubclass(PermissionError, adopter.LATE_FAILURES) + + needs_filter_repo = pytest.mark.skipif( shutil.which("git-filter-repo") is None, reason="git filter-repo is not installed: `pip install git-filter-repo`") From f645fcb80706022c500da9200d6e8b34c74c14bc Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 8 Nov 2026 13:55:56 +0000 Subject: [PATCH 5/9] adopt-project.py: `plan-case-twins` says a set in the assembly root could stop `execute` after both legs are pushed as `adopt-lost`, and no longer names a stop at `git rm` mounting the legs, which #1183's `-f` took away (#1183) #1237's review (r1237, SHOULD 1): the finding's words "or after both legs are pushed, as `adopt-lost` or at `git rm` mounting the legs" went stale with #1183's `-f`. Measured first, git 2.43 on a FUSE disk that ignores case, this head with `_case_twin_findings` patched in a scratch copy to name no finding, 1 run each: - twins kept in one leg (`A` and `a` in the code leg, and in the spec leg; `D/x` and `d/x`; `F/x` and `f`; `src/A` and `src/a`): `execute` exit 0, `adoption verified` (r1237 measured the same at c9f0cc18; at 58aa9d2b four of them stopped at `git rm`); - in the root: `A` and `a` verified; `F/x` beside `f` exit 1, `adopt-lost: F/x`, after both legs were pushed; `D/x` beside `d/x` `check` exit 1, `plan-file-reencoded` (`will not add file alias`), `execute` refused before any leg. So "after both legs are pushed, as `adopt-lost`" still holds, for a set in the assembly root, and the `git rm` stop is gone. The three places change together: the finding's words; `twins_kept` in `tests/test_adopt_check_throwaway_repos.py`, which pins them; and the history comment above the finding, which now says the `git rm` stop lasted "until #1183" and records these runs. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 13 ++++++++++--- tests/test_adopt_check_throwaway_repos.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index 00621758..2ddd5cf8 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -9286,7 +9286,14 @@ def _unsafe_path_findings(plan: Plan) -> tuple[list[str], list[str]]: # set kept as it was in the repository made. Twins in the code leg were # adopted so in 2 runs, and in 3 others, for #1189's review, `execute` # stopped after both legs were pushed, mounting them, at `git rm -r` ("the -# following files have local modifications"). +# following files have local modifications"), until #1183. Measured again +# for #1237's review with this finding taken out, 1 run each: twins kept in +# one leg (`A` and `a` in the code leg, and in the spec leg; `D/x` and +# `d/x`; `F/x` and `f`; `src/A` and `src/a`) were adopted; in the root, +# `A` and `a` were adopted, `F/x` beside `f` was still `adopt-lost` after +# both legs were pushed, and `D/x` beside `d/x` met `will not add file +# alias` in `check` (`plan-file-reencoded`). So the finding names the +# root's `adopt-lost`, and no `git rm`. # # So `check` names them, on every disk, as #520 names a branch in another # case: one plan is then not good on one machine and lost on another. @@ -9436,8 +9443,8 @@ def _case_twin_findings(plan: Plan, tree: list, "macOS or Windows disk does by default, holds each set as one " "name: a clone made there checks out only one path of each set, " "and `execute` could stop there before any leg exists, at git's " - "`will not add file alias`, or after both legs are pushed, as " - "`adopt-lost` or at `git rm` mounting the legs, so `execute` " + "`will not add file alias`, or, for a set in the assembly root, " + "after both legs are pushed, as `adopt-lost`, so `execute` " "refuses them on every disk.") return found, [CASE_TWINS_REMEDIATION] if found else [], notes diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index 821a4a24..c798c860 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -540,8 +540,8 @@ def twins_kept(sets: str, where: str, paths: str, named: str) -> str: "does by default, holds each set as one name: a clone made there " "checks out only one path of each set, and `execute` could stop " "there before any leg exists, at git's `will not add file " - "alias`, or after both legs are pushed, as `adopt-lost` or at " - "`git rm` mounting the legs, so `execute` refuses them on every " + "alias`, or, for a set in the assembly root, after both legs are " + "pushed, as `adopt-lost`, so `execute` refuses them on every " "disk.") From 63c5271857439ca953d2a272be2f2eca23d5297e Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 8 Nov 2026 13:56:26 +0000 Subject: [PATCH 6/9] adopt-project.py: `_split_case_twins`' docstring also names the sets the plan keeps whole in one leg, which reach the mount only as `check`'s one-blob NOTE and whose paths are removed with `-f` too (#1183) #1237's review (r1237, NIT 1): the docstring said the function returns "a set the plan sends into different repositories, or drops a path of". It also returns a set the plan keeps whole in one leg: every other such set is `plan-case-twins`, refused before any leg, so one reaches the mount only as a set of one blob in one mode (`_one_blob_twins`), and its paths, at or under the paths the leg takes, are removed with `-f`; none is written back. Docstring only. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/adopt-project.py b/adopt-project.py index 2ddd5cf8..a1d93c76 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -14422,7 +14422,11 @@ def _split_case_twins(held: list[str], holds (`_index_paths`, `_case_twin_sets`), that has a path at or under one of `removed`, the paths `_mount_the_legs` takes out of the root: a set the plan sends into different repositories, or drops a path of - (#1183). + (#1183); and a set it keeps whole in one leg, which reaches the mount + only as one of one blob in one mode, the NOTE `check` prints for it + (`_one_blob_twins`), every other being `plan-case-twins`. The paths of + such a set are removed from the root with `-f` too, and none is written + back, since the root keeps none of them (#1237's review, NIT 1). A DISK THAT IGNORES CASE HOLDS EACH SUCH SET AS ONE FILE. Measured for #1183 with git 2.43, on a FUSE disk that ignores case and keeps it: the From 9259da2b277258794a001a74c755e2e14eb49a0a Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 8 Nov 2026 15:17:40 +0000 Subject: [PATCH 7/9] adopt-project.py: before the mount's `git rm -f`, each gitlink the index holds at or under a removed path whose own path the disk holds as no folder (a symlink, a file, or nothing) leaves the index alone (`git rm --cached`), so a gitlink whose case twin is a symlink no longer empties the folder it leads to (#1183) r1237b's MUST 1. `git rm` of a gitlink's entry removes the folder at its path with all it holds, and follows a symlink there. `_beyond_a_symlink` asked only the folders on the way, never the path itself, so a dropped gitlink `D` the clone left as its twin `d`, a symlink to a folder outside the work directory, was given `git rm -f -- D`, which emptied that folder and stopped at `fatal: could not remove 'D'` after both legs were pushed. - `_index_modes` replaces `_index_paths`: one `git ls-files -s -z`, read as `_index_entries` reads it, giving each path and its mode. - `_taken_beyond_a_symlink` asks a gitlink's own path with `lstat` once no folder on the way is a symlink, and takes it with `--cached` where it is no folder (`_is_itself_a`, `stat.S_ISDIR`). - `_is_itself_a` is `_is_a_symlink`'s `lstat` with the kind as an argument; `_on_the_disk` is `_beyond_a_symlink`'s bytes join, shared. - The `GIT_RM_FORCED` and `GIT_RM_CACHED` comments and the `_beyond_a_symlink`, `_git_rm`, `_taken_beyond_a_symlink` and `_mount_the_legs` docstrings no longer say the path itself never needs asking. Rows: five shapes (the gitlink made the link and dropped, sent to a leg, or in a dropped folder; made a file; below a file) and the folder control. At 63c52718 the five fail: three empty the folder outside, two stop at `could not remove`. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 128 ++++++++++++------ tests/test_adopt_check_throwaway_repos.py | 150 ++++++++++++++++++++++ 2 files changed, 241 insertions(+), 37 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index a1d93c76..7e72553e 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -1884,12 +1884,13 @@ def _is_there(path: Path) -> bool: return _is_a(path, lambda _mode: True) -def _is_a_symlink(path: str | bytes) -> bool: - """Whether `path` is itself a symlink, asked with `os.lstat`, which - follows none, as `_is_a` asks `stat` (#430): nothing there is False, - and the `OSError` of an `lstat` the operating system REFUSED is raised, - never read as nothing there. NOT `Path.is_symlink()`, which from 3.14 - answers False to ANY `OSError`. `path` is a `str` or the bytes +def _is_itself_a(path: str | bytes, kind: Callable[[int], bool]) -> bool: + """Whether `path` is itself there and of `kind` (`stat.S_ISLNK`, + `stat.S_ISDIR`), asked with `os.lstat`, which follows no symlink, as + `_is_a` asks `stat` (#430): nothing there is False, and the `OSError` + of an `lstat` the operating system REFUSED is raised, never read as + nothing there. NOT `Path.is_symlink()` or `is_dir()`, which from 3.14 + answer False to ANY `OSError`. `path` is a `str` or the bytes `_path_argument` hands git, so that no locale is in the way (#527).""" try: mode = os.lstat(path).st_mode @@ -1900,7 +1901,12 @@ def _is_a_symlink(path: str | bytes) -> bool: except ValueError: # a name the operating system cannot take at all (a NUL in it) return False - return stat.S_ISLNK(mode) + return kind(mode) + + +def _is_a_symlink(path: str | bytes) -> bool: + """Whether `path` is itself a symlink (`_is_itself_a`).""" + return _is_itself_a(path, stat.S_ISLNK) def _os_reason(exc: OSError) -> str: @@ -14363,12 +14369,17 @@ def _refuse_an_unmountable_path(assembly: Path, path: str) -> None: #: disk still holds what the index holds, and that question was also what #: kept git from removing a file through a symlink: given a path beyond a #: symlink the disk holds on the way to it, `git rm -f` unlinks the file -#: the symlink leads to, wherever that is (r1237). So it is never given -#: one: `_taken_beyond_a_symlink` takes each such path first. +#: the symlink leads to, wherever that is (r1237). The path itself counts +#: too where the index holds a gitlink there: git removes the folder at a +#: gitlink's path with all it holds, and follows a symlink there, so `git +#: rm -f` of a gitlink whose path is a symlink empties the folder it leads +#: to (r1237b). So it is never given either: `_taken_beyond_a_symlink` +#: takes each such path first. GIT_RM_FORCED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "-f", "--") #: `GIT_RM` with `--cached`, for a path the index holds beyond a symlink -#: the disk holds on the way to it: git takes it out of the index and -#: touches no file on the disk (r1237). See `_taken_beyond_a_symlink`. +#: the disk holds on the way to it, or a gitlink whose path the disk holds +#: as no folder: git takes it out of the index and touches nothing on the +#: disk (r1237, r1237b). See `_taken_beyond_a_symlink`. GIT_RM_CACHED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "--cached", "--") @@ -14379,7 +14390,8 @@ def _git_rm(assembly: Path, path: str, forced: bool = False, assembly: what `run` does, with `path` handed to git as the bytes the tree holds (`_path_argument`, #527). `forced` is `GIT_RM_FORCED`, for a path a case twin is at or under (#1183), and `cached` `GIT_RM_CACHED`, - for one the index holds beyond a symlink (r1237). + for one the index holds beyond a symlink (r1237), or a gitlink whose + path the disk holds as no folder (r1237b). `run` takes only text (`check_program`), so this asks `check_program` of the command as the person reads it, and a git that refuses raises @@ -14409,17 +14421,23 @@ def _at_or_under(path: str, top: str) -> bool: return path == top or path.startswith(top + "/") -def _index_paths(assembly: Path) -> list[str]: +def _index_modes(assembly: Path) -> dict[str, str]: """Each path the assembly's index holds, in its order, as `_git_path` - spells it: `git ls-files -z`, which writes nothing.""" - raw = git_out(["ls-files", "-z"], cwd=assembly, binary=True) - return [_git_path(record) for record in raw.split(b"\x00") if record] + spells it, and its mode (`GITLINK_MODE` for a gitlink): `git ls-files + -s -z`, which writes nothing, read as `_index_entries` reads it.""" + raw = git_out(["ls-files", "-s", "-z"], cwd=assembly, binary=True) + modes = {} + for record in raw.split(b"\x00"): + head, _, name = record.partition(b"\t") + if name: + modes[_git_path(name)] = head.split()[0].decode("ascii") + return modes def _split_case_twins(held: list[str], removed: list[str]) -> list[list[str]]: """Each set of case twins of `held`, the paths the assembly's index - holds (`_index_paths`, `_case_twin_sets`), that has a path at or under + holds (`_index_modes`, `_case_twin_sets`), that has a path at or under one of `removed`, the paths `_mount_the_legs` takes out of the root: a set the plan sends into different repositories, or drops a path of (#1183); and a set it keeps whole in one leg, which reaches the mount @@ -14458,32 +14476,44 @@ def _split_case_twins(held: list[str], for twin in named for top in removed)] +def _on_the_disk(assembly: Path, path: str) -> str | bytes: + """`path`, as git spells it, in the assembly, as `os.lstat` is handed + it: the bytes the tree holds (`_path_argument`), so that no locale is + in the way (#527), or the text on Windows.""" + here = _path_argument(path) + return (os.path.join(os.fsencode(assembly), here) + if isinstance(here, bytes) else os.path.join(assembly, here)) + + def _beyond_a_symlink(assembly: Path, path: str) -> str | None: """The first folder on the way to `path`, a path the assembly's index holds, from the top, that the disk holds as a symlink, as git spells it; None where there is none (r1237). Each folder is asked with - `lstat` (`_is_a_symlink`), handed as the bytes the tree holds - (`_path_argument`). `path` itself is not asked: `git rm` of a symlink - unlinks the symlink, never what it leads to.""" + `lstat` (`_is_a_symlink`, `_on_the_disk`), and none beyond the first + symlink, which would be asked in the folder it leads to. `path` itself + is not asked here: `git rm` of a file's or a symlink's entry unlinks + what is at the path, never what a symlink there leads to. A gitlink's + entry is another matter, and `_taken_beyond_a_symlink` asks a + gitlink's own path (r1237b).""" parts = path.split("/") for end in range(1, len(parts)): above = "/".join(parts[:end]) - here = _path_argument(above) - if _is_a_symlink(os.path.join(os.fsencode(assembly), here) - if isinstance(here, bytes) - else os.path.join(assembly, here)): + if _is_a_symlink(_on_the_disk(assembly, above)): return above return None def _taken_beyond_a_symlink(assembly: Path, top: str, - held: list[str]) -> bool: + held: dict[str, str]) -> bool: """`git rm --cached` (`GIT_RM_CACHED`) of each path of `held`, the - paths the assembly's index holds, at or under `top` that the disk holds - beyond a symlink (`_beyond_a_symlink`), asked before `top` is removed - with `-f`: of the symlink's own path where that is at or under `top`, - which takes each path beyond it at once, else of the path. Whether that - took every path `held` at or under `top`, leaving none for `-f`. + paths the assembly's index holds and their modes (`_index_modes`), at + or under `top` that the disk holds beyond a symlink + (`_beyond_a_symlink`), asked before `top` is removed with `-f`: of the + symlink's own path where that is at or under `top`, which takes each + path beyond it at once, else of the path; and of each gitlink there + whose own path the disk holds as no folder (`_is_itself_a`): a + symlink, a file, or nothing. Whether that took every path `held` at or + under `top`, leaving none for `-f`. WHERE A FOLDER ON THE WAY IS A SYMLINK (#1237's review, r1237). A disk that ignores case holds a folder `D/` and a symlink `d` as one name, @@ -14503,6 +14533,24 @@ def _taken_beyond_a_symlink(assembly: Path, top: str, keeps, written back from the index. The root's tree is the plan's, as is each leg's, which `git filter-repo` made from the objects. + WHERE THE INDEX HOLDS A GITLINK (#1237's second review, r1237b). `git + rm` of a gitlink's entry is no unlink: git removes the folder at its + path with everything in it, and opens that path as a folder, following + a symlink there. Measured with git 2.43, for a gitlink `D` the disk + holds as a symlink to a folder outside, `git rm -f -- D` emptied that + folder and then said `fatal: could not remove 'D'`; on a FUSE disk that + ignores case, for a plan that drops the gitlink `D` and `.gitmodules` + and keeps `d`, a symlink, in the root (or drops the folder `M/` that + holds both), `check` printed `plan ok` and `execute` emptied the folder + `d` leads to after both legs were pushed. For a gitlink the disk holds + as a file, or below one (`M` a file, `M/D` the gitlink), `git rm -f` + stopped at the same `could not remove`, and deleted nothing. So the + gitlink's own path is asked too, once no folder on the way is a + symlink, and where it is no folder it is taken out of the index alone: + `git rm --cached` touched nothing there in each case. A folder there, + the empty one a clone writes for a gitlink, is removed with `-f` as + before. + REFUSING THERE WAS REJECTED: it would come after both legs are pushed, for a plan that is good on every disk, and a re-run then meets `leg-remote-exists`.""" @@ -14514,8 +14562,13 @@ def _taken_beyond_a_symlink(assembly: Path, top: str, link = _beyond_a_symlink(assembly, path) if link is not None: gone = link if _at_or_under(link, top) else path - _git_rm(assembly, gone, cached=True) - taken.append(gone) + elif held[path] == GITLINK_MODE and not _is_itself_a( + _on_the_disk(assembly, path), stat.S_ISDIR): + gone = path + else: + continue + _git_rm(assembly, gone, cached=True) + taken.append(gone) return bool(taken) and all(any(_at_or_under(path, gone) for gone in taken) for path in under) @@ -14555,16 +14608,17 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, is removed with `-f`, and each twin the root keeps is written back from the index after the last `git rm` (`_split_case_twins`), so that a disk that ignores case loses none of them. Before that `-f`, each path the - index holds at or under it beyond a symlink the disk holds on the way - is taken out of the index alone (`_taken_beyond_a_symlink`, r1237), so - that `-f` removes no file through a symlink. + index holds at or under it beyond a symlink the disk holds on the way, + and each gitlink there whose own path the disk holds as no folder, is + taken out of the index alone (`_taken_beyond_a_symlink`, r1237, + r1237b), so that `-f` removes nothing through a symlink. """ removed = sorted((p.rstrip("/") for p in (paths_for["spec"] + paths_for["code"] + paths_for["drop"])), key=lambda p: (p != GITMODULES, p)) - held = _index_paths(assembly) - split = [twin for named in _split_case_twins(held, removed) + held = _index_modes(assembly) + split = [twin for named in _split_case_twins(list(held), removed) for twin in named] for path in removed: forced = any(_at_or_under(twin, path) for twin in split) diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index c798c860..95d029e5 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -1383,6 +1383,156 @@ def test_a_folder_on_the_way_nobody_may_search_is_raised_not_read_as_none( assert issubclass(PermissionError, adopter.LATE_FAILURES) +# --------------------------------------------------------------------------- +# r1237b: a gitlink whose case twin is a symlink +# --------------------------------------------------------------------------- + +#: A path's kind in `with_kinds`: a symlink leading to `{victim}`, a folder +#: outside the assembly, and a gitlink. +LINK_OUT = ("link", "{victim}") +GITLINK = ("gitlink", None) + + +def with_kinds(kinds: dict): + """An `add` for `an_assembly`: each of `paths` goes into the source + commit with a blob of its own, as `with_paths` writes it, but a path + `kinds` gives `("link", target)` is a symlink leading to `target`, and + one it gives `GITLINK` a gitlink to the source's first commit, which + the clone checks out as an empty folder.""" + def add(repository: Path, paths: tuple[str, ...], + own_blobs: bool = True) -> None: + assert own_blobs + first = git("rev-parse", "HEAD", cwd=repository).stdout.strip() + for path in paths: + kind, target = kinds.get(path, ("file", None)) + if kind == "gitlink": + mode, oid = "160000", first + else: + mode, data = (("120000", target.encode()) if kind == "link" + else ("100644", (path + "\n").encode())) + oid = blob_of(repository, data) + git("update-index", "--add", "--cacheinfo", + f"{mode},{oid},{path}", cwd=repository) + git("commit", "-qm", "Add paths", cwd=repository) + return add + + +def as_the_twin(assembly: Path, at: str, twin: str) -> None: + """What a clone on a disk that ignores case leaves at `at`, a folder + (a gitlink's empty one among them), beside its twin `twin`, which + sorts after it: `twin` alone, which both names reach, as r1237 and + r1237b measured. Made on a disk that keeps case: the folder gives way + to a symlink leading where `twin` leads, or to a hard link of `twin`'s + file. Where the clone's own disk ignores case, git did this itself, + and nothing is done.""" + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + return + shutil.rmtree(assembly / at) + if (assembly / twin).is_symlink(): + os.symlink(os.readlink(assembly / twin), assembly / at) + else: + os.link(assembly / twin, assembly / at) + + +def outside_the_assembly(base: Path) -> Path: + """`base/victim`, a folder outside the assembly holding `x` and + `keep`.""" + victim = base / "victim" + victim.mkdir() + (victim / "x").write_bytes(b"victim\n") + (victim / "keep").write_bytes(b"keep\n") + return victim + + +#: `(the paths a source commit gains, the kind of each that is no file +#: (`with_kinds`), each `(at, twin)` the clone left as `twin` alone +#: (`as_the_twin`), the paths the plan sends to a leg or drops, the `git +#: rm` and `git checkout-index` the mount runs)`. On 63c52718 the mount +#: ran `git rm -f` of the gitlink `D` (or of `M`), which emptied the folder +#: the symlink leads to and stopped at `fatal: could not remove 'D'` +#: (`'M/D'`), or stopped there where the gitlink was a file or below one. +AT_A_GITLINK_OR_BEYOND_A_LINK = [ + pytest.param(("D", "d"), {"D": GITLINK, "d": LINK_OUT}, (("D", "d"),), + {"drop": ["D"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-gitlink-the-clone-left-the-link-dropped"), + pytest.param(("D", "d"), {"D": GITLINK, "d": LINK_OUT}, (("D", "d"),), + {"code": ["D"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-gitlink-the-clone-left-the-link-sent-to-a-leg"), + pytest.param(("M/D", "M/d", "M/z"), {"M/D": GITLINK, "M/d": LINK_OUT}, + (("M/D", "M/d"),), {"drop": ["M/"]}, + [([*CACHED_AS, "M/D"], None), ([*FORCED_AS, "M"], None)], + id="a-gitlink-the-clone-left-the-link-in-a-dropped-folder"), + pytest.param(("D", "d"), {"D": GITLINK}, (("D", "d"),), {"drop": ["D"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-gitlink-the-clone-left-a-file"), + pytest.param(("M/D", "m"), {"M/D": GITLINK}, (("M", "m"),), + {"drop": ["M/"]}, + [([*CACHED_AS, "M/D"], None), (WRITTEN_AS, b"m\0")], + id="a-gitlink-below-a-file-the-clone-left"), +] + + +@pytest.mark.parametrize(("paths", "kinds", "made", "moved", "commands"), + AT_A_GITLINK_OR_BEYOND_A_LINK) +def test_nothing_is_removed_through_a_symlink_at_a_gitlink_or_beyond( + adopter, monkeypatch, tmp_path, paths, kinds, made, moved, commands): + """`git rm` of a gitlink's entry removes the folder at its path with + all it holds, and follows a symlink there: on 63c52718, where the clone + left the gitlink `D` the symlink `d`, the mount's `git rm -f -- D` + emptied the folder `d` leads to, outside the assembly, and stopped + after both legs were pushed (r1237b). Now a gitlink whose own path the + disk holds as no folder leaves the index alone, `git rm --cached`, as + each path beyond a symlink on the way does; the folder holds what it + held, and the root's index holds each path the plan keeps there with + its own blob, and none that goes.""" + victim = outside_the_assembly(tmp_path) + assembly, blobs = an_assembly(tmp_path, paths, with_kinds( + {path: (kind, target and target.format(victim=victim)) + for path, (kind, target) in kinds.items()})) + for at, twin in made: + as_the_twin(assembly, at, twin) + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, moved) + assert what_it_holds(victim) == {"x": b"victim\n", "keep": b"keep\n"} + assert ran == commands + removed = [path.rstrip("/") for leg in moved.values() for path in leg] + left = staged(assembly) + # A disk that keeps case holds what was made at `at` beside `twin`, and + # `git add -A` adds it there; on a disk that ignores case the two are + # one, `twin`'s own. + for at, twin in made: + assert left.pop(at, blobs[twin]) == blobs[twin] + assert left == {path: oid for path, oid in blobs.items() + if not any(at_or_under(path, top) for top in removed)} + + +def test_a_gitlink_the_disk_holds_a_folder_at_is_still_removed_with_f( + adopter, monkeypatch, tmp_path): + """The control: the gitlink `D` is dropped and `d`, a symlink to a + folder outside the assembly, stays in the root, on a disk that keeps + case, where `D` is the empty folder the clone wrote for the gitlink. + `D` is removed with `-f`, as on 63c52718: the folder is gone, the + folder `d` leads to holds what it held, and `d` is written back.""" + victim = outside_the_assembly(tmp_path) + assembly, blobs = an_assembly(tmp_path, ("D", "d"), with_kinds( + {"D": GITLINK, "d": ("link", str(victim))})) + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + pytest.skip("on a disk that ignores case the clone leaves `D` the " + "symlink `d`: the rows above") + assert [child.name for child in (assembly / "D").iterdir()] == [] + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, {"drop": ["D"]}) + assert ran == [([*FORCED_AS, "D"], None), (WRITTEN_AS, b"d\0")] + assert not os.path.lexists(assembly / "D") + assert what_it_holds(victim) == {"x": b"victim\n", "keep": b"keep\n"} + assert staged(assembly) == {path: oid for path, oid in blobs.items() + if path != "D"} + + needs_filter_repo = pytest.mark.skipif( shutil.which("git-filter-repo") is None, reason="git filter-repo is not installed: `pip install git-filter-repo`") From 3ff03feb4a71653ffe01c6d3f4b622783eba7445 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 8 Nov 2026 15:18:11 +0000 Subject: [PATCH 8/9] tests/test_adopt_check_throwaway_repos.py: rows pin that the mount asks every path the index holds under a removed path, not only the twins, that a path under one is the path or below it after a `/`, that the first symlink on the way is named and nothing beyond it asked, and that `lstat` is handed bytes under a C locale (#1183) r1237b's SHOULD 1, NIT 1 and NIT 2: rows for its surviving mutants R11, R04, R02 and R03. Each passes at 63c52718 and fails under its mutant. - `a-file-beside-the-folder-the-clone-left-the-link` (R11): `S/D/x` and `S/z` go to code, `s/d` is a link kept in the root, `S/D` the link: `--cached -- S/D`, `-f -- S`, then `checkout-index` of `s/d`. - `two-folders-one-name-a-prefix-of-the-other` (R04): `D/` and `DD/` to code, `d` and `dd` links kept: `--cached` of each, no stop. - `a-symlink-beyond-the-first-symlink` and `test_nothing_beyond_the_first_symlink_on_the_way_is_asked` (R02): `D` is named; past it, a folder nobody may search raised. - `test_the_folder_on_the_way_is_asked_as_bytes_under_a_c_locale` (R03): `_mount_the_legs` in #527's child under `LC_ALL=C PYTHONUTF8=0`; handed text, `lstat` raises `UnicodeEncodeError`, which `_is_itself_a` reads as nothing there, and `git rm -f` unlinks `x` in the folder outside. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_check_throwaway_repos.py | 85 +++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index 95d029e5..afce3e49 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -62,9 +62,11 @@ from __future__ import annotations import importlib.util +import json import os import shutil import subprocess +import sys import tempfile from argparse import Namespace from pathlib import Path @@ -83,6 +85,9 @@ from test_adopt_root_symlinks import links_as_links # noqa: F401 from test_adopt_root_symlinks import unwritable_work # noqa: F401 from test_adopt_root_symlinks import planned as planned_with_links +#: #527's child, `_mount_the_legs` under `LC_ALL=C PYTHONUTF8=0`, and the +#: environment it runs in. +from test_adopt_git_rm_bytes import MOUNT_IN_THE_C_LOCALE, c_locale_or_skip #: The one skip for a mode that root reads through and Windows ignores. from test_validate_pins_unreadable_workflow import CANNOT_RUN_AS_ROOT @@ -1333,11 +1338,14 @@ def test_a_folder_that_is_no_symlink_is_still_removed_with_f( #: `(what the disk holds, each folder or symlink by its path, a path the #: index holds, the folder `_beyond_a_symlink` names)`. A symlink leads to -#: `elsewhere`, a folder beside them. +#: `elsewhere`, a folder beside them; one made beyond a symlink is made in +#: the folder that symlink leads to. ON_THE_WAY = [ pytest.param({"D": "link"}, "D/x", "D", id="the-folder-a-symlink"), pytest.param({"D": "folder", "D/E": "link"}, "D/E/x", "D/E", id="a-folder-below-it-a-symlink"), + pytest.param({"D": "link", "D/E": "link"}, "D/E/x", "D", + id="a-symlink-beyond-the-first-symlink"), pytest.param({"D": "folder"}, "D/x", None, id="a-folder"), pytest.param({"d": "link"}, "d", None, id="the-path-itself-a-symlink"), pytest.param({}, "nowhere/D/x", None, id="nothing-there"), @@ -1349,9 +1357,11 @@ def test_a_folder_that_is_no_symlink_is_still_removed_with_f( def test_the_first_symlink_on_the_way_is_named_and_nothing_else( adopter, tmp_path, layout, path, named): """Each folder on the way to the path is asked with `lstat`, from the - top: the first that is a symlink is named; the path itself, which `git - rm` unlinks as the symlink it is, is not asked, and a folder that is - not there, or a file, is no symlink.""" + top: the first that is a symlink is named, and not a symlink beyond + it; the path itself, which `git rm` of a file's or a symlink's entry + unlinks as what it is, is not asked here (a gitlink's is, by + `_taken_beyond_a_symlink`: the rows under r1237b below), and a folder + that is not there, or a file, is no symlink.""" (tmp_path / "elsewhere").mkdir() assembly = tmp_path / "assembly" assembly.mkdir() @@ -1383,8 +1393,27 @@ def test_a_folder_on_the_way_nobody_may_search_is_raised_not_read_as_none( assert issubclass(PermissionError, adopter.LATE_FAILURES) +@CANNOT_RUN_AS_ROOT +def test_nothing_beyond_the_first_symlink_on_the_way_is_asked(adopter, + tmp_path): + """`D` is a symlink to `elsewhere`, a folder this user may not search: + `D` is named, and nothing in `elsewhere` is asked. Asked past the first + symlink, `lstat` of `D/E` is refused there, and the `PermissionError` + would stop the mount after both legs are pushed (r1237b's R02).""" + elsewhere = tmp_path / "elsewhere" + (elsewhere / "E").mkdir(parents=True) + assembly = tmp_path / "assembly" + assembly.mkdir() + os.symlink(elsewhere, assembly / "D") + elsewhere.chmod(0) + try: + assert adopter._beyond_a_symlink(assembly, "D/E/x") == "D" + finally: + elsewhere.chmod(0o755) + + # --------------------------------------------------------------------------- -# r1237b: a gitlink whose case twin is a symlink +# r1237b: a gitlink whose case twin is a symlink, and every path held asked # --------------------------------------------------------------------------- #: A path's kind in `with_kinds`: a symlink leading to `{victim}`, a folder @@ -1452,6 +1481,10 @@ def outside_the_assembly(base: Path) -> Path: #: ran `git rm -f` of the gitlink `D` (or of `M`), which emptied the folder #: the symlink leads to and stopped at `fatal: could not remove 'D'` #: (`'M/D'`), or stopped there where the gitlink was a file or below one. +#: The last two shapes pin what 63c52718 already ran: every path the index +#: holds under a removed path is asked, not only the twins (r1237b's +#: SHOULD 1), and a path under one is the path or below it after a `/` +#: (its NIT 1). AT_A_GITLINK_OR_BEYOND_A_LINK = [ pytest.param(("D", "d"), {"D": GITLINK, "d": LINK_OUT}, (("D", "d"),), {"drop": ["D"]}, @@ -1472,6 +1505,17 @@ def outside_the_assembly(base: Path) -> Path: {"drop": ["M/"]}, [([*CACHED_AS, "M/D"], None), (WRITTEN_AS, b"m\0")], id="a-gitlink-below-a-file-the-clone-left"), + pytest.param(("S/D/x", "S/z", "s/d"), {"s/d": LINK_OUT}, + (("S/D", "s/d"),), {"code": ["S/"]}, + [([*CACHED_AS, "S/D"], None), ([*FORCED_AS, "S"], None), + (WRITTEN_AS, b"s/d\0")], + id="a-file-beside-the-folder-the-clone-left-the-link"), + pytest.param(("D/x", "DD/y", "d", "dd"), + {"d": LINK_OUT, "dd": LINK_OUT}, (("D", "d"), ("DD", "dd")), + {"code": ["D/", "DD/"]}, + [([*CACHED_AS, "D"], None), ([*CACHED_AS, "DD"], None), + (WRITTEN_AS, b"d\0dd\0")], + id="two-folders-one-name-a-prefix-of-the-other"), ] @@ -1533,6 +1577,37 @@ def test_a_gitlink_the_disk_holds_a_folder_at_is_still_removed_with_f( if path != "D"} +def test_the_folder_on_the_way_is_asked_as_bytes_under_a_c_locale(tmp_path): + """`Ü/x` goes to the code leg and `ü`, a symlink to a folder + outside the assembly, stays in the root, the clone having left + `Ü` that symlink; `_mount_the_legs` runs in a child under + `LC_ALL=C PYTHONUTF8=0`, whose file names are ASCII. `lstat` is handed + the bytes the tree holds, so `Ü` is the symlink and `Ü/x` + leaves the index alone. Handed the text, `lstat` raises + `UnicodeEncodeError`, a `ValueError` `_is_itself_a` reads as nothing + there, and `git rm -f` unlinks `x` in the folder the symlink leads to + (r1237b's R03).""" + env = c_locale_or_skip() + victim = outside_the_assembly(tmp_path) + folder, link = "Ü", "ü" + assembly, _ = an_assembly(tmp_path, (f"{folder}/x", link), with_kinds( + {link: ("link", str(victim))})) + as_the_twin(assembly, folder, link) + child = subprocess.run( + [sys.executable, "-c", MOUNT_IN_THE_C_LOCALE, + os.fsencode(ADOPT).hex(), os.fsencode(assembly).hex(), + os.fsencode(tmp_path / "work").hex(), + json.dumps({"spec": [], "code": [f"{folder}/"], "drop": []})], + capture_output=True, env=env, stdin=subprocess.DEVNULL, check=False) + assert child.returncode == 0, child.stderr.decode("utf-8", "replace") + answer = json.loads(child.stdout.decode("ascii")) + assert what_it_holds(victim) == {"x": b"victim\n", "keep": b"keep\n"} + assert "raised" not in answer, answer + held = z_rows(assembly, "ls-files", "-z") + assert f"{folder}/x" not in held + assert link in held + + needs_filter_repo = pytest.mark.skipif( shutil.which("git-filter-repo") is None, reason="git filter-repo is not installed: `pip install git-filter-repo`") From b557859fd578a189370ccbe7c5df60935525e3ae Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 8 Nov 2026 15:18:32 +0000 Subject: [PATCH 9/9] tests/test_adopt_check_throwaway_repos.py: the module's #1183 paragraph says that `-f` also skipped what kept git from removing through a symlink, a folder's twin or a gitlink's, and that such a path now leaves the index alone (#1183) r1237b's NIT 3: the paragraph described only `-f` and the write-back. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_check_throwaway_repos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index afce3e49..deaa069b 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -57,6 +57,16 @@ twin the root keeps back from the index. The rows that drive the mount make that one file on any disk, as hard links; the rows that run `execute` whole skip on a disk that keeps case, but for one control. + +`-f` also skipped what kept git from removing through a symlink. Where the +clone left a folder as its twin, a symlink to a folder outside the work +directory, `git rm -f` unlinked the file in that folder (r1237); where it +left a gitlink so, `git rm -f`, which removes the folder at a gitlink's +path with all it holds, emptied that folder (r1237b). Now each path the +index holds beyond a symlink on the way, and each gitlink whose own path +the disk holds as no folder (a symlink, a file, or nothing), leaves the +index alone (`git rm --cached`), and nothing on the disk is touched. The +rows make that symlink, or that file, by hand on a disk that keeps case. """ from __future__ import annotations