From aaa9099bdcf3f3a6cde2a9cc98bb1566f8635348 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:29:58 +0000 Subject: [PATCH 01/12] adopt-project.py: a plan that sends a source's `.gitmodules` to one leg and a submodule it registers to another, or keeps either in the root, is now a `check` finding and an `execute` refusal, and `plan` asks the one-leg question on `.gitmodules` itself (#166) A plan answering `.gitmodules -> code, upstream/ -> spec` (or `.gitmodules -> drop, upstream/ -> code`) used to pass `check`, run `execute` to "adoption verified", and leave an assembly whose `git clone --recurse-submodules` exits 128 on `No url found for submodule path 'spec/upstream/dependency'`. Reproduced on main before this change; refused before either leg exists after it. ONE helper, `submodule_plan_problems(entries, tree)`, read by both `cmd_check` (as findings) and `_refuse_an_unrunnable_plan` (as a refusal), so the plan `check` passes is the plan `execute` runs. The leg of a path is the `leg:` of the single entry `_covering` it: - `.gitmodules` and every gitlink (mode 160000): `root` is refused, one `plan-submodule-root` per path. The assembly root has exactly two mounts, `execute` writes it a fresh `.gitmodules` for them, and `validate.yml` reads every URL there as a leg. - When the source has a `.gitmodules`, every gitlink kept in spec or code must share its leg, one `plan-submodule-split` per gitlink; so `.gitmodules` cannot be dropped while one is kept. A gitlink dropped beside a `.gitmodules` that went to a leg is allowed: its registration stays behind inert (verified: recursive clone exit 0, `bootstrap ok`). All dropped together is allowed. - No `.gitmodules`: only the root rule, so an orphan gitlink adopted into a leg still adopts. - An unresolved, invalid, uncovered or doubly covered entry is skipped: it is already its own finding. A source with neither (the common case) returns nothing at once; its plan file and `check` output are byte-identical to main's. `contracts/path-classification.yaml` gains `ambiguous-gitmodules`, asked first with the other named ambiguities, whose question states the constraint: "A source's own `.gitmodules` goes to the SAME leg as every submodule it registers; `execute` writes the assembly root a fresh one for its two mounts", and that `root` is not an answer. Only the top-level file matches. With `root` refused, the root-kept submodule path #165 documents in `_verify` is no longer reachable from `execute`; `_verify` and `_mount_the_legs` are untouched and #165 stays open. Tests: the plan question; the `check` matrix of eleven answers, a gitlink a rule classified silently (`tools/dep`), an orphan gitlink; the refusal through importlib, all without git-filter-repo; and one end-to-end refusal under `needs_filter_repo` asserting no leg repository was made. The two policy tests take `.gitmodules` (its rule id) and a nested `docs/theme/.gitmodules` (spec). Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 125 +++++++++++++++- contracts/path-classification.yaml | 25 ++++ tests/test_adopt_plan.py | 19 ++- tests/test_adopt_submodules.py | 226 +++++++++++++++++++++++++++++ 4 files changed, 384 insertions(+), 11 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index a48a5f80..b771adb4 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -140,9 +140,13 @@ def _checked_pin_name(raw: str) -> str: LEG_VALUES = ("spec", "code", "root", "drop") FILE_PROTOCOL = ["-c", "protocol.file.allow=always"] #: The file `git submodule add` records a mount in. `_mount_the_legs` checks, -#: writes, stages and edits it -- named once so those four uses cannot -#: spell it differently. +#: writes, stages and edits it, and `check` and `execute` ask which leg the +#: plan gave the source's own -- named once so those uses cannot spell it +#: differently. GITMODULES = ".gitmodules" +#: The mode `git ls-tree` reports for a SUBMODULE: a commit of another +#: repository, recorded in this tree as a gitlink rather than as a file. +GITLINK_MODE = "160000" #: THE SPEC-ONLY CASE. A repository can honestly have nothing for one leg — #: InkRouter's IRRS and IRSS are specifications with no implementation yet @@ -975,6 +979,103 @@ def _leg_findings(plan: Plan) -> list[str]: return findings +#: What a plan that splits a source's own submodules is told to do instead. +#: Said once, by `execute`'s refusal; `check` states the same problems as +#: findings and the `.gitmodules` entry's `question:` states the rule. +SUBMODULE_REMEDIATION = ( + "Remediation: send `.gitmodules` and every submodule it registers to ONE " + "leg, spec or code, or `drop` them together. To be rid of ONE of several " + "registered submodules, adopt it with the others and `git rm` it in the " + "leg afterwards, which takes its registration with it. The assembly root " + "never keeps a submodule of the source's: `execute` writes it a fresh " + "`.gitmodules` for its two mounts, and its `validate` workflow reads every " + "URL in that file as a leg.") + + +def _answered_legs(entries: list, paths: list[str]) -> dict: + """`{path: (leg, entry)}` for each path ONE answered plan entry covers. + + A path covered by no entry or by two, or by an entry whose `leg:` is null + or not one of the four words, is LEFT OUT. Each of those is already a + finding of its own -- `plan-uncovered`, `plan-covered-twice`, + `plan-unresolved`, `plan-bad-leg` -- and a question nobody has answered + yet cannot have been answered inconsistently. + """ + entry_paths = [str(e.get("path")) for e in entries] + legs = {str(e.get("path")): str(e.get("leg")) for e in entries} + out: dict[str, tuple[str, str]] = {} + for path in paths: + covering = _covering(entry_paths, path) + if len(covering) == 1 and legs[covering[0]] in LEG_VALUES: + out[path] = (legs[covering[0]], covering[0]) + return out + + +def _as_entry(path: str, entry: str) -> str: + """`path`, and the plan entry to edit when that is a directory above it.""" + return path if entry == path else f"{path} (entry {entry})" + + +def submodule_plan_problems(entries: list, tree: list) -> list[tuple[str, str]]: + """`(code, detail)` for each way the plan splits the source's OWN submodules. + + ONE DEFINITION, asked by `check` for findings and by `execute` for a + refusal, so that the plan `check` passes is the plan `execute` will run + (#166). `tree` is `Source.tree()`: a gitlink is visible only by its mode, + which no path glob in `contracts/path-classification.yaml` can see. + + A SUBMODULE IS ONE FACT IN TWO PLACES: the gitlink in the tree and its + registration in `.gitmodules`. A plan that sent the two to different legs + used to run to the end and say `adoption verified` -- every blob was + accounted for -- and leave an assembly whose `git clone + --recurse-submodules` exits 128 on `No url found for submodule path`. So: + + * `root` is not an answer for `.gitmodules` or for any gitlink. The + assembly root has exactly two mounts, `execute` writes it a FRESH + `.gitmodules` for them, and the shape's own `validate.yml` reads every + URL in that file as a leg; a root `.gitmodules` the plan kept fails + verification today whatever it registers (#165). + * When the source HAS a `.gitmodules`, every gitlink the plan KEEPS, in + the spec or the code leg, goes where `.gitmodules` goes -- which is + also why `.gitmodules` cannot be dropped while one is kept. + * A gitlink DROPPED beside a `.gitmodules` that went to a leg is + allowed. Its registration stays behind in that leg, inert: git walks + the gitlinks in the INDEX and looks each one up in `.gitmodules`, + never the other way round, so a registration with no gitlink is never + cloned and never fails. Dropping all of them with the file is allowed + too. + * A source with gitlinks and NO `.gitmodules` is held to the first rule + only. Its orphan gitlinks were already unclonable in the source and + are no worse in a leg; refusing them would refuse plans that adopted + before this check existed. + + A source with neither -- the common case -- returns nothing at once, so + `check` and `execute` say exactly what they said before. + """ + gitlinks = [path for path, mode, _, _ in tree if mode == GITLINK_MODE] + registry = [path for path, _, _, _ in tree if path == GITMODULES] + if not gitlinks and not registry: + return [] + answered = _answered_legs(entries, registry + gitlinks) + problems = [ + ("plan-submodule-root", + f"{_as_entry(path, entry)} has leg: root, and the assembly root keeps " + "no submodule of the source's: `execute` writes it a fresh " + f"{GITMODULES} for its two leg mounts") + for path, (leg, entry) in answered.items() if leg == "root"] + if GITMODULES in answered: + registry_leg = answered[GITMODULES][0] + problems += [ + ("plan-submodule-split", + f"the submodule {_as_entry(path, entry)} has leg: {leg} but the " + f"source's {GITMODULES} has leg: {registry_leg}, so the leg that " + "holds the gitlink holds no URL for it and the assembly's " + "recursive clone fails there") + for path, (leg, entry) in answered.items() + if leg in EXTRACTED_LEGS and leg != registry_leg] + return problems + + def _topics_line(topic: str, local: bool) -> str: """The `topics` plan line, the one `scaffold-project.py` also prints. @@ -1032,9 +1133,12 @@ def cmd_check(args) -> int: "split.") entry_paths = [str(e.get("path")) for e in plan.entries] - tree_paths = [path for path, _, _, _ in source.tree()] + tree = source.tree() + tree_paths = [path for path, _, _, _ in tree] findings.extend(_coverage_findings(entry_paths, tree_paths)) findings.extend(_leg_findings(plan)) + findings.extend(f"FINDING {code}: {detail}" for code, detail + in submodule_plan_problems(plan.entries, tree)) names = plan.names() pins = set(plan.pins) @@ -1117,11 +1221,15 @@ def _confirm(args, plan: Plan, names: dict) -> None: def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: - """The two states a plan can be in that must not be executed. + """The three states a plan can be in that must not be executed. An unanswered question is never an implicit `root`, and a plan written against a tree that has since moved proves nothing about the tree that - would be split — which is how a path goes missing. + would be split — which is how a path goes missing. And a plan that + splits a submodule from its registration, or keeps one in the root, + builds an assembly that cannot be cloned: refused HERE, before either leg + exists, because once `_create_leg_remotes` has run, a re-run with the + corrected plan meets two legs that already exist (#166). """ unresolved = [e for e in plan.entries if e.get("leg") is None] if unresolved: @@ -1140,6 +1248,13 @@ def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: "Remediation: re-run `plan`, re-answer anything new, then " "`check`. Splitting a tree the plan has not seen is how a path " "goes missing.") + problems = submodule_plan_problems(plan.entries, source.tree()) + if problems: + details = [detail for _, detail in problems[:8]] + if len(problems) > 8: + details.append(f"and {len(problems) - 8} more") + raise Refusal(problems[0][0], "; ".join(details), + SUBMODULE_REMEDIATION) def _create_leg_remotes(plan: Plan, names: dict, repositories: dict, diff --git a/contracts/path-classification.yaml b/contracts/path-classification.yaml index 51d3453b..173dffdc 100644 --- a/contracts/path-classification.yaml +++ b/contracts/path-classification.yaml @@ -93,6 +93,31 @@ rules: Is this notebook an executable part of the implementation (code) or a worked argument supporting a decision (spec)? + # A SUBMODULE IS ONE FACT IN TWO PLACES: the gitlink in the tree and its + # registration in this file. Sent to different legs, the leg holding the + # gitlink has no URL for it, and the assembly's `git clone + # --recurse-submodules` exits 128 after an adoption that verified every + # blob (#166). No glob here can tell a gitlink from a file -- only its mode + # in `git ls-tree` can -- so the question is asked on the one file every + # registration lives in, and `adopt-project.py check` holds the gitlinks to + # the answer. Without this rule the default asked it as if `.gitmodules` + # were any other file, and `root` looked like an answer. + - id: ambiguous-gitmodules + paths: + - ".gitmodules" + leg: ambiguous + reason: >- + The repository registers submodules of its own. `.gitmodules` and the + gitlinks it names are one fact recorded in two places, and which leg + those dependencies belong to is a fact about this repository. + question: >- + A source's own `.gitmodules` goes to the SAME leg as every submodule it + registers; `execute` writes the assembly root a fresh one for its two + mounts. Which leg owns the submodules, spec or code? Send `.gitmodules` + there and every submodule path with it, or drop them all together. + `root` is not an answer for this file or for any submodule: `check` + finds it and `execute` refuses it, as they do a plan that splits them. + # -- the assembly root ---------------------------------------------------- - id: root-spec-kit diff --git a/tests/test_adopt_plan.py b/tests/test_adopt_plan.py index 8e186c41..e840bb90 100644 --- a/tests/test_adopt_plan.py +++ b/tests/test_adopt_plan.py @@ -79,20 +79,27 @@ def test_the_path_policy_loads_and_declares_four_classes(): (".cursor/rules.md", "root"), (".roo/config.md", "root"), ("Makefile", "root"), + # Only the TOP-LEVEL `.gitmodules` registers anything, so only that one is + # asked the submodule question; a nested one is content of its directory. + ("docs/theme/.gitmodules", "spec"), ]) def test_a_path_classifies_where_the_rulings_put_it(path, leg): verdict = PathPolicy.load(POLICY).classify_file(path) assert verdict.leg == leg, f"{path} -> {verdict.leg} ({verdict.rule})" -@pytest.mark.parametrize("path", [ - "examples/golden-run/expected.yaml", - ".claude/commands/ship.md", - "release.yaml", - "notebooks/explore.ipynb", +@pytest.mark.parametrize("path,rule", [ + ("examples/golden-run/expected.yaml", "ambiguous-examples"), + (".claude/commands/ship.md", "ambiguous-assistant-project-memory"), + ("release.yaml", "ambiguous-top-level-yaml"), + ("notebooks/explore.ipynb", "ambiguous-notebooks"), + # #166: its question states the one-leg rule, which `default`'s does not. + (".gitmodules", "ambiguous-gitmodules"), ]) -def test_an_ambiguous_path_carries_the_question_and_no_leg(path): +def test_an_ambiguous_path_carries_the_question_and_no_leg(path, rule): verdict = PathPolicy.load(POLICY).classify_file(path) + assert verdict.rule == rule, ( + "a NAMED question, not the default's: which leg owns this path?") assert verdict.leg is None assert verdict.review_required assert verdict.question, "an ambiguous path with no question is a shrug" diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index 73fea75d..8c3ed4cd 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -246,3 +246,229 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( "spec", "code") assert registrations(assembly) == LEG_REGISTRATIONS assert git("ls-files", "--", ".deps", cwd=assembly).stdout == "" + + +# --- the plan's own answer: one leg for a submodule and its registration ---- +# +# #166. `check` and `execute` read the SAME `submodule_plan_problems`, so a +# plan `check` passes is a plan `execute` runs. None of these needs +# `git filter-repo` but the one that reaches `execute` past its preflight. + +#: The two ids, as `check` prints them and `execute` raises them. +SPLIT = "plan-submodule-split" +ROOT = "plan-submodule-root" + +#: A source whose plan asks nothing but the submodule's questions: `specs/` +#: and `src/` classify themselves and `README.md` stays in the root. +SMALL_TREE = {"README.md": "# Thing\n", "src/app/util.py": "VALUE = 1\n", + "specs/001-feature/tasks.md": "- [ ] one\n"} + + +def registered_source(base: Path, registered=("upstream/dependency",), + orphan: str | None = None) -> Path: + """A source registering a real dependency at each of `registered`. + + `orphan` adds a gitlink at that path with NO registration, written + straight into the index as `git add` of an embedded clone would write it, + so that a source with a submodule and no `.gitmodules` is planned too. + """ + dependency = make_source_repo(base / "dependency", + tree={"value.txt": "dependency\n"}, edits=()) + source = make_source_repo(base / "Thing", tree=SMALL_TREE) + for path in registered: + git(*FILE_PROTOCOL, "submodule", "add", "-q", str(dependency), path, + cwd=source) + if orphan: + head = git("rev-parse", "HEAD", cwd=dependency).stdout.strip() + git("update-index", "--add", "--cacheinfo", f"160000,{head},{orphan}", + cwd=source) + git("-c", "user.name=Source Human", "-c", + "user.email=source@invalid.example", "commit", "-qm", + "Register dependencies", cwd=source) + return source + + +@pytest.fixture(scope="module") +def registered_plan(tmp_path_factory): + """ONE source with `upstream/dependency` registered, and its plan. + + Each check test answers a COPY of the plan; the source is only ever read. + """ + base = tmp_path_factory.mktemp("registered") + source = registered_source(base) + plan = base / "adoption-plan.yaml" + written = write_plan(source, plan) + assert written.returncode == 0, written.stderr + written.stdout + return plan, written + + +def answered(plan: Path, tmp_path: Path, answers) -> Path: + """A copy of `plan` with each `(path, leg)` in `answers` resolved; a leg + of None leaves that entry's question unanswered.""" + copy = tmp_path / "adoption-plan.yaml" + copy.write_text(plan.read_text(encoding="utf-8"), encoding="utf-8") + for path, leg in answers: + if leg is not None: + resolve(copy, path, leg) + return copy + + +def finding_codes(stderr: str) -> list[str]: + return [line.split(":", 1)[0].split()[1] for line in stderr.splitlines() + if line.startswith("FINDING ")] + + +def test_plan_asks_the_submodule_question_on_gitmodules(registered_plan): + plan, written = registered_plan + rows = {str(row["path"]): row for row in load_yaml(plan)["paths"]} + row = rows[".gitmodules"] + assert row["leg"] is None + assert row["review_required"] is True + assert row["rule"] == "ambiguous-gitmodules" + assert ("A source's own `.gitmodules` goes to the SAME leg as every " + "submodule it registers; `execute` writes the assembly root a " + "fresh one for its two mounts.") in row["question"] + assert "`root` is not an answer" in row["question"] + # Printed with the other unresolved paths, which is where it is read. + assert "goes to the SAME leg as every submodule" in written.stdout + + +@pytest.mark.parametrize("modules_leg, dependency_leg, expected", [ + ("code", "code", []), + ("spec", "spec", []), + # A dropped gitlink leaves its registration behind, inert. + ("code", "drop", []), + ("drop", "drop", []), + ("code", "spec", [SPLIT]), + ("drop", "code", [SPLIT]), + ("root", "root", [ROOT, ROOT]), + ("spec", "root", [ROOT]), + ("root", "code", [ROOT, SPLIT]), + # An unanswered `.gitmodules` is `plan-unresolved`'s, and is not also + # reported as a split; a gitlink kept in the root is refused regardless. + (None, "code", ["plan-unresolved"]), + (None, "root", ["plan-unresolved", ROOT]), +]) +def test_check_holds_a_submodule_to_the_leg_of_its_registration( + registered_plan, tmp_path, modules_leg, dependency_leg, expected): + plan = answered(registered_plan[0], tmp_path, + ((".gitmodules", modules_leg), + ("upstream/", dependency_leg))) + result = run_script(ADOPT, "check", "--plan", str(plan)) + assert finding_codes(result.stderr) == expected, result.stderr + if not expected: + assert result.returncode == 0, result.stderr + assert "plan ok" in result.stdout + return + assert result.returncode == 1 + assert f"{len(expected)} finding(s) in {plan}" in result.stderr + if SPLIT in expected: + assert (f"FINDING {SPLIT}: the submodule " + f"upstream/dependency (entry upstream/) has leg: " + f"{dependency_leg} but the source's .gitmodules has leg: " + f"{modules_leg}") in result.stderr + if modules_leg == "root": + assert f"FINDING {ROOT}: .gitmodules has leg: root" in result.stderr + if dependency_leg == "root": + assert (f"FINDING {ROOT}: upstream/dependency (entry upstream/) has " + "leg: root") in result.stderr + + +def test_check_finds_a_submodule_a_rule_classified_silently(tmp_path): + """`tools/dep` is under `tools/**`, so `plan` calls it code and asks + nothing; only `.gitmodules` is asked. Answered `spec`, the two split.""" + source = registered_source(tmp_path, registered=("tools/dep",)) + plan = tmp_path / "plan" / "adoption-plan.yaml" + plan.parent.mkdir() + assert write_plan(source, plan).returncode == 0 + rows = {str(row["path"]): row for row in load_yaml(plan)["paths"]} + assert rows["tools/"]["leg"] == "code" + resolve(plan, ".gitmodules", "spec") + result = run_script(ADOPT, "check", "--plan", str(plan)) + assert result.returncode == 1, result.stderr + result.stdout + assert (f"FINDING {SPLIT}: the submodule tools/dep (entry tools/) has " + "leg: code but the source's .gitmodules has leg: spec") \ + in result.stderr + + +@pytest.mark.parametrize("leg, expected", [ + ("code", []), + ("root", [ROOT]), +]) +def test_an_orphan_gitlink_may_go_to_a_leg_but_not_the_root(tmp_path, leg, + expected): + """No `.gitmodules` at all: the gitlink was unclonable in the source too, + and a leg is no worse a place for it. The root is still not one.""" + source = registered_source(tmp_path, registered=(), + orphan="upstream/orphan") + assert git("ls-tree", "HEAD", "--", ".gitmodules", + cwd=source).stdout == "" + plan = tmp_path / "plan" / "adoption-plan.yaml" + plan.parent.mkdir() + assert write_plan(source, plan).returncode == 0 + resolve(plan, "upstream/", leg) + result = run_script(ADOPT, "check", "--plan", str(plan)) + assert finding_codes(result.stderr) == expected, result.stderr + assert result.returncode == (1 if expected else 0) + + +@pytest.fixture(scope="module") +def adopter(): + """`adopt-project.py` as a module, for the refusal `execute` makes.""" + module_spec = importlib.util.spec_from_file_location("adopt_refusals", + ADOPT) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("modules_leg, dependency_leg, code", [ + ("code", "spec", SPLIT), + ("drop", "code", SPLIT), + ("root", "root", ROOT), + ("spec", "spec", None), + ("code", "drop", None), +]) +def test_execute_refuses_what_check_finds(adopter, registered_plan, tmp_path, + modules_leg, dependency_leg, code): + plan_path = answered(registered_plan[0], tmp_path, + ((".gitmodules", modules_leg), + ("upstream/", dependency_leg))) + plan = adopter.Plan.load(plan_path) + source = plan.open_source(None, tmp_path / "work") + if code is None: + adopter._refuse_an_unrunnable_plan(plan, source) + return + with pytest.raises(adopter.Refusal) as refused: + adopter._refuse_an_unrunnable_plan(plan, source) + assert refused.value.code == code + assert "upstream/dependency" in refused.value.detail + assert "`git rm` it in the leg afterwards" in refused.value.remediation + assert "never keeps a submodule" in refused.value.remediation + + +@needs_filter_repo +def test_execute_refuses_a_split_submodule_before_any_leg_exists(tmp_path): + """Before #166 this plan was extracted, mounted, pushed and verified, and + its assembly's recursive clone then exited 128. Refused now, and refused + before `_create_leg_remotes`: a re-run must not meet two legs that a + refused plan already made.""" + source = registered_source(tmp_path) + head = git("rev-parse", "HEAD", cwd=source).stdout.strip() + plan = tmp_path / "plan" / "adoption-plan.yaml" + plan.parent.mkdir() + assert write_plan(source, plan).returncode == 0 + resolve(plan, ".gitmodules", "code") + resolve(plan, "upstream/", "spec") + remotes = tmp_path / "remotes" + result = run_script(ADOPT, "execute", "--plan", str(plan), "--yes", + "--local-remote-dir", str(remotes), + "--work-dir", str(tmp_path / "work")) + assert result.returncode == 2, result.stderr + result.stdout + assert (f"REFUSED {SPLIT}: the submodule upstream/dependency (entry " + "upstream/) has leg: spec but the source's .gitmodules has leg: " + "code") in result.stderr + assert "creating the leg repositories" not in result.stdout + assert list(remotes.glob("*.git")) == [], "a refused plan made a leg" + assert git("rev-parse", "HEAD", cwd=source).stdout.strip() == head + assert git("branch", "--list", "adopt/*", cwd=source).stdout == "" From 09d78891ff19ca8bd17ffbf62f4917e692052c1b Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:11:56 +0000 Subject: [PATCH 02/12] adopt-project.py: a submodule a plan keeps in a leg must now be registered in the source's `.gitmodules` with a url, because the assembly's recursive clone dies on one that is not even where the source's own clone skips it, and a `.gitmodules` that `git config` cannot read is refused instead of being read as registering nothing (#166) aaa9099 allowed a gitlink with no registration into a leg on the premise that it "was already unclonable in the source and is no worse in a leg". That premise is false. At the top level, `git clone --recurse-submodules` skips a gitlink that has no URL; once it is inside a leg, the clone recurses with `--init` and dies on it. The adversarial review's reproduction (`upstream/orphan`, no `.gitmodules`, `upstream/ -> code`) gave `check` 0, `execute` 0 "adoption verified", source clone rc 0, and assembly clone rc 128 `No url found for submodule path 'code/upstream/orphan'`. The same happens beside a `.gitmodules` that registers other paths. The sentence is withdrawn from the docstring; this message withdraws it from aaa9099's ("No `.gitmodules`: only the root rule, so an orphan gitlink adopted into a leg still adopts"). The rule, still in ONE helper, so `check` and `execute` cannot disagree. The helper is `submodule_plan_problems(entries, tree, registered)`, and it is now a pure function of the registrations its callers read: - `Source.registered_submodules(tree)` returns the paths that have a `submodule..path` and a non-empty `submodule..url`. It reads them with `git config -z --blob :.gitmodules --list`. It uses `--list` and not `--get-regexp`, because the second exits 1 both for "no key matched" and for "bad config line" (probed on git 2.43); `--list` exits 0 or 128. A key with no `` is ignored, as git ignores it. With no `.gitmodules`, nothing is registered. - A gitlink kept in spec or code that is not registered is `plan-submodule-unregistered`, naming the gitlink and its leg. It is that and never also a split: no leg holds a URL for it. Dropped, it is fine. - A `.gitmodules` that `git config` cannot read is `plan-gitmodules-unreadable` when the plan keeps it, or any gitlink, in a leg. It is never read as registering nothing. - Unchanged: `root` refused for `.gitmodules` and every gitlink; a registered gitlink kept away from `.gitmodules` is `plan-submodule-split`; a dropped gitlink beside a leg's `.gitmodules` is inert. - Problems are listed root, unreadable, unregistered, split: the order to fix them in. `execute`'s refusal takes the first problem's code and names eight details, then "and N more". - The remediation says how to fix an unregistered gitlink: drop it, or register it in the source (an entry with path and url, committed) and re-run `plan`. Text that was untrue where it is printed: - The `ambiguous-gitmodules` question keeps the issue's sentence verbatim and "`root` is not an answer". Its "`check` finds it and `execute` refuses it" becomes "`adopt-project.py` refuses a plan that gives it". `shape-doctor.py --placement-plan` prints this question for already-split projects, whose plan no `check` or `execute` reads. The question also gains: "A submodule it does not register with a path and a url cannot be kept in a leg at all: drop it, or register it first." - Its `reason:` no longer claims that the repository registers submodules, since it is printed for an empty `.gitmodules` too. - The `plan-submodule-root` detail and the remediation said "`execute` writes it a fresh .gitmodules", which is false when `.gitmodules` is itself `root`. They now say that the root's `.gitmodules` "is reserved for its two leg mounts". - The doctor's verdict on the reviewer's adopted assembly is unchanged: `note`, "nothing is in the wrong leg", exit 0. Only the `reason:` and `question:` lines of its placement plan differ. Sonar hygiene: no literal in adopt-project.py or the test file newly reaches three uses. - `_git_path` is the one decoder for `Source.tree()` and the registrations, which must be compared in one spelling. - `GIT_CONFIG` replaces the subcommand literal in `_elector` and in `_mount_the_legs`' url write. That is a one-token change there; `_verify` is untouched. - `SUBMODULE_KEY_RE` parses the keys. - `_submodule_findings` keeps `cmd_check` at main's shape. Tests, tests/test_adopt_submodules.py (28 -> 52 cases): - The orphan test now expects `plan-submodule-unregistered` for code and spec, a pass for drop, and `plan-submodule-root` for root. - New: an unregistered gitlink beside a registered one (code, spec, drop). - New: a registration that is url-less, empty-url, name-less, at another path, or in an empty file is not one. Upper-case keys are one. - New: an unreadable `.gitmodules` (five answers). - The importlib refusal test now compares the refusal with `check`'s findings, detail for detail, across all four codes and four legal plans. - New: a mixed refusal of eleven or ten problems pins the first code and the 8-detail cap. - The `needs_filter_repo` end-to-end refusal is parametrised over the split plan and the reviewer's orphan plan. It asserts that the `--local-remote-dir` does not exist, that the `--work-dir` is empty, and that the source's HEAD, refs and status are unchanged. - Repeated literals go into constants and helpers (`adopt`, `check_plan`, `plan_of`, `add_submodule`, `commit_as_source_human`). The first test in the file uses them too. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 213 ++++++++++++--- contracts/path-classification.yaml | 23 +- tests/test_adopt_submodules.py | 425 +++++++++++++++++++++-------- 3 files changed, 503 insertions(+), 158 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index b771adb4..a9bebbc1 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -215,6 +215,23 @@ def emit(lines: list[str], key: str, value, indent: int = 0) -> None: # --------------------------------------------------------------------------- +#: A `.gitmodules` key as `git config --list` prints it: the section and the +#: variable lower-cased by git, the `` exactly as written. A key with no +#: `` does not match, because git skips one when it clones. +SUBMODULE_KEY_RE = re.compile(r"submodule\.(?P.+)\.(?Ppath|url)") +#: The `git` subcommand that reads and writes a config file: a source's +#: `.gitmodules`, a leg mount's url, the elector's name. +GIT_CONFIG = "config" + + +def _git_path(raw: bytes) -> str: + """A path git printed under `-z`, as `Source.tree()` spells it: UTF-8, + and a byte that is not UTF-8 kept as a surrogate rather than refused. A + gitlink and its `.gitmodules` registration are compared as strings, so + both are decoded here.""" + return raw.decode("utf-8", "surrogateescape") + + class Source: """A repository being read, and nothing more. Nothing here writes to it.""" @@ -271,11 +288,48 @@ def tree(self) -> list[tuple[str, str, str, int]]: continue head, _, path = record.partition(b"\t") mode, kind, oid, size = head.decode().split(maxsplit=3) - out.append((path.decode("utf-8", "surrogateescape"), mode, oid, + out.append((_git_path(path), mode, oid, 0 if size.strip() == "-" else int(size))) del kind return sorted(out) + def registered_submodules(self, tree: list) -> frozenset[str] | None: + """Each path the `.gitmodules` at the pinned commit registers WITH a + url, or None when `git config` cannot read that file. + + A REGISTRATION is a `submodule..path` and a non-empty + `submodule..url` under the same name: the path is what git + looks a gitlink up by, and the url is what it must then find to clone + it. Matched exactly as git matches them, and a key with no `` + is ignored as git ignores it. `tree` is `self.tree()`, which says + whether there is a file to read at all; with none, nothing is + registered. + + `--list`, NOT `--get-regexp`: the second exits 1 both when no key + matches and on a `bad config line`, so a file git cannot read would + come back as one that registers nothing. `--list` exits 0 on any file + it parsed, empty or not, and 128 on one it could not, and None says + so: a read that failed is not an answer (#166). + """ + if not any(path == GITMODULES for path, _, _, _ in tree): + return frozenset() + try: + raw = git_out([GIT_CONFIG, "-z", "--blob", + f"{self.commit}:{GITMODULES}", "--list"], + cwd=self.path, binary=True) + except Refusal: + return None + paths: dict[str, str] = {} + urls: dict[str, str] = {} + for record in raw.split(b"\x00"): + key, _, value = _git_path(record).partition("\n") + match = SUBMODULE_KEY_RE.fullmatch(key) + if match: + registry = paths if match["key"] == "path" else urls + registry[match["name"]] = value + return frozenset(path for name, path in paths.items() + if urls.get(name)) + def commit_count(self) -> int: return int(git_out(["rev-list", "--count", self.commit], cwd=self.path)) @@ -986,10 +1040,21 @@ def _leg_findings(plan: Plan) -> list[str]: "Remediation: send `.gitmodules` and every submodule it registers to ONE " "leg, spec or code, or `drop` them together. To be rid of ONE of several " "registered submodules, adopt it with the others and `git rm` it in the " - "leg afterwards, which takes its registration with it. The assembly root " - "never keeps a submodule of the source's: `execute` writes it a fresh " - "`.gitmodules` for its two mounts, and its `validate` workflow reads every " - "URL in that file as a leg.") + "leg afterwards, which takes its registration with it. A submodule the " + "source does not register -- no `.gitmodules` entry with its path and a " + "url -- cannot be kept in a leg at all: a top-level `git clone " + "--recurse-submodules` skips it, but the assembly's clone fails on it " + "inside the leg. `drop` it, or register it in the source (that entry, " + "committed) and re-run `plan`; a `.gitmodules` that `git config` cannot " + "read is repaired in the source and re-planned the same way, or dropped " + "with every submodule. The assembly root never keeps a submodule of the " + "source's: its `.gitmodules` is reserved for its two leg mounts, and its " + "`validate` workflow reads every URL in that file as a leg.") + +#: Why a kept gitlink with no URL beside it is refused, said in the same +#: words by the two rules that find one. +NO_URL_IN_THE_LEG = ("so the leg that holds the gitlink holds no URL for it " + "and the assembly's recursive clone fails there") def _answered_legs(entries: list, paths: list[str]) -> dict: @@ -1016,38 +1081,63 @@ def _as_entry(path: str, entry: str) -> str: return path if entry == path else f"{path} (entry {entry})" -def submodule_plan_problems(entries: list, tree: list) -> list[tuple[str, str]]: +def submodule_plan_problems(entries: list, tree: list, + registered: frozenset | None + ) -> list[tuple[str, str]]: """`(code, detail)` for each way the plan splits the source's OWN submodules. ONE DEFINITION, asked by `check` for findings and by `execute` for a refusal, so that the plan `check` passes is the plan `execute` will run (#166). `tree` is `Source.tree()`: a gitlink is visible only by its mode, which no path glob in `contracts/path-classification.yaml` can see. + `registered` is `Source.registered_submodules(tree)`, the gitlink paths + the source's `.gitmodules` names with a url, or None when `git config` + cannot read that file. The caller reads both, so this is a function of + its arguments alone. A SUBMODULE IS ONE FACT IN TWO PLACES: the gitlink in the tree and its registration in `.gitmodules`. A plan that sent the two to different legs used to run to the end and say `adoption verified` -- every blob was accounted for -- and leave an assembly whose `git clone - --recurse-submodules` exits 128 on `No url found for submodule path`. So: - - * `root` is not an answer for `.gitmodules` or for any gitlink. The - assembly root has exactly two mounts, `execute` writes it a FRESH - `.gitmodules` for them, and the shape's own `validate.yml` reads every - URL in that file as a leg; a root `.gitmodules` the plan kept fails - verification today whatever it registers (#165). - * When the source HAS a `.gitmodules`, every gitlink the plan KEEPS, in - the spec or the code leg, goes where `.gitmodules` goes -- which is - also why `.gitmodules` cannot be dropped while one is kept. + --recurse-submodules` exits 128 on `No url found for submodule path`. So, + in the order the problems are listed: + + * `root` is not an answer for `.gitmodules` or for any gitlink + (`plan-submodule-root`). The assembly root has exactly two mounts and + its `.gitmodules` is reserved for them -- `execute` writes a fresh one + when the source's moves to a leg or is dropped -- and the shape's own + `validate.yml` reads every URL in that file as a leg; a root + `.gitmodules` the plan kept fails verification today whatever it + registers (#165). + * A `.gitmodules` `git config` cannot read is never taken to register + nothing (`plan-gitmodules-unreadable`). It is a problem when the plan + keeps the file in a leg, where git fails on it when the assembly's + clone recurses there, or keeps any gitlink in a leg, whose + registration then cannot be checked. + * Every gitlink the plan KEEPS, in the spec or the code leg, must be + REGISTERED (`plan-submodule-unregistered`). A leg is WORSE than the + source for one that is not: a top-level `git clone + --recurse-submodules` of the source skips a gitlink with no URL, but + the assembly's clone recurses INTO the leg with `--init` and dies on + it. That holds for an orphan with no `.gitmodules` at all, as `git + add` of an embedded clone leaves one, and for a gitlink beside a + `.gitmodules` that registers other paths. Dropped, it is fine. + * Every REGISTERED gitlink the plan keeps goes where `.gitmodules` goes + (`plan-submodule-split`) -- which is also why `.gitmodules` cannot be + dropped while one is kept. * A gitlink DROPPED beside a `.gitmodules` that went to a leg is allowed. Its registration stays behind in that leg, inert: git walks the gitlinks in the INDEX and looks each one up in `.gitmodules`, never the other way round, so a registration with no gitlink is never cloned and never fails. Dropping all of them with the file is allowed too. - * A source with gitlinks and NO `.gitmodules` is held to the first rule - only. Its orphan gitlinks were already unclonable in the source and - are no worse in a leg; refusing them would refuse plans that adopted - before this check existed. + + THE ORDER IS THE ORDER TO FIX THEM IN, and `execute`'s refusal is named + by the first: a `.gitmodules` in the root makes every registered gitlink + kept in a leg a split, an unreadable one hides which gitlinks are + registered, and an unregistered gitlink may be fixed in the SOURCE, by a + commit and a fresh `plan`, which discards any answer given to a split + before it. A source with neither -- the common case -- returns nothing at once, so `check` and `execute` say exactly what they said before. @@ -1060,22 +1150,65 @@ def submodule_plan_problems(entries: list, tree: list) -> list[tuple[str, str]]: problems = [ ("plan-submodule-root", f"{_as_entry(path, entry)} has leg: root, and the assembly root keeps " - "no submodule of the source's: `execute` writes it a fresh " - f"{GITMODULES} for its two leg mounts") + f"no submodule of the source's: its {GITMODULES} is reserved for its " + "two leg mounts") for path, (leg, entry) in answered.items() if leg == "root"] - if GITMODULES in answered: - registry_leg = answered[GITMODULES][0] + registry_leg = answered.get(GITMODULES, (None, None))[0] + kept = _kept_in_a_leg(answered, gitlinks) + if registered is not None: + return problems + _kept_submodule_problems(kept, registered, + registry_leg) + if kept or registry_leg in EXTRACTED_LEGS: + problems.append(( + "plan-gitmodules-unreadable", + f"`git config` cannot read the source's {GITMODULES}, so which " + "submodules it registers is unknown, and the plan keeps it or a " + "submodule in a leg, where the assembly's recursive clone fails " + "on it")) + return problems + + +def _kept_in_a_leg(answered: dict, gitlinks: list[str]) -> list[tuple]: + """`(path, leg, entry)` for each gitlink `answered` keeps in a leg.""" + return [(path, *answered[path]) for path in gitlinks + if path in answered and answered[path][0] in EXTRACTED_LEGS] + + +def _kept_submodule_problems(kept: list, registered: frozenset, + registry_leg: str | None + ) -> list[tuple[str, str]]: + """The last two rules of `submodule_plan_problems`, for the gitlinks the + plan keeps in a leg: `(path, leg, entry)` each. + + A gitlink is unregistered or split, never both. With no registration it + has no URL in ANY leg, and moving `.gitmodules` would not give it one. + `registry_leg` is None when `.gitmodules` is absent or not yet answered, + which leaves no leg for a gitlink to be split from. + """ + problems = [ + ("plan-submodule-unregistered", + f"the submodule {_as_entry(path, entry)} has leg: {leg} but no " + f"{GITMODULES} entry in the source registers its path with a url, " + + NO_URL_IN_THE_LEG) + for path, leg, entry in kept if path not in registered] + if registry_leg is not None: problems += [ ("plan-submodule-split", f"the submodule {_as_entry(path, entry)} has leg: {leg} but the " - f"source's {GITMODULES} has leg: {registry_leg}, so the leg that " - "holds the gitlink holds no URL for it and the assembly's " - "recursive clone fails there") - for path, (leg, entry) in answered.items() - if leg in EXTRACTED_LEGS and leg != registry_leg] + f"source's {GITMODULES} has leg: {registry_leg}, " + + NO_URL_IN_THE_LEG) + for path, leg, entry in kept + if path in registered and leg != registry_leg] return problems +def _submodule_findings(plan: Plan, source: Source, tree: list) -> list[str]: + """`submodule_plan_problems` as `check` prints them, one FINDING each.""" + return [f"FINDING {code}: {detail}" for code, detail + in submodule_plan_problems(plan.entries, tree, + source.registered_submodules(tree))] + + def _topics_line(topic: str, local: bool) -> str: """The `topics` plan line, the one `scaffold-project.py` also prints. @@ -1137,8 +1270,7 @@ def cmd_check(args) -> int: tree_paths = [path for path, _, _, _ in tree] findings.extend(_coverage_findings(entry_paths, tree_paths)) findings.extend(_leg_findings(plan)) - findings.extend(f"FINDING {code}: {detail}" for code, detail - in submodule_plan_problems(plan.entries, tree)) + findings.extend(_submodule_findings(plan, source, tree)) names = plan.names() pins = set(plan.pins) @@ -1226,10 +1358,13 @@ def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: An unanswered question is never an implicit `root`, and a plan written against a tree that has since moved proves nothing about the tree that would be split — which is how a path goes missing. And a plan that - splits a submodule from its registration, or keeps one in the root, - builds an assembly that cannot be cloned: refused HERE, before either leg - exists, because once `_create_leg_remotes` has run, a re-run with the - corrected plan meets two legs that already exist (#166). + splits a submodule from its registration, keeps one nothing registers, + or keeps one in the root, builds an assembly that cannot be cloned: + refused HERE, before either leg exists, because once + `_create_leg_remotes` has run, a re-run with the corrected plan meets two + legs that already exist (#166). The refusal is named by the FIRST + problem, which `submodule_plan_problems` lists in the order to fix them + in, and its detail names up to eight of them and counts the rest. """ unresolved = [e for e in plan.entries if e.get("leg") is None] if unresolved: @@ -1248,7 +1383,9 @@ def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: "Remediation: re-run `plan`, re-answer anything new, then " "`check`. Splitting a tree the plan has not seen is how a path " "goes missing.") - problems = submodule_plan_problems(plan.entries, source.tree()) + tree = source.tree() + problems = submodule_plan_problems(plan.entries, tree, + source.registered_submodules(tree)) if problems: details = [detail for _, detail in problems[:8]] if len(problems) > 8: @@ -1401,7 +1538,7 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, for role, path in (("spec", spec_path), ("code", code_path)): run(["git", *FILE_PROTOCOL, "submodule", "add", "-q", str(work_root / names[role]), path], cwd=assembly) - run(["git", "config", "-f", GITMODULES, f"submodule.{path}.url", + run(["git", GIT_CONFIG, "-f", GITMODULES, f"submodule.{path}.url", urls[role]], cwd=assembly) run(["git", "remote", "set-url", "origin", urls[role]], cwd=assembly / path) @@ -1914,7 +2051,7 @@ def _check_names(policy: NamingPolicy, names: dict[str, str], def _elector() -> str: try: - return git_out(["config", "user.name"], cwd=SHAPE_ROOT) + return git_out([GIT_CONFIG, "user.name"], cwd=SHAPE_ROOT) except Refusal: return "" diff --git a/contracts/path-classification.yaml b/contracts/path-classification.yaml index 173dffdc..4485ee79 100644 --- a/contracts/path-classification.yaml +++ b/contracts/path-classification.yaml @@ -100,23 +100,30 @@ rules: # blob (#166). No glob here can tell a gitlink from a file -- only its mode # in `git ls-tree` can -- so the question is asked on the one file every # registration lives in, and `adopt-project.py check` holds the gitlinks to - # the answer. Without this rule the default asked it as if `.gitmodules` - # were any other file, and `root` looked like an answer. + # the answer and refuses to keep one this file does not register. Without + # this rule the default asked it as if `.gitmodules` were any other file, + # and `root` looked like an answer. `shape-doctor.py --placement-plan` + # prints this `reason:` and `question:` too, for a project already split + # and a file no `check` or `execute` ever reads, so what refuses a wrong + # answer is named as the tool and not as those two subcommands. - id: ambiguous-gitmodules paths: - ".gitmodules" leg: ambiguous reason: >- - The repository registers submodules of its own. `.gitmodules` and the - gitlinks it names are one fact recorded in two places, and which leg - those dependencies belong to is a fact about this repository. + `.gitmodules` is where a repository registers its submodules. Each + registration and the gitlink it names are one fact recorded in two + places, and which leg those dependencies belong to is a fact about this + repository. question: >- A source's own `.gitmodules` goes to the SAME leg as every submodule it registers; `execute` writes the assembly root a fresh one for its two mounts. Which leg owns the submodules, spec or code? Send `.gitmodules` - there and every submodule path with it, or drop them all together. - `root` is not an answer for this file or for any submodule: `check` - finds it and `execute` refuses it, as they do a plan that splits them. + there and every submodule path with it, or drop them all together. A + submodule it does not register with a path and a url cannot be kept in + a leg at all: drop it, or register it first. `root` is not an answer + for this file or for any submodule, and `adopt-project.py` refuses a + plan that gives it, as it does a plan that splits them. # -- the assembly root ---------------------------------------------------- diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index 8c3ed4cd..1ed3ef0b 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -28,6 +28,24 @@ reason="git filter-repo is not installed: `pip install git-filter-repo`") +def adopt(command: str, plan: Path, *args: str): + """`adopt-project.py --plan [args]`.""" + return run_script(ADOPT, command, "--plan", str(plan), *args) + + +def add_submodule(source: Path, dependency: Path, path: str) -> None: + """`git submodule add` of the local `dependency` at `path`, as a person + adding a dependency to `source` would.""" + git(*FILE_PROTOCOL, "submodule", "add", "-q", str(dependency), path, + cwd=source) + + +def commit_as_source_human(source: Path, message: str) -> None: + git("-c", "user.name=Source Human", "-c", + "user.email=source@invalid.example", "commit", "-qm", message, + cwd=source) + + @needs_filter_repo @pytest.mark.parametrize("dependency_leg", ["code", "spec"]) def test_adopt_moves_existing_gitmodules_with_its_dependency(tmp_path, @@ -39,11 +57,8 @@ def test_adopt_moves_existing_gitmodules_with_its_dependency(tmp_path, git("clone", "-q", "--bare", str(dependency), str(dependency_remote), cwd=tmp_path) source = make_source_repo(tmp_path / "Thing") - git(*FILE_PROTOCOL, "submodule", "add", "-q", str(dependency_remote), - "upstream/dependency", cwd=source) - git("-c", "user.name=Source Human", "-c", - "user.email=source@invalid.example", "commit", "-qm", - "Pin existing dependency", cwd=source) + add_submodule(source, dependency_remote, "upstream/dependency") + commit_as_source_human(source, "Pin existing dependency") source_head = git("rev-parse", "HEAD", cwd=source).stdout.strip() original_modules = git("rev-parse", "HEAD:.gitmodules", cwd=source).stdout.strip() @@ -57,11 +72,11 @@ def test_adopt_moves_existing_gitmodules_with_its_dependency(tmp_path, (".gitmodules", dependency_leg), ("upstream/", dependency_leg)): resolve(plan, path, leg) - checked = run_script(ADOPT, "check", "--plan", str(plan)) + checked = adopt("check", plan) assert checked.returncode == 0, checked.stderr + checked.stdout - result = run_script(ADOPT, "execute", "--plan", str(plan), "--yes", - "--local-remote-dir", str(tmp_path / "remotes"), - "--work-dir", str(tmp_path / "work")) + result = adopt("execute", plan, "--yes", + "--local-remote-dir", str(tmp_path / "remotes"), + "--work-dir", str(tmp_path / "work")) assert result.returncode == 0, result.stderr + result.stdout clone = tmp_path / "assembly" @@ -144,12 +159,9 @@ def build(registered=("upstream/kept", "upstream/moved"), tree=None): edits=()) source = make_source_repo(tmp_path / "source", tree=tree) for path in registered: - git(*FILE_PROTOCOL, "submodule", "add", "-q", str(dependency), - path, cwd=source) + add_submodule(source, dependency, path) if registered: - git("-c", "user.name=Source Human", "-c", - "user.email=source@invalid.example", "commit", "-qm", - "Register dependencies", cwd=source) + commit_as_source_human(source, "Register dependencies") work = tmp_path / "work" work.mkdir() names = {role: f"Northwind-{role}" for role in ("spec", "code")} @@ -254,81 +266,149 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( # plan `check` passes is a plan `execute` runs. None of these needs # `git filter-repo` but the one that reaches `execute` past its preflight. -#: The two ids, as `check` prints them and `execute` raises them. +#: The ids, as `check` prints them and `execute` raises them. SPLIT = "plan-submodule-split" ROOT = "plan-submodule-root" +UNREGISTERED = "plan-submodule-unregistered" +UNREADABLE = "plan-gitmodules-unreadable" + +#: The plan entries these sources ask about, and the file every plan below is +#: written to, each in a directory of its own. +GITMODULES = ".gitmodules" +UPSTREAM = "upstream/" +EXTRA = "extra/" +PLAN_FILE = "adoption-plan.yaml" + +#: What the assembly root's `.gitmodules` is, in the `plan-submodule-root` +#: detail and in the remediation: true whether or not `.gitmodules` is the +#: entry answered `root`, which "`execute` writes it a fresh one" was not. +RESERVED = "is reserved for its two leg mounts" #: A source whose plan asks nothing but the submodule's questions: `specs/` #: and `src/` classify themselves and `README.md` stays in the root. SMALL_TREE = {"README.md": "# Thing\n", "src/app/util.py": "VALUE = 1\n", "specs/001-feature/tasks.md": "- [ ] one\n"} +# The sources these tests answer plans for, as `registered_source` +# arguments: a test names the one it answers, and `plan_of` plans each once. + +#: `upstream/dependency`, registered with a url: +REGISTERED = {} +#: `upstream/orphan`, and no `.gitmodules` at all: +LONE_ORPHAN = {"registered": (), "orphan": "upstream/orphan"} +#: `upstream/dependency` registered, and `extra/orphan` beside it not: +BESIDE = {"orphan": "extra/orphan"} +#: `upstream/dependency`, beside a `.gitmodules` with a bad config line: +UNREADABLE_FILE = {"gitmodules": '[submodule "upstream/dependency"\n' + "\tpath = upstream/dependency\n"} +#: `d0/s0` to `d9/s9` registered, and `extra/orphan` beside them not: +CROWDED = {**BESIDE, "registered": tuple(f"d{i}/s{i}" for i in range(10))} + def registered_source(base: Path, registered=("upstream/dependency",), - orphan: str | None = None) -> Path: + orphan: str | None = None, + gitmodules: str | None = None) -> Path: """A source registering a real dependency at each of `registered`. `orphan` adds a gitlink at that path with NO registration, written straight into the index as `git add` of an embedded clone would write it, so that a source with a submodule and no `.gitmodules` is planned too. + `gitmodules`, when given, replaces the file `git submodule add` wrote, as + a hand edit would. """ dependency = make_source_repo(base / "dependency", tree={"value.txt": "dependency\n"}, edits=()) source = make_source_repo(base / "Thing", tree=SMALL_TREE) for path in registered: - git(*FILE_PROTOCOL, "submodule", "add", "-q", str(dependency), path, - cwd=source) + add_submodule(source, dependency, path) if orphan: head = git("rev-parse", "HEAD", cwd=dependency).stdout.strip() git("update-index", "--add", "--cacheinfo", f"160000,{head},{orphan}", cwd=source) - git("-c", "user.name=Source Human", "-c", - "user.email=source@invalid.example", "commit", "-qm", - "Register dependencies", cwd=source) + if gitmodules is not None: + (source / GITMODULES).write_text(gitmodules, encoding="utf-8") + git("add", "--", GITMODULES, cwd=source) + commit_as_source_human(source, "Register dependencies") return source -@pytest.fixture(scope="module") -def registered_plan(tmp_path_factory): - """ONE source with `upstream/dependency` registered, and its plan. - - Each check test answers a COPY of the plan; the source is only ever read. - """ - base = tmp_path_factory.mktemp("registered") - source = registered_source(base) - plan = base / "adoption-plan.yaml" +def planned(source: Path, base: Path): + """`(plan, completed)`: `plan` run on `source`, into a directory of its + own under `base`.""" + plan = base / "plan" / PLAN_FILE + plan.parent.mkdir() written = write_plan(source, plan) assert written.returncode == 0, written.stderr + written.stdout return plan, written -def answered(plan: Path, tmp_path: Path, answers) -> Path: - """A copy of `plan` with each `(path, leg)` in `answers` resolved; a leg +@pytest.fixture(scope="module") +def plan_of(tmp_path_factory): + """`plan_of(SOURCE)`: `planned` for one of the sources above, built once + per module. The source is only ever read, and each test answers a COPY + of its plan.""" + made: dict[str, tuple] = {} + + def build(source: dict): + key = repr(sorted(source.items())) + if key not in made: + base = tmp_path_factory.mktemp("source") + made[key] = planned(registered_source(base, **source), base) + return made[key] + + return build + + +def answered(plan: Path, tmp_path: Path, answers: dict) -> Path: + """A copy of `plan` with each `{path: leg}` in `answers` resolved; a leg of None leaves that entry's question unanswered.""" - copy = tmp_path / "adoption-plan.yaml" - copy.write_text(plan.read_text(encoding="utf-8"), encoding="utf-8") - for path, leg in answers: + copy = tmp_path / PLAN_FILE + shutil.copyfile(plan, copy) + for path, leg in answers.items(): if leg is not None: resolve(copy, path, leg) return copy +def plan_rows(plan: Path) -> dict: + """`{path: entry}` for every entry of the plan file.""" + return {str(row["path"]): row for row in load_yaml(plan)["paths"]} + + +def check_plan(plan: Path): + return adopt("check", plan) + + +def findings(stderr: str) -> list[tuple[str, str]]: + """`(code, detail)` for each `FINDING : ` line `check` + printed, in its order.""" + return [tuple(line.split(" ", 1)[1].split(": ", 1)) + for line in stderr.splitlines() if line.startswith("FINDING ")] + + def finding_codes(stderr: str) -> list[str]: - return [line.split(":", 1)[0].split()[1] for line in stderr.splitlines() - if line.startswith("FINDING ")] + return [code for code, _ in findings(stderr)] -def test_plan_asks_the_submodule_question_on_gitmodules(registered_plan): - plan, written = registered_plan - rows = {str(row["path"]): row for row in load_yaml(plan)["paths"]} - row = rows[".gitmodules"] +def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): + plan, written = plan_of(REGISTERED) + row = plan_rows(plan)[GITMODULES] + question = row["question"] assert row["leg"] is None assert row["review_required"] is True assert row["rule"] == "ambiguous-gitmodules" assert ("A source's own `.gitmodules` goes to the SAME leg as every " "submodule it registers; `execute` writes the assembly root a " - "fresh one for its two mounts.") in row["question"] - assert "`root` is not an answer" in row["question"] + "fresh one for its two mounts.") in question + assert ("A submodule it does not register with a path and a url cannot " + "be kept in a leg at all") in question + # `shape-doctor.py --placement-plan` prints this question too, for a file + # no `check` or `execute` ever reads: the TOOL refuses, true in both. + assert ("`root` is not an answer for this file or for any submodule, and " + "`adopt-project.py` refuses a plan that gives it") in question + # Printed for an EMPTY `.gitmodules` too, so it claims no registration. + assert row["reason"].startswith( + "`.gitmodules` is where a repository registers its submodules.") # Printed with the other unresolved paths, which is where it is read. assert "goes to the SAME leg as every submodule" in written.stdout @@ -350,11 +430,10 @@ def test_plan_asks_the_submodule_question_on_gitmodules(registered_plan): (None, "root", ["plan-unresolved", ROOT]), ]) def test_check_holds_a_submodule_to_the_leg_of_its_registration( - registered_plan, tmp_path, modules_leg, dependency_leg, expected): - plan = answered(registered_plan[0], tmp_path, - ((".gitmodules", modules_leg), - ("upstream/", dependency_leg))) - result = run_script(ADOPT, "check", "--plan", str(plan)) + plan_of, tmp_path, modules_leg, dependency_leg, expected): + plan = answered(plan_of(REGISTERED)[0], tmp_path, + {GITMODULES: modules_leg, UPSTREAM: dependency_leg}) + result = check_plan(plan) assert finding_codes(result.stderr) == expected, result.stderr if not expected: assert result.returncode == 0, result.stderr @@ -368,7 +447,9 @@ def test_check_holds_a_submodule_to_the_leg_of_its_registration( f"{dependency_leg} but the source's .gitmodules has leg: " f"{modules_leg}") in result.stderr if modules_leg == "root": - assert f"FINDING {ROOT}: .gitmodules has leg: root" in result.stderr + assert (f"FINDING {ROOT}: .gitmodules has leg: root, and the " + "assembly root keeps no submodule of the source's: its " + f".gitmodules {RESERVED}") in result.stderr if dependency_leg == "root": assert (f"FINDING {ROOT}: upstream/dependency (entry upstream/) has " "leg: root") in result.stderr @@ -377,14 +458,11 @@ def test_check_holds_a_submodule_to_the_leg_of_its_registration( def test_check_finds_a_submodule_a_rule_classified_silently(tmp_path): """`tools/dep` is under `tools/**`, so `plan` calls it code and asks nothing; only `.gitmodules` is asked. Answered `spec`, the two split.""" - source = registered_source(tmp_path, registered=("tools/dep",)) - plan = tmp_path / "plan" / "adoption-plan.yaml" - plan.parent.mkdir() - assert write_plan(source, plan).returncode == 0 - rows = {str(row["path"]): row for row in load_yaml(plan)["paths"]} - assert rows["tools/"]["leg"] == "code" - resolve(plan, ".gitmodules", "spec") - result = run_script(ADOPT, "check", "--plan", str(plan)) + plan = planned(registered_source(tmp_path, registered=("tools/dep",)), + tmp_path)[0] + assert plan_rows(plan)["tools/"]["leg"] == "code" + resolve(plan, GITMODULES, "spec") + result = check_plan(plan) assert result.returncode == 1, result.stderr + result.stdout assert (f"FINDING {SPLIT}: the submodule tools/dep (entry tools/) has " "leg: code but the source's .gitmodules has leg: spec") \ @@ -392,26 +470,97 @@ def test_check_finds_a_submodule_a_rule_classified_silently(tmp_path): @pytest.mark.parametrize("leg, expected", [ - ("code", []), + ("code", [UNREGISTERED]), + ("spec", [UNREGISTERED]), + ("drop", []), ("root", [ROOT]), ]) -def test_an_orphan_gitlink_may_go_to_a_leg_but_not_the_root(tmp_path, leg, - expected): - """No `.gitmodules` at all: the gitlink was unclonable in the source too, - and a leg is no worse a place for it. The root is still not one.""" - source = registered_source(tmp_path, registered=(), - orphan="upstream/orphan") - assert git("ls-tree", "HEAD", "--", ".gitmodules", - cwd=source).stdout == "" - plan = tmp_path / "plan" / "adoption-plan.yaml" - plan.parent.mkdir() - assert write_plan(source, plan).returncode == 0 - resolve(plan, "upstream/", leg) - result = run_script(ADOPT, "check", "--plan", str(plan)) +def test_an_orphan_gitlink_may_be_dropped_but_not_kept(plan_of, tmp_path, + leg, expected): + """No `.gitmodules` at all. The source's own `git clone + --recurse-submodules` skips a gitlink with no URL; the assembly's + recurses INTO the leg and dies on it, so a leg is worse than the source + for it, not "no worse". Dropped, it goes nowhere.""" + plan = answered(plan_of(LONE_ORPHAN)[0], tmp_path, {UPSTREAM: leg}) + assert GITMODULES not in plan_rows(plan) + result = check_plan(plan) + assert finding_codes(result.stderr) == expected, result.stderr + assert result.returncode == (1 if expected else 0) + if UNREGISTERED in expected: + assert (f"FINDING {UNREGISTERED}: the submodule upstream/orphan " + f"(entry upstream/) has leg: {leg} but no .gitmodules entry " + "in the source registers its path with a url") \ + in result.stderr + + +@pytest.mark.parametrize("orphan_leg, expected", [ + ("code", [UNREGISTERED]), + # In the OTHER leg it is still unregistered, and only that: no leg holds + # a URL for it, so moving `.gitmodules` would not help. + ("spec", [UNREGISTERED]), + ("drop", []), +]) +def test_check_refuses_to_keep_a_gitlink_its_gitmodules_does_not_register( + plan_of, tmp_path, orphan_leg, expected): + plan = answered(plan_of(BESIDE)[0], tmp_path, + {GITMODULES: "code", UPSTREAM: "code", EXTRA: orphan_leg}) + result = check_plan(plan) + assert finding_codes(result.stderr) == expected, result.stderr + assert result.returncode == (1 if expected else 0) + if expected: + assert (f"FINDING {UNREGISTERED}: the submodule extra/orphan (entry " + f"extra/) has leg: {orphan_leg}") in result.stderr + + +@pytest.mark.parametrize("gitmodules, expected", [ + ('[submodule "upstream/dependency"]\n\tpath = upstream/dependency\n', + [UNREGISTERED]), + ('[submodule "upstream/dependency"]\n\tpath = upstream/dependency\n' + "\turl =\n", [UNREGISTERED]), + # git skips a `submodule.path` with no ``, so this does too. + ("[submodule]\n\tpath = upstream/dependency\n\turl = ../dependency\n", + [UNREGISTERED]), + ('[submodule "elsewhere"]\n\tpath = elsewhere/dependency\n' + "\turl = ../dependency\n", [UNREGISTERED]), + ("", [UNREGISTERED]), + # Section and key names are case-insensitive to git, and so here. + ('[Submodule "upstream/dependency"]\n\tPATH = upstream/dependency\n' + "\tURL = ../dependency\n", []), +], ids=["no-url", "empty-url", "no-name", "another-path", "empty-file", + "upper-case-keys"]) +def test_a_registration_is_a_named_path_with_a_url(tmp_path, gitmodules, + expected): + """`upstream/dependency` is a gitlink in every case; only the + `.gitmodules` beside it, both kept in code, changes.""" + plan = answered( + planned(registered_source(tmp_path, gitmodules=gitmodules), + tmp_path)[0], + tmp_path, {GITMODULES: "code", UPSTREAM: "code"}) + result = check_plan(plan) assert finding_codes(result.stderr) == expected, result.stderr assert result.returncode == (1 if expected else 0) +@pytest.mark.parametrize("modules_leg, dependency_leg, expected", [ + ("code", "code", [UNREADABLE]), + # The file alone, kept, fails the leg's recursive clone on its own. + ("code", "drop", [UNREADABLE]), + ("drop", "spec", [UNREADABLE]), + ("drop", "drop", []), + ("root", "drop", [ROOT]), +]) +def test_a_gitmodules_git_cannot_read_is_not_one_that_registers_nothing( + plan_of, tmp_path, modules_leg, dependency_leg, expected): + plan = answered(plan_of(UNREADABLE_FILE)[0], tmp_path, + {GITMODULES: modules_leg, UPSTREAM: dependency_leg}) + result = check_plan(plan) + assert finding_codes(result.stderr) == expected, result.stderr + assert result.returncode == (1 if expected else 0) + if UNREADABLE in expected: + assert (f"FINDING {UNREADABLE}: `git config` cannot read the " + "source's .gitmodules") in result.stderr + + @pytest.fixture(scope="module") def adopter(): """`adopt-project.py` as a module, for the refusal `execute` makes.""" @@ -422,53 +571,105 @@ def adopter(): return module -@pytest.mark.parametrize("modules_leg, dependency_leg, code", [ - ("code", "spec", SPLIT), - ("drop", "code", SPLIT), - ("root", "root", ROOT), - ("spec", "spec", None), - ("code", "drop", None), -]) -def test_execute_refuses_what_check_finds(adopter, registered_plan, tmp_path, - modules_leg, dependency_leg, code): - plan_path = answered(registered_plan[0], tmp_path, - ((".gitmodules", modules_leg), - ("upstream/", dependency_leg))) +def refused_by_execute(adopter, plan_path: Path, tmp_path: Path): + """What `_refuse_an_unrunnable_plan` raises for `plan_path`, or None.""" plan = adopter.Plan.load(plan_path) source = plan.open_source(None, tmp_path / "work") - if code is None: + try: adopter._refuse_an_unrunnable_plan(plan, source) + except adopter.Refusal as refusal: + return refusal + return None + + +@pytest.mark.parametrize("source, answers, code", [ + (REGISTERED, {GITMODULES: "code", UPSTREAM: "spec"}, SPLIT), + (REGISTERED, {GITMODULES: "drop", UPSTREAM: "code"}, SPLIT), + (REGISTERED, {GITMODULES: "root", UPSTREAM: "root"}, ROOT), + (BESIDE, {GITMODULES: "code", UPSTREAM: "code", EXTRA: "code"}, + UNREGISTERED), + (LONE_ORPHAN, {UPSTREAM: "code"}, UNREGISTERED), + (UNREADABLE_FILE, {GITMODULES: "code", UPSTREAM: "code"}, UNREADABLE), + (REGISTERED, {GITMODULES: "spec", UPSTREAM: "spec"}, None), + (REGISTERED, {GITMODULES: "code", UPSTREAM: "drop"}, None), + (BESIDE, {GITMODULES: "code", UPSTREAM: "code", EXTRA: "drop"}, None), + (LONE_ORPHAN, {UPSTREAM: "drop"}, None), +]) +def test_execute_refuses_what_check_finds(adopter, plan_of, tmp_path, + source, answers, code): + plan = answered(plan_of(source)[0], tmp_path, answers) + found = findings(check_plan(plan).stderr) + refusal = refused_by_execute(adopter, plan, tmp_path) + if code is None: + assert found == [] and refusal is None, (found, refusal) return - with pytest.raises(adopter.Refusal) as refused: - adopter._refuse_an_unrunnable_plan(plan, source) - assert refused.value.code == code - assert "upstream/dependency" in refused.value.detail - assert "`git rm` it in the leg afterwards" in refused.value.remediation - assert "never keeps a submodule" in refused.value.remediation + # ONE definition: the refusal is what `check` found, in its words. + assert refusal is not None, found + assert refusal.code == found[0][0] == code + assert refusal.detail == "; ".join(detail for _, detail in found) + assert "`git rm` it in the leg afterwards" in refusal.remediation + assert "never keeps a submodule" in refusal.remediation + assert RESERVED in refusal.remediation + assert ("cannot be kept in a leg at all" in refusal.remediation + and "and re-run `plan`" in refusal.remediation) + + +@pytest.mark.parametrize("first_leg, code", [ + ("root", ROOT), + ("code", UNREGISTERED), +]) +def test_a_mixed_refusal_is_named_by_the_problem_to_fix_first( + adopter, plan_of, tmp_path, first_leg, code): + """`.gitmodules` and `extra/orphan` in code, `d1/` to `d9/` split to + spec, and `d0/` in the root or not: eleven problems or ten, listed root, + then unregistered, then split. The refusal takes the FIRST one's code + and names eight of them, then counts the rest.""" + answers = {GITMODULES: "code", EXTRA: "code", "d0/": first_leg, + **{f"d{i}/": "spec" for i in range(1, 10)}} + plan = answered(plan_of(CROWDED)[0], tmp_path, answers) + found = findings(check_plan(plan).stderr) + assert [found_code for found_code, _ in found] == ( + [ROOT] * (first_leg == "root") + [UNREGISTERED] + [SPLIT] * 9) + refusal = refused_by_execute(adopter, plan, tmp_path) + assert refusal.code == code + assert refusal.detail.split("; ") == ( + [detail for _, detail in found[:8]] + [f"and {len(found) - 8} more"]) + + +def source_state(source: Path) -> list[str]: + """HEAD, every ref and `git status`: what a refused `execute` must not + move.""" + return [git(*args, cwd=source).stdout for args in ( + ("rev-parse", "HEAD"), ("for-each-ref",), ("status", "--porcelain"))] @needs_filter_repo -def test_execute_refuses_a_split_submodule_before_any_leg_exists(tmp_path): - """Before #166 this plan was extracted, mounted, pushed and verified, and - its assembly's recursive clone then exited 128. Refused now, and refused - before `_create_leg_remotes`: a re-run must not meet two legs that a - refused plan already made.""" - source = registered_source(tmp_path) - head = git("rev-parse", "HEAD", cwd=source).stdout.strip() - plan = tmp_path / "plan" / "adoption-plan.yaml" - plan.parent.mkdir() - assert write_plan(source, plan).returncode == 0 - resolve(plan, ".gitmodules", "code") - resolve(plan, "upstream/", "spec") - remotes = tmp_path / "remotes" - result = run_script(ADOPT, "execute", "--plan", str(plan), "--yes", - "--local-remote-dir", str(remotes), - "--work-dir", str(tmp_path / "work")) +@pytest.mark.parametrize("source, answers, refused", [ + (REGISTERED, {GITMODULES: "code", UPSTREAM: "spec"}, + f"REFUSED {SPLIT}: the submodule upstream/dependency (entry upstream/) " + "has leg: spec but the source's .gitmodules has leg: code"), + (LONE_ORPHAN, {UPSTREAM: "code"}, + f"REFUSED {UNREGISTERED}: the submodule upstream/orphan (entry " + "upstream/) has leg: code but no .gitmodules entry in the source " + "registers its path with a url"), +], ids=["split", "unregistered-orphan"]) +def test_execute_refuses_a_submodule_plan_before_anything_exists( + tmp_path, source, answers, refused): + """Before #166 both plans were extracted, mounted, pushed and verified, + and each assembly's recursive clone then exited 128 -- the orphan's + although the source's own recursive clone exits 0. Refused now, and + refused before `_create_leg_remotes`: a re-run must not meet two legs + that a refused plan already made.""" + thing = registered_source(tmp_path, **source) + before = source_state(thing) + plan = answered(planned(thing, tmp_path)[0], tmp_path, answers) + remotes, work = tmp_path / "remotes", tmp_path / "work" + result = adopt("execute", plan, "--yes", "--local-remote-dir", + str(remotes), "--work-dir", str(work)) assert result.returncode == 2, result.stderr + result.stdout - assert (f"REFUSED {SPLIT}: the submodule upstream/dependency (entry " - "upstream/) has leg: spec but the source's .gitmodules has leg: " - "code") in result.stderr + assert refused in result.stderr assert "creating the leg repositories" not in result.stdout - assert list(remotes.glob("*.git")) == [], "a refused plan made a leg" - assert git("rev-parse", "HEAD", cwd=source).stdout.strip() == head - assert git("branch", "--list", "adopt/*", cwd=source).stdout == "" + # Nothing was made but the empty work directory `--work-dir` names. + assert not remotes.exists(), "a refused plan made the remotes directory" + assert list(work.iterdir()) == [], "a refused plan wrote into its work" + assert source_state(thing) == before From 326fb2598bafca11c219876753ffe68d16b50355 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:06:59 +0000 Subject: [PATCH 03/12] adopt-project.py: `plan` asks the `.gitmodules` question on every entry that holds one of the source's submodules, whatever rule matched it, so it no longer proposes the `root` that `check` refuses, and `check` now prints the submodule remediation after its findings (#166) The second review of aaa9099, by lane openRepoShape-2, found that `plan` proposes what `check` refuses. `walk` sees paths, not modes. So a submodule vendored at `.specify/vendored`, `.cursor` or `.agents` came out `leg: root` with no question (rules `root-spec-kit` and `root-assistant-instructions`). One at `docs/theme` came out `spec` (`spec-governance`), also unasked. `check` then answered `plan-submodule-root`, or a split once `.gitmodules` was answered. Reproduced at 09d7889: `.specify/` written `root`, then `plan-submodule-root: .specify/vendored (entry .specify/)`. - `ask_about_submodules` runs in `cmd_plan` right after `walk`. Every entry that `_covering` says covers a gitlink takes the verdict `.gitmodules` itself gets: `leg: null`, `review_required: true`, rule `ambiguous-gitmodules` and its reason. It also takes that question, plus "This entry holds the submodule(s) : it is asked this question, not classified by its rule ``." When the entry holds other paths too, the question adds that it can be split. `walk`'s folding is unchanged; only the covering entry is replaced. - `check` on the untouched plan now reports `plan-unresolved` there, never `plan-submodule-root`. - The 2026-09-02 ruling is unchanged for files. The `ambiguous-gitmodules` header comment says that a vendored SUBMODULE under a root-ruled path is asked rather than rooted, because the root keeps no submodule of the source's. - A source with no gitlink gets its entries back untouched. On the suite's SYNTHETIC_TREE source, these are byte-identical to main's: the plan file, `plan`'s output, and `check`'s output both unresolved and resolved. Not fixed here, and now said so: `execute` refuses every SUBMODULE problem `check` finds, but not yet every finding. An entry that is uncovered, covered twice or has a misspelled leg still reaches the split; that is #168, which merges right after this. The docstrings that said "the plan `check` passes is the plan `execute` runs" now say exactly this, in `submodule_plan_problems`, `_answered_legs` and `_refuse_an_unrunnable_plan`. The same review's smaller findings: - The question and the remediation offered no way to be rid of one submodule except `git rm` after adopting. The rule has always allowed dropping one on its own, and both texts now say so: its registration stays behind in the leg, inert. - `check` prints SUBMODULE_REMEDIATION once after its findings when any of them is a submodule problem, as `execute`'s refusal does. `_submodule_findings` returns it, so `cmd_check` gains no branch. - The module docstring's list of what `check` validates now includes the submodule rule. docs/cli.md is generated from the first line only and is unchanged. - #162's kept-root mount test and the `_mount_the_legs` comment are re-labelled as mount defence in depth for a plan that `check` and `execute` refuse (#166). In `_mount_the_legs` only the comment changes. Tests in tests/test_adopt_submodules.py, 52 -> 77 cases: - `plan` asks on the entry holding a gitlink at `.specify/vendored`, `.cursor`, `.agents`, `docs/theme` and `upstream/dependency`, and on one entry holding two gitlinks. Each row is the `.gitmodules` row apart from its path and size. Its question names the gitlinks and the rule, and `check` on the untouched plan reports only `plan-unresolved`. - The `tools/dep` test now answers `tools/` as its rule would have, since `plan` asks there. - Branches no test reached before: a `.gitmodules` with no gitlink beside it; two submodules split across legs; a gitlink that is its own entry. Also uncovered, doubly covered and bad-leg entries: `check` reports each by its own finding, and the helper skips it. - The `check` matrix asserts the remediation is printed once, after the findings, exactly when there is a submodule finding. - One `needs_filter_repo` end-to-end test runs `execute` on accepted plans: both code, both spec, and the submodule dropped on its own. `git clone --recurse-submodules` of the assembly and `bootstrap.py` in the clone then both exit 0. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 138 ++++++++++---- contracts/path-classification.yaml | 23 ++- tests/test_adopt_submodules.py | 282 +++++++++++++++++++++++++++-- 3 files changed, 383 insertions(+), 60 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index a9bebbc1..ffc25368 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -19,11 +19,14 @@ plan walks the source at its default branch, classifies every top-level path against `contracts/path-classification.yaml`, and writes - `adoption-plan.yaml`. Paths it cannot honestly call carry + `adoption-plan.yaml`. Paths it cannot honestly call, and every + path holding one of the source's own submodules, carry `leg: null`, `review_required: true` and the QUESTION to ask. check validates a plan against the source: every path covered exactly once, no unresolved legs, leg names conforming to the naming - policy. It prints what will happen and changes nothing. + policy, and the source's own submodules kept on one leg with + their `.gitmodules`, registered there and never in the root. It + prints what will happen and changes nothing. execute creates the two legs, extracts them with `git filter-repo`, makes the one split commit on a branch of the source, sets the `xf-project-` topic on all three (skipped for local remotes), @@ -420,6 +423,50 @@ def walk(policy: PathPolicy, files: list[tuple[str, int]], return sorted(entries, key=lambda e: e.path) +def ask_about_submodules(policy: PathPolicy, entries: list[Entry], + tree: list) -> list[Entry]: + """`entries`, with every entry that holds a gitlink of the source's + ASKED the `.gitmodules` question instead of answered by its rule (#166). + + `walk` sees paths, not modes, so it classifies a gitlink as the file its + path looks like: a submodule vendored at `.specify/vendored` or `.cursor` + comes out `root` by the 2026-09-02 ruling, one under `docs/` comes out + `spec`, and neither is asked. `check` refuses the first and may find the + second split from its `.gitmodules`, and `plan` must never propose what + `check` refuses. So each entry that covers a gitlink, WHATEVER RULE + MATCHED IT, gets `leg: null` and the question `.gitmodules` itself is + asked, naming the submodules it holds and the rule it is no longer + classified by. The ruling still roots the FILES under those paths: an + entry holding more than its submodules says so, and is split by the + person answering it. `walk`'s folding is left alone; only the covering + entry is replaced. A source with no gitlink, the common case, gets its + entries back untouched. + """ + gitlinks = [path for path, mode, _, _ in tree if mode == GITLINK_MODE] + if not gitlinks: + return entries + asked = policy.classify_file(GITMODULES) + out = [] + for entry in entries: + held = [path for path in gitlinks if _covering([entry.path], path)] + out.append(_asked_about(entry, held, asked) if held else entry) + return out + + +def _asked_about(entry: Entry, held: list[str], asked: Verdict) -> Entry: + """`entry` as `ask_about_submodules` writes it: the `.gitmodules` + verdict, whose `leg:` is null, and its question naming `held`.""" + question = (f"{asked.question or ''} This entry holds the submodule(s) " + f"{', '.join(held)}: it is asked this question, not " + f"classified by its rule `{entry.rule}`.") + if entry.files > len(held): + question += (" Split the entry if its other paths belong in a " + "different leg from the submodules.") + return Entry(entry.path, Verdict(None, asked.rule, asked.reason, + asked.confidence, question), + entry.files, entry.bytes) + + def cross_leg_references(source: "Source", entries: list["Entry"], spec_path: str, code_path: str) -> list[str]: """Files that will move to the CODE leg and name a path in the SPEC leg. @@ -758,7 +805,9 @@ def cmd_plan(args) -> int: names, pins = _checked_plan_inputs(args, source, naming) tree = source.tree() - entries = walk(policy, [(path, size) for path, _, _, size in tree]) + entries = ask_about_submodules( + policy, walk(policy, [(path, size) for path, _, _, size in tree]), + tree) materialized = _predict_collisions(entries) follow_ups = follow_ups_for(source, entries, names, args.spec_path, args.code_path, materialized) @@ -1034,22 +1083,24 @@ def _leg_findings(plan: Plan) -> list[str]: #: What a plan that splits a source's own submodules is told to do instead. -#: Said once, by `execute`'s refusal; `check` states the same problems as -#: findings and the `.gitmodules` entry's `question:` states the rule. +#: Said by `check`, once after its findings, and by `execute`'s refusal; the +#: `question:` of `.gitmodules`, and of every entry holding a submodule, +#: states the rule. SUBMODULE_REMEDIATION = ( "Remediation: send `.gitmodules` and every submodule it registers to ONE " - "leg, spec or code, or `drop` them together. To be rid of ONE of several " - "registered submodules, adopt it with the others and `git rm` it in the " - "leg afterwards, which takes its registration with it. A submodule the " - "source does not register -- no `.gitmodules` entry with its path and a " - "url -- cannot be kept in a leg at all: a top-level `git clone " - "--recurse-submodules` skips it, but the assembly's clone fails on it " - "inside the leg. `drop` it, or register it in the source (that entry, " - "committed) and re-run `plan`; a `.gitmodules` that `git config` cannot " - "read is repaired in the source and re-planned the same way, or dropped " - "with every submodule. The assembly root never keeps a submodule of the " - "source's: its `.gitmodules` is reserved for its two leg mounts, and its " - "`validate` workflow reads every URL in that file as a leg.") + "leg, spec or code, or `drop` them together. Any one submodule may also " + "be dropped on its own: its registration stays behind in the leg " + "`.gitmodules` went to, inert, because git looks a registration up only " + "for a gitlink it finds. A submodule the source does not register -- no " + "`.gitmodules` entry with its path and a url -- cannot be kept in a leg " + "at all: a top-level `git clone --recurse-submodules` skips it, but the " + "assembly's clone fails on it inside the leg. `drop` it, or register it " + "in the source (that entry, committed) and re-run `plan`; a " + "`.gitmodules` that `git config` cannot read is repaired in the source " + "and re-planned the same way, or dropped with every submodule. The " + "assembly root never keeps a submodule of the source's: its " + "`.gitmodules` is reserved for its two leg mounts, and its `validate` " + "workflow reads every URL in that file as a leg.") #: Why a kept gitlink with no URL beside it is refused, said in the same #: words by the two rules that find one. @@ -1062,9 +1113,10 @@ def _answered_legs(entries: list, paths: list[str]) -> dict: A path covered by no entry or by two, or by an entry whose `leg:` is null or not one of the four words, is LEFT OUT. Each of those is already a - finding of its own -- `plan-uncovered`, `plan-covered-twice`, + finding of `check`'s own -- `plan-uncovered`, `plan-covered-twice`, `plan-unresolved`, `plan-bad-leg` -- and a question nobody has answered - yet cannot have been answered inconsistently. + yet cannot have been answered inconsistently. `execute` refuses only the + null one of those four today; #168 makes it refuse the other three. """ entry_paths = [str(e.get("path")) for e in entries] legs = {str(e.get("path")): str(e.get("leg")) for e in entries} @@ -1087,8 +1139,12 @@ def submodule_plan_problems(entries: list, tree: list, """`(code, detail)` for each way the plan splits the source's OWN submodules. ONE DEFINITION, asked by `check` for findings and by `execute` for a - refusal, so that the plan `check` passes is the plan `execute` will run - (#166). `tree` is `Source.tree()`: a gitlink is visible only by its mode, + refusal, so that `execute` refuses every SUBMODULE problem `check` finds + (#166). That is not yet every finding: an entry `check` rejects for + another reason -- uncovered, covered twice, a misspelled leg -- is left + out here (see `_answered_legs`) and still reaches the split, and #168 + extends `execute`'s refusal to every finding of `check`. `tree` is + `Source.tree()`: a gitlink is visible only by its mode, which no path glob in `contracts/path-classification.yaml` can see. `registered` is `Source.registered_submodules(tree)`, the gitlink paths the source's `.gitmodules` names with a url, or None when `git config` @@ -1202,11 +1258,17 @@ def _kept_submodule_problems(kept: list, registered: frozenset, return problems -def _submodule_findings(plan: Plan, source: Source, tree: list) -> list[str]: - """`submodule_plan_problems` as `check` prints them, one FINDING each.""" - return [f"FINDING {code}: {detail}" for code, detail - in submodule_plan_problems(plan.entries, tree, - source.registered_submodules(tree))] +def _submodule_findings(plan: Plan, source: Source, + tree: list) -> tuple[list[str], list[str]]: + """`submodule_plan_problems` as `check` prints them, one FINDING each, + and what to print ONCE after every finding: SUBMODULE_REMEDIATION when + there is one of these, nothing when there is none. `check` is where the + person is told to fix the plan, so it says how, in the words `execute`'s + refusal would use one step later.""" + found = [f"FINDING {code}: {detail}" for code, detail + in submodule_plan_problems(plan.entries, tree, + source.registered_submodules(tree))] + return found, [SUBMODULE_REMEDIATION] if found else [] def _topics_line(topic: str, local: bool) -> str: @@ -1270,7 +1332,8 @@ def cmd_check(args) -> int: tree_paths = [path for path, _, _, _ in tree] findings.extend(_coverage_findings(entry_paths, tree_paths)) findings.extend(_leg_findings(plan)) - findings.extend(_submodule_findings(plan, source, tree)) + submodule_findings, remediation = _submodule_findings(plan, source, tree) + findings.extend(submodule_findings) names = plan.names() pins = set(plan.pins) @@ -1292,8 +1355,8 @@ def cmd_check(args) -> int: for line in plan.seeding_record_disagreements(seeded): print(line) - for finding in findings: - print(finding, file=sys.stderr) + for line in [*findings, *remediation]: + print(line, file=sys.stderr) if findings: print(f"\n{len(findings)} finding(s) in {plan.path}", file=sys.stderr) return 1 @@ -1365,6 +1428,11 @@ def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: legs that already exist (#166). The refusal is named by the FIRST problem, which `submodule_plan_problems` lists in the order to fix them in, and its detail names up to eight of them and counts the rest. + + So `execute` refuses every SUBMODULE problem `check` finds, in `check`'s + words. It does not yet refuse every other finding: a path no entry + covers, or two do, or whose entry has a misspelled leg, still passes + here, and #168 extends this refusal to every finding of `check`. """ unresolved = [e for e in plan.entries if e.get("leg") is None] if unresolved: @@ -1528,10 +1596,14 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, # two mounts: `-f` because a source `.gitignore` such as `.*` would hide it # from `git add`, and only in this case, so a source with no submodules is # mounted exactly as it was before. It is removed FIRST above because - # `git rm` of a submodule edits the file that registers it. Whether the - # registrations it held went to the same leg as the gitlinks they name is - # the plan's question, not this mount's (#166); a .gitmodules the plan KEPT - # here is verified byte for byte today and fails (#165). + # `git rm` of a submodule edits the file that registers it. #166 ruled + # that the assembly root keeps no submodule of the source's: a plan that + # keeps `.gitmodules` or a gitlink here, or splits one from its + # registration, is a `check` finding and an `execute` refusal before this + # runs. Appending the mounts to a `.gitmodules` left here, rather than + # replacing it, is MOUNT DEFENCE IN DEPTH for a plan that slipped past + # them (an entry `check` rejects for another reason, until #168), not a + # shape a plan may choose; such a file still fails verification (#165). if GITMODULES in removed: write_lf(assembly / GITMODULES, "") run(["git", "add", "-f", "--", GITMODULES], cwd=assembly) diff --git a/contracts/path-classification.yaml b/contracts/path-classification.yaml index 4485ee79..c3dc330a 100644 --- a/contracts/path-classification.yaml +++ b/contracts/path-classification.yaml @@ -99,10 +99,17 @@ rules: # --recurse-submodules` exits 128 after an adoption that verified every # blob (#166). No glob here can tell a gitlink from a file -- only its mode # in `git ls-tree` can -- so the question is asked on the one file every - # registration lives in, and `adopt-project.py check` holds the gitlinks to - # the answer and refuses to keep one this file does not register. Without - # this rule the default asked it as if `.gitmodules` were any other file, - # and `root` looked like an answer. `shape-doctor.py --placement-plan` + # registration lives in, and `adopt-project.py plan` asks it again, naming + # the submodules, on every entry that holds a gitlink, WHATEVER RULE HERE + # MATCHED THAT ENTRY; `check` then holds the gitlinks to the answer and + # refuses to keep one this file does not register. So a vendored SUBMODULE + # under a root-ruled path -- `.specify/vendored`, a `.cursor` or `.agents` + # that is a submodule -- is ASKED rather than rooted, because the root + # keeps no submodule of the source's. The 2026-09-02 ruling is unchanged + # for FILES: they stay in the root, and whoever answers an entry that holds + # both splits it, as the entry's question says. + # Without this rule the default asked it as if `.gitmodules` were any other + # file, and `root` looked like an answer. `shape-doctor.py --placement-plan` # prints this `reason:` and `question:` too, for a project already split # and a file no `check` or `execute` ever reads, so what refuses a wrong # answer is named as the tool and not as those two subcommands. @@ -121,9 +128,11 @@ rules: mounts. Which leg owns the submodules, spec or code? Send `.gitmodules` there and every submodule path with it, or drop them all together. A submodule it does not register with a path and a url cannot be kept in - a leg at all: drop it, or register it first. `root` is not an answer - for this file or for any submodule, and `adopt-project.py` refuses a - plan that gives it, as it does a plan that splits them. + a leg at all: drop it, or register it first. Any one submodule may also + be dropped on its own; its registration stays behind in that leg, + inert. `root` is not an answer for this file or for any submodule, and + `adopt-project.py` refuses a plan that gives it, as it does a plan that + splits them. # -- the assembly root ---------------------------------------------------- diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index 1ed3ef0b..1384a25d 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -176,12 +176,22 @@ def build(registered=("upstream/kept", "upstream/moved"), tree=None): @pytest.fixture def mount_fixture(mount_world): - """Two registrations in the source: one kept in the assembly, one moved.""" + """Two registrations in the source: one left in the assembly, one moved. + Since #166 a plan cannot leave one there; the mount is driven directly.""" return mount_world() def test_mount_preserves_registration_kept_in_assembly(mount_fixture, tmp_path): + """MOUNT DEFENCE IN DEPTH, for a plan `check` and `execute` now refuse. + + #166 ruled that the assembly root keeps no submodule of the source's: + `.gitmodules` and `upstream/kept` left in the root are `plan-submodule- + root` findings and an `execute` refusal before the mount is reached. + Should such a plan slip past them (until #168, an entry `check` rejects + for another reason does), the mount still appends its two legs to the + registrations it finds rather than replacing them. + """ adopter, source, work, names, urls = mount_fixture assembly = clone_assembly(source, tmp_path) kept = git("rev-parse", "HEAD:upstream/kept", cwd=source).stdout.strip() @@ -262,15 +272,23 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( # --- the plan's own answer: one leg for a submodule and its registration ---- # -# #166. `check` and `execute` read the SAME `submodule_plan_problems`, so a -# plan `check` passes is a plan `execute` runs. None of these needs -# `git filter-repo` but the one that reaches `execute` past its preflight. +# #166. `check` and `execute` read the SAME `submodule_plan_problems`, so +# `execute` refuses every submodule problem `check` finds (#168 extends that +# to every finding), and `plan` asks the question on every entry that holds +# a submodule, so it never proposes what `check` refuses. None of these +# needs `git filter-repo` but the ones that reach `execute` past its +# preflight. #: The ids, as `check` prints them and `execute` raises them. SPLIT = "plan-submodule-split" ROOT = "plan-submodule-root" UNREGISTERED = "plan-submodule-unregistered" UNREADABLE = "plan-gitmodules-unreadable" +SUBMODULE_CODES = (ROOT, UNREADABLE, UNREGISTERED, SPLIT) +#: `check`'s own ids for an entry it rejects before asking about submodules. +UNRESOLVED = "plan-unresolved" +UNCOVERED = "plan-uncovered" +BAD_LEG = "plan-bad-leg" #: The plan entries these sources ask about, and the file every plan below is #: written to, each in a directory of its own. @@ -278,6 +296,10 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( UPSTREAM = "upstream/" EXTRA = "extra/" PLAN_FILE = "adoption-plan.yaml" +#: The submodule most of these sources register, under UPSTREAM, and the +#: one file in the repository it is a commit of. +DEPENDENCY = "upstream/dependency" +DEPENDENCY_TREE = {"value.txt": "dependency\n"} #: What the assembly root's `.gitmodules` is, in the `plan-submodule-root` #: detail and in the remediation: true whether or not `.gitmodules` is the @@ -303,22 +325,32 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( "\tpath = upstream/dependency\n"} #: `d0/s0` to `d9/s9` registered, and `extra/orphan` beside them not: CROWDED = {**BESIDE, "registered": tuple(f"d{i}/s{i}" for i in range(10))} +#: `upstream/dependency` and `extra/second`, both registered: +TWO = {"registered": (DEPENDENCY, "extra/second")} +#: `dep`, registered at the top level, so it is a plan entry of its own: +TOP = {"registered": ("dep",)} +#: A `.gitmodules` registering a submodule the tree no longer holds, and no +#: gitlink anywhere: +STALE = {"registered": (), "gitmodules": '[submodule "gone"]\n' + "\tpath = gone\n\turl = ../gone\n"} -def registered_source(base: Path, registered=("upstream/dependency",), +def registered_source(base: Path, registered=(DEPENDENCY,), orphan: str | None = None, - gitmodules: str | None = None) -> Path: + gitmodules: str | None = None, + extra: dict | None = None) -> Path: """A source registering a real dependency at each of `registered`. `orphan` adds a gitlink at that path with NO registration, written straight into the index as `git add` of an embedded clone would write it, so that a source with a submodule and no `.gitmodules` is planned too. `gitmodules`, when given, replaces the file `git submodule add` wrote, as - a hand edit would. + a hand edit would. `extra` adds files to SMALL_TREE. """ - dependency = make_source_repo(base / "dependency", - tree={"value.txt": "dependency\n"}, edits=()) - source = make_source_repo(base / "Thing", tree=SMALL_TREE) + dependency = make_source_repo(base / "dependency", tree=DEPENDENCY_TREE, + edits=()) + source = make_source_repo(base / "Thing", + tree={**SMALL_TREE, **(extra or {})}) for path in registered: add_submodule(source, dependency, path) if orphan: @@ -379,6 +411,24 @@ def check_plan(plan: Path): return adopt("check", plan) +def carelessly_edited(plan: Path, removed: str | None, + added: str | None) -> None: + """`plan` with the entry for `removed` deleted, and an entry `added: + spec` put first, as a careless hand edit would leave it. Bytes, because + `plan` writes LF on every platform.""" + lines = plan.read_bytes().splitlines(keepends=True) + if removed: + start = lines.index(f" - path: {removed}\n".encode()) + end = start + 1 + while lines[end].startswith(b" "): + end += 1 + del lines[start:end] + if added: + first = lines.index(b"paths:\n") + 1 + lines[first:first] = [f" - path: {added}\n leg: spec\n".encode()] + plan.write_bytes(b"".join(lines)) + + def findings(stderr: str) -> list[tuple[str, str]]: """`(code, detail)` for each `FINDING : ` line `check` printed, in its order.""" @@ -402,6 +452,9 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): "fresh one for its two mounts.") in question assert ("A submodule it does not register with a path and a url cannot " "be kept in a leg at all") in question + # No stricter than the rule: one submodule may be dropped by itself. + assert ("Any one submodule may also be dropped on its own; its " + "registration stays behind in that leg, inert.") in question # `shape-doctor.py --placement-plan` prints this question too, for a file # no `check` or `execute` ever reads: the TOOL refuses, true in both. assert ("`root` is not an answer for this file or for any submodule, and " @@ -413,6 +466,45 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): assert "goes to the SAME leg as every submodule" in written.stdout +@pytest.mark.parametrize("gitlinks, extra, entry, rule", [ + # The 2026-09-02 ruling roots `.specify/` and the assistant directories, + # and still roots their FILES; a SUBMODULE there is asked instead. + ((".specify/vendored",), {".specify/memory/constitution.md": "# c\n"}, + ".specify/", "root-spec-kit"), + ((".cursor",), {}, ".cursor", "root-assistant-instructions"), + # Asked already, but whether it is root: `.gitmodules`' question instead. + ((".agents",), {}, ".agents", "ambiguous-assistant-project-memory"), + (("docs/theme",), {"docs/index.md": "# Docs\n"}, "docs/", + "spec-governance"), + ((DEPENDENCY,), {}, UPSTREAM, "default"), + (("vendor/a", "vendor/b"), {}, "vendor/", "default"), +], ids=["specify-vendored", "cursor", "agents", "docs-theme", "upstream", + "two-in-one-entry"]) +def test_plan_asks_on_every_entry_that_holds_a_submodule( + plan_of, tmp_path, gitlinks, extra, entry, rule): + """`plan` never proposes what `check` refuses: the entry covering a + gitlink is asked `.gitmodules`' question, whatever rule matched it, and + names its submodules; the untouched plan is then UNRESOLVED there, never + a submodule kept in the root.""" + plan = plan_of({"registered": gitlinks, "extra": extra})[0] + rows = plan_rows(plan) + row = rows[entry] + # Everything `.gitmodules` is asked -- `leg: null`, `review_required`, + # its rule, reason and question -- and only its own size besides. + expected = {**rows[GITMODULES], "path": entry, "files": row["files"], + "bytes": row["bytes"]} + expected["question"] += ( + f" This entry holds the submodule(s) {', '.join(gitlinks)}: it is " + f"asked this question, not classified by its rule `{rule}`." + + (" Split the entry if its other paths belong in a different leg " + "from the submodules." if extra else "")) + assert row == expected + assert (row["leg"], row["rule"]) == (None, "ambiguous-gitmodules") + result = check_plan(answered(plan, tmp_path, {})) + assert set(finding_codes(result.stderr)) == {UNRESOLVED}, result.stderr + assert f"FINDING {UNRESOLVED}: {entry} still has" in result.stderr + + @pytest.mark.parametrize("modules_leg, dependency_leg, expected", [ ("code", "code", []), ("spec", "spec", []), @@ -426,15 +518,23 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): ("root", "code", [ROOT, SPLIT]), # An unanswered `.gitmodules` is `plan-unresolved`'s, and is not also # reported as a split; a gitlink kept in the root is refused regardless. - (None, "code", ["plan-unresolved"]), - (None, "root", ["plan-unresolved", ROOT]), + (None, "code", [UNRESOLVED]), + (None, "root", [UNRESOLVED, ROOT]), ]) def test_check_holds_a_submodule_to_the_leg_of_its_registration( - plan_of, tmp_path, modules_leg, dependency_leg, expected): + adopter, plan_of, tmp_path, modules_leg, dependency_leg, expected): plan = answered(plan_of(REGISTERED)[0], tmp_path, {GITMODULES: modules_leg, UPSTREAM: dependency_leg}) result = check_plan(plan) assert finding_codes(result.stderr) == expected, result.stderr + # The remediation, ONCE after the last finding and before the count, + # when any of them is a submodule problem: `check` says how to fix it. + told = result.stderr.count(adopter.SUBMODULE_REMEDIATION) + assert told == (1 if set(expected) & set(SUBMODULE_CODES) else 0) + if told: + at = result.stderr.index(adopter.SUBMODULE_REMEDIATION) + assert result.stderr.rindex("FINDING ") < at \ + < result.stderr.index(" finding(s) in ") if not expected: assert result.returncode == 0, result.stderr assert "plan ok" in result.stdout @@ -456,11 +556,14 @@ def test_check_holds_a_submodule_to_the_leg_of_its_registration( def test_check_finds_a_submodule_a_rule_classified_silently(tmp_path): - """`tools/dep` is under `tools/**`, so `plan` calls it code and asks - nothing; only `.gitmodules` is asked. Answered `spec`, the two split.""" + """`tools/dep` is under `tools/**`, which a plan written before #166 + called code without asking; `plan` asks there now. Answered as that rule + would have it, with `.gitmodules` in spec, the two split, and `check` + finds it however the plan came to say so.""" plan = planned(registered_source(tmp_path, registered=("tools/dep",)), tmp_path)[0] - assert plan_rows(plan)["tools/"]["leg"] == "code" + assert plan_rows(plan)["tools/"]["review_required"] is True + resolve(plan, "tools/", "code") resolve(plan, GITMODULES, "spec") result = check_plan(plan) assert result.returncode == 1, result.stderr + result.stdout @@ -561,6 +664,87 @@ def test_a_gitmodules_git_cannot_read_is_not_one_that_registers_nothing( "source's .gitmodules") in result.stderr +@pytest.mark.parametrize("modules_leg, expected", [ + ("code", []), + ("spec", []), + ("drop", []), + ("root", [ROOT]), +]) +def test_a_gitmodules_with_no_submodule_beside_it(plan_of, tmp_path, + modules_leg, expected): + """A `.gitmodules` whose one registration names no gitlink in the tree: + in a leg it registers nothing git will look up, so it goes anywhere but + the root, whose `.gitmodules` is the two mounts'.""" + plan = answered(plan_of(STALE)[0], tmp_path, {GITMODULES: modules_leg}) + result = check_plan(plan) + assert finding_codes(result.stderr) == expected, result.stderr + assert result.returncode == (1 if expected else 0) + + +@pytest.mark.parametrize("upstream_leg, extra_leg, split", [ + ("code", "code", []), + ("code", "spec", ["extra/second (entry extra/) has leg: spec"]), + ("spec", "code", ["upstream/dependency (entry upstream/) has leg: spec"]), + ("spec", "spec", ["extra/second (entry extra/) has leg: spec", + "upstream/dependency (entry upstream/) has leg: spec"]), +]) +def test_check_finds_each_submodule_split_from_its_registration( + plan_of, tmp_path, upstream_leg, extra_leg, split): + """Two registered submodules and `.gitmodules` in code: each one kept + away from it is its own `plan-submodule-split`, and only those.""" + plan = answered(plan_of(TWO)[0], tmp_path, + {GITMODULES: "code", UPSTREAM: upstream_leg, + EXTRA: extra_leg}) + found = findings(check_plan(plan).stderr) + assert [code for code, _ in found] == [SPLIT] * len(split), found + assert [detail.split(" but ")[0] for _, detail in found] \ + == [f"the submodule {where}" for where in split] + + +@pytest.mark.parametrize("dep_leg, expected", [ + ("code", []), + ("spec", [(SPLIT, "the submodule dep has leg: spec but the source's " + ".gitmodules has leg: code")]), + ("root", [(ROOT, "dep has leg: root, and the assembly root keeps no " + "submodule of the source's")]), +]) +def test_a_submodule_that_is_its_own_entry_is_named_alone(plan_of, tmp_path, + dep_leg, expected): + """`dep` is a top-level gitlink, so its entry is itself: the finding + names it with no `(entry ...)` to point at.""" + plan = answered(plan_of(TOP)[0], tmp_path, {GITMODULES: "code", + "dep": dep_leg}) + found = findings(check_plan(plan).stderr) + assert [code for code, _ in found] == [code for code, _ in expected] + for (_, detail), (_, start) in zip(found, expected): + assert detail.startswith(start), detail + + +@pytest.mark.parametrize("answers, removed, added, own", [ + ({GITMODULES: "code"}, UPSTREAM, None, UNCOVERED), + ({UPSTREAM: "code"}, GITMODULES, None, UNCOVERED), + ({GITMODULES: "code", UPSTREAM: "code"}, None, DEPENDENCY, + "plan-covered-twice"), + ({GITMODULES: "code", UPSTREAM: "Code"}, None, None, BAD_LEG), + ({GITMODULES: "Code", UPSTREAM: "code"}, None, None, BAD_LEG), +], ids=["gitlink-uncovered", "gitmodules-uncovered", "gitlink-covered-twice", + "gitlink-bad-leg", "gitmodules-bad-leg"]) +def test_the_submodule_rule_leaves_a_rejected_entry_to_its_own_finding( + adopter, plan_of, tmp_path, answers, removed, added, own): + """An entry `check` already rejects is not ALSO a submodule problem: the + helper skips it, and `check` reports it once, by its own finding.""" + plan = answered(plan_of(REGISTERED)[0], tmp_path, answers) + carelessly_edited(plan, removed, added) + result = check_plan(plan) + assert finding_codes(result.stderr) == [own], result.stderr + assert adopter.SUBMODULE_REMEDIATION not in result.stderr + loaded = adopter.Plan.load(plan) + source = loaded.open_source(None, tmp_path / "work") + tree = source.tree() + assert adopter.submodule_plan_problems( + loaded.entries, tree, source.registered_submodules(tree)) == [] + + @pytest.fixture(scope="module") def adopter(): """`adopt-project.py` as a module, for the refusal `execute` makes.""" @@ -597,6 +781,9 @@ def refused_by_execute(adopter, plan_path: Path, tmp_path: Path): ]) def test_execute_refuses_what_check_finds(adopter, plan_of, tmp_path, source, answers, code): + """The refusal `execute` makes, asked of `_refuse_an_unrunnable_plan` + directly; that `cmd_execute` makes it before any leg exists is pinned by + the end-to-end refusal below, which needs `git filter-repo`.""" plan = answered(plan_of(source)[0], tmp_path, answers) found = findings(check_plan(plan).stderr) refusal = refused_by_execute(adopter, plan, tmp_path) @@ -607,7 +794,10 @@ def test_execute_refuses_what_check_finds(adopter, plan_of, tmp_path, assert refusal is not None, found assert refusal.code == found[0][0] == code assert refusal.detail == "; ".join(detail for _, detail in found) - assert "`git rm` it in the leg afterwards" in refusal.remediation + assert refusal.remediation == adopter.SUBMODULE_REMEDIATION + # No stricter than the rule: one submodule may be dropped by itself. + assert ("Any one submodule may also be dropped on its own: its " + "registration stays behind") in refusal.remediation assert "never keeps a submodule" in refusal.remediation assert RESERVED in refusal.remediation assert ("cannot be kept in a leg at all" in refusal.remediation @@ -636,6 +826,14 @@ def test_a_mixed_refusal_is_named_by_the_problem_to_fix_first( [detail for _, detail in found[:8]] + [f"and {len(found) - 8} more"]) +def execute_in(plan: Path, base: Path): + """`execute --yes` of `plan`, its leg remotes under `base/remotes` and its + work in `base/work`: `(result, remotes, work)`.""" + remotes, work = base / "remotes", base / "work" + return adopt("execute", plan, "--yes", "--local-remote-dir", str(remotes), + "--work-dir", str(work)), remotes, work + + def source_state(source: Path) -> list[str]: """HEAD, every ref and `git status`: what a refused `execute` must not move.""" @@ -663,9 +861,7 @@ def test_execute_refuses_a_submodule_plan_before_anything_exists( thing = registered_source(tmp_path, **source) before = source_state(thing) plan = answered(planned(thing, tmp_path)[0], tmp_path, answers) - remotes, work = tmp_path / "remotes", tmp_path / "work" - result = adopt("execute", plan, "--yes", "--local-remote-dir", - str(remotes), "--work-dir", str(work)) + result, remotes, work = execute_in(plan, tmp_path) assert result.returncode == 2, result.stderr + result.stdout assert refused in result.stderr assert "creating the leg repositories" not in result.stdout @@ -673,3 +869,49 @@ def test_execute_refuses_a_submodule_plan_before_anything_exists( assert not remotes.exists(), "a refused plan made the remotes directory" assert list(work.iterdir()) == [], "a refused plan wrote into its work" assert source_state(thing) == before + + +@needs_filter_repo +@pytest.mark.parametrize("modules_leg, dependency_leg", [ + ("code", "code"), + ("spec", "spec"), + # Dropped on its own: its registration stays behind in the leg, inert. + ("code", "drop"), +], ids=["both-code", "both-spec", "dropped-on-its-own"]) +def test_an_accepted_submodule_plan_adopts_clones_and_bootstraps( + tmp_path, modules_leg, dependency_leg): + """The plans the refusals above leave legal really adopt: `execute` + verifies, and the assembly's `git clone --recurse-submodules` and its + `bootstrap.py` both exit 0, which a split plan's clone did not (#166).""" + thing = registered_source(tmp_path) + plan = answered(planned(thing, tmp_path)[0], tmp_path, + {GITMODULES: modules_leg, UPSTREAM: dependency_leg}) + checked = check_plan(plan) + assert checked.returncode == 0, checked.stderr + checked.stdout + result = execute_in(plan, tmp_path)[0] + assert result.returncode == 0, result.stderr + result.stdout + assert "adoption verified" in result.stdout + + clone = tmp_path / "clone" + cloned = git(*FILE_PROTOCOL, "clone", "-q", "--recurse-submodules", "-b", + "adopt/three-repo-shape", str(thing), str(clone), + cwd=tmp_path, check=False) + assert cloned.returncode == 0, cloned.stderr + bootstrapped = run_script(clone / "scripts/bootstrap.py", cwd=clone) + assert bootstrapped.returncode == 0, (bootstrapped.stderr + + bootstrapped.stdout) + assert "bootstrap ok" in bootstrapped.stdout + leg = clone / modules_leg + gitlink = git("ls-tree", "HEAD", "--", DEPENDENCY, cwd=leg).stdout + # The registration travels with `.gitmodules`, whether or not its + # gitlink did. + assert git("config", "-f", GITMODULES, "--get", + f"submodule.{DEPENDENCY}.path", cwd=leg).stdout.strip() \ + == DEPENDENCY + if dependency_leg == "drop": + assert gitlink == "" and not (leg / UPSTREAM).exists() + return + assert gitlink.startswith("160000 commit ") + # Cloned into the leg, at the commit the source pinned. + assert {name: (leg / DEPENDENCY / name).read_text(encoding="utf-8") + for name in DEPENDENCY_TREE} == DEPENDENCY_TREE From ca5fb3d41aa42f15265c50e3b12d92d5e1361560 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:28:25 +0000 Subject: [PATCH 04/12] adopt-project.py: `check` reads a source's `.gitmodules` as git's own submodule reader reads it in the leg, so a 0xFF byte or an `[include]` no longer changes the answer and a registration git ignores is no registration, says what git said when it cannot read one, and an entry holding a submodule among other paths names the entries that replace it (#166) Round 3, from lane openRepoShape-2's re-verification of 09d7889 (N1-N3, P3.3). N1. `Source.registered_submodules` read the blob with `git config --blob`, which git's submodule code never does: it reads bytes through a `char` (signed on x86), so a 0xFF byte ends the file there, and it follows `[include]`. A Latin-1 comment before a registration was a false refusal; one before a `path` override hid it, and the adoption verified and then failed to clone (#166's symptom); a relative include was a false `plan-gitmodules-unreadable`; an absolute one read a file on the operator's machine. The blob is now piped into `git config -z --no-includes --file - --list`. `--no-includes` is not optional: probed on git 2.43, `--file -` FOLLOWS includes by default (stdin sets no file, so git turns `respect_includes` on), unlike `--file `. The `plan-gitmodules-unreadable` detail now says what git said, on one line ("git cannot read the source's .gitmodules for its submodules (fatal: bad config line 2 in standard input), ..."), including when the `.gitmodules` in the tree is no blob at all (`git cat-file` fails). N2. `_registrations` mirrors git's `parse_config` for `path` and `url`: a name that is empty or has a `..` component at `/` or `\` is ignored (`check_submodule_name`); a value starting with `-` is ignored and the one before it stands; a path belongs to the last name given it, and a name moved to another path removes its old path's entry whoever holds it (git's `cache_remove_path`); a `path` or `url` with no value, which `--list` prints but git's reader dies on, makes the file unreadable, except under a name git already ignores. Every one probed against a recursive `git submodule update --init` on git 2.43. "Matched exactly as git matches them" is gone from the docstring; it now says which of git's rules are mirrored. N3. Adding `src/vendored: drop` beside `src/` is `plan-covered-twice`. An asked entry holding more than one path now names the entries that replace it, each submodule an entry of its own, descending only into a directory that holds one ("replace it with an entry for each of `src/app/`, `src/vendored`, each with its own `leg:`"), and SUBMODULE_REMEDIATION says to REPLACE a directory entry, not add beside it. An entry holding only submodules, two or more, is split too, so one can be dropped on its own. P3.3 (partial). The `ambiguous-gitmodules` question keeps #166's sentence, framed "When a repository is adopted, ..." so it is true where `shape-doctor.py --placement-plan` prints it for a leg already split. Docstrings and comments that said "until #168" are corrected now that #168 is merged here: `_refuse_what_check_finds` refuses the findings `submodule_plan_problems` leaves to `check`'s own. Tests: the registration test gains the 0xFF, name, `-`, last-name, moved-name and no-value shapes; new tests for `[include]` relative and absolute, for git's words in the unreadable finding, and for the split the question names (beside: covered twice; the named split: plan ok); the accepted-plan e2e gains upper-case keys, a 0xFF comment and an include. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 232 +++++++++++++++++------- contracts/path-classification.yaml | 24 +-- tests/test_adopt_submodules.py | 279 +++++++++++++++++++++++------ 3 files changed, 408 insertions(+), 127 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index 3b4a74a3..83a2dbc7 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -221,8 +221,9 @@ def emit(lines: list[str], key: str, value, indent: int = 0) -> None: #: A `.gitmodules` key as `git config --list` prints it: the section and the #: variable lower-cased by git, the `` exactly as written. A key with no -#: `` does not match, because git skips one when it clones. -SUBMODULE_KEY_RE = re.compile(r"submodule\.(?P.+)\.(?Ppath|url)") +#: `` does not match, because git skips one when it clones; an EMPTY +#: `` does, and `_a_name_git_ignores` then ignores it as git does. +SUBMODULE_KEY_RE = re.compile(r"submodule\.(?P.*)\.(?Ppath|url)") #: The `git` subcommand that reads and writes a config file: a source's #: `.gitmodules`, a leg mount's url, the elector's name. GIT_CONFIG = "config" @@ -236,6 +237,57 @@ def _git_path(raw: bytes) -> str: return raw.decode("utf-8", "surrogateescape") +def _one_line(text: str) -> str: + """What git said, on one line: `check` prints one FINDING per line.""" + return " ".join(text.split()) + + +def _a_name_git_ignores(name: str) -> bool: + """git's `check_submodule_name`: a submodule name that is empty, or has a + `..` component between `/` or `\\` separators on every platform, is + skipped with `warning: ignoring suspicious submodule name`. Its `path` + and `url` then register nothing, though `git config` lists both.""" + return not name or ".." in re.split(r"[/\\]", name) + + +def _registrations(listed: bytes) -> frozenset[str] | str: + """The paths `git config -z --list` output registers with a url, read as + git's submodule reader reads them (submodule-config.c, `parse_config`); + or, for a `path` or `url` with no value at all, what that reader dies on. + + In the order git applies them: a name it ignores registers nothing; a + `path` or `url` with no `=` is fatal to it (`-z` prints such a key with + no newline); a value starting with `-` is ignored and the one before it + stands; a later value overrides an earlier one for its name, and a path + belongs to the LAST name given it. git keeps that path-to-name map by + path, so a name moved to another path removes whatever its old path + held, even when a later name had taken the path over -- the `pop` below, + which is git's `cache_remove_path`. + """ + path_of: dict[str, str] = {} + name_at: dict[str, str] = {} + url_of: dict[str, str] = {} + for record in listed.split(b"\x00"): + key, newline, value = _git_path(record).partition("\n") + match = SUBMODULE_KEY_RE.fullmatch(key) + if not match or _a_name_git_ignores(match["name"]): + continue + if not newline: + return f"missing value for '{key}'" + if value.startswith("-"): + continue + name = match["name"] + if match["key"] == "url": + url_of[name] = value + continue + if name in path_of: + name_at.pop(path_of[name], None) + path_of[name] = value + name_at[value] = name + return frozenset(path for path, name in name_at.items() + if url_of.get(name)) + + class Source: """A repository being read, and nothing more. Nothing here writes to it.""" @@ -297,42 +349,51 @@ def tree(self) -> list[tuple[str, str, str, int]]: del kind return sorted(out) - def registered_submodules(self, tree: list) -> frozenset[str] | None: + def registered_submodules(self, tree: list) -> frozenset[str] | str: """Each path the `.gitmodules` at the pinned commit registers WITH a - url, or None when `git config` cannot read that file. + url; or, as a string, what git says when it cannot read that file + for its submodules. A REGISTRATION is a `submodule..path` and a non-empty `submodule..url` under the same name: the path is what git looks a gitlink up by, and the url is what it must then find to clone - it. Matched exactly as git matches them, and a key with no `` - is ignored as git ignores it. `tree` is `self.tree()`, which says - whether there is a file to read at all; with none, nothing is - registered. + it. `tree` is `self.tree()`, which says whether there is a file to + read at all; with none, nothing is registered. + + READ AS THE LEG WILL READ IT. In the assembly, git reads the + CHECKED-OUT file, follows no `[include]` in it, and filters what it + parses (`_registrations` mirrors those filters). So the blob is piped + into `git config --no-includes --file - --list`, and NOT read with + `--blob`, which differs twice: it reads bytes through a `char`, + signed on x86, so a 0xFF byte -- in a Latin-1 comment, say -- reads + as the end of the file, hiding a registration or an override after + it (git 2.43 on x86_64, `--list` exits 0); and it follows + `[include]`, so a relative one is fatal (`relative config includes + must come from files`) and an absolute one reads a file on this + machine. `--no-includes` because `--file -` follows includes by + default, as `--file ` does not. `--list`, NOT `--get-regexp`: the second exits 1 both when no key matches and on a `bad config line`, so a file git cannot read would come back as one that registers nothing. `--list` exits 0 on any file - it parsed, empty or not, and 128 on one it could not, and None says - so: a read that failed is not an answer (#166). + it parsed, empty or not, and 128 on one it could not, and git's words + come back instead of paths: a read that failed is not an answer + (#166), and the person told so is told why. """ if not any(path == GITMODULES for path, _, _, _ in tree): return frozenset() - try: - raw = git_out([GIT_CONFIG, "-z", "--blob", - f"{self.commit}:{GITMODULES}", "--list"], - cwd=self.path, binary=True) - except Refusal: - return None - paths: dict[str, str] = {} - urls: dict[str, str] = {} - for record in raw.split(b"\x00"): - key, _, value = _git_path(record).partition("\n") - match = SUBMODULE_KEY_RE.fullmatch(key) - if match: - registry = paths if match["key"] == "path" else urls - registry[match["name"]] = value - return frozenset(path for name, path in paths.items() - if urls.get(name)) + blob = subprocess.run( + ["git", "cat-file", "blob", f"{self.commit}:{GITMODULES}"], + cwd=self.path, capture_output=True, check=False) + # A `.gitmodules` that is no blob -- a gitlink, say -- leaves nothing + # for `git config` to read, and what `cat-file` said is the answer. + listed = blob if blob.returncode else subprocess.run( + ["git", GIT_CONFIG, "-z", "--no-includes", "--file", "-", + "--list"], cwd=self.path, input=blob.stdout, capture_output=True, + check=False) + if listed.returncode: + return _one_line(listed.stderr.decode(errors="replace")) + return _registrations(listed.stdout) def commit_count(self) -> int: return int(git_out(["rev-list", "--count", self.commit], cwd=self.path)) @@ -437,32 +498,69 @@ def ask_about_submodules(policy: PathPolicy, entries: list[Entry], `check` refuses. So each entry that covers a gitlink, WHATEVER RULE MATCHED IT, gets `leg: null` and the question `.gitmodules` itself is asked, naming the submodules it holds and the rule it is no longer - classified by. The ruling still roots the FILES under those paths: an - entry holding more than its submodules says so, and is split by the - person answering it. `walk`'s folding is left alone; only the covering - entry is replaced. A source with no gitlink, the common case, gets its - entries back untouched. + classified by. The ruling still roots the FILES under those paths, and + one of several submodules may be dropped on its own, so an entry holding + more than one path NAMES THE ENTRIES THAT REPLACE IT, each submodule one + of its own (`_own_entries`): splitting it is the only edit that answers + those paths apart, because an entry added beside it for a path it + already covers is `plan-covered-twice`. `walk`'s folding is left alone; + only the covering entry is replaced. A source with no gitlink, the + common case, gets its entries back untouched. """ gitlinks = [path for path, mode, _, _ in tree if mode == GITLINK_MODE] if not gitlinks: return entries + paths = [path for path, _, _, _ in tree] asked = policy.classify_file(GITMODULES) out = [] for entry in entries: held = [path for path in gitlinks if _covering([entry.path], path)] - out.append(_asked_about(entry, held, asked) if held else entry) + out.append(_asked_about(entry, held, asked, + _own_entries(entry.path, paths, held)) + if held else entry) return out -def _asked_about(entry: Entry, held: list[str], asked: Verdict) -> Entry: +def _own_entries(entry: str, paths: list[str], gitlinks: list[str] + ) -> list[str]: + """The plan entries that replace `entry` so that each of `gitlinks` it + covers is an entry of its own: one per child of the directory, and + descending only into a child directory that holds one of them. An entry + that is not a directory is already its own.""" + if not entry.endswith("/"): + return [entry] + out: list[str] = [] + for child in sorted({_child_of(entry, path) for path in paths + if path.startswith(entry)}): + if child.endswith("/") and any(path.startswith(child) + for path in gitlinks): + out += _own_entries(child, paths, gitlinks) + else: + out.append(child) + return out + + +def _child_of(directory: str, path: str) -> str: + """The entry one level below `directory` that covers `path`: the file + itself, or the directory it is in, with its slash.""" + head, slash, _ = path[len(directory):].partition("/") + return directory + head + slash + + +def _asked_about(entry: Entry, held: list[str], asked: Verdict, + own: list[str]) -> Entry: """`entry` as `ask_about_submodules` writes it: the `.gitmodules` - verdict, whose `leg:` is null, and its question naming `held`.""" + verdict, whose `leg:` is null, and its question naming `held` and, when + it holds more than one path, the `own` entries that replace it.""" question = (f"{asked.question or ''} This entry holds the submodule(s) " f"{', '.join(held)}: it is asked this question, not " f"classified by its rule `{entry.rule}`.") - if entry.files > len(held): - question += (" Split the entry if its other paths belong in a " - "different leg from the submodules.") + if len(own) > 1: + listed = ", ".join(f"`{path}`" for path in own) + question += (" Split the entry if its paths do not all belong in one " + f"leg: replace it with an entry for each of {listed}, " + "each with its own `leg:`. An entry added beside this " + "one for a path it covers is `plan-covered-twice`.") return Entry(entry.path, Verdict(None, asked.rule, asked.reason, asked.confidence, question), entry.files, entry.bytes) @@ -1097,8 +1195,13 @@ def _leg_findings(plan: Plan) -> list[str]: "at all: a top-level `git clone --recurse-submodules` skips it, but the " "assembly's clone fails on it inside the leg. `drop` it, or register it " "in the source (that entry, committed) and re-run `plan`; a " - "`.gitmodules` that `git config` cannot read is repaired in the source " - "and re-planned the same way, or dropped with every submodule. The " + "`.gitmodules` git cannot read for its submodules is repaired in the " + "source and re-planned the same way, or dropped with every submodule. " + "To answer a submodule apart from the rest of the directory entry that " + "holds it, REPLACE that entry with an entry for each of its children, " + "the submodule's own path among them, as that entry's `question:` lists " + "them: an entry for the submodule added BESIDE the directory's covers it " + "twice, which is `plan-covered-twice`. The " "assembly root never keeps a submodule of the source's: its " "`.gitmodules` is reserved for its two leg mounts, and its `validate` " "workflow reads every URL in that file as a leg.") @@ -1116,8 +1219,9 @@ def _answered_legs(entries: list, paths: list[str]) -> dict: or not one of the four words, is LEFT OUT. Each of those is already a finding of `check`'s own -- `plan-uncovered`, `plan-covered-twice`, `plan-unresolved`, `plan-bad-leg` -- and a question nobody has answered - yet cannot have been answered inconsistently. `execute` refuses only the - null one of those four today; #168 makes it refuse the other three. + yet cannot have been answered inconsistently. `execute` refuses all four + too: the null one in `_refuse_an_unrunnable_plan`, the other three in + #168's `_refuse_what_check_finds`. """ entry_paths = [str(e.get("path")) for e in entries] legs = {str(e.get("path")): str(e.get("leg")) for e in entries} @@ -1135,22 +1239,22 @@ def _as_entry(path: str, entry: str) -> str: def submodule_plan_problems(entries: list, tree: list, - registered: frozenset | None + registered: frozenset | str ) -> list[tuple[str, str]]: """`(code, detail)` for each way the plan splits the source's OWN submodules. ONE DEFINITION, asked by `check` for findings and by `execute` for a refusal, so that `execute` refuses every SUBMODULE problem `check` finds - (#166). That is not yet every finding: an entry `check` rejects for - another reason -- uncovered, covered twice, a misspelled leg -- is left - out here (see `_answered_legs`) and still reaches the split, and #168 - extends `execute`'s refusal to every finding of `check`. `tree` is + (#166). An entry `check` rejects for another reason -- uncovered, covered + twice, a misspelled leg -- is left out here (see `_answered_legs`): + `check` reports it by its own finding, and `execute` refuses it in + #168's `_refuse_what_check_finds`. `tree` is `Source.tree()`: a gitlink is visible only by its mode, which no path glob in `contracts/path-classification.yaml` can see. - `registered` is `Source.registered_submodules(tree)`, the gitlink paths - the source's `.gitmodules` names with a url, or None when `git config` - cannot read that file. The caller reads both, so this is a function of - its arguments alone. + `registered` is `Source.registered_submodules(tree)`: the gitlink paths + the source's `.gitmodules` names with a url, or, as a string, what git + said when it could not read that file for its submodules. The caller + reads both, so this is a function of its arguments alone. A SUBMODULE IS ONE FACT IN TWO PLACES: the gitlink in the tree and its registration in `.gitmodules`. A plan that sent the two to different legs @@ -1166,8 +1270,9 @@ def submodule_plan_problems(entries: list, tree: list, `validate.yml` reads every URL in that file as a leg; a root `.gitmodules` the plan kept fails verification today whatever it registers (#165). - * A `.gitmodules` `git config` cannot read is never taken to register - nothing (`plan-gitmodules-unreadable`). It is a problem when the plan + * A `.gitmodules` git cannot read for its submodules is never taken to + register nothing (`plan-gitmodules-unreadable`, saying what git + said). It is a problem when the plan keeps the file in a leg, where git fails on it when the assembly's clone recurses there, or keeps any gitlink in a leg, whose registration then cannot be checked. @@ -1212,16 +1317,16 @@ def submodule_plan_problems(entries: list, tree: list, for path, (leg, entry) in answered.items() if leg == "root"] registry_leg = answered.get(GITMODULES, (None, None))[0] kept = _kept_in_a_leg(answered, gitlinks) - if registered is not None: + if isinstance(registered, frozenset): return problems + _kept_submodule_problems(kept, registered, registry_leg) if kept or registry_leg in EXTRACTED_LEGS: problems.append(( "plan-gitmodules-unreadable", - f"`git config` cannot read the source's {GITMODULES}, so which " - "submodules it registers is unknown, and the plan keeps it or a " - "submodule in a leg, where the assembly's recursive clone fails " - "on it")) + f"git cannot read the source's {GITMODULES} for its submodules " + f"({registered}), so which submodules it registers is unknown, " + "and the plan keeps it or a submodule in a leg, where the " + "assembly's recursive clone fails on it")) return problems @@ -1431,9 +1536,9 @@ def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: in, and its detail names up to eight of them and counts the rest. So `execute` refuses every SUBMODULE problem `check` finds, in `check`'s - words. It does not yet refuse every other finding: a path no entry - covers, or two do, or whose entry has a misspelled leg, still passes - here, and #168 extends this refusal to every finding of `check`. + words. A path no entry covers, or two do, or whose entry has a + misspelled leg, passes here and is refused by #168's + `_refuse_what_check_finds`, which `cmd_execute` runs next. """ unresolved = [e for e in plan.entries if e.get("leg") is None] if unresolved: @@ -1601,10 +1706,11 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, # that the assembly root keeps no submodule of the source's: a plan that # keeps `.gitmodules` or a gitlink here, or splits one from its # registration, is a `check` finding and an `execute` refusal before this - # runs. Appending the mounts to a `.gitmodules` left here, rather than + # runs, and #168 refuses every other finding of `check` there too. + # Appending the mounts to a `.gitmodules` left here, rather than # replacing it, is MOUNT DEFENCE IN DEPTH for a plan that slipped past - # them (an entry `check` rejects for another reason, until #168), not a - # shape a plan may choose; such a file still fails verification (#165). + # both, not a shape a plan may choose; such a file still fails + # verification (#165). if GITMODULES in removed: write_lf(assembly / GITMODULES, "") run(["git", "add", "-f", "--", GITMODULES], cwd=assembly) diff --git a/contracts/path-classification.yaml b/contracts/path-classification.yaml index c3dc330a..c294a3f6 100644 --- a/contracts/path-classification.yaml +++ b/contracts/path-classification.yaml @@ -112,7 +112,9 @@ rules: # file, and `root` looked like an answer. `shape-doctor.py --placement-plan` # prints this `reason:` and `question:` too, for a project already split # and a file no `check` or `execute` ever reads, so what refuses a wrong - # answer is named as the tool and not as those two subcommands. + # answer is named as the tool and not as those two subcommands, and what + # `execute` does is said of a repository WHEN IT IS ADOPTED, which is true + # of a leg already split as well. - id: ambiguous-gitmodules paths: - ".gitmodules" @@ -123,16 +125,16 @@ rules: places, and which leg those dependencies belong to is a fact about this repository. question: >- - A source's own `.gitmodules` goes to the SAME leg as every submodule it - registers; `execute` writes the assembly root a fresh one for its two - mounts. Which leg owns the submodules, spec or code? Send `.gitmodules` - there and every submodule path with it, or drop them all together. A - submodule it does not register with a path and a url cannot be kept in - a leg at all: drop it, or register it first. Any one submodule may also - be dropped on its own; its registration stays behind in that leg, - inert. `root` is not an answer for this file or for any submodule, and - `adopt-project.py` refuses a plan that gives it, as it does a plan that - splits them. + When a repository is adopted, a source's own `.gitmodules` goes to the + SAME leg as every submodule it registers; `execute` writes the assembly + root a fresh one for its two mounts. Which leg owns the submodules, spec + or code? Send `.gitmodules` there and every submodule path with it, or + drop them all together. A submodule it does not register with a path + and a url cannot be kept in a leg at all: drop it, or register it + first. Any one submodule may also be dropped on its own; its + registration stays behind in that leg, inert. `root` is not an answer + for this file or for any submodule, and `adopt-project.py` refuses a + plan that gives it, as it does a plan that splits them. # -- the assembly root ---------------------------------------------------- diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index 1384a25d..9b0137b4 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -7,6 +7,7 @@ from __future__ import annotations import importlib.util +import re import shutil import sys from pathlib import Path @@ -187,9 +188,9 @@ def test_mount_preserves_registration_kept_in_assembly(mount_fixture, #166 ruled that the assembly root keeps no submodule of the source's: `.gitmodules` and `upstream/kept` left in the root are `plan-submodule- - root` findings and an `execute` refusal before the mount is reached. - Should such a plan slip past them (until #168, an entry `check` rejects - for another reason does), the mount still appends its two legs to the + root` findings and an `execute` refusal before the mount is reached, and + #168 refuses every other finding of `check` there too. Should such a + plan slip past both, the mount still appends its two legs to the registrations it finds rather than replacing them. """ adopter, source, work, names, urls = mount_fixture @@ -273,11 +274,12 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( # --- the plan's own answer: one leg for a submodule and its registration ---- # # #166. `check` and `execute` read the SAME `submodule_plan_problems`, so -# `execute` refuses every submodule problem `check` finds (#168 extends that -# to every finding), and `plan` asks the question on every entry that holds -# a submodule, so it never proposes what `check` refuses. None of these -# needs `git filter-repo` but the ones that reach `execute` past its -# preflight. +# `execute` refuses every submodule problem `check` finds (and #168 every +# other finding), and `plan` asks the question on every entry that holds a +# submodule, so it never proposes what `check` refuses. `check` reads the +# source's `.gitmodules` as git's own submodule reader will read it in the +# leg. None of these needs `git filter-repo` but the ones that reach +# `execute` past its preflight. #: The ids, as `check` prints them and `execute` raises them. SPLIT = "plan-submodule-split" @@ -300,6 +302,23 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( #: one file in the repository it is a commit of. DEPENDENCY = "upstream/dependency" DEPENDENCY_TREE = {"value.txt": "dependency\n"} +#: Stands, in a hand-written `.gitmodules`, for the dependency's absolute +#: path, which `registered_source` knows only once it has made it: the url a +#: leg's own clone can follow, where `../dependency` is relative to the leg. +THE_DEPENDENCY = "" + + +def registration(name: str = DEPENDENCY, url: str = "../dependency") -> str: + """`DEPENDENCY` registered by hand, under `name`, with `url`.""" + return f'[submodule "{name}"]\n\tpath = {DEPENDENCY}\n\turl = {url}\n' + + +#: The submodule #166's N3 found unasked, under a directory its rule calls +#: code, and a file a directory holding a submodule may hold beside it. +VENDORED = "src/vendored" +DOCS_INDEX = "docs/index.md" +#: A section that moves `registration()`'s name to another path. +MOVED_ELSEWHERE = '[submodule "upstream/dependency"]\n\tpath = elsewhere\n' #: What the assembly root's `.gitmodules` is, in the `plan-submodule-root` #: detail and in the remediation: true whether or not `.gitmodules` is the @@ -337,7 +356,7 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( def registered_source(base: Path, registered=(DEPENDENCY,), orphan: str | None = None, - gitmodules: str | None = None, + gitmodules: str | bytes | None = None, extra: dict | None = None) -> Path: """A source registering a real dependency at each of `registered`. @@ -345,7 +364,9 @@ def registered_source(base: Path, registered=(DEPENDENCY,), straight into the index as `git add` of an embedded clone would write it, so that a source with a submodule and no `.gitmodules` is planned too. `gitmodules`, when given, replaces the file `git submodule add` wrote, as - a hand edit would. `extra` adds files to SMALL_TREE. + a hand edit would: written as BYTES, LF on every platform, so that a byte + no encoding gives is written as it is, and with THE_DEPENDENCY replaced + by the dependency's path. `extra` adds files to SMALL_TREE. """ dependency = make_source_repo(base / "dependency", tree=DEPENDENCY_TREE, edits=()) @@ -358,7 +379,10 @@ def registered_source(base: Path, registered=(DEPENDENCY,), git("update-index", "--add", "--cacheinfo", f"160000,{head},{orphan}", cwd=source) if gitmodules is not None: - (source / GITMODULES).write_text(gitmodules, encoding="utf-8") + raw = gitmodules if isinstance(gitmodules, bytes) \ + else gitmodules.encode() + (source / GITMODULES).write_bytes(raw.replace( + THE_DEPENDENCY.encode(), dependency.as_posix().encode())) git("add", "--", GITMODULES, cwd=source) commit_as_source_human(source, "Register dependencies") return source @@ -411,11 +435,11 @@ def check_plan(plan: Path): return adopt("check", plan) -def carelessly_edited(plan: Path, removed: str | None, - added: str | None) -> None: - """`plan` with the entry for `removed` deleted, and an entry `added: - spec` put first, as a careless hand edit would leave it. Bytes, because - `plan` writes LF on every platform.""" +def edited(plan: Path, removed: str | None, added: dict) -> None: + """`plan` with the entry for `removed` deleted, and an entry `path: leg` + for each of `added` put first, as a hand edit would leave it -- a careless + one, or one that splits an entry. Bytes, because `plan` writes LF on + every platform.""" lines = plan.read_bytes().splitlines(keepends=True) if removed: start = lines.index(f" - path: {removed}\n".encode()) @@ -423,9 +447,9 @@ def carelessly_edited(plan: Path, removed: str | None, while lines[end].startswith(b" "): end += 1 del lines[start:end] - if added: - first = lines.index(b"paths:\n") + 1 - lines[first:first] = [f" - path: {added}\n leg: spec\n".encode()] + first = lines.index(b"paths:\n") + 1 + lines[first:first] = [f" - path: {path}\n leg: {leg}\n".encode() + for path, leg in added.items()] plan.write_bytes(b"".join(lines)) @@ -447,9 +471,12 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): assert row["leg"] is None assert row["review_required"] is True assert row["rule"] == "ambiguous-gitmodules" - assert ("A source's own `.gitmodules` goes to the SAME leg as every " - "submodule it registers; `execute` writes the assembly root a " - "fresh one for its two mounts.") in question + # #166's sentence, framed so that it is also true where + # `shape-doctor.py --placement-plan` prints it, for a leg already split. + assert question.startswith( + "When a repository is adopted, a source's own `.gitmodules` goes to " + "the SAME leg as every submodule it registers; `execute` writes the " + "assembly root a fresh one for its two mounts.") assert ("A submodule it does not register with a path and a url cannot " "be kept in a leg at all") in question # No stricter than the rule: one submodule may be dropped by itself. @@ -466,26 +493,38 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): assert "goes to the SAME leg as every submodule" in written.stdout -@pytest.mark.parametrize("gitlinks, extra, entry, rule", [ +@pytest.mark.parametrize("gitlinks, extra, entry, rule, own", [ # The 2026-09-02 ruling roots `.specify/` and the assistant directories, # and still roots their FILES; a SUBMODULE there is asked instead. ((".specify/vendored",), {".specify/memory/constitution.md": "# c\n"}, - ".specify/", "root-spec-kit"), - ((".cursor",), {}, ".cursor", "root-assistant-instructions"), + ".specify/", "root-spec-kit", (".specify/memory/", ".specify/vendored")), + ((".cursor",), {}, ".cursor", "root-assistant-instructions", ()), # Asked already, but whether it is root: `.gitmodules`' question instead. - ((".agents",), {}, ".agents", "ambiguous-assistant-project-memory"), - (("docs/theme",), {"docs/index.md": "# Docs\n"}, "docs/", - "spec-governance"), - ((DEPENDENCY,), {}, UPSTREAM, "default"), - (("vendor/a", "vendor/b"), {}, "vendor/", "default"), + ((".agents",), {}, ".agents", "ambiguous-assistant-project-memory", ()), + (("docs/theme",), {DOCS_INDEX: "# Docs\n"}, "docs/", + "spec-governance", (DOCS_INDEX, "docs/theme")), + ((DEPENDENCY,), {}, UPSTREAM, "default", ()), + # Nothing but submodules, and still split, to drop one on its own. + (("vendor/a", "vendor/b"), {}, "vendor/", "default", + ("vendor/a", "vendor/b")), + # Under a rule that would have called it code, unasked (#166's N3). + ((VENDORED,), {}, "src/", "code-source-and-tests", + ("src/app/", VENDORED)), + # Deeper than one level: only the directory holding it is opened. + (("docs/theme/sub",), {DOCS_INDEX: "# Docs\n", + "docs/theme/README.md": "# Theme\n"}, + "docs/", "spec-governance", + (DOCS_INDEX, "docs/theme/README.md", "docs/theme/sub")), ], ids=["specify-vendored", "cursor", "agents", "docs-theme", "upstream", - "two-in-one-entry"]) + "two-in-one-entry", "src-vendored", "nested"]) def test_plan_asks_on_every_entry_that_holds_a_submodule( - plan_of, tmp_path, gitlinks, extra, entry, rule): + plan_of, tmp_path, gitlinks, extra, entry, rule, own): """`plan` never proposes what `check` refuses: the entry covering a gitlink is asked `.gitmodules`' question, whatever rule matched it, and - names its submodules; the untouched plan is then UNRESOLVED there, never - a submodule kept in the root.""" + names its submodules and, when it holds more than one path, the `own` + entries that replace it so that each submodule is answered on its own; + the untouched plan is then UNRESOLVED there, never a submodule kept in + the root.""" plan = plan_of({"registered": gitlinks, "extra": extra})[0] rows = plan_rows(plan) row = rows[entry] @@ -495,9 +534,14 @@ def test_plan_asks_on_every_entry_that_holds_a_submodule( "bytes": row["bytes"]} expected["question"] += ( f" This entry holds the submodule(s) {', '.join(gitlinks)}: it is " - f"asked this question, not classified by its rule `{rule}`." - + (" Split the entry if its other paths belong in a different leg " - "from the submodules." if extra else "")) + f"asked this question, not classified by its rule `{rule}`.") + if own: + expected["question"] += ( + " Split the entry if its paths do not all belong in one leg: " + "replace it with an entry for each of " + + ", ".join(f"`{path}`" for path in own) + + ", each with its own `leg:`. An entry added beside this one " + "for a path it covers is `plan-covered-twice`.") assert row == expected assert (row["leg"], row["rule"]) == (None, "ambiguous-gitmodules") result = check_plan(answered(plan, tmp_path, {})) @@ -572,6 +616,34 @@ def test_check_finds_a_submodule_a_rule_classified_silently(tmp_path): in result.stderr +#: The entries an asked entry's question names to replace it with. +REPLACE_WITH_RE = re.compile(r"replace it with an entry for each of (.+?), " + r"each with its own `leg:`") + + +def test_the_split_the_question_names_is_the_edit_that_works(tmp_path): + """#166's N3: `src/vendored`, an orphan, under `src/`, which its rule + calls code. `check` says to drop it; adding `src/vendored: drop` BESIDE + `src/` covers it twice, and the edit the entry's question names -- `src/` + replaced by `src/app/` and `src/vendored` -- is the one `check` passes.""" + plan = planned(registered_source(tmp_path, registered=(), + orphan=VENDORED), tmp_path)[0] + named = REPLACE_WITH_RE.search(plan_rows(plan)["src/"]["question"]) + own = re.findall(r"`([^`]+)`", named[1]) + assert own == ["src/app/", VENDORED] + beside = answered(plan, tmp_path, {"src/": "code"}) + edited(beside, None, {VENDORED: "drop"}) + assert finding_codes(check_plan(beside).stderr) == ["plan-covered-twice"] + split = tmp_path / "split" / PLAN_FILE + split.parent.mkdir() + shutil.copyfile(plan, split) + edited(split, "src/", {path: "drop" if path == VENDORED else "code" + for path in own}) + result = check_plan(split) + assert result.returncode == 0, result.stderr + assert "plan ok" in result.stdout + + @pytest.mark.parametrize("leg, expected", [ ("code", [UNREGISTERED]), ("spec", [UNREGISTERED]), @@ -615,11 +687,18 @@ def test_check_refuses_to_keep_a_gitlink_its_gitmodules_does_not_register( f"extra/) has leg: {orphan_leg}") in result.stderr +#: A comment no encoding gives, as a Latin-1 editor writes one: `git config +#: --blob` reads 0xFF as the end of the file. +LATIN_1_COMMENT = b"# caf\xe9 \xff\xfe\n" +#: Upper-case section and keys, which git lower-cases. +UPPER_CASE_KEYS = ('[Submodule "upstream/dependency"]\n' + "\tPATH = upstream/dependency\n\tURL = ../dependency\n") + + @pytest.mark.parametrize("gitmodules, expected", [ ('[submodule "upstream/dependency"]\n\tpath = upstream/dependency\n', [UNREGISTERED]), - ('[submodule "upstream/dependency"]\n\tpath = upstream/dependency\n' - "\turl =\n", [UNREGISTERED]), + (registration(url=""), [UNREGISTERED]), # git skips a `submodule.path` with no ``, so this does too. ("[submodule]\n\tpath = upstream/dependency\n\turl = ../dependency\n", [UNREGISTERED]), @@ -627,14 +706,45 @@ def test_check_refuses_to_keep_a_gitlink_its_gitmodules_does_not_register( "\turl = ../dependency\n", [UNREGISTERED]), ("", [UNREGISTERED]), # Section and key names are case-insensitive to git, and so here. - ('[Submodule "upstream/dependency"]\n\tPATH = upstream/dependency\n' - "\tURL = ../dependency\n", []), + (UPPER_CASE_KEYS, []), + # git reads the leg's CHECKED-OUT file, where a 0xFF byte is a byte: + # the registration after one is a registration, and so is an override. + (LATIN_1_COMMENT + registration().encode(), []), + (registration().encode() + b"# \xff\n" + MOVED_ELSEWHERE.encode(), + [UNREGISTERED]), + # git ignores a name that is empty or has a `..` component, at `/` or + # `\`, with "ignoring suspicious submodule name"; `a..b` is a name. + (registration("../escape"), [UNREGISTERED]), + (registration("a\\\\..\\\\b"), [UNREGISTERED]), + (registration(""), [UNREGISTERED]), + (registration("a..b"), []), + # A `url` or `path` that starts with `-` is ignored, "may be interpreted + # as a command-line option", and the value before it stands. + (registration(url="-oops"), [UNREGISTERED]), + (registration() + "\tpath = -upstream/dependency\n", []), + # A path belongs to the LAST name given it, url or none. + (registration() + '[submodule "b"]\n\tpath = upstream/dependency\n', + [UNREGISTERED]), + ('[submodule "b"]\n\tpath = upstream/dependency\n' + registration(), []), + # ...and a name moved elsewhere takes its old path's registration with + # it, even one a later name holds (git keeps that map BY PATH). + (registration() + registration("b") + MOVED_ELSEWHERE, [UNREGISTERED]), + # A key with no value is fatal to git, but not under a name it ignores. + (registration() + '[submodule "../b"]\n\tpath\n', []), ], ids=["no-url", "empty-url", "no-name", "another-path", "empty-file", - "upper-case-keys"]) + "upper-case-keys", "a-0xff-comment", "a-0xff-hides-an-override", + "name-dotdot", "name-dotdot-backslash", "name-empty", + "name-dots-not-a-component", "url-option-like", + "path-option-like-after-a-path", "last-name-for-a-path-has-no-url", + "first-name-for-a-path-has-no-url", "a-name-moved-off-its-path", + "no-value-under-an-ignored-name"]) def test_a_registration_is_a_named_path_with_a_url(tmp_path, gitmodules, expected): """`upstream/dependency` is a gitlink in every case; only the - `.gitmodules` beside it, both kept in code, changes.""" + `.gitmodules` beside it, both kept in code, changes. Each expectation + is what git's own submodule reader does with the same shape in a leg: + no finding exactly where `git submodule update --init` there populates + the dependency (probed on git 2.43).""" plan = answered( planned(registered_source(tmp_path, gitmodules=gitmodules), tmp_path)[0], @@ -660,8 +770,59 @@ def test_a_gitmodules_git_cannot_read_is_not_one_that_registers_nothing( assert finding_codes(result.stderr) == expected, result.stderr assert result.returncode == (1 if expected else 0) if UNREADABLE in expected: - assert (f"FINDING {UNREADABLE}: `git config` cannot read the " - "source's .gitmodules") in result.stderr + assert (f"FINDING {UNREADABLE}: git cannot read the source's " + ".gitmodules for its submodules (fatal: bad config line") \ + in result.stderr + + +@pytest.mark.parametrize("where, expected", [ + ("relative", []), + ("absolute", [UNREGISTERED]), +]) +def test_git_follows_no_include_in_a_gitmodules(tmp_path, where, expected): + """git's submodule reader never follows `[include]` in `.gitmodules`, so + neither does `check`. A registration beside a RELATIVE include is a + registration (`git config --blob` dies on one: "relative config includes + must come from files"), and one that lives only in an ABSOLUTE include, + a file on the machine running `check`, is none.""" + operator = tmp_path / "operator.gitconfig" + operator.write_bytes(registration().encode()) + gitmodules = { + "relative": registration() + "[include]\n\tpath = .gitmodules-local\n", + "absolute": f'[include]\n\tpath = "{operator.as_posix()}"\n', + }[where] + plan = answered( + planned(registered_source(tmp_path, gitmodules=gitmodules), + tmp_path)[0], + tmp_path, {GITMODULES: "code", UPSTREAM: "code"}) + result = check_plan(plan) + assert finding_codes(result.stderr) == expected, result.stderr + + +@pytest.mark.parametrize("source, said", [ + (UNREADABLE_FILE, "fatal: bad config line"), + # `git config --list` lists both; git's submodule reader dies on each. + ({"gitmodules": registration() + '[submodule "b"]\n\tpath\n'}, + "missing value for 'submodule.b.path'"), + ({"gitmodules": '[submodule "d"]\n\tpath = upstream/dependency\n\turl\n'}, + "missing value for 'submodule.d.url'"), + # A `.gitmodules` that is itself a gitlink has no blob to read. + ({"registered": (), "orphan": GITMODULES}, "fatal: git cat-file"), +], ids=["bad-config-line", "path-with-no-value", "url-with-no-value", + "gitmodules-is-a-gitlink"]) +def test_a_gitmodules_git_cannot_read_says_what_git_said(plan_of, tmp_path, + source, said): + """The finding names the file AND what git says about it, on the one + line `check` prints a finding on.""" + plan = plan_of(source)[0] + plan = answered(plan, tmp_path, {path: "code" for path in + (GITMODULES, UPSTREAM) + if path in plan_rows(plan)}) + found = findings(check_plan(plan).stderr) + assert [code for code, _ in found] == [UNREADABLE], found + assert found[0][1].startswith( + "git cannot read the source's .gitmodules for its submodules (" + + said), found @pytest.mark.parametrize("modules_leg, expected", [ @@ -734,7 +895,7 @@ def test_the_submodule_rule_leaves_a_rejected_entry_to_its_own_finding( """An entry `check` already rejects is not ALSO a submodule problem: the helper skips it, and `check` reports it once, by its own finding.""" plan = answered(plan_of(REGISTERED)[0], tmp_path, answers) - carelessly_edited(plan, removed, added) + edited(plan, removed, {added: "spec"} if added else {}) result = check_plan(plan) assert finding_codes(result.stderr) == [own], result.stderr assert adopter.SUBMODULE_REMEDIATION not in result.stderr @@ -802,6 +963,10 @@ def test_execute_refuses_what_check_finds(adopter, plan_of, tmp_path, assert RESERVED in refusal.remediation assert ("cannot be kept in a leg at all" in refusal.remediation and "and re-run `plan`" in refusal.remediation) + # The one edit that answers a submodule apart from its directory entry. + assert ("REPLACE that entry with an entry for each of its children" + in refusal.remediation + and "`plan-covered-twice`" in refusal.remediation) @pytest.mark.parametrize("first_leg, code", [ @@ -872,18 +1037,26 @@ def test_execute_refuses_a_submodule_plan_before_anything_exists( @needs_filter_repo -@pytest.mark.parametrize("modules_leg, dependency_leg", [ - ("code", "code"), - ("spec", "spec"), +@pytest.mark.parametrize("modules_leg, dependency_leg, gitmodules", [ + ("code", "code", None), + ("spec", "spec", None), # Dropped on its own: its registration stays behind in the leg, inert. - ("code", "drop"), -], ids=["both-code", "both-spec", "dropped-on-its-own"]) + ("code", "drop", None), + # Hand-written registrations `check` reads as the leg's clone reads them: + # upper-case keys, and two `git config --blob` refused. + ("code", "code", UPPER_CASE_KEYS.replace("../dependency", THE_DEPENDENCY)), + ("code", "code", + LATIN_1_COMMENT + registration(url=THE_DEPENDENCY).encode()), + ("code", "code", registration(url=THE_DEPENDENCY) + + "[include]\n\tpath = .gitmodules-local\n"), +], ids=["both-code", "both-spec", "dropped-on-its-own", "upper-case-keys", + "a-0xff-comment", "an-include-git-never-follows"]) def test_an_accepted_submodule_plan_adopts_clones_and_bootstraps( - tmp_path, modules_leg, dependency_leg): + tmp_path, modules_leg, dependency_leg, gitmodules): """The plans the refusals above leave legal really adopt: `execute` verifies, and the assembly's `git clone --recurse-submodules` and its `bootstrap.py` both exit 0, which a split plan's clone did not (#166).""" - thing = registered_source(tmp_path) + thing = registered_source(tmp_path, gitmodules=gitmodules) plan = answered(planned(thing, tmp_path)[0], tmp_path, {GITMODULES: modules_leg, UPSTREAM: dependency_leg}) checked = check_plan(plan) From ce5e52d23ce14e0e77465277f886698f4efcc90f Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:09:35 +0000 Subject: [PATCH 05/12] adopt-project.py: `plan` keeps main's follow-ups and seeding record for a source whose `src/` or `contracts/` holds a submodule, spells every path its question names so a non-UTF-8 or newline name no longer crashes it or empties `--out`, and caps the split it names; `check` reads a key with no value beside no submodule as git does, says where git fails on a `.gitmodules` it cannot read, and why a named entry registers nothing; `execute` refuses with the one list `check` prints (#166) Round 4, from this lane's re-verification of ca5fb3d (S1-S3, N1-N4) and lane openRepoShape-2's passes 4 and 5 (NEW-1, NEW-2, NEW-3, R5-1, R5-2). NEW-1 / S1. `cmd_plan` computed the collision prediction, the follow-ups and the seeding record after `ask_about_submodules` had set the entry holding a submodule to `leg: null`. A submodule under `src/` dropped main's follow-up for a harness there that reads `contracts/`, and called the code leg empty: a SEEDED warning, `seeded_from_template: true`, and an `--allow-empty-leg code` it never needed. One under `contracts/` lost the harness line and the `CONTRACTS_DIR` line. All three are now read from `walk`'s entries, before the question is asked; a source with no submodule plans byte for byte as before. NEW-2 / S2 / R5-1. Every path the question names, the submodules and the entries that replace a directory alike, is `_spelled`: between backticks, a byte that is not UTF-8 as `\xe9`, a newline as `\n`, any other character that is not printable by a backslash, and a backslash or a backtick escaped so the backticks still delimit it. A raw surrogate made `write_lf` raise mid-write and leave `--out` empty; a raw newline read back as a space. The spelled newline, between double quotes, is the YAML entry that covers the file. R5-2. Past SPLIT_ENTRIES_NAMED (8) entries the question no longer names them one by one: "replace it with an entry for each path directly in `src/`, 402 in all", or "in one of `docs/`, `docs/theme/`, other than those directories" when it is nested. That edit is the one `check` passes. S3. A `.gitmodules` whose one defect is a key with no value is fatal to git's submodule reader alone, which git runs only for a gitlink. Beside no submodule it is now read as registering nothing: main adopts and clones such a source, and this branch no longer refuses it. Measuring every shape end to end against main also found where git really fails on a file it cannot read, so `plan-gitmodules-unreadable` now says so: beside a submodule, `git filter-repo`'s `git fast-export` reads the checked-out `.gitmodules` for each gitlink and fails the extraction of either leg, wherever the plan sends the two. That includes both of them dropped, which this branch passed and main and this branch both died on after making both leg remotes; it is refused now. Only a plan that extracts no leg escapes it. Otherwise the finding names the assembly's recursive clone, which fails in a leg that keeps a file with a line git cannot parse (no submodule needed), or a `.gitmodules` that is itself a gitlink. SUBMODULE_REMEDIATION no longer offers dropping such a file beside a submodule as an exit. NEW-3 / N1. `check` and `_refuse_what_check_finds` read one list, `_entry_findings`: coverage, each entry's `leg:` (#169's leg paths among them) and the submodule problems. A plan `check` rejected as uncovered and split was refused by `execute` for the split alone; it is refused for both, in one refusal, with #168's remedy and the submodule one. `_refuse_an_unrunnable_plan` is main's again, so its literal `8` cap is gone with the code that had it. #168's tests pass unchanged. N2. `.gitmodules` is now read into `Registrations`, which also keeps why a path an entry names registers nothing: the entry holding it has no url git reads, its name is one git ignores as suspicious, its path starts with `-`, or another entry's move dropped it. The `plan-submodule-unregistered` detail says that, and "no .gitmodules entry ... registers its path with a url" only when no entry names it. N3. git calls the file read through `--file -` "standard input"; the finding says `.gitmodules`. N4. `_SplitTree` reads `GITLINK_MODE` (`_verify`'s literal is left to #173), and `cat-file` is `GIT_CAT_FILE`, as `config` is `GIT_CONFIG`. Tests: new ones for each of the above (follow-ups and seeding for `src/vendor/dep` and `contracts/vendor/x`; four names no plan line can hold, built through `update-index --index-info` and skipped where git refuses them; a newline entry typed from the question; the capped split, flat and nested; why an entry registers nothing, eight ways; where git fails, six shapes; one refusal for an uncovered and split plan through `cmd_execute`); the unreadable-file matrix now refuses both dropped, and says the extraction; the end-to-end refusal gains that case and the end-to-end acceptance a key with no value beside no submodule. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 464 +++++++++++++++++++++----------- tests/test_adopt_submodules.py | 465 ++++++++++++++++++++++++++++----- 2 files changed, 721 insertions(+), 208 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index 4bd7f951..825779a3 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -244,6 +244,9 @@ def emit(lines: list[str], key: str, value, indent: int = 0) -> None: #: The `git` subcommand that reads and writes a config file: a source's #: `.gitmodules`, a leg mount's url, the elector's name. GIT_CONFIG = "config" +#: The `git` subcommand that prints one object: a source's `.gitmodules`, +#: and in the split commit a symlink's target and the shape pin. +GIT_CAT_FILE = "cat-file" def _git_path(raw: bytes) -> str: @@ -267,42 +270,84 @@ def _a_name_git_ignores(name: str) -> bool: return not name or ".." in re.split(r"[/\\]", name) -def _registrations(listed: bytes) -> frozenset[str] | str: - """The paths `git config -z --list` output registers with a url, read as - git's submodule reader reads them (submodule-config.c, `parse_config`); - or, for a `path` or `url` with no value at all, what that reader dies on. - - In the order git applies them: a name it ignores registers nothing; a - `path` or `url` with no `=` is fatal to it (`-z` prints such a key with - no newline); a value starting with `-` is ignored and the one before it - stands; a later value overrides an earlier one for its name, and a path - belongs to the LAST name given it. git keeps that path-to-name map by - path, so a name moved to another path removes whatever its old path - held, even when a later name had taken the path over -- the `pop` below, - which is git's `cache_remove_path`. +class Registrations: + """What git's submodule reader (submodule-config.c, `parse_config`) makes + of a `.gitmodules`: `path in registrations` when the file registers that + path WITH a url, and `why_not(path)` when an entry names it and git + registers nothing there. + + `read` takes each `submodule..path` or `.url` value in the file's + order and applies it as git does: a name git ignores registers nothing; a + value starting with `-` is ignored and the one before it stands; a later + value overrides an earlier one for its name, and a path belongs to the + LAST name given it. git keeps that path-to-name map by path, so a name + moved to another path removes whatever its old path held, even when a + later name had taken the path over -- `_move`'s `pop`, which is git's + `cache_remove_path`. How a named path came to register nothing is kept + in words: the person told that a submodule is not registered is looking + at an entry for it, and is told why git ignores that entry (#166). """ - path_of: dict[str, str] = {} - name_at: dict[str, str] = {} - url_of: dict[str, str] = {} + + def __init__(self) -> None: + self._path_of: dict[str, str] = {} + self._name_at: dict[str, str] = {} + self._url_of: dict[str, str] = {} + self._lost: dict[str, str] = {} + + def __contains__(self, path: str) -> bool: + name = self._name_at.get(path) + return name is not None and bool(self._url_of.get(name)) + + def why_not(self, path: str) -> str | None: + """Why git registers nothing at `path` though an entry names it; + None when no entry of the file does.""" + name = self._name_at.get(path) + if name is not None: + return (f"its {GITMODULES} entry {_spelled(name)} has no url git " + "reads") + return self._lost.get(path) + + def read(self, name: str, key: str, value: str) -> None: + if _a_name_git_ignores(name): + self._lose(key, value, f"git ignores its {GITMODULES} entry " + f"{_spelled(name)} as a suspicious name") + elif value.startswith("-"): + self._lose(key, value, f"git ignores its {GITMODULES} path, which " + "starts with `-`") + elif key == "url": + self._url_of[name] = value + else: + self._move(name, value) + + def _lose(self, key: str, path: str, why: str) -> None: + if key == "path": + self._lost[path] = why + + def _move(self, name: str, path: str) -> None: + old = self._path_of.get(name) + if old is not None and self._name_at.pop(old, None) is not None: + self._lost[old] = (f"git dropped its path when the {GITMODULES} " + f"entry {_spelled(name)} moved to " + f"{_spelled(path)}") + self._path_of[name] = path + self._name_at[path] = name + + +def _registrations(listed: bytes) -> Registrations | str: + """`git config -z --list` output, read by `Registrations`; or, for a + `path` or `url` with no value at all, what git's submodule reader dies + on. `-z` prints such a key with no newline, and git looks for its value + after it has filtered the name and before it filters the value.""" + registrations = Registrations() for record in listed.split(b"\x00"): key, newline, value = _git_path(record).partition("\n") match = SUBMODULE_KEY_RE.fullmatch(key) - if not match or _a_name_git_ignores(match["name"]): + if not match: continue - if not newline: + if not newline and not _a_name_git_ignores(match["name"]): return f"missing value for '{key}'" - if value.startswith("-"): - continue - name = match["name"] - if match["key"] == "url": - url_of[name] = value - continue - if name in path_of: - name_at.pop(path_of[name], None) - path_of[name] = value - name_at[value] = name - return frozenset(path for path, name in name_at.items() - if url_of.get(name)) + registrations.read(match["name"], match["key"], value) + return registrations class Source: @@ -366,10 +411,10 @@ def tree(self) -> list[tuple[str, str, str, int]]: del kind return sorted(out) - def registered_submodules(self, tree: list) -> frozenset[str] | str: - """Each path the `.gitmodules` at the pinned commit registers WITH a - url; or, as a string, what git says when it cannot read that file - for its submodules. + def registered_submodules(self, tree: list) -> Registrations | str: + """What the `.gitmodules` at the pinned commit registers, each path + WITH a url; or, as a string, what git says when it cannot read that + file for its submodules. A REGISTRATION is a `submodule..path` and a non-empty `submodule..url` under the same name: the path is what git @@ -395,12 +440,20 @@ def registered_submodules(self, tree: list) -> frozenset[str] | str: come back as one that registers nothing. `--list` exits 0 on any file it parsed, empty or not, and 128 on one it could not, and git's words come back instead of paths: a read that failed is not an answer - (#166), and the person told so is told why. + (#166), and the person told so is told why. git calls the file it + read through `--file -` "standard input"; they are told `.gitmodules`. + + A KEY WITH NO VALUE -- `path` or `url` with nothing after it -- is + fatal to git's submodule reader alone, and git runs that reader only + for a gitlink. So in a tree that holds none, nothing ever reads the + file that way, the assembly's clone and `execute` pass over it as + main always did, and it registers nothing (#166's S3). A line git + cannot parse at all is another matter: every reader dies on it. """ if not any(path == GITMODULES for path, _, _, _ in tree): - return frozenset() + return Registrations() blob = subprocess.run( - ["git", "cat-file", "blob", f"{self.commit}:{GITMODULES}"], + ["git", GIT_CAT_FILE, "blob", f"{self.commit}:{GITMODULES}"], cwd=self.path, capture_output=True, check=False) # A `.gitmodules` that is no blob -- a gitlink, say -- leaves nothing # for `git config` to read, and what `cat-file` said is the answer. @@ -409,8 +462,13 @@ def registered_submodules(self, tree: list) -> frozenset[str] | str: "--list"], cwd=self.path, input=blob.stdout, capture_output=True, check=False) if listed.returncode: - return _one_line(listed.stderr.decode(errors="replace")) - return _registrations(listed.stdout) + return _one_line(listed.stderr.decode(errors="replace")).replace( + "standard input", GITMODULES) + read = _registrations(listed.stdout) + if isinstance(read, str) and not any( + mode == GITLINK_MODE for _, mode, _, _ in tree): + return Registrations() + return read def commit_count(self) -> int: return int(git_out(["rev-list", "--count", self.commit], cwd=self.path)) @@ -518,11 +576,12 @@ def ask_about_submodules(policy: PathPolicy, entries: list[Entry], classified by. The ruling still roots the FILES under those paths, and one of several submodules may be dropped on its own, so an entry holding more than one path NAMES THE ENTRIES THAT REPLACE IT, each submodule one - of its own (`_own_entries`): splitting it is the only edit that answers - those paths apart, because an entry added beside it for a path it - already covers is `plan-covered-twice`. `walk`'s folding is left alone; - only the covering entry is replaced. A source with no gitlink, the - common case, gets its entries back untouched. + of its own (`_own_entries`), or past SPLIT_ENTRIES_NAMED says how to + make them: splitting it is the only edit that answers those paths + apart, because an entry added beside it for a path it already covers is + `plan-covered-twice`. `walk`'s folding is left alone; only the covering + entry is replaced. A source with no gitlink, the common case, gets its + entries back untouched. """ gitlinks = [path for path, mode, _, _ in tree if mode == GITLINK_MODE] if not gitlinks: @@ -564,25 +623,80 @@ def _child_of(directory: str, path: str) -> str: return directory + head + slash +#: How many of the entries that replace a directory entry holding a +#: submodule its question names one by one. Past it, the question says +#: which directories' paths they are and counts them, as `plan-unresolved` +#: names eight paths and counts the rest: four hundred files beside one +#: submodule would otherwise be four hundred names on one line of the plan, +#: and again in `check`'s `plan-unresolved` finding (#166's R5-2). +SPLIT_ENTRIES_NAMED = 8 + + def _asked_about(entry: Entry, held: list[str], asked: Verdict, own: list[str]) -> Entry: """`entry` as `ask_about_submodules` writes it: the `.gitmodules` verdict, whose `leg:` is null, and its question naming `held` and, when - it holds more than one path, the `own` entries that replace it.""" + it holds more than one path, the `own` entries that replace it. Each + path is `_spelled`, because the question is a line of the plan.""" question = (f"{asked.question or ''} This entry holds the submodule(s) " - f"{', '.join(held)}: it is asked this question, not " + f"{_listed(held)}: it is asked this question, not " f"classified by its rule `{entry.rule}`.") if len(own) > 1: - listed = ", ".join(f"`{path}`" for path in own) question += (" Split the entry if its paths do not all belong in one " - f"leg: replace it with an entry for each of {listed}, " - "each with its own `leg:`. An entry added beside this " - "one for a path it covers is `plan-covered-twice`.") + f"leg: replace it with " + f"{_replacements(entry.path, held, own)}, each with its " + "own `leg:`. An entry added beside this one for a path " + "it covers is `plan-covered-twice`.") return Entry(entry.path, Verdict(None, asked.rule, asked.reason, asked.confidence, question), entry.files, entry.bytes) +def _replacements(entry: str, held: list[str], own: list[str]) -> str: + """The entries `own` that replace the directory entry `entry`, as its + question names them: each one while there are few, and past + SPLIT_ENTRIES_NAMED the directories they are the paths of -- `entry` + and each one `_own_entries` opened on the way to a submodule in `held`, + which are no entries themselves -- and how many there are.""" + if len(own) <= SPLIT_ENTRIES_NAMED: + return f"an entry for each of {_listed(own)}" + opened = sorted({path[:end + 1] for path in held + for end in range(len(entry) - 1, len(path)) + if path[end] == "/"}) + where = (_listed(opened) if len(opened) == 1 else + f"one of {_listed(opened)}, other than those directories") + return f"an entry for each path directly in {where}, {len(own)} in all" + + +def _listed(paths: list[str]) -> str: + return ", ".join(_spelled(path) for path in paths) + + +def _spelled(path: str) -> str: + """`path` between backticks, as a question or a finding names it. + + The plan is UTF-8 text, one value to a line, and a path is bytes: + `Source.tree` keeps one that is not UTF-8 as a surrogate, which no UTF-8 + file can hold -- written raw, it raised mid-write and left `--out` empty + (#166's NEW-2) -- and a name may hold a newline, which would end the + line. So a character that is not printable is spelled with a backslash, + `\\xe9` for the byte 0xE9 and `\\n` for a newline, and so are a + backslash and a backtick, so that the spelling reads one way only and + the backticks still delimit it. + """ + return "`" + "".join(map(_spelled_character, path)) + "`" + + +def _spelled_character(char: str) -> str: + if char in "\\`": + return "\\" + char + if char.isprintable(): + return char + if "\udc80" <= char <= "\udcff": + return f"\\x{ord(char) - 0xDC00:02x}" + return ascii(char)[1:-1] + + def cross_leg_references(source: "Source", entries: list["Entry"], spec_path: str, code_path: str) -> list[str]: """Files that will move to the CODE leg and name a path in the SPEC leg. @@ -702,7 +816,8 @@ def assigned_paths(entries) -> dict[str, list[str]]: def render_plan(args, source: Source, entries: list[Entry], names: dict, - pins: list[str], follow_ups: list[str]) -> str: + pins: list[str], follow_ups: list[str], + seeded: list[str]) -> str: lines = [ "schema_version: 1", f"kind: {PLAN_KIND}", @@ -752,7 +867,6 @@ def render_plan(args, source: Source, entries: list[Entry], names: dict, "# its `open` is listed here (2026-09-02).", "pins: [" + ", ".join(pins) + "]", ""] - seeded = seeded_legs(assigned_paths(entries)) lines += [ "# A leg that NO entry below assigns a path to is SEEDED from the", "# shape's own leg template as one initial commit, instead of being", @@ -1141,7 +1255,7 @@ def _print_triad_advisory(path: Path, commit: str | None = None) -> None: def _print_plan_report(args, source: Source, tree: list, entries: list, - follow_ups: list, out: Path) -> None: + follow_ups: list, seeded: list, out: Path) -> None: """What `plan` says to the terminal once the file is on disk. Split out of `cmd_plan` for #138. THE UNRESOLVED PATHS ARE LAST on @@ -1159,8 +1273,7 @@ def _print_plan_report(args, source: Source, tree: list, entries: list, print(f"\nfollow-ups ({len(follow_ups)}):") for item in follow_ups: print(f" - {item}") - for line in seeding_warnings(seeded_legs(assigned_paths(entries)), - set(args.allow_empty_leg or [])): + for line in seeding_warnings(seeded, set(args.allow_empty_leg or [])): print(line) unresolved = [e for e in entries if e.leg is None] print(f"\nplan written to {out}") @@ -1180,16 +1293,26 @@ def cmd_plan(args) -> int: names, pins = _checked_plan_inputs(args, source, naming) tree = source.tree() - entries = ask_about_submodules( - policy, walk(policy, [(path, size) for path, _, _, size in tree]), - tree) - materialized = _predict_collisions(entries) - follow_ups = follow_ups_for(source, entries, names, args.spec_path, + # WHAT THE RULES IMPLY is read from `walk`'s entries, BEFORE + # `ask_about_submodules` asks again on the ones that hold a submodule: the + # shape files a rooted path will collide with, the follow-ups, and the + # legs a split would seed. Asking takes back no rule's answer for the + # files under such an entry; read after it, a `leg: null` dropped main's + # follow-up for a harness under `src/` that reads `contracts/`, and + # called the code leg empty, so the plan recorded it as seeded and told + # the person to re-run with `--allow-empty-leg code`, which it never + # needed (#166's NEW-1). A source with no submodule plans as it did. + ruled = walk(policy, [(path, size) for path, _, _, size in tree]) + materialized = _predict_collisions(ruled) + follow_ups = follow_ups_for(source, ruled, names, args.spec_path, args.code_path, materialized) + seeded = seeded_legs(assigned_paths(ruled)) + entries = ask_about_submodules(policy, ruled, tree) out = Path(args.out) - write_lf(out, render_plan(args, source, entries, names, pins, follow_ups)) + write_lf(out, render_plan(args, source, entries, names, pins, follow_ups, + seeded)) - _print_plan_report(args, source, tree, entries, follow_ups, out) + _print_plan_report(args, source, tree, entries, follow_ups, seeded, out) return 0 @@ -1485,7 +1608,8 @@ def _leg_findings(plan: Plan) -> list[str]: "assembly's clone fails on it inside the leg. `drop` it, or register it " "in the source (that entry, committed) and re-run `plan`; a " "`.gitmodules` git cannot read for its submodules is repaired in the " - "source and re-planned the same way, or dropped with every submodule. " + "source and re-planned the same way (in a source with no submodule, " + "dropping it is enough). " "To answer a submodule apart from the rest of the directory entry that " "holds it, REPLACE that entry with an entry for each of its children, " "the submodule's own path among them, as that entry's `question:` lists " @@ -1499,6 +1623,18 @@ def _leg_findings(plan: Plan) -> list[str]: #: words by the two rules that find one. NO_URL_IN_THE_LEG = ("so the leg that holds the gitlink holds no URL for it " "and the assembly's recursive clone fails there") +#: Why a kept gitlink is not registered, when no entry of `.gitmodules` +#: names its path at all; `Registrations.why_not` says it otherwise. +NO_ENTRY_NAMES_IT = (f"no {GITMODULES} entry in the source registers its " + "path with a url") +#: Where git fails on a `.gitmodules` it cannot read for its submodules: +#: the two ends of `_where_git_fails`, each said only where it is true. +FAILS_IN_THE_EXTRACTION = ( + "and the source holds a submodule, so `git filter-repo` fails on that " + "file when `execute` extracts a leg, wherever the plan sends the two") +FAILS_IN_THE_CLONE = ( + "so which submodules it registers is unknown, and the plan keeps it or a " + "submodule in a leg, where the assembly's recursive clone fails on it") def _answered_legs(entries: list, paths: list[str]) -> dict: @@ -1528,22 +1664,21 @@ def _as_entry(path: str, entry: str) -> str: def submodule_plan_problems(entries: list, tree: list, - registered: frozenset | str + registered: Registrations | str ) -> list[tuple[str, str]]: """`(code, detail)` for each way the plan splits the source's OWN submodules. - ONE DEFINITION, asked by `check` for findings and by `execute` for a - refusal, so that `execute` refuses every SUBMODULE problem `check` finds - (#166). An entry `check` rejects for another reason -- uncovered, covered - twice, a misspelled leg -- is left out here (see `_answered_legs`): - `check` reports it by its own finding, and `execute` refuses it in - #168's `_refuse_what_check_finds`. `tree` is - `Source.tree()`: a gitlink is visible only by its mode, + ONE DEFINITION, in the one list of findings `check` prints and `execute` + refuses (`_entry_findings`), so that `execute` refuses every SUBMODULE + problem `check` finds, in its words (#166). An entry `check` rejects for + another reason -- uncovered, covered twice, a misspelled leg -- is left + out here (see `_answered_legs`): it is in that list by its own finding. + `tree` is `Source.tree()`: a gitlink is visible only by its mode, which no path glob in `contracts/path-classification.yaml` can see. - `registered` is `Source.registered_submodules(tree)`: the gitlink paths - the source's `.gitmodules` names with a url, or, as a string, what git - said when it could not read that file for its submodules. The caller - reads both, so this is a function of its arguments alone. + `registered` is `Source.registered_submodules(tree)`: what the source's + `.gitmodules` registers, or, as a string, what git said when it could + not read that file for its submodules. The caller reads both, so this is + a function of its arguments alone. A SUBMODULE IS ONE FACT IN TWO PLACES: the gitlink in the tree and its registration in `.gitmodules`. A plan that sent the two to different legs @@ -1561,10 +1696,10 @@ def submodule_plan_problems(entries: list, tree: list, registers (#165). * A `.gitmodules` git cannot read for its submodules is never taken to register nothing (`plan-gitmodules-unreadable`, saying what git - said). It is a problem when the plan - keeps the file in a leg, where git fails on it when the assembly's - clone recurses there, or keeps any gitlink in a leg, whose - registration then cannot be checked. + said, and where git fails on it: `_where_git_fails`). Beside a + submodule it fails the extraction of either leg, wherever the plan + sends the two; otherwise it is a problem where the plan keeps it, or + a gitlink, in a leg the assembly's clone then recurses into. * Every gitlink the plan KEEPS, in the spec or the code leg, must be REGISTERED (`plan-submodule-unregistered`). A leg is WORSE than the source for one that is not: a top-level `git clone @@ -1583,7 +1718,7 @@ def submodule_plan_problems(entries: list, tree: list, cloned and never fails. Dropping all of them with the file is allowed too. - THE ORDER IS THE ORDER TO FIX THEM IN, and `execute`'s refusal is named + THE ORDER IS THE ORDER TO FIX THEM IN, and a refusal they begin is named by the first: a `.gitmodules` in the root makes every registered gitlink kept in a leg a split, an unreadable one hides which gitlinks are registered, and an unregistered gitlink may be fixed in the SOURCE, by a @@ -1606,26 +1741,60 @@ def submodule_plan_problems(entries: list, tree: list, for path, (leg, entry) in answered.items() if leg == "root"] registry_leg = answered.get(GITMODULES, (None, None))[0] kept = _kept_in_a_leg(answered, gitlinks) - if isinstance(registered, frozenset): + if not isinstance(registered, str): return problems + _kept_submodule_problems(kept, registered, registry_leg) - if kept or registry_leg in EXTRACTED_LEGS: + failure = _where_git_fails(entries, tree, registry_leg, kept) + if failure: problems.append(( "plan-gitmodules-unreadable", f"git cannot read the source's {GITMODULES} for its submodules " - f"({registered}), so which submodules it registers is unknown, " - "and the plan keeps it or a submodule in a leg, where the " - "assembly's recursive clone fails on it")) + f"({registered}), {failure}")) return problems +def _where_git_fails(entries: list, tree: list, registry_leg: str | None, + kept: list) -> str | None: + """Where git fails on a source `.gitmodules` it cannot read for its + submodules, as the plan answers it; None where nothing reads it so. + + Measured end to end with git 2.43 (#166's S3), the first failure first: + + * `git filter-repo` runs `git fast-export`, which reads the CHECKED-OUT + `.gitmodules` for each gitlink in the history it exports. So such a + FILE in a source that holds a submodule fails the extraction of + either leg, wherever the plan sends the two -- after both leg + repositories were made. A plan that extracts no leg, every path + rooted or dropped, is the only one that escapes it. + * Otherwise the assembly's recursive clone fails in a leg the plan + keeps the file in, or a gitlink: on a line git cannot parse, even + with no gitlink in the leg, and on a `.gitmodules` that is itself a + gitlink, which the clone checks out as a directory and which + registers nothing. + + A key with no value is fatal to git's submodule reader alone, which + git runs only for a gitlink, so `Source.registered_submodules` reads it + as registering nothing in a tree with none, and it never reaches here + without one. + """ + a_file = any(path == GITMODULES and mode in FILE_MODES + for path, mode, _, _ in tree) + beside = any(mode == GITLINK_MODE for _, mode, _, _ in tree) + if a_file and beside and any(str(e.get("leg")) in EXTRACTED_LEGS + for e in entries): + return FAILS_IN_THE_EXTRACTION + if kept or registry_leg in EXTRACTED_LEGS: + return FAILS_IN_THE_CLONE + return None + + def _kept_in_a_leg(answered: dict, gitlinks: list[str]) -> list[tuple]: """`(path, leg, entry)` for each gitlink `answered` keeps in a leg.""" return [(path, *answered[path]) for path in gitlinks if path in answered and answered[path][0] in EXTRACTED_LEGS] -def _kept_submodule_problems(kept: list, registered: frozenset, +def _kept_submodule_problems(kept: list, registered: Registrations, registry_leg: str | None ) -> list[tuple[str, str]]: """The last two rules of `submodule_plan_problems`, for the gitlinks the @@ -1633,13 +1802,16 @@ def _kept_submodule_problems(kept: list, registered: frozenset, A gitlink is unregistered or split, never both. With no registration it has no URL in ANY leg, and moving `.gitmodules` would not give it one. - `registry_leg` is None when `.gitmodules` is absent or not yet answered, - which leaves no leg for a gitlink to be split from. + An unregistered one is told why: an entry of `.gitmodules` may name its + path and still register nothing, and saying "no entry" beside it would + send the person looking for one that is there. `registry_leg` is None + when `.gitmodules` is absent or not yet answered, which leaves no leg + for a gitlink to be split from. """ problems = [ ("plan-submodule-unregistered", - f"the submodule {_as_entry(path, entry)} has leg: {leg} but no " - f"{GITMODULES} entry in the source registers its path with a url, " + f"the submodule {_as_entry(path, entry)} has leg: {leg} but " + f"{registered.why_not(path) or NO_ENTRY_NAMES_IT}, " + NO_URL_IN_THE_LEG) for path, leg, entry in kept if path not in registered] if registry_leg is not None: @@ -1666,6 +1838,28 @@ def _submodule_findings(plan: Plan, source: Source, return found, [SUBMODULE_REMEDIATION] if found else [] +def _entry_findings(plan: Plan, source: Source + ) -> tuple[list[str], list[str]]: + """Every FINDING `check` prints about the plan's ENTRIES, in its order, + and what it prints once after them: coverage, then each entry's `leg:`, + then the source's own submodules. + + ONE LIST, read by `check` and by `execute`'s `_refuse_what_check_finds`. + #168 and #166 each added findings to `check` and a refusal to `execute` + while both were open, and composed them separately so as not to edit the + same lines; three compositions then disagreed, and a plan `check` + rejected as uncovered AND split was refused by `execute` for the split + alone, to be refused again for the rest once that was fixed. Composed + once, the two commands cannot differ about a plan. + """ + tree = source.tree() + findings = (_coverage_findings([str(e.get("path")) for e in plan.entries], + [path for path, _, _, _ in tree]) + + _leg_findings(plan)) + submodule_findings, remediation = _submodule_findings(plan, source, tree) + return findings + submodule_findings, remediation + + def _topics_line(topic: str, local: bool) -> str: """The `topics` plan line, the one `scaffold-project.py` also prints. @@ -1723,13 +1917,8 @@ def cmd_check(args) -> int: "tree that has moved proves nothing about the tree that will be " "split.") - entry_paths = [str(e.get("path")) for e in plan.entries] - tree = source.tree() - tree_paths = [path for path, _, _, _ in tree] - findings.extend(_coverage_findings(entry_paths, tree_paths)) - findings.extend(_leg_findings(plan)) - submodule_findings, remediation = _submodule_findings(plan, source, tree) - findings.extend(submodule_findings) + entry_findings, remediation = _entry_findings(plan, source) + findings.extend(entry_findings) names = plan.names() pins = set(plan.pins) @@ -1812,23 +2001,11 @@ def _confirm(args, plan: Plan, names: dict) -> None: def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: - """The three states a plan can be in that must not be executed. + """The two states a plan can be in that must not be executed. An unanswered question is never an implicit `root`, and a plan written against a tree that has since moved proves nothing about the tree that - would be split — which is how a path goes missing. And a plan that - splits a submodule from its registration, keeps one nothing registers, - or keeps one in the root, builds an assembly that cannot be cloned: - refused HERE, before either leg exists, because once - `_create_leg_remotes` has run, a re-run with the corrected plan meets two - legs that already exist (#166). The refusal is named by the FIRST - problem, which `submodule_plan_problems` lists in the order to fix them - in, and its detail names up to eight of them and counts the rest. - - So `execute` refuses every SUBMODULE problem `check` finds, in `check`'s - words. A path no entry covers, or two do, or whose entry has a - misspelled leg, passes here and is refused by #168's - `_refuse_what_check_finds`, which `cmd_execute` runs next. + would be split — which is how a path goes missing. """ unresolved = [e for e in plan.entries if e.get("leg") is None] if unresolved: @@ -1847,15 +2024,6 @@ def _refuse_an_unrunnable_plan(plan: Plan, source: Source) -> None: "Remediation: re-run `plan`, re-answer anything new, then " "`check`. Splitting a tree the plan has not seen is how a path " "goes missing.") - tree = source.tree() - problems = submodule_plan_problems(plan.entries, tree, - source.registered_submodules(tree)) - if problems: - details = [detail for _, detail in problems[:8]] - if len(problems) > 8: - details.append(f"and {len(problems) - 8} more") - raise Refusal(problems[0][0], "; ".join(details), - SUBMODULE_REMEDIATION) def _create_leg_remotes(plan: Plan, names: dict, repositories: dict, @@ -1995,8 +2163,8 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, # `git rm` of a submodule edits the file that registers it. #166 ruled # that the assembly root keeps no submodule of the source's: a plan that # keeps `.gitmodules` or a gitlink here, or splits one from its - # registration, is a `check` finding and an `execute` refusal before this - # runs, and #168 refuses every other finding of `check` there too. + # registration, is a `check` finding, and `execute` refuses every + # finding of `check` before this runs (#168). # Appending the mounts to a `.gitmodules` left here, rather than # replacing it, is MOUNT DEFENCE IN DEPTH for a plan that slipped past # both, not a shape a plan may choose; such a file still fails @@ -2101,11 +2269,13 @@ def _refuse_what_check_finds(plan: Plan, source: Source) -> None: say yes after `check`, but `execute` is the one gate that cannot be skipped, so it says no to the same findings. - THE FINDINGS ARE `check`'S OWN: the same two functions over the same two - lists, so the two commands cannot disagree about what a plan covers. The - refusal's code is the FIRST finding's own (`plan-uncovered`, ...); its - detail is every finding as `check` prints it, capped at - `REFUSED_FINDINGS_SHOWN` with an "and N more" tail. + THE FINDINGS ARE `check`'S OWN: `_entry_findings`, the one list `check` + prints, so the two commands cannot disagree about what a plan covers. + The refusal's code is the FIRST finding's own (`plan-uncovered`, ...); + its detail is every finding as `check` prints it, capped at + `REFUSED_FINDINGS_SHOWN` with an "and N more" tail; its remediation is + this one, then whatever `check` prints after its findings -- #166's + submodule remediation, when one of them is a submodule problem. THE ORDER IS PART OF THE FIX. This runs AFTER `_refuse_an_unrunnable_plan`: `_leg_findings` also reports `plan-unresolved`, which that function has @@ -2122,16 +2292,16 @@ def _refuse_what_check_finds(plan: Plan, source: Source) -> None: already made the work directory and an `org/repo` source has already been cloned into it: they come first, and neither is a leg.) - ONE LIST, LATER. #166 adds `submodule_plan_problems` to `check` and to - `_refuse_an_unrunnable_plan`. Once both branches have merged, `check`, - `_refuse_an_unrunnable_plan` and this function can read ONE shared list of - findings. That is deliberately not done here, so that the two branches do - not edit the same functions and conflict. + ONE LIST. While #168 and #166 were both open, each composed its own + findings, so that the two branches would not edit the same functions; + #166 refused its submodule problems in `_refuse_an_unrunnable_plan`. + With both merged, `check` and this function read `_entry_findings` and + nothing else, and `_refuse_an_unrunnable_plan` is back to its two + states: a plan `check` rejects as uncovered and split is refused for + both at once, and a submodule problem is refused here like any other, + after `_leg_paths` has checked every path as a `git` argument. """ - entry_paths = [str(e.get("path")) for e in plan.entries] - tree_paths = [path for path, _, _, _ in source.tree()] - findings = (_coverage_findings(entry_paths, tree_paths) - + _leg_findings(plan)) + findings, remediation = _entry_findings(plan, source) if not findings: return # `FINDING : ` is the one shape both functions write. @@ -2142,13 +2312,14 @@ def _refuse_what_check_finds(plan: Plan, source: Source) -> None: if len(findings) > REFUSED_FINDINGS_SHOWN: lines.append(f" and {len(findings) - REFUSED_FINDINGS_SHOWN} more") raise Refusal( - code, "\n".join(lines), - "Remediation: no leg repository was created and nothing was pushed. " - "Correct the plan -- answer each entry's `leg:` and cover every " - "source path exactly once -- and run `check` until it prints `plan " - "ok`, then run `execute` again. A leg made from a plan `check` " - "rejects is one the corrected plan cannot make again: under " - "`--local-remote-dir` it meets `leg-remote-exists`.") + code, "\n".join(lines), "\n".join([ + "Remediation: no leg repository was created and nothing was " + "pushed. Correct the plan -- answer each entry's `leg:` and cover " + "every source path exactly once -- and run `check` until it " + "prints `plan ok`, then run `execute` again. A leg made from a " + "plan `check` rejects is one the corrected plan cannot make " + "again: under `--local-remote-dir` it meets `leg-remote-exists`.", + *remediation])) def _refuse_unconsented_seeding(args, plan: Plan, seeded: list) -> None: @@ -2649,7 +2820,7 @@ def __init__(self, assembly: Path, commit: str) -> None: for path, mode, oid, _ in _tree_of(assembly, commit)} #: The mounted legs: a directory in a checkout, so `..` may leave one. self.mounts = {path for path, (mode, _) in self.entries.items() - if mode == "160000"} + if mode == GITLINK_MODE} self._dirs: set[str] | None = None def blob_at(self, path: str) -> str: @@ -2717,7 +2888,7 @@ def _target_of(self, at: str) -> str | None: mode, oid = self.entries.get(at, ("", "")) if mode != SYMLINK_MODE: return None - return git_out(["cat-file", "blob", oid], cwd=self.assembly, + return git_out([GIT_CAT_FILE, "blob", oid], cwd=self.assembly, binary=True).decode("utf-8", "surrogateescape") @@ -2730,7 +2901,8 @@ def _pinned_shape_paths(assembly: Path, pin: str) -> list[str] | None: the one outcome it exists to prevent. """ try: - loaded = parse_yaml(git_out(["cat-file", "blob", pin], cwd=assembly)) + loaded = parse_yaml(git_out([GIT_CAT_FILE, "blob", pin], + cwd=assembly)) except (YamlError, UnicodeDecodeError): return None rows = loaded.get("files") if isinstance(loaded, dict) else None diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index 9b0137b4..f8cd8e07 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -9,7 +9,9 @@ import importlib.util import re import shutil +import subprocess import sys +from argparse import Namespace from pathlib import Path import pytest @@ -76,7 +78,7 @@ def test_adopt_moves_existing_gitmodules_with_its_dependency(tmp_path, checked = adopt("check", plan) assert checked.returncode == 0, checked.stderr + checked.stdout result = adopt("execute", plan, "--yes", - "--local-remote-dir", str(tmp_path / "remotes"), + "--local-remote-dir", str(tmp_path / REMOTES), "--work-dir", str(tmp_path / "work")) assert result.returncode == 0, result.stderr + result.stdout @@ -162,7 +164,7 @@ def build(registered=("upstream/kept", "upstream/moved"), tree=None): for path in registered: add_submodule(source, dependency, path) if registered: - commit_as_source_human(source, "Register dependencies") + commit_as_source_human(source, REGISTER) work = tmp_path / "work" work.mkdir() names = {role: f"Northwind-{role}" for role in ("spec", "code")} @@ -241,7 +243,7 @@ def test_mount_stages_gitmodules_past_a_gitignore_that_hides_it( none to replace and `git submodule add` writes its own; a source whose file the plan moved gets a fresh one, staged with `-f`. """ - tree = {"README.md": "# Thing\n", ".gitignore": IGNORE_DOTFILES, + tree = {README: "# Thing\n", ".gitignore": IGNORE_DOTFILES, "src/app.py": "VALUE = 1\n"} adopter, source, work, names, urls = mount_world(registered, tree) assert git("check-ignore", "-q", "--no-index", ".gitmodules", @@ -273,13 +275,13 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( # --- the plan's own answer: one leg for a submodule and its registration ---- # -# #166. `check` and `execute` read the SAME `submodule_plan_problems`, so -# `execute` refuses every submodule problem `check` finds (and #168 every -# other finding), and `plan` asks the question on every entry that holds a -# submodule, so it never proposes what `check` refuses. `check` reads the -# source's `.gitmodules` as git's own submodule reader will read it in the -# leg. None of these needs `git filter-repo` but the ones that reach -# `execute` past its preflight. +# #166. `check` and `execute` read the SAME list of findings, so `execute` +# refuses every submodule problem `check` finds beside every other one (#168), +# and `plan` asks the question on every entry that holds a submodule, so it +# never proposes what `check` refuses. `check` reads the source's +# `.gitmodules` as git's own submodule reader will read it in the leg. None +# of these needs `git filter-repo` but the ones that reach `execute` past its +# preflight. #: The ids, as `check` prints them and `execute` raises them. SPLIT = "plan-submodule-split" @@ -317,6 +319,18 @@ def registration(name: str = DEPENDENCY, url: str = "../dependency") -> str: #: code, and a file a directory holding a submodule may hold beside it. VENDORED = "src/vendored" DOCS_INDEX = "docs/index.md" +#: The paths the questions below name, each in more than two of them. +APP = "src/app/" +DOCS = "docs/" +THEME_README = "docs/theme/README.md" +THEME_SUB = "docs/theme/sub" +README = "README.md" +#: A later section that gives `registration()`'s path to a name with no url. +SHADOWING = '[submodule "b"]\n\tpath = upstream/dependency\n' +#: The commit a source registers its dependencies in. +REGISTER = "Register dependencies" +#: Where `execute` makes the legs' bare remotes, under a test's own directory. +REMOTES = "remotes" #: A section that moves `registration()`'s name to another path. MOVED_ELSEWHERE = '[submodule "upstream/dependency"]\n\tpath = elsewhere\n' @@ -327,7 +341,7 @@ def registration(name: str = DEPENDENCY, url: str = "../dependency") -> str: #: A source whose plan asks nothing but the submodule's questions: `specs/` #: and `src/` classify themselves and `README.md` stays in the root. -SMALL_TREE = {"README.md": "# Thing\n", "src/app/util.py": "VALUE = 1\n", +SMALL_TREE = {README: "# Thing\n", "src/app/util.py": "VALUE = 1\n", "specs/001-feature/tasks.md": "- [ ] one\n"} # The sources these tests answer plans for, as `registered_source` @@ -384,7 +398,7 @@ def registered_source(base: Path, registered=(DEPENDENCY,), (source / GITMODULES).write_bytes(raw.replace( THE_DEPENDENCY.encode(), dependency.as_posix().encode())) git("add", "--", GITMODULES, cwd=source) - commit_as_source_human(source, "Register dependencies") + commit_as_source_human(source, REGISTER) return source @@ -501,7 +515,7 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): ((".cursor",), {}, ".cursor", "root-assistant-instructions", ()), # Asked already, but whether it is root: `.gitmodules`' question instead. ((".agents",), {}, ".agents", "ambiguous-assistant-project-memory", ()), - (("docs/theme",), {DOCS_INDEX: "# Docs\n"}, "docs/", + (("docs/theme",), {DOCS_INDEX: "# Docs\n"}, DOCS, "spec-governance", (DOCS_INDEX, "docs/theme")), ((DEPENDENCY,), {}, UPSTREAM, "default", ()), # Nothing but submodules, and still split, to drop one on its own. @@ -509,12 +523,10 @@ def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): ("vendor/a", "vendor/b")), # Under a rule that would have called it code, unasked (#166's N3). ((VENDORED,), {}, "src/", "code-source-and-tests", - ("src/app/", VENDORED)), + (APP, VENDORED)), # Deeper than one level: only the directory holding it is opened. - (("docs/theme/sub",), {DOCS_INDEX: "# Docs\n", - "docs/theme/README.md": "# Theme\n"}, - "docs/", "spec-governance", - (DOCS_INDEX, "docs/theme/README.md", "docs/theme/sub")), + ((THEME_SUB,), {DOCS_INDEX: "# Docs\n", THEME_README: "# Theme\n"}, + DOCS, "spec-governance", (DOCS_INDEX, THEME_README, THEME_SUB)), ], ids=["specify-vendored", "cursor", "agents", "docs-theme", "upstream", "two-in-one-entry", "src-vendored", "nested"]) def test_plan_asks_on_every_entry_that_holds_a_submodule( @@ -533,8 +545,9 @@ def test_plan_asks_on_every_entry_that_holds_a_submodule( expected = {**rows[GITMODULES], "path": entry, "files": row["files"], "bytes": row["bytes"]} expected["question"] += ( - f" This entry holds the submodule(s) {', '.join(gitlinks)}: it is " - f"asked this question, not classified by its rule `{rule}`.") + " This entry holds the submodule(s) " + + ", ".join(f"`{path}`" for path in gitlinks) + + f": it is asked this question, not classified by its rule `{rule}`.") if own: expected["question"] += ( " Split the entry if its paths do not all belong in one leg: " @@ -630,7 +643,7 @@ def test_the_split_the_question_names_is_the_edit_that_works(tmp_path): orphan=VENDORED), tmp_path)[0] named = REPLACE_WITH_RE.search(plan_rows(plan)["src/"]["question"]) own = re.findall(r"`([^`]+)`", named[1]) - assert own == ["src/app/", VENDORED] + assert own == [APP, VENDORED] beside = answered(plan, tmp_path, {"src/": "code"}) edited(beside, None, {VENDORED: "drop"}) assert finding_codes(check_plan(beside).stderr) == ["plan-covered-twice"] @@ -644,6 +657,164 @@ def test_the_split_the_question_names_is_the_edit_that_works(tmp_path): assert "plan ok" in result.stdout +def children(source: Path, directory: str) -> list[str]: + """The plan entries for what is directly in `directory` at HEAD: a + directory with its slash, anything else as it is.""" + listed = git("ls-tree", "HEAD", "--", directory, cwd=source).stdout + return [path + "/" if kind == "tree" else path + for kind, path in (line.split()[1::2] + for line in listed.splitlines())] + + +@pytest.mark.parametrize("gitlink, extra, entry, opened, where", [ + (VENDORED, {f"src/f{i:02}.py": "x\n" for i in range(12)}, "src/", + ("src/",), "`src/`, 14 in all"), + # Deeper than one level: each directory on the way is opened. + (THEME_SUB, {**{f"docs/f{i}.md": "x\n" for i in range(8)}, + THEME_README: "# Theme\n"}, DOCS, (DOCS, "docs/theme/"), + "one of `docs/`, `docs/theme/`, other than those directories, " + "10 in all"), +], ids=["one-directory", "nested"]) +def test_a_long_split_is_described_not_listed(adopter, tmp_path, gitlink, + extra, entry, opened, where): + """#166's R5-2. Past SPLIT_ENTRIES_NAMED the question does not name the + entries that replace a directory, one by one, on one line of the plan + and again in `check`'s finding: it says which directories they are the + paths of and counts them, and that edit is the one `check` passes.""" + source = registered_source(tmp_path, registered=(gitlink,), extra=extra) + plan = planned(source, tmp_path)[0] + question = plan_rows(plan)[entry]["question"] + assert question.endswith( + " Split the entry if its paths do not all belong in one leg: replace " + f"it with an entry for each path directly in {where}, each with its " + "own `leg:`. An entry added beside this one for a path it covers is " + "`plan-covered-twice`.") + assert "f00" not in question and "f0." not in question + own = [path for directory in opened for path in children(source, directory) + if path not in opened] + assert len(own) > adopter.SPLIT_ENTRIES_NAMED + split = answered(plan, tmp_path, {GITMODULES: "code"}) + edited(split, entry, {path: "code" for path in own}) + result = check_plan(split) + assert result.returncode == 0, result.stderr + + +#: A harness under `src/` that reads `contracts/`: `plan` names it in a +#: follow-up, and the root Makefile's `CONTRACTS_DIR` in another. +HARNESS = {"contracts/api.yaml": "kind: api\n", + "src/app/harness.py": "SCHEMA = 'contracts/api.yaml'\n"} + + +@pytest.mark.parametrize("gitlink, entry", [ + ("src/vendor/dep", "src/"), + ("contracts/vendor/x", "contracts/"), +]) +def test_a_submodule_takes_nothing_from_what_the_rules_imply(tmp_path, + gitlink, entry): + """#166's NEW-1. The entry holding the submodule is asked, and what the + RULES imply about the rest of the source is what it is without one: the + harness follow-up and the `CONTRACTS_DIR` line, the seeding record, and + no SEEDED warning. Read after the question, `src/: null` lost the first + and called the code leg empty, and `contracts/: null` lost both.""" + plain = make_source_repo(tmp_path / "plain", + tree={**SMALL_TREE, **HARNESS}) + without = load_yaml(planned(plain, tmp_path)[0]) + held = tmp_path / "held" + plan, written = planned(registered_source(held, registered=(gitlink,), + extra=HARNESS), held) + assert plan_rows(plan)[entry]["leg"] is None + data = load_yaml(plan) + follow_ups = data["follow_ups"] + assert follow_ups == without["follow_ups"] + assert [item.split(" ", 1)[0] for item in follow_ups + if "`contracts/`" in item or "CONTRACTS_DIR" in item] \ + == ["code/src/app/harness.py", "the"], follow_ups + assert data["seeding"] == without["seeding"] == { + "spec": {"seeded_from_template": False}, + "code": {"seeded_from_template": False}} + assert "will be SEEDED" not in written.stdout + + +def git_bytes(source: Path, *args: str, stdin: bytes) -> bytes: + """`git ` in `source`, fed and answering BYTES.""" + done = subprocess.run(["git", *args], cwd=source, input=stdin, + capture_output=True, check=False) + if done.returncode: + # Where git will not record such a name -- Windows refuses a newline + # in a path -- the case cannot be built there, and is not a failure. + pytest.skip(f"git {args[0]} refuses it here: {done.stderr!r}") + return done.stdout + + +def raw_source(base: Path, gitlink: bytes, beside: bytes | None) -> Path: + """A source with the submodule `gitlink`, registered, and a file at + `beside`: names given as BYTES and put straight into the index, as `git + add` records them, so that a name no UTF-8 file or line can hold is + built with no filesystem that has to hold it.""" + dependency = make_source_repo(base / "dependency", tree=DEPENDENCY_TREE, + edits=()) + source = make_source_repo(base / "raw-source", tree=SMALL_TREE) + head = git("rev-parse", "HEAD", cwd=dependency).stdout.strip() + index = b"160000 " + head.encode() + b"\t" + gitlink + b"\x00" + if beside: + blob = git_bytes(source, "hash-object", "-w", "--stdin", + stdin=b"x\n").strip() + index += b"100644 " + blob + b"\t" + beside + b"\x00" + git_bytes(source, "update-index", "-z", "--index-info", stdin=index) + # Quoted as git config quotes a value: a bare backslash is an escape. + quoted = gitlink.replace(b"\\", b"\\\\").replace(b'"', b'\\"') + (source / GITMODULES).write_bytes( + b'[submodule "dep"]\n\tpath = "' + quoted + b'"\n\turl = ' + + dependency.as_posix().encode() + b"\n") + git("add", "--", GITMODULES, cwd=source) + commit_as_source_human(source, REGISTER) + return source + + +@pytest.mark.parametrize("gitlink, beside, entry, named", [ + # Not UTF-8: `Source.tree` keeps the byte as a surrogate, which no UTF-8 + # plan can hold. + (b"upstream/d\xe9p", None, UPSTREAM, + " This entry holds the submodule(s) `upstream/d\\xe9p`: it is asked"), + (b"src/vendored", b"src/caf\xe9.py", "src/", + "an entry for each of `src/app/`, `src/caf\\xe9.py`, `src/vendored`,"), + # A newline would end the plan's line, and read back as a space. + (b"src/vendored", b"src/a\nb.py", "src/", + "an entry for each of `src/a\\nb.py`, `src/app/`, `src/vendored`,"), + # A backtick, and a backslash, would break the backticks around a name. + (b"src/ven`d\\or", None, "src/", + "an entry for each of `src/app/`, `src/ven\\`d\\\\or`,"), +], ids=["non-utf8-submodule", "non-utf8-file-beside", "newline-beside", + "backtick-and-backslash"]) +def test_a_name_no_plan_line_can_hold_is_spelled_in_the_question( + tmp_path, gitlink, beside, entry, named): + """#166's NEW-2 and R5-1. A name that is not UTF-8, or that holds a + newline or a backtick, is SPELLED in the question with a backslash, and + `plan` writes its plan over the one `--out` held -- where it used to raise + mid-write and leave that file empty.""" + source = raw_source(tmp_path, gitlink, beside) + plan = tmp_path / PLAN_FILE + plan.write_text("a plan answered before this run\n", encoding="utf-8") + written = write_plan(source, plan) + assert written.returncode == 0, written.stderr + assert named in plan_rows(plan)[entry]["question"] + + +def test_a_newline_spelled_in_the_question_is_the_entry_to_write(tmp_path): + """The spelling is the entry: `src/a\\nb.py` between double quotes is a + YAML scalar holding the newline, so the split the question names covers + the file, where the name read back as `src/a b.py` covered nothing.""" + source = raw_source(tmp_path, b"src/vendored", b"src/a\nb.py") + plan = planned(source, tmp_path)[0] + named = REPLACE_WITH_RE.search(plan_rows(plan)["src/"]["question"]) + own = re.findall(r"`([^`]+)`", named[1]) + assert own == ["src/a\\nb.py", APP, VENDORED] + split = answered(plan, tmp_path, {GITMODULES: "code"}) + edited(split, "src/", {f'"{path}"': "code" for path in own}) + result = check_plan(split) + assert result.returncode == 0, result.stderr + + @pytest.mark.parametrize("leg, expected", [ ("code", [UNREGISTERED]), ("spec", [UNREGISTERED]), @@ -723,9 +894,9 @@ def test_check_refuses_to_keep_a_gitlink_its_gitmodules_does_not_register( (registration(url="-oops"), [UNREGISTERED]), (registration() + "\tpath = -upstream/dependency\n", []), # A path belongs to the LAST name given it, url or none. - (registration() + '[submodule "b"]\n\tpath = upstream/dependency\n', + (registration() + SHADOWING, [UNREGISTERED]), - ('[submodule "b"]\n\tpath = upstream/dependency\n' + registration(), []), + (SHADOWING + registration(), []), # ...and a name moved elsewhere takes its old path's registration with # it, even one a later name holds (git keeps that map BY PATH). (registration() + registration("b") + MOVED_ELSEWHERE, [UNREGISTERED]), @@ -754,25 +925,123 @@ def test_a_registration_is_a_named_path_with_a_url(tmp_path, gitmodules, assert result.returncode == (1 if expected else 0) +#: The submodule most of these sources hold, as a finding names it. +NAMED_DEPENDENCY = f"{DEPENDENCY} (entry {UPSTREAM})" +#: Both of them kept in code. +BOTH_IN_CODE = {GITMODULES: "code", UPSTREAM: "code"} +#: What `why_not` says of an entry that holds the path and no url. +NO_URL_READ = "its .gitmodules entry `{}` has no url git reads" + + +@pytest.mark.parametrize("source, answers, submodule, why", [ + ({"gitmodules": '[submodule "upstream/dependency"]\n' + "\tpath = upstream/dependency\n"}, + BOTH_IN_CODE, NAMED_DEPENDENCY, NO_URL_READ.format(DEPENDENCY)), + ({"gitmodules": registration(url="-oops")}, BOTH_IN_CODE, + NAMED_DEPENDENCY, NO_URL_READ.format(DEPENDENCY)), + # Shadowed: the path belongs to the LAST name given it. + ({"gitmodules": registration() + SHADOWING}, + BOTH_IN_CODE, NAMED_DEPENDENCY, NO_URL_READ.format("b")), + ({"gitmodules": registration("../escape")}, BOTH_IN_CODE, + NAMED_DEPENDENCY, + "git ignores its .gitmodules entry `../escape` as a suspicious name"), + ({"gitmodules": registration() + MOVED_ELSEWHERE}, BOTH_IN_CODE, + NAMED_DEPENDENCY, "git dropped its path when the .gitmodules entry " + "`upstream/dependency` moved to `elsewhere`"), + ({"gitmodules": registration() + registration("b") + MOVED_ELSEWHERE}, + BOTH_IN_CODE, NAMED_DEPENDENCY, + "git dropped its path when the .gitmodules entry " + "`upstream/dependency` moved to `elsewhere`"), + ({"registered": (), "orphan": "-dep", + "gitmodules": '[submodule "d"]\n\tpath = -dep\n\turl = ../dep\n'}, + {GITMODULES: "code", '"-dep"': "code"}, "-dep", + "git ignores its .gitmodules path, which starts with `-`"), + # No entry names it: what was said before. + ({"gitmodules": '[submodule "elsewhere"]\n' + "\tpath = elsewhere/dependency\n\turl = ../dependency\n"}, + BOTH_IN_CODE, NAMED_DEPENDENCY, + "no .gitmodules entry in the source registers its path with a url"), +], ids=["no-url", "url-option-like", "shadowed-by-a-later-name", + "suspicious-name", "moved-off-its-path", "moved-under-a-later-name", + "path-option-like", "no-entry"]) +def test_an_unregistered_submodule_is_told_why_git_ignores_its_entry( + adopter, tmp_path, source, answers, submodule, why): + """#166's N2. An entry of `.gitmodules` that names the submodule's path + and that git ignores is named, with the reason, rather than "no entry" + beside an entry the person can see.""" + plan = answered(planned(registered_source(tmp_path, **source), + tmp_path)[0], tmp_path, answers) + assert findings(check_plan(plan).stderr) == [( + UNREGISTERED, f"the submodule {submodule} has leg: code but {why}, " + + adopter.NO_URL_IN_THE_LEG)] + + @pytest.mark.parametrize("modules_leg, dependency_leg, expected", [ ("code", "code", [UNREADABLE]), - # The file alone, kept, fails the leg's recursive clone on its own. ("code", "drop", [UNREADABLE]), ("drop", "spec", [UNREADABLE]), - ("drop", "drop", []), - ("root", "drop", [ROOT]), + # Beside a submodule `git filter-repo` fails on it, wherever the two go. + ("drop", "drop", [UNREADABLE]), + ("root", "drop", [ROOT, UNREADABLE]), ]) def test_a_gitmodules_git_cannot_read_is_not_one_that_registers_nothing( - plan_of, tmp_path, modules_leg, dependency_leg, expected): + adopter, plan_of, tmp_path, modules_leg, dependency_leg, expected): plan = answered(plan_of(UNREADABLE_FILE)[0], tmp_path, {GITMODULES: modules_leg, UPSTREAM: dependency_leg}) result = check_plan(plan) assert finding_codes(result.stderr) == expected, result.stderr - assert result.returncode == (1 if expected else 0) - if UNREADABLE in expected: - assert (f"FINDING {UNREADABLE}: git cannot read the source's " - ".gitmodules for its submodules (fatal: bad config line") \ - in result.stderr + assert result.returncode == 1 + assert (f"FINDING {UNREADABLE}: git cannot read the source's " + ".gitmodules for its submodules (fatal: bad config line 2 in " + f".gitmodules), {adopter.FAILS_IN_THE_EXTRACTION}\n") \ + in result.stderr + + +#: A `.gitmodules` whose one defect is a key with no value, which only git's +#: SUBMODULE reader dies on, and git runs that for a gitlink alone. +BARE_KEY = '[submodule "b"]\n\tpath\n' +#: A line no git reader can parse, and nothing else. +BAD_LINE = UNREADABLE_FILE["gitmodules"] + + +@pytest.mark.parametrize("source, answers, legs, where", [ + # S3: beside no submodule a key with no value is read by nothing, and + # main adopts and clones it. + ({"registered": (), "gitmodules": BARE_KEY}, {GITMODULES: "code"}, {}, + None), + ({"registered": (), "gitmodules": '[submodule "d"]\n\tpath = d\n\turl\n'}, + {GITMODULES: "code"}, {}, None), + # Every reader dies on a bad line: in a leg, the assembly's clone. + ({"registered": (), "gitmodules": BAD_LINE}, {GITMODULES: "code"}, {}, + "FAILS_IN_THE_CLONE"), + ({"registered": (), "gitmodules": BAD_LINE}, {GITMODULES: "drop"}, {}, + None), + # Beside a submodule, a key with no value fails `git filter-repo` too... + ({"gitmodules": registration() + BARE_KEY}, + {GITMODULES: "drop", UPSTREAM: "drop"}, {}, "FAILS_IN_THE_EXTRACTION"), + # ...unless no leg is extracted at all. + (UNREADABLE_FILE, {GITMODULES: "drop", UPSTREAM: "drop"}, + {"src/": "drop", "specs/": "drop"}, None), +], ids=["no-path-value-beside-no-submodule", + "no-url-value-beside-no-submodule", + "bad-line-beside-no-submodule-kept", + "bad-line-beside-no-submodule-dropped", + "no-value-beside-a-dropped-submodule", "no-leg-extracted"]) +def test_git_fails_on_a_gitmodules_it_cannot_read_where_it_reads_it( + adopter, plan_of, tmp_path, source, answers, legs, where): + """#166's S3, measured end to end on main and this branch alike: the + finding is made where git would fail, and says where that is -- the + extraction, for a file beside a submodule, or else the clone of a leg + that keeps it.""" + plan = answered(plan_of(source)[0], tmp_path, answers) + for path, leg in legs.items(): + edited(plan, path, {path: leg}) + found = findings(check_plan(plan).stderr) + if where is None: + assert found == [], found + return + assert [code for code, _ in found] == [UNREADABLE], found + assert found[0][1].endswith(getattr(adopter, where)), found @pytest.mark.parametrize("where, expected", [ @@ -800,7 +1069,8 @@ def test_git_follows_no_include_in_a_gitmodules(tmp_path, where, expected): @pytest.mark.parametrize("source, said", [ - (UNREADABLE_FILE, "fatal: bad config line"), + # git calls the file it read from `--file -` "standard input". + (UNREADABLE_FILE, "fatal: bad config line 2 in .gitmodules)"), # `git config --list` lists both; git's submodule reader dies on each. ({"gitmodules": registration() + '[submodule "b"]\n\tpath\n'}, "missing value for 'submodule.b.path'"), @@ -917,16 +1187,21 @@ def adopter(): def refused_by_execute(adopter, plan_path: Path, tmp_path: Path): - """What `_refuse_an_unrunnable_plan` raises for `plan_path`, or None.""" + """What `_refuse_what_check_finds` raises for `plan_path`, or None.""" plan = adopter.Plan.load(plan_path) source = plan.open_source(None, tmp_path / "work") try: - adopter._refuse_an_unrunnable_plan(plan, source) + adopter._refuse_what_check_finds(plan, source) except adopter.Refusal as refusal: return refusal return None +def as_refused(found: list[tuple[str, str]]) -> list[str]: + """`check`'s findings as `execute`'s refusal lists them, one a line.""" + return [f" FINDING {code}: {detail}" for code, detail in found] + + @pytest.mark.parametrize("source, answers, code", [ (REGISTERED, {GITMODULES: "code", UPSTREAM: "spec"}, SPLIT), (REGISTERED, {GITMODULES: "drop", UPSTREAM: "code"}, SPLIT), @@ -935,6 +1210,7 @@ def refused_by_execute(adopter, plan_path: Path, tmp_path: Path): UNREGISTERED), (LONE_ORPHAN, {UPSTREAM: "code"}, UNREGISTERED), (UNREADABLE_FILE, {GITMODULES: "code", UPSTREAM: "code"}, UNREADABLE), + (UNREADABLE_FILE, {GITMODULES: "drop", UPSTREAM: "drop"}, UNREADABLE), (REGISTERED, {GITMODULES: "spec", UPSTREAM: "spec"}, None), (REGISTERED, {GITMODULES: "code", UPSTREAM: "drop"}, None), (BESIDE, {GITMODULES: "code", UPSTREAM: "code", EXTRA: "drop"}, None), @@ -942,7 +1218,7 @@ def refused_by_execute(adopter, plan_path: Path, tmp_path: Path): ]) def test_execute_refuses_what_check_finds(adopter, plan_of, tmp_path, source, answers, code): - """The refusal `execute` makes, asked of `_refuse_an_unrunnable_plan` + """The refusal `execute` makes, asked of `_refuse_what_check_finds` directly; that `cmd_execute` makes it before any leg exists is pinned by the end-to-end refusal below, which needs `git filter-repo`.""" plan = answered(plan_of(source)[0], tmp_path, answers) @@ -951,11 +1227,11 @@ def test_execute_refuses_what_check_finds(adopter, plan_of, tmp_path, if code is None: assert found == [] and refusal is None, (found, refusal) return - # ONE definition: the refusal is what `check` found, in its words. + # ONE list: the refusal is what `check` found, in its words. assert refusal is not None, found assert refusal.code == found[0][0] == code - assert refusal.detail == "; ".join(detail for _, detail in found) - assert refusal.remediation == adopter.SUBMODULE_REMEDIATION + assert refusal.detail.splitlines()[1:] == as_refused(found) + assert refusal.remediation.endswith("\n" + adopter.SUBMODULE_REMEDIATION) # No stricter than the rule: one submodule may be dropped by itself. assert ("Any one submodule may also be dropped on its own: its " "registration stays behind") in refusal.remediation @@ -978,7 +1254,8 @@ def test_a_mixed_refusal_is_named_by_the_problem_to_fix_first( """`.gitmodules` and `extra/orphan` in code, `d1/` to `d9/` split to spec, and `d0/` in the root or not: eleven problems or ten, listed root, then unregistered, then split. The refusal takes the FIRST one's code - and names eight of them, then counts the rest.""" + and names as many as `execute` names of any findings, then counts the + rest.""" answers = {GITMODULES: "code", EXTRA: "code", "d0/": first_leg, **{f"d{i}/": "spec" for i in range(1, 10)}} plan = answered(plan_of(CROWDED)[0], tmp_path, answers) @@ -987,40 +1264,85 @@ def test_a_mixed_refusal_is_named_by_the_problem_to_fix_first( [ROOT] * (first_leg == "root") + [UNREGISTERED] + [SPLIT] * 9) refusal = refused_by_execute(adopter, plan, tmp_path) assert refusal.code == code - assert refusal.detail.split("; ") == ( - [detail for _, detail in found[:8]] + [f"and {len(found) - 8} more"]) + shown = adopter.REFUSED_FINDINGS_SHOWN + assert refusal.detail.splitlines()[1:] == ( + as_refused(found[:shown]) + [f" and {len(found) - shown} more"]) + + +def test_execute_refuses_every_finding_check_prints_at_once( + adopter, plan_of, tmp_path, monkeypatch): + """#166's NEW-3. `check` lists an uncovered path and a split submodule; + `execute` used to refuse the split alone, in a function of its own, and + the uncovered path only once that was fixed. Through `cmd_execute`, as a + person meets it: one refusal naming both, before any leg exists, with + #168's remedy and the submodule one.""" + plan = answered(plan_of(REGISTERED)[0], tmp_path, + {GITMODULES: "code", UPSTREAM: "spec"}) + edited(plan, README, {}) + found = findings(check_plan(plan).stderr) + assert [code for code, _ in found] == [UNCOVERED, SPLIT], found + # Every refusal here comes before the tool would be used. + monkeypatch.setattr(adopter, "_require_filter_repo", lambda: None) + remotes = tmp_path / REMOTES + with pytest.raises(adopter.Refusal) as caught: + adopter.cmd_execute(Namespace( + plan=str(plan), source=None, local_remote_dir=remotes, + allow_empty_leg=[], yes=True, work_dir=tmp_path / "work")) + refusal = caught.value + assert refusal.code == UNCOVERED + assert refusal.detail.splitlines()[1:] == as_refused(found) + assert "no leg repository was created" in refusal.remediation + assert refusal.remediation.endswith("\n" + adopter.SUBMODULE_REMEDIATION) + assert not remotes.exists() def execute_in(plan: Path, base: Path): """`execute --yes` of `plan`, its leg remotes under `base/remotes` and its work in `base/work`: `(result, remotes, work)`.""" - remotes, work = base / "remotes", base / "work" + remotes, work = base / REMOTES, base / "work" return adopt("execute", plan, "--yes", "--local-remote-dir", str(remotes), "--work-dir", str(work)), remotes, work +def clone_the_split(source: Path, tmp_path: Path): + """`git clone --recurse-submodules` of the branch `execute` pushed the + split to, into `tmp_path/clone`: what a person runs next.""" + return git(*FILE_PROTOCOL, "clone", "-q", "--recurse-submodules", "-b", + "adopt/three-repo-shape", str(source), str(tmp_path / "clone"), + cwd=tmp_path, check=False) + + def source_state(source: Path) -> list[str]: """HEAD, every ref and `git status`: what a refused `execute` must not - move.""" + move. Submodules are ignored by the status, which otherwise reads + `.gitmodules` for them and dies on one it cannot read.""" return [git(*args, cwd=source).stdout for args in ( - ("rev-parse", "HEAD"), ("for-each-ref",), ("status", "--porcelain"))] + ("rev-parse", "HEAD"), ("for-each-ref",), + ("status", "--porcelain", "--ignore-submodules=all"))] @needs_filter_repo -@pytest.mark.parametrize("source, answers, refused", [ - (REGISTERED, {GITMODULES: "code", UPSTREAM: "spec"}, - f"REFUSED {SPLIT}: the submodule upstream/dependency (entry upstream/) " - "has leg: spec but the source's .gitmodules has leg: code"), - (LONE_ORPHAN, {UPSTREAM: "code"}, - f"REFUSED {UNREGISTERED}: the submodule upstream/orphan (entry " - "upstream/) has leg: code but no .gitmodules entry in the source " - "registers its path with a url"), -], ids=["split", "unregistered-orphan"]) +@pytest.mark.parametrize("source, answers, code, refused", [ + (REGISTERED, {GITMODULES: "code", UPSTREAM: "spec"}, SPLIT, + "the submodule upstream/dependency (entry upstream/) has leg: spec but " + "the source's .gitmodules has leg: code"), + (LONE_ORPHAN, {UPSTREAM: "code"}, UNREGISTERED, + "the submodule upstream/orphan (entry upstream/) has leg: code but no " + ".gitmodules entry in the source registers its path with a url"), + # Both dropped, and `git filter-repo` still fails on the file: on main + # that was after both leg remotes existed (#166's S3). + (UNREADABLE_FILE, {GITMODULES: "drop", UPSTREAM: "drop"}, UNREADABLE, + "git cannot read the source's .gitmodules for its submodules (fatal: " + "bad config line 2 in .gitmodules), and the source holds a submodule, " + "so `git filter-repo` fails on that file when `execute` extracts a " + "leg"), +], ids=["split", "unregistered-orphan", "unreadable-beside-a-dropped-one"]) def test_execute_refuses_a_submodule_plan_before_anything_exists( - tmp_path, source, answers, refused): - """Before #166 both plans were extracted, mounted, pushed and verified, - and each assembly's recursive clone then exited 128 -- the orphan's - although the source's own recursive clone exits 0. Refused now, and + tmp_path, source, answers, code, refused): + """Before #166 each plan was extracted, mounted, pushed and verified, + and the assembly's recursive clone then exited 128 -- the orphan's + although the source's own recursive clone exits 0 -- or, for the file + git cannot read, `execute` died extracting a leg. Refused now, and refused before `_create_leg_remotes`: a re-run must not meet two legs that a refused plan already made.""" thing = registered_source(tmp_path, **source) @@ -1028,7 +1350,8 @@ def test_execute_refuses_a_submodule_plan_before_anything_exists( plan = answered(planned(thing, tmp_path)[0], tmp_path, answers) result, remotes, work = execute_in(plan, tmp_path) assert result.returncode == 2, result.stderr + result.stdout - assert refused in result.stderr + assert f"REFUSED {code}: 1 finding(s) in " in result.stderr + assert f"\n FINDING {code}: {refused}" in result.stderr assert "creating the leg repositories" not in result.stdout # Nothing was made but the empty work directory `--work-dir` names. assert not remotes.exists(), "a refused plan made the remotes directory" @@ -1066,9 +1389,7 @@ def test_an_accepted_submodule_plan_adopts_clones_and_bootstraps( assert "adoption verified" in result.stdout clone = tmp_path / "clone" - cloned = git(*FILE_PROTOCOL, "clone", "-q", "--recurse-submodules", "-b", - "adopt/three-repo-shape", str(thing), str(clone), - cwd=tmp_path, check=False) + cloned = clone_the_split(thing, tmp_path) assert cloned.returncode == 0, cloned.stderr bootstrapped = run_script(clone / "scripts/bootstrap.py", cwd=clone) assert bootstrapped.returncode == 0, (bootstrapped.stderr @@ -1088,3 +1409,23 @@ def test_an_accepted_submodule_plan_adopts_clones_and_bootstraps( # Cloned into the leg, at the commit the source pinned. assert {name: (leg / DEPENDENCY / name).read_text(encoding="utf-8") for name in DEPENDENCY_TREE} == DEPENDENCY_TREE + + +@needs_filter_repo +def test_a_key_with_no_value_beside_no_submodule_adopts_and_clones(tmp_path): + """#166's S3, end to end: a `.gitmodules` whose one defect is a key with + no value, in a source with no submodule, kept in a leg. git runs its + submodule reader for a gitlink alone, so nothing reads the file that way: + main adopted and cloned it, and `check` no longer refuses it.""" + thing = registered_source(tmp_path, registered=(), gitmodules=BARE_KEY) + plan = answered(planned(thing, tmp_path)[0], tmp_path, + {GITMODULES: "code"}) + checked = check_plan(plan) + assert checked.returncode == 0, checked.stderr + checked.stdout + result = execute_in(plan, tmp_path)[0] + assert result.returncode == 0, result.stderr + result.stdout + assert "adoption verified" in result.stdout + clone = tmp_path / "clone" + cloned = clone_the_split(thing, tmp_path) + assert cloned.returncode == 0, cloned.stderr + assert (clone / "code" / GITMODULES).read_bytes() == BARE_KEY.encode() From 6aaedc5b92d81950c9e5cad3fa00418e6ae56d2d Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 09:14:09 +0000 Subject: [PATCH 06/12] tests: the unreadable-`.gitmodules` finding names where git fails on it, so a `.gitmodules` that is itself a gitlink is pinned to the clone of its leg (#166) `_where_git_fails` names the extraction only for a FILE git cannot read beside a submodule: a `.gitmodules` that is itself a gitlink leaves `git fast-export` nothing to read, and git fails in the clone of the leg that keeps it (measured on main: execute 0, recursive clone 128). No test held that condition; `test_a_gitmodules_git_cannot_read_says_what_git_said` now asserts the ending of each of its four findings. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_submodules.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index f8cd8e07..d6a389d0 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -1068,22 +1068,26 @@ def test_git_follows_no_include_in_a_gitmodules(tmp_path, where, expected): assert finding_codes(result.stderr) == expected, result.stderr -@pytest.mark.parametrize("source, said", [ +@pytest.mark.parametrize("source, said, where", [ # git calls the file it read from `--file -` "standard input". - (UNREADABLE_FILE, "fatal: bad config line 2 in .gitmodules)"), + (UNREADABLE_FILE, "fatal: bad config line 2 in .gitmodules)", + "FAILS_IN_THE_EXTRACTION"), # `git config --list` lists both; git's submodule reader dies on each. - ({"gitmodules": registration() + '[submodule "b"]\n\tpath\n'}, - "missing value for 'submodule.b.path'"), + ({"gitmodules": registration() + BARE_KEY}, + "missing value for 'submodule.b.path'", "FAILS_IN_THE_EXTRACTION"), ({"gitmodules": '[submodule "d"]\n\tpath = upstream/dependency\n\turl\n'}, - "missing value for 'submodule.d.url'"), - # A `.gitmodules` that is itself a gitlink has no blob to read. - ({"registered": (), "orphan": GITMODULES}, "fatal: git cat-file"), + "missing value for 'submodule.d.url'", "FAILS_IN_THE_EXTRACTION"), + # A `.gitmodules` that is itself a gitlink has no blob to read, and no + # file for `git fast-export` to read either: the clone of its leg is + # where git fails on it. + ({"registered": (), "orphan": GITMODULES}, "fatal: git cat-file", + "FAILS_IN_THE_CLONE"), ], ids=["bad-config-line", "path-with-no-value", "url-with-no-value", "gitmodules-is-a-gitlink"]) -def test_a_gitmodules_git_cannot_read_says_what_git_said(plan_of, tmp_path, - source, said): +def test_a_gitmodules_git_cannot_read_says_what_git_said( + adopter, plan_of, tmp_path, source, said, where): """The finding names the file AND what git says about it, on the one - line `check` prints a finding on.""" + line `check` prints a finding on, and where git fails on it.""" plan = plan_of(source)[0] plan = answered(plan, tmp_path, {path: "code" for path in (GITMODULES, UPSTREAM) @@ -1093,6 +1097,7 @@ def test_a_gitmodules_git_cannot_read_says_what_git_said(plan_of, tmp_path, assert found[0][1].startswith( "git cannot read the source's .gitmodules for its submodules (" + said), found + assert found[0][1].endswith(getattr(adopter, where)), found @pytest.mark.parametrize("modules_leg, expected", [ From 6fcf6455e654c1aab6c7db3fe9897068aebe49c7 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:54:41 +0000 Subject: [PATCH 07/12] tests: `resolve` writes the plan back LF and `raw_source` skips a source git will not record a name in, so the 13 tests of test_adopt_submodules.py that failed only on `tests-windows` no longer fail there (#166) PR #194 was green on Linux and macOS and red on `tests-windows`: 13 tests, two causes, both in the test helpers. adopt-project.py is unchanged: the plan it writes is LF on every platform (`write_lf`). 1. `edited()` found no `paths:` and no ` - path: ` line in the plan (10 tests). `conftest.resolve` wrote the answered plan back with `Path.write_text`, which on Windows turns every LF into CRLF, and `edited` reads the plan as BYTES, for the LF `plan` writes. It now writes with `open(..., newline="\n")`: `write_text` has no `newline` before 3.10 and this standard runs on 3.9. 2. `raw_source` builds a source holding a name no filesystem need hold with `git update-index -z --index-info`, which exits 0 and says `Ignoring path` for a name git will not record, so the Windows runner committed a source with no `src/a\nb.py` and no `src/ven`d\or`, and the question never named them (3 tests: KeyError 'question', and a list without the file). The helper now reads the committed tree back with `git ls-tree -r -z` and skips the case when a name it was given is not there: asked of git, not of the operating system. Where git keeps the name the case runs, with its assertion unchanged. Two tests keep both from coming back on a platform that does not show them. `windows_text_mode` opens text files for writing as Windows does (CRLF for LF unless a `newline=` is named), around the one `answered()` it is for: with `resolve` reverted the plan holds CRLF and `edited` raises the ValueError of the CI log. A `.git/...` name, which every git leaves out of the tree, makes `raw_source` skip. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 17 +++++-- tests/test_adopt_submodules.py | 88 +++++++++++++++++++++++++++++++--- 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ea70077c..4e136610 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -365,11 +365,18 @@ def write_plan(source: Path, out: Path, project: str = "Northwind", def resolve(plan_path: Path, path: str, leg: str, why: str = "answered by the test") -> None: - """Answer one `review_required` entry the way a human or an AI would.""" + """Answer one `review_required` entry the way a human or an AI would. + + The plan goes back LF, as `plan` wrote it: `Path.write_text` translates + every `\\n` to CRLF on Windows, and a helper that reads the plan as bytes + afterwards (`edited` in `test_adopt_submodules.py`) then finds none of its + lines. `open` with `newline`, because `write_text` has no such parameter + before 3.10 and this standard runs on 3.9. + """ text = plan_path.read_text(encoding="utf-8") needle = f" - path: {path}\n leg: null\n" assert needle in text, f"{path} is not an unresolved entry in the plan" - plan_path.write_text( - text.replace(needle, f" - path: {path}\n leg: {leg}\n" - f" resolution: \"{why}\"\n", 1), - encoding="utf-8") + with plan_path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(text.replace( + needle, f" - path: {path}\n leg: {leg}\n" + f" resolution: \"{why}\"\n", 1)) diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index d6a389d0..c71f1164 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -6,7 +6,10 @@ from __future__ import annotations +import builtins +import contextlib import importlib.util +import io import re import shutil import subprocess @@ -453,7 +456,8 @@ def edited(plan: Path, removed: str | None, added: dict) -> None: """`plan` with the entry for `removed` deleted, and an entry `path: leg` for each of `added` put first, as a hand edit would leave it -- a careless one, or one that splits an entry. Bytes, because `plan` writes LF on - every platform.""" + every platform, and `conftest.resolve` keeps it so: an entry is found + by its LF-ended lines.""" lines = plan.read_bytes().splitlines(keepends=True) if removed: start = lines.index(f" - path: {removed}\n".encode()) @@ -478,6 +482,46 @@ def finding_codes(stderr: str) -> list[str]: return [code for code, _ in findings(stderr)] +@contextlib.contextmanager +def windows_text_mode(): + """Text mode as Windows has it: a file opened for writing as text, with + no `newline=` named, gets CRLF for every `\\n` it is given. Only the + Windows job ever saw that, so it is put here where every platform does. + A context and not a fixture, so that it covers the one call it is for: + `plan_of` caches what it builds for the whole module.""" + real_open = io.open + + def open_as_windows(file, mode="r", buffering=-1, encoding=None, + errors=None, newline=None, *rest, **named): + if newline is None and "b" not in mode and set(mode) & set("wax+"): + newline = "\r\n" + return real_open(file, mode, buffering, encoding, errors, newline, + *rest, **named) + + # `Path.write_text` opens through `io.open`; a bare `open` is `builtins`'. + with pytest.MonkeyPatch.context() as patch: + patch.setattr(io, "open", open_as_windows) + patch.setattr(builtins, "open", open_as_windows) + yield + + +def test_a_plan_answered_where_text_mode_writes_crlf_is_still_edited( + plan_of, tmp_path): + """PR #194's `tests-windows`, ten failures: `conftest.resolve` wrote the + answered plan with `Path.write_text`, which on Windows turns every LF + into CRLF, and `edited` -- which reads the LF `plan` writes on every + platform -- found neither `paths:` nor ` - path: ` in it.""" + asked = plan_of(REGISTERED)[0] + with windows_text_mode(): + plan = answered(asked, tmp_path, + {GITMODULES: "code", UPSTREAM: "code"}) + assert b"\r" not in plan.read_bytes() + edited(plan, UPSTREAM, {DEPENDENCY: "spec"}) + rows = plan_rows(plan) + assert UPSTREAM not in rows + assert rows[DEPENDENCY]["leg"] == "spec" + + def test_plan_asks_the_submodule_question_on_gitmodules(plan_of): plan, written = plan_of(REGISTERED) row = plan_rows(plan)[GITMODULES] @@ -740,26 +784,35 @@ def git_bytes(source: Path, *args: str, stdin: bytes) -> bytes: done = subprocess.run(["git", *args], cwd=source, input=stdin, capture_output=True, check=False) if done.returncode: - # Where git will not record such a name -- Windows refuses a newline - # in a path -- the case cannot be built there, and is not a failure. + # Where git fails outright on a name it will not record, the case + # cannot be built there, and is not a failure. `update-index` does + # not fail on one: it ignores it and exits 0, which `raw_source` + # finds by reading the tree back. pytest.skip(f"git {args[0]} refuses it here: {done.stderr!r}") return done.stdout +#: Ends a name in `-z` output, and in what `update-index -z` is given. +NUL = b"\x00" + + def raw_source(base: Path, gitlink: bytes, beside: bytes | None) -> Path: """A source with the submodule `gitlink`, registered, and a file at `beside`: names given as BYTES and put straight into the index, as `git add` records them, so that a name no UTF-8 file or line can hold is - built with no filesystem that has to hold it.""" + built with no filesystem that has to hold it. Skipped where git does not + keep a name in the tree, which only git can say: `update-index` ignores a + path it will not record, and exits 0 (on the Windows runner it left out a + backslash and a newline).""" dependency = make_source_repo(base / "dependency", tree=DEPENDENCY_TREE, edits=()) source = make_source_repo(base / "raw-source", tree=SMALL_TREE) head = git("rev-parse", "HEAD", cwd=dependency).stdout.strip() - index = b"160000 " + head.encode() + b"\t" + gitlink + b"\x00" + index = b"160000 " + head.encode() + b"\t" + gitlink + NUL if beside: blob = git_bytes(source, "hash-object", "-w", "--stdin", stdin=b"x\n").strip() - index += b"100644 " + blob + b"\t" + beside + b"\x00" + index += b"100644 " + blob + b"\t" + beside + NUL git_bytes(source, "update-index", "-z", "--index-info", stdin=index) # Quoted as git config quotes a value: a bare backslash is an escape. quoted = gitlink.replace(b"\\", b"\\\\").replace(b'"', b'\\"') @@ -768,6 +821,12 @@ def raw_source(base: Path, gitlink: bytes, beside: bytes | None) -> Path: + dependency.as_posix().encode() + b"\n") git("add", "--", GITMODULES, cwd=source) commit_as_source_human(source, REGISTER) + held = subprocess.run( + ["git", "ls-tree", "-r", "-z", "--name-only", "HEAD"], cwd=source, + capture_output=True, check=True).stdout.split(NUL) + missing = [name for name in (gitlink, beside) if name and name not in held] + if missing: + pytest.skip(f"git does not keep {missing!r} in a tree here") return source @@ -815,6 +874,23 @@ def test_a_newline_spelled_in_the_question_is_the_entry_to_write(tmp_path): assert result.returncode == 0, result.stderr +@pytest.mark.parametrize("gitlink, beside", [ + (b".git/vendored", None), + (b"src/vendored", b".git/x"), +], ids=["submodule", "file-beside"]) +def test_a_source_whose_name_git_will_not_record_is_skipped_not_failed( + tmp_path, gitlink, beside): + """PR #194's `tests-windows`, the three name failures: the runner's git + left a backslash and a newline out of the source's tree, and the question + never named them. `git update-index --index-info` exits 0 for a path it + will not record and says `Ignoring path`; every platform does it for a + `.git` component, which is how Linux runs this. Asked of the tree the + tool reads, `raw_source` skips that case where it used to fail.""" + with pytest.raises(pytest.skip.Exception, + match=r"git does not keep .*\.git/"): + raw_source(tmp_path, gitlink, beside) + + @pytest.mark.parametrize("leg, expected", [ ("code", [UNREGISTERED]), ("spec", [UNREGISTERED]), From 3eb55e598846e81f3cb4aa4222c1d529475b3c27 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:33:35 +0000 Subject: [PATCH 08/12] adopt-project.py: a source submodule's relative url kept in a leg must name the same repository from the leg as from the source, or `check` finds it and `execute` refuses it before any leg exists, so `url = ../dependency` no longer adopts as `adoption verified` into an assembly whose recursive clone cannot fetch it (#190) git resolves a `./` or `../` submodule url against the remote of the repository holding `.gitmodules`: the source's before the split, the leg's after it. On main and on #166's head the re-verifier's `relative-url` case checked 0, executed 0 with `adoption verified`, and the assembly's `git clone --recurse-submodules` exited 1 on `.../remotes/dependency' does not exist`. Each relative url the plan keeps in a leg is now resolved by git itself (`git submodule init` in a scratch repository whose `origin` is the remote in question), from the source's default remote (the adopted branch's, else `origin`, else its own directory) and from the leg's remote, and the two are compared with `remote_key`, and for two paths on this disk through its symlinks. Where they name one repository nothing changes; `check` says in one NOTE that it resolved from the GitHub remotes the plan names. Where they differ, or git cannot resolve one, `check` reports `plan-submodule-relative-url` naming the submodule, its url and both resolutions, with a remediation that works (commit the absolute url in the source and re-plan, or drop the submodule), and `execute` refuses it through #168's one list -- resolved again from the remotes it is about to create, so under `--local-remote-dir` its refusal says so. Refused, never rewritten: a rewritten leg `.gitmodules` is a blob `_verify` accounts for by sha and a url nobody committed, so `_verify` is unchanged. A credential in the source's remote is redacted wherever it is printed. A source with no submodule, or none kept with a relative url, starts no git process for this and prints exactly what the base printed. `tests/test_adopt_submodules.py`'s hand-written registrations take the dependency's absolute url: seven cases kept `../dependency` in code with no remote and expected no finding, which is #190's broken outcome. Stacked on #166 (fix/adopt-split-submodule-plan). Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 344 +++++++++-- tests/test_adopt_relative_submodule_url.py | 635 +++++++++++++++++++++ tests/test_adopt_submodules.py | 11 +- 3 files changed, 951 insertions(+), 39 deletions(-) create mode 100644 tests/test_adopt_relative_submodule_url.py diff --git a/adopt-project.py b/adopt-project.py index 825779a3..c5865929 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -29,8 +29,9 @@ Windows keeps for itself such as `.git` or `CON`, or a value `execute` would refuse; the two never equal or nested, in any case), and the source's own submodules kept on one leg with - their `.gitmodules`, registered there and never in the root. It - prints what will happen and changes nothing. + their `.gitmodules`, registered there and never in the root, each + relative url naming the same repository from the leg as from the + source. It prints what will happen and changes nothing. execute creates the two legs, extracts them with `git filter-repo`, makes the one split commit on a branch of the source, sets the `xf-project-` topic on all three (skipped for local remotes), @@ -42,7 +43,9 @@ right about `openspec/` and cannot be right about `examples/golden-run/` without knowing whether the specification cites it. `execute` REFUSES while any `leg:` is still null, and on every other finding `check` reports, before -it has created a leg repository or pushed anything. +it has created a leg repository or pushed anything -- a source submodule's +relative url resolved again from the leg remotes it is about to create, which +under `--local-remote-dir` are not the GitHub ones `check` resolves from. WHY `git filter-repo` AND NOT A VENDORED COPY. Extracting history correctly is a solved problem with one correct implementation, and a vendored copy of it @@ -75,6 +78,7 @@ import argparse import datetime as _dt +import os import re import shutil import subprocess @@ -89,7 +93,8 @@ free_plan_secret_hint, COMMIT_RE, NEUTRAL_PRODUCT_OWNER, PROJECT_ID_RE, TREE_DIGEST_DEFINITION, SAFE_ARG_RE, VISIBILITY_CHOICES, NamingPolicy, Refusal, YamlError, - accepts_role, checked_value, git_out, load_yaml, parse_yaml, tree_digest, + accepts_role, checked_value, git_out, load_yaml, parse_yaml, redacted, + remote_key, remote_local_path, tree_digest, ) from shape_materialize import ( # noqa: E402 ADOPT_MAKEFILE_BLOCK, RULESET_HINT, SHAPE_REPOSITORY, @@ -298,6 +303,12 @@ def __contains__(self, path: str) -> bool: name = self._name_at.get(path) return name is not None and bool(self._url_of.get(name)) + def url(self, path: str) -> str: + """The url git reads for the submodule registered at `path`; empty + when it registers none there.""" + name = self._name_at.get(path) + return "" if name is None else self._url_of.get(name, "") + def why_not(self, path: str) -> str | None: """Why git registers nothing at `path` though an entry names it; None when no entry of the file does.""" @@ -1825,24 +1836,252 @@ def _kept_submodule_problems(kept: list, registered: Registrations, return problems -def _submodule_findings(plan: Plan, source: Source, - tree: list) -> tuple[list[str], list[str]]: +# --------------------------------------------------------------------------- +# A kept submodule's RELATIVE url, resolved from the leg that holds it (#190) +# --------------------------------------------------------------------------- +# +# git resolves a submodule url that begins `./` or `../` against the remote of +# the repository whose `.gitmodules` holds it: that repository's default +# remote, or, with none, its own directory (`git help submodule`). Before the +# split that is the SOURCE; after it, the file is in a LEG, so it is the leg's +# remote. `url = ../dependency` names a sibling of the source before and a +# sibling of the leg after, and on main the run said `adoption verified` and +# the assembly's `git clone --recurse-submodules` then failed wherever the +# dependency did not happen to sit beside the leg: under `--local-remote-dir`, +# or with an `--org` other than the source's. +# +# REFUSED, NEVER REWRITTEN. Writing the absolute url into the leg's +# `.gitmodules` would change a blob the verification accounts for by sha, in +# every commit of the leg's history that holds one, and put in the leg a url +# nobody committed. So a url whose two resolutions differ is a finding of +# `check` and a refusal of `execute` before any leg exists, and the person +# commits the absolute url in the source or drops the submodule. Where both +# name the same repository -- legs made in the source's own GitHub +# organisation, or on disk beside the dependency -- nothing changes, and the +# verification has nothing new to account for. +# +# ASKED OF GIT. Each resolution is `git submodule init`'s, run in a scratch +# repository whose `origin` is the remote in question: the code the +# assembly's recursive clone runs, not a copy of it. `repo_shape.join_remote` +# is a copy, and git's arithmetic differs from it where it matters here: git +# takes `..\` for a relative url on every platform and strips it on Windows +# alone, and it fails on a `..` too many where the copy consumes nothing. A +# url that is not relative is never resolved, because git copies it as it +# stands, so a source with no submodule, or none with a relative url, starts +# no git process here. + +RELATIVE_URL = "plan-submodule-relative-url" +#: git's own test for a url it resolves against a remote +#: (`starts_with_dot_slash`, `starts_with_dot_dot_slash`): `./` or `../`, +#: with either separator on every platform. +RELATIVE_URL_RE = re.compile(r"\.\.?[/\\]") +#: What a plan that keeps a relative url git resolves elsewhere from its leg +#: is told: by `check`, once after its findings, and by `execute`'s refusal. +RELATIVE_URL_REMEDIATION = ( + "Remediation for a relative submodule url: git resolves it against the " + "remote of the repository whose `.gitmodules` holds it -- in the " + "assembly, the leg -- so it must name the same repository from the leg " + "as from the source. Commit the submodule's absolute url in the source's " + "`.gitmodules` and re-run `plan`, or `drop` the submodule: its " + "registration then stays behind in the leg, inert. `check` resolves it " + "from the GitHub remotes the plan's `org:` names and `execute` from the " + "remotes it is about to create, before it creates any; under " + "`--local-remote-dir` those are bare repositories on disk, where " + "`check`'s `plan ok` settles nothing about it.") + + +class _Resolution: + """One relative url as git resolved it from one repository: `url`, or, + when git could not, `said`, which is what git said.""" + + def __init__(self, url: str | None, said: str = "") -> None: + self.url = url + self.said = said + + def same_as(self, other: "_Resolution") -> bool: + """Both resolved, to one repository however each is spelled. + + `remote_key` is the standard's one rule for when two spellings are + one repository: the source's `origin` may be + `git@github.com:Org/Thing.git` and a leg's is + `https://github.com/org/...`. It compares two filesystem paths by + their text alone, so they are then asked of this machine, whose + symlinks may make two spellings one directory -- the order + `same_repository_here` asks them in, so the disk can add an answer + and never take one away. + """ + if self.url is None or other.url is None: + return False + if remote_key(self.url) == remote_key(other.url): + return True + paths = [remote_local_path(url) for url in (self.url, other.url)] + return None not in paths \ + and os.path.realpath(paths[0]) == os.path.realpath(paths[1]) + + def __str__(self) -> str: + # `redacted`: a source's remote may carry a token, and so does every + # url resolved against it. + return (f"to {_spelled(redacted(self.url))}" if self.url is not None + else f"not at all ({self.said})") + + +def _kept_relative_urls(entries: list, tree: list, + registered: Registrations | str + ) -> tuple[str | None, list[tuple[str, str, str]]]: + """`(leg, [(path, entry, url), ...])`: each submodule the plan keeps in + the leg its `.gitmodules` goes to, registered with a relative url, and + that leg. A submodule kept anywhere else, or not registered, is already + a finding of `submodule_plan_problems`; a dropped one is inert; and a + `.gitmodules` git cannot read has no url to read.""" + gitlinks = [path for path, mode, _, _ in tree if mode == GITLINK_MODE] + if not gitlinks or isinstance(registered, str): + return None, [] + answered = _answered_legs(entries, [GITMODULES, *gitlinks]) + leg = answered.get(GITMODULES, (None, None))[0] + return leg, [(path, entry, registered.url(path)) for path, kept, entry + in _kept_in_a_leg(answered, gitlinks) + if kept == leg and path in registered + and RELATIVE_URL_RE.match(registered.url(path))] + + +def _default_remote(source: Source) -> tuple[str, str]: + """`(base, which)`: what git resolves the source's relative urls against, + and how a finding names it. git's rule -- the checked-out branch's + remote, else `origin`; that remote's url, else, with none, the + repository's own directory -- asked of the branch being ADOPTED, which a + clone of the source checks out, and not of whatever branch this working + copy happens to have checked out.""" + name = _config_value(source.path, f"branch.{source.branch}.remote") \ + or "origin" + url = _config_value(source.path, f"remote.{name}.url") + if url is not None: + return url, f"its remote {_spelled(name)}" + return (source.path.as_posix(), + f"its own directory, as it has no remote {_spelled(name)}") + + +def _config_value(repository: Path, key: str) -> str | None: + """`key` in `repository`'s git config, as git reads it -- the last + value, when there are several; None when unset.""" + got = subprocess.run(["git", GIT_CONFIG, "-z", "--get", key], + cwd=repository, capture_output=True, check=False) + return None if got.returncode else _git_path(got.stdout.rstrip(b"\x00")) + + +def _resolved(base: str, urls: list[str]) -> list[_Resolution]: + """Each of `urls` as `git submodule init` resolves it in a repository + whose `origin` is `base`, in a scratch repository made for the purpose + and removed after it. A url git cannot resolve is what git said: that is + where the clone fails.""" + scratch = Path(tempfile.mkdtemp(prefix="openreposhape-url-")) + try: + run(["git", "init", "-q", str(scratch)]) + run(["git", GIT_CONFIG, "remote.origin.url", base], cwd=scratch) + for index, url in enumerate(urls): + for key, value in (("path", f"u{index}"), ("url", url)): + run(["git", GIT_CONFIG, "-f", GITMODULES, + f"submodule.u{index}.{key}", value], cwd=scratch) + # A gitlink names a commit nobody fetches here; any object id of the + # repository's own format will do, and this one is at hand. + oid = run(["git", "hash-object", GITMODULES], cwd=scratch) + for index in range(len(urls)): + run(["git", "update-index", "--add", "--cacheinfo", + f"{GITLINK_MODE},{oid},u{index}"], cwd=scratch) + # One `init` per url, because git stops at the first it cannot + # resolve and each finding names its own. + return [_resolution_of(scratch, f"u{index}") + for index in range(len(urls))] + finally: + shutil.rmtree(scratch, ignore_errors=True) + + +def _resolution_of(scratch: Path, name: str) -> _Resolution: + """`name`'s url, as `git submodule init` copies it into the config.""" + init = subprocess.run(["git", "submodule", "init", "-q", "--", name], + cwd=scratch, capture_output=True, check=False) + if init.returncode: + return _Resolution(None, _one_line(init.stderr.decode( + errors="replace"))) + return _Resolution(_config_value(scratch, f"submodule.{name}.url")) + + +def _relative_url_findings(plan: Plan, source: Source, tree: list, + registered: Registrations | str, + local_remote_dir: Path | None + ) -> tuple[list[str], list[str]]: + """`(findings, notes)` for the relative urls of the submodules the plan + keeps in a leg: a `plan-submodule-relative-url` FINDING for each that + git resolves to another repository from its leg than from the source, + or cannot resolve from one of them; and, when there is none of those, + the one NOTE `check` prints about what it could not know. + + THE LEG'S REMOTE IS THE ONE `execute` WOULD CREATE: under + `local_remote_dir`, the bare repository there; otherwise the GitHub + repository the plan's `org:` and leg name give. `check` takes no + `--local-remote-dir`, so it decides on GitHub's, and `execute` decides + again on the remotes it is about to create, before it creates any. They + are read only when there is a relative url to resolve, and the NOTE + says, where `check` resolved them all alike, that it was GitHub's it + asked: a plan it passes can still be refused by `execute + --local-remote-dir`.""" + leg, kept = _kept_relative_urls(plan.entries, tree, registered) + if not kept: + return [], [] + remote = _remote_urls(checked_value("org", plan.get("org")), plan.names(), + local_remote_dir, source)[leg] + base, which = _default_remote(source) + urls = [url for _, _, url in kept] + found, alike = [], [] + for (path, entry, url), here, there in zip(kept, _resolved(base, urls), + _resolved(remote, urls)): + if here.same_as(there): + alike.append(f"{_spelled(url)} of {_spelled(path)}, {there}") + continue + found.append( + f"FINDING {RELATIVE_URL}: the submodule {_as_entry(path, entry)} " + f"has leg: {leg}, and its url in the source's {GITMODULES} is the " + f"relative {_spelled(url)}, which git resolves against the remote " + "of the repository holding that file: from the source, at " + f"{_spelled(redacted(base))} ({which}), {here}, and from the " + f"{leg} leg, at {_spelled(remote)}, {there}; the assembly's " + "recursive clone resolves it from the leg") + if found or local_remote_dir is not None: + return found, [] + return [], [ + f"NOTE `check` resolved the relative url of each submodule the plan " + f"keeps in the {leg} leg from that leg's GitHub remote, " + f"{_spelled(remote)}, and each names the same repository from there " + f"as from the source: {'; '.join(alike)}. `execute " + "--local-remote-dir` resolves it again from the bare repository it " + "would create instead, and refuses there, before it creates any leg, " + "a url that names another repository."] + + +def _submodule_findings(plan: Plan, source: Source, tree: list, + local_remote_dir: Path | None = None + ) -> tuple[list[str], list[str], list[str]]: """`submodule_plan_problems` as `check` prints them, one FINDING each, - and what to print ONCE after every finding: SUBMODULE_REMEDIATION when - there is one of these, nothing when there is none. `check` is where the + then each `_relative_url_findings` one; what to print ONCE after every + finding: SUBMODULE_REMEDIATION when there is one of the first, + RELATIVE_URL_REMEDIATION when there is one of the second, nothing when + there is neither; and the NOTEs `check` prints. `check` is where the person is told to fix the plan, so it says how, in the words `execute`'s refusal would use one step later.""" + registered = source.registered_submodules(tree) found = [f"FINDING {code}: {detail}" for code, detail - in submodule_plan_problems(plan.entries, tree, - source.registered_submodules(tree))] - return found, [SUBMODULE_REMEDIATION] if found else [] + in submodule_plan_problems(plan.entries, tree, registered)] + moved, notes = _relative_url_findings(plan, source, tree, registered, + local_remote_dir) + return found + moved, ([SUBMODULE_REMEDIATION] if found else []) \ + + ([RELATIVE_URL_REMEDIATION] if moved else []), notes -def _entry_findings(plan: Plan, source: Source - ) -> tuple[list[str], list[str]]: +def _entry_findings(plan: Plan, source: Source, + local_remote_dir: Path | None = None + ) -> tuple[list[str], list[str], list[str]]: """Every FINDING `check` prints about the plan's ENTRIES, in its order, - and what it prints once after them: coverage, then each entry's `leg:`, - then the source's own submodules. + what it prints once after them, and the NOTEs it prints beside them: + coverage, then each entry's `leg:`, then the source's own submodules. ONE LIST, read by `check` and by `execute`'s `_refuse_what_check_finds`. #168 and #166 each added findings to `check` and a refusal to `execute` @@ -1850,14 +2089,18 @@ def _entry_findings(plan: Plan, source: Source same lines; three compositions then disagreed, and a plan `check` rejected as uncovered AND split was refused by `execute` for the split alone, to be refused again for the rest once that was fixed. Composed - once, the two commands cannot differ about a plan. + once, the two commands cannot differ about a plan -- but in the one + place a remote decides it: `local_remote_dir` is `execute`'s, and a + relative submodule url is resolved from the leg remotes it names, where + `check` has none and resolves from GitHub's (#190). """ tree = source.tree() findings = (_coverage_findings([str(e.get("path")) for e in plan.entries], [path for path, _, _, _ in tree]) + _leg_findings(plan)) - submodule_findings, remediation = _submodule_findings(plan, source, tree) - return findings + submodule_findings, remediation + submodule_findings, remediation, notes = _submodule_findings( + plan, source, tree, local_remote_dir) + return findings + submodule_findings, remediation, notes def _topics_line(topic: str, local: bool) -> str: @@ -1917,7 +2160,7 @@ def cmd_check(args) -> int: "tree that has moved proves nothing about the tree that will be " "split.") - entry_findings, remediation = _entry_findings(plan, source) + entry_findings, remediation, notes = _entry_findings(plan, source) findings.extend(entry_findings) names = plan.names() @@ -1939,6 +2182,8 @@ def cmd_check(args) -> int: print(line) for line in plan.seeding_record_disagreements(seeded): print(line) + for line in notes: + print(line) for line in [*findings, *remediation]: print(line, file=sys.stderr) @@ -2209,21 +2454,28 @@ def _repository_urls(args, plan: Plan, names: dict, itself under `--local-remote-dir`, because an in-place adoption pushes its split branch back to the repository it read. """ - local = args.local_remote_dir is not None org = checked_value("org", plan.get("org")) repositories = {role: f"{org}/{name}" for role, name in names.items()} - if local: - # NOT created yet: nothing exists on disk until the human has said - # yes, so a refused run leaves the directory it would have used - # absent rather than empty. - remote_dir = args.local_remote_dir.resolve() - urls = {role: str(remote_dir / f"{name}.git") - for role, name in names.items()} - urls["assembly"] = str(source.path) - else: - urls = {role: f"https://github.com/{org}/{name}.git" + return (args.local_remote_dir is not None, repositories, + _remote_urls(org, names, args.local_remote_dir, source)) + + +def _remote_urls(org: str, names: dict, local_remote_dir: Path | None, + source: Source) -> dict: + """The url of each of the three repositories, by role: what `execute` + pushes to, and what a relative submodule url kept in a leg is resolved + from (#190), which `check` asks with no `local_remote_dir`.""" + if local_remote_dir is None: + return {role: f"https://github.com/{org}/{name}.git" for role, name in names.items()} - return local, repositories, urls + # NOT created yet: nothing exists on disk until the human has said yes, + # so a refused run leaves the directory it would have used absent rather + # than empty. + remote_dir = local_remote_dir.resolve() + urls = {role: str(remote_dir / f"{name}.git") + for role, name in names.items()} + urls["assembly"] = str(source.path) + return urls def _leg_paths(plan: Plan) -> dict: @@ -2242,7 +2494,8 @@ def _leg_paths(plan: Plan) -> dict: REFUSED_FINDINGS_SHOWN = 8 -def _refuse_what_check_finds(plan: Plan, source: Source) -> None: +def _refuse_what_check_finds(plan: Plan, source: Source, + local_remote_dir: Path | None = None) -> None: """Refuse a plan `check` rejects, BEFORE a leg repository exists (#168). Of `check`'s findings, `execute` used to refuse only `plan-unresolved`, @@ -2300,14 +2553,21 @@ def _refuse_what_check_finds(plan: Plan, source: Source) -> None: states: a plan `check` rejects as uncovered and split is refused for both at once, and a submodule problem is refused here like any other, after `_leg_paths` has checked every path as a `git` argument. + + BUT FROM THE REMOTES THIS RUN WOULD CREATE (#190). A relative submodule + url kept in a leg is resolved from that leg's remote, and under + `local_remote_dir` that is a bare repository `check` cannot know of: such + a finding is this run's own, and the refusal says so. A url `check` finds + from GitHub's remotes may name the same repository from these, and one it + passes may not. """ - findings, remediation = _entry_findings(plan, source) + findings, remediation, _ = _entry_findings(plan, source, local_remote_dir) if not findings: return # `FINDING : ` is the one shape both functions write. code = findings[0].split()[1].rstrip(":") lines = [f"{len(findings)} finding(s) in {plan.path}, as `check` prints " - "them:"] + f"them{_as_resolved_here(findings, local_remote_dir)}:"] lines += [f" {finding}" for finding in findings[:REFUSED_FINDINGS_SHOWN]] if len(findings) > REFUSED_FINDINGS_SHOWN: lines.append(f" and {len(findings) - REFUSED_FINDINGS_SHOWN} more") @@ -2322,6 +2582,20 @@ def _refuse_what_check_finds(plan: Plan, source: Source) -> None: *remediation])) +def _as_resolved_here(findings: list[str], + local_remote_dir: Path | None) -> str: + """What the refusal's first line adds when a finding in it is not one + `check` prints: a relative url resolved from `local_remote_dir`'s + remotes, where `check` resolves it from GitHub's. Nothing otherwise.""" + if local_remote_dir is None or not any( + finding.startswith(f"FINDING {RELATIVE_URL}:") + for finding in findings): + return "" + return (", but with each relative submodule url resolved from the leg " + "remotes this run would create under --local-remote-dir, where " + "`check` resolves it from GitHub's") + + def _refuse_unconsented_seeding(args, plan: Plan, seeded: list) -> None: """A leg with no path is SEEDED, and seeding takes a human's word. @@ -2548,7 +2822,7 @@ def cmd_execute(args) -> int: spec_path, code_path, branch, tracking = _checked_plan_values(plan) local, repositories, urls = _repository_urls(args, plan, names, source) paths_for = _leg_paths(plan) - _refuse_what_check_finds(plan, source) + _refuse_what_check_finds(plan, source, args.local_remote_dir) seeded = seeded_legs(paths_for) _refuse_unconsented_seeding(args, plan, seeded) diff --git a/tests/test_adopt_relative_submodule_url.py b/tests/test_adopt_relative_submodule_url.py new file mode 100644 index 00000000..1bbc1caf --- /dev/null +++ b/tests/test_adopt_relative_submodule_url.py @@ -0,0 +1,635 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A source submodule's RELATIVE url, resolved from the leg that holds it (#190). + +git resolves `url = ../dependency` against the remote of the repository +whose `.gitmodules` holds it: the source's before the split, the LEG's after +it. On main a plan that kept such a submodule in a leg checked out, executed, +and said `adoption verified`, and the assembly's `git clone +--recurse-submodules` then failed wherever the dependency did not sit beside +the leg. `check` now resolves it from both places, as git does, and finds it +where the two differ; `execute` resolves it again from the remotes it would +create and refuses it before it creates any. + +No network: a GitHub url here is only ever a string git resolves against. +Every repository is local, and the end-to-end tests clone from disk. +""" + +from __future__ import annotations + +import importlib.util +import os +import re +import shutil +import sys +from argparse import Namespace +from pathlib import Path + +import pytest + +from conftest import (ADOPT, FILE_PROTOCOL, git, make_source_repo, resolve, + run_script, write_plan) + +needs_filter_repo = pytest.mark.skipif( + shutil.which("git-filter-repo") is None, + reason="git filter-repo is not installed: `pip install git-filter-repo`") + +#: What the test with a symlink ON THE DISK needs; see +#: tests/test_adopt_ignored_shape_files.py for the same guard. +needs_disk_symlinks = pytest.mark.skipif( + sys.platform == "win32", + reason="a symlink needs a privilege Windows does not grant by default") + +#: The finding, as `check` prints it and `execute` raises it. +RELATIVE = "plan-submodule-relative-url" +SPLIT = "plan-submodule-split" +GITMODULES = ".gitmodules" +UPSTREAM = "upstream/" +EXTRA = "extra/" +#: The submodule every source here holds, a second one some hold, and the +#: one file of the repository each is a commit of. +DEPENDENCY = "upstream/dependency" +SECOND = "extra/second" +VALUE = "value.txt" +DEPENDENCY_TREE = {VALUE: "dependency\n"} +#: The url #190 is about: the dependency beside whatever repository holds +#: `.gitmodules`. +SIBLING = "../dependency" +#: The dependency INSIDE the repository holding `.gitmodules`, and two +#: levels up from it: git's arithmetic, not a sibling's. +INSIDE = "./dependency" +TWO_UP = "../../dependency" +#: The second submodule's url, beside whatever holds `.gitmodules`. +SECOND_SIBLING = "../second" +#: The source's own name for its remote, beside the repositories it makes. +THING_REMOTE = "Thing.git" +#: Where each test's `execute` works, and what a verified one says. +WORK = "work" +VERIFIED = "adoption verified" +#: A source whose plan asks nothing but the submodules' questions. +SMALL_TREE = {"README.md": "# Thing\n", "src/app/util.py": "VALUE = 1\n", + "specs/001-feature/tasks.md": "- [ ] one\n"} +#: The plan's own organisation, and one that is not the source's. +ORG, OTHER_ORG = "testorg", "otherorg" +#: Where each test's `execute` makes its leg remotes. +REMOTES = "remotes" +#: The code leg's GitHub remote, as `check` resolves from it. +CODE_LEG = "https://github.com/{org}/Northwind-code.git" +#: The code leg's bare remote, as `execute --local-remote-dir` names it. +CODE_BARE = "Northwind-code.git" +#: The source's `origin` on GitHub, in the spellings a clone may carry. +HTTPS_ORIGIN = "https://github.com/testorg/Thing.git" +SCP_ORIGIN = "git@github.com:testorg/Thing.git" +#: An owner spelled in another case: one organisation to GitHub. +SCP_OTHER_CASE = "git@github.com:TestOrg/Thing.git" +#: What `check` says of the source's base when it is its `origin`. +ITS_ORIGIN = "its remote `origin`" +#: Every source's dependency, from the plan's own organisation on GitHub. +GITHUB_DEPENDENCY = "https://github.com/testorg/dependency" +#: The one leg every plan here keeps the submodules in. +BOTH_IN_CODE = {GITMODULES: "code", UPSTREAM: "code"} + + +@pytest.fixture(scope="module") +def adopter(): + """`adopt-project.py` as a module: its remediation, its spelling of a + url in a finding, and `execute`'s refusal.""" + module_spec = importlib.util.spec_from_file_location("adopt_relative", + ADOPT) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + return module + + +def source_with(base: Path, url: str, origin: str | None = None, + dependency: Path | None = None, + second: str | None = None) -> Path: + """`base/Thing`, holding DEPENDENCY registered with `url`, and SECOND + registered with `second` when it is given. + + Each submodule is ADDED from the dependency's absolute path, so nothing + is fetched from a url that is only a string, and its url is then set in + `.gitmodules` and committed, as a hand edit would. `origin`, when given, + is the source's remote: git resolves a relative url against it only + when asked to. + """ + dependency = dependency or base / "dependency" + make_source_repo(dependency, tree=DEPENDENCY_TREE, edits=()) + source = make_source_repo(base / "Thing", tree=SMALL_TREE) + for path, value in ((DEPENDENCY, url), (SECOND, second)): + if value is None: + continue + git(*FILE_PROTOCOL, "submodule", "add", "-q", str(dependency), path, + cwd=source) + git("config", "-f", GITMODULES, f"submodule.{path}.url", value, + cwd=source) + commit(source, "Register the dependency") + if origin is not None: + git("remote", "add", "origin", origin, cwd=source) + return source + + +def commit(source: Path, message: str) -> None: + git("add", "--", GITMODULES, cwd=source) + git("-c", "user.name=Source Human", "-c", + "user.email=source@invalid.example", "commit", "-qm", message, + cwd=source) + + +def planned(source: Path, base: Path, org: str = ORG, + answers: dict | None = None, name: str | None = None) -> Path: + """A plan of `source` for `org`, in a directory of its own under `base` + (`name`, else `plan-`), with `.gitmodules` and DEPENDENCY both kept + in code unless `answers` says otherwise.""" + plan = base / (name or f"plan-{org}") / "adoption-plan.yaml" + plan.parent.mkdir(parents=True) + written = write_plan(source, plan, org=org) + assert written.returncode == 0, written.stderr + written.stdout + for path, leg in (answers or BOTH_IN_CODE).items(): + resolve(plan, path, leg) + return plan + + +def check(plan: Path): + return run_script(ADOPT, "check", "--plan", str(plan)) + + +def found(stderr: str) -> list[tuple[str, str]]: + """`(code, detail)` for each `FINDING : ` line.""" + return [tuple(line.split(" ", 1)[1].split(": ", 1)) + for line in stderr.splitlines() if line.startswith("FINDING ")] + + +def notes(stdout: str) -> list[str]: + """The NOTE lines `check` prints about the relative urls it resolved.""" + return [line for line in stdout.splitlines() + if line.startswith("NOTE `check` resolved")] + + +def moved(adopter, url: str, base: str, which: str, here: str, remote: str, + there: str, path: str = DEPENDENCY, entry: str = UPSTREAM) -> str: + """The detail of the finding, for the submodule at `path` kept in code. + `here` and `there` are git's resolutions, spelled; every other url is + spelled here as the tool spells it, which on Windows doubles each + backslash of a path.""" + spelled = adopter._spelled + return (f"the submodule {path} (entry {entry}) has leg: code, and its " + f"url in the source's {GITMODULES} is the relative " + f"{spelled(url)}, which git resolves against the remote of the " + f"repository holding that file: from the source, at " + f"{spelled(base)} ({which}), {here}, and from the code leg, at " + f"{spelled(remote)}, {there}; the assembly's recursive clone " + "resolves it from the leg") + + +def alike(org: str, *resolved: tuple[str, str, str]) -> str: + """The NOTE `check` prints when every `(url, path, resolution)` names, + from the code leg on GitHub, the repository it names from the source.""" + return ("NOTE `check` resolved the relative url of each submodule the " + "plan keeps in the code leg from that leg's GitHub remote, " + f"`{CODE_LEG.format(org=org)}`, and each names the same " + "repository from there as from the source: " + + "; ".join(f"`{url}` of `{path}`, to `{to}`" + for url, path, to in resolved) + + ". `execute --local-remote-dir` resolves it again from the bare " + "repository it would create instead, and refuses there, before " + "it creates any leg, a url that names another repository.") + + +#: `from the code leg, at ``, to ``; ` in a finding. +LEG_RESOLUTION_RE = re.compile( + r"from the code leg, at `(?:[^`\\]|\\.)*`, to `((?:[^`\\]|\\.)*)`; ") + + +def resolved_from_the_leg(detail: str) -> Path: + """The url git resolved from the code leg, as the path it names: the + spelling's backslash escapes undone, so the comparison is a Path's and + holds whichever separator git joined it with.""" + spelled = LEG_RESOLUTION_RE.search(detail) + assert spelled, detail + return Path(re.sub(r"\\(.)", r"\1", spelled[1])) + + +def passes(result) -> None: + assert result.returncode == 0, result.stderr + result.stdout + assert "plan ok" in result.stdout + assert RELATIVE not in result.stderr + result.stdout + + +# --- `check`, on the GitHub remotes the plan names -------------------------- + + +@pytest.mark.parametrize("origin, resolved", [ + (HTTPS_ORIGIN, GITHUB_DEPENDENCY), + (SCP_ORIGIN, "git@github.com:testorg/dependency"), + (SCP_OTHER_CASE, "git@github.com:TestOrg/dependency"), +], ids=["https", "scp", "scp-other-case"]) +def test_check_keeps_a_relative_url_its_github_leg_resolves_alike( + adopter, tmp_path, origin, resolved): + """(a) Legs in the source's own organisation sit beside its dependency + on GitHub, so `../dependency` names one repository from both, however + the source's `origin` spells it: no finding, and the NOTE says which + remotes `check` asked. In another organisation it names another + repository, and that is found, with the submodule, its url, the leg's + remote and both resolutions, and the remediation once after it.""" + source = source_with(tmp_path, SIBLING, origin) + same = check(planned(source, tmp_path)) + passes(same) + assert notes(same.stdout) == [alike(ORG, (SIBLING, DEPENDENCY, + GITHUB_DEPENDENCY))] + + result = check(planned(source, tmp_path, OTHER_ORG)) + assert result.returncode == 1, result.stderr + result.stdout + assert found(result.stderr) == [(RELATIVE, moved( + adopter, SIBLING, origin, ITS_ORIGIN, f"to `{resolved}`", + CODE_LEG.format(org=OTHER_ORG), + "to `https://github.com/otherorg/dependency`"))] + assert notes(result.stdout) == [] + # Said once, after the findings, and it is not #166's remediation. + assert result.stderr.count(adopter.RELATIVE_URL_REMEDIATION) == 1 + assert adopter.SUBMODULE_REMEDIATION not in result.stderr + assert result.stderr.index("FINDING ") \ + < result.stderr.index(adopter.RELATIVE_URL_REMEDIATION) \ + < result.stderr.index(" finding(s) in ") + + +@pytest.mark.parametrize("url, there", [ + # Inside the repository that holds it: another one for every leg. + (INSIDE, + "to `https://github.com/testorg/Northwind-code.git/dependency`"), + # More `..` than the remote has components: git says so, in words + # this pins only the start of, since they are git's to change. + ("../" * 8 + "dependency", "not at all (fatal: "), +], ids=["dot-slash", "too-many-dotdots"]) +def test_check_finds_a_relative_url_no_github_leg_resolves_alike( + tmp_path, url, there): + """`./dependency` names a repository INSIDE the one holding + `.gitmodules`, so no leg resolves it to the source's; a url git cannot + resolve from the leg is one the assembly's clone fails on. Both are found + in the plan's own organisation, where `../dependency` is not.""" + source = source_with(tmp_path, url, HTTPS_ORIGIN) + result = check(planned(source, tmp_path)) + assert result.returncode == 1, result.stderr + result.stdout + [(code, detail)] = found(result.stderr) + assert code == RELATIVE + assert f"from the code leg, at `{CODE_LEG.format(org=ORG)}`, {there}" \ + in detail, detail + assert detail.endswith("; the assembly's recursive clone resolves it " + "from the leg"), detail + + +def test_a_source_with_no_remote_resolves_from_its_own_directory(adopter, + tmp_path): + """With no remote, git resolves a relative url against the source's own + directory, so `../dependency` is the directory beside the source -- and + a GitHub leg resolves it to GitHub. `check` says which it used.""" + source = source_with(tmp_path, SIBLING) + result = check(planned(source, tmp_path)) + assert result.returncode == 1, result.stderr + result.stdout + resolved = adopter._spelled((tmp_path / "dependency").as_posix()) + assert found(result.stderr) == [(RELATIVE, moved( + adopter, SIBLING, source.as_posix(), + "its own directory, as it has no remote `origin`", f"to {resolved}", + CODE_LEG.format(org=ORG), f"to `{GITHUB_DEPENDENCY}`"))] + + +def test_the_source_resolves_from_the_remote_its_branch_tracks(tmp_path): + """git's default remote is the one the branch tracks, and `origin` only + when it tracks none: a fork's `origin` is not where its `main` comes + from, and `upstream` is.""" + source = source_with(tmp_path, SIBLING, + "https://github.com/someone/Thing.git") + git("remote", "add", "upstream", HTTPS_ORIGIN, cwd=source) + git("config", "branch.main.remote", "upstream", cwd=source) + passes(check(planned(source, tmp_path))) + git("config", "--unset", "branch.main.remote", cwd=source) + result = check(planned(source, tmp_path, name="untracked")) + assert [code for code, _ in found(result.stderr)] == [RELATIVE] + assert ("from the source, at `https://github.com/someone/Thing.git` " + f"({ITS_ORIGIN}), to `https://github.com/someone/dependency`" + ) in result.stderr + + +def test_each_relative_url_is_resolved_on_its_own(adopter, tmp_path): + """Two submodules in one leg, resolved in one scratch repository: the + one whose url names another repository from the leg is found, by its + own path and url, and the one that does not is not; with both alike, + the NOTE names both, in the tree's order.""" + source = source_with(tmp_path, INSIDE, HTTPS_ORIGIN, + second=SECOND_SIBLING) + answers = {**BOTH_IN_CODE, EXTRA: "code"} + result = check(planned(source, tmp_path, answers=answers)) + assert found(result.stderr) == [(RELATIVE, moved( + adopter, INSIDE, HTTPS_ORIGIN, ITS_ORIGIN, + "to `https://github.com/testorg/Thing.git/dependency`", + CODE_LEG.format(org=ORG), + "to `https://github.com/testorg/Northwind-code.git/dependency`"))] + assert notes(result.stdout) == [] + + both = source_with(tmp_path / "both", SIBLING, HTTPS_ORIGIN, + second=SECOND_SIBLING) + result = check(planned(both, tmp_path / "both", answers=answers)) + passes(result) + assert notes(result.stdout) == [alike( + ORG, (SECOND_SIBLING, SECOND, "https://github.com/testorg/second"), + (SIBLING, DEPENDENCY, GITHUB_DEPENDENCY))] + + +def test_a_dropped_submodule_leaves_its_relative_url_inert(tmp_path): + """git resolves the url of a submodule it finds a gitlink for, and of no + other: dropped on its own, its registration stays behind in the leg and + is never resolved, and nothing is said of it. Kept, it is found.""" + source = source_with(tmp_path, SIBLING) + dropped = check(planned(source, tmp_path, + answers={GITMODULES: "code", UPSTREAM: "drop"})) + passes(dropped) + assert notes(dropped.stdout) == [] + kept = check(planned(source, tmp_path, name="kept")) + assert [code for code, _ in found(kept.stderr)] == [RELATIVE] + + +def test_a_split_submodule_is_a_split_and_nothing_more(tmp_path): + """A submodule kept away from its `.gitmodules` is #166's split: the + leg that holds it holds no url for it, relative or not, so it is not + also a relative url resolved from a leg.""" + source = source_with(tmp_path, SIBLING) + result = check(planned(source, tmp_path, + answers={GITMODULES: "code", UPSTREAM: "spec"})) + assert [code for code, _ in found(result.stderr)] == [SPLIT] + + +def test_a_credential_in_the_source_remote_is_never_printed(adopter, + tmp_path): + """A source's `origin` may carry a token, and every url resolved against + it carries it too. The finding names both with the credential redacted, + as the doctor does, and the token appears nowhere `check` writes.""" + secret = "s3cret-token" + origin = f"https://x-access-token:{secret}@github.com/testorg/Thing.git" + source = source_with(tmp_path, SIBLING, origin) + result = check(planned(source, tmp_path, OTHER_ORG)) + assert secret not in result.stderr + result.stdout + assert found(result.stderr) == [(RELATIVE, moved( + adopter, SIBLING, + "https://***@github.com/testorg/Thing.git", ITS_ORIGIN, + "to `https://***@github.com/testorg/dependency`", + CODE_LEG.format(org=OTHER_ORG), + "to `https://github.com/otherorg/dependency`"))] + # And from the plan's own organisation it is one repository: the + # credential is not part of a repository's identity. + same = check(planned(source, tmp_path)) + passes(same) + assert secret not in same.stderr + same.stdout + + +def test_an_absolute_url_resolves_nothing(adopter, tmp_path, monkeypatch): + """(d) A url that is not relative is copied by git as it stands, so it + is never resolved and no scratch repository is made for it: a source + with an absolute url checks exactly as it did on #166's base, with no + finding and no NOTE. The same plan with a relative url is resolved.""" + def refuse(*_): + raise AssertionError("a url was resolved") + + monkeypatch.setattr(adopter, "_resolved", refuse) + dependency = tmp_path / "dependency" + absolute = source_with(tmp_path / "absolute", dependency.as_posix(), + dependency=dependency) + plan = adopter.Plan.load(planned(absolute, tmp_path / "absolute")) + opened = plan.open_source(None, tmp_path / WORK) + assert adopter._entry_findings(plan, opened) == ([], [], []) + + relative = source_with(tmp_path / "relative", SIBLING) + plan = adopter.Plan.load(planned(relative, tmp_path / "relative")) + opened = plan.open_source(None, tmp_path / WORK) + with pytest.raises(AssertionError, match="a url was resolved"): + adopter._entry_findings(plan, opened) + + +# --- `execute`, on the remotes it would create ------------------------------- + + +def refusal_of(adopter, plan: Path, tmp_path: Path, + remotes: Path | None = None): + """What `_refuse_what_check_finds` raises for `plan` against the leg + remotes `execute` would make under `remotes`, or GitHub's; None when it + raises nothing.""" + loaded = adopter.Plan.load(plan) + opened = loaded.open_source(None, tmp_path / WORK) + try: + adopter._refuse_what_check_finds(loaded, opened, remotes) + except adopter.Refusal as refusal: + return refusal + return None + + +@pytest.mark.parametrize("origin, url, org", [ + (HTTPS_ORIGIN, SIBLING, ORG), + (SCP_ORIGIN, SIBLING, ORG), + (SCP_OTHER_CASE, SIBLING, ORG), + # git's arithmetic, not the organisation: two `..` leave both behind. + (HTTPS_ORIGIN, TWO_UP, OTHER_ORG), +], ids=["https", "scp", "scp-other-case", "dotdot-dotdot-other-org"]) +def test_execute_resolves_again_from_the_remotes_it_would_create( + adopter, tmp_path, monkeypatch, origin, url, org): + """(c) `check` cannot know where `execute` will make the legs, so it + resolves from GitHub's, where these plans name one repository from both + and pass, and says so. `execute --local-remote-dir` resolves again from + the bare repositories it would make there, where they do not, and + refuses before it makes any -- saying the finding is its own and not one + `check` printed, and that `check`'s `plan ok` settles nothing here.""" + source = source_with(tmp_path, url, origin) + plan = planned(source, tmp_path, org) + checked = check(plan) + passes(checked) + assert len(notes(checked.stdout)) == 1 + # Every refusal here comes before the tool would be used. + monkeypatch.setattr(adopter, "_require_filter_repo", lambda: None) + remotes = tmp_path / REMOTES + with pytest.raises(adopter.Refusal) as caught: + adopter.cmd_execute(Namespace( + plan=str(plan), source=None, local_remote_dir=remotes, + allow_empty_leg=[], yes=True, work_dir=tmp_path / WORK)) + refusal = caught.value + assert refusal.code == RELATIVE + first, finding = refusal.detail.splitlines() + assert first == ( + f"1 finding(s) in {plan}, as `check` prints them, but with each " + "relative submodule url resolved from the leg remotes this run would " + "create under --local-remote-dir, where `check` resolves it from " + "GitHub's:") + leg = adopter._spelled(str(remotes.resolve() / CODE_BARE)) + assert finding.startswith(f" FINDING {RELATIVE}: the submodule " + f"{DEPENDENCY} (entry {UPSTREAM})"), finding + assert f", and from the code leg, at {leg}, to " in finding + assert refusal.remediation.endswith( + "\n" + adopter.RELATIVE_URL_REMEDIATION) + assert "`check`'s `plan ok` settles nothing" in refusal.remediation + assert "no leg repository was created" in refusal.remediation + assert not remotes.exists() + + +def test_execute_without_local_remotes_refuses_what_check_prints( + adopter, tmp_path): + """Against GitHub, `execute` resolves from the remotes `check` resolved + from, so its refusal is `check`'s list, line for line, under the first + line `check`'s refusals have always had.""" + source = source_with(tmp_path, SIBLING) + plan = planned(source, tmp_path) + printed = found(check(plan).stderr) + assert [code for code, _ in printed] == [RELATIVE] + refusal = refusal_of(adopter, plan, tmp_path) + assert refusal.code == RELATIVE + assert refusal.detail.splitlines() == [ + f"1 finding(s) in {plan}, as `check` prints them:", + *(f" FINDING {code}: {detail}" for code, detail in printed)] + + +def test_the_refusal_says_nothing_of_local_remotes_it_did_not_use( + adopter, tmp_path): + """Under `--local-remote-dir` a plan refused for anything BUT a relative + url is refused in `check`'s words: the first line adds nothing about + remotes it resolved nothing from.""" + source = source_with(tmp_path, SIBLING) + plan = planned(source, tmp_path, + answers={GITMODULES: "code", UPSTREAM: "spec"}) + refusal = refusal_of(adopter, plan, tmp_path, tmp_path / REMOTES) + assert refusal.code == SPLIT + assert refusal.detail.splitlines()[0] \ + == f"1 finding(s) in {plan}, as `check` prints them:" + + +@needs_disk_symlinks +def test_one_directory_through_a_symlink_is_one_repository(adopter, + tmp_path): + """The source's `origin` reaches the remotes directory through a + symlink, and `--local-remote-dir` names it directly: two spellings, one + directory, so `../dependency` names one repository from both and is not + refused. A directory that is not the same one is.""" + remotes = tmp_path / REMOTES + remotes.mkdir() + link = tmp_path / "link" + try: + os.symlink(remotes, link, target_is_directory=True) + except OSError as exc: + # A platform, or a filesystem, that refuses one: not a failure. + pytest.skip(f"this platform cannot create a symlink here: {exc}") + linked = source_with(tmp_path / "linked", SIBLING, + str(link / THING_REMOTE)) + plan = planned(linked, tmp_path / "linked") + assert refusal_of(adopter, plan, tmp_path, remotes) is None + + elsewhere = source_with(tmp_path / "elsewhere", SIBLING, + str(tmp_path / "elsewhere" / THING_REMOTE)) + plan = planned(elsewhere, tmp_path / "elsewhere") + refusal = refusal_of(adopter, plan, tmp_path, remotes) + assert refusal is not None and refusal.code == RELATIVE + + +def execute(plan: Path, work: Path, remotes: Path): + return run_script(ADOPT, "execute", "--plan", str(plan), "--yes", + "--local-remote-dir", str(remotes), + "--work-dir", str(work)) + + +def clone_the_split(source: Path, clone: Path): + """`git clone --recurse-submodules` of the branch `execute` pushed the + split to: what a person runs next.""" + return git(*FILE_PROTOCOL, "clone", "-q", "--recurse-submodules", "-b", + "adopt/three-repo-shape", str(source), str(clone), + cwd=clone.parent, check=False) + + +@needs_filter_repo +@pytest.mark.parametrize("url, there", [ + # #190 itself: the re-verifier's `relative-url` case. + (SIBLING, "remotes/dependency"), + (INSIDE, "remotes/Northwind-code.git/dependency"), + (TWO_UP, "dependency"), +], ids=["dotdot", "dot-slash", "dotdot-dotdot"]) +def test_execute_refuses_a_relative_url_its_leg_resolves_elsewhere( + tmp_path, url, there): + """A source with no remote of its own, as the re-verifier built it: + `check` finds it from GitHub's remotes, and `execute --local-remote-dir` + from the bare ones it would make. On main both passed, `execute` said + `adoption verified`, and the assembly's recursive clone exited 1. Now + `execute` exits 2 before the remotes directory exists, and the source + is untouched.""" + source = source_with(tmp_path, url) + head = git("rev-parse", "HEAD", cwd=source).stdout + plan = planned(source, tmp_path) + assert [code for code, _ in found(check(plan).stderr)] == [RELATIVE] + remotes, work = tmp_path / REMOTES, tmp_path / WORK + result = execute(plan, work, remotes) + assert result.returncode == 2, result.stderr + result.stdout + assert f"REFUSED {RELATIVE}: 1 finding(s) in " in result.stderr + [detail] = [line.split(": ", 1)[1] for line in result.stderr.splitlines() + if line.startswith(f" FINDING {RELATIVE}: ")] + assert resolved_from_the_leg(detail) == tmp_path.resolve() / there + assert "creating the leg repositories" not in result.stdout + assert VERIFIED not in result.stdout + assert not remotes.exists(), "a refused plan made the remotes directory" + assert list(work.iterdir()) == [], "a refused plan wrote into its work" + assert git("rev-parse", "HEAD", cwd=source).stdout == head + + +@needs_filter_repo +def test_the_remediation_adopts_and_clones(tmp_path): + """The refusal's remediation works: the absolute url committed in the + source's `.gitmodules`, a fresh plan answered as before, and `check` + passes with nothing to say of it, `execute` verifies into the remotes + the refused run never made, and the assembly's recursive clone fetches + the dependency into the leg.""" + source = source_with(tmp_path, SIBLING) + remotes = tmp_path / REMOTES + refused = execute(planned(source, tmp_path), tmp_path / WORK, remotes) + assert f"REFUSED {RELATIVE}: " in refused.stderr, refused.stderr + + git("config", "-f", GITMODULES, f"submodule.{DEPENDENCY}.url", + (tmp_path / "dependency").as_posix(), cwd=source) + commit(source, "Register the dependency by its absolute url") + plan = planned(source, tmp_path, name="replanned") + checked = check(plan) + passes(checked) + assert notes(checked.stdout) == [] + result = execute(plan, tmp_path / "work-again", remotes) + assert result.returncode == 0, result.stderr + result.stdout + assert VERIFIED in result.stdout + clone = tmp_path / "clone" + cloned = clone_the_split(source, clone) + assert cloned.returncode == 0, cloned.stderr + assert (clone / "code" / DEPENDENCY / VALUE).read_text() \ + == DEPENDENCY_TREE[VALUE] + + +@needs_filter_repo +@pytest.mark.parametrize("url, origin, legs", [ + # The dependency beside the source's remote, and the legs beside both. + (SIBLING, THING_REMOTE, "."), + # Two levels: the source's remote and the legs in sibling directories. + (TWO_UP, f"sources/{THING_REMOTE}", "legs"), +], ids=["dotdot", "dotdot-dotdot"]) +def test_a_relative_url_its_leg_resolves_alike_adopts_and_clones( + tmp_path, url, origin, legs): + """(a) end to end. The source's `origin` and the legs are local, in a + layout where its relative url names the dependency from both: `check`, + which resolves from GitHub's remotes, finds it, and `execute`, which + resolves from the ones it makes, adopts and verifies, and the assembly's + `git clone --recurse-submodules` and `bootstrap.py` both exit 0 with the + dependency checked out in the leg, whose `.gitmodules` still says what + the source committed.""" + remotes = tmp_path / REMOTES + dependency = remotes / "dependency" + source = source_with(tmp_path, url, str(remotes / origin), dependency) + plan = planned(source, tmp_path) + assert [code for code, _ in found(check(plan).stderr)] == [RELATIVE] + result = execute(plan, tmp_path / WORK, remotes / legs) + assert result.returncode == 0, result.stderr + result.stdout + assert VERIFIED in result.stdout + clone = tmp_path / "clone" + cloned = clone_the_split(source, clone) + assert cloned.returncode == 0, cloned.stderr + leg = clone / "code" + assert git("config", "-f", GITMODULES, "--get", + f"submodule.{DEPENDENCY}.url", cwd=leg).stdout.strip() == url + assert (leg / DEPENDENCY / VALUE).read_text() == DEPENDENCY_TREE[VALUE] + bootstrapped = run_script(clone / "scripts/bootstrap.py", cwd=clone) + assert bootstrapped.returncode == 0, (bootstrapped.stderr + + bootstrapped.stdout) diff --git a/tests/test_adopt_submodules.py b/tests/test_adopt_submodules.py index d6a389d0..04da1fee 100644 --- a/tests/test_adopt_submodules.py +++ b/tests/test_adopt_submodules.py @@ -310,8 +310,11 @@ def test_mount_removes_gitmodules_before_a_submodule_that_sorts_ahead( THE_DEPENDENCY = "" -def registration(name: str = DEPENDENCY, url: str = "../dependency") -> str: - """`DEPENDENCY` registered by hand, under `name`, with `url`.""" +def registration(name: str = DEPENDENCY, url: str = THE_DEPENDENCY) -> str: + """`DEPENDENCY` registered by hand, under `name`, with `url`: by + default the dependency's absolute path, because `check` finds a RELATIVE + url its leg resolves elsewhere (#190), and these sources have no remote + to resolve one alike from.""" return f'[submodule "{name}"]\n\tpath = {DEPENDENCY}\n\turl = {url}\n' @@ -863,7 +866,7 @@ def test_check_refuses_to_keep_a_gitlink_its_gitmodules_does_not_register( LATIN_1_COMMENT = b"# caf\xe9 \xff\xfe\n" #: Upper-case section and keys, which git lower-cases. UPPER_CASE_KEYS = ('[Submodule "upstream/dependency"]\n' - "\tPATH = upstream/dependency\n\tURL = ../dependency\n") + f"\tPATH = upstream/dependency\n\tURL = {THE_DEPENDENCY}\n") @pytest.mark.parametrize("gitmodules, expected", [ @@ -1372,7 +1375,7 @@ def test_execute_refuses_a_submodule_plan_before_anything_exists( ("code", "drop", None), # Hand-written registrations `check` reads as the leg's clone reads them: # upper-case keys, and two `git config --blob` refused. - ("code", "code", UPPER_CASE_KEYS.replace("../dependency", THE_DEPENDENCY)), + ("code", "code", UPPER_CASE_KEYS), ("code", "code", LATIN_1_COMMENT + registration(url=THE_DEPENDENCY).encode()), ("code", "code", registration(url=THE_DEPENDENCY) From 8a4663bd4496f7586347c34864e94d4e46e3bdd7 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:59:45 +0000 Subject: [PATCH 09/12] tests/test_adopt_relative_submodule_url.py: `execute`'s refusal is asked of a relative url kept in the SPEC leg and of `..\dependency`, so #190 coming back for the spec leg alone, or for Windows' separator alone, no longer passes every test (#190) Lane 2's review of b0c959e found two mutants of adopt-project.py that survived all 193 targeted tests: - M14, `if kept == leg == "code" and ...` in `_kept_relative_urls`, which resolves only the code leg's relative urls and reintroduces #190 for the spec leg (its rows P6 and P8 then end `adoption verified` over a failing recursive clone); - M13, `RELATIVE_URL_RE = re.compile(r"\.\.?/")`, which no longer takes `..\` for relative, where git does on every platform and strips it on Windows: there it is #190's own case. `test_execute_refuses_a_relative_url_its_leg_resolves_elsewhere` gains a `leg` parameter and two rows: - `spec-leg`: `.gitmodules` and the submodule kept in spec, url `./dependency`, so the answer names the spec leg's own remote (`remotes/Northwind-spec.git/dependency`), not the code leg's; - `backslash`: `..\dependency` in code, which git appends whole where `\` separates nothing (`remotes/Northwind-code.git/..\dependency`) and strips on Windows (`remotes/dependency`). `resolved_from_the_leg` reads the leg's name out of the finding and checks it. Each mutant, run over this file at c46dc57, fails exactly its row and nothing else: M14 1 failed (`[spec-leg]`), 26 passed; M13 1 failed (`[backslash]`), 26 passed. Unmutated, all five rows pass. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_relative_submodule_url.py | 41 ++++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/test_adopt_relative_submodule_url.py b/tests/test_adopt_relative_submodule_url.py index 1bbc1caf..65405768 100644 --- a/tests/test_adopt_relative_submodule_url.py +++ b/tests/test_adopt_relative_submodule_url.py @@ -60,6 +60,10 @@ TWO_UP = "../../dependency" #: The second submodule's url, beside whatever holds `.gitmodules`. SECOND_SIBLING = "../second" +#: SIBLING with Windows' separator. git takes `..\` for a relative url on +#: every platform and strips it only where `\` separates directories, so on +#: Windows this is #190's own case. +BACKSLASH = "..\\dependency" #: The source's own name for its remote, beside the repositories it makes. THING_REMOTE = "Thing.git" #: Where each test's `execute` works, and what a verified one says. @@ -195,18 +199,19 @@ def alike(org: str, *resolved: tuple[str, str, str]) -> str: "it creates any leg, a url that names another repository.") -#: `from the code leg, at ``, to ``; ` in a finding. +#: `from the leg, at ``, to ``; ` in a finding. LEG_RESOLUTION_RE = re.compile( - r"from the code leg, at `(?:[^`\\]|\\.)*`, to `((?:[^`\\]|\\.)*)`; ") + r"from the (spec|code) leg, at `(?:[^`\\]|\\.)*`, " + r"to `((?:[^`\\]|\\.)*)`; ") -def resolved_from_the_leg(detail: str) -> Path: - """The url git resolved from the code leg, as the path it names: the +def resolved_from_the_leg(detail: str, leg: str = "code") -> Path: + """The url git resolved from `leg`, as the path it names: the spelling's backslash escapes undone, so the comparison is a Path's and holds whichever separator git joined it with.""" spelled = LEG_RESOLUTION_RE.search(detail) - assert spelled, detail - return Path(re.sub(r"\\(.)", r"\1", spelled[1])) + assert spelled and spelled[1] == leg, detail + return Path(re.sub(r"\\(.)", r"\1", spelled[2])) def passes(result) -> None: @@ -538,23 +543,29 @@ def clone_the_split(source: Path, clone: Path): @needs_filter_repo -@pytest.mark.parametrize("url, there", [ +@pytest.mark.parametrize("url, leg, there", [ # #190 itself: the re-verifier's `relative-url` case. - (SIBLING, "remotes/dependency"), - (INSIDE, "remotes/Northwind-code.git/dependency"), - (TWO_UP, "dependency"), -], ids=["dotdot", "dot-slash", "dotdot-dotdot"]) + (SIBLING, "code", "remotes/dependency"), + (INSIDE, "code", "remotes/Northwind-code.git/dependency"), + (TWO_UP, "code", "dependency"), + # The SPEC leg's own remote, which `./` names from inside. + (INSIDE, "spec", "remotes/Northwind-spec.git/dependency"), + # Where `\` separates nothing, git appends `..\dependency` whole. + (BACKSLASH, "code", + "remotes/dependency" if sys.platform == "win32" + else "remotes/Northwind-code.git/..\\dependency"), +], ids=["dotdot", "dot-slash", "dotdot-dotdot", "spec-leg", "backslash"]) def test_execute_refuses_a_relative_url_its_leg_resolves_elsewhere( - tmp_path, url, there): + tmp_path, url, leg, there): """A source with no remote of its own, as the re-verifier built it: `check` finds it from GitHub's remotes, and `execute --local-remote-dir` from the bare ones it would make. On main both passed, `execute` said `adoption verified`, and the assembly's recursive clone exited 1. Now `execute` exits 2 before the remotes directory exists, and the source - is untouched.""" + is untouched -- from either leg, and for `..\\` as for `../`.""" source = source_with(tmp_path, url) head = git("rev-parse", "HEAD", cwd=source).stdout - plan = planned(source, tmp_path) + plan = planned(source, tmp_path, answers={GITMODULES: leg, UPSTREAM: leg}) assert [code for code, _ in found(check(plan).stderr)] == [RELATIVE] remotes, work = tmp_path / REMOTES, tmp_path / WORK result = execute(plan, work, remotes) @@ -562,7 +573,7 @@ def test_execute_refuses_a_relative_url_its_leg_resolves_elsewhere( assert f"REFUSED {RELATIVE}: 1 finding(s) in " in result.stderr [detail] = [line.split(": ", 1)[1] for line in result.stderr.splitlines() if line.startswith(f" FINDING {RELATIVE}: ")] - assert resolved_from_the_leg(detail) == tmp_path.resolve() / there + assert resolved_from_the_leg(detail, leg) == tmp_path.resolve() / there assert "creating the leg repositories" not in result.stdout assert VERIFIED not in result.stdout assert not remotes.exists(), "a refused plan made the remotes directory" From 8a0adb9a6f16c33fa262b459fc328cd71cf29ad8 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:12:01 +0000 Subject: [PATCH 10/12] adopt-project.py: git's answer for a relative url, resolved against a RELATIVE source remote, is joined to the source's working tree before it is compared, so a source whose `origin` is `../src/Thing.git`, with the legs beside its dependency, no longer has its relative url refused when its recursive clone works (#190) Lane 2's review of b0c959e, P3-2: a source whose `remote.origin.url` is the relative `../src/Thing.git` and whose `.gitmodules` keeps `../dep` in a leg was refused by `execute --local-remote-dir`, though the legs sat beside the dependency and main adopted it into an assembly whose recursive clone worked. git resolves `../dep` against that remote to `../src/dep` and keeps it RELATIVE; `same_as` compared it as written with the leg's absolute answer, and `remote_local_path` names no path for a relative remote, deliberately, so the two never matched. What git does with it (git 2.43.0, probed here): `git submodule init` stores `../src/dep`, and `git submodule update` opens it from the TOP of the working tree, even when run from a subdirectory. So `_default_remote` now also returns that working tree for a remote that is a path and not an absolute one -- git's own test for the answers it leaves relative -- and `_resolved` joins git's answer to it. The join is not normalised: `same_as` asks the disk (`realpath`) whether the two name one directory, which is where a `..` after a symlink is decided. The finding says why its path is joined: `(its remote `origin`, relative to its working tree)`. The review suggested anchoring the BASE first, `normpath(source.path / url)`. That differs from git wherever git's arithmetic on the relative spelling is not normpath's: with `origin` `.` git resolves `../dep` to `dep`, the working tree's own `dep`, where the anchored base gives the directory beside it; with `../src/Thing.git/.` it gives `../src/Thing.git/dep`; and one `..` too many is git's `fatal: cannot strip one component off url '..'`, which an anchored base would resolve. Joining git's own answer keeps every one of those as git has it. Tests (tests/test_adopt_relative_submodule_url.py): - `test_a_relative_origin_resolves_from_the_source_working_tree`, with the origin spelled relative and absolute: under the legs' directory nothing is refused; with the legs elsewhere it is refused, the finding naming `/Thing/../remotes/dependency` for the relative spelling and saying why, and the absolute spelling exactly as before; - `test_a_relative_url_its_leg_resolves_alike_adopts_and_clones` gains `[relative-origin]`: `execute` verifies, and the assembly's recursive clone and `bootstrap.py` exit 0 with the dependency in the leg. Each part reverted on its own, over the whole file (30 tests): no join, 2 failed (both `[relative-origin]` rows); the `which` clause dropped, 1 failed; joined to the process's cwd, 2 failed; the clause for every local path, 1 failed (`[absolute-origin]`); the join normalised, 1 failed. Unmutated, 30 passed. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 76 +++++++++++++++------- tests/test_adopt_relative_submodule_url.py | 50 +++++++++++++- 2 files changed, 98 insertions(+), 28 deletions(-) diff --git a/adopt-project.py b/adopt-project.py index 23644b49..0e97a087 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -99,7 +99,7 @@ COMMIT_RE, NEUTRAL_PRODUCT_OWNER, PROJECT_ID_RE, TREE_DIGEST_DEFINITION, VISIBILITY_CHOICES, NamingPolicy, Refusal, YamlError, accepts_role, checked_value, git_out, load_yaml, parse_yaml, redacted, - remote_key, remote_local_path, tree_digest, + remote_is_a_path, remote_key, remote_local_path, tree_digest, ) from shape_materialize import ( # noqa: E402 ADOPT_MAKEFILE_BLOCK, RULESET_HINT, SHAPE_REPOSITORY, @@ -1958,9 +1958,10 @@ def same_as(self, other: "_Resolution") -> bool: `git@github.com:Org/Thing.git` and a leg's is `https://github.com/org/...`. It compares two filesystem paths by their text alone, so they are then asked of this machine, whose - symlinks may make two spellings one directory -- the order - `same_repository_here` asks them in, so the disk can add an answer - and never take one away. + symlinks -- and the `..` of a relative answer joined to the source's + working tree (`_default_remote`) -- may make two spellings one + directory: the order `same_repository_here` asks them in, so the + disk can add an answer and never take one away. """ if self.url is None or other.url is None: return False @@ -1996,20 +1997,35 @@ def _kept_relative_urls(entries: list, tree: list, and RELATIVE_URL_RE.match(registered.url(path))] -def _default_remote(source: Source) -> tuple[str, str]: - """`(base, which)`: what git resolves the source's relative urls against, - and how a finding names it. git's rule -- the checked-out branch's - remote, else `origin`; that remote's url, else, with none, the - repository's own directory -- asked of the branch being ADOPTED, which a - clone of the source checks out, and not of whatever branch this working - copy happens to have checked out.""" +def _default_remote(source: Source) -> tuple[str, str, Path | None]: + """`(base, which, working_tree)`: what git resolves the source's relative + urls against, how a finding names it, and where the clone opens git's + answer when that answer is a relative path. git's rule -- the + checked-out branch's remote, else `origin`; that remote's url, else, + with none, the repository's own directory -- asked of the branch being + ADOPTED, which a clone of the source checks out, and not of whatever + branch this working copy happens to have checked out. + + A RELATIVE REMOTE GIVES A RELATIVE ANSWER. Against `../src/Thing.git`, + `git submodule init` resolves `../dependency` to `../src/dependency` and + keeps it so, and `git submodule update` opens that from the top of the + working tree. Compared as git leaves it, it equalled no leg's answer, + and a layout whose clone works was refused (#190, P3-2). So + `working_tree` is the source's own for such a remote -- a path, and not + an absolute one (`remote_local_path` names absolute paths only), which + is git's own test for the answers it leaves relative -- and None for + every other base.""" name = _config_value(source.path, f"branch.{source.branch}.remote") \ or "origin" url = _config_value(source.path, f"remote.{name}.url") - if url is not None: - return url, f"its remote {_spelled(name)}" - return (source.path.as_posix(), - f"its own directory, as it has no remote {_spelled(name)}") + if url is None: + return (source.path.as_posix(), + f"its own directory, as it has no remote {_spelled(name)}", + None) + if remote_is_a_path(url) and remote_local_path(url) is None: + return (url, f"its remote {_spelled(name)}, relative to its working " + "tree", source.path) + return url, f"its remote {_spelled(name)}", None def _config_value(repository: Path, key: str) -> str | None: @@ -2020,11 +2036,13 @@ def _config_value(repository: Path, key: str) -> str | None: return None if got.returncode else _git_path(got.stdout.rstrip(b"\x00")) -def _resolved(base: str, urls: list[str]) -> list[_Resolution]: +def _resolved(base: str, urls: list[str], + working_tree: Path | None = None) -> list[_Resolution]: """Each of `urls` as `git submodule init` resolves it in a repository whose `origin` is `base`, in a scratch repository made for the purpose - and removed after it. A url git cannot resolve is what git said: that is - where the clone fails.""" + and removed after it; and, given a `working_tree`, git's relative answer + joined to it, which is what the clone opens (`_default_remote`). A url + git cannot resolve is what git said: that is where the clone fails.""" scratch = Path(tempfile.mkdtemp(prefix="openreposhape-url-")) try: run(["git", "init", "-q", str(scratch)]) @@ -2041,20 +2059,27 @@ def _resolved(base: str, urls: list[str]) -> list[_Resolution]: f"{GITLINK_MODE},{oid},u{index}"], cwd=scratch) # One `init` per url, because git stops at the first it cannot # resolve and each finding names its own. - return [_resolution_of(scratch, f"u{index}") + return [_resolution_of(scratch, f"u{index}", working_tree) for index in range(len(urls))] finally: shutil.rmtree(scratch, ignore_errors=True) -def _resolution_of(scratch: Path, name: str) -> _Resolution: - """`name`'s url, as `git submodule init` copies it into the config.""" +def _resolution_of(scratch: Path, name: str, + working_tree: Path | None) -> _Resolution: + """`name`'s url, as `git submodule init` copies it into the config, and + joined to `working_tree` when there is one.""" init = subprocess.run(["git", "submodule", "init", "-q", "--", name], cwd=scratch, capture_output=True, check=False) if init.returncode: return _Resolution(None, _one_line(init.stderr.decode( errors="replace"))) - return _Resolution(_config_value(scratch, f"submodule.{name}.url")) + url = _config_value(scratch, f"submodule.{name}.url") + if working_tree is None or url is None: + return _Resolution(url) + # Not normalised: `..` after a symlink is the disk's to resolve, and + # `same_as` asks the disk (`realpath`) for exactly that. + return _Resolution((working_tree / url).as_posix()) def _relative_url_findings(plan: Plan, source: Source, tree: list, @@ -2081,11 +2106,12 @@ def _relative_url_findings(plan: Plan, source: Source, tree: list, return [], [] remote = _remote_urls(checked_value("org", plan.get("org")), plan.names(), local_remote_dir, source)[leg] - base, which = _default_remote(source) + base, which, working_tree = _default_remote(source) urls = [url for _, _, url in kept] found, alike = [], [] - for (path, entry, url), here, there in zip(kept, _resolved(base, urls), - _resolved(remote, urls)): + for (path, entry, url), here, there in zip( + kept, _resolved(base, urls, working_tree), + _resolved(remote, urls)): if here.same_as(there): alike.append(f"{_spelled(url)} of {_spelled(path)}, {there}") continue diff --git a/tests/test_adopt_relative_submodule_url.py b/tests/test_adopt_relative_submodule_url.py index 65405768..dda2bdf3 100644 --- a/tests/test_adopt_relative_submodule_url.py +++ b/tests/test_adopt_relative_submodule_url.py @@ -76,6 +76,9 @@ ORG, OTHER_ORG = "testorg", "otherorg" #: Where each test's `execute` makes its leg remotes. REMOTES = "remotes" +#: The source's remote beside them, as `git remote add origin +#: ../remotes/Thing.git` writes it from `/Thing`: RELATIVE. +RELATIVE_ORIGIN = f"../{REMOTES}/{THING_REMOTE}" #: The code leg's GitHub remote, as `check` resolves from it. CODE_LEG = "https://github.com/{org}/Northwind-code.git" #: The code leg's bare remote, as `execute --local-remote-dir` names it. @@ -528,6 +531,42 @@ def test_one_directory_through_a_symlink_is_one_repository(adopter, assert refusal is not None and refusal.code == RELATIVE +@pytest.mark.parametrize("relative", [True, False], + ids=["relative-origin", "absolute-origin"]) +def test_a_relative_origin_resolves_from_the_source_working_tree( + adopter, tmp_path, relative): + """git resolves `../dependency` against the RELATIVE origin + `../remotes/Thing.git` to `../remotes/dependency`, keeps it relative, + and the clone opens it from the source's working tree: the directory + beside the bare legs `execute --local-remote-dir` would make there, + which name it too, so nothing is refused, as on main. With the legs + anywhere else it names another repository and is refused, and the + finding names the directory git opens and says why. The same remote + spelled absolute resolves to an absolute path, said as before.""" + remotes = tmp_path / REMOTES + origin = RELATIVE_ORIGIN if relative else str(remotes / THING_REMOTE) + source = source_with(tmp_path, SIBLING, origin, remotes / "dependency") + plan = planned(source, tmp_path) + assert refusal_of(adopter, plan, tmp_path, remotes) is None + + elsewhere = tmp_path / "elsewhere" + refusal = refusal_of(adopter, plan, tmp_path, elsewhere) + assert refusal is not None and refusal.code == RELATIVE + [_, finding] = refusal.detail.splitlines() + if relative: + which = f"{ITS_ORIGIN}, relative to its working tree" + opened = (tmp_path.resolve() / "Thing" + / f"../{REMOTES}/dependency").as_posix() + else: + # git keeps the base's own separators and joins with `/`. + which, opened = ITS_ORIGIN, f"{remotes}/dependency" + assert (f"from the source, at {adopter._spelled(origin)} ({which}), to " + f"{adopter._spelled(opened)}, and from the code leg, at ") \ + in finding, finding + assert resolved_from_the_leg(finding) \ + == elsewhere.resolve() / "dependency" + + def execute(plan: Path, work: Path, remotes: Path): return run_script(ADOPT, "execute", "--plan", str(plan), "--yes", "--local-remote-dir", str(remotes), @@ -616,7 +655,9 @@ def test_the_remediation_adopts_and_clones(tmp_path): (SIBLING, THING_REMOTE, "."), # Two levels: the source's remote and the legs in sibling directories. (TWO_UP, f"sources/{THING_REMOTE}", "legs"), -], ids=["dotdot", "dotdot-dotdot"]) + # The first layout with the remote written relative to the source. + (SIBLING, RELATIVE_ORIGIN, "."), +], ids=["dotdot", "dotdot-dotdot", "relative-origin"]) def test_a_relative_url_its_leg_resolves_alike_adopts_and_clones( tmp_path, url, origin, legs): """(a) end to end. The source's `origin` and the legs are local, in a @@ -625,10 +666,13 @@ def test_a_relative_url_its_leg_resolves_alike_adopts_and_clones( resolves from the ones it makes, adopts and verifies, and the assembly's `git clone --recurse-submodules` and `bootstrap.py` both exit 0 with the dependency checked out in the leg, whose `.gitmodules` still says what - the source committed.""" + the source committed. An `origin` beginning `..` is written as it + stands, relative to the source's working tree; any other is a path in + the remotes directory.""" remotes = tmp_path / REMOTES dependency = remotes / "dependency" - source = source_with(tmp_path, url, str(remotes / origin), dependency) + source = source_with(tmp_path, url, origin if origin.startswith("..") + else str(remotes / origin), dependency) plan = planned(source, tmp_path) assert [code for code, _ in found(check(plan).stderr)] == [RELATIVE] result = execute(plan, tmp_path / WORK, remotes / legs) From c2b53020d5f16d2957afa56f5aebe735988ab982 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:18:14 +0000 Subject: [PATCH 11/12] repo_shape.py: `remote_key` takes a port off a url's host and reads `ssh.github.com` as `github.com`, so `ssh://git@github.com:22/...` and `ssh://git@ssh.github.com:443/...` key like `git@github.com:...` and a source cloned that way no longer has its relative submodule url refused (#190) Lane 2's review of b0c959e, P3-4: `remote_key` read the `host:port` colon of a `://` url as git's scp separator, so `ssh://git@github.com:22/testorg/dep` keyed as `github.com/22/testorg/dep`, and GitHub's documented SSH-over-the-HTTPS- port spelling `ssh://git@ssh.github.com:443/testorg/dep` as `ssh.github.com/443/testorg/dep`. `adopt-project.py`'s relative-url check compares keys and nothing else, so `check` found, and `execute` refused, `../dep` in a source whose `origin` was spelled that way, in the plan's own organisation, where main said `plan ok`. It failed safe; it was still a false refusal for a GitHub spelling. `remote_key` now takes `:` off the host of a `://` url before it looks for the scp colon (`REMOTE_PORT_RE`), and maps `ssh.github.com` to `github.com` (`REMOTE_HOST_ALIASES`). Only a url with a scheme has a port: in the scp spelling `git@host:22/Repo.git` the `22` is the path's and stays, and a drive's colon has no digits after it, so `file:///D:/...` still keys as `D:\...` does. Over 24 spellings, the 8 that change are the port and `ssh.github.com` ones, each to its port-free `github.com` key; paths, drives, file urls, bare `Org/Repo` and scp keys are unchanged. `same_repository` asks the same keys, and its forge-tail match had hid this for an `Owner/Repo`; at a host's top level it had not, and `ssh://git@host:2222/Repo.git` and `git@host:Repo.git` are now one repository to `make siblings` and the doctor as well. THIS IS A SHAPE COPY. `scripts/repo_shape.py` is copied into every assembly root and family holder and digest-pinned there, so each one's next `update-shape.py check` shows `upstream-changed scripts/repo_shape.py`, as after #192's `load_yaml` (22c9fdd) and #157's `same_repository` (71cf5de). That is the right place for it: the project's own copy runs `same_repository` (`siblings.py`), and the fix is to the rule, not a helper only this checkout calls. No test pins its digest: digests are computed from the bytes copied, and like those two commits this one changes no pin. Tests: - tests/test_shape_doctor.py, `test_a_port_and_githubs_ssh_host_are_ not_part_of_a_remotes_key`, beside the one-repository table: six port and `ssh.github.com` spellings key as the scp one; a top-level repository with a port is `same_repository` with its scp spelling; the scp `:22/` path, another repository and another host do not key alike; a drive's file url still keys as the drive path. - tests/test_adopt_relative_submodule_url.py, `test_check_keeps_a_relative_url_its_github_leg_resolves_alike` gains `[ssh-port]` and `[ssh-github-443]`: no finding and the NOTE in the plan's own organisation, found in another. Each half reverted, over the new unit test, the one-repository table and the five `check` rows (7 tests): main's `remote_key` fails 3 (the unit test, `[ssh-port]`, `[ssh-github-443]`); the port kept, the same 3; no `ssh.github.com` alias, 2 (the unit test, `[ssh-github-443]`); a port taken off the scp spelling's path too, 1 (the unit test). The table passes under every one, as it must: no row of it changes. The three `check` rows that existed say exactly what they said. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- scripts/repo_shape.py | 28 +++++++++++++++++-- tests/test_adopt_relative_submodule_url.py | 21 +++++++++----- tests/test_shape_doctor.py | 32 ++++++++++++++++++++++ 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/scripts/repo_shape.py b/scripts/repo_shape.py index 1b07e105..369ee144 100644 --- a/scripts/repo_shape.py +++ b/scripts/repo_shape.py @@ -1794,6 +1794,17 @@ def repo_basename(repository: str) -> str: #: draws for a value on a command line, for the same reason. REMOTE_DRIVE_RE = re.compile(r"^[A-Za-z]:$") +#: A `:` after the HOST of a `://` url: where its transport +#: connects, never which repository it reaches. Only such a url has one; in +#: git's scp spelling `host:22/Repo.git` the `22` is the path's, and stays. +#: Digits are required, so a drive's bare colon is never read as one. +REMOTE_PORT_RE = re.compile(r":\d+$") + +#: A host that is another door into the repositories of the host it maps +#: to: GitHub's SSH-over-the-HTTPS-port endpoint, which GitHub's own +#: documentation spells `ssh://git@ssh.github.com:443/Org/Repo.git`. +REMOTE_HOST_ALIASES = {"ssh.github.com": "github.com"} + def _remote_body(url: str) -> str: """A remote with its scheme, its credential and its trailing `/` gone. @@ -1907,11 +1918,24 @@ def remote_key(url: str) -> str: was written in. It is asked of the remote AS IT WAS WRITTEN, because the scp colon this function is about to turn into a `/` is the one thing that tells a Windows drive from a host. + + A PORT COMES OFF TOO, AND `ssh.github.com` IS `github.com` (#190). + `ssh://git@github.com:22/Org/Repo.git` and GitHub's documented + `ssh://git@ssh.github.com:443/Org/Repo.git` are + `git@github.com:Org/Repo.git` reached through another door; with the + port's colon taken for the scp one they keyed as `github.com/22/org/repo`, + which no other spelling of that repository does, and a caller comparing + keys -- `adopt-project.py`'s relative-url check -- called them two. """ text = _remote_body(url) fold_case = _remote_names_a_host(url) - if ":" in text.split("/", 1)[0]: # the scp spelling's one separator - text = text.replace(":", "/", 1) + first, slash, rest = text.partition("/") + if REMOTE_SCHEME_RE.match(url.strip()): # only a url's host has a port + first = REMOTE_PORT_RE.sub("", first) + # The scp spelling's one separator, `host:path`, becomes a `/`. + host, slash, rest = (first.replace(":", "/", 1) + slash + + rest).partition("/") + text = REMOTE_HOST_ALIASES.get(host.lower(), host) + slash + rest if text.lower().endswith(".git"): text = text[:-4] return (text.lower() if fold_case else text).strip("/") diff --git a/tests/test_adopt_relative_submodule_url.py b/tests/test_adopt_relative_submodule_url.py index dda2bdf3..b9ce9886 100644 --- a/tests/test_adopt_relative_submodule_url.py +++ b/tests/test_adopt_relative_submodule_url.py @@ -230,15 +230,22 @@ def passes(result) -> None: (HTTPS_ORIGIN, GITHUB_DEPENDENCY), (SCP_ORIGIN, "git@github.com:testorg/dependency"), (SCP_OTHER_CASE, "git@github.com:TestOrg/dependency"), -], ids=["https", "scp", "scp-other-case"]) + # A port, and GitHub's SSH-over-the-HTTPS-port host: GitHub all the same. + ("ssh://git@github.com:22/testorg/Thing.git", + "ssh://git@github.com:22/testorg/dependency"), + ("ssh://git@ssh.github.com:443/testorg/Thing.git", + "ssh://git@ssh.github.com:443/testorg/dependency"), +], ids=["https", "scp", "scp-other-case", "ssh-port", "ssh-github-443"]) def test_check_keeps_a_relative_url_its_github_leg_resolves_alike( adopter, tmp_path, origin, resolved): """(a) Legs in the source's own organisation sit beside its dependency on GitHub, so `../dependency` names one repository from both, however - the source's `origin` spells it: no finding, and the NOTE says which - remotes `check` asked. In another organisation it names another - repository, and that is found, with the submodule, its url, the leg's - remote and both resolutions, and the remediation once after it.""" + the source's `origin` spells it, a port and `ssh.github.com` included: + no finding, and the NOTE says which remotes `check` asked. In another + organisation it names another repository, and that is found, with the + submodule, its url, the leg's remote and both resolutions, and the + remediation once after it. A url with a scheme is printed through + `redacted`, which hides its user too (`ssh://***@github.com:22/...`).""" source = source_with(tmp_path, SIBLING, origin) same = check(planned(source, tmp_path)) passes(same) @@ -248,8 +255,8 @@ def test_check_keeps_a_relative_url_its_github_leg_resolves_alike( result = check(planned(source, tmp_path, OTHER_ORG)) assert result.returncode == 1, result.stderr + result.stdout assert found(result.stderr) == [(RELATIVE, moved( - adopter, SIBLING, origin, ITS_ORIGIN, f"to `{resolved}`", - CODE_LEG.format(org=OTHER_ORG), + adopter, SIBLING, adopter.redacted(origin), ITS_ORIGIN, + f"to `{adopter.redacted(resolved)}`", CODE_LEG.format(org=OTHER_ORG), "to `https://github.com/otherorg/dependency`"))] assert notes(result.stdout) == [] # Said once, after the findings, and it is not #166's remediation. diff --git a/tests/test_shape_doctor.py b/tests/test_shape_doctor.py index ac50577e..a672b549 100644 --- a/tests/test_shape_doctor.py +++ b/tests/test_shape_doctor.py @@ -1827,6 +1827,38 @@ def test_the_doctor_and_siblings_run_ONE_definition_of_one_repository( "https://host/IRRS.git", "https://host/x") == "https://host/IRRS.git" +def test_a_port_and_githubs_ssh_host_are_not_part_of_a_remotes_key(): + """`remote_key` TAKES A PORT, AND `ssh.github.com`, OFF THE HOST (#190). + + `ssh://git@github.com:22/Acme/Widget.git`, and + `ssh://git@ssh.github.com:443/Acme/Widget.git` as GitHub documents SSH + over the HTTPS port, are the scp `git@github.com:Acme/Widget.git` + reached another way, and keyed as `github.com/22/acme/widget`. + `same_repository`'s forge-tail match hid that for an `Owner/Repo`, but + not for a repository at a host's top level, and `adopt-project.py` + compares keys and nothing else, so it refused a source cloned that way. + """ + scp = repo_shape.remote_key("git@github.com:Acme/Widget.git") + for url in ("ssh://git@github.com:22/Acme/Widget.git", + "ssh://git@ssh.github.com:443/Acme/Widget.git", + "ssh://git@SSH.GitHub.com:443/Acme/Widget.git", + "https://github.com:443/Acme/Widget.git", + "https://x-access-token:token@github.com:443/Acme/Widget.git", + "git@ssh.github.com:Acme/Widget.git"): + assert repo_shape.remote_key(url) == scp, url + assert repo_shape.same_repository("ssh://git@host:2222/Repo.git", + "git@host:Repo.git") + # ...and nothing that is not a port: the scp spelling has none, so its + # `22` is a path component; a drive's colon carries no digits; and + # another repository, or another host, is still another. + for url in ("git@github.com:22/Acme/Widget.git", + "ssh://git@github.com:22/Acme/Other.git", + "ssh://git@gitlab.com:22/Acme/Widget.git"): + assert repo_shape.remote_key(url) != scp, url + assert repo_shape.remote_key("file:///E:/mirrors/Widget.git") \ + == repo_shape.remote_key("E:\\mirrors\\Widget.git") + + def test_a_symlinked_prefix_is_ONE_directory_to_both_callers(tmp_path, siblings): """THE PREFIX CASE #157 MOVED OUT OF THE RULE AND INTO THE CALLERS. From c0b1ebbe24c84233e7ea7efb6cb2893d707065a1 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:04:15 +0000 Subject: [PATCH 12/12] tests: #190's relative-url finding is expected with the submodule and its entry between backticks, as #212 spells every path a finding names, and #215's `.gitmodules` values are registered under an absolute url, so the two merges of main no longer fail 22 tests (#190) Test-only. adopt-project.py is unchanged, and so is every other file. #212 (5751460, merged by GitHub's update-branch at 8ce5614) made `_as_entry` spell the submodule and its entry. #190's `plan-submodule-relative-url` names the submodule through `_as_entry`, so it now prints `` the submodule `upstream/dependency` (entry `upstream/`) ``. The code is right under main's rule. The branch's expectations were raw: `moved()` and the assertion in `test_execute_resolves_again_from_the_remotes_it_would_create`. Both now put backticks around the path and the entry, as main's own tests/test_adopt_submodules.py does. Every other path and url #190's finding and NOTE name already goes through `_spelled`, and every url through `redacted` as well. Where git cannot resolve a url, the finding quotes git's own words on one line, as main's `plan-gitmodules-unreadable` does. #215 (c2ff2b0, merged at 18e161e) registers its submodule with `url = ../b` (`REGISTRATION`), in a source with no remote, with the tip gitlink kept in code. Under #190 that is a finding of its own: from the source `../b` is the directory beside it, and from the code leg it is `https://github.com/testorg/b`. The tests that expect `check` 0 or an adoption beside a value git reads then failed. They are about values, not urls, so the url is now absolute, `https://example.invalid/b`, which git copies as it stands. This is what #190 already did for #166's `registration()`. The one bytes row that still writes `url = ../b` is left as it is: its `.gitmodules` is one git cannot read, so no url is read from it. Revert: with this commit reverted, at 18e161e, the two files give 22 failed, 197 passed. - tests/test_adopt_relative_submodule_url.py, 12: `test_check_keeps_a_relative_url_its_github_leg_resolves_alike[https, scp, scp-other-case, ssh-port, ssh-github-443]`, `test_a_source_with_no_remote_resolves_from_its_own_directory`, `test_each_relative_url_is_resolved_on_its_own`, `test_a_credential_in_the_source_remote_is_never_printed` and `test_execute_resolves_again_from_the_remotes_it_would_create[https, scp, scp-other-case, dotdot-dotdot-other-org]`. These same 12 are the only failures at 8ce5614 over 26 targeted files (12 failed, 1376 passed, 2 skipped). - tests/test_adopt_gitmodules_values.py, 10: `test_check_passes_a_value_git_reads[*-tip]` (5), `test_the_finding_names_the_key_git_dies_on[under-a-suspicious-name, under-an-empty-name]`, `test_the_issue_shape_is_refused_and_the_repaired_source_then_adopts` and `test_a_value_git_reads_still_adopts[*-tip]` (2). Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_gitmodules_values.py | 7 ++++++- tests/test_adopt_relative_submodule_url.py | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_adopt_gitmodules_values.py b/tests/test_adopt_gitmodules_values.py index 255edaa3..5a6eca0d 100644 --- a/tests/test_adopt_gitmodules_values.py +++ b/tests/test_adopt_gitmodules_values.py @@ -38,7 +38,12 @@ UNREADABLE = "plan-gitmodules-unreadable" REMOTES = "remotes" -REGISTRATION = '[submodule "b"]\n\tpath = vendor/b\n\turl = ../b\n' +#: An ABSOLUTE url, which git copies as it stands. A relative one kept in a +#: leg is resolved from the leg too (#190), and in a source with no remote +#: `../b` is the directory beside it, which no GitHub leg reaches: a finding +#: of its own beside every value here, which is not what these tests ask. +REGISTRATION = ('[submodule "b"]\n\tpath = vendor/b\n' + '\turl = https://example.invalid/b\n') #: A source whose plan asks about `.gitmodules`, and about `vendor/` where #: the tip holds the submodule: `specs/` and `src/` classify themselves and #: `README.md` stays in the root. diff --git a/tests/test_adopt_relative_submodule_url.py b/tests/test_adopt_relative_submodule_url.py index b9ce9886..e03c5a35 100644 --- a/tests/test_adopt_relative_submodule_url.py +++ b/tests/test_adopt_relative_submodule_url.py @@ -177,10 +177,11 @@ def moved(adopter, url: str, base: str, which: str, here: str, remote: str, """The detail of the finding, for the submodule at `path` kept in code. `here` and `there` are git's resolutions, spelled; every other url is spelled here as the tool spells it, which on Windows doubles each - backslash of a path.""" + backslash of a path. The submodule and its entry are between backticks, + as every path a finding names is (#212).""" spelled = adopter._spelled - return (f"the submodule {path} (entry {entry}) has leg: code, and its " - f"url in the source's {GITMODULES} is the relative " + return (f"the submodule `{path}` (entry `{entry}`) has leg: code, and " + f"its url in the source's {GITMODULES} is the relative " f"{spelled(url)}, which git resolves against the remote of the " f"repository holding that file: from the source, at " f"{spelled(base)} ({which}), {here}, and from the code leg, at " @@ -472,7 +473,7 @@ def test_execute_resolves_again_from_the_remotes_it_would_create( "GitHub's:") leg = adopter._spelled(str(remotes.resolve() / CODE_BARE)) assert finding.startswith(f" FINDING {RELATIVE}: the submodule " - f"{DEPENDENCY} (entry {UPSTREAM})"), finding + f"`{DEPENDENCY}` (entry `{UPSTREAM}`)"), finding assert f", and from the code leg, at {leg}, to " in finding assert refusal.remediation.endswith( "\n" + adopter.RELATIVE_URL_REMEDIATION)