adopt-project.py: answered_paths: also records each entry a leg's own walk puts elsewhere, so a directory split by hand out of an extension-majority fold no longer reads MISPLACED on the doctor's first run after adoption verified (#801)

adopt-project.py

@@ -11331,6 +11331,29 @@ def answered_path_record(policy: PathPolicy, entries: list, tree: list,
     again by `ask_about_submodules` and `ask_about_refused_paths` as `plan`
     asks them. A question of the walk above is recorded as before.
 
+    AND AGAINST WHAT EACH LEG'S OWN WALK SAYS (#801). The doctor walks a
+    leg, not the source, and `walk` folds a directory by the files under it
+    in THAT leg. A directory the source's walk folds, split by hand, leaves
+    part of it in each leg, and a part may fold another way or not at all:
+    `engine/` is 3 of 4 `.py`, exactly the 75% `extension_majority:`, so it
+    folds to code; with `engine/c.py` answered `spec`, the code leg's
+    `engine/` is 2 of 3, which does not fold, and there `engine/x.md` is
+    spec by its own extension. Answers against their extensions can fold a
+    directory in a leg as the source's walk did not, over an entry beside
+    them left as `plan` wrote it. So `walk` is run again over each leg's
+    own files -- the paths `_answered_legs` keeps in it, gitlinks left out,
+    as the doctor runs it over the leg -- and an entry covering a path that
+    walk does not put in that leg is recorded too, whether the walk asks of
+    the path or puts it elsewhere. A row is added only where the doctor's
+    walk of that leg disputes the path. A plan left as `plan` wrote it,
+    over a source with no gitlink, gets none: each entry is whole in the
+    leg `plan` gave it, and no directory above one can fold to another leg
+    there, since at most a quarter of an entry's files read as one. Before
+    it walks, the doctor also sets aside the files every repository carries
+    (`everywhere_patterns`: names at the top of a leg, and `.github/`),
+    which change how no other directory folds, and a name no report can
+    carry, which can: this walk keeps both, as the one above does.
+
     A SOURCE WITH NO SUCH ENTRY GETS "" and so the `project.yaml` it always
     got, byte for byte.
     """
@@ -11343,16 +11366,33 @@ def answered_path_record(policy: PathPolicy, entries: list, tree: list,
             policy, [(path, size) for path, _, _, size in tree]), tree),
         tree)}
     answered = _answered_legs(entries, [path for path, _ in files])
+    in_its_leg = _walked_in_each_leg(policy, files, answered)
     kept = sorted({entry: leg for path, (leg, entry) in answered.items()
                    if leg in EXTRACTED_LEGS and entry != GITMODULES
-                   and leg != _walked_leg(called, path)
-                   and (_walked_leg(called, path) is None
-                        or leg != _walked_leg(shown, path))}.items())
+                   and (leg != in_its_leg.get(path)
+                        or (leg != _walked_leg(called, path)
+                            and (_walked_leg(called, path) is None
+                                 or leg != _walked_leg(shown, path))))
+                   }.items())
     return _record_block(ANSWER_RECORD_PREAMBLE,
                          [(entry, leg, entry) for entry, leg in kept],
                          entries, commit)
 
 
+def _walked_in_each_leg(policy: PathPolicy, files: list,
+                        answered: dict) -> dict:
+    """`{path: leg}` for each of `files` that `answered` keeps in a leg:
+    the leg `walk` gives it over THAT leg's files alone, as the doctor
+    walks the leg after the split; None for a question (#801)."""
+    out: dict = {}
+    for role in EXTRACTED_LEGS:
+        held = [(path, size) for path, size in files
+                if path in answered and answered[path][0] == role]
+        called = {entry.path: entry.leg for entry in walk(policy, held)}
+        out.update((path, _walked_leg(called, path)) for path, _ in held)
+    return out
+
+
 def _walked_leg(called: dict, path: str) -> str | None:
     """The leg `walk` gave `path`, from `called` (`{entry path: leg}`): its
     own entry's, or the folded directory's above it; None for a question.

tests/test_answered_paths_per_leg.py

@@ -0,0 +1,309 @@
+# SPDX-License-Identifier: Apache-2.0
+"""A directory split by hand is recorded as each leg's own walk reads it,
+so the doctor's first run after `adoption verified` reads it COMPLIANT
+(#801: #457's L1-83).
+
+THE DEFECT, on main bf1f099b. The source's `engine/` holds `a.py`, `b.py`,
+`c.py` and `x.md`: 3 of 4 files `.py`, exactly the policy's
+`extension_majority: 0.75`, so `plan` proposes `engine/` -> code. Split by
+hand -- `engine/c.py` into the spec leg, the other three into the code leg,
+each with a `resolution:` -- the plan passed `check`, and `execute` said
+`adoption verified`. The doctor's first run then said `MISPLACED (1 path)`,
+exit 1: `code/engine/x.md: spec by rule extension-majority`.
+`answered_path_record` walked the whole source, where `engine/` folds to
+code and `x.md` needs no row. The doctor walks the code leg, where
+`engine/` is 2 of 3 `.py` (67%), and reads each file by its own extension.
+
+The same split reaches the same disagreement with `engine/c.py` dropped or
+kept in the root, in a directory the majority groups to spec, nested one
+directory down, and as a question where the file left over has an
+extension no class names: a `note` there, and not `ok`. So do answers
+against their extensions, which fold a directory in a leg that the source
+did not fold, over a file beside them left as `plan` wrote it.
+
+THE FIX. `answered_path_record` also walks each leg's own files, as the
+doctor walks that leg, and records each entry kept in a leg that covers a
+path that walk puts elsewhere. The doctor is unchanged, and a plan left as
+`plan` wrote it gets the record main writes, byte for byte.
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+import pytest
+
+from conftest import REPO, clone_just_pushed, git, run_script
+from test_doctor_recorded_answers import placement, tree_of
+from test_shape_doctor import (  # noqa: F401
+    ORG, adoption_module, commit_all, needs_filter_repo, standard,
+)
+
+sys.path.insert(0, str(REPO / "scripts"))
+from repo_shape import parse_yaml  # noqa: E402
+
+BLOB = "100644"
+COMPLIANT = "COMPLIANT"
+SOURCE_COMMIT = "ef" * 20
+#: The `resolution:` every entry split out of a directory carries.
+BY_HAND = "split by hand"
+ENGINE = "engine/"
+A_PY = "engine/a.py"
+B_PY = "engine/b.py"
+C_PY = "engine/c.py"
+X_MD = "engine/x.md"
+Q_ZZZ = "engine/q.zzz"
+NOTES = "engine/notes/"
+C_MD = "notes/c.md"
+Y_PY = "notes/y.py"
+THE_ISSUE = "the-issue"
+#: `stuff/`, which no rule names: 2 `.py` and 3 `.md`, which do not fold.
+STUFF = "stuff/"
+STUFF_PY = ("stuff/a.py", "stuff/b.py")
+STUFF_MD = ("stuff/m1.md", "stuff/m2.md", "stuff/m3.md")
+#: The record's key, and the doctor's for what it answered.
+ANSWERED = "answered_paths"
+UTF8 = "utf-8"
+#: Beside every source below: a front door the root keeps and a
+#: requirement the spec leg keeps, so that neither leg is seeded.
+BESIDE = {"README.md": "# Thing\n", "requirements/r.md": "# Requirement\n"}
+#: Three `.py` files under `engine/`.
+PY_FILES = {A_PY: "A = 1\n", B_PY: "B = 1\n", C_PY: "C = 1\n"}
+#: The issue's `engine/`: 3 of 4 files `.py`, exactly the 75% majority.
+ENGINE_FILES = {**PY_FILES, X_MD: "# Notes\n"}
+#: The two `.py` files the split leaves in the code leg.
+KEPT = ((A_PY, "code"), (B_PY, "code"))
+
+#: `{case: (the source's files beside BESIDE, the entry `plan` writes for
+#: them, the entries a person splits it into, the rows `answered_paths:`
+#: needs)}`. Each split is one the doctor's first run disputed on main.
+SPLITS = {
+    THE_ISSUE: (ENGINE_FILES, ENGINE,
+                  (*KEPT, (C_PY, "spec"), (X_MD, "code")),
+                  [(C_PY, "spec"), (X_MD, "code")]),
+    "the-other-file-dropped": (ENGINE_FILES, ENGINE,
+                               (*KEPT, (C_PY, "drop"), (X_MD, "code")),
+                               [(X_MD, "code")]),
+    "the-other-file-in-the-root": (ENGINE_FILES, ENGINE,
+                                   (*KEPT, (C_PY, "root"), (X_MD, "code")),
+                                   [(X_MD, "code")]),
+    # 3 of 4 `.md`, so `notes/` goes to spec; one `.md` taken to code
+    # leaves `notes/y.py` read as code in the spec leg.
+    "a-spec-majority-split": (
+        {"notes/a.md": "# A\n", "notes/b.md": "# B\n", C_MD: "# C\n",
+         Y_PY: "Y = 1\n"}, "notes/",
+        (("notes/a.md", "spec"), ("notes/b.md", "spec"), (C_MD, "code"),
+         (Y_PY, "spec")),
+        [(C_MD, "code"), (Y_PY, "spec")]),
+    # One directory down: the row the code leg needs is the directory's.
+    "a-nested-split": (
+        {**PY_FILES, NOTES + "x.md": "# Notes\n"}, ENGINE,
+        (*KEPT, (C_PY, "spec"), (NOTES, "code")),
+        [(C_PY, "spec"), (NOTES, "code")]),
+    # An extension no class names is a question once it is outvoted no
+    # more: a `note` on main, not MISPLACED.
+    "a-question-left-over": (
+        {**PY_FILES, Q_ZZZ: "q\n"}, ENGINE,
+        (*KEPT, (C_PY, "spec"), (Q_ZZZ, "code")),
+        [(C_PY, "spec"), (Q_ZZZ, "code")]),
+    # No directory split: `plan` lists `stuff/`'s five files, `b.py` is
+    # answered spec and the three `.md` code, against their extension, and
+    # `a.py` is left as `plan` wrote it. The code leg's `stuff/` is then 3
+    # of 4 `.md` and folds to spec, over `a.py` too.
+    "answers-fold-a-file-left-as-written": (
+        {**dict.fromkeys(STUFF_PY, "P = 1\n"),
+         **dict.fromkeys(STUFF_MD, "# M\n")}, STUFF,
+        ((STUFF_PY[0], "code"), (STUFF_PY[1], "spec"),
+         *((path, "code") for path in STUFF_MD)),
+        [(STUFF_PY[0], "code"), (STUFF_PY[1], "spec"),
+         *((path, "code") for path in STUFF_MD)]),
+}
+
+#: The controls, which main records the same way: a split whose rest still
+#: clears 75%, and `engine/` as `plan` wrote it.
+CONTROLS = {
+    "the-rest-clears-the-majority": (
+        {**ENGINE_FILES, "engine/d.py": "D = 1\n"}, ENGINE,
+        (*KEPT, (C_PY, "spec"), ("engine/d.py", "code"), (X_MD, "code")),
+        [(C_PY, "spec")]),
+    "left-as-plan-wrote-it": (ENGINE_FILES, ENGINE, ((ENGINE, "code"),), []),
+}
+CASES = {**SPLITS, **CONTROLS}
+
+
+def record_of(adopt, files: dict, parts: tuple) -> str:
+    """`answered_path_record` over BESIDE and `files`, for a plan keeping
+    the README in the root, `requirements/` in the spec leg, and each
+    `(path, leg)` of `parts` with BY_HAND as its `resolution:`."""
+    entries = [{"path": "README.md", "leg": "root"},
+               {"path": "requirements/", "leg": "spec"}]
+    entries += [{"path": path, "leg": leg, "resolution": BY_HAND}
+                for path, leg in parts]
+    tree = tree_of(*((path, BLOB) for path in {**BESIDE, **files}))
+    return adopt.answered_path_record(
+        adopt.PathPolicy.load(adopt.PATH_POLICY), entries, tree,
+        SOURCE_COMMIT)
+
+
+def rows_of(text: str) -> list:
+    """`(path, leg)` for each `answered_paths:` row `text` holds, or []."""
+    return [(row["path"], row["leg"])
+            for row in ((parse_yaml(text) or {}).get(ANSWERED) or [])]
+
+
+# --- the record `execute` writes ---------------------------------------------
+
+def test_the_issues_split_records_the_file_the_code_legs_walk_disputes(
+        standard):
+    """#801's LINE, THE RECORD. In the code leg `engine/` is 2 of 3 `.py`,
+    so `engine/x.md` reads as spec there: it is recorded in the code leg
+    with the plan's reason and commit, beside `engine/c.py` in the spec
+    leg, which main already recorded."""
+    block = record_of(adoption_module(standard), ENGINE_FILES,
+                      SPLITS[THE_ISSUE][2])
+    assert parse_yaml(block)[ANSWERED] == [
+        {"path": path, "leg": leg, "resolution": BY_HAND,
+         "source_commit": SOURCE_COMMIT}
+        for path, leg in ((C_PY, "spec"), (X_MD, "code"))], block
+
+
+@pytest.mark.parametrize("case", list(CASES))
+def test_each_entry_a_legs_own_walk_puts_elsewhere_is_recorded(standard,
+                                                               case):
+    """ONE ROW PER ENTRY KEPT IN A LEG THAT COVERS A PATH THAT LEG'S OWN
+    WALK PUTS ELSEWHERE, as the doctor walks the leg: a file left outvoted
+    no more, a directory one level down, a question. Where the rest still
+    clears the majority, or `engine/` is kept whole, the record is main's.
+    """
+    files, _, parts, rows = CASES[case]
+    assert rows_of(record_of(adoption_module(standard), files, parts)) == rows
+
+
+def test_a_plan_left_as_plan_wrote_it_gets_mains_record_byte_for_byte(
+        standard):
+    """BYTE FOR BYTE. `engine/` kept whole beside a question answered
+    (`examples/`) and an answer against a rule (`docs/`, #655): the block
+    is main's, its comment and those two rows and nothing for `engine/`;
+    and `engine/` alone gets "", so the `project.yaml` it always got."""
+    adopt = adoption_module(standard)
+    files = {**ENGINE_FILES, "docs/index.md": "# Docs\n",
+             "examples/golden/case.json": '{"case": 1}\n'}
+    answered = (("docs/", "code"), ("examples/", "spec"))
+    block = record_of(adopt, files, ((ENGINE, "code"), *answered))
+    assert block == "\n".join(adopt.ANSWER_RECORD_PREAMBLE) + "\n" + "".join(
+        f'  - path: "{path}"\n    leg: {leg}\n    resolution: "{BY_HAND}"\n'
+        f'    source_commit: "{SOURCE_COMMIT}"\n' for path, leg in answered)
+    assert record_of(adopt, ENGINE_FILES, ((ENGINE, "code"),)) == ""
+
+
+# --- end to end: `execute`, then the doctor ---------------------------------
+
+def split_by_hand(text: str, entry: str, parts: tuple) -> str:
+    """The plan `text` with the block -- its `- path:` line and every line
+    indented under it -- of `entry`, or of each entry `plan` listed under
+    it, replaced by one entry per `(path, leg)` of `parts` where the first
+    stood, each with BY_HAND as its `resolution:`."""
+    blocks = re.findall(rf"(?m)^  - path: {re.escape(entry)}.*\n(?:    .*\n)*",
+                        text)
+    assert blocks, text
+    out = text.replace(blocks[0], "".join(
+        f'  - path: {path}\n    leg: {leg}\n    resolution: "{BY_HAND}"\n'
+        for path, leg in parts), 1)
+    for block in blocks[1:]:
+        out = out.replace(block, "", 1)
+    return out
+
+
+def adopt_split(standard: Path, base: Path, case: str) -> dict:
+    """CASES[case]'s source committed, planned, its entry split by hand --
+    or, where the case keeps it whole, left exactly as `plan` wrote it --
+    executed into local remotes, and cloned from the split branch."""
+    files, entry, parts, _ = CASES[case]
+    source = base / "Thing"
+    for name, body in {**BESIDE, **files}.items():
+        (source / name).parent.mkdir(parents=True, exist_ok=True)
+        (source / name).write_text(body, encoding=UTF8)
+    git("init", "-q", "-b", "main", ".", cwd=source)
+    commit_all(source, "a directory its extensions fold")
+    tool = standard / "adopt-project.py"
+    plan = base / "plan.yaml"
+    planned = run_script(tool, "plan", "--out", str(plan), "--source",
+                         str(source), "--org", ORG, "--project", "Northwind",
+                         "--elected-on", "2026-10-17", "--elected-by", "T")
+    assert planned.returncode == 0, planned.stderr
+    text = plan.read_text(encoding=UTF8)
+    if [path for path, _ in parts] == [entry]:
+        assert f"  - path: {entry}\n    leg: {parts[0][1]}\n" in text, text
+    else:
+        plan.write_bytes(split_by_hand(text, entry, parts).encode(UTF8))
+    executed = run_script(tool, "execute", "--yes", "--plan", str(plan),
+                          "--work-dir", str(base / "work"),
+                          "--local-remote-dir", str(base / "remotes"))
+    assert (executed.returncode, "adoption verified" in executed.stdout) == (
+        0, True), executed.stdout + executed.stderr
+    cloned = clone_just_pushed(source, base / "clone",
+                               branch="adopt/three-repo-shape")
+    assert cloned.returncode == 0, cloned.stderr
+    return {"clone": base / "clone",
+            "commit": git("rev-parse", "HEAD", cwd=source).stdout.strip()}
+
+
+@pytest.fixture(scope="module")
+def adopted_the_issue(standard, tmp_path_factory) -> dict:
+    """The issue's split, adopted."""
+    return adopt_split(standard, tmp_path_factory.mktemp(THE_ISSUE),
+                       THE_ISSUE)
+
+
+@needs_filter_repo
+def test_the_issues_split_is_compliant_on_the_doctors_first_run(
+        standard, adopted_the_issue):
+    """#801's LINE, END TO END. The split's manifest records `engine/x.md`
+    in the code leg beside `engine/c.py` in the spec leg, and the doctor's
+    first run, which said `MISPLACED (1 path)` on main, reads the adoption
+    COMPLIANT with the row `ok`, the code leg's `engine/x.md` answered by
+    its row."""
+    clone = adopted_the_issue["clone"]
+    manifest = parse_yaml((clone / "project.yaml").read_text(encoding=UTF8))
+    assert manifest[ANSWERED] == [
+        {"path": path, "leg": leg, "resolution": BY_HAND,
+         "source_commit": adopted_the_issue["commit"]}
+        for path, leg in ((C_PY, "spec"), (X_MD, "code"))], manifest
+    result, verdict, row = placement(standard, clone)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert (verdict, row["status"], row["detail"]["misplaced"]) == (
+        COMPLIANT, "ok", []), row
+    assert [(item["path"], item["answered_by"])
+            for item in row["detail"][ANSWERED]] == [
+        ("spec/engine/", [C_PY]), ("code/engine/x.md", [X_MD])], row["detail"]
+
+
+@pytest.fixture(scope="module", params=[case for case in CASES
+                                        if case not in (
+                                            THE_ISSUE,
+                                            "the-other-file-in-the-root")])
+def adopted_by_hand(request, standard, tmp_path_factory) -> dict:
+    """Each other case but the root's, which differs from the drop's only
+    in the root, adopted."""
+    return {"case": request.param,
+            **adopt_split(standard, tmp_path_factory.mktemp(request.param),
+                          request.param)}
+
+
+@needs_filter_repo
+def test_each_split_by_hand_is_compliant_on_the_doctors_first_run(
+        standard, adopted_by_hand):
+    """END TO END, EVERY OTHER WAY IN. Each manifest holds the rows its
+    legs' walks need, and no `answered_paths:` at all where none is
+    needed; the doctor's first run reads each adoption COMPLIANT with the
+    row `ok`."""
+    clone = adopted_by_hand["clone"]
+    text = (clone / "project.yaml").read_text(encoding=UTF8)
+    rows = CASES[adopted_by_hand["case"]][3]
+    assert (rows_of(text), ANSWERED in text) == (rows, bool(rows)), \
+        text
+    result, verdict, row = placement(standard, clone)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert (verdict, row["status"]) == (COMPLIANT, "ok"), row