shape-doctor.py: in a root that lost scripts/repo_shape.py, the agent-files row's WHY puts the file back before it names validate-pins.py to confirm the digest (or says why no command does), and names the standard's copy where the root has none (#776)

shape-doctor.py

@@ -2086,14 +2086,17 @@ def validator_command(shape: Path, used: Path, own_copy: bool, args: list,
 
 
 def lost_repo_shape_cure(ctx: Context, own: str, lost: Path, rerun: str,
-                         again: bool = True) -> tuple[str | None, str]:
+                         again: bool = True,
+                         purpose: str = "") -> tuple[str | None, str]:
     """`(the NEXT, the why)` for a root's own validator `own` that crashed
     because `lost`, the `repo_shape.py` it imports from beside itself, is not
     there, where `rerun` is the line that runs `own` (#697).
 
     `again` says "run it AGAIN" where the doctor ran the gate and it crashed,
     as the validator rows' crash does; the `legs` and `members` rows' lockstep
-    line is one the reader never ran, and says "run it" (#750).
+    line is one the reader never ran, and says "run it" (#750). `purpose` is
+    what the gate is run for, where the row says (", to confirm the digest",
+    #776), and follows "once the file is back".
 
     THE NEXT IS THE CURE, NOT THE VALIDATOR. The row used to print `rerun`,
     and a reader whose own `PYTHONPATH`, or a `.pth` in whose site-packages,
@@ -2117,7 +2120,7 @@ def lost_repo_shape_cure(ctx: Context, own: str, lost: Path, rerun: str,
     """
     relative = lost.relative_to(ctx.root).as_posix()
     then = (f"run `{rerun}`{' again' if again else ''} once the file is "
-            "back; before that, a reader whose own PYTHONPATH or "
+            f"back{purpose}; before that, a reader whose own PYTHONPATH or "
             "site-packages reaches another repo_shape sees it run on that "
             "one, which says nothing about this root's own copy")
     if head_holds(ctx.root, relative):
@@ -6239,6 +6242,25 @@ def own_bootstrap(root: Path, make_target: bool = False) -> bool:
     return re.search(r"(?m)^bootstrap\s*:", makefile) is not None
 
 
+def gate_line(ctx: Context, gate: tuple) -> tuple[Path, bool, str | None]:
+    """`(the copy of `gate` a row names, whether it is the root's own, the
+    line that runs it on the root)`: the line is None where neither copy is
+    readable, or where no `PYTHONPATH` can name the standard's template copy
+    (#278), because a line that cannot run is not printed (#276).
+
+    ONE DEFINITION for the rows that hand a reader the lockstep gate:
+    `lockstep_next`, for the `legs` and `members` rows, and the `agent-files`
+    row's confirm clause (#776). Both asked `validator_for` and
+    `pythonpath_unnamable` in the same words.
+    """
+    used, _, own_copy = validator_for(ctx, *gate)
+    if (not used.is_file()
+            or pythonpath_unnamable(ctx.shape, own_copy) is not None):
+        return used, own_copy, None
+    return used, own_copy, validator_command(
+        ctx.shape, used, own_copy, ["--root", str(ctx.root)])
+
+
 def lockstep_next(ctx: Context, gate: tuple, noun: str) -> tuple:
     """`(the line, why it is the line)` for what NO checkout cures: the
     lockstep gate, built the way the `pins` and `family` rows build their
@@ -6264,12 +6286,9 @@ def lockstep_next(ctx: Context, gate: tuple, noun: str) -> tuple:
     where HEAD no longer holds it. Its `why` says to run the gate once the
     file is back, and the sentence above follows it.
     """
-    used, _, own_copy = validator_for(ctx, *gate)
-    if (not used.is_file()
-            or pythonpath_unnamable(ctx.shape, own_copy) is not None):
+    used, own_copy, command = gate_line(ctx, gate)
+    if command is None:
         return None, None
-    command = validator_command(ctx.shape, used, own_copy,
-                                ["--root", str(ctx.root)])
     said = (f"no submodule update cures a {noun} whose gitlink "
             "or pin is missing or disagrees, or a mount git cannot "
             f"read a HEAD in. {used.name} prints what it finds and "
@@ -9199,6 +9218,46 @@ def pinned_paths(root: Path) -> set:
 LABEL_AGENT_FILES = "agent files"
 
 
+def confirm_digest(ctx: Context) -> str:
+    """The end of the `agent-files` WHY for a pinned copy that is gone: what
+    confirms its digest once it is back (#776).
+
+    THE GATE IS THE ROOT KIND'S, as the validator rows have it: an assembly
+    root's `validate-pins.py` (`PINS_GATE`), a family holder's
+    `validate-family.py` (`FAMILY_GATE`), the root's own copy else the
+    standard's template copy, named as the validator rows name it
+    (`gate_line`). The row runs in both kinds of root, and a holder has no
+    `validate-pins.py`: the assembly-root gate refuses there with a remedy
+    that names a file the holder lacks.
+
+    The sentence used to name the assembly root's own copy whether or not the
+    root had one, and where the root has lost the `scripts/repo_shape.py`
+    that copy imports from beside itself it died on `ModuleNotFoundError` and
+    exited 1, as the validator row says. So there the sentence is #737's: put
+    the file back first, with the cure `lost_repo_shape_cure` gives that row,
+    or, where HEAD does not hold it, why no command does, and the gate after.
+    Where the template copy is named because the root has none of its own, it
+    also reports that missing copy where the pin has a row for it, and exits
+    1, which the sentence says. Where no line can be named (`gate_line`)
+    nothing is said of a gate. Everywhere else it is, byte for byte, what it
+    was.
+    """
+    gate = FAMILY_GATE if ctx.kind == FAMILY else PINS_GATE
+    used, own_copy, command = gate_line(ctx, gate)
+    if command is None:
+        return ""
+    if own_copy and not imports_repo_shape(ctx.shape, used, own_copy):
+        cure, why = lost_repo_shape_cure(
+            ctx, gate[0], repo_shape_for(ctx.shape, used, own_copy), command,
+            again=False, purpose=", to confirm the digest")
+        return ". " + (f"First `{cure}`: {why}" if cure else why)
+    said = f", then `{command}` to confirm the digest"
+    if not own_copy and gate[0] in pinned_paths(ctx.root):
+        said += (f". Until {gate[0]} is back that exits 1, reporting the "
+                 "file as pinned but absent")
+    return said
+
+
 def check_agent_files(ctx: Context) -> Row:
     present = {name: (ctx.root / name).is_file() for name in AGENT_FILES}
     absent = [name for name, there in present.items() if not there]
@@ -9223,10 +9282,8 @@ def check_agent_files(ctx: Context) -> Row:
                f"{quote_arg(first)}")
         why = ("the pin has a "
                "`files:` row for it, so it was deleted rather than never "
-               "copied; restore it from this repository's own history, then "
-               f"`{PYTHON} "
-               f"{quote_arg(ctx.root / 'scripts' / 'validate-pins.py')} "
-               f"--root {quote_arg(ctx.root)}` to confirm the digest")
+               "copied; restore it from this repository's own history"
+               + confirm_digest(ctx))
     elif "AGENTS-shape.md" in absent:
         fix = (f"{PYTHON} {quote_arg(ctx.shape / 'update-shape.py')} "
                f"check --root {quote_arg(ctx.root)} --upstream "

tests/test_doctor_next_is_a_command.py

@@ -110,7 +110,7 @@
 #: Doctor B's crashing validator and unloaded manifest (#278, #298), from the
 #: module that owns them.
 from test_doctor_validator_rows import (
-    crashing, lockstep_lost, lost_gate, unloaded,
+    agent_files_lost, crashing, lockstep_lost, lost_gate, unloaded,
 )
 
 
@@ -259,6 +259,16 @@ def _agent_files_pinned(standard, project, holder, tmp_path):
     return rows_of(doctor(standard, project, "--json"))["agent-files"]
 
 
+def _agent_files_pinned_lost_repo_shape(case):
+    """The builder of the same row where the root's own gate lost its
+    `repo_shape.py`, in an assembly root (`pins`) and in a holder
+    (`family`): its WHY puts the file back before the gate (#776)."""
+    def build(standard, project, holder, tmp_path):
+        root = agent_files_lost(case, project, holder, tmp_path)
+        return rows_of(doctor(standard, root, "--json"))["agent-files"]
+    return build
+
+
 def _agent_files_unpinned(standard, project, holder, tmp_path):
     # No `contracts/shape-pin.yaml` names `AGENTS-shape.md`: the branch
     # that sends the reader to `update-shape.py check` and `apply --add`.
@@ -701,6 +711,16 @@ def build(standard, project, holder, tmp_path):
         _agent_files_pinned,
         "the pin has a `files:` row for it, so it was deleted rather than "
         "never copied; restore it from this repository's own history, then "),
+    "check_agent_files-pinned-lost-repo-shape": (
+        _agent_files_pinned_lost_repo_shape("pins"),
+        "the pin has a `files:` row for it, so it was deleted rather than "
+        "never copied; restore it from this repository's own history. "
+        "First `git -C "),
+    "check_agent_files-pinned-lost-repo-shape-holder": (
+        _agent_files_pinned_lost_repo_shape("family"),
+        "the pin has a `files:` row for it, so it was deleted rather than "
+        "never copied; restore it from this repository's own history. "
+        "First `git -C "),
     "check_agent_files-unpinned": (
         _agent_files_unpinned,
         "no pin row names it, so it is reported upstream-added and taken, on "

tests/test_doctor_validator_rows.py

@@ -40,6 +40,7 @@
 
 import json
 import os
+import re
 import shlex
 import shutil
 import subprocess
@@ -996,6 +997,218 @@ def test_a_lockstep_line_whose_repo_shape_left_head_too_is_no_line(
         assert said in row["why"], (said, row["why"])
 
 
+# --- #776: the agent-files WHY names no gate that dies on the lost file -----
+#
+# The `agent-files` row of a pinned copy that is gone says to restore it and
+# then run `validate-pins.py` "to confirm the digest": a line that dies on
+# `import repo_shape` where the root lost that file, as the rows above did,
+# and one a family holder has no copy of. The holder's own gate is
+# `validate-family.py`, as its validator row has it.
+
+#: The copy the row is about: pinned, so its absence is a FINDING.
+AGENT_COPY = "AGENTS-shape.md"
+#: What the row's WHY says before it names a gate, which no case changes.
+AGENT_FILES_WHY = ("the pin has a `files:` row for it, so it was deleted "
+                   "rather than never copied; restore it from this "
+                   "repository's own history")
+#: The two kinds of root the row runs in, by the gate row each one's gate is.
+AGENT_GATES = ["pins", "family"]
+
+
+def argv_of(line: str) -> list:
+    """A line the row printed, as the argv it runs: `shlex` on POSIX, where a
+    path with a blank is quoted, and `split` on Windows, where the paths here
+    need no quoting and a backslash is no escape."""
+    return line.split() if os.name == "nt" else shlex.split(line)
+
+
+def agent_files_lost(case: str, project: Path, holder: dict, tmp_path: Path,
+                     committed: bool = False) -> Path:
+    """The root whose own `case` gate lost its `scripts/repo_shape.py`
+    (`committed`, or not) and whose pinned `AGENTS-shape.md` is deleted from
+    the tree, which HEAD still holds."""
+    root = lost_gate(case, project, holder, tmp_path, committed)
+    (root / AGENT_COPY).unlink()
+    return root
+
+
+def agent_files_row(standard: Path, root: Path) -> dict:
+    row = rows_of(doctor(standard, root, "--json"))["agent-files"]
+    assert row["status"] == "FINDING", row
+    return row
+
+
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_agent_files_why_of_a_root_that_lost_repo_shape_names_the_cure(
+        standard, project, holder, tmp_path, case):
+    """The row's NEXT is what it was. The gate it names to confirm the digest
+    dies on the import before the file is back; the WHY says to put the file
+    back first, and the three lines it names, run in its order -- the NEXT,
+    the cure, the gate -- end with the gate passing. In a holder the gate is
+    `validate-family.py`."""
+    root = agent_files_lost(case, project, holder, tmp_path)
+    row = agent_files_row(standard, root)
+    module = doctor_module(standard)
+    restore = module.restore_from_head(root, LOST)
+    gate = rerun_of(standard, root, case)
+    assert row["next"] == (f"git -C {module.quote_arg(root)} checkout -- "
+                           f"{AGENT_COPY}"), row["next"]
+    assert row["why"].startswith(AGENT_FILES_WHY + ". "), row["why"]
+    assert f". First `{restore}`: " in row["why"], row["why"]
+    assert f"`{gate}` once the file is back, to confirm" in row["why"]
+    dies = run_argv(argv_of(gate))
+    assert dies.returncode == 1, dies.stdout + dies.stderr
+    assert "ModuleNotFoundError" in dies.stdout + dies.stderr
+    for line in (row["next"], restore, gate):
+        ran = run_argv(argv_of(line))
+        assert ran.returncode == 0, (line, ran.stdout + ran.stderr)
+
+
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_agent_files_why_where_head_lacks_repo_shape_says_why_and_no_cure(
+        standard, project, holder, tmp_path, case):
+    """HEAD does not hold a file whose deletion was COMMITTED, so the WHY has
+    no cure to offer: it says where the file comes back from, and the gate to
+    run then, to confirm the digest. The NEXT is still the copy's own."""
+    root = agent_files_lost(case, project, holder, tmp_path, committed=True)
+    row = agent_files_row(standard, root)
+    gate = rerun_of(standard, root, case)
+    assert row["next"] is not None, row
+    assert "First `" not in row["why"], row["why"]
+    for said in (f". HEAD does not hold {LOST}", "parent of the commit",
+                 f"`{gate}` once the file is back, to confirm the digest"):
+        assert said in row["why"], (said, row["why"])
+
+
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_agent_files_why_of_a_root_that_keeps_repo_shape_names_its_gate(
+        standard, project, holder, tmp_path, case):
+    """The control: the root's own gate and the same two arguments, spelled
+    out here. For an assembly root this is the WHY it always was, byte for
+    byte; a holder is told its own `validate-family.py`, which exits 0 when
+    run after the NEXT, and not an assembly-root gate it has no copy of."""
+    root = own_root("family" if case == "family" else "manifest", project,
+                    holder, tmp_path)
+    (root / AGENT_COPY).unlink()
+    module = doctor_module(standard)
+    own = root / TEMPLATE_FALLBACK[case]["own"]
+    row = agent_files_row(standard, root)
+    assert row["why"] == (
+        f"{AGENT_FILES_WHY}, then `{module.PYTHON} {module.quote_arg(own)} "
+        f"--root {module.quote_arg(root)}` to confirm the digest"), row["why"]
+    for line in (row["next"], rerun_of(standard, root, case)):
+        ran = run_argv(argv_of(line))
+        assert ran.returncode == 0, (line, ran.stdout + ran.stderr)
+
+
+def without_its_own_gate(case: str, project: Path, holder: dict,
+                         tmp_path: Path, pin_row: bool = True) -> Path:
+    """The root with no gate of its own to name, and `AGENTS-shape.md` gone.
+    The shape pin keeps its `files:` row for the gate, as every scaffolded
+    root's does, unless `pin_row` is False."""
+    root = own_root("family" if case == "family" else "manifest", project,
+                    holder, tmp_path)
+    own = TEMPLATE_FALLBACK[case]["own"]
+    git("rm", "-q", "--", own, cwd=root)
+    (root / AGENT_COPY).unlink()
+    if not pin_row:
+        pin = root / "contracts" / "shape-pin.yaml"
+        row = re.compile(rb'(?m)^  - path: ' + re.escape(own.encode())
+                         + rb'\n    sha256: "[0-9a-f]{64}"\n')
+        edited, removed = row.subn(b"", pin.read_bytes())
+        assert removed == 1, own
+        pin.write_bytes(edited)
+    return root
+
+
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_agent_files_why_of_a_root_with_no_gate_names_the_standards(
+        standard, project, holder, tmp_path, case):
+    """A root with no gate of its own has the standard's template copy named,
+    with the `PYTHONPATH` it needs, as the validator rows name it, and not a
+    file the root does not have; and the WHY says that copy also reports the
+    gate the root lacks, the pin having a row for it."""
+    root = without_its_own_gate(case, project, holder, tmp_path)
+    shape = standard.resolve()
+    spec = TEMPLATE_FALLBACK[case]
+    line = doctor_module(standard).validator_command(
+        shape, shape / spec["template"], False, ["--root", root])
+    assert agent_files_row(standard, root)["why"] == (
+        f"{AGENT_FILES_WHY}, then `{line}` to confirm the digest. Until "
+        f"{spec['own']} is back that exits 1, reporting the file as pinned "
+        "but absent")
+
+
+@WINDOWS_SKIP
+@pytest.mark.skipif(shutil.which("sh") is None, reason="needs sh")
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_template_line_the_agent_files_why_names_reports_the_missing_gate(
+        standard, project, holder, tmp_path, case):
+    """Pasted after the NEXT, from `/`, the standard's line does confirm the
+    restored copy, and exits 1 for the gate the root lacks, as the WHY says."""
+    root = without_its_own_gate(case, project, holder, tmp_path)
+    row = agent_files_row(standard, root)
+    restored = pasted_from_elsewhere(row["next"], tmp_path)
+    assert restored.returncode == 0, restored.stdout + restored.stderr
+    shape = standard.resolve()
+    line = doctor_module(standard).validator_command(
+        shape, shape / TEMPLATE_FALLBACK[case]["template"], False,
+        ["--root", root])
+    gate = pasted_from_elsewhere(line, tmp_path)
+    whole = gate.stdout + gate.stderr
+    assert gate.returncode == 1, whole
+    assert "shape-copy-missing" in whole, whole
+    assert TEMPLATE_FALLBACK[case]["own"] in whole, whole
+
+
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_agent_files_why_says_nothing_of_a_gate_the_pin_has_no_row_for(
+        standard, project, holder, tmp_path, case):
+    """The root has no gate of its own, and the shape pin has no `files:` row
+    for it either: the standard's copy has nothing to report missing, so the
+    WHY names the line and carries no sentence about an exit 1 for the gate."""
+    root = without_its_own_gate(case, project, holder, tmp_path, pin_row=False)
+    shape = standard.resolve()
+    line = doctor_module(standard).validator_command(
+        shape, shape / TEMPLATE_FALLBACK[case]["template"], False,
+        ["--root", root])
+    assert agent_files_row(standard, root)["why"] == (
+        f"{AGENT_FILES_WHY}, then `{line}` to confirm the digest")
+
+
+@WINDOWS_SKIP
+@pytest.mark.skipif(shutil.which("sh") is None, reason="needs sh")
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_template_line_for_a_gate_the_pin_has_no_row_for_passes(
+        standard, project, holder, tmp_path, case):
+    """Pasted after the NEXT, from `/`, the standard's line confirms the
+    restored copy and exits 0: nothing is pinned and absent, so the WHY was
+    right to say nothing of an exit 1."""
+    root = without_its_own_gate(case, project, holder, tmp_path, pin_row=False)
+    row = agent_files_row(standard, root)
+    restored = pasted_from_elsewhere(row["next"], tmp_path)
+    assert restored.returncode == 0, restored.stdout + restored.stderr
+    shape = standard.resolve()
+    line = doctor_module(standard).validator_command(
+        shape, shape / TEMPLATE_FALLBACK[case]["template"], False,
+        ["--root", root])
+    gate = pasted_from_elsewhere(line, tmp_path)
+    whole = gate.stdout + gate.stderr
+    assert gate.returncode == 0, whole
+    assert "shape-copy-missing" not in whole, whole
+
+
+@pytest.mark.parametrize("case", AGENT_GATES)
+def test_the_agent_files_why_names_no_gate_where_no_line_can_be_named(
+        standard, project, holder, tmp_path, case):
+    """Neither copy of the gate is readable: nothing is said of one, as the
+    lockstep rows print no line there (#276)."""
+    hollow = standard_under(standard, tmp_path, "hollow")
+    (hollow / TEMPLATE_FALLBACK[case]["template"]).unlink()
+    root = without_its_own_gate(case, project, holder, tmp_path)
+    assert agent_files_row(hollow, root)["why"] == AGENT_FILES_WHY
+
+
 # --- #278: the lockstep line of the `legs` and `members` rows ----------------
 
 def assert_no_pythonpath_line(*streams: str) -> None: