scripts/shape_materialize.py and adopt-project.py: a --pin-source that is no clone's top level is refused pin-source-not-a-clone, and check finds a recorded one before a partial clone's NOTE, so a folder inside a clone no longer records the digest of part of its tree, or of nothing (#800)

adopt-project.py

@@ -312,6 +312,9 @@
 #: How `plan`'s refusals of a `--pin-source` begin their remediation.
 PLAN_NOT_WRITTEN = "no plan was written."
 from repo_shape import SHA256_RE, repo_basename  # noqa: E402
+# Whether a recorded `--pin-source` is a clone's top level, asked as `plan`
+# asks it (#800).
+from shape_materialize import pin_source_not_top_level  # noqa: E402
 # One reading of an `origin`, shared with `shape-doctor.py` (#521).
 from repo_shape import origin_is_a_path, url_repository  # noqa: E402
 
@@ -336,6 +339,9 @@
 PIN_DIGEST_MISMATCH = "plan-pin-digest-mismatch"
 PIN_SOURCE_UNREADABLE = "plan-pin-source-unreadable"
 PIN_NOT_A_COMMIT = "plan-pin-not-a-commit"
+#: And about a clone `plan` read a digest from that is no clone's top level,
+#: which `plan` now refuses as `pin-source-not-a-clone` (#800).
+PIN_SOURCE_NOT_A_CLONE = "plan-pin-source-not-a-clone"
 #: How `plan` records where it read a digest (`_as_planned`): from a clone on
 #: this machine, `--pin-source` as typed, or from the forge.
 LOCAL_CLONE_RE = re.compile(r"local-clone \((?P<path>.+)\)")
@@ -694,10 +700,17 @@ def _pin_findings(plan: "Plan", tree: list) -> list[str]:
 # directory `check` or `execute` runs in, as `plan` read it from the one it
 # ran in, and every sentence naming one says where that is
 # (`_read_from_here`). Anything else at the path that cannot answer -- a
-# commit it lacks, a directory that is no clone, a file -- is
-# `plan-pin-source-unreadable`, as `validate-pins.py` finds a source it reads
-# that cannot answer. `check` and `execute` ask this through
-# `_entry_findings`, the one list.
+# commit it lacks, a file -- is `plan-pin-source-unreadable`, as
+# `validate-pins.py` finds a source it reads that cannot answer. `check` and
+# `execute` ask this through `_entry_findings`, the one list.
+#
+# A DIRECTORY THAT IS NO CLONE'S TOP LEVEL is asked about FIRST (#800), before
+# a partial clone's NOTE: a folder inside a clone, or a directory that is no
+# clone at all, is `plan-pin-source-not-a-clone`, as `plan` refuses it
+# `pin-source-not-a-clone`. In a folder `git ls-tree` lists only what is
+# under it, so the digest recomputed there was the wrong one `plan` read,
+# and `check` said it recomputes; inside a partial clone it said only that
+# it was not asked. The question reads no object, so it reads no network.
 
 #: Why a pin's digest `gh api` read was not rechecked.
 NOT_RECHECKED_GH = "`plan` read it with `gh api`, and `check` reads no network"
@@ -808,6 +821,30 @@ def _recomputed_pin_finding(said: str, values: dict) -> str | None:
             f"(`neutral-pin-digest-mismatch`): {_pin_fix(repository)}.")
 
 
+def _not_a_top_level_finding(said: str, how: str,
+                             repository: str) -> str | None:
+    """The FINDING for a pin whose `digest_source:` names a directory that is
+    no clone's top level (#800), asked as `plan` asks a `--pin-source`
+    (`pin_source_not_top_level`); None for a top level, and for a path that
+    is no directory or that cannot be entered, which the questions after
+    this one name as they always did."""
+    clone = _recorded_clone(how)
+    try:
+        found = (pin_source_not_top_level(Path(clone))
+                 if clone is not None and Path(clone).is_dir() else None)
+    except OSError:
+        return None
+    if found is None:
+        return None
+    why, top = found
+    fix = (f"re-run `plan` with `--pin-source {_shown(str(top))}`, its top "
+           "level" if top is not None else _pin_fix(repository))
+    return (f"FINDING {PIN_SOURCE_NOT_A_CLONE}: {said} records that `plan` "
+            f"read its digest from {_shown(clone)}{_read_from_here(clone)}, "
+            f"which is not a clone's top level: {why}. `plan` refuses such a "
+            f"--pin-source as `pin-source-not-a-clone`: {fix}.")
+
+
 def _pin_digest_findings(plan: "Plan") -> tuple[list[str], list[str]]:
     """`(findings, notes)` for each pin `execute` would write: a FINDING for
     a digest the clone `plan` read it from recomputes otherwise at the
@@ -818,8 +855,10 @@ def _pin_digest_findings(plan: "Plan") -> tuple[list[str], list[str]]:
     notes: list[str] = []
     for said, _, values in _read_pins(plan)[0]:
         how = values["PIN_DIGEST_SOURCE"]
-        why = _why_not_rechecked(how)
-        found = None if why else _recomputed_pin_finding(said, values)
+        found = _not_a_top_level_finding(said, how, values["PIN_REPOSITORY"])
+        why = None if found else _why_not_rechecked(how)
+        if not (found or why):
+            found = _recomputed_pin_finding(said, values)
         if why:
             notes.append(f"NOTE {said}: digest NOT rechecked here: {why}. The "
                          "root's own `validate-pins.py` recomputes it "
@@ -13375,7 +13414,10 @@ def main(argv: list[str] | None = None) -> int:
                            "Given with no --pin to read it, the flag is "
                            "refused as `pin-source-unused`; an empty value "
                            "is refused as `pin-source-no-path`, and `.` "
-                           "means this directory.")
+                           "means this directory. `pin-source-not-a-clone` "
+                           "refuses any other path than a clone's top level, "
+                           "or a bare clone's, where a --pin's digest is "
+                           "read.")
     plan.add_argument("--allow-empty-leg", action="append", default=[],
                       choices=EXTRACTED_LEGS,
                       help="record in the plan that this leg having NO path "

docs/cli.md

@@ -366,7 +366,9 @@ options:
                         --pin and no --referent-chain to read it, it is
                         refused as `pin-source-unused`; an empty value is
                         refused as `pin-source-no-path`, and `.` means this
-                        directory.
+                        directory. A --pin's digest is read only at a clone's
+                        top level, or a bare clone's: any other path is
+                        refused as `pin-source-not-a-clone`.
   --work-dir WORK_DIR   where the working trees are built (default: a
                         temporary directory)
   --no-next-steps       leave out the closing NEXT STEPS (the clone, the cd
@@ -516,6 +518,9 @@ options:
                         Given with no --pin to read it, the flag is refused as
                         `pin-source-unused`; an empty value is refused as
                         `pin-source-no-path`, and `.` means this directory.
+                        `pin-source-not-a-clone` refuses any other path than a
+                        clone's top level, or a bare clone's, where a --pin's
+                        digest is read.
   --allow-empty-leg {spec,code}
                         record in the plan that this leg having NO path is
                         intended, so it may be SEEDED from the shape's

scaffold-project.py

@@ -381,7 +381,10 @@ def main(argv: list[str] | None = None) -> int:
                              "--pin and no --referent-chain to read it, it "
                              "is refused as `pin-source-unused`; an empty "
                              "value is refused as `pin-source-no-path`, and "
-                             "`.` means this directory.")
+                             "`.` means this directory. A --pin's digest is "
+                             "read only at a clone's top level, or a bare "
+                             "clone's: any other path is refused as "
+                             "`pin-source-not-a-clone`.")
     parser.add_argument("--work-dir", type=Path, default=None,
                         help="where the working trees are built "
                              "(default: a temporary directory)")

scripts/shape_materialize.py

@@ -75,6 +75,9 @@
 # How a `--pin-source` that is not printable is named, the one spelling the
 # scaffold's chain WARNING gives it too (#610).
 from repo_shape import shown_path  # noqa: E402
+# What git answers about where a `--pin-source` is, and why, in its own
+# words, where it cannot (#800).
+from repo_shape import git_out  # noqa: E402
 # What the Free-plan secrets hint goes by (#660): GitHub's three
 # visibilities, and the hint's own words, which stay in the pinned copy.
 from repo_shape import FREE_PLAN_HINT, VISIBILITY_CHOICES  # noqa: E402
@@ -2994,6 +2997,8 @@ def pin_digest_from_source(source: Path, commit: str, repository: str) -> str:
     try:
         return _digest_from_pin_source(source, commit)
     except (Refusal, OSError) as exc:
+        if isinstance(exc, Refusal) and exc.code == PIN_SOURCE_NOT_A_CLONE:
+            raise  # its own refusal, which names the top level (#800)
         why = (exc.detail if isinstance(exc, Refusal)
                else exc.strerror or str(exc))
         named = shown_source(source)
@@ -3065,18 +3070,108 @@ def shown_source(source) -> str:
 
 def _digest_from_pin_source(source: Path, commit: str) -> str:
     """`tree_digest` over `source`, with a path that is no directory said
-    in `validate-pins.py`'s words (#339). Any other `OSError`, from `stat()`
-    or from git's `cwd=`, is left for `pin_digest_from_source` to name."""
+    in `validate-pins.py`'s words (#339), and a directory that is no
+    clone's top level refused `pin-source-not-a-clone` before anything is
+    digested (#800). Any other `OSError`, from `stat()` or from git's
+    `cwd=`, is left for `pin_digest_from_source` to name."""
     try:
         mode = source.stat().st_mode
     except (FileNotFoundError, NotADirectoryError):
         raise Refusal("pin-source-missing", "no such directory") from None
     if not stat.S_ISDIR(mode):
         raise Refusal("pin-source-not-a-directory",
                       "it is a file, not a directory")
+    _refuse_unless_a_clone_top_level(source)
     return tree_digest(source, commit)
 
 
+# ---------------------------------------------------------------------------
+# A `--pin-source` is a clone's top level (#800)
+# ---------------------------------------------------------------------------
+#
+# `tree_digest` runs `git ls-tree -r` IN the `--pin-source`, and there, with
+# no `--full-tree`, git lists only the part of the tree under the folder it
+# runs in. So a folder inside a clone was read as a clone: `glass/sub`
+# recorded the digest of `sub/` alone, a folder the commit does not hold
+# recorded the sha256 of nothing, and `adopt-project.py plan` and the
+# scaffold, `--dry-run` alike, wrote it with no word; `check` recomputed the
+# same number there, or, inside a partial clone, NOTEd that it did not ask.
+#
+# SO GIT IS ASKED FIRST, before any digest is read: a working tree's top
+# level, a linked worktree's among them, has an empty `rev-parse
+# --show-prefix` outside the git directory, and a bare clone's git directory
+# is the path itself. Any other path is `pin-source-not-a-clone`, naming the
+# top level git finds, or saying that it finds none. What a top level digests
+# is unchanged, byte for byte, and `repo_shape.tree_digest`, a pinned shape
+# copy, is not changed. `adopt-project.py check` asks a recorded path the
+# same question (`pin_source_not_top_level`) before a partial clone's NOTE.
+
+#: The refusal of a `--pin-source` that is no clone's top level.
+PIN_SOURCE_NOT_A_CLONE = "pin-source-not-a-clone"
+#: What it says to pass where git names no top level to pass.
+NO_TOP_LEVEL_REMEDIATION = (
+    "Remediation: pass --pin-source the top level of a clone of the pinned "
+    "product (`git rev-parse --show-toplevel` prints it in that clone's "
+    "working tree), or drop --pin-source and let `gh api` read the forge.")
+
+
+def _rev_parse(source: Path, *asked: str) -> str:
+    """What `git rev-parse <asked>` prints in `source`, its last line break
+    dropped and a path in it as `sys.argv` holds one. A git that fails is
+    `git_out`'s refusal, naming the folder and git's reason, spelled."""
+    printed = git_out(["rev-parse", *asked], cwd=source, binary=True)
+    return os.fsdecode(printed).removesuffix("\n")
+
+
+def pin_source_not_top_level(source: Path) -> tuple[str, Path | None] | None:
+    """None where `source` is a clone's top level (#800). Otherwise `(why,
+    top)`: why it is not, and the top level git finds for it, or None where
+    git finds none to pass. A `source` `subprocess` cannot enter raises the
+    `OSError` of `cwd=`, which is the caller's to name.
+
+    The answers come in one question where they can: `--show-prefix` is
+    printed last, so a folder whose name holds a line break is read whole."""
+    try:
+        bare, inside, prefix = _rev_parse(
+            source, "--is-bare-repository", "--is-inside-git-dir",
+            "--show-prefix").split("\n", 2)
+        if bare == "true":
+            top = Path(_rev_parse(source, "--absolute-git-dir"))
+            if os.path.samefile(top, source):
+                return None
+            return (f"it is a folder inside the bare clone at "
+                    f"{shown_source(top)}", top)
+        if inside == "true":
+            gitdir = Path(_rev_parse(source, "--absolute-git-dir"))
+            return (f"it is inside {shown_source(gitdir)}, the git directory "
+                    "of a clone, where git finds no top level", None)
+        if not prefix:
+            return None
+        top = Path(_rev_parse(source, "--show-toplevel"))
+    except Refusal as exc:
+        return f"git finds no clone there: {exc.detail}", None
+    return (f"it is a folder inside the clone at {shown_source(top)}, and a "
+            "digest read there covers only what the commit holds under that "
+            "folder", top)
+
+
+def _refuse_unless_a_clone_top_level(source: Path) -> None:
+    """`pin-source-not-a-clone` for a `source` that is no clone's top level
+    (`pin_source_not_top_level`), naming the top level to pass instead."""
+    found = pin_source_not_top_level(source)
+    if found is None:
+        return
+    why, top = found
+    raise Refusal(
+        PIN_SOURCE_NOT_A_CLONE,
+        f"--pin-source {shown_source(source)} is not a clone's top level: "
+        f"{why}",
+        NO_TOP_LEVEL_REMEDIATION if top is None else
+        f"Remediation: pass --pin-source {shown_source(top)}, the clone's "
+        "top level, where a pin's digest is read over its commit's whole "
+        "tree.")
+
+
 def _resolved_pin_source(source: Path) -> Path:
     """`source` made absolute, or AS TYPED when it cannot be (#339).