repo_shape.py and bootstrap.py: COMMIT_RE, SHA256_RE and COMMIT_40 end in \Z, not $, so a pin's commit: or tree_sha256: that ends in a line break is refused as not 40 or 64 hex by every validator and adopt-project.py (#1017)

scripts/repo_shape.py

@@ -65,8 +65,11 @@
 #: Exactly 40 hex, case-insensitive, normalised to lowercase before comparison.
 #: 40 means 40 — an abbreviated oid, a branch name or a tag cannot pass, which
 #: is `neutral-product-pin`'s "a tag can be moved" written as a regex.
-COMMIT_RE = re.compile(r"^[0-9a-fA-F]{40}$")
-SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
+#: BOTH END IN `\Z`, NEVER `$` (#1017): `$` also matches just before a trailing
+#: line feed, so a `commit:` or `tree_sha256:` read as `"<hex>\n"` (a
+#: double-quoted escape, or a block scalar) passed `.match` as well formed.
+COMMIT_RE = re.compile(r"^[0-9a-fA-F]{40}\Z")
+SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}\Z")
 
 #: A project id is the lowercase machine name; the GitHub topic is derived.
 PROJECT_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")

templates/assembly-root/scripts/bootstrap.py

@@ -160,7 +160,9 @@
 #: a dependency again, which is the whole thing the copies exist to avoid.
 SHAPE_UPSTREAM_ENV = "SHAPE_UPSTREAM_PATH"
 SHAPE_UPSTREAM_CACHE = ".shape-upstream-tip"
-COMMIT_40 = re.compile(r"^[0-9a-fA-F]{40}$")
+#: `\Z`, NEVER `$` (#1017): `$` also matches before a trailing line feed, as it
+#: did in `repo_shape.COMMIT_RE`, which this is a copy of.
+COMMIT_40 = re.compile(r"^[0-9a-fA-F]{40}\Z")
 #: Where a remote-tracking ref lives. Only an upstream there holds a branch's
 #: commits on the remote's behalf; an upstream of `.` is another local branch.
 REMOTE_TRACKING = "refs/remotes/"

tests/test_adopt_pins.py

@@ -1246,6 +1246,17 @@ def malformed(glass) -> dict:
                                                digest=digest[:12]),
                          f"{entry} `tree_sha256: '{digest[:12]}'`, which is "
                          "not 64 hex"),
+        # `$` matched just before a trailing line feed (#1017), so the
+        # double-quoted `\n` after the hex, which the plan's reader reads
+        # as a line feed, was a well formed commit and a well formed digest.
+        "commit-line-break": ("pins:\n" + pin_entry(commit=commit + "\\n",
+                                                   digest=digest),
+                              f"{entry} `commit: '{commit}\\n'`, which is "
+                              "not 40 hex"),
+        "digest-line-break": ("pins:\n" + pin_entry(commit=commit,
+                                                   digest=digest + "\\n"),
+                              f"{entry} `tree_sha256: '{digest}\\n'`, which "
+                              "is not 64 hex"),
         "another-product": ("pins:\n" + pin_entry(
                                 repository="opensoft/openPane",
                                 commit=commit, digest=digest),
@@ -1354,9 +1365,9 @@ def cases(glass):
     "old-plan-forged")]
 CASE_IDS += [("plan-pin-malformed", name) for name in (
     "not-a-list", "a-number", "short-commit", "short-digest",
-    "another-product", "no-owner", "no-source", "source-line-break",
-    "source-bare-path", "source-no-space", "source-by-hand",
-    "source-clone-nul", "a-number-commit")]
+    "commit-line-break", "digest-line-break", "another-product", "no-owner",
+    "no-source", "source-line-break", "source-bare-path", "source-no-space",
+    "source-by-hand", "source-clone-nul", "a-number-commit")]
 CASE_IDS += [("plan-pin-collides", name)
              for name in ("case-pair", "shape", "variable-pair")]
 

tests/test_manifest_validator.py

@@ -38,6 +38,29 @@ def test_a_scaffolded_manifest_is_valid(project):
     assert "manifest ok" in result.stdout
 
 
+def shape_of(project) -> dict:
+    return parse_yaml((project / "project.yaml").read_text())["shape"]
+
+
+def test_a_shape_commit_that_ends_in_a_line_break_is_a_finding(project):
+    """`$` matched just before a trailing line feed (#1017), so a
+    `shape.commit` read as the 40 hex and a line feed (the double-quoted
+    `\\n`, or a block scalar) was `manifest ok`."""
+    commit = shape_of(project)["commit"]
+    edit(project, f'commit: "{commit}"', f'commit: "{commit}\\n"')
+    result = validate(project)
+    assert result.returncode == 1, result.stderr
+    assert "manifest-shape-commit" in result.stderr
+
+
+def test_a_shape_digest_that_ends_in_a_line_break_is_a_finding(project):
+    digest = shape_of(project)["digests"]["tree_sha256"]
+    edit(project, f'tree_sha256: "{digest}"', f'tree_sha256: "{digest}\\n"')
+    result = validate(project)
+    assert result.returncode == 1, result.stderr
+    assert "manifest-shape-digest" in result.stderr
+
+
 def test_wrong_kind_is_a_finding(project):
     edit(project, "kind: project-manifest", "kind: project-register")
     result = validate(project)