update-shape.py, family.py and bump-leg.py: the branch HEAD is on is read as the bytes git wrote, so a branch ending in U+00A0 is no longer read as its neighbour, and a name that is not UTF-8 is refused by name before anything is written (#765)

scripts/bump-leg.py

@@ -1141,14 +1141,45 @@ def current_branch(root: Path) -> str | None:
     trailing U+00A0, U+2028 or U+0085, which a branch name may end in, and
     read a root on `feat<U+00A0>` as `feat`, a branch of its own that the
     push line then pushed instead of the bump.
+
+    READ AS THE BYTES GIT WROTE (#765). Exactly one newline and exactly
+    `refs/heads/` come off the bytes, and the rest is decoded with
+    `surrogateescape`, as `family.py` reads its holder's branch. Decoded
+    with `errors="replace"`, a branch `caf<0xE9>` read as `caf<U+FFFD>`, a
+    branch nobody has, and the push line named that on both sides and
+    failed pasted (`src refspec ... does not match any`). A NAME THAT IS NOT
+    UTF-8 IS REFUSED here, `bump-leg-branch-not-utf8`, the dry run alike,
+    before the fetch and before anything is written: no line this tool
+    prints can spell it, and a push line naming another branch is worse
+    than none. `git branch -m` gives the branch a name that can be spelled.
     """
     proc = subprocess.run(["git", "symbolic-ref", "--quiet", "HEAD"],
-                          cwd=str(root), capture_output=True, text=True,
-                          encoding="utf-8", errors="replace", check=False)
-    ref = proc.stdout.rstrip("\n")
-    if not ref.startswith(HEADS):
+                          cwd=str(root), capture_output=True, check=False)
+    ref = proc.stdout[:-1] if proc.stdout.endswith(b"\n") else proc.stdout
+    heads = HEADS.encode("ascii")
+    if not ref.startswith(heads):
         return None
-    return ref[len(HEADS):] or None
+    raw = ref[len(heads):]
+    branch = raw.decode("utf-8", "surrogateescape")
+    try:
+        branch.encode("utf-8")
+    except UnicodeEncodeError:
+        # NAMED BY ITS BYTES, `b'caf<0xE9>'`, as `trailer-not-utf8` names a
+        # trailer, and THE ROOT as `shown_path` names a path, as
+        # `_branch_to_bump_from`'s other refusals name it (#766). The remedy
+        # is a command to paste, and holds the root raw inside its shell
+        # quotes, as their `switch -c` does.
+        raise Refusal(
+            "bump-leg-branch-not-utf8",
+            f"{shown_path(root)} is on the branch {raw!r}, whose name is not "
+            "UTF-8, so no line this tool prints can name it: its push line "
+            "would name a branch that is not there. Nothing was fetched, "
+            "written or committed.",
+            "Remediation: give the branch a name that is UTF-8 text, then "
+            "run the same command again: "
+            + shlex.join(["git", "-C", str(root), "branch", "-m"])
+            + " <a new name>") from None
+    return branch or None
 
 
 # ---------------------------------------------------------------------------

scripts/family.py

@@ -771,13 +771,49 @@ def _read_member_project(path: Path, project: str, repository: str,
 
 def _holder_branch(root: Path) -> str | None:
     """The branch the holder's HEAD is on, or None: detached, or a HEAD
-    that names something other than a branch."""
+    that names something other than a branch.
+
+    READ AS THE BYTES GIT WROTE (#765). Exactly one newline and exactly
+    `refs/heads/` come off the bytes, and the rest is decoded with
+    `surrogateescape`. `strip()` also took a trailing U+00A0, U+2028 or
+    U+0085, which a branch name may end in: a holder on `feat<U+00A0>` was
+    read as `feat`, a branch of its own, and the NEXT line pushed that one
+    and not the commit the act made. A NAME THAT IS NOT UTF-8 IS REFUSED
+    here, `family-branch-not-utf8`: no line this tool prints can spell it,
+    and a push line naming another branch is worse than none. The real run
+    of each act asks this before it reads the member or writes anything,
+    and its dry run at the line that would name the branch."""
     try:
-        ref = _git_bytes(["symbolic-ref", "-q", "HEAD"], root).decode(
-            "utf-8", "surrogateescape").strip()
+        ref = _git_bytes(["symbolic-ref", "-q", "HEAD"], root)
     except CommandFailed:
         return None
-    return ref[len(HEADS_PREFIX):] if ref.startswith(HEADS_PREFIX) else None
+    ref = ref[:-1] if ref.endswith(b"\n") else ref
+    heads = HEADS_PREFIX.encode("ascii")
+    if not ref.startswith(heads):
+        return None
+    raw = ref[len(heads):]
+    branch = raw.decode("utf-8", "surrogateescape")
+    try:
+        branch.encode("utf-8")
+    except UnicodeEncodeError:
+        # NAMED BY ITS BYTES, `b'caf<0xE9>'`, as `trailer-not-utf8` names a
+        # trailer, and THE HOLDER as `shown_path` names a path, as
+        # `_checked_branch`'s other refusals name it (#766). The remedy is
+        # quoted for a POSIX shell, as bump-leg's and update-shape's are, so
+        # README's sentence on the lines quoted for PowerShell stays true;
+        # a command to paste, it holds the holder's path raw inside its
+        # quotes, as their `switch -c` does (#459).
+        raise Refusal(
+            "family-branch-not-utf8",
+            f"{shown_path(root)} is on the branch {raw!r}, whose name is not "
+            "UTF-8, so no line this tool prints can name it: the NEXT line "
+            "would push a branch that is not there. Nothing was committed, "
+            "and the holder's work tree and index are as they were.",
+            "Remediation: give the branch a name that is UTF-8 text, then "
+            "run the same command again:\n    "
+            + shell_line(["git", "-C", root, "branch", "-m"], "posix")
+            + " <a new name>") from None
+    return branch
 
 
 #: A branch-name component the suggestion may use as it stands: dot-separated

tests/conftest.py

@@ -1177,6 +1177,64 @@ def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None:
     probe.rmdir()
 
 
+#: A branch git makes and no text can spell (#765): `caf` and the byte 0xE9,
+#: Latin-1's e-acute, a UTF-8 lead byte with nothing after it. Built with
+#: `chr`, not `os.fsdecode`, which raises on Windows, where this module
+#: must still import.
+NOT_UTF_8_BRANCH = "caf" + chr(0xDCE9)
+#: How a refusal names `NOT_UTF_8_BRANCH`: by its bytes, `b'caf<0xE9>'`, as
+#: `trailer-not-utf8` names a trailer.
+NOT_UTF_8_BRANCH_NAMED = repr(NOT_UTF_8_BRANCH.encode("utf-8",
+                                                      "surrogateescape"))
+
+
+def git_answer_bytes(repo: Path, *args: str) -> bytes:
+    """`git <args>` in `repo`, its stdout as the bytes git wrote, whatever
+    the exit: `git` above decodes as text, which a name that is not UTF-8
+    fails."""
+    return subprocess.run(["git", *args], cwd=str(repo), capture_output=True,
+                          check=False).stdout
+
+
+def held_by_git(repo: Path) -> dict:
+    """What a refusal must leave in `repo` as it was, as bytes (#765): the
+    commit HEAD names, the ref it is attached to, every ref and its commit,
+    the index and the status; and for each git directory under
+    `.git/modules`, a leg's or a member's, its refs, its object count and
+    its FETCH_HEAD with the time it was written, which every fetch into it
+    rewrites."""
+    held = {"head": git_answer_bytes(repo, "rev-parse", "HEAD"),
+            "attached": git_answer_bytes(repo, "symbolic-ref", "-q", "HEAD"),
+            "refs": git_answer_bytes(repo, "for-each-ref",
+                                     "--format=%(objectname) %(refname)"),
+            "index": git_answer_bytes(repo, "ls-files", "--stage", "-z"),
+            "status": git_answer_bytes(repo, "status", "--porcelain", "-z")}
+    modules = repo / ".git" / "modules"
+    gitdirs = sorted(found.parent for found in modules.rglob("HEAD")
+                     if (found.parent / "objects").is_dir()
+                     ) if modules.is_dir() else []
+    for gitdir in gitdirs:
+        fetched = gitdir / "FETCH_HEAD"
+        held[gitdir.relative_to(modules).as_posix()] = (
+            git_answer_bytes(gitdir, "for-each-ref",
+                             "--format=%(objectname) %(refname)"),
+            git_answer_bytes(gitdir, "count-objects", "-v"),
+            (fetched.read_bytes(), fetched.stat().st_mtime_ns)
+            if fetched.is_file() else None)
+    return held
+
+
+def on_a_branch_that_is_not_utf_8(repo: Path) -> dict:
+    """`repo` switched to a new branch `NOT_UTF_8_BRANCH` where HEAD is, or
+    the test skipped where this system cannot make the name
+    (`skip_where_a_name_cannot_be_not_utf_8`); returns `held_by_git(repo)`
+    on it."""
+    skip_where_a_name_cannot_be_not_utf_8(repo)
+    subprocess.run(["git", "switch", "-q", "-c", NOT_UTF_8_BRANCH],
+                   cwd=str(repo), capture_output=True, check=True)
+    return held_by_git(repo)
+
+
 def windows_fsencode(text: str | bytes) -> bytes:
     """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every
     lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes

tests/test_bump_leg_branch_and_push_line.py

@@ -32,10 +32,15 @@
     they already refuse it exits 0 and the real run ends
     `bump-leg-validators-red`. Its closing paragraph says so.
 
+AND THE BRANCH'S BYTES (#765): the name is read as git wrote it, so a
+trailing no-break space stays on it, and a name that is not UTF-8 is refused
+by name (`bump-leg-branch-not-utf8`) where `errors="replace"` read another.
+
 THE ROOT ABOVE THEM (#766). The three refusals and the dry run's `the real
 run needs a branch` line printed the root as it was, so a root under a name
 with an ESC, a line break or a bidi override sent it to the terminal raw.
-Each names it as `repo_shape.shown_path` does now.
+Each names it as `repo_shape.shown_path` does now, and so does #765's
+`bump-leg-branch-not-utf8`, raised in the same check.
 
 NO NETWORK, NO `gh`: the bare remotes are the session's own, and every root
 here pushes to a private copy of its remote (`make_root`), so a push reaches
@@ -52,7 +57,9 @@
 
 import pytest
 
-from conftest import FILE_PROTOCOL, PROJECT, REPO, cloned_just_pushed, git
+from conftest import (FILE_PROTOCOL, NOT_UTF_8_BRANCH_NAMED, PROJECT, REPO,
+                      cloned_just_pushed, git, held_by_git,
+                      on_a_branch_that_is_not_utf_8)
 #: The names no line may print raw, and how a line spells them (#766).
 from conftest import (POSIX_NAMES, SHOWN_NAMES, UNSAFE_NAMES,
                       assert_shown_safe, copied_under, named_as,
@@ -241,16 +248,23 @@ def test_a_branch_beside_a_tag_of_its_name_is_pushed_by_its_full_ref(
     assert upstream_of(root, "v1") == "refs/remotes/origin/v1"
 
 
+@pytest.mark.parametrize("neighbour", ["feat", "main"],
+                         ids=["beside-a-branch", "beside-the-tracking-branch"])
 def test_a_branch_ending_in_a_no_break_space_is_pushed_as_itself(
-        make_root, scaffolded, advanced):
+        make_root, scaffolded, advanced, neighbour):
     """`feat<U+00A0>` beside a branch `feat`: `strip()` took the no-break
     space off the name it read, so the bump committed on `feat<U+00A0>` and
     the line pushed `feat`, the other branch, at the commit before the bump
-    (rv672's S2). Only git's newline is taken off now: the line names the
-    branch the bump committed on, and run, pushes that commit."""
+    (rv672's S2). And `main<U+00A0>` beside the tracking branch `main` read
+    as `main`, so the bump was refused `bump-leg-on-tracking-branch` on a
+    branch that is not it (#765). Only git's newline is taken off now: the
+    line names the branch the bump committed on, and run, pushes that
+    commit, leaving origin's neighbour where it was."""
     root, remote = make_root("plain")
-    name = "feat\u00a0"
-    git("branch", "feat", cwd=root)
+    name = neighbour + chr(0xA0)
+    if neighbour != "main":
+        git("branch", neighbour, cwd=root)
+    was = head_of(remote, f"refs/heads/{neighbour}")
     git("checkout", "-q", "-b", name, cwd=root)
 
     result = bump(root, scaffolded, advanced)
@@ -261,7 +275,33 @@ def test_a_branch_ending_in_a_no_break_space_is_pushed_as_itself(
     pushed = run_the_line(line, root.parent)
     assert pushed.returncode == 0, pushed.stderr + pushed.stdout
     assert head_of(remote, f"refs/heads/{name}") == head_of(root)
-    assert head_of(remote, "refs/heads/feat") == "", "`feat` was not pushed"
+    assert head_of(remote, f"refs/heads/{neighbour}") == was, (
+        f"`{neighbour}` was not pushed")
+
+
+@pytest.mark.parametrize("extra", [(), ("--dry-run",)],
+                         ids=["bump", "dry-run"])
+def test_a_branch_named_with_a_byte_that_is_not_utf8_is_refused_by_name(
+        project, scaffolded, advanced, extra):
+    """`caf<0xE9>` (#765): decoded with `errors="replace"`, it read as
+    `caf<U+FFFD>`, a branch nobody has, so the bump committed on `caf<0xE9>`
+    and the line named `caf<U+FFFD>` on both sides, which pasted fails `src
+    refspec ... does not match any`. Refused by name now, by its bytes, the
+    dry run alike, before the fetch: no line names the branch but the
+    refusal, which says how to rename it, and the root, its refs and each
+    leg's git directory are as they were, FETCH_HEAD and objects too."""
+    before = on_a_branch_that_is_not_utf_8(project)
+
+    result = bump(project, scaffolded, advanced, *extra)
+
+    assert result.returncode == 2, result.stderr + result.stdout
+    assert (f"REFUSED bump-leg-branch-not-utf8: {project.resolve()} is on "
+            f"the branch {NOT_UTF_8_BRANCH_NAMED}, whose name is not UTF-8"
+            in result.stderr), result.stderr
+    assert result.stderr.rstrip().endswith(" branch -m <a new name>"), (
+        result.stderr)
+    assert result.stdout == ""
+    assert held_by_git(project) == before
 
 
 # --- the dirty root, spelled -------------------------------------------------
@@ -606,3 +646,25 @@ def test_the_dry_runs_branch_line_names_the_root_safe_to_show(
     assert said[0].startswith(
         f"the real run needs a branch: {spelt(root)} is on 'main', the "
         "tracking branch. Create one first with: git -C "), said
+
+
+@pytest.mark.parametrize("name", SHOWN_NAMES)
+def test_a_root_on_a_branch_that_is_not_utf8_is_named_safe_to_show(
+        scaffolded, advanced, tmp_path, name):
+    """#765's `bump-leg-branch-not-utf8` is raised in `_branch_to_bump_from`,
+    the check that gives `bump-leg-root-detached` and
+    `bump-leg-on-tracking-branch`, and names the root as they do (#766): as
+    `repr` spells it, on the one line it is, and a printable root as it is,
+    byte for byte. The remedy's `branch -m` is a command to paste, quoted
+    for a shell, and is not this row's."""
+    root = copied_under(scaffolded["clone"], tmp_path, name)
+    on_a_branch_that_is_not_utf_8(root)
+    result = bump(root, scaffolded, advanced)
+    assert result.returncode == 2, result.stderr + result.stdout
+    assert result.stdout == ""
+    assert refusal_detail(result.stderr, "bump-leg-branch-not-utf8") == [
+        f"REFUSED bump-leg-branch-not-utf8: {named_as(root, name)} is on the "
+        f"branch {NOT_UTF_8_BRANCH_NAMED}, whose name is not UTF-8, so no "
+        "line this tool prints can name it: its push line would name a "
+        "branch that is not there. Nothing was fetched, written or "
+        "committed."]