From 1a9d08bed71e1c694a276d336100d5269639976f Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 14:00:54 +0000 Subject: [PATCH 1/3] =?UTF-8?q?bootstrap.py:=20step=20(a)=20decodes=20what?= =?UTF-8?q?=20git=20prints=20as=20the=20system=20decodes=20a=20name=20it?= =?UTF-8?q?=20is=20given,=20and=20leaves=20a=20good=20branch=20this=20syst?= =?UTF-8?q?em's=20encoding=20cannot=20write=20with=20a=20line=20of=20its?= =?UTF-8?q?=20own,=20so=20a=20branch=20past=20the=20pin=20whose=20upstream?= =?UTF-8?q?=20is=20not=20UTF-8=20is=20moved=20back,=20and=20`tracking=5Fbr?= =?UTF-8?q?anch:=20"feature/=E6=97=A5=E6=9C=AC"`=20under=20`LC=5FALL=3DC?= =?UTF-8?q?=20PYTHONUTF8=3D0`=20no=20longer=20ends=20`make=20bootstrap`=20?= =?UTF-8?q?in=20a=20traceback=20(#710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_try_git` read every answer through `repo_shape.git_out`'s strict UTF-8 decode. It now reads bytes and decodes them with `os.fsdecode`, as #745 (#704) made `_head_of` and `_in_progress` read theirs. That decode is the inverse of the `os.fsencode` that `subprocess` applies to each argument, so a name git prints goes back to git as the same bytes under any POSIX locale. Measured on main 24a977c0 before this change: - lane 5's site on #710: a tracking branch past the pin with no commit of its own, its name and upstream not UTF-8, and the leg detached at the pin, ended in `UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff`, exit 1, from `_upstream_holding`. It is now moved back. - under `LC_ALL=C PYTHONUTF8=0`, `main` tracking `origin/日本` past the pin ended in `UnicodeEncodeError` once the name, read as UTF-8 text, was handed back to `rev-parse`. It is now moved back. - the bytes are stripped and not the text, because `str.strip` took a trailing U+00A0 off `refs/remotes/origin/a`, and the branch was then not moved back. `git_out` is not changed. The readers of `origin`'s URL in `shape-doctor.py` and `shape_advisory.py` take its `UnicodeDecodeError` to mean the URL names nothing (#385), and `adopt-project.py`'s `_default_branch` decodes locally since #717 (#425). So the decode rule lives in `_try_git`, step (a)'s one reader, and not in `git_out` for every caller. L4-124, as Brett Heap ruled at 2026-10-17T13:01:07Z ("go with (a) for L4-124"): `_argument_fault` also faults a character `os.fsencode` cannot write (`UNWRITABLE`). It asks for one only once the name holds no NUL and no lone surrogate, because the line it gets says the name is good. The leg is left as it is. `_say_not_tried` gives that fault its own second line, which says the name is good, names the encoding, and gives `LC_ALL=C.UTF-8` and `PYTHONUTF8=1`; #455's "Give that one a name git can create" stays for the other two faults. The run then goes on to the validators, which pass the name, and ends `bootstrap ok`, as a branch git refuses does. L4-125's own route (a leg on `ab`, bootstrapped twice) and L4-126 (the manifest's `"a\udcffb"` followed back to `main`) already exit 0 on main through #745's `_head_of`. L4-126 is answered by option (b), the root cause, as R455 recommended. The new tests pin both. No pinned copy or recorded digest names this file in the repository's tests, as #644 and #745 found when they changed it, so nothing is re-pinned. tests/test_bootstrap_non_utf8_branch.py runs each case on the harness of tests/test_bootstrap_after_bump.py, and holds the rule under an emulated `ascii` `os.fsencode` on every platform. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 72 +++- tests/test_bootstrap_non_utf8_branch.py | 371 +++++++++++++++++++ 2 files changed, 437 insertions(+), 6 deletions(-) create mode 100644 tests/test_bootstrap_non_utf8_branch.py diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index f92f8047..7e562ceb 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -41,7 +41,12 @@ file is. A branch no command can be given (a NUL, or a lone surrogate this system cannot encode) is never tried: the leg is left as it is and the line says why (#455); step (b)'s manifest - validator names the key when the manifest holds it. Advancing a pin + validator names the key when the manifest holds it. Nor is a branch + holding a character this system's encoding cannot write: that name + is good, so its line says to run bootstrap under a UTF-8 locale + (#710). Every name git prints back is read as bytes and decoded as + the system decodes a name it is given, so it reaches git again as + the bytes it was (#710). Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -131,6 +136,10 @@ "Remediation: put git on PATH and run this again. Step (a) reads each " "leg's pin and moves the leg through git, and the validators read the " "pins through git, so without it nothing here can be placed or checked.") +#: What `_argument_fault` calls a character this system's encoding cannot +#: write (#710): the one fault of a name that is good, so `_say_not_tried` +#: gives it a line of its own. +UNWRITABLE = "a character this system's encoding cannot write" REGISTER_RELPATH = Path("governance") / "review-authority" / "register.yaml" VALIDATORS = ( ("naming", ["scripts/validate-repository-naming.py", "--project", @@ -151,8 +160,31 @@ def _run(root: Path, args: list[str]) -> int: def _try_git(args: list[str], cwd: Path) -> str | None: + """What `git ` printed in `cwd`, stripped, or `None` where git + exited non-zero. + + STEP (a)'S ONE DECODE RULE (#710). What git prints is read as BYTES and + decoded as this system decodes a name it is given (`os.fsdecode`), as + `_head_of` and `_in_progress` read theirs: the inverse of the + `os.fsencode` that `subprocess` applies to each argument, so a name git + prints reaches git again as the same bytes, under any POSIX locale; on + Windows both are UTF-8, which is what git for Windows writes a ref name + in. Through `git_out`'s strict UTF-8, `_upstream_holding` ended `make + bootstrap` in a `UnicodeDecodeError` traceback over an upstream named + with a byte that is not UTF-8 (`refs/remotes/origin/ab`), and, under + an `ascii` encoding (`LC_ALL=C PYTHONUTF8=0`), in a `UnicodeEncodeError` + over one named in UTF-8, since the text it read could not be handed + back to `rev-parse`. `git_out` itself keeps its strict decode: the + readers of `origin`'s URL in `shape-doctor.py` and `shape_advisory.py` + take its `UnicodeDecodeError` to mean the URL names nothing (#385). + + The BYTES are stripped, of the ASCII whitespace git ends a line with, + and not the text: `str.strip` also takes away a U+00A0 or a U+3000 that + ends a name, so `refs/remotes/origin/a` was read as `.../a`, and + a branch with no commit of its own past the pin was not moved back. + """ try: - return git_out(args, cwd=cwd) + return os.fsdecode(git_out(args, cwd=cwd, binary=True).strip()) except Refusal: return None @@ -199,8 +231,18 @@ def _argument_fault(branch: str) -> str | None: exception `_try_git` would then have to swallow: a `UnicodeDecodeError`, which `git_out` raises over output that is not UTF-8, is a `ValueError` too, and catching that there would read a leg whose HEAD is on such a - branch as detached. Only a NUL and a lone surrogate are looked for; a - character the system's encoding merely lacks is not this fix's. + branch as detached. + + A CHARACTER THIS SYSTEM'S ENCODING CANNOT WRITE IS A FAULT TOO (#710, + Brett Heap's ruling (a) on L4-124), named `UNWRITABLE`. Under + `LC_ALL=C PYTHONUTF8=0` the encoding is `ascii`, and + `tracking_branch: "feature/日本"`, which `validate-manifest.py` passes + and git takes under a UTF-8 locale, ended step (a) in a + `UnicodeEncodeError` traceback from `subprocess.py`. The name is good, + so it is asked only once the name holds neither a NUL nor a lone + surrogate: a line saying the name is good is then true. The leg is left + as it is, and `_say_not_tried` says how to run bootstrap so that the + system can write the name. A surrogate that is ONE BYTE of a name is not refused. POSIX spells a name that is not UTF-8 with U+DC80 to U+DCFF and `os.fsencode` writes each back @@ -216,6 +258,9 @@ def _argument_fault(branch: str) -> str | None: return "U+0000, a NUL" if "\ud800" <= char <= "\udfff" and _fsencoded_length(char) != 1: return f"U+{ord(char):04X}, a lone surrogate" + for char in branch: + if _fsencoded_length(char) == 0: + return f"U+{ord(char):04X}, {UNWRITABLE}" return None @@ -230,10 +275,22 @@ def _fsencoded_length(char: str) -> int: def _say_not_tried(where: str, named: str, fault: str) -> None: """A branch `_argument_fault` found: no command was run with it. The leg stays where it is, and the line says which of the three places the - person's value can have come from holds it (#455).""" + person's value can have come from holds it (#455). + + A character this system's encoding cannot write (`UNWRITABLE`, #710) + has its own line, as Brett Heap ruled on L4-124: the name is good, so + "give that one a name git can create" would be wrong. It says so, names + the encoding (`sys.getfilesystemencoding()`, the one `os.fsencode` + writes with) and gives the two ways to run bootstrap under UTF-8, which + writes every character: a UTF-8 locale, or Python's UTF-8 mode.""" print(f" {where}: could not create branch {named}: it holds {fault}, " "which no command can be given as an argument; the leg is left as " "it is") + if fault.endswith(UNWRITABLE): + print(" the name is good, and this system's encoding, " + f"{sys.getfilesystemencoding()}, cannot write it: run bootstrap " + "again under `LC_ALL=C.UTF-8`, or with `PYTHONUTF8=1`.") + return print(" the branch is the leg's `branch:`, else `--branch`, else " "`tracking_branch`. Give that one a name git can create and run " "bootstrap again.") @@ -301,7 +358,10 @@ def _upstream_holding(sub: Path, branch: str, local: str) -> str | None: remote-tracking ref is not there (never fetched, or pruned) and one whose tip is not the upstream's. The two are compared by object id. `for-each-ref` names the upstream whether or not its ref exists, and - takes the branch after `refs/heads/`, where git reads no option. + takes the branch after `refs/heads/`, where git reads no option. Its + name is read by `_try_git`'s rule, so an upstream named with a byte + that is not UTF-8, or in UTF-8 under an `ascii` encoding, is the same + ref when `rev-parse` is handed it, and no longer a traceback (#710). """ upstream = _try_git(["for-each-ref", "--format=%(upstream)", f"refs/heads/{branch}"], sub) diff --git a/tests/test_bootstrap_non_utf8_branch.py b/tests/test_bootstrap_non_utf8_branch.py new file mode 100644 index 00000000..ea2e0506 --- /dev/null +++ b/tests/test_bootstrap_non_utf8_branch.py @@ -0,0 +1,371 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A name git prints is handed back to git as the bytes it was, and a good +name this system's encoding cannot write is said, not a traceback (#710). + +Step (a) of the assembly root's `scripts/bootstrap.py` reads names back from +git (the ref HEAD is on, a branch's upstream) and hands them to git again as +arguments. Each read went through `repo_shape.git_out`, which decodes what git +printed as strict UTF-8. #745 (#704) read HEAD's ref as bytes; `_try_git`, the +reader of every other answer, still decoded strictly. It now reads bytes and +decodes them with `os.fsdecode`, the inverse of the `os.fsencode` that +`subprocess` applies to an argument, so the name goes back to git as the same +bytes under any POSIX locale. `git_out` itself is unchanged: the readers of +`origin`'s URL in `shape-doctor.py` and `shape_advisory.py` take its +`UnicodeDecodeError` to mean the URL names nothing (#385). + +THE ISSUE'S LINES, each measured on main 24a977c0 (#745's squash) before this +change, in a scaffold's recursive clone bootstrapped once: + +- LANE 5'S SITE (issuecomment-6226993443). A tracking branch past the pin with + no commit of its own, its name and upstream not UTF-8, and the leg detached + at the pin: `UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in + position 21`, exit 1, from `_upstream_holding`. Now `moved back`. +- THE SAME READ, STRIPPED AS TEXT. `str.strip` takes a U+00A0 off the end of + the upstream's name, so `refs/remotes/origin/a` was asked as `.../a` + and the branch was not moved back (`not moving an existing branch`). The + bytes are stripped now, so the name keeps every byte. +- THE SAME READ UNDER `ascii`. `main` tracking `origin/日本` past the pin, under + `LC_ALL=C PYTHONUTF8=0`: `UnicodeEncodeError: 'ascii' codec can't encode + characters in position 20-21`, exit 1, since the upstream's name read as + UTF-8 text could not be handed back to `rev-parse`. Now `moved back`. +- L4-125, the issue's own route: a leg on `ab` (`git branch $'a\\xffb'`), + bootstrapped twice. Exit 0 on main already, since #745 read HEAD's ref as + bytes; held here, as the issue asks, by `make bootstrap` and `--branch`. +- L4-126, answered by option (b) as R455 recommended: `tracking_branch: + "a\\udcffb"` is created by step (a) and named by the validator (exit 1); the + manifest edited back to `main`, the next run places every leg (exit 0, a + traceback before #745). +- L4-124, Brett Heap's ruling (a) of 2026-10-17T13:01:07Z. Under + `LC_ALL=C PYTHONUTF8=0`, `tracking_branch: "feature/日本"` passed the + validator (`manifest ok`) and ended step (a) in `UnicodeEncodeError: 'ascii' + codec can't encode characters in position 19-20`, exit 1. `_argument_fault` + now faults a character `os.fsencode` cannot write, the leg is left as it is, + and the line of its own says the name is good, names the encoding and gives + the two ways to run bootstrap under UTF-8; each is run here as printed. + +A run that needs the byte 0xff in a file name skips where the disk refuses it +(or Windows, where `_argument_fault` refuses U+DCFF), and a run under `ascii` +skips where `LC_ALL=C PYTHONUTF8=0` gives Python no `ascii` encoding (macOS and +Windows keep UTF-8). The rule itself is held on every platform by an +emulated `ascii` `os.fsencode`. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import git, run_script +from test_bootstrap_after_bump import ( # noqa: F401 + MAIN, bootstrap, bootstrapped, head_ref, moved_back, rev, root, +) +from test_bootstrap_branch_bytes import ( # noqa: F401 + bootstrap_module, legs_state, tracking, +) +from test_bootstrap_spelling import edit +from test_stdout_encoding import child_environ, run_tool_bytes + +#: Where L4-124 was measured: the C locale with Python's UTF-8 mode off, +#: which makes the filesystem encoding `ascii`. +ASCII = {"LC_ALL": "C", "PYTHONUTF8": "0"} +#: The two ways out the line gives, each as the environment it names. +WAYS_OUT = [ + pytest.param({"LC_ALL": "C.UTF-8", "PYTHONUTF8": "0"}, + id="LC_ALL=C.UTF-8"), + pytest.param({"LC_ALL": "C", "PYTHONUTF8": "1"}, id="PYTHONUTF8=1"), +] +#: The good name L4-124 measured, and what an `ascii` stdout prints of it. +GOOD = "feature/日本" +GOOD_PRINTED = r"feature/\u65e5\u672c" +#: What `_argument_fault` calls the character (`UNWRITABLE`). +UNWRITABLE = "a character this system's encoding cannot write" + + +def own_line(encoding: str) -> str: + """The line of its own under a good name the encoding cannot write.""" + return (f" the name is good, and this system's encoding, " + f"{encoding}, cannot write it: run bootstrap again under " + "`LC_ALL=C.UTF-8`, or with `PYTHONUTF8=1`.") + + +def unwritten(leg: str, printed: str, point: str) -> list[str]: + """Step (a)'s two lines for a leg whose good branch was not tried.""" + return [f" [{leg}] {leg}: could not create branch {printed}: it holds " + f"{point}, {UNWRITABLE}, which no command can be given as an " + "argument; the leg is left as it is", own_line("ascii")] + + +def needs_the_byte(tmp_path: Path) -> None: + """Skip where a name holding the byte 0xff cannot be made: Windows, + where `_argument_fault` refuses U+DCFF, and a disk that refuses it.""" + if os.name == "nt": + pytest.skip("U+DCFF is three bytes on Windows, and step (a) does not " + "try such a branch at all") + try: + (tmp_path / "probe-\udcff").write_bytes(b"") + except (OSError, ValueError): + pytest.skip("this disk takes no file name with the byte 0xff") + + +def encoding_under(tmp_path: Path, changes: dict[str, str]) -> str: + """The filesystem encoding a child Python gets under `changes`.""" + return subprocess.run( + [sys.executable, "-c", + "import sys; print(sys.getfilesystemencoding())"], + env=child_environ(tmp_path, None, changes), capture_output=True, + text=True, check=True).stdout.strip() + + +def under_ascii(tmp_path: Path) -> dict[str, str]: + """`ASCII`, or a skip where it gives a child Python no `ascii`.""" + if encoding_under(tmp_path, ASCII) != "ascii": + pytest.skip("`LC_ALL=C PYTHONUTF8=0` gives this system's Python no " + "`ascii` filesystem encoding") + return ASCII + + +def run_under(root: Path, tmp_path: Path, changes: dict[str, str], + *args: str) -> tuple[int, str, bytes]: + """`bootstrap.py ` under `changes`, with no `PYTHONIOENCODING`, so + the stdout's encoding is the locale's: `(exit, stdout, stderr)`.""" + code, out, err = run_tool_bytes(root / "scripts" / "bootstrap.py", *args, + encoding=None, tmp_path=tmp_path, cwd=root, + changes=changes) + return code, out.decode("utf-8"), err + + +def head_bytes(leg: Path) -> bytes: + return subprocess.run(["git", "symbolic-ref", "--quiet", "HEAD"], + cwd=str(leg), capture_output=True, + check=True).stdout + + +def step_a(out: str) -> list[str]: + lines = out.split("\n") + return lines[lines.index("(a) legs on tracking branches, pins untouched") + + 1:lines.index("(b) neutral validators") - 1] + + +def past_the_pin(leg: Path, upstream: str, branch: str) -> tuple[str, str]: + """`branch` one commit past the pin, with no commit of its own: its tip + IS `refs/remotes/origin/`, which it tracks, and the leg is + detached at the pin. Returns the pin and that commit.""" + pin = rev(leg) + ahead = git("commit-tree", rev(leg, f"{pin}^{{tree}}"), "-p", pin, "-m", + "the remote moved on", cwd=leg).stdout.strip() + git("update-ref", f"refs/remotes/origin/{upstream}", ahead, cwd=leg) + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-f", "--track", branch, f"origin/{upstream}", + cwd=leg) + return pin, ahead + + +# --- the upstream's name goes back to git as the bytes it was --------------- + +@pytest.mark.parametrize("raw, byte", [ + pytest.param("a\udcffb", True, id="the-byte-0xff"), + pytest.param("a\u00a0", False, id="ending-in-U+00A0"), +]) +def test_a_branch_past_the_pin_is_moved_back_whatever_its_upstream_is_named( + root, tmp_path, bootstrap_module, raw, byte): + """LANE 5'S SITE: on main, a `UnicodeDecodeError` traceback over the + byte 0xff, and, over a name ending in U+00A0, `not moving an existing + branch`. Both are moved back now, the line naming the upstream and the + command as `move_back_to_pin` spells them, and HEAD is on the branch + whose name is the bytes git was given.""" + if byte: + needs_the_byte(tmp_path) + leg = root / "spec" + pin, ahead = past_the_pin(leg, raw, raw) + + result = run_script(root / "scripts" / "bootstrap.py", "--skip-validators", + "--branch", raw, cwd=root) + assert result.returncode == 0, result.stderr + result.stdout + assert "Traceback" not in result.stderr, result.stderr + spelled = bootstrap_module.spelled + command = ("git -C spec merge --ff-only " + f"{spelled(shlex.quote('origin/' + raw))}") + assert moved_back(ahead, pin, spelled(raw), command=command) in ( + result.stdout), result.stdout + assert rev(leg, f"refs/heads/{raw}") == pin and rev(leg) == pin + assert head_bytes(leg) == b"refs/heads/" + os.fsencode(raw) + b"\n" + + +def test_under_an_ascii_encoding_an_upstream_named_in_utf8_is_moved_back( + root, tmp_path, bootstrap_module): + """`main` tracking `origin/日本`, past the pin, under `LC_ALL=C + PYTHONUTF8=0`: a `UnicodeEncodeError` traceback on main, since the name + read as UTF-8 text could not be handed back to `rev-parse`. Decoded as + `os.fsdecode` decodes it, it is the same ref, and the remote-tracking + ref still holds the commit.""" + changes = under_ascii(tmp_path) + leg = root / "spec" + pin, ahead = past_the_pin(leg, "日本", "main") + + code, out, err = run_under(root, tmp_path, changes, "--skip-validators") + assert code == 0 and b"Traceback" not in err, out + err.decode("utf-8") + held = bootstrap_module.spelled( + "origin/" + "日本".encode("utf-8").decode("ascii", "surrogateescape")) + assert (f" [spec] spec: branch main moved back {ahead[:12]} -> " + f"{pin[:12]}, the pin; nothing is lost, {held} still holds those " + f"commits, and `git -C spec merge --ff-only '{held}'` brings the " + "branch forward\n") in out, out + assert rev(leg, MAIN) == pin and head_ref(leg) == MAIN + assert rev(leg, "refs/remotes/origin/日本") == ahead + + +# --- L4-125 and L4-126: every later run places the leg --------------------- + +@pytest.mark.parametrize("given", [False, True], + ids=["make-bootstrap", "--branch-ab"]) +def test_a_leg_on_a_branch_whose_name_is_not_utf8_is_placed_on_every_run( + root, tmp_path, given): + """L4-125: `git branch $'a\\xffb'`, the leg checked out on it, then + bootstrap twice. Without `--branch` each run puts the leg on `main`, + the first saying the leg left `ab`; with `--branch $'a\\xffb'` each + run says `on` it. Every run exits 0 with the validators run.""" + needs_the_byte(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("branch", "a\udcffb", cwd=leg) + git("checkout", "-q", "a\udcffb", cwd=leg) + args = ("--branch", "a\udcffb") if given else () + on = r"a\udcffb" if given else "main" + left = (" the leg was on a\\udcffb, which keeps its commits; it " + "is on main now\n") + for run in (1, 2): + result = run_script(root / "scripts" / "bootstrap.py", *args, + cwd=root) + assert result.returncode == 0, result.stderr + result.stdout + assert "Traceback" not in result.stderr, result.stderr + assert result.stdout.endswith("bootstrap ok\n"), result.stdout + placed = f" [spec] spec: on {on} at {pin[:12]} (branch tip == pin)\n" + assert placed in result.stdout, result.stdout + assert (placed + left in result.stdout) is ( + not given and run == 1), result.stdout + assert head_bytes(leg) == (b"refs/heads/a\xffb\n" if given + else b"refs/heads/main\n") + + +def test_a_tracking_branch_the_validator_names_is_followed_and_then_placed( + root, tmp_path): + """L4-126, option (b): step (a) creates `ab` from the manifest's + `"a\\udcffb"` and the validator names it, exit 1. The finding followed, + `main` again, the next run puts every leg on `main` and exits 0, where + it ended in a `UnicodeDecodeError` before #745.""" + needs_the_byte(tmp_path) + pins = {leg: rev(root / leg) for leg in ("spec", "code")} + tracking(root, r'"a\udcffb"') + first = bootstrap(root) + assert first.returncode == 1, first.stderr + first.stdout + assert "Traceback" not in first.stderr, first.stderr + assert ("FINDING manifest-tracking-branch: tracking_branch is " + "'a\\udcffb', which holds U+DCFF") in first.stderr, first.stderr + assert "bootstrap FAILED: manifest (exit 1)" in first.stderr + for leg, pin in pins.items(): + assert (f" [{leg}] {leg}: branch a\\udcffb created at {pin[:12]} " + "— at the pin\n") in first.stdout, first.stdout + assert head_bytes(root / leg) == b"refs/heads/a\xffb\n" + + edit(root, "project.yaml", r'tracking_branch: "a\udcffb"', + 'tracking_branch: "main"') + second = bootstrap(root) + assert second.returncode == 0, second.stderr + second.stdout + assert second.stdout.endswith("bootstrap ok\n"), second.stdout + for leg, pin in pins.items(): + assert (f" [{leg}] {leg}: on main at {pin[:12]} (branch tip == " + f"pin)\n the leg was on a\\udcffb, which keeps its " + "commits; it is on main now\n") in second.stdout, second.stdout + assert head_ref(root / leg) == MAIN + + +# --- L4-124: a good name the encoding cannot write is said ------------------ + +@pytest.mark.parametrize("way_out", WAYS_OUT) +def test_a_good_name_this_encoding_cannot_write_is_said_and_the_way_out_works( + root, tmp_path, way_out): + """Under `LC_ALL=C PYTHONUTF8=0`, `tracking_branch: "feature/日本"`: + the validators pass it, as they did, and step (a) ended in a + `UnicodeEncodeError` traceback. Now each leg is left exactly as it was + under the two lines, and the run ends `bootstrap ok`. Then the way out, + run as the line gives it, puts each leg on `feature/日本`.""" + changes = under_ascii(tmp_path) + if encoding_under(tmp_path, way_out) != "utf-8": + pytest.skip(f"{way_out} gives this system's Python no UTF-8") + tracking(root, f'"{GOOD}"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + assert code == 0 and b"Traceback" not in err, out + err.decode("utf-8") + assert step_a(out) == (unwritten("spec", GOOD_PRINTED, "U+65E5") + + unwritten("code", GOOD_PRINTED, "U+65E5")), out + assert "manifest ok: Atlas (atlas), 3 legs" in out, out + assert out.endswith("bootstrap ok\n"), out + assert legs_state(root) == before + + code, out, err = run_under(root, tmp_path, way_out) + assert code == 0 and b"Traceback" not in err, out + err.decode("utf-8") + for leg in ("spec", "code"): + assert f" [{leg}] {leg}: branch {GOOD} created at " in out, out + assert head_bytes(root / leg) == ( + b"refs/heads/" + GOOD.encode("utf-8") + b"\n") + + +def ascii_fsencode(text): + """`os.fsencode` where the filesystem encoding is `ascii`: bytes as they + are, text through `surrogateescape`, as POSIX's handler is.""" + if isinstance(text, bytes): + return text + return text.encode("ascii", "surrogateescape") + + +@pytest.mark.parametrize("branch, fault", [ + (GOOD, f"U+65E5, {UNWRITABLE}"), + ("a\xe9b", f"U+00E9, {UNWRITABLE}"), + # A NUL or a lone surrogate ANYWHERE is named first: that name is not + # good, so the line saying it is would be false. + ("日\x00", "U+0000, a NUL"), + ("日\ud83d", "U+D83D, a lone surrogate"), +]) +def test_under_an_ascii_encoding_what_it_cannot_write_is_the_fault( + bootstrap_module, monkeypatch, branch, fault): + monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) + assert bootstrap_module._argument_fault(branch) == fault + + +@pytest.mark.parametrize("branch", [ + "main", "release/1.2", "", + "a\udcffb", # the byte 0xff, as POSIX spells it + "x\udcc3\udca9", # `--branch xé` typed under the C locale +]) +def test_under_an_ascii_encoding_what_it_writes_has_no_fault( + bootstrap_module, monkeypatch, branch): + """A name the encoding writes, the bytes of one typed under it + included, is carried as it was: `--branch` reaches git as the bytes + the person typed.""" + monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) + assert bootstrap_module._argument_fault(branch) is None + + +def test_the_line_for_a_name_the_encoding_cannot_write_is_its_own( + bootstrap_module, monkeypatch, capsys): + """The second line is the ruling's own, naming the encoding the + process has, and #455's NUL keeps its line.""" + monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) + say = bootstrap_module._say_not_tried + say("[spec] spec", GOOD, bootstrap_module._argument_fault(GOOD)) + say("[code] code", "a\\x00b", bootstrap_module._argument_fault("a\x00b")) + lines = capsys.readouterr().out.split("\n") + assert lines[:2] == [ + f" [spec] spec: could not create branch {GOOD}: it holds U+65E5, " + f"{UNWRITABLE}, which no command can be given as an argument; the " + "leg is left as it is", own_line(sys.getfilesystemencoding())], lines + assert lines[3] == ( + " the branch is the leg's `branch:`, else `--branch`, else " + "`tracking_branch`. Give that one a name git can create and run " + "bootstrap again."), lines From 0a5e82bf15ce0dd212b0edffb100998bd0175165 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 16:20:40 +0000 Subject: [PATCH 2/3] =?UTF-8?q?bootstrap.py:=20the=20L4-124=20line=20calls?= =?UTF-8?q?=20a=20branch=20good=20only=20where=20git=20takes=20it=20in=20U?= =?UTF-8?q?TF-8=20and=20the=20name=20holds=20no=20surrogate,=20says=20`did?= =?UTF-8?q?=20not=20try`,=20and=20on=20Windows=20git's=20bytes=20are=20rea?= =?UTF-8?q?d=20as=20UTF-8,=20so=20`feature/=E6=97=A5=E6=9C=AC..x`=20or=20`?= =?UTF-8?q?"=E6=97=A5\udcff"`=20under=20`LC=5FALL=3DC=20PYTHONUTF8=3D0`=20?= =?UTF-8?q?is=20no=20longer=20called=20good,=20and=20the=20advisory=20test?= =?UTF-8?q?=20passes=20(#710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix round 1 for RV824's review of #824 (lane 3, issuecomment-6236261109), under the lane 4 coordinator's rulings. - M1: `_try_git`'s docstring named the advisory module, which `tests/test_shape_advisory.py` refuses in any template. It now says "in `shape-doctor.py` and in the advisory it reads". - S1: three changes, so the ruled line never calls a bad name good. - `_argument_fault` asks `UNWRITABLE` last, and names any surrogate in the name first, a byte-carrying one included, because `validate-manifest.py` names every surrogate (#280). - `_git_refuses_in_utf8` asks `git check-ref-format --branch` about the name's UTF-8 bytes before the line calls it good. `feature/日本..x`, `日本.lock` and `日本 x` get git's own words and #455's way out. - The new tests have rows for those three names and for `"日\udcff"`. - S2: `tests/test_bootstrap_branch_bytes.py`'s no-fault test skips `feature/日本`, `ab` and `a\xe9b` where `os.fsencode` cannot write them (`written`), and its docstring states the rule. The parameter list and docstring change; #723's lines do not. - S3: each `assert a and b` in the new test file is now two asserts. - N2: the first line for `UNWRITABLE` says `did not try`. - N3: on Windows `_try_git` decodes git's bytes as UTF-8, since `subprocess` hands git its arguments through `CreateProcessW`. `os.fsdecode` there is `mbcs` with `replace` under `PYTHONLEGACYWINDOWSFSENCODING=1`. #745's `_head_of` is not changed. - N4: a row sends `UNWRITABLE` through a leg's own `branch:`. - N5: the four new-file tests that hand git a name only UTF-8 can write, or read a child's UTF-8 line, skip where pytest itself is not under UTF-8. - N6: a comment at the decode records #710's ruling, Option 2: `git_out` keeps its strict UTF-8. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 138 +++++++++----- tests/test_bootstrap_branch_bytes.py | 23 ++- tests/test_bootstrap_non_utf8_branch.py | 190 +++++++++++++++++-- 3 files changed, 286 insertions(+), 65 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 7e562ceb..4b688413 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -42,11 +42,12 @@ or a lone surrogate this system cannot encode) is never tried: the leg is left as it is and the line says why (#455); step (b)'s manifest validator names the key when the manifest holds it. Nor is a branch - holding a character this system's encoding cannot write: that name - is good, so its line says to run bootstrap under a UTF-8 locale - (#710). Every name git prints back is read as bytes and decoded as - the system decodes a name it is given, so it reaches git again as - the bytes it was (#710). Advancing a pin + holding a character this system's encoding cannot write (#710): where + git takes the name in UTF-8, its line says the name is good and to + run bootstrap under a UTF-8 locale, and where git refuses it there + too, the line gives git's words. Every name git prints back is read + as bytes and decoded so that it reaches git again as the bytes it + was (#710). Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -164,19 +165,27 @@ def _try_git(args: list[str], cwd: Path) -> str | None: exited non-zero. STEP (a)'S ONE DECODE RULE (#710). What git prints is read as BYTES and - decoded as this system decodes a name it is given (`os.fsdecode`), as - `_head_of` and `_in_progress` read theirs: the inverse of the - `os.fsencode` that `subprocess` applies to each argument, so a name git - prints reaches git again as the same bytes, under any POSIX locale; on - Windows both are UTF-8, which is what git for Windows writes a ref name - in. Through `git_out`'s strict UTF-8, `_upstream_holding` ended `make - bootstrap` in a `UnicodeDecodeError` traceback over an upstream named - with a byte that is not UTF-8 (`refs/remotes/origin/ab`), and, under - an `ascii` encoding (`LC_ALL=C PYTHONUTF8=0`), in a `UnicodeEncodeError` - over one named in UTF-8, since the text it read could not be handed - back to `rev-parse`. `git_out` itself keeps its strict decode: the - readers of `origin`'s URL in `shape-doctor.py` and `shape_advisory.py` - take its `UnicodeDecodeError` to mean the URL names nothing (#385). + decoded the way the name is encoded when git is handed it again. On + POSIX that is `os.fsdecode`, as `_head_of` and `_in_progress` read + theirs: the inverse of the `os.fsencode` that `subprocess` applies to + each argument, so a name git prints reaches git again as the same bytes + under any locale. On Windows `subprocess` encodes no argument: it hands + the command line to `CreateProcessW` as UTF-16, and git for Windows + reads it, and writes a ref name, in UTF-8. So the bytes are decoded as + UTF-8 there, as `git_out` decoded them, and not by `os.fsdecode`, which + `PYTHONLEGACYWINDOWSFSENCODING=1` makes `mbcs` with `replace`, losing + every character the code page lacks (RV824's N3). Through `git_out`'s + strict UTF-8, `_upstream_holding` ended `make bootstrap` in a + `UnicodeDecodeError` traceback over an upstream named with a byte that + is not UTF-8 (`refs/remotes/origin/ab`), and, under an `ascii` + encoding (`LC_ALL=C PYTHONUTF8=0`), in a `UnicodeEncodeError` over one + named in UTF-8, since the text it read could not be handed back to + `rev-parse`. + + `git_out` itself keeps its strict decode, as #710's ruling took Option + 2: the readers of `origin`'s URL in `shape-doctor.py` and in the + advisory it reads take its `UnicodeDecodeError` to mean the URL names + nothing (#385). The BYTES are stripped, of the ASCII whitespace git ends a line with, and not the text: `str.strip` also takes away a U+00A0 or a U+3000 that @@ -184,9 +193,12 @@ def _try_git(args: list[str], cwd: Path) -> str | None: a branch with no commit of its own past the pin was not moved back. """ try: - return os.fsdecode(git_out(args, cwd=cwd, binary=True).strip()) + raw = git_out(args, cwd=cwd, binary=True).strip() except Refusal: return None + # #710's ruling, Option 2: the decode is step (a)'s own, here, and + # `repo_shape.git_out` keeps its strict UTF-8 for every other caller. + return raw.decode("utf-8") if os.name == "nt" else os.fsdecode(raw) @contextmanager @@ -238,11 +250,14 @@ def _argument_fault(branch: str) -> str | None: `LC_ALL=C PYTHONUTF8=0` the encoding is `ascii`, and `tracking_branch: "feature/日本"`, which `validate-manifest.py` passes and git takes under a UTF-8 locale, ended step (a) in a - `UnicodeEncodeError` traceback from `subprocess.py`. The name is good, - so it is asked only once the name holds neither a NUL nor a lone - surrogate: a line saying the name is good is then true. The leg is left - as it is, and `_say_not_tried` says how to run bootstrap so that the - system can write the name. + `UnicodeEncodeError` traceback from `subprocess.py`. The leg is left as + it is, and `_say_not_tried` says why. Its line can call the name good, + so this fault is asked last, and a name that holds a surrogate at all + is named for the surrogate instead, even one that is one byte of a name + (RV824's S1): `validate-manifest.py` names every surrogate (#280), so + `"日\\udcff"` is not a good name, and #455's line is the true one. + Whether git takes the name is asked before a line calls it good + (`_git_refuses_in_utf8`). A surrogate that is ONE BYTE of a name is not refused. POSIX spells a name that is not UTF-8 with U+DC80 to U+DCFF and `os.fsencode` writes each back @@ -258,10 +273,15 @@ def _argument_fault(branch: str) -> str | None: return "U+0000, a NUL" if "\ud800" <= char <= "\udfff" and _fsencoded_length(char) != 1: return f"U+{ord(char):04X}, a lone surrogate" - for char in branch: - if _fsencoded_length(char) == 0: - return f"U+{ord(char):04X}, {UNWRITABLE}" - return None + unwritable = next((char for char in branch + if _fsencoded_length(char) == 0), None) + if unwritable is None: + return None + surrogate = next((char for char in branch + if "\ud800" <= char <= "\udfff"), None) + if surrogate is not None: + return f"U+{ord(surrogate):04X}, a lone surrogate" + return f"U+{ord(unwritable):04X}, {UNWRITABLE}" def _fsencoded_length(char: str) -> int: @@ -272,30 +292,59 @@ def _fsencoded_length(char: str) -> int: return 0 -def _say_not_tried(where: str, named: str, fault: str) -> None: +def _say_not_tried(where: str, named: str, fault: str, + refused: list[str] | None = None) -> None: """A branch `_argument_fault` found: no command was run with it. The leg stays where it is, and the line says which of the three places the person's value can have come from holds it (#455). - A character this system's encoding cannot write (`UNWRITABLE`, #710) - has its own line, as Brett Heap ruled on L4-124: the name is good, so - "give that one a name git can create" would be wrong. It says so, names - the encoding (`sys.getfilesystemencoding()`, the one `os.fsencode` - writes with) and gives the two ways to run bootstrap under UTF-8, which - writes every character: a UTF-8 locale, or Python's UTF-8 mode.""" - print(f" {where}: could not create branch {named}: it holds {fault}, " - "which no command can be given as an argument; the leg is left as " - "it is") - if fault.endswith(UNWRITABLE): + A character this system's encoding cannot write (`UNWRITABLE`, #710) is + a branch step (a) DID NOT TRY, not one it could not create: the branch + can be there already, made by a run under UTF-8 (RV824's N2). Where git + takes the name in UTF-8 (`refused` is `None`), the line is the one Brett + Heap ruled on L4-124: the name is good, the encoding is named + (`sys.getfilesystemencoding()`, the one `os.fsencode` writes with), and + the two ways to run bootstrap under UTF-8 follow, a UTF-8 locale or + Python's UTF-8 mode. Where git refuses it in UTF-8 too, `refused` is + git's lines (`_git_refuses_in_utf8`): the name is not called good, git's + words are said, and #455's way out follows, since the name itself has + to change (RV824's S1).""" + unwritable = fault.endswith(UNWRITABLE) + tried = "did not try" if unwritable else "could not create" + print(f" {where}: {tried} branch {named}: it holds {fault}, which no " + "command can be given as an argument; the leg is left as it is") + if unwritable and refused is None: print(" the name is good, and this system's encoding, " f"{sys.getfilesystemencoding()}, cannot write it: run bootstrap " "again under `LC_ALL=C.UTF-8`, or with `PYTHONUTF8=1`.") return + if unwritable: + print(" and git refuses it as a branch name in UTF-8 too:") + _say_lines(refused) print(" the branch is the leg's `branch:`, else `--branch`, else " "`tracking_branch`. Give that one a name git can create and run " "bootstrap again.") +def _git_refuses_in_utf8(sub: Path, branch: str) -> list[str] | None: + """What git says of `branch` as a branch name in the bytes a run under + UTF-8 would hand it, or `None` when git takes it (#710, RV824's S1). + + A name `_argument_fault` faults as `UNWRITABLE` is not called good until + git has been asked: `feature/日本..x`, `日本.lock` and `日本 x` are names + this system's encoding cannot write AND names git refuses in any + encoding, and the way out the line gave ended in `fatal: ... is not a + valid branch name`. `git check-ref-format --branch` is git's own answer. + It is asked of `branch.encode("utf-8", "surrogateescape")`, handed to + `subprocess` through `os.fsdecode`, which POSIX's `os.fsencode` writes + back as exactly those bytes under any locale. On Windows no character + is `UNWRITABLE` (its `os.fsencode` writes UTF-8, or replaces), so this + is never asked there. + """ + utf8 = os.fsdecode(branch.encode("utf-8", "surrogateescape")) + return _git_says(["check-ref-format", "--branch", utf8], sub) + + def _short(sha: str | None) -> str: return sha[:12] if sha else "?" @@ -359,9 +408,10 @@ def _upstream_holding(sub: Path, branch: str, local: str) -> str | None: whose tip is not the upstream's. The two are compared by object id. `for-each-ref` names the upstream whether or not its ref exists, and takes the branch after `refs/heads/`, where git reads no option. Its - name is read by `_try_git`'s rule, so an upstream named with a byte - that is not UTF-8, or in UTF-8 under an `ascii` encoding, is the same - ref when `rev-parse` is handed it, and no longer a traceback (#710). + name is read by `_try_git`'s rule, so on POSIX an upstream named with + a byte that is not UTF-8, or in UTF-8 under an `ascii` encoding, is the + same ref when `rev-parse` is handed it, and no longer a traceback + (#710). """ upstream = _try_git(["for-each-ref", "--format=%(upstream)", f"refs/heads/{branch}"], sub) @@ -872,7 +922,9 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: return fault = _argument_fault(branch) if fault is not None: - _say_not_tried(where, named, fault) + _say_not_tried(where, named, fault, + _git_refuses_in_utf8(sub, branch) + if fault.endswith(UNWRITABLE) else None) return pin = recorded_gitlink(root, path) local = _try_git(["rev-parse", "--verify", "--quiet", diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index bab82102..72a6fee9 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -78,6 +78,20 @@ "and run bootstrap again.") +def written(branch: str): + """`branch` as a row of the no-fault test, skipped where this system's + encoding cannot write it: there `_argument_fault` faults it as #710's + `UNWRITABLE`, which `tests/test_bootstrap_non_utf8_branch.py` holds + (RV824's S2). Asked at collection, of `os.fsencode` itself.""" + try: + os.fsencode(branch) + except UnicodeEncodeError: + return pytest.param(branch, marks=pytest.mark.skip( + reason="this system's encoding cannot write it, so it is #710's " + "UNWRITABLE fault, held in test_bootstrap_non_utf8_branch")) + return branch + + def not_tried(leg: str, printed: str, point: str, kind: str) -> list[str]: return [f" [{leg}] {leg}: could not create branch {printed}: it holds " f"{point}, {kind}, which no command can be given as an " @@ -377,11 +391,14 @@ def bootstrap_module(): @pytest.mark.parametrize("branch", [ - "main", "release/1.2", "feature/日本", "a‍b", "a\xe9b", "a b", + "main", "release/1.2", written("feature/日本"), written("a‍b"), + written("a\xe9b"), "a b", "a\x1bb", "a..b", "", "\udc80", "a\udcffb"]) def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): - """Nothing but a NUL and a lone surrogate with no byte is a fault: a - control, a space, `..`, a character the system's encoding may lack, and + """Where the system's encoding writes every character of the branch, + nothing but a NUL and a lone surrogate with no byte is a fault: a + control, a space, `..`, a character that is not ASCII (`written` skips + one the encoding cannot write, #710's `UNWRITABLE`), and the byte-carrying surrogates U+DC80 to U+DCFF are the commands'. On a system whose `os.fsencode` does not carry U+DC80 and U+DCFF (Windows) the next test is the one that holds.""" diff --git a/tests/test_bootstrap_non_utf8_branch.py b/tests/test_bootstrap_non_utf8_branch.py index ea2e0506..4a5dded6 100644 --- a/tests/test_bootstrap_non_utf8_branch.py +++ b/tests/test_bootstrap_non_utf8_branch.py @@ -42,16 +42,24 @@ now faults a character `os.fsencode` cannot write, the leg is left as it is, and the line of its own says the name is good, names the encoding and gives the two ways to run bootstrap under UTF-8; each is run here as printed. +- RV824's review of #824, on the ruled line: it is said only where git takes + the name in UTF-8 (`git check-ref-format --branch`), so `feature/日本..x`, + `日本.lock` and `日本 x` get git's own words instead (S1); a name that also + holds a surrogate is named for the surrogate, as the validator names it + (S1); the first line says `did not try` (N2); a leg's own `branch:` takes + the same path (N4); and on Windows git's bytes are read as UTF-8 (N3). A run that needs the byte 0xff in a file name skips where the disk refuses it (or Windows, where `_argument_fault` refuses U+DCFF), and a run under `ascii` skips where `LC_ALL=C PYTHONUTF8=0` gives Python no `ascii` encoding (macOS and Windows keep UTF-8). The rule itself is held on every platform by an -emulated `ascii` `os.fsencode`. +emulated `ascii` `os.fsencode`. A test that hands git a name only UTF-8 can +write skips where pytest itself does not run under UTF-8 (N5). """ from __future__ import annotations +import codecs import os import shlex import subprocess @@ -65,7 +73,8 @@ MAIN, bootstrap, bootstrapped, head_ref, moved_back, rev, root, ) from test_bootstrap_branch_bytes import ( # noqa: F401 - bootstrap_module, legs_state, tracking, + SECOND_LINE, bootstrap_module, leg_branch, legs_state, not_tried, + tracking, ) from test_bootstrap_spelling import edit from test_stdout_encoding import child_environ, run_tool_bytes @@ -84,6 +93,17 @@ GOOD_PRINTED = r"feature/\u65e5\u672c" #: What `_argument_fault` calls the character (`UNWRITABLE`). UNWRITABLE = "a character this system's encoding cannot write" +#: The line under a name git refuses in UTF-8 too, before git's own words. +GIT_REFUSES = " and git refuses it as a branch name in UTF-8 too:" +#: A test that hands git a name only UTF-8 can write, or reads a line a +#: child under this process's locale writes in UTF-8 (`—`), needs this +#: process itself under UTF-8 (RV824's N5). CI's is; a run of pytest under +#: `LC_ALL=C PYTHONUTF8=0` is not, and `conftest.git` cannot hand git such +#: a name there. +NEEDS_A_UTF8_PROCESS = pytest.mark.skipif( + codecs.lookup(sys.getfilesystemencoding()).name != "utf-8", + reason="this pytest process's filesystem encoding is not UTF-8, so it " + "cannot hand git the name or read the child's UTF-8 line") def own_line(encoding: str) -> str: @@ -95,9 +115,14 @@ def own_line(encoding: str) -> str: def unwritten(leg: str, printed: str, point: str) -> list[str]: """Step (a)'s two lines for a leg whose good branch was not tried.""" - return [f" [{leg}] {leg}: could not create branch {printed}: it holds " + return [did_not_try(leg, printed, point), own_line("ascii")] + + +def did_not_try(leg: str, printed: str, point: str) -> str: + """The first line for a leg whose branch holds `UNWRITABLE` (N2).""" + return (f" [{leg}] {leg}: did not try branch {printed}: it holds " f"{point}, {UNWRITABLE}, which no command can be given as an " - "argument; the leg is left as it is", own_line("ascii")] + "argument; the leg is left as it is") def needs_the_byte(tmp_path: Path) -> None: @@ -169,7 +194,8 @@ def past_the_pin(leg: Path, upstream: str, branch: str) -> tuple[str, str]: @pytest.mark.parametrize("raw, byte", [ pytest.param("a\udcffb", True, id="the-byte-0xff"), - pytest.param("a\u00a0", False, id="ending-in-U+00A0"), + pytest.param("a\u00a0", False, id="ending-in-U+00A0", + marks=NEEDS_A_UTF8_PROCESS), ]) def test_a_branch_past_the_pin_is_moved_back_whatever_its_upstream_is_named( root, tmp_path, bootstrap_module, raw, byte): @@ -192,10 +218,12 @@ def test_a_branch_past_the_pin_is_moved_back_whatever_its_upstream_is_named( f"{spelled(shlex.quote('origin/' + raw))}") assert moved_back(ahead, pin, spelled(raw), command=command) in ( result.stdout), result.stdout - assert rev(leg, f"refs/heads/{raw}") == pin and rev(leg) == pin + assert rev(leg, f"refs/heads/{raw}") == pin + assert rev(leg) == pin assert head_bytes(leg) == b"refs/heads/" + os.fsencode(raw) + b"\n" +@NEEDS_A_UTF8_PROCESS def test_under_an_ascii_encoding_an_upstream_named_in_utf8_is_moved_back( root, tmp_path, bootstrap_module): """`main` tracking `origin/日本`, past the pin, under `LC_ALL=C @@ -208,14 +236,16 @@ def test_under_an_ascii_encoding_an_upstream_named_in_utf8_is_moved_back( pin, ahead = past_the_pin(leg, "日本", "main") code, out, err = run_under(root, tmp_path, changes, "--skip-validators") - assert code == 0 and b"Traceback" not in err, out + err.decode("utf-8") + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") held = bootstrap_module.spelled( "origin/" + "日本".encode("utf-8").decode("ascii", "surrogateescape")) assert (f" [spec] spec: branch main moved back {ahead[:12]} -> " f"{pin[:12]}, the pin; nothing is lost, {held} still holds those " f"commits, and `git -C spec merge --ff-only '{held}'` brings the " "branch forward\n") in out, out - assert rev(leg, MAIN) == pin and head_ref(leg) == MAIN + assert rev(leg, MAIN) == pin + assert head_ref(leg) == MAIN assert rev(leg, "refs/remotes/origin/日本") == ahead @@ -252,6 +282,7 @@ def test_a_leg_on_a_branch_whose_name_is_not_utf8_is_placed_on_every_run( else b"refs/heads/main\n") +@NEEDS_A_UTF8_PROCESS def test_a_tracking_branch_the_validator_names_is_followed_and_then_placed( root, tmp_path): """L4-126, option (b): step (a) creates `ab` from the manifest's @@ -301,7 +332,8 @@ def test_a_good_name_this_encoding_cannot_write_is_said_and_the_way_out_works( before = legs_state(root) code, out, err = run_under(root, tmp_path, changes) - assert code == 0 and b"Traceback" not in err, out + err.decode("utf-8") + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") assert step_a(out) == (unwritten("spec", GOOD_PRINTED, "U+65E5") + unwritten("code", GOOD_PRINTED, "U+65E5")), out assert "manifest ok: Atlas (atlas), 3 legs" in out, out @@ -309,13 +341,124 @@ def test_a_good_name_this_encoding_cannot_write_is_said_and_the_way_out_works( assert legs_state(root) == before code, out, err = run_under(root, tmp_path, way_out) - assert code == 0 and b"Traceback" not in err, out + err.decode("utf-8") + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") for leg in ("spec", "code"): assert f" [{leg}] {leg}: branch {GOOD} created at " in out, out assert head_bytes(root / leg) == ( b"refs/heads/" + GOOD.encode("utf-8") + b"\n") +@pytest.mark.parametrize("name, printed", [ + pytest.param("feature/日本..x", r"feature/\u65e5\u672c..x", id="dot-dot"), + pytest.param("日本.lock", r"\u65e5\u672c.lock", id="dot-lock"), + pytest.param("日本 x", r"\u65e5\u672c x", id="a-space"), +]) +def test_a_name_git_refuses_in_utf8_too_is_never_called_good( + root, tmp_path, name, printed): + """RV824's S1: each of these names this system's encoding cannot + write, and git refuses in any encoding (`..`, `.lock`, a space), was + called good, with a way out that ended in git's `is not a valid branch + name`. Now `git check-ref-format --branch` is asked of the UTF-8 bytes + first: git's words are said, and #455's way out, and each leg is left + as it was. The validators pass the value, as they did, so the run + ends `bootstrap ok`, as a branch git refuses always has.""" + changes = under_ascii(tmp_path) + tracking(root, f'"{name}"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + assert "the name is good" not in out, out + lines = step_a(out) + for leg in ("spec", "code"): + at = lines.index(did_not_try(leg, printed, "U+65E5")) + said = lines[at + 2:lines.index(SECOND_LINE, at)] + assert lines[at + 1] == GIT_REFUSES, lines + assert said, lines + assert any(printed in line for line in said), said + assert lines.count(SECOND_LINE) == 2, lines + assert out.endswith("bootstrap ok\n"), out + assert legs_state(root) == before + + +def test_a_name_holding_a_surrogate_too_is_named_for_the_surrogate( + root, tmp_path): + """RV824's S1: `"日\\udcff"` under `LC_ALL=C PYTHONUTF8=0` was called + good in step (a), while step (b) of the same run named its U+DCFF. + The surrogate is the fault now, with #455's lines, and the run ends as + the validator ends it.""" + changes = under_ascii(tmp_path) + tracking(root, r'"日\udcff"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + said = err.decode("utf-8") + assert code == 1, out + said + assert "Traceback" not in said, said + assert "the name is good" not in out, out + printed = r"\u65e5\udcff" + assert step_a(out) == ( + not_tried("spec", printed, "U+DCFF", "a lone surrogate") + + not_tried("code", printed, "U+DCFF", "a lone surrogate")), out + assert "FINDING manifest-tracking-branch:" in said, said + assert "bootstrap FAILED: manifest (exit 1)" in said, said + assert legs_state(root) == before + + +def test_a_legs_own_branch_this_encoding_cannot_write_is_said_too( + root, tmp_path): + """RV824's N4: a leg's own `branch:` reaches `_argument_fault` as + `tracking_branch` does. Under `LC_ALL=C PYTHONUTF8=0`, `branch: + "feature/日本"` on the spec leg leaves that leg as it was under the + ruled lines, and the code leg, on `main`, is placed as it always was.""" + changes = under_ascii(tmp_path) + leg_branch(root, "spec", f'"{GOOD}"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + assert step_a(out) == unwritten("spec", GOOD_PRINTED, "U+65E5") + [ + f" [code] code: on main at {rev(root / 'code')[:12]} (branch tip == " + "pin)"], out + assert out.endswith("bootstrap ok\n"), out + assert legs_state(root) == before + + +class LegacyWindowsOs: + """`os` as Windows has it under `PYTHONLEGACYWINDOWSFSENCODING=1`: + `nt`, with an `os.fsdecode` that is `mbcs` with `replace`, so a + character the code page lacks is lost. Only what `bootstrap.py` reads + of it differs; everything else is this system's `os`.""" + + name = "nt" + + @staticmethod + def fsdecode(raw: bytes) -> str: + return raw.decode("ascii", "replace") + + def __getattr__(self, attr: str): + return getattr(os, attr) + + +@NEEDS_A_UTF8_PROCESS +def test_on_windows_what_git_prints_is_read_as_utf8( + root, bootstrap_module, monkeypatch): + """RV824's N3: on Windows `subprocess` hands git its arguments through + `CreateProcessW` and git reads them as UTF-8, so `_try_git` reads git's + bytes as UTF-8 there. Through `os.fsdecode` in the legacy mode, the + upstream `origin/日本` came back as `origin/??????`, which names no ref. + The stand-in `os` is the module's alone: git still runs here.""" + leg = root / "spec" + past_the_pin(leg, "日本", "main") + monkeypatch.setattr(bootstrap_module, "os", LegacyWindowsOs()) + assert bootstrap_module._try_git( + ["for-each-ref", "--format=%(upstream)", "refs/heads/main"], + leg) == "refs/remotes/origin/日本" + + def ascii_fsencode(text): """`os.fsencode` where the filesystem encoding is `ascii`: bytes as they are, text through `surrogateescape`, as POSIX's handler is.""" @@ -331,6 +474,10 @@ def ascii_fsencode(text): # good, so the line saying it is would be false. ("日\x00", "U+0000, a NUL"), ("日\ud83d", "U+D83D, a lone surrogate"), + # A surrogate that IS one byte of a name is the fault beside a character + # the encoding cannot write: the validator names it (RV824's S1). + ("日\udcff", "U+DCFF, a lone surrogate"), + ("\udc80日", "U+DC80, a lone surrogate"), ]) def test_under_an_ascii_encoding_what_it_cannot_write_is_the_fault( bootstrap_module, monkeypatch, branch, fault): @@ -355,17 +502,22 @@ def test_under_an_ascii_encoding_what_it_writes_has_no_fault( def test_the_line_for_a_name_the_encoding_cannot_write_is_its_own( bootstrap_module, monkeypatch, capsys): """The second line is the ruling's own, naming the encoding the - process has, and #455's NUL keeps its line.""" + process has; #455's NUL keeps its line; and a name git refuses gets + git's words and #455's way out, never "good" (RV824's S1).""" monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) say = bootstrap_module._say_not_tried say("[spec] spec", GOOD, bootstrap_module._argument_fault(GOOD)) say("[code] code", "a\\x00b", bootstrap_module._argument_fault("a\x00b")) + refused = "fatal: '日本.lock' is not a valid branch name" + say("[spec] spec", "日本.lock", bootstrap_module._argument_fault("日本.lock"), + [refused]) lines = capsys.readouterr().out.split("\n") - assert lines[:2] == [ - f" [spec] spec: could not create branch {GOOD}: it holds U+65E5, " - f"{UNWRITABLE}, which no command can be given as an argument; the " - "leg is left as it is", own_line(sys.getfilesystemencoding())], lines - assert lines[3] == ( - " the branch is the leg's `branch:`, else `--branch`, else " - "`tracking_branch`. Give that one a name git can create and run " - "bootstrap again."), lines + assert lines[:2] == [did_not_try("spec", GOOD, "U+65E5"), + own_line(sys.getfilesystemencoding())], lines + assert lines[2:4] == [ + " [code] code: could not create branch a\\x00b: it holds U+0000, a " + "NUL, which no command can be given as an argument; the leg is left " + "as it is", SECOND_LINE], lines + assert lines[4:] == [did_not_try("spec", "日本.lock", "U+65E5"), + GIT_REFUSES, f" {refused}", SECOND_LINE, + ""], lines From 7777c7d0859fba4c8067f1d01ed6aec8ca047591 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 18:52:16 +0000 Subject: [PATCH 3/3] bootstrap.py: git is asked whether a branch this system's encoding cannot write is good in the root, not in the leg, so a leg whose `.git` file names a git dir that is gone no longer turns a good name into one git refuses (#710) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix round 2 for RV824's re-check of #824 (lane 3, issuecomment-6237945041, READY), its R1-N1. `_git_refuses_in_utf8` ran `git check-ref-format --branch` in the leg. `--branch` makes git set up the repository it runs in, so in a spec leg whose `.git` file named a missing git dir it exited 128 (`fatal: not a git repository`). Under `LC_ALL=C PYTHONUTF8=0`, `tracking_branch: "feature/日本"` then read as a name git refuses on that leg, while the code leg in the same run called it good. Git is asked in the root now, the repository bootstrap was found in, and the answer is about the name alone: every leg says the name is good, and the pins validator names the broken leg, as it did. `tests/test_bootstrap_non_utf8_branch.py` gains that row. The mutant that asks in the leg again fails it. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 13 ++++++++--- tests/test_bootstrap_non_utf8_branch.py | 24 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 4b688413..99333635 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -326,7 +326,7 @@ def _say_not_tried(where: str, named: str, fault: str, "bootstrap again.") -def _git_refuses_in_utf8(sub: Path, branch: str) -> list[str] | None: +def _git_refuses_in_utf8(root: Path, branch: str) -> list[str] | None: """What git says of `branch` as a branch name in the bytes a run under UTF-8 would hand it, or `None` when git takes it (#710, RV824's S1). @@ -340,9 +340,16 @@ def _git_refuses_in_utf8(sub: Path, branch: str) -> list[str] | None: back as exactly those bytes under any locale. On Windows no character is `UNWRITABLE` (its `os.fsencode` writes UTF-8, or replaces), so this is never asked there. + + ASKED IN THE ROOT, not in the leg (RV824's R1-N1). `--branch` makes git + set up the repository it runs in, so in a leg whose `.git` file names a + git dir that is gone it exits 128 (`not a git repository`), and a good + name read as one git refuses there, while the same name was good in the + next leg. The root is the repository this command was found in, and + the answer is about the name alone. """ utf8 = os.fsdecode(branch.encode("utf-8", "surrogateescape")) - return _git_says(["check-ref-format", "--branch", utf8], sub) + return _git_says(["check-ref-format", "--branch", utf8], root) def _short(sha: str | None) -> str: @@ -923,7 +930,7 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: fault = _argument_fault(branch) if fault is not None: _say_not_tried(where, named, fault, - _git_refuses_in_utf8(sub, branch) + _git_refuses_in_utf8(root, branch) if fault.endswith(UNWRITABLE) else None) return pin = recorded_gitlink(root, path) diff --git a/tests/test_bootstrap_non_utf8_branch.py b/tests/test_bootstrap_non_utf8_branch.py index 4a5dded6..fcd595af 100644 --- a/tests/test_bootstrap_non_utf8_branch.py +++ b/tests/test_bootstrap_non_utf8_branch.py @@ -427,6 +427,30 @@ def test_a_legs_own_branch_this_encoding_cannot_write_is_said_too( assert legs_state(root) == before + +def test_a_leg_whose_git_dir_is_gone_still_has_a_good_name_called_good( + root, tmp_path): + """RV824's R1-N1: git is asked about the name in the ROOT. Asked in + the leg, a spec leg whose `.git` file names a git dir that is gone made + `check-ref-format --branch` exit 128 (`fatal: not a git repository`), + and `feature/日本` read as a name git refuses on that leg alone while + the code leg called it good. Every leg calls it good now, and the pins + validator names the broken leg, as it did: the run ends on it, as + `FAILED` or as #764's `REFUSED`, whichever this tree ends a run that + exits 2 with.""" + changes = under_ascii(tmp_path) + tracking(root, f'"{GOOD}"') + (root / "spec" / ".git").write_bytes(b"gitdir: ../.git/modules/gone\n") + + code, out, err = run_under(root, tmp_path, changes) + said = err.decode("utf-8") + assert "Traceback" not in said, said + assert step_a(out) == (unwritten("spec", GOOD_PRINTED, "U+65E5") + + unwritten("code", GOOD_PRINTED, "U+65E5")), out + assert code != 0, out + said + assert ": pins (exit 2)" in said, said + + class LegacyWindowsOs: """`os` as Windows has it under `PYTHONLEGACYWINDOWSFSENCODING=1`: `nt`, with an `os.fsdecode` that is `mbcs` with `replace`, so a