diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index f92f8047..99333635 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -41,7 +41,13 @@ file is. A branch no command can be given (a NUL, or a lone surrogate this system cannot encode) is never tried: the leg is left as it is and the line says why (#455); step (b)'s manifest - validator names the key when the manifest holds it. Advancing a pin + validator names the key when the manifest holds it. Nor is a branch + holding a character this system's encoding cannot write (#710): where + git takes the name in UTF-8, its line says the name is good and to + run bootstrap under a UTF-8 locale, and where git refuses it there + too, the line gives git's words. Every name git prints back is read + as bytes and decoded so that it reaches git again as the bytes it + was (#710). Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -131,6 +137,10 @@ "Remediation: put git on PATH and run this again. Step (a) reads each " "leg's pin and moves the leg through git, and the validators read the " "pins through git, so without it nothing here can be placed or checked.") +#: What `_argument_fault` calls a character this system's encoding cannot +#: write (#710): the one fault of a name that is good, so `_say_not_tried` +#: gives it a line of its own. +UNWRITABLE = "a character this system's encoding cannot write" REGISTER_RELPATH = Path("governance") / "review-authority" / "register.yaml" VALIDATORS = ( ("naming", ["scripts/validate-repository-naming.py", "--project", @@ -151,10 +161,44 @@ def _run(root: Path, args: list[str]) -> int: def _try_git(args: list[str], cwd: Path) -> str | None: + """What `git ` printed in `cwd`, stripped, or `None` where git + exited non-zero. + + STEP (a)'S ONE DECODE RULE (#710). What git prints is read as BYTES and + decoded the way the name is encoded when git is handed it again. On + POSIX that is `os.fsdecode`, as `_head_of` and `_in_progress` read + theirs: the inverse of the `os.fsencode` that `subprocess` applies to + each argument, so a name git prints reaches git again as the same bytes + under any locale. On Windows `subprocess` encodes no argument: it hands + the command line to `CreateProcessW` as UTF-16, and git for Windows + reads it, and writes a ref name, in UTF-8. So the bytes are decoded as + UTF-8 there, as `git_out` decoded them, and not by `os.fsdecode`, which + `PYTHONLEGACYWINDOWSFSENCODING=1` makes `mbcs` with `replace`, losing + every character the code page lacks (RV824's N3). Through `git_out`'s + strict UTF-8, `_upstream_holding` ended `make bootstrap` in a + `UnicodeDecodeError` traceback over an upstream named with a byte that + is not UTF-8 (`refs/remotes/origin/ab`), and, under an `ascii` + encoding (`LC_ALL=C PYTHONUTF8=0`), in a `UnicodeEncodeError` over one + named in UTF-8, since the text it read could not be handed back to + `rev-parse`. + + `git_out` itself keeps its strict decode, as #710's ruling took Option + 2: the readers of `origin`'s URL in `shape-doctor.py` and in the + advisory it reads take its `UnicodeDecodeError` to mean the URL names + nothing (#385). + + The BYTES are stripped, of the ASCII whitespace git ends a line with, + and not the text: `str.strip` also takes away a U+00A0 or a U+3000 that + ends a name, so `refs/remotes/origin/a` was read as `.../a`, and + a branch with no commit of its own past the pin was not moved back. + """ try: - return git_out(args, cwd=cwd) + raw = git_out(args, cwd=cwd, binary=True).strip() except Refusal: return None + # #710's ruling, Option 2: the decode is step (a)'s own, here, and + # `repo_shape.git_out` keeps its strict UTF-8 for every other caller. + return raw.decode("utf-8") if os.name == "nt" else os.fsdecode(raw) @contextmanager @@ -199,8 +243,21 @@ def _argument_fault(branch: str) -> str | None: exception `_try_git` would then have to swallow: a `UnicodeDecodeError`, which `git_out` raises over output that is not UTF-8, is a `ValueError` too, and catching that there would read a leg whose HEAD is on such a - branch as detached. Only a NUL and a lone surrogate are looked for; a - character the system's encoding merely lacks is not this fix's. + branch as detached. + + A CHARACTER THIS SYSTEM'S ENCODING CANNOT WRITE IS A FAULT TOO (#710, + Brett Heap's ruling (a) on L4-124), named `UNWRITABLE`. Under + `LC_ALL=C PYTHONUTF8=0` the encoding is `ascii`, and + `tracking_branch: "feature/日本"`, which `validate-manifest.py` passes + and git takes under a UTF-8 locale, ended step (a) in a + `UnicodeEncodeError` traceback from `subprocess.py`. The leg is left as + it is, and `_say_not_tried` says why. Its line can call the name good, + so this fault is asked last, and a name that holds a surrogate at all + is named for the surrogate instead, even one that is one byte of a name + (RV824's S1): `validate-manifest.py` names every surrogate (#280), so + `"日\\udcff"` is not a good name, and #455's line is the true one. + Whether git takes the name is asked before a line calls it good + (`_git_refuses_in_utf8`). A surrogate that is ONE BYTE of a name is not refused. POSIX spells a name that is not UTF-8 with U+DC80 to U+DCFF and `os.fsencode` writes each back @@ -216,7 +273,15 @@ def _argument_fault(branch: str) -> str | None: return "U+0000, a NUL" if "\ud800" <= char <= "\udfff" and _fsencoded_length(char) != 1: return f"U+{ord(char):04X}, a lone surrogate" - return None + unwritable = next((char for char in branch + if _fsencoded_length(char) == 0), None) + if unwritable is None: + return None + surrogate = next((char for char in branch + if "\ud800" <= char <= "\udfff"), None) + if surrogate is not None: + return f"U+{ord(surrogate):04X}, a lone surrogate" + return f"U+{ord(unwritable):04X}, {UNWRITABLE}" def _fsencoded_length(char: str) -> int: @@ -227,18 +292,66 @@ def _fsencoded_length(char: str) -> int: return 0 -def _say_not_tried(where: str, named: str, fault: str) -> None: +def _say_not_tried(where: str, named: str, fault: str, + refused: list[str] | None = None) -> None: """A branch `_argument_fault` found: no command was run with it. The leg stays where it is, and the line says which of the three places the - person's value can have come from holds it (#455).""" - print(f" {where}: could not create branch {named}: it holds {fault}, " - "which no command can be given as an argument; the leg is left as " - "it is") + person's value can have come from holds it (#455). + + A character this system's encoding cannot write (`UNWRITABLE`, #710) is + a branch step (a) DID NOT TRY, not one it could not create: the branch + can be there already, made by a run under UTF-8 (RV824's N2). Where git + takes the name in UTF-8 (`refused` is `None`), the line is the one Brett + Heap ruled on L4-124: the name is good, the encoding is named + (`sys.getfilesystemencoding()`, the one `os.fsencode` writes with), and + the two ways to run bootstrap under UTF-8 follow, a UTF-8 locale or + Python's UTF-8 mode. Where git refuses it in UTF-8 too, `refused` is + git's lines (`_git_refuses_in_utf8`): the name is not called good, git's + words are said, and #455's way out follows, since the name itself has + to change (RV824's S1).""" + unwritable = fault.endswith(UNWRITABLE) + tried = "did not try" if unwritable else "could not create" + print(f" {where}: {tried} branch {named}: it holds {fault}, which no " + "command can be given as an argument; the leg is left as it is") + if unwritable and refused is None: + print(" the name is good, and this system's encoding, " + f"{sys.getfilesystemencoding()}, cannot write it: run bootstrap " + "again under `LC_ALL=C.UTF-8`, or with `PYTHONUTF8=1`.") + return + if unwritable: + print(" and git refuses it as a branch name in UTF-8 too:") + _say_lines(refused) print(" the branch is the leg's `branch:`, else `--branch`, else " "`tracking_branch`. Give that one a name git can create and run " "bootstrap again.") +def _git_refuses_in_utf8(root: Path, branch: str) -> list[str] | None: + """What git says of `branch` as a branch name in the bytes a run under + UTF-8 would hand it, or `None` when git takes it (#710, RV824's S1). + + A name `_argument_fault` faults as `UNWRITABLE` is not called good until + git has been asked: `feature/日本..x`, `日本.lock` and `日本 x` are names + this system's encoding cannot write AND names git refuses in any + encoding, and the way out the line gave ended in `fatal: ... is not a + valid branch name`. `git check-ref-format --branch` is git's own answer. + It is asked of `branch.encode("utf-8", "surrogateescape")`, handed to + `subprocess` through `os.fsdecode`, which POSIX's `os.fsencode` writes + back as exactly those bytes under any locale. On Windows no character + is `UNWRITABLE` (its `os.fsencode` writes UTF-8, or replaces), so this + is never asked there. + + ASKED IN THE ROOT, not in the leg (RV824's R1-N1). `--branch` makes git + set up the repository it runs in, so in a leg whose `.git` file names a + git dir that is gone it exits 128 (`not a git repository`), and a good + name read as one git refuses there, while the same name was good in the + next leg. The root is the repository this command was found in, and + the answer is about the name alone. + """ + utf8 = os.fsdecode(branch.encode("utf-8", "surrogateescape")) + return _git_says(["check-ref-format", "--branch", utf8], root) + + def _short(sha: str | None) -> str: return sha[:12] if sha else "?" @@ -301,7 +414,11 @@ def _upstream_holding(sub: Path, branch: str, local: str) -> str | None: remote-tracking ref is not there (never fetched, or pruned) and one whose tip is not the upstream's. The two are compared by object id. `for-each-ref` names the upstream whether or not its ref exists, and - takes the branch after `refs/heads/`, where git reads no option. + takes the branch after `refs/heads/`, where git reads no option. Its + name is read by `_try_git`'s rule, so on POSIX an upstream named with + a byte that is not UTF-8, or in UTF-8 under an `ascii` encoding, is the + same ref when `rev-parse` is handed it, and no longer a traceback + (#710). """ upstream = _try_git(["for-each-ref", "--format=%(upstream)", f"refs/heads/{branch}"], sub) @@ -812,7 +929,9 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: return fault = _argument_fault(branch) if fault is not None: - _say_not_tried(where, named, fault) + _say_not_tried(where, named, fault, + _git_refuses_in_utf8(root, branch) + if fault.endswith(UNWRITABLE) else None) return pin = recorded_gitlink(root, path) local = _try_git(["rev-parse", "--verify", "--quiet", diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index 5a79ff8e..5418705f 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -81,6 +81,20 @@ "and run bootstrap again.") +def written(branch: str): + """`branch` as a row of the no-fault test, skipped where this system's + encoding cannot write it: there `_argument_fault` faults it as #710's + `UNWRITABLE`, which `tests/test_bootstrap_non_utf8_branch.py` holds + (RV824's S2). Asked at collection, of `os.fsencode` itself.""" + try: + os.fsencode(branch) + except UnicodeEncodeError: + return pytest.param(branch, marks=pytest.mark.skip( + reason="this system's encoding cannot write it, so it is #710's " + "UNWRITABLE fault, held in test_bootstrap_non_utf8_branch")) + return branch + + def not_tried(leg: str, printed: str, point: str, kind: str) -> list[str]: return [f" [{leg}] {leg}: could not create branch {printed}: it holds " f"{point}, {kind}, which no command can be given as an " @@ -408,11 +422,14 @@ def validator_module(): @pytest.mark.parametrize("branch", [ - "main", "release/1.2", "feature/日本", "a‍b", "a\xe9b", "a b", + "main", "release/1.2", written("feature/日本"), written("a‍b"), + written("a\xe9b"), "a b", "a\x1bb", "a..b", "", "\udc80", "a\udcffb"]) def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): - """Nothing but a NUL and a lone surrogate with no byte is a fault: a - control, a space, `..`, a character the system's encoding may lack, and + """Where the system's encoding writes every character of the branch, + nothing but a NUL and a lone surrogate with no byte is a fault: a + control, a space, `..`, a character that is not ASCII (`written` skips + one the encoding cannot write, #710's `UNWRITABLE`), and the byte-carrying surrogates U+DC80 to U+DCFF are the commands'. On a system whose `os.fsencode` does not carry U+DC80 and U+DCFF (Windows) the next test is the one that holds.""" diff --git a/tests/test_bootstrap_non_utf8_branch.py b/tests/test_bootstrap_non_utf8_branch.py new file mode 100644 index 00000000..fcd595af --- /dev/null +++ b/tests/test_bootstrap_non_utf8_branch.py @@ -0,0 +1,547 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A name git prints is handed back to git as the bytes it was, and a good +name this system's encoding cannot write is said, not a traceback (#710). + +Step (a) of the assembly root's `scripts/bootstrap.py` reads names back from +git (the ref HEAD is on, a branch's upstream) and hands them to git again as +arguments. Each read went through `repo_shape.git_out`, which decodes what git +printed as strict UTF-8. #745 (#704) read HEAD's ref as bytes; `_try_git`, the +reader of every other answer, still decoded strictly. It now reads bytes and +decodes them with `os.fsdecode`, the inverse of the `os.fsencode` that +`subprocess` applies to an argument, so the name goes back to git as the same +bytes under any POSIX locale. `git_out` itself is unchanged: the readers of +`origin`'s URL in `shape-doctor.py` and `shape_advisory.py` take its +`UnicodeDecodeError` to mean the URL names nothing (#385). + +THE ISSUE'S LINES, each measured on main 24a977c0 (#745's squash) before this +change, in a scaffold's recursive clone bootstrapped once: + +- LANE 5'S SITE (issuecomment-6226993443). A tracking branch past the pin with + no commit of its own, its name and upstream not UTF-8, and the leg detached + at the pin: `UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in + position 21`, exit 1, from `_upstream_holding`. Now `moved back`. +- THE SAME READ, STRIPPED AS TEXT. `str.strip` takes a U+00A0 off the end of + the upstream's name, so `refs/remotes/origin/a` was asked as `.../a` + and the branch was not moved back (`not moving an existing branch`). The + bytes are stripped now, so the name keeps every byte. +- THE SAME READ UNDER `ascii`. `main` tracking `origin/日本` past the pin, under + `LC_ALL=C PYTHONUTF8=0`: `UnicodeEncodeError: 'ascii' codec can't encode + characters in position 20-21`, exit 1, since the upstream's name read as + UTF-8 text could not be handed back to `rev-parse`. Now `moved back`. +- L4-125, the issue's own route: a leg on `ab` (`git branch $'a\\xffb'`), + bootstrapped twice. Exit 0 on main already, since #745 read HEAD's ref as + bytes; held here, as the issue asks, by `make bootstrap` and `--branch`. +- L4-126, answered by option (b) as R455 recommended: `tracking_branch: + "a\\udcffb"` is created by step (a) and named by the validator (exit 1); the + manifest edited back to `main`, the next run places every leg (exit 0, a + traceback before #745). +- L4-124, Brett Heap's ruling (a) of 2026-10-17T13:01:07Z. Under + `LC_ALL=C PYTHONUTF8=0`, `tracking_branch: "feature/日本"` passed the + validator (`manifest ok`) and ended step (a) in `UnicodeEncodeError: 'ascii' + codec can't encode characters in position 19-20`, exit 1. `_argument_fault` + now faults a character `os.fsencode` cannot write, the leg is left as it is, + and the line of its own says the name is good, names the encoding and gives + the two ways to run bootstrap under UTF-8; each is run here as printed. +- RV824's review of #824, on the ruled line: it is said only where git takes + the name in UTF-8 (`git check-ref-format --branch`), so `feature/日本..x`, + `日本.lock` and `日本 x` get git's own words instead (S1); a name that also + holds a surrogate is named for the surrogate, as the validator names it + (S1); the first line says `did not try` (N2); a leg's own `branch:` takes + the same path (N4); and on Windows git's bytes are read as UTF-8 (N3). + +A run that needs the byte 0xff in a file name skips where the disk refuses it +(or Windows, where `_argument_fault` refuses U+DCFF), and a run under `ascii` +skips where `LC_ALL=C PYTHONUTF8=0` gives Python no `ascii` encoding (macOS and +Windows keep UTF-8). The rule itself is held on every platform by an +emulated `ascii` `os.fsencode`. A test that hands git a name only UTF-8 can +write skips where pytest itself does not run under UTF-8 (N5). +""" + +from __future__ import annotations + +import codecs +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import git, run_script +from test_bootstrap_after_bump import ( # noqa: F401 + MAIN, bootstrap, bootstrapped, head_ref, moved_back, rev, root, +) +from test_bootstrap_branch_bytes import ( # noqa: F401 + SECOND_LINE, bootstrap_module, leg_branch, legs_state, not_tried, + tracking, +) +from test_bootstrap_spelling import edit +from test_stdout_encoding import child_environ, run_tool_bytes + +#: Where L4-124 was measured: the C locale with Python's UTF-8 mode off, +#: which makes the filesystem encoding `ascii`. +ASCII = {"LC_ALL": "C", "PYTHONUTF8": "0"} +#: The two ways out the line gives, each as the environment it names. +WAYS_OUT = [ + pytest.param({"LC_ALL": "C.UTF-8", "PYTHONUTF8": "0"}, + id="LC_ALL=C.UTF-8"), + pytest.param({"LC_ALL": "C", "PYTHONUTF8": "1"}, id="PYTHONUTF8=1"), +] +#: The good name L4-124 measured, and what an `ascii` stdout prints of it. +GOOD = "feature/日本" +GOOD_PRINTED = r"feature/\u65e5\u672c" +#: What `_argument_fault` calls the character (`UNWRITABLE`). +UNWRITABLE = "a character this system's encoding cannot write" +#: The line under a name git refuses in UTF-8 too, before git's own words. +GIT_REFUSES = " and git refuses it as a branch name in UTF-8 too:" +#: A test that hands git a name only UTF-8 can write, or reads a line a +#: child under this process's locale writes in UTF-8 (`—`), needs this +#: process itself under UTF-8 (RV824's N5). CI's is; a run of pytest under +#: `LC_ALL=C PYTHONUTF8=0` is not, and `conftest.git` cannot hand git such +#: a name there. +NEEDS_A_UTF8_PROCESS = pytest.mark.skipif( + codecs.lookup(sys.getfilesystemencoding()).name != "utf-8", + reason="this pytest process's filesystem encoding is not UTF-8, so it " + "cannot hand git the name or read the child's UTF-8 line") + + +def own_line(encoding: str) -> str: + """The line of its own under a good name the encoding cannot write.""" + return (f" the name is good, and this system's encoding, " + f"{encoding}, cannot write it: run bootstrap again under " + "`LC_ALL=C.UTF-8`, or with `PYTHONUTF8=1`.") + + +def unwritten(leg: str, printed: str, point: str) -> list[str]: + """Step (a)'s two lines for a leg whose good branch was not tried.""" + return [did_not_try(leg, printed, point), own_line("ascii")] + + +def did_not_try(leg: str, printed: str, point: str) -> str: + """The first line for a leg whose branch holds `UNWRITABLE` (N2).""" + return (f" [{leg}] {leg}: did not try branch {printed}: it holds " + f"{point}, {UNWRITABLE}, which no command can be given as an " + "argument; the leg is left as it is") + + +def needs_the_byte(tmp_path: Path) -> None: + """Skip where a name holding the byte 0xff cannot be made: Windows, + where `_argument_fault` refuses U+DCFF, and a disk that refuses it.""" + if os.name == "nt": + pytest.skip("U+DCFF is three bytes on Windows, and step (a) does not " + "try such a branch at all") + try: + (tmp_path / "probe-\udcff").write_bytes(b"") + except (OSError, ValueError): + pytest.skip("this disk takes no file name with the byte 0xff") + + +def encoding_under(tmp_path: Path, changes: dict[str, str]) -> str: + """The filesystem encoding a child Python gets under `changes`.""" + return subprocess.run( + [sys.executable, "-c", + "import sys; print(sys.getfilesystemencoding())"], + env=child_environ(tmp_path, None, changes), capture_output=True, + text=True, check=True).stdout.strip() + + +def under_ascii(tmp_path: Path) -> dict[str, str]: + """`ASCII`, or a skip where it gives a child Python no `ascii`.""" + if encoding_under(tmp_path, ASCII) != "ascii": + pytest.skip("`LC_ALL=C PYTHONUTF8=0` gives this system's Python no " + "`ascii` filesystem encoding") + return ASCII + + +def run_under(root: Path, tmp_path: Path, changes: dict[str, str], + *args: str) -> tuple[int, str, bytes]: + """`bootstrap.py ` under `changes`, with no `PYTHONIOENCODING`, so + the stdout's encoding is the locale's: `(exit, stdout, stderr)`.""" + code, out, err = run_tool_bytes(root / "scripts" / "bootstrap.py", *args, + encoding=None, tmp_path=tmp_path, cwd=root, + changes=changes) + return code, out.decode("utf-8"), err + + +def head_bytes(leg: Path) -> bytes: + return subprocess.run(["git", "symbolic-ref", "--quiet", "HEAD"], + cwd=str(leg), capture_output=True, + check=True).stdout + + +def step_a(out: str) -> list[str]: + lines = out.split("\n") + return lines[lines.index("(a) legs on tracking branches, pins untouched") + + 1:lines.index("(b) neutral validators") - 1] + + +def past_the_pin(leg: Path, upstream: str, branch: str) -> tuple[str, str]: + """`branch` one commit past the pin, with no commit of its own: its tip + IS `refs/remotes/origin/`, which it tracks, and the leg is + detached at the pin. Returns the pin and that commit.""" + pin = rev(leg) + ahead = git("commit-tree", rev(leg, f"{pin}^{{tree}}"), "-p", pin, "-m", + "the remote moved on", cwd=leg).stdout.strip() + git("update-ref", f"refs/remotes/origin/{upstream}", ahead, cwd=leg) + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-f", "--track", branch, f"origin/{upstream}", + cwd=leg) + return pin, ahead + + +# --- the upstream's name goes back to git as the bytes it was --------------- + +@pytest.mark.parametrize("raw, byte", [ + pytest.param("a\udcffb", True, id="the-byte-0xff"), + pytest.param("a\u00a0", False, id="ending-in-U+00A0", + marks=NEEDS_A_UTF8_PROCESS), +]) +def test_a_branch_past_the_pin_is_moved_back_whatever_its_upstream_is_named( + root, tmp_path, bootstrap_module, raw, byte): + """LANE 5'S SITE: on main, a `UnicodeDecodeError` traceback over the + byte 0xff, and, over a name ending in U+00A0, `not moving an existing + branch`. Both are moved back now, the line naming the upstream and the + command as `move_back_to_pin` spells them, and HEAD is on the branch + whose name is the bytes git was given.""" + if byte: + needs_the_byte(tmp_path) + leg = root / "spec" + pin, ahead = past_the_pin(leg, raw, raw) + + result = run_script(root / "scripts" / "bootstrap.py", "--skip-validators", + "--branch", raw, cwd=root) + assert result.returncode == 0, result.stderr + result.stdout + assert "Traceback" not in result.stderr, result.stderr + spelled = bootstrap_module.spelled + command = ("git -C spec merge --ff-only " + f"{spelled(shlex.quote('origin/' + raw))}") + assert moved_back(ahead, pin, spelled(raw), command=command) in ( + result.stdout), result.stdout + assert rev(leg, f"refs/heads/{raw}") == pin + assert rev(leg) == pin + assert head_bytes(leg) == b"refs/heads/" + os.fsencode(raw) + b"\n" + + +@NEEDS_A_UTF8_PROCESS +def test_under_an_ascii_encoding_an_upstream_named_in_utf8_is_moved_back( + root, tmp_path, bootstrap_module): + """`main` tracking `origin/日本`, past the pin, under `LC_ALL=C + PYTHONUTF8=0`: a `UnicodeEncodeError` traceback on main, since the name + read as UTF-8 text could not be handed back to `rev-parse`. Decoded as + `os.fsdecode` decodes it, it is the same ref, and the remote-tracking + ref still holds the commit.""" + changes = under_ascii(tmp_path) + leg = root / "spec" + pin, ahead = past_the_pin(leg, "日本", "main") + + code, out, err = run_under(root, tmp_path, changes, "--skip-validators") + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + held = bootstrap_module.spelled( + "origin/" + "日本".encode("utf-8").decode("ascii", "surrogateescape")) + assert (f" [spec] spec: branch main moved back {ahead[:12]} -> " + f"{pin[:12]}, the pin; nothing is lost, {held} still holds those " + f"commits, and `git -C spec merge --ff-only '{held}'` brings the " + "branch forward\n") in out, out + assert rev(leg, MAIN) == pin + assert head_ref(leg) == MAIN + assert rev(leg, "refs/remotes/origin/日本") == ahead + + +# --- L4-125 and L4-126: every later run places the leg --------------------- + +@pytest.mark.parametrize("given", [False, True], + ids=["make-bootstrap", "--branch-ab"]) +def test_a_leg_on_a_branch_whose_name_is_not_utf8_is_placed_on_every_run( + root, tmp_path, given): + """L4-125: `git branch $'a\\xffb'`, the leg checked out on it, then + bootstrap twice. Without `--branch` each run puts the leg on `main`, + the first saying the leg left `ab`; with `--branch $'a\\xffb'` each + run says `on` it. Every run exits 0 with the validators run.""" + needs_the_byte(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("branch", "a\udcffb", cwd=leg) + git("checkout", "-q", "a\udcffb", cwd=leg) + args = ("--branch", "a\udcffb") if given else () + on = r"a\udcffb" if given else "main" + left = (" the leg was on a\\udcffb, which keeps its commits; it " + "is on main now\n") + for run in (1, 2): + result = run_script(root / "scripts" / "bootstrap.py", *args, + cwd=root) + assert result.returncode == 0, result.stderr + result.stdout + assert "Traceback" not in result.stderr, result.stderr + assert result.stdout.endswith("bootstrap ok\n"), result.stdout + placed = f" [spec] spec: on {on} at {pin[:12]} (branch tip == pin)\n" + assert placed in result.stdout, result.stdout + assert (placed + left in result.stdout) is ( + not given and run == 1), result.stdout + assert head_bytes(leg) == (b"refs/heads/a\xffb\n" if given + else b"refs/heads/main\n") + + +@NEEDS_A_UTF8_PROCESS +def test_a_tracking_branch_the_validator_names_is_followed_and_then_placed( + root, tmp_path): + """L4-126, option (b): step (a) creates `ab` from the manifest's + `"a\\udcffb"` and the validator names it, exit 1. The finding followed, + `main` again, the next run puts every leg on `main` and exits 0, where + it ended in a `UnicodeDecodeError` before #745.""" + needs_the_byte(tmp_path) + pins = {leg: rev(root / leg) for leg in ("spec", "code")} + tracking(root, r'"a\udcffb"') + first = bootstrap(root) + assert first.returncode == 1, first.stderr + first.stdout + assert "Traceback" not in first.stderr, first.stderr + assert ("FINDING manifest-tracking-branch: tracking_branch is " + "'a\\udcffb', which holds U+DCFF") in first.stderr, first.stderr + assert "bootstrap FAILED: manifest (exit 1)" in first.stderr + for leg, pin in pins.items(): + assert (f" [{leg}] {leg}: branch a\\udcffb created at {pin[:12]} " + "— at the pin\n") in first.stdout, first.stdout + assert head_bytes(root / leg) == b"refs/heads/a\xffb\n" + + edit(root, "project.yaml", r'tracking_branch: "a\udcffb"', + 'tracking_branch: "main"') + second = bootstrap(root) + assert second.returncode == 0, second.stderr + second.stdout + assert second.stdout.endswith("bootstrap ok\n"), second.stdout + for leg, pin in pins.items(): + assert (f" [{leg}] {leg}: on main at {pin[:12]} (branch tip == " + f"pin)\n the leg was on a\\udcffb, which keeps its " + "commits; it is on main now\n") in second.stdout, second.stdout + assert head_ref(root / leg) == MAIN + + +# --- L4-124: a good name the encoding cannot write is said ------------------ + +@pytest.mark.parametrize("way_out", WAYS_OUT) +def test_a_good_name_this_encoding_cannot_write_is_said_and_the_way_out_works( + root, tmp_path, way_out): + """Under `LC_ALL=C PYTHONUTF8=0`, `tracking_branch: "feature/日本"`: + the validators pass it, as they did, and step (a) ended in a + `UnicodeEncodeError` traceback. Now each leg is left exactly as it was + under the two lines, and the run ends `bootstrap ok`. Then the way out, + run as the line gives it, puts each leg on `feature/日本`.""" + changes = under_ascii(tmp_path) + if encoding_under(tmp_path, way_out) != "utf-8": + pytest.skip(f"{way_out} gives this system's Python no UTF-8") + tracking(root, f'"{GOOD}"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + assert step_a(out) == (unwritten("spec", GOOD_PRINTED, "U+65E5") + + unwritten("code", GOOD_PRINTED, "U+65E5")), out + assert "manifest ok: Atlas (atlas), 3 legs" in out, out + assert out.endswith("bootstrap ok\n"), out + assert legs_state(root) == before + + code, out, err = run_under(root, tmp_path, way_out) + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + for leg in ("spec", "code"): + assert f" [{leg}] {leg}: branch {GOOD} created at " in out, out + assert head_bytes(root / leg) == ( + b"refs/heads/" + GOOD.encode("utf-8") + b"\n") + + +@pytest.mark.parametrize("name, printed", [ + pytest.param("feature/日本..x", r"feature/\u65e5\u672c..x", id="dot-dot"), + pytest.param("日本.lock", r"\u65e5\u672c.lock", id="dot-lock"), + pytest.param("日本 x", r"\u65e5\u672c x", id="a-space"), +]) +def test_a_name_git_refuses_in_utf8_too_is_never_called_good( + root, tmp_path, name, printed): + """RV824's S1: each of these names this system's encoding cannot + write, and git refuses in any encoding (`..`, `.lock`, a space), was + called good, with a way out that ended in git's `is not a valid branch + name`. Now `git check-ref-format --branch` is asked of the UTF-8 bytes + first: git's words are said, and #455's way out, and each leg is left + as it was. The validators pass the value, as they did, so the run + ends `bootstrap ok`, as a branch git refuses always has.""" + changes = under_ascii(tmp_path) + tracking(root, f'"{name}"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + assert "the name is good" not in out, out + lines = step_a(out) + for leg in ("spec", "code"): + at = lines.index(did_not_try(leg, printed, "U+65E5")) + said = lines[at + 2:lines.index(SECOND_LINE, at)] + assert lines[at + 1] == GIT_REFUSES, lines + assert said, lines + assert any(printed in line for line in said), said + assert lines.count(SECOND_LINE) == 2, lines + assert out.endswith("bootstrap ok\n"), out + assert legs_state(root) == before + + +def test_a_name_holding_a_surrogate_too_is_named_for_the_surrogate( + root, tmp_path): + """RV824's S1: `"日\\udcff"` under `LC_ALL=C PYTHONUTF8=0` was called + good in step (a), while step (b) of the same run named its U+DCFF. + The surrogate is the fault now, with #455's lines, and the run ends as + the validator ends it.""" + changes = under_ascii(tmp_path) + tracking(root, r'"日\udcff"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + said = err.decode("utf-8") + assert code == 1, out + said + assert "Traceback" not in said, said + assert "the name is good" not in out, out + printed = r"\u65e5\udcff" + assert step_a(out) == ( + not_tried("spec", printed, "U+DCFF", "a lone surrogate") + + not_tried("code", printed, "U+DCFF", "a lone surrogate")), out + assert "FINDING manifest-tracking-branch:" in said, said + assert "bootstrap FAILED: manifest (exit 1)" in said, said + assert legs_state(root) == before + + +def test_a_legs_own_branch_this_encoding_cannot_write_is_said_too( + root, tmp_path): + """RV824's N4: a leg's own `branch:` reaches `_argument_fault` as + `tracking_branch` does. Under `LC_ALL=C PYTHONUTF8=0`, `branch: + "feature/日本"` on the spec leg leaves that leg as it was under the + ruled lines, and the code leg, on `main`, is placed as it always was.""" + changes = under_ascii(tmp_path) + leg_branch(root, "spec", f'"{GOOD}"') + before = legs_state(root) + + code, out, err = run_under(root, tmp_path, changes) + assert code == 0, out + err.decode("utf-8") + assert b"Traceback" not in err, err.decode("utf-8") + assert step_a(out) == unwritten("spec", GOOD_PRINTED, "U+65E5") + [ + f" [code] code: on main at {rev(root / 'code')[:12]} (branch tip == " + "pin)"], out + assert out.endswith("bootstrap ok\n"), out + assert legs_state(root) == before + + + +def test_a_leg_whose_git_dir_is_gone_still_has_a_good_name_called_good( + root, tmp_path): + """RV824's R1-N1: git is asked about the name in the ROOT. Asked in + the leg, a spec leg whose `.git` file names a git dir that is gone made + `check-ref-format --branch` exit 128 (`fatal: not a git repository`), + and `feature/日本` read as a name git refuses on that leg alone while + the code leg called it good. Every leg calls it good now, and the pins + validator names the broken leg, as it did: the run ends on it, as + `FAILED` or as #764's `REFUSED`, whichever this tree ends a run that + exits 2 with.""" + changes = under_ascii(tmp_path) + tracking(root, f'"{GOOD}"') + (root / "spec" / ".git").write_bytes(b"gitdir: ../.git/modules/gone\n") + + code, out, err = run_under(root, tmp_path, changes) + said = err.decode("utf-8") + assert "Traceback" not in said, said + assert step_a(out) == (unwritten("spec", GOOD_PRINTED, "U+65E5") + + unwritten("code", GOOD_PRINTED, "U+65E5")), out + assert code != 0, out + said + assert ": pins (exit 2)" in said, said + + +class LegacyWindowsOs: + """`os` as Windows has it under `PYTHONLEGACYWINDOWSFSENCODING=1`: + `nt`, with an `os.fsdecode` that is `mbcs` with `replace`, so a + character the code page lacks is lost. Only what `bootstrap.py` reads + of it differs; everything else is this system's `os`.""" + + name = "nt" + + @staticmethod + def fsdecode(raw: bytes) -> str: + return raw.decode("ascii", "replace") + + def __getattr__(self, attr: str): + return getattr(os, attr) + + +@NEEDS_A_UTF8_PROCESS +def test_on_windows_what_git_prints_is_read_as_utf8( + root, bootstrap_module, monkeypatch): + """RV824's N3: on Windows `subprocess` hands git its arguments through + `CreateProcessW` and git reads them as UTF-8, so `_try_git` reads git's + bytes as UTF-8 there. Through `os.fsdecode` in the legacy mode, the + upstream `origin/日本` came back as `origin/??????`, which names no ref. + The stand-in `os` is the module's alone: git still runs here.""" + leg = root / "spec" + past_the_pin(leg, "日本", "main") + monkeypatch.setattr(bootstrap_module, "os", LegacyWindowsOs()) + assert bootstrap_module._try_git( + ["for-each-ref", "--format=%(upstream)", "refs/heads/main"], + leg) == "refs/remotes/origin/日本" + + +def ascii_fsencode(text): + """`os.fsencode` where the filesystem encoding is `ascii`: bytes as they + are, text through `surrogateescape`, as POSIX's handler is.""" + if isinstance(text, bytes): + return text + return text.encode("ascii", "surrogateescape") + + +@pytest.mark.parametrize("branch, fault", [ + (GOOD, f"U+65E5, {UNWRITABLE}"), + ("a\xe9b", f"U+00E9, {UNWRITABLE}"), + # A NUL or a lone surrogate ANYWHERE is named first: that name is not + # good, so the line saying it is would be false. + ("日\x00", "U+0000, a NUL"), + ("日\ud83d", "U+D83D, a lone surrogate"), + # A surrogate that IS one byte of a name is the fault beside a character + # the encoding cannot write: the validator names it (RV824's S1). + ("日\udcff", "U+DCFF, a lone surrogate"), + ("\udc80日", "U+DC80, a lone surrogate"), +]) +def test_under_an_ascii_encoding_what_it_cannot_write_is_the_fault( + bootstrap_module, monkeypatch, branch, fault): + monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) + assert bootstrap_module._argument_fault(branch) == fault + + +@pytest.mark.parametrize("branch", [ + "main", "release/1.2", "", + "a\udcffb", # the byte 0xff, as POSIX spells it + "x\udcc3\udca9", # `--branch xé` typed under the C locale +]) +def test_under_an_ascii_encoding_what_it_writes_has_no_fault( + bootstrap_module, monkeypatch, branch): + """A name the encoding writes, the bytes of one typed under it + included, is carried as it was: `--branch` reaches git as the bytes + the person typed.""" + monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) + assert bootstrap_module._argument_fault(branch) is None + + +def test_the_line_for_a_name_the_encoding_cannot_write_is_its_own( + bootstrap_module, monkeypatch, capsys): + """The second line is the ruling's own, naming the encoding the + process has; #455's NUL keeps its line; and a name git refuses gets + git's words and #455's way out, never "good" (RV824's S1).""" + monkeypatch.setattr(bootstrap_module.os, "fsencode", ascii_fsencode) + say = bootstrap_module._say_not_tried + say("[spec] spec", GOOD, bootstrap_module._argument_fault(GOOD)) + say("[code] code", "a\\x00b", bootstrap_module._argument_fault("a\x00b")) + refused = "fatal: '日本.lock' is not a valid branch name" + say("[spec] spec", "日本.lock", bootstrap_module._argument_fault("日本.lock"), + [refused]) + lines = capsys.readouterr().out.split("\n") + assert lines[:2] == [did_not_try("spec", GOOD, "U+65E5"), + own_line(sys.getfilesystemencoding())], lines + assert lines[2:4] == [ + " [code] code: could not create branch a\\x00b: it holds U+0000, a " + "NUL, which no command can be given as an argument; the leg is left " + "as it is", SECOND_LINE], lines + assert lines[4:] == [did_not_try("spec", "日本.lock", "U+65E5"), + GIT_REFUSES, f" {refused}", SECOND_LINE, + ""], lines