From 118c85a9f4618e04d092ee14bd53d902c3f3c3b3 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 12 Oct 2026 18:04:10 +0000 Subject: [PATCH 1/4] =?UTF-8?q?templates/assembly-root/scripts/bootstrap.p?= =?UTF-8?q?y=20and=20templates/family-root/scripts/bootstrap.py:=20`main`?= =?UTF-8?q?=20escapes=20what=20stdout=20cannot=20encode=20and=20spells=20e?= =?UTF-8?q?ach=20manifest,=20register=20and=20member=20value=20it=20prints?= =?UTF-8?q?,=20so=20`name:=20"a\ud83db"`,=20or=20`=E6=97=A5=E6=9C=AC`=20un?= =?UTF-8?q?der=20`PYTHONIOENCODING=3Dascii`,=20is=20no=20longer=20a=20trac?= =?UTF-8?q?eback=20before=20the=20validators=20run,=20and=20an=20ESC=20or?= =?UTF-8?q?=20CR=20in=20a=20name=20no=20longer=20reaches=20the=20terminal?= =?UTF-8?q?=20(#391)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `main` opened with `print(f"bootstrap: {name} ({root})")`, before any step, and the family root's with the same line. `name: "a\ud83db"` reads as a string holding a lone surrogate, which no UTF-8 stdout can write: `scripts/bootstrap.py` exited 1 with a `UnicodeEncodeError`, `make bootstrap` exited 2, and `validate-manifest.py`, the validator that names the problem, was never reached. `日本` under `PYTHONIOENCODING=ascii` ended the same way. The family root's line also holds a literal dash, so under that stdout it raised for every name, `InkRouter` included. A raw ESC or CR in a name is encodable on every stdout and reached the terminal as the control it is. Both files are shape copies, digest-pinned in the project that holds them, and a project carries `scripts/repo_shape.py` and no `shape_materialize.py`, so neither can call #214's helper. Each file now carries the same two functions: - `escape_unencodable_stdout`, the first statement of `main`. A `strict` or `surrogateescape` stdout is reconfigured to `backslashreplace`, the handler stderr already has, so what it holds is written as it was and what it cannot hold as `日`. Any other handler is left alone and the encoding is never changed. It is `validate-pins.py`'s guard from #295, which is inline there and cannot be imported (a hyphenated name and a validator). - `spelled`, which writes every character `str.isprintable` refuses as the escape Python spells it (`\x1b`, `\r`, `
`, `\ud83d`) and keeps every printable one, a backslash included, so a name a UTF-8 stdout can hold prints byte for byte. `repo_shape.double_quoted` was read first and cannot serve: it quotes, and escapes `"` and `\`. Every print of a value the script read from the manifest, a register row or the disk is spelled: the name and the root on the first line; a leg's role, path and tracking branch in step (a); a register row's holder, act, objects, state and expiry, its path, and the reader's reason when it cannot be read; in the family root, a member's project and directory name and `--members-make`. The member's own command is still handed the raw target. Output for a name every stdout holds is unchanged: the new tests spell out the whole of a plain project's and a plain holder's output. What the run does next is not this change's. `validate-manifest.py` still prints its own `manifest ok: ` through a stdout that has no handler, so on a tree without #280's fix the lone-surrogate row now ends in the validator's traceback and `bootstrap FAILED: manifest (exit 1)`, and with it at `FINDING manifest-name`. Tests: tests/test_bootstrap_spelling.py (new). Each file is run as a real project and a real holder under six stdouts; every other print above has a row; the two functions and the identity of their two copies are tested directly. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 98 ++- templates/family-root/scripts/bootstrap.py | 74 +- tests/test_bootstrap_spelling.py | 687 +++++++++++++++++++ 3 files changed, 834 insertions(+), 25 deletions(-) create mode 100644 tests/test_bootstrap_spelling.py diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index b6c27184..7d5d6485 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -80,6 +80,61 @@ ) +def escape_unencodable_stdout() -> None: + """A character stdout cannot encode is printed escaped, never raised on (#391). + + WHAT WAS WRONG. `main` printed the manifest's `name` with a plain `print` + before any step ran, so `name: "a\\ud83db"` (a lone surrogate, which no + UTF-8 stdout can write) or `name: "日本"` on a stdout that is not UTF-8 + (`PYTHONIOENCODING=ascii`, or the ANSI code page Windows gives a stdout that + is not the console) ended `make bootstrap` in a `UnicodeEncodeError` + traceback one step before `validate-manifest.py`, the validator that names + the problem, was reached. This file's own wording raised on such a stdout + too, wherever it carries a dash or a middle dot. + + A `strict` or `surrogateescape` stdout, the two handlers that raise, is + reconfigured ONCE, at the start of `main`, to `backslashreplace`, the + handler stderr already has: what the stdout holds is written as it was, and + what it cannot hold as `\\u65e5`. Any other handler, one somebody chose, is + left alone, and the ENCODING is never changed. `scripts/validate-pins.py` + does the same for its report (#295). The tools run from a checkout of the + standard use `shape_materialize.escape_what_stdout_cannot_encode` (#214), + which no project carries, and this file imports nothing but `repo_shape`. + This function and `spelled` are the same in the assembly root's + `bootstrap.py` and the family root's. + + GUARDED, because failing to set a handler is no reason to refuse the run: a + stream a caller replaced may have no `reconfigure`, a closed one raises + `ValueError`, and `pythonw` has no stdout at all. + """ + if getattr(sys.stdout, "errors", None) in ("strict", "surrogateescape"): + try: + sys.stdout.reconfigure(errors="backslashreplace") + except (AttributeError, ValueError, OSError): + pass + + +def spelled(value: object) -> str: + """`value` as ONE line of text a terminal can be shown (#391). + + What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on + every stdout, so it reaches the terminal as the control it is, and a + manifest's `name` or a register row is a person's text. Every character + `str.isprintable` refuses is written as the escape Python spells it + (`\\x1b`, `\\r`, `\\n`, `\\u2028`, `\\ud83d`), so the value stays on + its line and a lone surrogate is named, whatever handler the stdout has. A + printable character is kept as it is, a backslash included, so what a UTF-8 + stdout can hold prints byte for byte; the cost is that a name holding the + four characters `\\x1b` prints as one holding ESC does. + + Applied to every value this script prints that it read from the manifest, a + register row or the disk, and to none of its own wording. + """ + return "".join(char if char.isprintable() + else char.encode("unicode_escape").decode("ascii") + for char in str(value)) + + def _run(root: Path, args: list[str]) -> int: # Flush first: this process's own stdout is block-buffered when it is a # pipe, and a child writing straight to the fd would otherwise appear @@ -104,12 +159,15 @@ def _short(sha: str | None) -> str: def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: """Step (a) for one leg. Prints; never raises; never moves the pin.""" path = str(leg.get("path") or "") - role = leg.get("role") sub = root / path + # What is PRINTED of the leg is spelled (#391): `role`, `path` and `branch` + # are the manifest's text, read before any validator has had its say. + where = f"[{spelled(leg.get('role'))}] {spelled(path)}" + named = spelled(branch) if not (sub / ".git").exists(): - print(f" [{role}] {path}: NOT INITIALIZED. Run " - f"`git submodule update --init {path}` — the validators below " - f"will refuse until you do.") + print(f" {where}: NOT INITIALIZED. Run " + f"`git submodule update --init {spelled(path)}` — the " + f"validators below will refuse until you do.") return pin = recorded_gitlink(root, path) local = _try_git(["rev-parse", "--verify", "--quiet", @@ -119,10 +177,10 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: if local is None: if pin is None: - print(f" [{role}] {path}: no gitlink recorded; leaving as is") + print(f" {where}: no gitlink recorded; leaving as is") return if _try_git(["checkout", "-q", "-b", branch, pin], sub) is None: - print(f" [{role}] {path}: could not create branch {branch!r} at " + print(f" {where}: could not create branch {branch!r} at " f"the pin {_short(pin)}; leaving detached") return if remote is not None: @@ -130,19 +188,19 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: branch], sub) state = "at the pin" if remote is not None and remote != pin: - state = (f"at the pin; origin/{branch} is {_short(remote)}, which " + state = (f"at the pin; origin/{named} is {_short(remote)}, which " f"the pin deliberately does NOT follow") - print(f" [{role}] {path}: branch {branch} created at " + print(f" {where}: branch {named} created at " f"{_short(pin)} — {state}") return if local == pin: _try_git(["checkout", "-q", branch], sub) - print(f" [{role}] {path}: on {branch} at {_short(pin)} " + print(f" {where}: on {named} at {_short(pin)} " f"(branch tip == pin)") return - print(f" [{role}] {path}: PIN {_short(pin)} != branch {branch} tip " + print(f" {where}: PIN {_short(pin)} != branch {named} tip " f"{_short(local)}") print(" not moving an existing branch. The checkout stays at " "the pin; the recorded pin is authoritative and advancing it is an " @@ -236,7 +294,8 @@ def _load_register_rows(register_path: Path) -> list[dict] | None: try: data = load_yaml(register_path) except Refusal as exc: - print(f" unreadable, so nothing is claimed from it: {exc.detail}") + print(" unreadable, so nothing is claimed from it: " + f"{spelled(exc.detail)}") return None rows = [] if isinstance(data, dict): @@ -258,11 +317,12 @@ def _row_report_line(row: dict, repositories: set[str], if _matches(o, repositories, prefixes)] if not objects: return None - holder = row.get("holder_ref") or row.get("holder") or "?" - act = row.get("act") or row.get("acts") or "?" - state = row.get("state") or "?" - expires = row.get("expires_at") or "-" - return (f" {holder} · {act} · {', '.join(objects)} · " + # A register row is a file's text (#391): each value is spelled. + holder = spelled(row.get("holder_ref") or row.get("holder") or "?") + act = spelled(row.get("act") or row.get("acts") or "?") + state = spelled(row.get("state") or "?") + expires = spelled(row.get("expires_at") or "-") + return (f" {holder} · {act} · {', '.join(map(spelled, objects))} · " f"{state} · expires {expires}") @@ -277,7 +337,8 @@ def _report_one_register(register_path: Path, root: Path, """ # POSIX, on every platform: this line names a file in the repository, # and git and the reader both spell it that way. - print(f" register: {register_path.relative_to(root).as_posix()}") + print(" register: " + f"{spelled(register_path.relative_to(root).as_posix())}") rows = _load_register_rows(register_path) if rows is None: return @@ -378,6 +439,7 @@ def _run_neutral_validators(root: Path, manifest: dict | None, def main(argv: list[str] | None = None) -> int: + escape_unencodable_stdout() parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--root", type=Path, default=None) parser.add_argument("--branch", default=None, @@ -397,7 +459,7 @@ def main(argv: list[str] | None = None) -> int: sub_legs = [leg for leg in legs if leg.get("role") != "assembly"] name = (manifest or {}).get("name") or root.name - print(f"bootstrap: {name} ({root})") + print(f"bootstrap: {spelled(name)} ({spelled(root)})") print("\n(a) legs on tracking branches, pins untouched") _checkout_all_tracking_branches(root, sub_legs, manifest, args.branch) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 6668790c..0943b47c 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -68,6 +68,61 @@ REWRITE_TARGETS = ("https://github.com/", "git@github.com:") +def escape_unencodable_stdout() -> None: + """A character stdout cannot encode is printed escaped, never raised on (#391). + + WHAT WAS WRONG. `main` printed the manifest's `name` with a plain `print` + before any step ran, so `name: "a\\ud83db"` (a lone surrogate, which no + UTF-8 stdout can write) or `name: "日本"` on a stdout that is not UTF-8 + (`PYTHONIOENCODING=ascii`, or the ANSI code page Windows gives a stdout that + is not the console) ended `make bootstrap` in a `UnicodeEncodeError` + traceback one step before `validate-manifest.py`, the validator that names + the problem, was reached. This file's own wording raised on such a stdout + too, wherever it carries a dash or a middle dot. + + A `strict` or `surrogateescape` stdout, the two handlers that raise, is + reconfigured ONCE, at the start of `main`, to `backslashreplace`, the + handler stderr already has: what the stdout holds is written as it was, and + what it cannot hold as `\\u65e5`. Any other handler, one somebody chose, is + left alone, and the ENCODING is never changed. `scripts/validate-pins.py` + does the same for its report (#295). The tools run from a checkout of the + standard use `shape_materialize.escape_what_stdout_cannot_encode` (#214), + which no project carries, and this file imports nothing but `repo_shape`. + This function and `spelled` are the same in the assembly root's + `bootstrap.py` and the family root's. + + GUARDED, because failing to set a handler is no reason to refuse the run: a + stream a caller replaced may have no `reconfigure`, a closed one raises + `ValueError`, and `pythonw` has no stdout at all. + """ + if getattr(sys.stdout, "errors", None) in ("strict", "surrogateescape"): + try: + sys.stdout.reconfigure(errors="backslashreplace") + except (AttributeError, ValueError, OSError): + pass + + +def spelled(value: object) -> str: + """`value` as ONE line of text a terminal can be shown (#391). + + What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on + every stdout, so it reaches the terminal as the control it is, and a + manifest's `name` or a register row is a person's text. Every character + `str.isprintable` refuses is written as the escape Python spells it + (`\\x1b`, `\\r`, `\\n`, `\\u2028`, `\\ud83d`), so the value stays on + its line and a lone surrogate is named, whatever handler the stdout has. A + printable character is kept as it is, a backslash included, so what a UTF-8 + stdout can hold prints byte for byte; the cost is that a name holding the + four characters `\\x1b` prints as one holding ESC does. + + Applied to every value this script prints that it read from the manifest, a + register row or the disk, and to none of its own wording. + """ + return "".join(char if char.isprintable() + else char.encode("unicode_escape").decode("ascii") + for char in str(value)) + + def credential() -> tuple[str, str]: """`(source, token)`; `("none", "")` when the environment carries none.""" for name in CREDENTIAL_ENV: @@ -133,11 +188,14 @@ def member_path(root: Path, row: dict) -> Path: def run_member_target(root: Path, row: dict, target: str) -> str | None: """(b) One member's own `make `. Returns a failure line, or None.""" - project = str(row.get("project") or "?") + # A member row is the family manifest's text (#391): what is PRINTED of it, + # and of `--members-make`, is spelled. The path is built from the raw row. + project = spelled(row.get("project") or "?") path = member_path(root, row) + named = spelled(target) if not (path / ".git").exists(): - print(f" [{project}] {path.name}: NOT CHECKED OUT; skipped. See the " - "fetch above.") + print(f" [{project}] {spelled(path.name)}: NOT CHECKED OUT; skipped. " + "See the fetch above.") return None if not (path / "Makefile").is_file(): print(f" [{project}] no Makefile; skipped. A member without one is " @@ -148,16 +206,17 @@ def run_member_target(root: Path, row: dict, target: str) -> str | None: if not make: print(f" [{project}] `make` is not on PATH; skipped.") return None - print(f" --- {project}: make {target} ---") + print(f" --- {project}: make {named} ---") sys.stdout.flush() proc = subprocess.run([make, target], cwd=str(path), check=False) sys.stdout.flush() if proc.returncode != 0: - return f"{project} (make {target} exited {proc.returncode})" + return f"{project} (make {named} exited {proc.returncode})" return None def main(argv: list[str] | None = None) -> int: + escape_unencodable_stdout() parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--root", type=Path, default=None) parser.add_argument("--no-fetch", action="store_true", @@ -177,7 +236,8 @@ def main(argv: list[str] | None = None) -> int: print(str(exc), file=sys.stderr) return 2 - print(f"bootstrap: {name} ({root}) — {len(rows)} member(s)") + print(f"bootstrap: {spelled(name)} ({spelled(root)}) — " + f"{len(rows)} member(s)") print("\n(a) members and their legs") if args.no_fetch: @@ -185,7 +245,7 @@ def main(argv: list[str] | None = None) -> int: else: fetch_members(root) - print(f"\n(b) each member's own `make {args.members_make}`") + print(f"\n(b) each member's own `make {spelled(args.members_make)}`") failed: list[str] = [] if args.skip_members: print(" skipped (--skip-members)") diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py new file mode 100644 index 00000000..7ccd45f9 --- /dev/null +++ b/tests/test_bootstrap_spelling.py @@ -0,0 +1,687 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Both roots' `bootstrap.py` print what the manifest says and are never ended +by it (#391). + +`scripts/bootstrap.py` (what `make bootstrap` runs) opened with +`print(f"bootstrap: {name} ({root})")`, before any step, and the family root's +with the same line. `name: "a\\ud83db"` is a string holding a lone surrogate, which +no UTF-8 stdout can write, so the run ended in a `UnicodeEncodeError` traceback, +`make bootstrap` exited 2, and the validator that names the problem +(`validate-manifest.py`, the second of the three) was never reached. A valid name +the stdout cannot hold (`日本` under `PYTHONIOENCODING=ascii`) ended the same way, +and the family root's line also holds a literal dash, so under that stdout it +raised for EVERY name. A raw ESC or CR in a name is encodable everywhere and +reached the terminal as the control it is. + +Both files are shape copies (digest-pinned in every project that holds them) and +a project carries `scripts/repo_shape.py` and no `shape_materialize.py`, so they +cannot call #214's helper. Each now carries two small functions, the same in +both: `escape_unencodable_stdout` reconfigures a `strict` or `surrogateescape` +stdout to `backslashreplace` once, at the start of `main`; `spelled` writes every +character `str.isprintable` refuses as the escape Python spells it, and is +applied to every value the script prints that it read from the manifest, a +register row or the disk. Four kinds of test, all against local bare remotes: + +- the rows of the issue, run for real in a scaffolded project and in a family + holder, under every stdout that matters, answered in BYTES and decoded + strictly in the encoding the run was given (a byte the stdout could not have + held fails the test); +- every other print that carries a value: the leg's role, path and tracking + branch, a register row and the path of a register, the root's own directory + name, a member row and `--members-make`; +- what the run does NEXT: it reaches the validators and ends as they end. What + they say about the surrogate is the other fix's (#280, on + `fix/yaml-flow-brackets-surrogates`), so that row asks the validator itself + what it says and holds `bootstrap` to that; +- the two functions themselves, and a guard that the two files carry the same + two. + +A plain name must print exactly what it printed before: a golden of the whole +output, from known values. +""" + +from __future__ import annotations + +import ast +import importlib.util +import io +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import REPO, git, run_script +from test_stdout_encoding import child_environ, run_tool_bytes, shown + +sys.path.insert(0, str(REPO / "scripts")) + +ASSEMBLY_TEMPLATE = (REPO / "templates" / "assembly-root" / "scripts" + / "bootstrap.py") +FAMILY_TEMPLATE = (REPO / "templates" / "family-root" / "scripts" + / "bootstrap.py") +FAMILY_TOOL = REPO / "scripts" / "family.py" +FAMILY_NAME = "InkRouter" + +#: Every stdout the rows below are run under, as `PYTHONIOENCODING` spells it. +#: - `utf-8:strict` is the control: it holds every name but a surrogate. +#: - `utf-8:surrogateescape` is what a POSIX or `C.UTF-8` locale gives. +#: - `cp1252:surrogateescape` is Windows' default for a stdout that is not the +#: console, and `cp1252:strict` and `ascii:strict` are what +#: `PYTHONIOENCODING=cp1252` and `=ascii` give: none holds `日本`. +#: - `utf-8:replace` is an operator's own lenient handler, left as chosen. +ENCODINGS = ("utf-8:strict", "utf-8:surrogateescape", + "cp1252:surrogateescape", "cp1252:strict", "ascii:strict", + "utf-8:replace") +#: The two that do not hold the dash, the middle dot or `日本`. +NARROW = ("utf-8:strict", "ascii:strict") + +#: `(id, the YAML scalar written after `name:`, what the line must say of it)`. +#: The scalar is YAML text, so `\ud83d` here is the six characters of an escape +#: the reader resolves; the third is the characters that must be PRINTED. +CORE_NAMES = ( + ("plain", '"Atlas"', "Atlas"), + ("lone-surrogate", r'"a\ud83db"', r"a\ud83db"), + ("esc-and-cr", r'"a\x1bb\rc"', r"a\x1bb\rc"), + ("japanese", '"日本"', "日本"), +) +MORE_NAMES = ( + ("newline-tab-del", r'"a\nb\tc\x7fd"', r"a\nb\tc\x7fd"), + ("nbsp-and-line-separator", r'"a\xa0b\u2028c"', r"a\xa0b\u2028c"), + ("astral", '"🙂"', "🙂"), + ("quote-and-backslash", r'"He said \"hi\" \\o/"', r'He said "hi" \o/'), + ("two-lone-surrogates", r'"\ud83d\ude00"', r"\ud83d\ude00"), +) +NAME_ROWS = [pytest.param(scalar, printed, encoding, + id=f"{label}-{encoding.replace(':', '-')}") + for label, scalar, printed in CORE_NAMES + for encoding in ENCODINGS] +NAME_ROWS += [pytest.param(scalar, printed, encoding, + id=f"{label}-{encoding.replace(':', '-')}") + for label, scalar, printed in MORE_NAMES + for encoding in NARROW] + +DEGRADE_LINE = "authority is not wallet-carried in this org" +#: The dash in step (a)'s own wording. +DASH = "\u2014" + + +def run(script: Path, *args: str, encoding: str, tmp_path: Path, cwd: Path, + changes: dict | None = None) -> tuple[int, str, str]: + """`(exit, stdout, stderr)` of one run with its stdout in `encoding`, read + as BYTES and decoded STRICTLY in that encoding. `\\r\\n` is read as `\\n` + (Windows writes it), and a lone `\\r` is left to be seen.""" + code, stdout, stderr = run_tool_bytes( + script, *args, encoding=encoding, tmp_path=tmp_path, cwd=cwd, + changes=changes) + codec = encoding.split(":")[0] + return (code, stdout.decode(codec).replace("\r\n", "\n"), + stderr.decode(codec, "replace").replace("\r\n", "\n")) + + +def edit(root: Path, filename: str, old: str, new: str) -> None: + """`old` to `new` in `root/filename`, as bytes, whatever line ending the + clone has. `old` must be there once.""" + path = root / filename + body = path.read_bytes() + assert body.count(old.encode("utf-8")) == 1, (old, body) + path.write_bytes(body.replace(old.encode("utf-8"), new.encode("utf-8"))) + + +def bootstrap_of(root: Path) -> Path: + return root / "scripts" / "bootstrap.py" + + +def spelled_root(root: Path, encoding: str) -> str: + """The root as the first line prints it: resolved, with what the stdout + cannot hold escaped, and a line separator in a directory name escaped + whatever the stdout.""" + text = str(root.resolve()).replace("\u2028", "\\u2028") + return shown(text, encoding) + + +# --- the issue's rows, in a scaffolded project ------------------------------ + +@pytest.mark.parametrize("scalar, printed, encoding", NAME_ROWS) +def test_the_name_line_is_printed_whatever_the_name_and_the_stdout( + project, tmp_path, scalar, printed, encoding): + """On origin/main every row but `plain` and `japanese` under a stdout that + holds it ended in `UnicodeEncodeError` from the `print` of the name; a raw + ESC or CR reached the terminal under all of them. Exit 0, no traceback, + the name on the line it belongs to, spelled, and not one control character + anywhere in the output. `--skip-validators` because the validators are + this row's NEXT step, and are not this fix's (they have their own test).""" + edit(project, "project.yaml", 'name: "Atlas"', f"name: {scalar}") + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding=encoding, tmp_path=tmp_path, cwd=project) + assert code == 0, out + err + assert "Traceback" not in err, err + lines = out.split("\n") + assert lines[0] == (f"bootstrap: {shown(printed, encoding)} " + f"({spelled_root(project, encoding)})"), out + assert lines[1] == "", out + assert out.endswith("bootstrap ok\n"), out + assert not {"\x1b", "\r", "\x7f", "\u2028", "\xa0"} & set(out), out + + +def test_a_plain_name_prints_exactly_what_it_printed_before(project, tmp_path): + """BYTE-IDENTICAL for a name every stdout holds. The first run places the + legs (and says so in other words), the second is steady, and its WHOLE + output is spelled out here, from the pins and the root. These are the + lines origin/main printed; nothing in them is a value this fix touches + except the first line and the two legs'.""" + for _ in range(2): + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=project) + assert code == 0 and err == "", out + err + spec, leg = (git("rev-parse", f"HEAD:{name}", cwd=project).stdout.strip() + [:12] for name in ("spec", "code")) + assert out.split("\n") == [ + f"bootstrap: Atlas ({project.resolve()})", + "", + "(a) legs on tracking branches, pins untouched", + f" [spec] spec: on main at {spec} (branch tip == pin)", + f" [code] code: on main at {leg} (branch tip == pin)", + "", + "(b) neutral validators", + " skipped (--skip-validators)", + "", + "(c) review authority", + DEGRADE_LINE, + " (no `governance/review-authority/register.yaml` in the assembly " + "root or in any leg. Wallet-carried authority is an OVERLAY; the " + "shape confers nothing without it, and nothing with it.)", + "", + "bootstrap ok", + "", + ], out + + +def test_a_name_a_utf8_stdout_holds_is_written_as_the_bytes_it_is( + project, tmp_path): + """Read as BYTES, so a character kept is a character kept: `日本`, an + emoji and a quote and a backslash are the UTF-8 bytes they always were.""" + edit(project, "project.yaml", 'name: "Atlas"', 'name: "日本 🙂 \\"q\\" \\\\"') + code, stdout, stderr = run_tool_bytes( + bootstrap_of(project), "--skip-validators", encoding="utf-8:strict", + tmp_path=tmp_path, cwd=project) + assert code == 0 and b"Traceback" not in stderr, stderr + first = stdout.split(b"\n")[0].rstrip(b"\r") + assert first.startswith('bootstrap: 日本 🙂 "q" \\ ('.encode("utf-8")), first + + +# --- what the run does NEXT -------------------------------------------------- + +def test_a_lone_surrogate_name_reaches_the_validators_and_ends_as_they_end( + project, tmp_path): + """The run is no longer ended by its own first line. It reaches step (b), + where the manifest validator fails the run, and exits as `main`'s own + failed-validator path does: `bootstrap FAILED: manifest (exit 1)`, exit 1. + + WHAT THE VALIDATOR SAYS is the other fix's (#280). Where it names the + surrogate (`FINDING manifest-name`), bootstrap must say it and no + traceback may be anywhere; where it does not yet, the validator's own + `print` raises in ITS process, and no traceback may come from bootstrap's. + The validator is asked by itself, so the row holds on both trees.""" + edit(project, "project.yaml", 'name: "Atlas"', r'name: "a\ud83db"') + alone_code, _, alone_err = run( + project / "scripts" / "validate-manifest.py", encoding="utf-8:strict", + tmp_path=tmp_path, cwd=project) + assert alone_code == 1, alone_err + code, out, err = run(bootstrap_of(project), encoding="utf-8:strict", + tmp_path=tmp_path, cwd=project) + assert out.startswith(r"bootstrap: a\ud83db ("), out + assert "\n(b) neutral validators\n" in out, out + assert code == 1, out + err + assert "bootstrap FAILED: manifest (exit 1)" in err, err + assert 'bootstrap.py", line' not in err, err + if "FINDING manifest-name" in alone_err: + assert "FINDING manifest-name" in err, err + assert "Traceback" not in err, err + + +@pytest.mark.skipif(shutil.which("make") is None or + shutil.which("python3") is None, + reason="the Makefile runs `python3` under `make`") +def test_make_bootstrap_ends_as_the_validators_end_not_in_a_traceback( + project, tmp_path): + """`make bootstrap` exited 2 with the traceback and `make: *** [Makefile:44: + bootstrap] Error 1`; it still exits 2 (make's own code for a failed + recipe), and now for the validator's reason.""" + edit(project, "project.yaml", 'name: "Atlas"', r'name: "a\ud83db"') + proc = subprocess.run(["make", "bootstrap"], cwd=str(project), + env=child_environ(tmp_path, "utf-8:strict"), + capture_output=True, stdin=subprocess.DEVNULL, + check=False) + out, err = (text.decode("utf-8", "replace") + for text in (proc.stdout, proc.stderr)) + assert proc.returncode == 2, out + err + assert r"bootstrap: a\ud83db (" in out, out + assert "bootstrap FAILED: manifest (exit 1)" in err, err + assert 'bootstrap.py", line' not in err, err + + +# --- every other print that carries a value --------------------------------- + +def steady(project: Path, tmp_path: Path, encoding: str = "utf-8:strict"): + """The output of a SECOND run: the first places the legs on their branches + (and the lines it says that in are not what these rows are about).""" + run(bootstrap_of(project), "--skip-validators", encoding=encoding, + tmp_path=tmp_path, cwd=project) + return run(bootstrap_of(project), "--skip-validators", encoding=encoding, + tmp_path=tmp_path, cwd=project) + + +def pin12(project: Path, leg: str) -> str: + return git("rev-parse", f"HEAD:{leg}", cwd=project).stdout.strip()[:12] + + +def test_a_legs_role_is_spelled(project, tmp_path): + edit(project, "project.yaml", " - role: spec", r' - role: "s\x1bp"') + code, out, err = steady(project, tmp_path) + assert code == 0 and "Traceback" not in err, out + err + assert (rf" [s\x1bp] spec: on main at {pin12(project, 'spec')} " + "(branch tip == pin)") in out.split("\n"), out + assert "\x1b" not in out, out + + +@pytest.mark.parametrize("path, spelling", [ + (r'"sp\x1bec"', r"sp\x1bec"), + (r'"sp\ud83dec"', r"sp\ud83dec"), + (r'"sp\u2028ec"', r"sp\u2028ec"), +]) +def test_a_legs_path_is_spelled_where_it_is_not_initialized( + project, tmp_path, path, spelling): + """A path the manifest holds that is no directory: step (a) says so, and + says what to run, before any validator has had its say. A lone surrogate + here raised from the `print`.""" + edit(project, "project.yaml", " path: spec", f" path: {path}") + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=project) + assert code == 0 and "Traceback" not in err, out + err + assert (f" [spec] {spelling}: NOT INITIALIZED. Run `git submodule update " + f"--init {spelling}` {DASH} the validators below will refuse " + "until you do.") in out.split("\n"), out + assert not {"\x1b", "\u2028"} & set(out), out + + +def test_a_tracking_branch_git_will_not_make_is_spelled(project, tmp_path): + """Passes on origin/main too, and is here to keep it: that line prints the + branch with `!r`, and a repr has always escaped a control.""" + edit(project, "project.yaml", "tracking_branch: main", + r'tracking_branch: "a\x1bb"') + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=project) + assert code == 0 and "Traceback" not in err, out + err + for leg in ("spec", "code"): + assert (rf" [{leg}] {leg}: could not create branch 'a\x1bb' at the " + f"pin {pin12(project, leg)}; leaving detached" + ) in out.split("\n"), out + assert "\x1b" not in out, out + + +def test_a_branch_name_is_spelled_in_every_line_that_names_it(project, + tmp_path): + """A branch git accepts and a terminal should not be handed raw: U+2028. + The lines that name one: created (and where the remote's is), on, and the + disagreement.""" + edit(project, "project.yaml", "tracking_branch: main", + r'tracking_branch: "tb\u2028x"') + named = r"tb\u2028x" + spec = project / "spec" + pin = git("rev-parse", "HEAD:spec", cwd=project).stdout.strip() + code_pin = pin12(project, "code") + tree = git("rev-parse", f"{pin}^{{tree}}", cwd=spec).stdout.strip() + other = git("commit-tree", tree, "-p", pin, "-m", "elsewhere", + cwd=spec).stdout.strip() + git("update-ref", "refs/remotes/origin/tb\u2028x", other, cwd=spec) + + def lines() -> list[str]: + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=project) + assert code == 0 and "Traceback" not in err, out + err + assert "\u2028" not in out, out + return out.split("\n") + + first = lines() + assert (f" [spec] spec: branch {named} created at {pin[:12]} {DASH} at " + f"the pin; origin/{named} is {other[:12]}, which the pin " + "deliberately does NOT follow") in first, first + assert (f" [code] code: branch {named} created at {code_pin} {DASH} at " + "the pin") in first, first + + assert (f" [spec] spec: on {named} at {pin[:12]} (branch tip == pin)" + ) in lines() + + git("commit", "-q", "--allow-empty", "-m", "local work", cwd=spec) + tip = git("rev-parse", "HEAD", cwd=spec).stdout.strip()[:12] + assert (f" [spec] spec: PIN {pin[:12]} != branch {named} tip {tip}" + ) in lines() + + +#: A register row is a file's text. Each value carries a control or a line +#: separator, and the object is a leg's repository with a path after it, so +#: the row is one the readout reports. +REGISTER = ( + "register_version: 1\n" + "rows:\n" + " - row_id: row-0001\n" + ' holder_ref: "agent:\\x1b[31mred"\n' + " target_repo: testorg/Atlas-code\n" + " objects:\n" + ' - "testorg/Atlas-code:co\\x1bde"\n' + ' act: "re\\rview"\n' + ' state: "ac\\x1btive"\n' + ' expires_at: "2026\\u2028"\n') +DOT = "\u00b7" +REGISTER_ROW = (rf" agent:\x1b[31mred {DOT} re\rview {DOT} testorg/Atlas-code, " + rf"testorg/Atlas-code:co\x1bde {DOT} ac\x1btive {DOT} " + rf"expires 2026\u2028") + + +@pytest.mark.parametrize("encoding", NARROW) +def test_a_register_row_is_spelled(project, tmp_path, encoding): + """Under `ascii` as well: the row's own separator is a middle dot, which + that stdout cannot hold, so the row raised whatever it said.""" + register = project / "governance" / "review-authority" / "register.yaml" + register.parent.mkdir(parents=True) + register.write_bytes(REGISTER.encode("utf-8")) + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding=encoding, tmp_path=tmp_path, cwd=project) + assert code == 0 and "Traceback" not in err, out + err + assert shown(REGISTER_ROW, encoding) in out.split("\n"), out + assert not {"\x1b", "\r", "\u2028"} & set(out), out + + +def test_a_register_and_its_path_are_spelled_when_it_cannot_be_read( + project, tmp_path): + """The register a leg holds under a directory whose name carries a line + separator, which does not parse: its path is on the `register:` line and + again, with the reader's reason, on the `unreadable` one. Under a stdout + that HOLDS a line separator, so a raw one would be written raw.""" + edit(project, "project.yaml", " path: spec", r' path: "sp\u2028ec"') + register = (project / "sp\u2028ec" / "governance" / "review-authority" + / "register.yaml") + register.parent.mkdir(parents=True) + register.write_bytes(b"rows:\n\t- row_id: x\n") + code, out, err = run(bootstrap_of(project), "--skip-validators", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=project) + assert code == 0 and "Traceback" not in err, out + err + lines = out.split("\n") + assert (r" register: sp\u2028ec/governance/review-authority/register.yaml" + ) in lines, out + unreadable = [line for line in lines if "unreadable, so nothing" in line] + assert len(unreadable) == 1 and r"sp\u2028ec" in unreadable[0], out + assert unreadable[0].endswith("tab used for indentation"), unreadable + assert "\u2028" not in out, out + + +def test_a_roots_own_directory_name_is_spelled(project, tmp_path): + """The second thing the first line prints. Under a stdout that HOLDS a + line separator, so a raw one would be written raw.""" + renamed = tmp_path / "At\u2028las" + shutil.move(str(project), str(renamed)) + code, out, err = run(bootstrap_of(renamed), "--skip-validators", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=renamed) + assert code == 0 and "Traceback" not in err, out + err + assert out.split("\n")[0] == ( + f"bootstrap: Atlas ({spelled_root(renamed, 'utf-8:strict')})"), out + assert "\u2028" not in out, out + + +# --- the family root -------------------------------------------------------- + +@pytest.fixture(scope="module") +def family_template(tmp_path_factory) -> Path: + """One real family holder, `family.py init` into local bare remotes, no + members.""" + base = tmp_path_factory.mktemp("bootstrap-spelling-family") + created = run_script( + FAMILY_TOOL, "init", "--org", "InkRouter", "--family", FAMILY_NAME, + "--created-by", "Test Human", "--created-on", "2026-09-04", + "--local-remote-dir", str(base / "remotes"), + "--work-dir", str(base / "fam")) + assert created.returncode == 0, created.stderr + created.stdout + return base / "fam" / FAMILY_NAME + + +@pytest.fixture +def holder(family_template, tmp_path) -> Path: + """A private, mutable copy of the family holder.""" + target = tmp_path / FAMILY_NAME + shutil.copytree(family_template, target, symlinks=True) + return target + + +def family_line(printed: str, holder: Path, encoding: str) -> str: + """The first line, whole: its dash is the family's own wording, so under a + stdout that cannot hold it the dash is escaped too.""" + return shown(f"bootstrap: {printed} ({holder.resolve()}) \u2014 0 member(s)", + encoding) + + +@pytest.mark.parametrize("scalar, printed, encoding", NAME_ROWS) +def test_the_family_name_line_is_printed_whatever_the_name_and_the_stdout( + holder, tmp_path, scalar, printed, encoding): + """The same rows, one template over. The line holds a literal dash, so on + origin/main it raised under `ascii` for the plain name too.""" + edit(holder, "family.yaml", f'name: "{FAMILY_NAME}"', f"name: {scalar}") + code, out, err = run(bootstrap_of(holder), "--no-fetch", "--skip-members", + encoding=encoding, tmp_path=tmp_path, cwd=holder) + assert code == 0, out + err + assert "Traceback" not in err, err + assert out.split("\n")[0] == family_line(printed, holder, encoding), out + assert out.endswith("family bootstrap ok\n"), out + assert not {"\x1b", "\r", "\x7f", "\u2028", "\xa0"} & set(out), out + + +def test_a_plain_family_prints_exactly_what_it_printed_before(holder, + tmp_path): + code, out, err = run(bootstrap_of(holder), "--no-fetch", "--skip-members", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=holder) + assert code == 0 and err == "", out + err + assert out.split("\n") == [ + f"bootstrap: {FAMILY_NAME} ({holder.resolve()}) \u2014 0 member(s)", + "", + "(a) members and their legs", + " skipped (--no-fetch)", + "", + "(b) each member's own `make bootstrap`", + " skipped (--skip-members)", + "", + "family bootstrap ok", + "", + ], out + + +def test_a_family_roots_own_directory_name_is_spelled(holder, tmp_path): + """The second thing the family's first line prints, under a stdout that + HOLDS a line separator, so a raw one would be written raw.""" + renamed = tmp_path / "Ink\u2028Router" + shutil.move(str(holder), str(renamed)) + code, out, err = run(bootstrap_of(renamed), "--no-fetch", "--skip-members", + encoding="utf-8:strict", tmp_path=tmp_path, + cwd=renamed) + assert code == 0 and "Traceback" not in err, out + err + assert out.split("\n")[0] == ( + f"bootstrap: {FAMILY_NAME} ({spelled_root(renamed, 'utf-8:strict')}) " + f"{DASH} 0 member(s)"), out + assert "\u2028" not in out, out + + +def test_a_members_row_and_the_make_target_are_spelled(holder, tmp_path): + """Four rows, none a real submodule, run with `--no-fetch` under a stdout + that HOLDS a line separator, so a raw one would be written raw: + - a member that is not checked out, named by a directory with a line + separator in it; + - a member with no Makefile; + - a member whose own `make` runs: `MAKE` is the interpreter and the + TARGET is a script in the member, named with a line separator, so the + member runs only if the RAW target reached the command; + - a member whose command fails, which is what the failure line names. + """ + edit(holder, "family.yaml", "members: []", ( + "members:\n" + ' - project: "Ir\\x1bs"\n' + ' path: "members/Ir\\u2028s"\n' + ' - project: "Bo\\u2028x"\n' + " path: members/Box\n" + ' - project: "Ma\\x1bke"\n' + " path: members/Make\n" + ' - project: "Fa\\x1bil"\n' + " path: members/Fail\n")) + members = holder / "members" + target = "ta\u2028rget.py" + for name in ("Box", "Make", "Fail"): + (members / name / ".git").mkdir(parents=True) + for name in ("Make", "Fail"): + (members / name / "Makefile").write_bytes(b"") + (members / "Make" / target).write_bytes( + b"open('ran', 'w').close()\n") + + code, out, err = run(bootstrap_of(holder), "--no-fetch", + "--members-make", target, encoding="utf-8:strict", + tmp_path=tmp_path, cwd=holder, + changes={"MAKE": sys.executable}) + assert "Traceback" not in err, out + err + assert code == 1, out + err + lines = out.split("\n") + assert r" [Ir\x1bs] Ir\u2028s: NOT CHECKED OUT; skipped. See the fetch " \ + "above." in lines, out + assert (r" [Bo\u2028x] no Makefile; skipped. A member without one is " + f"not misconfigured {DASH} it simply has nothing for this " + "command to call.") in lines, out + assert r"(b) each member's own `make ta\u2028rget.py`" in lines, out + assert r" --- Ma\x1bke: make ta\u2028rget.py ---" in lines, out + assert r" --- Fa\x1bil: make ta\u2028rget.py ---" in lines, out + assert (members / "Make" / "ran").exists(), \ + "the member's command was handed the escaped target, not the raw one" + assert err.strip().endswith( + r"family bootstrap FAILED: Fa\x1bil (make ta\u2028rget.py exited 2)" + ), err + assert not {"\x1b", "\u2028"} & set(out), out + + +# --- the two functions, and the two files that carry them ------------------- + +def load(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module", params=[ASSEMBLY_TEMPLATE, FAMILY_TEMPLATE], + ids=["assembly-root", "family-root"]) +def template(request): + """Each root's `bootstrap.py`, loaded by path under a name of its own, so + nothing here leaves a module called `bootstrap` importable.""" + return load(request.param, "bootstrap_spelling_" + f"{request.param.parts[-3].replace('-', '_')}") + + +@pytest.mark.parametrize("text", [ + "Atlas", "日本", "Zo\u00eb", "\U0001f642", 'He said "hi" \\o/', "a b", "", + "/tmp/x y/z", "C:\\Users\\a\\Atlas", "a=b: c, [d] {e}"]) +def test_spelled_keeps_what_a_terminal_can_be_shown(template, text): + assert template.spelled(text) == text + + +@pytest.mark.parametrize("char, spelling", [ + ("\x00", r"\x00"), ("\x1b", r"\x1b"), ("\r", r"\r"), ("\n", r"\n"), + ("\t", r"\t"), ("\x7f", r"\x7f"), ("\x85", r"\x85"), ("\xa0", r"\xa0"), + ("\u200b", r"\u200b"), ("\u2028", r"\u2028"), ("\ud83d", r"\ud83d"), + ("\udce9", r"\udce9"), ("\U000e0001", r"\U000e0001")]) +def test_spelled_writes_the_escape_python_spells(template, char, spelling): + assert template.spelled(f"a{char}b") == f"a{spelling}b" + + +def test_spelled_takes_what_a_manifest_can_hold(template): + """`name: 5` and `role: [a]` are values too: the text a print would have + made of them, spelled.""" + assert template.spelled(5) == "5" + assert template.spelled(None) == "None" + assert template.spelled(["a\x1b"]) == "['a\\x1b']" + assert template.spelled(Path("b\u2028")) == "b\\u2028" + + +def test_spelled_leaves_one_line_of_printable_text(template): + every = "".join(chr(code) for code in (*range(0x100), 0x2028, 0x2029, + 0xd800, 0xdc80, 0xdfff, 0xe000, + 0xfeff, 0x1f642, 0xe0001)) + out = template.spelled(every) + assert out.isprintable() and "\n" not in out, out + + +@pytest.mark.parametrize("errors, expected", [ + ("strict", "backslashreplace"), + ("surrogateescape", "backslashreplace"), + ("backslashreplace", "backslashreplace"), + ("replace", "replace"), + ("ignore", "ignore"), + ("xmlcharrefreplace", "xmlcharrefreplace"), + ("namereplace", "namereplace"), +]) +def test_a_stdout_that_raises_is_made_to_escape_and_no_other_is_touched( + template, monkeypatch, errors, expected): + stream = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors=errors, + newline="\n") + monkeypatch.setattr(sys, "stdout", stream) + template.escape_unencodable_stdout() + assert (stream.errors, stream.encoding) == (expected, "cp1252") + print("\u65e5 \u2014", end="") + stream.flush() + written = stream.buffer.getvalue() + if expected == "backslashreplace": + assert written == b"\\u65e5 \x97", written + + +def test_a_stdout_that_cannot_be_reconfigured_fails_nothing(template, + monkeypatch): + """No `reconfigure`, a closed stream and no stdout at all (`pythonw`).""" + + class Replaced: + errors = "strict" + + for stdout in (Replaced(), None): + monkeypatch.setattr(sys, "stdout", stdout) + template.escape_unencodable_stdout() + closed = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors="strict") + closed.close() + monkeypatch.setattr(sys, "stdout", closed) + template.escape_unencodable_stdout() + + +def source_of(path: Path, name: str) -> str: + text = path.read_text(encoding="utf-8").replace("\r\n", "\n") + node = next(node for node in ast.parse(text).body + if isinstance(node, ast.FunctionDef) and node.name == name) + return ast.get_source_segment(text, node) + + +@pytest.mark.parametrize("name", ["escape_unencodable_stdout", "spelled"]) +def test_the_two_roots_carry_the_same_function(name): + """Both files are shape copies that cannot import anything of this + repository's, so the helper is written twice; this is what keeps it one.""" + assert (source_of(ASSEMBLY_TEMPLATE, name) + == source_of(FAMILY_TEMPLATE, name)), name + + +@pytest.mark.parametrize("path", [ASSEMBLY_TEMPLATE, FAMILY_TEMPLATE], + ids=["assembly-root", "family-root"]) +def test_main_escapes_stdout_before_it_prints_or_parses_anything(path): + """The reconfigure is `main`'s FIRST statement: `--help` and an argument + error print too.""" + text = path.read_text(encoding="utf-8") + main = next(node for node in ast.parse(text).body + if isinstance(node, ast.FunctionDef) and node.name == "main") + first = main.body[0] + assert ast.unparse(first) == "escape_unencodable_stdout()", ast.unparse(first) From af26b19161f731df8c66822c3fe36e19497b30e5 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 12 Oct 2026 19:08:12 +0000 Subject: [PATCH 2/4] templates/assembly-root/scripts/bootstrap.py and templates/family-root/scripts/bootstrap.py: `spelled` escapes the controls, the surrogates, the line and paragraph separators and the bidirectional controls and nothing else, so a Persian name with its `\u200c`, an emoji sequence with its `\u200d`, and a name with `\u3000`, `\xa0` or `\xad` print byte for byte as they did before #391 (#391) Review of the first head found that `spelled` wrote as an escape every character `str.isprintable` refuses. That is wider than a control, a surrogate or a line separator: it also refuses every space but ASCII space, every format character and every unassigned code point. So a valid name that `validate-manifest.py` accepts, and prints raw a few lines below, was escaped by bootstrap's own first line: `\u200c` (the zero-width non-joiner a Persian name needs), `\u200d` (the joiner of an emoji sequence), `\u3000` (the ideographic space between a Japanese family name and a given name), `\xa0`, `\xad` and `\u200f`. The `git submodule update --init ` command printed for a leg path holding one of them was no longer right to paste, and what was escaped moved with the interpreter's Unicode database. The test is now the category (the coordinator's decision; Brett may reverse it). `spelled` writes the escape Python spells for EXACTLY these characters: - the controls (Cc); - the surrogates (Cs); - the line and paragraph separators (Zl, Zp); - the bidirectional controls `\u061c`, `\u200e`, `\u200f`, `\u202a` to `\u202e` and `\u2066` to `\u2069`, so a name cannot reorder the text around it on a terminal. Every other character prints as it did before #391. The four categories are fixed ranges, so the answer no longer depends on the Unicode database. The two sets are module-level constants in both copies, and the identity test compares them with the two functions. `unicodedata` is the standard library's. Tests: `tests/test_bootstrap_spelling.py` pins each newly raw character (`\u200c` in a Persian name, `\u200d` in an emoji sequence, `\u3000` in a Japanese full name, `\xa0`, `\xad`, and a zero-width space with a byte-order mark) as the bytes a plain `print` wrote on the base, in a scaffolded project and in a family holder, and with the validators' own `manifest ok:` line beside it. `\u202e`, `\u200f` and `\u2028` are pinned as escaped. Every code point is swept once against a set written from the specification, which is itself checked against `unicodedata.category`. The lone-surrogate rows now assert `FINDING manifest-name` unconditionally, since #280 landed in #390 (cb7063f), and the `make bootstrap` row asserts there is no `Traceback`. Reverting the set to `isprintable` fails 59 of the new rows. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 53 ++++- templates/family-root/scripts/bootstrap.py | 53 ++++- tests/test_bootstrap_spelling.py | 227 ++++++++++++++++--- 3 files changed, 274 insertions(+), 59 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 7d5d6485..2a5af60a 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -51,6 +51,7 @@ import re import subprocess import sys +import unicodedata from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -80,6 +81,16 @@ ) +#: The characters `spelled` writes as an escape, besides the bidirectional +#: controls below: the controls (Cc), the surrogates (Cs), and the line and +#: paragraph separators (Zl, Zp). Nothing else is escaped, `spelled` says why. +ESCAPED_CATEGORIES = frozenset(("Cc", "Cs", "Zl", "Zp")) +#: The bidirectional controls: U+061C, U+200E and U+200F, the embeddings and +#: overrides U+202A to U+202E, and the isolates U+2066 to U+2069. +ESCAPED_BIDI_CONTROLS = frozenset( + "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") + + def escape_unencodable_stdout() -> None: """A character stdout cannot encode is printed escaped, never raised on (#391). @@ -115,24 +126,44 @@ def escape_unencodable_stdout() -> None: def spelled(value: object) -> str: - """`value` as ONE line of text a terminal can be shown (#391). + """`value` as ONE line of text, with what a terminal must not be handed + written as escapes (#391). What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on every stdout, so it reaches the terminal as the control it is, and a - manifest's `name` or a register row is a person's text. Every character - `str.isprintable` refuses is written as the escape Python spells it - (`\\x1b`, `\\r`, `\\n`, `\\u2028`, `\\ud83d`), so the value stays on - its line and a lone surrogate is named, whatever handler the stdout has. A - printable character is kept as it is, a backslash included, so what a UTF-8 - stdout can hold prints byte for byte; the cost is that a name holding the - four characters `\\x1b` prints as one holding ESC does. + manifest's `name` or a register row is a person's text. EXACTLY these + characters are written as the escape Python spells it (`\\x1b`, `\\r`, + `\\u2028`, `\\ud83d`, `\\u202e`), and no others: + + - the controls (Unicode category Cc: C0, DEL and C1), which move a cursor, + clear a screen or end a line; + - the surrogates (Cs), which no UTF-8 text holds, so a lone one is named + whatever handler the stdout has; + - the line and paragraph separators (Zl and Zp, U+2028 and U+2029), which + end a line in some viewers, so the value would not stay on its line; + - the bidirectional controls (U+061C, U+200E, U+200F, U+202A to U+202E and + U+2066 to U+2069), which reorder the text around them so that a name + reads as another name. + + EVERY OTHER CHARACTER PRINTS AS IT IS, as it did before this function + existed: the zero-width non-joiner a Persian or Indic name needs, the + zero-width joiner that makes an emoji sequence, the ideographic space of a + Japanese full name, the no-break space, the soft hyphen, and any other + format, space or unassigned character. `str.isprintable` is not the test: + it refuses all of those, which `validate-manifest.py` accepts and a UTF-8 + terminal shows correctly, and what it counts as unassigned moves with the + interpreter's Unicode database. A backslash prints as it is too, so a name + holding the four characters `\\x1b` prints as one holding ESC does. Applied to every value this script prints that it read from the manifest, a register row or the disk, and to none of its own wording. """ - return "".join(char if char.isprintable() - else char.encode("unicode_escape").decode("ascii") - for char in str(value)) + return "".join( + char.encode("unicode_escape").decode("ascii") + if (char in ESCAPED_BIDI_CONTROLS + or unicodedata.category(char) in ESCAPED_CATEGORIES) + else char + for char in str(value)) def _run(root: Path, args: list[str]) -> int: diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 0943b47c..41ec4fbc 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -50,6 +50,7 @@ import shutil import subprocess import sys +import unicodedata from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -68,6 +69,16 @@ REWRITE_TARGETS = ("https://github.com/", "git@github.com:") +#: The characters `spelled` writes as an escape, besides the bidirectional +#: controls below: the controls (Cc), the surrogates (Cs), and the line and +#: paragraph separators (Zl, Zp). Nothing else is escaped, `spelled` says why. +ESCAPED_CATEGORIES = frozenset(("Cc", "Cs", "Zl", "Zp")) +#: The bidirectional controls: U+061C, U+200E and U+200F, the embeddings and +#: overrides U+202A to U+202E, and the isolates U+2066 to U+2069. +ESCAPED_BIDI_CONTROLS = frozenset( + "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") + + def escape_unencodable_stdout() -> None: """A character stdout cannot encode is printed escaped, never raised on (#391). @@ -103,24 +114,44 @@ def escape_unencodable_stdout() -> None: def spelled(value: object) -> str: - """`value` as ONE line of text a terminal can be shown (#391). + """`value` as ONE line of text, with what a terminal must not be handed + written as escapes (#391). What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on every stdout, so it reaches the terminal as the control it is, and a - manifest's `name` or a register row is a person's text. Every character - `str.isprintable` refuses is written as the escape Python spells it - (`\\x1b`, `\\r`, `\\n`, `\\u2028`, `\\ud83d`), so the value stays on - its line and a lone surrogate is named, whatever handler the stdout has. A - printable character is kept as it is, a backslash included, so what a UTF-8 - stdout can hold prints byte for byte; the cost is that a name holding the - four characters `\\x1b` prints as one holding ESC does. + manifest's `name` or a register row is a person's text. EXACTLY these + characters are written as the escape Python spells it (`\\x1b`, `\\r`, + `\\u2028`, `\\ud83d`, `\\u202e`), and no others: + + - the controls (Unicode category Cc: C0, DEL and C1), which move a cursor, + clear a screen or end a line; + - the surrogates (Cs), which no UTF-8 text holds, so a lone one is named + whatever handler the stdout has; + - the line and paragraph separators (Zl and Zp, U+2028 and U+2029), which + end a line in some viewers, so the value would not stay on its line; + - the bidirectional controls (U+061C, U+200E, U+200F, U+202A to U+202E and + U+2066 to U+2069), which reorder the text around them so that a name + reads as another name. + + EVERY OTHER CHARACTER PRINTS AS IT IS, as it did before this function + existed: the zero-width non-joiner a Persian or Indic name needs, the + zero-width joiner that makes an emoji sequence, the ideographic space of a + Japanese full name, the no-break space, the soft hyphen, and any other + format, space or unassigned character. `str.isprintable` is not the test: + it refuses all of those, which `validate-manifest.py` accepts and a UTF-8 + terminal shows correctly, and what it counts as unassigned moves with the + interpreter's Unicode database. A backslash prints as it is too, so a name + holding the four characters `\\x1b` prints as one holding ESC does. Applied to every value this script prints that it read from the manifest, a register row or the disk, and to none of its own wording. """ - return "".join(char if char.isprintable() - else char.encode("unicode_escape").decode("ascii") - for char in str(value)) + return "".join( + char.encode("unicode_escape").decode("ascii") + if (char in ESCAPED_BIDI_CONTROLS + or unicodedata.category(char) in ESCAPED_CATEGORIES) + else char + for char in str(value)) def credential() -> tuple[str, str]: diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index 7ccd45f9..e150b9b1 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -17,10 +17,15 @@ a project carries `scripts/repo_shape.py` and no `shape_materialize.py`, so they cannot call #214's helper. Each now carries two small functions, the same in both: `escape_unencodable_stdout` reconfigures a `strict` or `surrogateescape` -stdout to `backslashreplace` once, at the start of `main`; `spelled` writes every -character `str.isprintable` refuses as the escape Python spells it, and is -applied to every value the script prints that it read from the manifest, a -register row or the disk. Four kinds of test, all against local bare remotes: +stdout to `backslashreplace` once, at the start of `main`; `spelled` writes the +controls (category Cc), the surrogates (Cs), the line and paragraph separators +(Zl, Zp) and the bidirectional controls (U+061C, U+200E, U+200F, U+202A to +U+202E, U+2066 to U+2069) as the escape Python spells them, and NOTHING else, +and is applied to every value the script prints that it read from the manifest, +a register row or the disk. `str.isprintable` is not the test: it also refuses +the zero-width non-joiner and joiner, the ideographic and no-break spaces and +the soft hyphen, which are valid text that printed raw before this fix and +still do. Four kinds of test, all against local bare remotes: - the rows of the issue, run for real in a scaffolded project and in a family holder, under every stdout that matters, answered in BYTES and decoded @@ -29,10 +34,9 @@ - every other print that carries a value: the leg's role, path and tracking branch, a register row and the path of a register, the root's own directory name, a member row and `--members-make`; -- what the run does NEXT: it reaches the validators and ends as they end. What - they say about the surrogate is the other fix's (#280, on - `fix/yaml-flow-brackets-surrogates`), so that row asks the validator itself - what it says and holds `bootstrap` to that; +- what the run does NEXT: it reaches the validators and ends as they end, at + the `FINDING manifest-name` that `validate-manifest.py` now gives a name + holding a lone surrogate (#280, landed in #390), with no traceback; - the two functions themselves, and a guard that the two files carry the same two. @@ -48,6 +52,7 @@ import shutil import subprocess import sys +import unicodedata from pathlib import Path import pytest @@ -85,21 +90,46 @@ ("lone-surrogate", r'"a\ud83db"', r"a\ud83db"), ("esc-and-cr", r'"a\x1bb\rc"', r"a\x1bb\rc"), ("japanese", '"日本"', "日本"), + # Valid text that `str.isprintable` refuses: it prints raw, as it did + # before the fix, wherever the stdout holds it, and the cp1252 stdouts + # hold this one. + ("nbsp", r'"a\xa0b"', "a\xa0b"), ) +#: Valid names that print RAW, as on origin/main: a Persian name with its +#: zero-width non-joiner, an emoji sequence with its zero-width joiners, a +#: Japanese full name with the ideographic space, and the soft hyphen. (Each +#: is spelled by code point here, so no invisible character sits in this file.) +RAW_NAMES = ( + ("persian-zwnj", r'"\u0645\u06cc\u200c\u062e\u0648\u0627\u0647\u0645"', + "\u0645\u06cc\u200c\u062e\u0648\u0627\u0647\u0645"), + ("emoji-zwj", r'"\U0001f468\u200d\U0001f469\u200d\U0001f467"', + "\U0001f468\u200d\U0001f469\u200d\U0001f467"), + ("ideographic-space", '"山田\\u3000太郎"', "山田\u3000太郎"), + ("nbsp", r'"a\xa0b"', "a\xa0b"), + ("soft-hyphen", r'"co\xadop"', "co\xadop"), + ("zero-width-space-and-bom", r'"a\u200bb\ufeffc"', "a\u200bb\ufeffc"), +) +#: Names that print ESCAPED: a newline, tab and DEL; U+2028; and the +#: right-to-left override and mark, which reorder the text around them. MORE_NAMES = ( ("newline-tab-del", r'"a\nb\tc\x7fd"', r"a\nb\tc\x7fd"), - ("nbsp-and-line-separator", r'"a\xa0b\u2028c"', r"a\xa0b\u2028c"), + ("line-separator", r'"a\u2028b"', r"a\u2028b"), + ("rlo-and-rlm", r'"a\u202eb\u200fc"', r"a\u202eb\u200fc"), ("astral", '"🙂"', "🙂"), ("quote-and-backslash", r'"He said \"hi\" \\o/"', r'He said "hi" \o/'), ("two-lone-surrogates", r'"\ud83d\ude00"', r"\ud83d\ude00"), ) +#: What must reach NO stdout raw, whatever the row: the controls, the line +#: separator and the bidirectional override and mark. +NEVER_RAW = {"\x1b", "\r", "\x7f", "\u2028", "\u202e", "\u200f", "\x85"} NAME_ROWS = [pytest.param(scalar, printed, encoding, id=f"{label}-{encoding.replace(':', '-')}") for label, scalar, printed in CORE_NAMES for encoding in ENCODINGS] NAME_ROWS += [pytest.param(scalar, printed, encoding, id=f"{label}-{encoding.replace(':', '-')}") - for label, scalar, printed in MORE_NAMES + for label, scalar, printed in MORE_NAMES + RAW_NAMES + if (label, scalar, printed) not in CORE_NAMES for encoding in NARROW] DEGRADE_LINE = "authority is not wallet-carried in this org" @@ -162,7 +192,7 @@ def test_the_name_line_is_printed_whatever_the_name_and_the_stdout( f"({spelled_root(project, encoding)})"), out assert lines[1] == "", out assert out.endswith("bootstrap ok\n"), out - assert not {"\x1b", "\r", "\x7f", "\u2028", "\xa0"} & set(out), out + assert not NEVER_RAW & set(out), out def test_a_plain_name_prints_exactly_what_it_printed_before(project, tmp_path): @@ -199,17 +229,30 @@ def test_a_plain_name_prints_exactly_what_it_printed_before(project, tmp_path): ], out +#: The valid names whose first line must be the bytes origin/main wrote: those +#: that print raw, and a mixture of printable text and a quote and a backslash. +BYTE_NAMES = [pytest.param(scalar, printed, id=label) + for label, scalar, printed in RAW_NAMES] + [ + pytest.param('"日本 🙂 \\"q\\" \\\\"', '日本 🙂 "q" \\', id="printable-mixture")] + + +@pytest.mark.parametrize("scalar, name", BYTE_NAMES) def test_a_name_a_utf8_stdout_holds_is_written_as_the_bytes_it_is( - project, tmp_path): - """Read as BYTES, so a character kept is a character kept: `日本`, an - emoji and a quote and a backslash are the UTF-8 bytes they always were.""" - edit(project, "project.yaml", 'name: "Atlas"', 'name: "日本 🙂 \\"q\\" \\\\"') + project, tmp_path, scalar, name): + """BYTE-IDENTICAL to origin/main, read as BYTES so a character kept is a + character kept: the line is exactly what the plain `print` it replaced + wrote. `日本`, an emoji, a quote and a backslash, and the valid names + `str.isprintable` refuses: a Persian name with its zero-width non-joiner, + an emoji sequence with its zero-width joiners, a Japanese full name with + the ideographic space, the no-break space, the soft hyphen and the + zero-width space and byte-order mark.""" + edit(project, "project.yaml", 'name: "Atlas"', f"name: {scalar}") code, stdout, stderr = run_tool_bytes( bootstrap_of(project), "--skip-validators", encoding="utf-8:strict", tmp_path=tmp_path, cwd=project) assert code == 0 and b"Traceback" not in stderr, stderr first = stdout.split(b"\n")[0].rstrip(b"\r") - assert first.startswith('bootstrap: 日本 🙂 "q" \\ ('.encode("utf-8")), first + assert first == f"bootstrap: {name} ({project.resolve()})".encode("utf-8") # --- what the run does NEXT -------------------------------------------------- @@ -220,16 +263,17 @@ def test_a_lone_surrogate_name_reaches_the_validators_and_ends_as_they_end( where the manifest validator fails the run, and exits as `main`'s own failed-validator path does: `bootstrap FAILED: manifest (exit 1)`, exit 1. - WHAT THE VALIDATOR SAYS is the other fix's (#280). Where it names the - surrogate (`FINDING manifest-name`), bootstrap must say it and no - traceback may be anywhere; where it does not yet, the validator's own - `print` raises in ITS process, and no traceback may come from bootstrap's. - The validator is asked by itself, so the row holds on both trees.""" + WHAT THE VALIDATOR SAYS is #280's, landed in #390: `FINDING manifest-name` + names the lone surrogate, in the validator by itself and again in the run + that carries it, and no traceback is anywhere. (Before #390 the + validator's own `manifest ok:` print raised in ITS process.)""" edit(project, "project.yaml", 'name: "Atlas"', r'name: "a\ud83db"') alone_code, _, alone_err = run( project / "scripts" / "validate-manifest.py", encoding="utf-8:strict", tmp_path=tmp_path, cwd=project) assert alone_code == 1, alone_err + assert "FINDING manifest-name" in alone_err, alone_err + assert "Traceback" not in alone_err, alone_err code, out, err = run(bootstrap_of(project), encoding="utf-8:strict", tmp_path=tmp_path, cwd=project) assert out.startswith(r"bootstrap: a\ud83db ("), out @@ -237,9 +281,32 @@ def test_a_lone_surrogate_name_reaches_the_validators_and_ends_as_they_end( assert code == 1, out + err assert "bootstrap FAILED: manifest (exit 1)" in err, err assert 'bootstrap.py", line' not in err, err - if "FINDING manifest-name" in alone_err: - assert "FINDING manifest-name" in err, err - assert "Traceback" not in err, err + assert "FINDING manifest-name" in err, err + assert "Traceback" not in err, err + + +#: The valid names the manifest validator accepts and prints itself. +VALIDATED_NAMES = [pytest.param(scalar, printed, id=label) + for label, scalar, printed in RAW_NAMES + if label in ("persian-zwnj", "emoji-zwj", + "ideographic-space")] + + +@pytest.mark.parametrize("scalar, name", VALIDATED_NAMES) +def test_a_valid_name_is_printed_as_the_validator_prints_it(project, tmp_path, + scalar, name): + """The whole run, validators included, over a valid name that + `str.isprintable` refuses. `validate-manifest.py` accepts it and prints it + raw on its `manifest ok:` line, a few lines below this script's own, so + escaping it here would show one name twice, two ways.""" + edit(project, "project.yaml", 'name: "Atlas"', f"name: {scalar}") + code, out, err = run(bootstrap_of(project), encoding="utf-8:strict", + tmp_path=tmp_path, cwd=project) + assert code == 0 and "Traceback" not in err, out + err + lines = out.split("\n") + assert lines[0] == (f"bootstrap: {name} " + f"({spelled_root(project, 'utf-8:strict')})"), out + assert any(line.startswith(f"manifest ok: {name} (") for line in lines), out @pytest.mark.skipif(shutil.which("make") is None or @@ -259,8 +326,10 @@ def test_make_bootstrap_ends_as_the_validators_end_not_in_a_traceback( for text in (proc.stdout, proc.stderr)) assert proc.returncode == 2, out + err assert r"bootstrap: a\ud83db (" in out, out + assert "FINDING manifest-name" in err, err assert "bootstrap FAILED: manifest (exit 1)" in err, err assert 'bootstrap.py", line' not in err, err + assert "Traceback" not in err, err # --- every other print that carries a value --------------------------------- @@ -479,7 +548,7 @@ def test_the_family_name_line_is_printed_whatever_the_name_and_the_stdout( assert "Traceback" not in err, err assert out.split("\n")[0] == family_line(printed, holder, encoding), out assert out.endswith("family bootstrap ok\n"), out - assert not {"\x1b", "\r", "\x7f", "\u2028", "\xa0"} & set(out), out + assert not NEVER_RAW & set(out), out def test_a_plain_family_prints_exactly_what_it_printed_before(holder, @@ -502,6 +571,21 @@ def test_a_plain_family_prints_exactly_what_it_printed_before(holder, ], out +@pytest.mark.parametrize("scalar, name", BYTE_NAMES) +def test_a_family_name_a_utf8_stdout_holds_is_written_as_the_bytes_it_is( + holder, tmp_path, scalar, name): + """The family root's first line, as BYTES: exactly what the plain `print` + it replaced wrote on origin/main, for every valid name that prints raw.""" + edit(holder, "family.yaml", f'name: "{FAMILY_NAME}"', f"name: {scalar}") + code, stdout, stderr = run_tool_bytes( + bootstrap_of(holder), "--no-fetch", "--skip-members", + encoding="utf-8:strict", tmp_path=tmp_path, cwd=holder) + assert code == 0 and b"Traceback" not in stderr, stderr + first = stdout.split(b"\n")[0].rstrip(b"\r") + assert first == (f"bootstrap: {name} ({holder.resolve()}) \u2014 0 member(s)" + ).encode("utf-8") + + def test_a_family_roots_own_directory_name_is_spelled(holder, tmp_path): """The second thing the family's first line prints, under a stdout that HOLDS a line separator, so a raw one would be written raw.""" @@ -590,20 +674,75 @@ def template(request): @pytest.mark.parametrize("text", [ "Atlas", "日本", "Zo\u00eb", "\U0001f642", 'He said "hi" \\o/', "a b", "", - "/tmp/x y/z", "C:\\Users\\a\\Atlas", "a=b: c, [d] {e}"]) + "/tmp/x y/z", "C:\\Users\\a\\Atlas", "a=b: c, [d] {e}", + # What `str.isprintable` refuses and a UTF-8 terminal shows correctly: + # Zs (the no-break and ideographic spaces), Cf (the zero-width non-joiner + # and joiner, the soft hyphen, the zero-width space, the byte-order mark, + # a tag character), Co (private use) and Cn (unassigned). Printed raw + # before the fix, so printed raw now. + "a\xa0b", "\u5c71\u7530\u3000\u592a\u90ce", "co\xadop", + "\u0645\u06cc\u200c\u062e\u0648\u0627\u0647\u0645", + "\U0001f468\u200d\U0001f469\u200d\U0001f467", "a\u200bb", "\ufeffa", + "\U000e0001", "\ue000", "\u0378", "\u2000\u2009\u202f\u205f"]) def test_spelled_keeps_what_a_terminal_can_be_shown(template, text): assert template.spelled(text) == text @pytest.mark.parametrize("char, spelling", [ ("\x00", r"\x00"), ("\x1b", r"\x1b"), ("\r", r"\r"), ("\n", r"\n"), - ("\t", r"\t"), ("\x7f", r"\x7f"), ("\x85", r"\x85"), ("\xa0", r"\xa0"), - ("\u200b", r"\u200b"), ("\u2028", r"\u2028"), ("\ud83d", r"\ud83d"), - ("\udce9", r"\udce9"), ("\U000e0001", r"\U000e0001")]) + ("\t", r"\t"), ("\x7f", r"\x7f"), ("\x85", r"\x85"), ("\x9f", r"\x9f"), + ("\u2028", r"\u2028"), ("\u2029", r"\u2029"), ("\ud83d", r"\ud83d"), + ("\udce9", r"\udce9"), ("\ud800", r"\ud800"), ("\udfff", r"\udfff"), + # The bidirectional controls: the right-to-left override and mark, the + # left-to-right mark, the Arabic letter mark, every embedding and isolate. + ("\u202e", r"\u202e"), ("\u200f", r"\u200f"), ("\u200e", r"\u200e"), + ("\u061c", r"\u061c"), ("\u202a", r"\u202a"), ("\u202b", r"\u202b"), + ("\u202c", r"\u202c"), ("\u202d", r"\u202d"), ("\u2066", r"\u2066"), + ("\u2067", r"\u2067"), ("\u2068", r"\u2068"), ("\u2069", r"\u2069")]) def test_spelled_writes_the_escape_python_spells(template, char, spelling): assert template.spelled(f"a{char}b") == f"a{spelling}b" +#: The code points `spelled` escapes, from the specification and not from the +#: implementation: the controls (Cc), the surrogates (Cs), the line and +#: paragraph separators (Zl, Zp), and the bidirectional controls. +ESCAPED_CODES = frozenset( + {*range(0x00, 0x20), *range(0x7f, 0xa0), *range(0xd800, 0xe000), + 0x2028, 0x2029, 0x061c, 0x200e, 0x200f, *range(0x202a, 0x202f), + *range(0x2066, 0x206a)}) + + +def escape_of(code: int) -> str: + """The escape Python spells `code` as: its own name for a tab, a line feed + and a carriage return, else `\\xNN` or `\\uNNNN`.""" + named = {9: r"\t", 10: r"\n", 13: r"\r"} + if code in named: + return named[code] + return rf"\x{code:02x}" if code < 0x100 else rf"\u{code:04x}" + + +def test_spelled_escapes_exactly_the_specified_characters(template): + """EVERY code point, once, in one string: it is escaped when it is in the + specified set and left alone when it is not. A character class by + category (Cc, Cs, Zl, Zp) and not `str.isprintable`, so Zs, Cf, Co and Cn + stay raw.""" + codes = range(sys.maxunicode + 1) + expected = "".join(escape_of(code) if code in ESCAPED_CODES + else chr(code) for code in codes) + assert template.spelled("".join(map(chr, codes))) == expected + + +def test_the_specified_set_is_the_four_categories_and_the_bidi_controls(): + """The test's own set agrees with the category names the spec gives, on + this interpreter's Unicode database (the four categories are fixed ranges, + so it holds on every one).""" + categories = {code for code in range(sys.maxunicode + 1) + if unicodedata.category(chr(code)) in ("Cc", "Cs", "Zl", "Zp")} + bidi = {0x061c, 0x200e, 0x200f, *range(0x202a, 0x202f), *range(0x2066, 0x206a)} + assert ESCAPED_CODES == categories | bidi + assert {unicodedata.category(chr(code)) for code in bidi} <= {"Cf"} + + def test_spelled_takes_what_a_manifest_can_hold(template): """`name: 5` and `role: [a]` are values too: the text a print would have made of them, spelled.""" @@ -613,12 +752,15 @@ def test_spelled_takes_what_a_manifest_can_hold(template): assert template.spelled(Path("b\u2028")) == "b\\u2028" -def test_spelled_leaves_one_line_of_printable_text(template): +def test_spelled_leaves_one_line_free_of_what_must_not_reach_a_terminal( + template): every = "".join(chr(code) for code in (*range(0x100), 0x2028, 0x2029, - 0xd800, 0xdc80, 0xdfff, 0xe000, - 0xfeff, 0x1f642, 0xe0001)) + 0x202e, 0x200f, 0xd800, 0xdc80, + 0xdfff, 0xe000, 0xfeff, 0x1f642, + 0xe0001)) out = template.spelled(every) - assert out.isprintable() and "\n" not in out, out + assert "\n" not in out and not NEVER_RAW & set(out), out + assert not {chr(code) for code in ESCAPED_CODES} & set(out), out @pytest.mark.parametrize("errors, expected", [ @@ -661,16 +803,27 @@ class Replaced: def source_of(path: Path, name: str) -> str: + """The source of the module-level function or assignment called `name`.""" text = path.read_text(encoding="utf-8").replace("\r\n", "\n") - node = next(node for node in ast.parse(text).body - if isinstance(node, ast.FunctionDef) and node.name == name) + + def named(node: ast.stmt) -> bool: + if isinstance(node, ast.FunctionDef): + return node.name == name + return (isinstance(node, ast.Assign) + and [target.id for target in node.targets + if isinstance(target, ast.Name)] == [name]) + + node = next(node for node in ast.parse(text).body if named(node)) return ast.get_source_segment(text, node) -@pytest.mark.parametrize("name", ["escape_unencodable_stdout", "spelled"]) +@pytest.mark.parametrize("name", ["escape_unencodable_stdout", "spelled", + "ESCAPED_CATEGORIES", + "ESCAPED_BIDI_CONTROLS"]) def test_the_two_roots_carry_the_same_function(name): """Both files are shape copies that cannot import anything of this - repository's, so the helper is written twice; this is what keeps it one.""" + repository's, so the helper is written twice; this is what keeps it one. + The two sets `spelled` reads are part of it.""" assert (source_of(ASSEMBLY_TEMPLATE, name) == source_of(FAMILY_TEMPLATE, name)), name From 2733c6a1497201c2325c920cb22a58af34f232a4 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 12 Oct 2026 20:23:25 +0000 Subject: [PATCH 3/4] tests/test_bootstrap_spelling.py: load each root's bootstrap.py without writing bytecode into templates/, so the template tree a later test reads holds no .pyc CI run 38297896001 (Linux and macOS) failed three cases of `tests/test_scaffold_leg_path_collisions.py::test_the_paths_refused_are_the_paths_the_materializer_writes` ([no-pin], [one-pin], [two-pins]) with a `UnicodeDecodeError` on a CPython `.pyc` header: the test reads every file under `templates/assembly-root` as UTF-8 text, and there was a `.pyc` in it. This file wrote it. Its `load()` runs each root's `scripts/bootstrap.py` with `SourceFileLoader`, which caches the bytecode it compiles in a `__pycache__` beside the source, so the module-scoped `template` fixture left `templates/-root/scripts/__pycache__/bootstrap.cpython-3XX.pyc` in the tree. `.gitignore` ignores `__pycache__/`, so `git status` never showed it, and `test_scaffold_leg_path_collisions.py` sorts after this file, so it read it. Reproduced locally on CPython 3.12 with the two files in one process, in that order: 3 failed, 361 passed. A tree a materializer walks is open to the same stray. `load()` now sets `sys.dont_write_bytecode` around `exec_module` and puts it back in a `finally`. It keeps importlib and the module's `__spec__` as they were, and the flag also covers the `repo_shape` a root imports as it runs, which compiling the source into a bare `types.ModuleType` would have left to the normal import machinery. One regression test, `test_loading_a_root_leaves_no_file_where_it_was_loaded_from`, snapshots every path under `templates/`, loads both roots, and asserts the snapshot is unchanged. It does not ask that no `__pycache__` exists, because a checkout may hold a stale one from an older run. A tree that already holds the cache would pass for any `load`, so each root is also loaded from a copy in a directory that holds nothing. With the flag removed, the test fails on a fresh tree (the real-tree assertion) and on a stale one (the copy assertion). Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- tests/test_bootstrap_spelling.py | 38 +++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index e150b9b1..0a545fdf 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -657,12 +657,48 @@ def test_a_members_row_and_the_make_target_are_spelled(holder, tmp_path): # --- the two functions, and the two files that carry them ------------------- def load(path: Path, name: str): + """`path` run as a module called `name`, and no bytecode written for it. + + `SourceFileLoader` caches what it compiles in a `__pycache__` beside the + source, and these sources are the TEMPLATES: a tree other tests read file + by file as text (`test_scaffold_leg_path_collisions.py` does, and a `.pyc` + is not UTF-8), and a tree a materializer walks. `.gitignore` hides the + stray, so `git status` never says it is there. The flag is process-wide, so + it is restored; it also covers the `repo_shape` a root imports as it runs. + """ spec = importlib.util.spec_from_file_location(name, path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + wrote_bytecode = sys.dont_write_bytecode + sys.dont_write_bytecode = True + try: + spec.loader.exec_module(module) + finally: + sys.dont_write_bytecode = wrote_bytecode return module +def test_loading_a_root_leaves_no_file_where_it_was_loaded_from(tmp_path, + monkeypatch): + """The tree under `templates/` is the same after both roots are loaded. + + A checkout may hold a stale `__pycache__` from an older run, so nothing + here asks that none exists; it asks that loading changes nothing. A tree + that already holds the cache would pass for any `load`, so each root is + loaded from a copy as well, in a directory that holds nothing.""" + monkeypatch.setattr(sys, "path", list(sys.path)) # a root puts its own in + templates = REPO / "templates" + before = set(templates.rglob("*")) + for number, path in enumerate((ASSEMBLY_TEMPLATE, FAMILY_TEMPLATE)): + copy = tmp_path / f"copy{number}" / "scripts" / path.name + copy.parent.mkdir(parents=True) + shutil.copyfile(path, copy) + before_copy = set(copy.parent.rglob("*")) + load(path, f"bootstrap_spelling_probe_{number}") + assert set(templates.rglob("*")) == before, path + load(copy, f"bootstrap_spelling_probe_copy_{number}") + assert set(copy.parent.rglob("*")) == before_copy, copy.parent + + @pytest.fixture(scope="module", params=[ASSEMBLY_TEMPLATE, FAMILY_TEMPLATE], ids=["assembly-root", "family-root"]) def template(request): From d0aea3e0c9fd3fda74a681bc50ca31180cbe1231 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 12 Oct 2026 20:53:10 +0000 Subject: [PATCH 4/4] scripts/repo_shape.py carries spelled() and escape_unencodable_stdout(), which both roots' bootstrap.py now import instead of defining twice (#391) Why. SonarCloud failed PR #456 on duplication (14.7% on new code against a 3% gate): spelled() and escape_unencodable_stdout(), with the two sets spelled() reads, were written out once in each root's scripts/bootstrap.py, and a test held the two copies identical. The premise behind the copies, that a root cannot import anything of this repository's, was false: every project and every family holder carries scripts/repo_shape.py, the pinned shared library, and both bootstraps already import Refusal, find_repo_root and load_yaml from it. Brett Heap's decision (2026-10-12): move the two helpers into repo_shape.py and have both bootstraps import them; no Sonar exclusion. Where. A new section at the END of scripts/repo_shape.py, after die(): ESCAPED_CATEGORIES, ESCAPED_BIDI_CONTROLS, escape_unencodable_stdout() and spelled(), copied from the bootstrap source byte for byte. Their code is unchanged (the AST of each, docstrings aside, equals both old copies); only the docstring sentences the move made untrue are reworded ("this file", "this script", "the same in the assembly root's bootstrap.py and the family root's"). `import unicodedata` joins the imports after `import sys`, and the module docstring's "WHAT THIS MODULE OWNS" list gains one bullet after `Refusal`. Those three places are clear of the open PRs that also edit this file (#394, #409, #416, #453): each of them merges with this commit without a conflict in repo_shape.py. Both templates/*/scripts/bootstrap.py lose the two definitions, the two sets and `import unicodedata`, and add the two names to their existing `from repo_shape import` line (the family root's becomes the parenthesised form). escape_unencodable_stdout() is still main's first statement, and the module level is still only `sys.path.insert` and the import. Against origin/main, each bootstrap now differs only in that import line and the spelled() call sites. Tests (tests/test_bootstrap_spelling.py). The test test_the_two_roots_carry_the_same_function is replaced by an ast check that each root's `from repo_shape import` names both and that nothing in the file defines or assigns either function or either set; one more test per root asserts that the name its module exposes IS repo_shape's object. The unit rows of spelled() (keep-raw, escape, the full code-point sweep, the non-string values, the one-line check) and of escape_unencodable_stdout() now run once against repo_shape itself instead of once per root. The end-to-end rows run a real scaffolded project and a real family holder, which carry scripts/repo_shape.py, so they exercise the moved code where it ships; they and the bytecode regression test are unchanged. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- scripts/repo_shape.py | 100 ++++++++++++++++ templates/assembly-root/scripts/bootstrap.py | 89 +------------- templates/family-root/scripts/bootstrap.py | 90 +------------- tests/test_bootstrap_spelling.py | 117 +++++++++++-------- 4 files changed, 175 insertions(+), 221 deletions(-) diff --git a/scripts/repo_shape.py b/scripts/repo_shape.py index 67b66a78..2bcae52f 100644 --- a/scripts/repo_shape.py +++ b/scripts/repo_shape.py @@ -27,6 +27,9 @@ - `Refusal` — the fail-closed exception every validator raises, carrying a remediation string, because a refusal that names what is wrong without naming what to run puts the exit in tribal memory instead of the message. + - `spelled` and `escape_unencodable_stdout` — how both roots' + `bootstrap.py` print a person's text: what a terminal must not be handed + raw is escaped, and a stdout that cannot encode it never raises (#391). This file is COPIED into a scaffolded assembly root by `scaffold-project.py` and its sha256 is recorded in that project's `contracts/shape-pin.yaml`, so an @@ -41,6 +44,7 @@ import re import subprocess import sys +import unicodedata from pathlib import Path from typing import Any @@ -2514,3 +2518,99 @@ def find_repo_root(start: Path) -> Path: def die(exc: Refusal, stream=sys.stderr) -> int: print(str(exc), file=stream) return 2 + + +# --------------------------------------------------------------------------- +# Printing a person's text (#391) +# --------------------------------------------------------------------------- +# +# BOTH ROOTS' `bootstrap.py` print what a manifest, a register row or the +# disk says, before any validator has had its say about it. Each imports +# these two from here, as it imports the YAML reader: this file is the one +# library a project and a family holder carry beside it. + +#: The characters `spelled` writes as an escape, besides the bidirectional +#: controls below: the controls (Cc), the surrogates (Cs), and the line and +#: paragraph separators (Zl, Zp). Nothing else is escaped, `spelled` says why. +ESCAPED_CATEGORIES = frozenset(("Cc", "Cs", "Zl", "Zp")) +#: The bidirectional controls: U+061C, U+200E and U+200F, the embeddings and +#: overrides U+202A to U+202E, and the isolates U+2066 to U+2069. +ESCAPED_BIDI_CONTROLS = frozenset( + "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") + + +def escape_unencodable_stdout() -> None: + """A character stdout cannot encode is printed escaped, never raised on (#391). + + WHAT WAS WRONG. Each root's `bootstrap.py` `main` printed the manifest's + `name` with a plain `print` before any step ran, so `name: "a\\ud83db"` (a + lone surrogate, which no UTF-8 stdout can write) or `name: "日本"` on a + stdout that is not UTF-8 (`PYTHONIOENCODING=ascii`, or the ANSI code page + Windows gives a stdout that is not the console) ended `make bootstrap` in a + `UnicodeEncodeError` traceback one step before `validate-manifest.py`, the + validator that names the problem, was reached. Their own wording raised on + such a stdout too, wherever it carries a dash or a middle dot. + + A `strict` or `surrogateescape` stdout, the two handlers that raise, is + reconfigured ONCE, at the start of each root's `main`, to + `backslashreplace`, the handler stderr already has: what the stdout holds + is written as it was, and what it cannot hold as `\\u65e5`. Any other + handler, one somebody chose, is left alone, and the ENCODING is never + changed. `scripts/validate-pins.py` does the same for its report (#295). + The tools run from a checkout of the standard use + `shape_materialize.escape_what_stdout_cannot_encode` (#214), which no + project carries. This module is what every project and every family holder + carries beside its `bootstrap.py`, so both roots import this function and + `spelled` from here, where each is written once. + + GUARDED, because failing to set a handler is no reason to refuse the run: a + stream a caller replaced may have no `reconfigure`, a closed one raises + `ValueError`, and `pythonw` has no stdout at all. + """ + if getattr(sys.stdout, "errors", None) in ("strict", "surrogateescape"): + try: + sys.stdout.reconfigure(errors="backslashreplace") + except (AttributeError, ValueError, OSError): + pass + + +def spelled(value: object) -> str: + """`value` as ONE line of text, with what a terminal must not be handed + written as escapes (#391). + + What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on + every stdout, so it reaches the terminal as the control it is, and a + manifest's `name` or a register row is a person's text. EXACTLY these + characters are written as the escape Python spells it (`\\x1b`, `\\r`, + `\\u2028`, `\\ud83d`, `\\u202e`), and no others: + + - the controls (Unicode category Cc: C0, DEL and C1), which move a cursor, + clear a screen or end a line; + - the surrogates (Cs), which no UTF-8 text holds, so a lone one is named + whatever handler the stdout has; + - the line and paragraph separators (Zl and Zp, U+2028 and U+2029), which + end a line in some viewers, so the value would not stay on its line; + - the bidirectional controls (U+061C, U+200E, U+200F, U+202A to U+202E and + U+2066 to U+2069), which reorder the text around them so that a name + reads as another name. + + EVERY OTHER CHARACTER PRINTS AS IT IS, as it did before this function + existed: the zero-width non-joiner a Persian or Indic name needs, the + zero-width joiner that makes an emoji sequence, the ideographic space of a + Japanese full name, the no-break space, the soft hyphen, and any other + format, space or unassigned character. `str.isprintable` is not the test: + it refuses all of those, which `validate-manifest.py` accepts and a UTF-8 + terminal shows correctly, and what it counts as unassigned moves with the + interpreter's Unicode database. A backslash prints as it is too, so a name + holding the four characters `\\x1b` prints as one holding ESC does. + + Each root's `bootstrap.py` applies it to every value it prints that it + read from the manifest, a register row or the disk, and to none of its + own wording. + """ + return "".join( + char.encode("unicode_escape").decode("ascii") + if (char in ESCAPED_BIDI_CONTROLS + or unicodedata.category(char) in ESCAPED_CATEGORIES) + else char + for char in str(value)) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 2a5af60a..d2b25a9c 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -51,12 +51,12 @@ import re import subprocess import sys -import unicodedata from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from repo_shape import ( # noqa: E402 - Refusal, find_repo_root, git_out, load_yaml, recorded_gitlink, + Refusal, escape_unencodable_stdout, find_repo_root, git_out, load_yaml, + recorded_gitlink, spelled, ) DEGRADE_LINE = "authority is not wallet-carried in this org" @@ -81,91 +81,6 @@ ) -#: The characters `spelled` writes as an escape, besides the bidirectional -#: controls below: the controls (Cc), the surrogates (Cs), and the line and -#: paragraph separators (Zl, Zp). Nothing else is escaped, `spelled` says why. -ESCAPED_CATEGORIES = frozenset(("Cc", "Cs", "Zl", "Zp")) -#: The bidirectional controls: U+061C, U+200E and U+200F, the embeddings and -#: overrides U+202A to U+202E, and the isolates U+2066 to U+2069. -ESCAPED_BIDI_CONTROLS = frozenset( - "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") - - -def escape_unencodable_stdout() -> None: - """A character stdout cannot encode is printed escaped, never raised on (#391). - - WHAT WAS WRONG. `main` printed the manifest's `name` with a plain `print` - before any step ran, so `name: "a\\ud83db"` (a lone surrogate, which no - UTF-8 stdout can write) or `name: "日本"` on a stdout that is not UTF-8 - (`PYTHONIOENCODING=ascii`, or the ANSI code page Windows gives a stdout that - is not the console) ended `make bootstrap` in a `UnicodeEncodeError` - traceback one step before `validate-manifest.py`, the validator that names - the problem, was reached. This file's own wording raised on such a stdout - too, wherever it carries a dash or a middle dot. - - A `strict` or `surrogateescape` stdout, the two handlers that raise, is - reconfigured ONCE, at the start of `main`, to `backslashreplace`, the - handler stderr already has: what the stdout holds is written as it was, and - what it cannot hold as `\\u65e5`. Any other handler, one somebody chose, is - left alone, and the ENCODING is never changed. `scripts/validate-pins.py` - does the same for its report (#295). The tools run from a checkout of the - standard use `shape_materialize.escape_what_stdout_cannot_encode` (#214), - which no project carries, and this file imports nothing but `repo_shape`. - This function and `spelled` are the same in the assembly root's - `bootstrap.py` and the family root's. - - GUARDED, because failing to set a handler is no reason to refuse the run: a - stream a caller replaced may have no `reconfigure`, a closed one raises - `ValueError`, and `pythonw` has no stdout at all. - """ - if getattr(sys.stdout, "errors", None) in ("strict", "surrogateescape"): - try: - sys.stdout.reconfigure(errors="backslashreplace") - except (AttributeError, ValueError, OSError): - pass - - -def spelled(value: object) -> str: - """`value` as ONE line of text, with what a terminal must not be handed - written as escapes (#391). - - What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on - every stdout, so it reaches the terminal as the control it is, and a - manifest's `name` or a register row is a person's text. EXACTLY these - characters are written as the escape Python spells it (`\\x1b`, `\\r`, - `\\u2028`, `\\ud83d`, `\\u202e`), and no others: - - - the controls (Unicode category Cc: C0, DEL and C1), which move a cursor, - clear a screen or end a line; - - the surrogates (Cs), which no UTF-8 text holds, so a lone one is named - whatever handler the stdout has; - - the line and paragraph separators (Zl and Zp, U+2028 and U+2029), which - end a line in some viewers, so the value would not stay on its line; - - the bidirectional controls (U+061C, U+200E, U+200F, U+202A to U+202E and - U+2066 to U+2069), which reorder the text around them so that a name - reads as another name. - - EVERY OTHER CHARACTER PRINTS AS IT IS, as it did before this function - existed: the zero-width non-joiner a Persian or Indic name needs, the - zero-width joiner that makes an emoji sequence, the ideographic space of a - Japanese full name, the no-break space, the soft hyphen, and any other - format, space or unassigned character. `str.isprintable` is not the test: - it refuses all of those, which `validate-manifest.py` accepts and a UTF-8 - terminal shows correctly, and what it counts as unassigned moves with the - interpreter's Unicode database. A backslash prints as it is too, so a name - holding the four characters `\\x1b` prints as one holding ESC does. - - Applied to every value this script prints that it read from the manifest, a - register row or the disk, and to none of its own wording. - """ - return "".join( - char.encode("unicode_escape").decode("ascii") - if (char in ESCAPED_BIDI_CONTROLS - or unicodedata.category(char) in ESCAPED_CATEGORIES) - else char - for char in str(value)) - - def _run(root: Path, args: list[str]) -> int: # Flush first: this process's own stdout is block-buffered when it is a # pipe, and a child writing straight to the fd would otherwise appear diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 41ec4fbc..d9ddcc30 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -50,11 +50,12 @@ import shutil import subprocess import sys -import unicodedata from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from repo_shape import Refusal, find_repo_root, load_yaml # noqa: E402 +from repo_shape import ( # noqa: E402 + Refusal, escape_unencodable_stdout, find_repo_root, load_yaml, spelled, +) MANIFEST = "family.yaml" DEFAULT_MEMBERS_DIR = "members" @@ -69,91 +70,6 @@ REWRITE_TARGETS = ("https://github.com/", "git@github.com:") -#: The characters `spelled` writes as an escape, besides the bidirectional -#: controls below: the controls (Cc), the surrogates (Cs), and the line and -#: paragraph separators (Zl, Zp). Nothing else is escaped, `spelled` says why. -ESCAPED_CATEGORIES = frozenset(("Cc", "Cs", "Zl", "Zp")) -#: The bidirectional controls: U+061C, U+200E and U+200F, the embeddings and -#: overrides U+202A to U+202E, and the isolates U+2066 to U+2069. -ESCAPED_BIDI_CONTROLS = frozenset( - "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") - - -def escape_unencodable_stdout() -> None: - """A character stdout cannot encode is printed escaped, never raised on (#391). - - WHAT WAS WRONG. `main` printed the manifest's `name` with a plain `print` - before any step ran, so `name: "a\\ud83db"` (a lone surrogate, which no - UTF-8 stdout can write) or `name: "日本"` on a stdout that is not UTF-8 - (`PYTHONIOENCODING=ascii`, or the ANSI code page Windows gives a stdout that - is not the console) ended `make bootstrap` in a `UnicodeEncodeError` - traceback one step before `validate-manifest.py`, the validator that names - the problem, was reached. This file's own wording raised on such a stdout - too, wherever it carries a dash or a middle dot. - - A `strict` or `surrogateescape` stdout, the two handlers that raise, is - reconfigured ONCE, at the start of `main`, to `backslashreplace`, the - handler stderr already has: what the stdout holds is written as it was, and - what it cannot hold as `\\u65e5`. Any other handler, one somebody chose, is - left alone, and the ENCODING is never changed. `scripts/validate-pins.py` - does the same for its report (#295). The tools run from a checkout of the - standard use `shape_materialize.escape_what_stdout_cannot_encode` (#214), - which no project carries, and this file imports nothing but `repo_shape`. - This function and `spelled` are the same in the assembly root's - `bootstrap.py` and the family root's. - - GUARDED, because failing to set a handler is no reason to refuse the run: a - stream a caller replaced may have no `reconfigure`, a closed one raises - `ValueError`, and `pythonw` has no stdout at all. - """ - if getattr(sys.stdout, "errors", None) in ("strict", "surrogateescape"): - try: - sys.stdout.reconfigure(errors="backslashreplace") - except (AttributeError, ValueError, OSError): - pass - - -def spelled(value: object) -> str: - """`value` as ONE line of text, with what a terminal must not be handed - written as escapes (#391). - - What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on - every stdout, so it reaches the terminal as the control it is, and a - manifest's `name` or a register row is a person's text. EXACTLY these - characters are written as the escape Python spells it (`\\x1b`, `\\r`, - `\\u2028`, `\\ud83d`, `\\u202e`), and no others: - - - the controls (Unicode category Cc: C0, DEL and C1), which move a cursor, - clear a screen or end a line; - - the surrogates (Cs), which no UTF-8 text holds, so a lone one is named - whatever handler the stdout has; - - the line and paragraph separators (Zl and Zp, U+2028 and U+2029), which - end a line in some viewers, so the value would not stay on its line; - - the bidirectional controls (U+061C, U+200E, U+200F, U+202A to U+202E and - U+2066 to U+2069), which reorder the text around them so that a name - reads as another name. - - EVERY OTHER CHARACTER PRINTS AS IT IS, as it did before this function - existed: the zero-width non-joiner a Persian or Indic name needs, the - zero-width joiner that makes an emoji sequence, the ideographic space of a - Japanese full name, the no-break space, the soft hyphen, and any other - format, space or unassigned character. `str.isprintable` is not the test: - it refuses all of those, which `validate-manifest.py` accepts and a UTF-8 - terminal shows correctly, and what it counts as unassigned moves with the - interpreter's Unicode database. A backslash prints as it is too, so a name - holding the four characters `\\x1b` prints as one holding ESC does. - - Applied to every value this script prints that it read from the manifest, a - register row or the disk, and to none of its own wording. - """ - return "".join( - char.encode("unicode_escape").decode("ascii") - if (char in ESCAPED_BIDI_CONTROLS - or unicodedata.category(char) in ESCAPED_CATEGORIES) - else char - for char in str(value)) - - def credential() -> tuple[str, str]: """`(source, token)`; `("none", "")` when the environment carries none.""" for name in CREDENTIAL_ENV: diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index 0a545fdf..9c65ccf0 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -15,8 +15,10 @@ Both files are shape copies (digest-pinned in every project that holds them) and a project carries `scripts/repo_shape.py` and no `shape_materialize.py`, so they -cannot call #214's helper. Each now carries two small functions, the same in -both: `escape_unencodable_stdout` reconfigures a `strict` or `surrogateescape` +cannot call #214's helper. Both now import two small functions from +`repo_shape.py`, the shared library every project and every family holder +carries beside its `bootstrap.py`, where each is written once: +`escape_unencodable_stdout` reconfigures a `strict` or `surrogateescape` stdout to `backslashreplace` once, at the start of `main`; `spelled` writes the controls (category Cc), the surrogates (Cs), the line and paragraph separators (Zl, Zp) and the bidirectional controls (U+061C, U+200E, U+200F, U+202A to @@ -37,8 +39,8 @@ - what the run does NEXT: it reaches the validators and ends as they end, at the `FINDING manifest-name` that `validate-manifest.py` now gives a name holding a lone surrogate (#280, landed in #390), with no traceback; -- the two functions themselves, and a guard that the two files carry the same - two. +- the two functions themselves, in `repo_shape.py`, and a guard that each root + imports both from there, runs `repo_shape`'s own objects and defines neither. A plain name must print exactly what it printed before: a golden of the whole output, from known values. @@ -61,6 +63,7 @@ from test_stdout_encoding import child_environ, run_tool_bytes, shown sys.path.insert(0, str(REPO / "scripts")) +import repo_shape # noqa: E402 ASSEMBLY_TEMPLATE = (REPO / "templates" / "assembly-root" / "scripts" / "bootstrap.py") @@ -654,7 +657,7 @@ def test_a_members_row_and_the_make_target_are_spelled(holder, tmp_path): assert not {"\x1b", "\u2028"} & set(out), out -# --- the two functions, and the two files that carry them ------------------- +# --- the two functions, in repo_shape.py, and the roots that import them ----- def load(path: Path, name: str): """`path` run as a module called `name`, and no bytecode written for it. @@ -720,8 +723,8 @@ def template(request): "\u0645\u06cc\u200c\u062e\u0648\u0627\u0647\u0645", "\U0001f468\u200d\U0001f469\u200d\U0001f467", "a\u200bb", "\ufeffa", "\U000e0001", "\ue000", "\u0378", "\u2000\u2009\u202f\u205f"]) -def test_spelled_keeps_what_a_terminal_can_be_shown(template, text): - assert template.spelled(text) == text +def test_spelled_keeps_what_a_terminal_can_be_shown(text): + assert repo_shape.spelled(text) == text @pytest.mark.parametrize("char, spelling", [ @@ -735,8 +738,8 @@ def test_spelled_keeps_what_a_terminal_can_be_shown(template, text): ("\u061c", r"\u061c"), ("\u202a", r"\u202a"), ("\u202b", r"\u202b"), ("\u202c", r"\u202c"), ("\u202d", r"\u202d"), ("\u2066", r"\u2066"), ("\u2067", r"\u2067"), ("\u2068", r"\u2068"), ("\u2069", r"\u2069")]) -def test_spelled_writes_the_escape_python_spells(template, char, spelling): - assert template.spelled(f"a{char}b") == f"a{spelling}b" +def test_spelled_writes_the_escape_python_spells(char, spelling): + assert repo_shape.spelled(f"a{char}b") == f"a{spelling}b" #: The code points `spelled` escapes, from the specification and not from the @@ -757,7 +760,7 @@ def escape_of(code: int) -> str: return rf"\x{code:02x}" if code < 0x100 else rf"\u{code:04x}" -def test_spelled_escapes_exactly_the_specified_characters(template): +def test_spelled_escapes_exactly_the_specified_characters(): """EVERY code point, once, in one string: it is escaped when it is in the specified set and left alone when it is not. A character class by category (Cc, Cs, Zl, Zp) and not `str.isprintable`, so Zs, Cf, Co and Cn @@ -765,7 +768,7 @@ def test_spelled_escapes_exactly_the_specified_characters(template): codes = range(sys.maxunicode + 1) expected = "".join(escape_of(code) if code in ESCAPED_CODES else chr(code) for code in codes) - assert template.spelled("".join(map(chr, codes))) == expected + assert repo_shape.spelled("".join(map(chr, codes))) == expected def test_the_specified_set_is_the_four_categories_and_the_bidi_controls(): @@ -779,22 +782,21 @@ def test_the_specified_set_is_the_four_categories_and_the_bidi_controls(): assert {unicodedata.category(chr(code)) for code in bidi} <= {"Cf"} -def test_spelled_takes_what_a_manifest_can_hold(template): +def test_spelled_takes_what_a_manifest_can_hold(): """`name: 5` and `role: [a]` are values too: the text a print would have made of them, spelled.""" - assert template.spelled(5) == "5" - assert template.spelled(None) == "None" - assert template.spelled(["a\x1b"]) == "['a\\x1b']" - assert template.spelled(Path("b\u2028")) == "b\\u2028" + assert repo_shape.spelled(5) == "5" + assert repo_shape.spelled(None) == "None" + assert repo_shape.spelled(["a\x1b"]) == "['a\\x1b']" + assert repo_shape.spelled(Path("b\u2028")) == "b\\u2028" -def test_spelled_leaves_one_line_free_of_what_must_not_reach_a_terminal( - template): +def test_spelled_leaves_one_line_free_of_what_must_not_reach_a_terminal(): every = "".join(chr(code) for code in (*range(0x100), 0x2028, 0x2029, 0x202e, 0x200f, 0xd800, 0xdc80, 0xdfff, 0xe000, 0xfeff, 0x1f642, 0xe0001)) - out = template.spelled(every) + out = repo_shape.spelled(every) assert "\n" not in out and not NEVER_RAW & set(out), out assert not {chr(code) for code in ESCAPED_CODES} & set(out), out @@ -809,11 +811,11 @@ def test_spelled_leaves_one_line_free_of_what_must_not_reach_a_terminal( ("namereplace", "namereplace"), ]) def test_a_stdout_that_raises_is_made_to_escape_and_no_other_is_touched( - template, monkeypatch, errors, expected): + monkeypatch, errors, expected): stream = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors=errors, newline="\n") monkeypatch.setattr(sys, "stdout", stream) - template.escape_unencodable_stdout() + repo_shape.escape_unencodable_stdout() assert (stream.errors, stream.encoding) == (expected, "cp1252") print("\u65e5 \u2014", end="") stream.flush() @@ -822,8 +824,7 @@ def test_a_stdout_that_raises_is_made_to_escape_and_no_other_is_touched( assert written == b"\\u65e5 \x97", written -def test_a_stdout_that_cannot_be_reconfigured_fails_nothing(template, - monkeypatch): +def test_a_stdout_that_cannot_be_reconfigured_fails_nothing(monkeypatch): """No `reconfigure`, a closed stream and no stdout at all (`pythonw`).""" class Replaced: @@ -831,37 +832,59 @@ class Replaced: for stdout in (Replaced(), None): monkeypatch.setattr(sys, "stdout", stdout) - template.escape_unencodable_stdout() + repo_shape.escape_unencodable_stdout() closed = io.TextIOWrapper(io.BytesIO(), encoding="cp1252", errors="strict") closed.close() monkeypatch.setattr(sys, "stdout", closed) - template.escape_unencodable_stdout() + repo_shape.escape_unencodable_stdout() -def source_of(path: Path, name: str) -> str: - """The source of the module-level function or assignment called `name`.""" - text = path.read_text(encoding="utf-8").replace("\r\n", "\n") +#: The two functions both roots call, and the two sets `spelled` reads. +MOVED = ("escape_unencodable_stdout", "spelled", "ESCAPED_CATEGORIES", + "ESCAPED_BIDI_CONTROLS") - def named(node: ast.stmt) -> bool: - if isinstance(node, ast.FunctionDef): - return node.name == name - return (isinstance(node, ast.Assign) - and [target.id for target in node.targets - if isinstance(target, ast.Name)] == [name]) - node = next(node for node in ast.parse(text).body if named(node)) - return ast.get_source_segment(text, node) - - -@pytest.mark.parametrize("name", ["escape_unencodable_stdout", "spelled", - "ESCAPED_CATEGORIES", - "ESCAPED_BIDI_CONTROLS"]) -def test_the_two_roots_carry_the_same_function(name): - """Both files are shape copies that cannot import anything of this - repository's, so the helper is written twice; this is what keeps it one. - The two sets `spelled` reads are part of it.""" - assert (source_of(ASSEMBLY_TEMPLATE, name) - == source_of(FAMILY_TEMPLATE, name)), name +@pytest.mark.parametrize("path", [ASSEMBLY_TEMPLATE, FAMILY_TEMPLATE], + ids=["assembly-root", "family-root"]) +def test_each_root_imports_both_functions_and_defines_neither(path): + """Written ONCE, in `scripts/repo_shape.py`: the shared library every + project and every family holder carries beside its `bootstrap.py`, and + the one each root already imported its YAML reader from. Each root's + module-level `from repo_shape import (...)` names both, and nothing in + the file defines or assigns any of the four names again, so there is no + second copy to drift from the first.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + bound = set() + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, + ast.ClassDef)): + bound.add(node.name) + elif isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = (node.targets if isinstance(node, ast.Assign) + else [node.target]) + bound.update(name.id for target in targets + for name in ast.walk(target) + if isinstance(name, ast.Name)) + assert not bound & set(MOVED), bound & set(MOVED) + imported = [(alias.name, alias.asname) for node in tree.body + if isinstance(node, ast.ImportFrom) and node.level == 0 + and node.module == "repo_shape" for alias in node.names] + for name in ("escape_unencodable_stdout", "spelled"): + assert (name, None) in imported, imported + others = [alias for node in ast.walk(tree) + if isinstance(node, (ast.Import, ast.ImportFrom)) + and getattr(node, "module", None) != "repo_shape" + for alias in node.names + if (alias.asname or alias.name) in MOVED] + assert not others, [alias.name for alias in others] + + +@pytest.mark.parametrize("name", ["escape_unencodable_stdout", "spelled"]) +def test_each_root_runs_repo_shapes_own_function(template, name): + """What a root calls by that name IS `repo_shape`'s object, the one the + unit rows above hold to the specification, so those rows are each + root's rows too.""" + assert getattr(template, name) is getattr(repo_shape, name), name @pytest.mark.parametrize("path", [ASSEMBLY_TEMPLATE, FAMILY_TEMPLATE],