From df1bb1867b104e0af52ec03a6118256f74a61794 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 15:46:23 +0000 Subject: [PATCH 1/7] the assembly root's validate-manifest.py and bootstrap.py: a path this user cannot read is named, never read as absent, so a `scripts/` nobody may search no longer SKIPs every validator into `bootstrap ok` (#430, slice D) Slice D of lane 1's plan on #430 (issuecomment-6251069236). Each guard in the two pinned copies asked `Path.is_file()`, `exists()` or `os.path.lexists`. Up to Python 3.13 the first two raised a refused `stat` out of the run, a traceback with exit 1; from 3.14 they answer False to any `OSError`, and `lexists` always has. Measured at 50178305: - validate-manifest.py: a `project.yaml` this user cannot reach was `manifest-missing`, and a declared referent's pin file in a `contracts/` it cannot search was the finding `naming-referent-missing`. - bootstrap.py: such a `project.yaml` was a project that elected nothing, and a `scripts/` it cannot search SKIPPED all three validators, each run ending `bootstrap ok`, exit 0. A leg mount it cannot search was NOT INITIALIZED, a `governance/` it cannot search had no register, and, on every Python, a leg whose `sequencer/` it cannot search was put on its tracking branch over a cherry-pick not done. Each file keeps a local `_is_a`, a copy of validate-pins.py's, with the one errno set: ENOENT, ENOTDIR, EBADF and ELOOP, and Windows' 21, 123 and 1921, mean absent; any other `OSError` is named `: cannot be read ()`, with validate-pins.py's cure, or its OTHER_REASON for a cause that is not a permission. Nothing new is imported from repo_shape.py (#370, #610). - A `project.yaml`, a validator, or a declared referent's pin file is a refusal, exit 2: `yaml-unreadable` and, for a validator, the new `validator-unreadable`, raised before step (b) runs any. A `project.yaml` this user cannot open says the same line as one it cannot reach. - A leg's `.git`, or an operation's marker in its git dir, is a line in step (a), and the leg is left as it is; a register is a line in step (c), and nothing is claimed from it, one that cannot be opened included. Neither step fails the run, as neither did; validate-pins.py refuses such a leg. A missing validator is still SKIPPED, a missing leg NOT INITIALIZED, and readable paths print what they printed. Both files are pinned copies: each project takes one `upstream-changed` row for each. tests/test_assembly_root_unreadable_paths.py holds every site, as a project runs it with the mode taken away and in-process under `refused_stat`, and the two must say the same. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 306 +++++++++- .../scripts/validate-manifest.py | 126 +++- tests/test_assembly_root_unreadable_paths.py | 548 ++++++++++++++++++ 3 files changed, 952 insertions(+), 28 deletions(-) create mode 100644 tests/test_assembly_root_unreadable_paths.py diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index f7531de7..088235ae 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -77,6 +77,13 @@ upstream notice refuse `git-not-runnable`, exit 2, where git cannot be started, as the validators do (#694): the run stops there and says neither line, though a validator may have found or refused by then. + A `project.yaml` this user cannot read is refused by name, exit 2, + before step (a), and a validator it cannot read before step (b) runs + any (#430): read as absent, the first was a project that elected + nothing and the second a validator SKIPPED, each `bootstrap ok`. A + leg's `.git`, an operation's marker in its git dir or a register that + it cannot read is named in its own step, which goes on: the leg is + left as it is, and nothing is claimed from the register. SCHEMA-NEUTRAL BY CONTRACT. A one-repository project runs the same command: step (a) is a no-op where there are no submodules, and steps (c) and (d) are @@ -89,12 +96,14 @@ import argparse import codecs +import errno import os import re import shlex +import stat import subprocess import sys -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from pathlib import Path @@ -299,6 +308,142 @@ def _git_runs(unread: str) -> Iterator[None]: f"({exc.strerror or exc})", GIT_NOT_RUNNABLE) from exc +#: The `OSError`s that `Path.is_file()` and `exists()` answered False to on +#: Python 3.9 to 3.13 (pathlib's `_ignore_error`), because each means there +#: is nothing at the name: no such file, a part of the path that is not a +#: directory, a bad descriptor, a symlink loop, and Windows' "not ready", +#: "invalid name" and "cannot resolve". Every other one was raised, and is +#: named here as a path this user cannot read (#430), as `validate-pins.py` +#: names it. +_ABSENT_ERRNOS = (errno.ENOENT, errno.ENOTDIR, errno.EBADF, errno.ELOOP) +_ABSENT_WINERRORS = (21, 123, 1921) +#: What a path the operating system refused for ANOTHER reason than +#: permission is told (#430): an I/O error, a name longer than the +#: filesystem takes, a file replaced while this ran. No `chmod` puts one +#: right. `validate-pins.py`'s `OTHER_REASON`, word for word. +OTHER_REASON = ( + "Where the operating system gave another reason, put a regular file this " + "user can open back in its place, or take away one nobody wants, and make " + "sure nothing is changing the directory while this runs. ") +#: The sentence that ends what puts such a path right, `validate-pins.py`'s. +RUN_AGAIN = "Then run the same command again." +#: Why a `project.yaml` this user cannot read is refused (#430). +MANIFEST_READ = ( + "this command reads project.yaml for the legs it places and the " + "validators it runs, and one it cannot read is not one that is absent, " + "which would place no leg and check nothing.") +#: Why a validator this user cannot read is refused (#430). +VALIDATOR_READ = ( + "step (b) runs each neutral validator it finds and skips only one that " + "is not there, and one it cannot read is a check that cannot run, never " + "a pass.") + + +def _is_a(path: Path, kind: Callable[[int], bool], *, + link: bool = False) -> bool: + """Whether `path` is there and of `kind` (`stat.S_ISREG`, or anything), + raising the `OSError` of a `stat` the operating system REFUSED (#430). + With `link`, of the name itself and not of what a symlink there names, + as `os.path.lexists` asks. + + `validate-pins.py`'s `_is_a`, as a copy: this file imports nothing new + from `repo_shape.py`, which may be older than it (#370, #610). NOT + `Path.is_file()` or `exists()`, whose answer depends on the Python, nor + `os.path.lexists`, whose does not. Up to 3.13 the first two raised a + refused `stat` out of the run, a traceback and exit 1, which is + `bootstrap FAILED`'s code; from 3.14 they answer False to ANY `OSError`, + and `lexists` always has. So a `project.yaml` this user cannot reach was + a project that elected nothing, and a `scripts/` it cannot search held + three validators SKIPPED as absent: `bootstrap ok` over a gate nobody + ran. A leg in a mount it cannot search was NOT INITIALIZED, a register + it cannot reach was none, and the marker of a cherry-pick in a git dir + it cannot search was no operation in progress, so the leg was moved and + the pick's `--continue` went on over the tracking branch. A name the + operating system cannot take at all (a NUL in it) is nothing there, as + `is_file()` answered it. + """ + try: + mode = (path.lstat() if link else path.stat()).st_mode + except OSError as exc: + if (exc.errno in _ABSENT_ERRNOS + or getattr(exc, "winerror", None) in _ABSENT_WINERRORS): + return False + raise + except ValueError: + return False + return kind(mode) + + +def _named(root: Path, path: Path) -> str: + """`path` as this file names one it cannot read (#430): relative to + `root`, POSIX on every platform, or whole where it is not under it (a + leg's git dir in another worktree), and spelled (#391).""" + try: + return spelled(path.relative_to(root).as_posix()) + except ValueError: + return spelled(str(path)) + + +def _cannot_read(root: Path, path: Path, exc: OSError) -> str: + """`: cannot be read ()`, `validate-pins.py`'s line for a path whose `stat` or + open the operating system refused (#430).""" + return (f"{_named(root, path)}: cannot be read " + f"({exc.strerror or type(exc).__name__})") + + +def _cure(exc: OSError) -> str: + """What puts right a path this command cannot read (#430), in + `validate-pins.py`'s words: the mode or the owner for a permission + (`PermissionError`, which is `EACCES` and `EPERM`), and `OTHER_REASON` + for any other cause, which is not told to `chmod`. It ends on + `RUN_AGAIN`.""" + if isinstance(exc, PermissionError): + return ("Make what this user may not open readable by this user: " + "`chmod u+r` on a file, `chmod u+rx` on a directory (and on " + "every directory above it inside the root), or fix who owns " + "it. " + RUN_AGAIN) + return OTHER_REASON + RUN_AGAIN + + +def _unreadable(code: str, root: Path, path: Path, exc: OSError, + why: str) -> Refusal: + """The refusal of a file the run cannot go on without, whose `stat` or + open the operating system refused (#430): `_cannot_read`'s line, then + `why` the file is read and `_cure`'s advice, as `validate-pins.py`'s + `_unreadable_file` says it.""" + return Refusal(code, _cannot_read(root, path, exc), + f"Remediation: {why} {_cure(exc)}") + + +def _is_file(root: Path, path: Path, code: str, why: str) -> bool: + """Whether `path` is a file, asked as `_is_a` asks it, or `code`'s + refusal naming it (`_unreadable`) where its `stat` was refused + (#430).""" + try: + return _is_a(path, stat.S_ISREG) + except OSError as exc: + raise _unreadable(code, root, path, exc, why) from exc + + +def _load_manifest(root: Path, path: Path): + """`load_yaml` of `project.yaml`, with one this user cannot OPEN refused + as one whose `stat` was refused is (`_unreadable`, #430), so a manifest + at mode 000 and one behind a folder nobody may search say one line. + + Read off the refusal's cause, as `validate-pins.py`'s `_load` reads it: + `load_yaml` has raised its refusal `from` the `OSError` since it was + written, so a `repo_shape.py` older than this file is read the same way. + Any other refusal keeps its words.""" + try: + return load_yaml(path) + except Refusal as exc: + if isinstance(exc.__cause__, OSError): + raise _unreadable("yaml-unreadable", root, path, exc.__cause__, + MANIFEST_READ) from exc.__cause__ + raise + + #: What `_argument_fault` calls a character that does not print, or white #: space (#715): git takes the name, and no person can tell it from the name #: without it, so `_say_not_tried` gives it a line of its own. @@ -689,6 +834,77 @@ def _on_a_branch(current: str | None) -> bool: return current is not None and current.startswith("refs/heads/") +class _Unreadable(Exception): + """A name in a leg that step (a) must look at and whose `stat` the + operating system refused (#430): `path`, and its `OSError`, `error`. + + NOT AN `OSError`, so `_git_runs` never takes it for git's, and NOT A + `Refusal`, so it ends no run: the leg is said to be left as it is + (`_say_unreadable`), and the next leg is placed, as it is after a leg + git refused to move.""" + + def __init__(self, path: Path, error: OSError): + super().__init__(path, error) + self.path = path + self.error = error + + +def _found(path: Path, *, link: bool = False) -> bool: + """Whether anything is at `path` in a leg (its `.git`, or the marker of + an operation in its git dir), asked as `_is_a` asks it, or `_Unreadable` + where the operating system refused the `stat` (#430).""" + try: + return _is_a(path, lambda _mode: True, link=link) + except OSError as exc: + raise _Unreadable(path, exc) from exc + + +def _say_unreadable(where: str, root: Path, unread: _Unreadable) -> None: + """A leg step (a) cannot look into (#430): the name it could not reach, + in `_cannot_read`'s words, that the leg is left as it is, and what puts + it right. No move is made.""" + print(f" {where}: {_cannot_read(root, unread.path, unread.error)}, so " + "the leg is left as it is") + print(f" {_cure(unread.error)}") + + +def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: + """Whether the leg at `sub` is a checkout step (a) can place, its `.git` + there, saying why not: NOT INITIALIZED, with the command that makes it + one, or `_say_unreadable` where the `stat` of its `.git` is refused + (#430). `Path.exists()` raised that refusal out of the run up to Python + 3.13, a traceback, and from 3.14 answered False, so an initialized leg + in a mount this user cannot search was NOT INITIALIZED.""" + try: + there = _found(sub / ".git") + except _Unreadable as unread: + _say_unreadable(where, root, unread) + return False + if not there: + print(f" {where}: NOT INITIALIZED. Run " + f"`git submodule update --init {spelled(path)}` — the " + f"validators below will refuse until you do.") + return there + + +def _stays_for_an_operation(root: Path, sub: Path, where: str, + path: str) -> bool: + """Whether the leg stays where it is for an operation in progress in it + (`_say_in_progress`, #704), or for the marker of one that step (a) + cannot reach (`_say_unreadable`, #430), said either way. Asked through + `os.path.lexists`, a marker it could not reach was no marker, so a leg + whose `sequencer/` this user cannot search was moved over a cherry-pick + that was not done.""" + try: + doing = _in_progress(sub) + except _Unreadable as unread: + _say_unreadable(where, root, unread) + return True + if doing is not None: + _say_in_progress(where, path, *doing) + return doing is not None + + def _in_progress(sub: Path) -> tuple[tuple[str, str, str], str] | None: """The `IN_PROGRESS` row of the operation git has in progress in the leg, with where its marker is (`git dir` or `ref store`), or `None` @@ -707,6 +923,9 @@ def _in_progress(sub: Path) -> tuple[tuple[str, str, str], str] | None: file, so a leg stopped in either was moved, and the move ended it as #704 measured on the files backend: `CHERRY_PICK_HEAD` or `REVERT_HEAD` gone, and what it had staged carried onto the tracking branch. + + A marker whose `stat` the operating system refuses is `_Unreadable` + (#430), never no marker: `os.path.lexists` answered False to it. """ try: said = git_out(["rev-parse", "--git-dir"], cwd=sub, binary=True) @@ -714,7 +933,7 @@ def _in_progress(sub: Path) -> tuple[tuple[str, str, str], str] | None: return None gitdir = sub / os.fsdecode(said.rstrip(b"\r\n")) for row in IN_PROGRESS: - if os.path.lexists(gitdir / row[0]): + if _found(gitdir / row[0], link=True): return row, "git dir" if row[0] in IN_REF_STORE and _in_ref_store(sub, row[0]): return row, "ref store" @@ -745,9 +964,11 @@ def _sequence_in_progress(gitdir: Path) -> tuple[str, str, str] | None: --continue`, which picks B onto wherever HEAD is by then. A move onto the tracking branch put B there. `sequencer/todo` is what is left, and its first word is `revert` for a revert and `pick` for a cherry-pick; - a file that cannot be read is taken as the cherry-pick, the commoner.""" + a file that cannot be read is taken as the cherry-pick, the commoner. + A `sequencer/` this user cannot search is `_Unreadable` (#430): the + leg is left as it is, and no `todo` is said to be missing.""" todo = gitdir / SEQUENCER_TODO - if not os.path.lexists(todo): + if not _found(todo, link=True): return None try: first = todo.read_bytes().split(None, 1)[:1] @@ -1046,17 +1267,16 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: `_checkout_all_tracking_branches` turns into `git-not-runnable`. A leg with an operation in progress is refused before any path is - chosen (`_say_in_progress`, #704), so no move below meets one.""" + chosen (`_say_in_progress`, #704), so no move below meets one. So is a + leg whose `.git`, or an operation's marker in whose git dir, this user + cannot reach (`_say_unreadable`, #430).""" path = str(leg.get("path") or "") sub = root / path # What is PRINTED of the leg is spelled (#391): `role`, `path` and `branch` # are the manifest's text, read before any validator has had its say. where = f"[{spelled(leg.get('role'))}] {spelled(path)}" named = spelled(branch) - if not (sub / ".git").exists(): - print(f" {where}: NOT INITIALIZED. Run " - f"`git submodule update --init {spelled(path)}` — the " - f"validators below will refuse until you do.") + if not _initialized(root, sub, where, path): return fault = _argument_fault(branch) if fault is not None: @@ -1069,9 +1289,7 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: f"refs/heads/{branch}"], sub) remote = _try_git(["rev-parse", "--verify", "--quiet", f"refs/remotes/origin/{branch}"], sub) - doing = _in_progress(sub) - if doing is not None: - _say_in_progress(where, path, *doing) + if _stays_for_an_operation(root, sub, where, path): return if local is None: @@ -1188,7 +1406,20 @@ def _register_paths(root: Path, legs: list[dict]) -> list[Path]: path = str(leg.get("path") or "") if path and path != ".": candidates.append(root / path / REGISTER_RELPATH) - return [c for c in candidates if c.is_file()] + return [c for c in candidates if _a_register(c)] + + +def _a_register(path: Path) -> bool: + """Whether step (c) takes `path` for a register: a file, or a name whose + `stat` the operating system refused (#430), which may be one and which + `_load_register_rows` then names as unreadable. `Path.is_file()` raised + that refusal out of step (c), which never fails, up to Python 3.13, a + traceback, and from 3.14 answered False, so the run said `authority is + not wallet-carried in this org` of a register it could not read.""" + try: + return _is_a(path, stat.S_ISREG) + except OSError: + return True def _row_objects(row: dict) -> list[str]: @@ -1220,19 +1451,30 @@ def _matches(obj: str, repositories: set[str], prefixes: list[str]) -> bool: return False -def _load_register_rows(register_path: Path) -> list[dict] | None: +def _load_register_rows(register_path: Path, + root: Path) -> list[dict] | None: """One register file's rows, tolerant of which key holds them. Split from `read_authority` for #132. Returns `None` when the file itself could not be read — already printed in that case — so the caller's loop moves on to the next register file exactly as the original loop's `continue` did. + + One this user cannot reach or open is named relative to the root, in + `_cannot_read`'s words, with what puts it right (#430). Read off the + refusal's cause, as `_load_manifest` reads it; any other refusal is + said in its own words, as it was. """ try: data = load_yaml(register_path) except Refusal as exc: - print(" unreadable, so nothing is claimed from it: " - f"{spelled(exc.detail)}") + cause = exc.__cause__ + unread = isinstance(cause, OSError) + detail = (_cannot_read(root, register_path, cause) if unread + else spelled(exc.detail)) + print(f" unreadable, so nothing is claimed from it: {detail}") + if unread: + print(f" {_cure(cause)}") return None rows = [] if isinstance(data, dict): @@ -1276,7 +1518,7 @@ def _report_one_register(register_path: Path, root: Path, # and git and the reader both spell it that way. print(" register: " f"{spelled(register_path.relative_to(root).as_posix())}") - rows = _load_register_rows(register_path) + rows = _load_register_rows(register_path, root) if rows is None: return if not rows: @@ -1321,12 +1563,16 @@ def _load_root_and_manifest(args) -> tuple[Path, dict | None]: from either step is `main`'s early exit 2, exactly as the original function's two separate `try`/`except Refusal` blocks around these same two steps both were. + + A `project.yaml` this user cannot reach or open is `yaml-unreadable`, + naming it (#430), never a project with no manifest: read as absent, no + leg was placed, no validator ran, and the run said `bootstrap ok`. """ root = find_repo_root(args.root or Path(__file__).resolve().parents[1]) manifest_path = root / "project.yaml" manifest: dict | None = None - if manifest_path.is_file(): - loaded = load_yaml(manifest_path) + if _is_file(root, manifest_path, "yaml-unreadable", MANIFEST_READ): + loaded = _load_manifest(root, manifest_path) manifest = loaded if isinstance(loaded, dict) else None return root, manifest @@ -1395,6 +1641,13 @@ def _run_neutral_validators(root: Path, manifest: dict | None, Split from `main` for #132. Returns the label and the exit code of each validator that failed, in the order they ran: `_end_of_run` says which were findings and which refusals (#764). + + Each validator is looked for BEFORE ANY RUNS, and one whose `stat` the + operating system refuses is `validator-unreadable`, naming it (#430): + only one that is not there is SKIPPED. `Path.is_file()` raised that + refusal out of the run up to Python 3.13, a traceback, and from 3.14 + answered False, so a `scripts/` this user cannot search SKIPPED all + three and the run said `bootstrap ok`. """ if skip: print(" skipped (--skip-validators)") @@ -1405,8 +1658,12 @@ def _run_neutral_validators(root: Path, manifest: dict | None, "governed for that.") return [] failed = [] - for label, argv_ in VALIDATORS: - if not (root / argv_[0]).is_file(): + present = [(label, argv_, _is_file(root, root / argv_[0], + "validator-unreadable", + VALIDATOR_READ)) + for label, argv_ in VALIDATORS] + for label, argv_, there in present: + if not there: print(f" {label}: {argv_[0]} is absent; SKIPPED") continue print(f" --- {label} ---") @@ -1477,9 +1734,10 @@ def main(argv: list[str] | None = None) -> int: print("\n(b) neutral validators") failed = _run_neutral_validators(root, manifest, args.skip_validators) except Refusal as exc: - # Only `git-not-runnable` reaches here (#694): nothing else in the - # two steps raises one. Stdout first, so a log of both streams - # reads in the order the run went. + # Only `git-not-runnable` (#694) and step (b)'s + # `validator-unreadable` (#430) reach here: nothing else in the two + # steps raises one. Stdout first, so a log of both streams reads in + # the order the run went. sys.stdout.flush() print(str(exc), file=sys.stderr) return 2 diff --git a/templates/assembly-root/scripts/validate-manifest.py b/templates/assembly-root/scripts/validate-manifest.py index fe9cb2f7..663c9a64 100755 --- a/templates/assembly-root/scripts/validate-manifest.py +++ b/templates/assembly-root/scripts/validate-manifest.py @@ -84,14 +84,19 @@ names a file, is the doctor's to say (#655). EXIT CODES: 0 valid · 1 findings · 2 refusal (the file is missing or unreadable) + -- and a declared referent's pin file this user cannot reach is + a refusal too, never a finding that it is missing (#430) """ from __future__ import annotations import argparse +import errno import inspect import re +import stat import sys +from collections.abc import Callable from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -107,6 +112,108 @@ QUALIFIED_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$") REQUIRED_ROLES = {"assembly", "spec", "code"} +#: The `OSError`s that `Path.is_file()` answered False to on Python 3.9 to +#: 3.13 (pathlib's `_ignore_error`), because each means there is nothing at +#: the name to open: no such file, a part of the path that is not a +#: directory, a bad descriptor, a symlink loop, and Windows' "not ready", +#: "invalid name" and "cannot resolve". Every other one was raised, and is +#: refused here by name (#430), as `validate-pins.py` refuses it. +_ABSENT_ERRNOS = (errno.ENOENT, errno.ENOTDIR, errno.EBADF, errno.ELOOP) +_ABSENT_WINERRORS = (21, 123, 1921) + +#: What a file the operating system refused for ANOTHER reason than +#: permission is told (#430): an I/O error, a name longer than the +#: filesystem takes, a file replaced while this ran. No `chmod` puts one +#: right. `validate-pins.py`'s `OTHER_REASON`, word for word. +OTHER_REASON = ( + "Where the operating system gave another reason, put a regular file this " + "user can open back in its place, or take away one nobody wants, and make " + "sure nothing is changing the directory while this runs. ") + + +def _is_a(path: Path, kind: Callable[[int], bool]) -> bool: + """Whether `path` is there and of `kind` (`stat.S_ISREG`), raising the + `OSError` of a `stat` the operating system REFUSED (#430). + + `validate-pins.py`'s `_is_a`, as a copy: this file imports nothing new + from `repo_shape.py`, which may be older than it (#370, #610). NOT + `Path.is_file()`, whose answer depends on the Python. Up to 3.13 it + raised a refused `stat` (`PermissionError`) out of the run, a traceback + and exit 1, which is a finding's code; from 3.14 it answers False to ANY + `OSError`, so a `project.yaml` this user cannot reach was refused + `manifest-missing`, and a declared referent's pin file in a `contracts/` + it cannot search was the finding `naming-referent-missing`: a file that + is there, said not to be. A name the operating system cannot take at all + (a NUL in it) is no file, as `is_file()` answered it. + """ + try: + mode = path.stat().st_mode + except OSError as exc: + if (exc.errno in _ABSENT_ERRNOS + or getattr(exc, "winerror", None) in _ABSENT_WINERRORS): + return False + raise + except ValueError: + return False + return kind(mode) + + +def _unreadable_file(root: Path, path: Path, exc: OSError) -> Refusal: + """`yaml-unreadable` for a file this validator must read or find and the + operating system will not let it (#430): its `stat` refused, or its + open. `validate-pins.py`'s `_unreadable_file`, in its words. + + NAMED ONCE, RELATIVE TO THE ROOT, POSIX on every platform, or whole + where it is not under the root. THE ADVICE FOLLOWS THE CAUSE: a + permission (`PermissionError`, which is `EACCES` and `EPERM`) is put + right by the mode or the owner, and any other cause is `OTHER_REASON`, + which is not told to `chmod`. + """ + try: + named = path.relative_to(root).as_posix() + except ValueError: + named = str(path) + advice = ( + "Make what this user may not open readable by this user: `chmod u+r` " + "on a file, `chmod u+rx` on a directory (and on every directory above " + "it inside the root), or fix who owns it. " + if isinstance(exc, PermissionError) else OTHER_REASON) + return Refusal( + "yaml-unreadable", + f"{named}: cannot be read ({exc.strerror or type(exc).__name__})", + "Remediation: this validator reads project.yaml, the naming policy " + "and the pin file each declared referent needs, and one it cannot " + "read is a question it cannot answer, never a pass. " + advice + + "Then run the same command again.") + + +def _is_file(root: Path, path: Path) -> bool: + """Whether `path` is a file, asked as `_is_a` asks it, or the refusal + naming it (`_unreadable_file`) where its `stat` was refused (#430).""" + try: + return _is_a(path, stat.S_ISREG) + except OSError as exc: + raise _unreadable_file(root, path, exc) from exc + + +def _load(root: Path, path: Path): + """`load_yaml(path)`, with a file this user cannot OPEN named as one + whose `stat` was refused is (`_unreadable_file`, #430), so `project.yaml` + at mode 000 and one in a folder nobody may search say one line. + + Read off the refusal's cause, as `validate-pins.py`'s `_load` reads it: + `load_yaml` has raised its refusal `from` the `OSError` since it was + written, so a `repo_shape.py` older than this file is read the same way. + Any other refusal (a file that is not UTF-8, or does not parse) keeps + its words.""" + try: + return load_yaml(path) + except Refusal as exc: + if isinstance(exc.__cause__, OSError): + raise _unreadable_file(root, path, + exc.__cause__) from exc.__cause__ + raise + def _declared_pins(manifest: dict) -> set[str]: """The neutral products this manifest DECLARES a pin on. @@ -394,6 +501,12 @@ def _naming_referent_missing_pin_finding(leg_role, naming: dict, found, Split from `_naming_referent_declared_findings` for #132. A direct pin is the referent's own pin file; a chain's is the FIRST LINK's, because that is the pin this project actually holds. + + A PIN FILE THIS USER CANNOT REACH IS NOT MISSING (#430): `_is_file` + raises `yaml-unreadable` naming it, which `main` refuses, exit 2, as + `validate-pins.py` refuses the same file. `Path.is_file()` raised that + `stat` out of the run up to Python 3.13, and from 3.14 answered False, + and this finding said a file that is there does not exist. """ resolution = found.referent satisfied = resolution.referent if resolution.reached else None @@ -403,7 +516,7 @@ def _naming_referent_missing_pin_finding(leg_role, naming: dict, found, if not (declared is True and held and root is not None): return [] pin_path = root / "contracts" / f"{held.lower()}-pin.yaml" - if pin_path.is_file(): + if _is_file(root, pin_path): return [] return [ f"FINDING naming-referent-missing: leg {leg_role!r}: " @@ -1082,23 +1195,28 @@ def main(argv: list[str] | None = None) -> int: try: root = find_repo_root(args.root or Path(__file__).resolve().parents[1]) manifest_path = root / "project.yaml" - if not manifest_path.is_file(): + # A `project.yaml` this user cannot reach or open is refused by name, + # `yaml-unreadable`, never `manifest-missing` (#430). + if not _is_file(root, manifest_path): raise Refusal( "manifest-missing", f"{manifest_path} does not exist. A project that has elected " "this schema declares it in `project.yaml`; a project that has " "not, does not run this validator.", ) - manifest = load_yaml(manifest_path) + manifest = _load(root, manifest_path) if not isinstance(manifest, dict): raise Refusal("manifest-unreadable", f"{manifest_path}: not a mapping") policy_path = args.policy or (root / "contracts" / "repository-naming.yaml") policy = NamingPolicy.load(policy_path) + # Inside the `try`: a declared referent's pin file this user cannot + # reach is a check that cannot run, so it is refused, exit 2, and no + # finding is said over it (#430). + notes = _findings(manifest, policy, root) except Refusal as exc: print(str(exc), file=sys.stderr) return 2 - notes = _findings(manifest, policy, root) # A WARNING is a note about what could not be CHECKED here (a chain link # whose tree is not on this disk), never about what is wrong. It is # printed and then forgotten by the exit code, deliberately. diff --git a/tests/test_assembly_root_unreadable_paths.py b/tests/test_assembly_root_unreadable_paths.py new file mode 100644 index 00000000..82c21179 --- /dev/null +++ b/tests/test_assembly_root_unreadable_paths.py @@ -0,0 +1,548 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The assembly root's pinned `validate-manifest.py` and `bootstrap.py` name a +path this user cannot read, and never read it as absent (#430, slice D of +lane 1's plan, #430 issuecomment-6251069236). + +THE DEFECT, measured at the v1.1.1 release commit 50178305 in a scaffold's +recursive clone, on Python 3.12.3 with the mode taken away, and under a +`sitecustomize` that answers as Python 3.14's `Path.is_file()`, `is_dir()` +and `exists()` do, False to ANY `OSError`: + + * `validate-manifest.py`, a `project.yaml` it cannot reach: a + `PermissionError` traceback, exit 1, on 3.12; `manifest-missing` on 3.14. + * `validate-manifest.py`, a declared referent's pin file it cannot reach: + a traceback, exit 1; the finding `naming-referent-missing` on 3.14. + * `bootstrap.py`, a `project.yaml` it cannot reach: a traceback; on 3.14 a + project that elected nothing, and `bootstrap ok`, exit 0. + * `bootstrap.py`, a `scripts/` it cannot search: a traceback in step (b); + on 3.14 the three validators `SKIPPED` as absent, and `bootstrap ok`. + * `bootstrap.py`, a leg mount it cannot search: a traceback in step (a); + on 3.14 the leg NOT INITIALIZED. + * `bootstrap.py`, a `governance/` it cannot search: a traceback in step + (c); on 3.14 `authority is not wallet-carried in this org`. + * `bootstrap.py`, a leg's `sequencer/` it cannot search over a cherry-pick + that is not done: on EVERY Python `os.path.lexists` read the marker as + absent, and the leg was put on its tracking branch, where the pick's + `--continue` goes on. + +THE RULE NOW, lane 1's for every slice. The path is `stat`ed: ENOENT, ENOTDIR, +EBADF and ELOOP (and Windows' 21, 123 and 1921) mean absent, and any other +`OSError` is named `: cannot be read ()`, +in `validate-pins.py`'s words, with its cure: `chmod u+r` a file and `chmod +u+rx` a directory and every one above it, or, for another cause, its +`OTHER_REASON`. + * A `project.yaml`, a validator or a declared referent's pin file is a + refusal, exit 2, and a `project.yaml` this user cannot OPEN says the same + line as one it cannot reach. + * A leg's `.git`, or an operation's marker in its git dir, is a line in + step (a), and the leg is left as it is; a register is a line in step + (c), and nothing is claimed from it. Neither step fails the run, as + neither did before: the validators refuse such a leg. + +THE HARNESS. Each refusal is asked twice and must say the same both times: +as a project runs the tool, with the mode taken away (POSIX, and not as +root, who reads through any mode), and in-process under `refused_stat` +(`tests/test_bump_leg_unopenable.py`), which answers as Python 3.14 does on +any Python, as root and on every platform. A `project.yaml` or a pin file is +reached through a symlink into a folder nobody may enter, since the root +itself cannot be one: no tool finds a root it cannot search. +""" + +from __future__ import annotations + +import errno +import os +import shutil +import stat +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from conftest import (NO_BYTECODE, REPO, clear_ambient_pin_sources, git, + load_without_bytecode, run_script) +from test_bump_leg_unopenable import refused_stat +from test_manifest_validator import as_a_descendant_by_chain, link_tree +from test_validate_pins_named_and_spelled import ( + AGAIN, OTHER, PERMISSION, failing, os_error, +) +from test_validate_pins_unreadable_workflow import CANNOT_RUN_AS_ROOT +from test_validate_pins_unreadable_workflows_dir import mode_of + +TEMPLATES = REPO / "templates" / "assembly-root" / "scripts" +#: A symlink a test makes, which a Windows account may not. +POSIX_LINKS = pytest.mark.skipif( + os.name == "nt", reason="makes a symlink, which Windows may refuse") + +#: What each refusal's remedy says the file is read for. +MANIFEST_READS = ( + "Remediation: this validator reads project.yaml, the naming policy and " + "the pin file each declared referent needs, and one it cannot read is a " + "question it cannot answer, never a pass. ") +BOOTSTRAP_READS = ( + "Remediation: this command reads project.yaml for the legs it places and " + "the validators it runs, and one it cannot read is not one that is " + "absent, which would place no leg and check nothing. ") +VALIDATOR_READS = ( + "Remediation: step (b) runs each neutral validator it finds and skips " + "only one that is not there, and one it cannot read is a check that " + "cannot run, never a pass. ") + +#: The pin file `as_a_descendant_by_chain` makes the root hold: its chain's +#: first link's. +PIN = "contracts/openxdox-pin.yaml" +REGISTER = "governance/review-authority/register.yaml" +GRANTS = (" Grants are read here for REPORTING only. This command confers " + "nothing and enforces nothing; a required check in the repository " + "that owns the object is what confers.") +DEGRADE = [ + "authority is not wallet-carried in this org", + " (no `governance/review-authority/register.yaml` in the assembly root " + "or in any leg. Wallet-carried authority is an OVERLAY; the shape confers " + "nothing without it, and nothing with it.)"] + + +def refused(code: str, named: str, reads: str, + reason: str = "Permission denied", + advice: str = PERMISSION) -> str: + """The whole of stderr for a file the tool cannot reach or open: the + file named once, relative to the root, with the system's reason, then + what it is read for and the cure for that reason.""" + return (f"REFUSED {code}: {named}: cannot be read ({reason})\n" + f"{reads}{advice}{AGAIN}\n") + + +def left_as_it_is(named: str, reason: str = "Permission denied", + advice: str = PERMISSION) -> list[str]: + """Step (a)'s two lines for the spec leg, where `named` cannot be read.""" + return [f" [spec] spec: {named}: cannot be read ({reason}), so the leg " + "is left as it is", f" {advice}{AGAIN}"] + + +def nothing_claimed(named: str) -> list[str]: + """Step (c)'s lines for a register at `named` this user cannot read.""" + return [f" register: {named}", + f" unreadable, so nothing is claimed from it: {named}: cannot " + "be read (Permission denied)", f" {PERMISSION}{AGAIN}"] + + +def placed(root: Path, role: str) -> str: + """Step (a)'s line for a leg of a fresh clone, put on `main` at its pin.""" + pin = git("rev-parse", f"HEAD:{role}", cwd=root).stdout.strip() + return f" [{role}] {role}: on main at {pin[:12]} (branch tip == pin)" + + +def skipping_validators(root: Path, step_a: list[str], + step_c: list[str]) -> str: + """The whole of stdout of `bootstrap.py --skip-validators` in `root`.""" + return "\n".join([ + f"bootstrap: Atlas ({root.resolve()})", "", + "(a) legs on tracking branches, pins untouched", *step_a, "", + "(b) neutral validators", " skipped (--skip-validators)", "", + "(c) review authority", *step_c, "", "bootstrap ok", ""]) + + +def validate(root: Path, *args: str): + return run_script(root / "scripts" / "validate-manifest.py", *args, + cwd=root) + + +def bootstrap(root: Path, *args: str): + return run_script(root / "scripts" / "bootstrap.py", *args, cwd=root, + env={"SHAPE_UPSTREAM_PATH": ""}) + + +def behind_a_locked_folder(root: Path, relative: str): + """Move `relative` into a folder beside the root, leave a symlink to it + in its place, and return the context in which nobody may enter it.""" + locked = root.parent / "locked" + locked.mkdir() + real = root / relative + shutil.move(str(real), str(locked / real.name)) + real.symlink_to(os.path.relpath(locked / real.name, real.parent)) + return mode_of(root.parent, "locked", 0) + + +def at_mode_000(root: Path, relative: str): + """The context in which nobody may read `relative`.""" + return mode_of(root, relative, 0) + + +UNREACHABLE = pytest.mark.parametrize( + "how", [behind_a_locked_folder, at_mode_000], + ids=["behind-a-folder-nobody-may-enter", "at-mode-000"]) + + +def still_detached(root: Path, role: str) -> bool: + """Whether the leg's HEAD is still detached, as a fresh clone left it.""" + return git("symbolic-ref", "-q", "HEAD", cwd=root / role, + check=False).returncode != 0 + + +@pytest.fixture(scope="module") +def manifest_gate(): + """The standard's `validate-manifest.py`, loaded by path.""" + return load_without_bytecode(TEMPLATES / "validate-manifest.py", + "unreadable_paths_validate_manifest") + + +@pytest.fixture(scope="module") +def boot(): + """The standard's assembly-root `bootstrap.py`, loaded by path.""" + return load_without_bytecode(TEMPLATES / "bootstrap.py", + "unreadable_paths_bootstrap") + + +def held_by_chain(root: Path) -> None: + """The manifest declares descent through a chain whose first link's pin + file the root holds, with the link's tree beside it: a valid manifest + that asks whether `PIN` is there.""" + as_a_descendant_by_chain(root) + (root / PIN).write_text("schema_version: 1\n" + "kind: pinned_contract_manifest\n") + link_tree(root.parent / "openXdox", "openDox") + + +# --- validate-manifest.py: project.yaml -------------------------------------- + +@CANNOT_RUN_AS_ROOT +@UNREACHABLE +def test_a_manifest_the_gate_cannot_reach_or_open_is_refused_by_name( + project, how): + """Behind a folder nobody may enter, the base's 3.12 ended in a + traceback; at mode 000 it named the absolute path, with no remedy. Both + are one refusal now, and with the mode put back the gate passes.""" + with how(project, "project.yaml"): + result = validate(project) + assert result.stderr == refused("yaml-unreadable", "project.yaml", + MANIFEST_READS), result.stderr + assert result.returncode == 2, result.stdout + assert result.stdout == "" + assert validate(project).returncode == 0 + + +def test_a_manifest_whose_stat_is_refused_is_the_same_refusal( + manifest_gate, project, monkeypatch, capsys): + """On every Python: 3.14's `is_file()` said False, and the base refused + `manifest-missing` over a file that is there.""" + refused_stat(monkeypatch, "project.yaml") + assert manifest_gate.main(["--root", str(project)]) == 2 + seen = capsys.readouterr() + assert seen.err == refused("yaml-unreadable", "project.yaml", + MANIFEST_READS) + assert seen.out == "" + + +# --- validate-manifest.py: a declared referent's pin file -------------------- + +@CANNOT_RUN_AS_ROOT +@POSIX_LINKS +def test_a_referent_pin_the_gate_cannot_reach_is_refused_not_missing( + project): + """The base's 3.12 ended in a traceback, and 3.14 found + `naming-referent-missing`, a file that is there said not to exist.""" + held_by_chain(project) + with behind_a_locked_folder(project, PIN): + result = validate(project) + assert result.stderr == refused("yaml-unreadable", PIN, MANIFEST_READS) + assert result.returncode == 2, result.stdout + assert validate(project).returncode == 0 + + +@CANNOT_RUN_AS_ROOT +def test_a_contracts_folder_the_gate_cannot_search_names_the_pin(project, + tmp_path): + """`contracts/` at mode 000, the naming policy given elsewhere with + `--policy`: the pin file is the first thing in it the gate looks for.""" + held_by_chain(project) + policy = tmp_path / "repository-naming.yaml" + shutil.copy2(project / "contracts" / "repository-naming.yaml", policy) + with at_mode_000(project, "contracts"): + result = validate(project, "--policy", str(policy)) + assert result.stderr == refused("yaml-unreadable", PIN, MANIFEST_READS) + assert result.returncode == 2, result.stdout + + +@pytest.mark.parametrize("failure, advice", [ + pytest.param(None, PERMISSION, id="permission"), + pytest.param(os_error(errno.EIO), OTHER, id="io-error"), +]) +def test_a_referent_pin_whose_stat_is_refused_is_refused_on_every_python( + manifest_gate, project, monkeypatch, capsys, failure, advice): + """`refused_stat` answers as 3.14 does, where the base said the pin was + missing; an I/O error is no permission, and is not told to `chmod`.""" + held_by_chain(project) + clear_ambient_pin_sources(monkeypatch) + if failure is None: + refused_stat(monkeypatch, "openxdox-pin.yaml") + else: + failing(monkeypatch, "stat", {"openxdox-pin.yaml": failure}) + assert manifest_gate.main(["--root", str(project)]) == 2 + reason = "Permission denied" if failure is None else failure.strerror + assert capsys.readouterr().err == refused( + "yaml-unreadable", PIN, MANIFEST_READS, reason, advice) + + +@POSIX_LINKS +@pytest.mark.parametrize("target", ["nowhere-pin.yaml", "openxdox-pin.yaml"], + ids=["dangling-link", "link-to-itself"]) +def test_a_referent_pin_that_is_not_there_is_still_missing(project, target): + """THE OTHER HALF, which the base answered the same: a link to nothing + (ENOENT) or to itself (ELOOP) is no pin file, and the finding stands.""" + held_by_chain(project) + pin = project / PIN + pin.unlink() + pin.symlink_to(target) + result = validate(project) + assert result.stderr == ( + "FINDING naming-referent-missing: leg 'assembly': openXdox is the " + "first link of this leg's recorded chain, reaching openDox, but " + f"{PIN} does not exist. A declared pin that is not in the tree is a " + "claim wearing the costume of a referent.\n\n1 finding(s) in " + f"{project.resolve() / 'project.yaml'}\n"), result.stderr + assert result.returncode == 1 + + +# --- bootstrap.py: project.yaml ---------------------------------------------- + +@CANNOT_RUN_AS_ROOT +@UNREACHABLE +def test_a_manifest_bootstrap_cannot_reach_or_open_is_refused_first(project, + how): + """The base's 3.12 ended in a traceback, and its 3.14 placed no leg, ran + no validator and said `bootstrap ok`; at mode 000 it named the absolute + path, with no remedy. Refused before the first line, and no leg moves.""" + with how(project, "project.yaml"): + result = bootstrap(project) + assert result.stderr == refused("yaml-unreadable", "project.yaml", + BOOTSTRAP_READS), result.stderr + assert result.returncode == 2 + assert result.stdout == "" + assert still_detached(project, "spec") + + +def test_a_manifest_bootstrap_cannot_stat_is_the_same_refusal(boot, project, + monkeypatch): + refused_stat(monkeypatch, "project.yaml") + with pytest.raises(boot.Refusal) as caught: + boot._load_root_and_manifest(SimpleNamespace(root=project)) + assert f"{caught.value}\n" == refused("yaml-unreadable", "project.yaml", + BOOTSTRAP_READS) + + +# --- bootstrap.py: the validators -------------------------------------------- + +@CANNOT_RUN_AS_ROOT +def test_a_scripts_folder_bootstrap_cannot_search_is_refused_not_skipped( + project): + """THE FALSE PASS. Run as this standard's `bootstrap` runs a project's, + from outside its `scripts/`: on the base's 3.14 each validator was + `SKIPPED` as absent and the run said `bootstrap ok`. The first one is + refused by name, after step (a) and before any validator runs.""" + with at_mode_000(project, "scripts"): + result = run_script(REPO / "bootstrap", cwd=project, + env={"SHAPE_UPSTREAM_PATH": "", **NO_BYTECODE}) + assert result.stderr == refused( + "validator-unreadable", "scripts/validate-repository-naming.py", + VALIDATOR_READS), result.stderr + assert result.returncode == 2 + assert result.stdout == "\n".join([ + f"bootstrap: Atlas ({project.resolve()})", "", + "(a) legs on tracking branches, pins untouched", + placed(project, "spec"), placed(project, "code"), "", + "(b) neutral validators", ""]) + + +def test_a_validator_whose_stat_is_refused_stops_step_b_before_any_runs( + boot, project, monkeypatch, capsys): + """On every Python, the second validator: the base SKIPPED it and ran + the other two. Nothing is run or said before the refusal.""" + manifest = boot.load_yaml(project / "project.yaml") + refused_stat(monkeypatch, "validate-manifest.py") + with pytest.raises(boot.Refusal) as caught: + boot._run_neutral_validators(project, manifest, False) + assert f"{caught.value}\n" == refused( + "validator-unreadable", "scripts/validate-manifest.py", + VALIDATOR_READS) + assert capsys.readouterr().out == "" + + +def test_a_validator_that_is_not_there_is_still_skipped(boot, project, + monkeypatch, capsys): + """THE OTHER HALF, which the base answered the same. The validators are + stood in for: what is asked is which of them step (b) runs, and a + pinned copy taken away is `validate-pins.py`'s own finding.""" + monkeypatch.delenv("SHAPE_UPSTREAM_PATH", raising=False) + ran = [] + monkeypatch.setattr(boot, "_run", + lambda root, argv: ran.append(argv[0]) or 0) + manifest = boot.load_yaml(project / "project.yaml") + (project / "scripts" / "validate-repository-naming.py").unlink() + assert boot._run_neutral_validators(project, manifest, False) == [] + assert ran == ["scripts/validate-manifest.py", "scripts/validate-pins.py"] + assert capsys.readouterr().out == ( + " naming: scripts/validate-repository-naming.py is absent; " + "SKIPPED\n --- manifest ---\n --- pins ---\n") + + +# --- bootstrap.py: a leg ----------------------------------------------------- + +@CANNOT_RUN_AS_ROOT +def test_a_leg_bootstrap_cannot_search_is_left_as_it_is(project): + """The base's 3.12 ended in a traceback in step (a), and its 3.14 said + NOT INITIALIZED of an initialized leg. Step (a) names the `.git` it + cannot reach and goes on, and so does step (c), where a register could + be; the validators refuse the leg, so the run ends refused.""" + with at_mode_000(project, "spec"): + skipped = bootstrap(project, "--skip-validators") + result = bootstrap(project) + assert skipped.stdout == skipping_validators( + project, [*left_as_it_is("spec/.git"), placed(project, "code")], + [*nothing_claimed(f"spec/{REGISTER}"), GRANTS]), skipped.stdout + assert skipped.stderr == "" + assert skipped.returncode == 0 + assert result.stderr.splitlines()[-1] == ( + "bootstrap REFUSED: pins (exit 2)"), result.stderr + assert result.returncode == 2 + assert still_detached(project, "spec") + + +@pytest.mark.parametrize("failure, advice", [ + pytest.param(None, PERMISSION, id="permission"), + pytest.param(os_error(errno.EIO), OTHER, id="io-error"), +]) +def test_a_leg_whose_stat_is_refused_is_not_uninitialized( + boot, project, monkeypatch, capsys, failure, advice): + if failure is None: + refused_stat(monkeypatch, ".git") + else: + failing(monkeypatch, "stat", {".git": failure}) + boot.checkout_tracking_branch(project.resolve(), + {"role": "spec", "path": "spec"}, "main") + reason = "Permission denied" if failure is None else failure.strerror + assert capsys.readouterr().out.splitlines() == left_as_it_is( + "spec/.git", reason, advice) + + +@CANNOT_RUN_AS_ROOT +def test_an_operation_bootstrap_cannot_see_keeps_the_leg_where_it_is( + project): + """A cherry-pick of several commits stopped and committed by hand + leaves `sequencer/todo`. With `sequencer/` at mode 000 the base, on + every Python, put the leg on `main`; now it is named and the leg stays + detached at its pin.""" + spec = project / "spec" + gitdir = Path(git("rev-parse", "--absolute-git-dir", + cwd=spec).stdout.strip()) + (gitdir / "sequencer").mkdir() + head = git("rev-parse", "HEAD", cwd=spec).stdout.strip() + (gitdir / "sequencer" / "todo").write_text(f"pick {head} x\n") + named = (gitdir / "sequencer" / "todo").relative_to(project.resolve()) + with mode_of(gitdir, "sequencer", 0): + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [*left_as_it_is(named.as_posix()), placed(project, "code")], + DEGRADE), result.stdout + assert result.returncode == 0 + assert still_detached(project, "spec") + + +@pytest.mark.parametrize("marker, named", [ + ("MERGE_HEAD", "MERGE_HEAD"), ("todo", "sequencer/todo")], + ids=["a-merge", "a-sequence"]) +def test_a_marker_whose_lstat_is_refused_keeps_the_leg_on_every_python( + boot, project, monkeypatch, capsys, marker, named): + """`os.path.lexists` answered False to a refused `lstat` on every + Python, so the leg was placed. Each of the two places a marker is + asked, the operations' and the sequence's.""" + failing(monkeypatch, "lstat", { + marker: PermissionError(errno.EACCES, "Permission denied")}) + boot.checkout_tracking_branch(project.resolve(), + {"role": "spec", "path": "spec"}, "main") + assert capsys.readouterr().out.splitlines() == left_as_it_is( + f".git/modules/spec/{named}") + assert still_detached(project, "spec") + + +# --- bootstrap.py: a register ------------------------------------------------ + +@CANNOT_RUN_AS_ROOT +@pytest.mark.parametrize("locked", ["governance", REGISTER], + ids=["a-folder-nobody-may-enter", "at-mode-000"]) +def test_a_register_bootstrap_cannot_read_is_named_not_absent(project, + locked): + """The base's 3.12 ended in a traceback in step (c), and its 3.14 said + `authority is not wallet-carried in this org`; at mode 000 it named the + absolute path. Step (c) names it, claims nothing from it, and the run + goes on as it does for a register that does not parse.""" + register = project / REGISTER + register.parent.mkdir(parents=True) + register.write_text("rows:\n - holder: agent:x\n act: review\n" + " target_repo: testorg/Atlas-code\n") + with at_mode_000(project, locked): + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [placed(project, "spec"), placed(project, "code")], + [*nothing_claimed(REGISTER), GRANTS]), result.stdout + assert result.returncode == 0 + + +def test_a_register_whose_stat_is_refused_is_not_dropped(boot, project, + monkeypatch): + """On every Python: 3.14's `is_file()` dropped it, and step (c) said the + org has no wallet-carried authority.""" + refused_stat(monkeypatch, "register.yaml") + legs = [{"role": "spec", "path": "spec"}] + assert boot._register_paths(project, legs) == [ + project / REGISTER, project / "spec" / REGISTER] + + +# --- the one errno set (slice Z holds every copy to it) ---------------------- + +ABSENT = [ + pytest.param(errno.ENOENT, None, id="ENOENT"), + pytest.param(errno.ENOTDIR, None, id="ENOTDIR"), + pytest.param(errno.EBADF, None, id="EBADF"), + pytest.param(errno.ELOOP, None, id="ELOOP"), + pytest.param(errno.EINVAL, 21, id="winerror-21"), + pytest.param(errno.EINVAL, 123, id="winerror-123"), + pytest.param(errno.EINVAL, 1921, id="winerror-1921"), +] +REFUSED = [ + pytest.param(errno.EACCES, None, id="EACCES"), + pytest.param(errno.EPERM, None, id="EPERM"), + pytest.param(errno.EIO, None, id="EIO"), + pytest.param(errno.ENAMETOOLONG, None, id="ENAMETOOLONG"), +] + + +@pytest.fixture(params=["validate-manifest.py", "bootstrap.py"]) +def copy_of_is_a(request, manifest_gate, boot): + """Each of the two files' own `_is_a`.""" + module = manifest_gate if request.param.startswith("validate") else boot + return module._is_a + + +def stat_failing(monkeypatch, tmp_path: Path, code: int, + winerror: int | None) -> tuple[Path, OSError]: + error = OSError(code, os.strerror(code)) + if winerror is not None: + error.winerror = winerror + failing(monkeypatch, "stat", {"asked": error}) + return tmp_path / "asked", error + + +@pytest.mark.parametrize("code, winerror", ABSENT) +def test_a_name_that_is_not_there_is_absent(copy_of_is_a, monkeypatch, + tmp_path, code, winerror): + path, _ = stat_failing(monkeypatch, tmp_path, code, winerror) + assert copy_of_is_a(path, stat.S_ISREG) is False + + +@pytest.mark.parametrize("code, winerror", REFUSED) +def test_every_other_refusal_is_raised(copy_of_is_a, monkeypatch, tmp_path, + code, winerror): + path, error = stat_failing(monkeypatch, tmp_path, code, winerror) + with pytest.raises(OSError) as caught: + copy_of_is_a(path, stat.S_ISREG) + assert caught.value is error From 0ef875c0cb89a94902f0573c705aa354b1f7e8a2 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 17:03:03 +0000 Subject: [PATCH 2/7] the assembly root's bootstrap.py: the step (a) helpers of the slice-D commit sit beside its other new helpers, so the #896 branch merges with it cleanly (#430, slice D) A pure move, no line changed: `_Unreadable`, `_found`, `_say_unreadable`, `_initialized` and `_stays_for_an_operation` move from between `_on_a_branch` and `_in_progress`, where lane 5's #896 branch (fix/896-casefold-branch-f, 80353dd1) adds `_folded_twin`, `TWIN_RISKS` and `_say_twin`, to after `_load_manifest`. `git merge-tree` of the two heads conflicted at that one spot, two insertions in one place; with the move it is clean. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 142 +++++++++---------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 088235ae..e3f953b3 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -444,6 +444,77 @@ def _load_manifest(root: Path, path: Path): raise +class _Unreadable(Exception): + """A name in a leg that step (a) must look at and whose `stat` the + operating system refused (#430): `path`, and its `OSError`, `error`. + + NOT AN `OSError`, so `_git_runs` never takes it for git's, and NOT A + `Refusal`, so it ends no run: the leg is said to be left as it is + (`_say_unreadable`), and the next leg is placed, as it is after a leg + git refused to move.""" + + def __init__(self, path: Path, error: OSError): + super().__init__(path, error) + self.path = path + self.error = error + + +def _found(path: Path, *, link: bool = False) -> bool: + """Whether anything is at `path` in a leg (its `.git`, or the marker of + an operation in its git dir), asked as `_is_a` asks it, or `_Unreadable` + where the operating system refused the `stat` (#430).""" + try: + return _is_a(path, lambda _mode: True, link=link) + except OSError as exc: + raise _Unreadable(path, exc) from exc + + +def _say_unreadable(where: str, root: Path, unread: _Unreadable) -> None: + """A leg step (a) cannot look into (#430): the name it could not reach, + in `_cannot_read`'s words, that the leg is left as it is, and what puts + it right. No move is made.""" + print(f" {where}: {_cannot_read(root, unread.path, unread.error)}, so " + "the leg is left as it is") + print(f" {_cure(unread.error)}") + + +def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: + """Whether the leg at `sub` is a checkout step (a) can place, its `.git` + there, saying why not: NOT INITIALIZED, with the command that makes it + one, or `_say_unreadable` where the `stat` of its `.git` is refused + (#430). `Path.exists()` raised that refusal out of the run up to Python + 3.13, a traceback, and from 3.14 answered False, so an initialized leg + in a mount this user cannot search was NOT INITIALIZED.""" + try: + there = _found(sub / ".git") + except _Unreadable as unread: + _say_unreadable(where, root, unread) + return False + if not there: + print(f" {where}: NOT INITIALIZED. Run " + f"`git submodule update --init {spelled(path)}` — the " + f"validators below will refuse until you do.") + return there + + +def _stays_for_an_operation(root: Path, sub: Path, where: str, + path: str) -> bool: + """Whether the leg stays where it is for an operation in progress in it + (`_say_in_progress`, #704), or for the marker of one that step (a) + cannot reach (`_say_unreadable`, #430), said either way. Asked through + `os.path.lexists`, a marker it could not reach was no marker, so a leg + whose `sequencer/` this user cannot search was moved over a cherry-pick + that was not done.""" + try: + doing = _in_progress(sub) + except _Unreadable as unread: + _say_unreadable(where, root, unread) + return True + if doing is not None: + _say_in_progress(where, path, *doing) + return doing is not None + + #: What `_argument_fault` calls a character that does not print, or white #: space (#715): git takes the name, and no person can tell it from the name #: without it, so `_say_not_tried` gives it a line of its own. @@ -834,77 +905,6 @@ def _on_a_branch(current: str | None) -> bool: return current is not None and current.startswith("refs/heads/") -class _Unreadable(Exception): - """A name in a leg that step (a) must look at and whose `stat` the - operating system refused (#430): `path`, and its `OSError`, `error`. - - NOT AN `OSError`, so `_git_runs` never takes it for git's, and NOT A - `Refusal`, so it ends no run: the leg is said to be left as it is - (`_say_unreadable`), and the next leg is placed, as it is after a leg - git refused to move.""" - - def __init__(self, path: Path, error: OSError): - super().__init__(path, error) - self.path = path - self.error = error - - -def _found(path: Path, *, link: bool = False) -> bool: - """Whether anything is at `path` in a leg (its `.git`, or the marker of - an operation in its git dir), asked as `_is_a` asks it, or `_Unreadable` - where the operating system refused the `stat` (#430).""" - try: - return _is_a(path, lambda _mode: True, link=link) - except OSError as exc: - raise _Unreadable(path, exc) from exc - - -def _say_unreadable(where: str, root: Path, unread: _Unreadable) -> None: - """A leg step (a) cannot look into (#430): the name it could not reach, - in `_cannot_read`'s words, that the leg is left as it is, and what puts - it right. No move is made.""" - print(f" {where}: {_cannot_read(root, unread.path, unread.error)}, so " - "the leg is left as it is") - print(f" {_cure(unread.error)}") - - -def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: - """Whether the leg at `sub` is a checkout step (a) can place, its `.git` - there, saying why not: NOT INITIALIZED, with the command that makes it - one, or `_say_unreadable` where the `stat` of its `.git` is refused - (#430). `Path.exists()` raised that refusal out of the run up to Python - 3.13, a traceback, and from 3.14 answered False, so an initialized leg - in a mount this user cannot search was NOT INITIALIZED.""" - try: - there = _found(sub / ".git") - except _Unreadable as unread: - _say_unreadable(where, root, unread) - return False - if not there: - print(f" {where}: NOT INITIALIZED. Run " - f"`git submodule update --init {spelled(path)}` — the " - f"validators below will refuse until you do.") - return there - - -def _stays_for_an_operation(root: Path, sub: Path, where: str, - path: str) -> bool: - """Whether the leg stays where it is for an operation in progress in it - (`_say_in_progress`, #704), or for the marker of one that step (a) - cannot reach (`_say_unreadable`, #430), said either way. Asked through - `os.path.lexists`, a marker it could not reach was no marker, so a leg - whose `sequencer/` this user cannot search was moved over a cherry-pick - that was not done.""" - try: - doing = _in_progress(sub) - except _Unreadable as unread: - _say_unreadable(where, root, unread) - return True - if doing is not None: - _say_in_progress(where, path, *doing) - return doing is not None - - def _in_progress(sub: Path) -> tuple[tuple[str, str, str], str] | None: """The `IN_PROGRESS` row of the operation git has in progress in the leg, with where its marker is (`git dir` or `ref store`), or `None` From b2887cf56baad8ccb3d078336589a822ef6717fb Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 18:24:54 +0000 Subject: [PATCH 3/7] bootstrap.py: a paste word that begins with `=` is quoted, a non-ASCII one only where the stdout writes the bytes git is handed, and a character's byte escapes get their `printf` word, so zsh no longer expands a leg's path (#961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #944 (#932) made each command step (a) prints for a person to paste name its leg path and its upstream as ONE shell word (`_paste_word`). rv932's review of it left five findings, and this takes them. They are all in the one function and its helper, in the module paragraph and in three docstrings. - A word that begins with `=` is quoted, as `repo_shape.shell_quoted` quotes it (#486), and written locally: bootstrap.py imports nothing new from repo_shape.py. zsh expands a bare `=ls` to `/usr/bin/ls` and stops a line at `=x` with `zsh: x not found`; 225 of 603 names that begin with `=` failed there, in two locales, and dash and bash read them all. - `_prints_as_it_is` asked the stdout's codec only, but git is handed `os.fsencode`'s bytes. A non-ASCII name now keeps its quoted word only where the stdout writes those same bytes; otherwise it gets the `printf` word, which is ASCII. Under a latin-1 stdout the line for `origin/café` named the byte 0xe9, where the ref is 0xc3 0xa9. - On POSIX the round-trip guard is gone: git IS handed `os.fsencode`'s bytes, so a name `os.fsdecode` reads as another (`sp` + U+DCC3 U+DCA9 + `c`, the directory `spéc`) has its word. The guard stays on Windows, where a command is not handed those bytes. A name with a NUL has no word either (`$(...)` drops it, so the word named another path), as one that ends in a line feed has none. - Two unit rows that kill the mutants rv932 found alive: a stdout with an unknown codec (R6) and a latin-1 filesystem codec through an in-process shim (R8). - The docstrings of `tests/test_bootstrap_after_bump.py`, of the pasteable lines test file and of the module paragraph say what the command holds. Not changed: the NOT INITIALIZED line in `checkout_tracking_branch`, which slice D (#958) moves, and `_say_twin`, `fast_forward_to_pin`, `place_at_pin` and `create_at_pin`, which #948 edits. A pinned copy: every project's `bootstrap.py` takes an `upstream-changed` row, and re-pins it. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- templates/assembly-root/scripts/bootstrap.py | 70 ++++-- tests/test_bootstrap_after_bump.py | 19 +- tests/test_bootstrap_pasteable_lines.py | 246 ++++++++++++++++--- 3 files changed, 274 insertions(+), 61 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 0c84142a..3ca312b1 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -50,11 +50,14 @@ was (#710). The commands it prints for a person to paste (the `merge --ff-only` that brings a moved-back branch forward, and the `--continue` and `--abort` of an operation in progress) name the - leg's path and the upstream as ONE shell word each: quoted, and, - where the name holds a byte that is not UTF-8 or a character a - terminal must not be handed, a `printf` that prints its bytes - again, so a paste names what is on disk and not its escape (#932). - Advancing a pin + leg's path and the upstream as ONE shell word each: quoted (a word + that begins with `=` too, which zsh would expand), and, where the + name holds a byte that is not UTF-8, a character a terminal must + not be handed, or a character this stdout cannot write, or writes + as other bytes than git is handed, a `printf` that prints those + bytes again, so a paste names what is on disk and not its escape + (#932, #961). A name no `printf` can print, one that ends in a line + feed or holds a NUL, is printed spelled, as it was. Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -772,23 +775,35 @@ def _sequence_in_progress(gitdir: Path) -> tuple[str, str, str] | None: def _prints_as_it_is(text: str) -> bool: - """Whether `print` writes `text` as it is (#932): `spelled` has no - escape to make in it, and this stdout holds every character of it, so - what the terminal shows is the text and a paste of it names the same.""" + """Whether `print` writes `text` as it is, and a paste of that gives git + the bytes it is handed (#932, #961): `spelled` has no escape to make in + it, this stdout holds every character of it and, where the text is not + ASCII, writes the same bytes as `os.fsencode`, so what the terminal + shows is the text and a paste of it names the same.""" + if spelled(text) != text: + return False + encoding = getattr(sys.stdout, "encoding", None) or "utf-8" try: - text.encode(getattr(sys.stdout, "encoding", None) or "utf-8") + written = text.encode(encoding) + return text.isascii() or written == os.fsencode(text) except (LookupError, UnicodeEncodeError): return False - return spelled(text) == text def _paste_word(name: str) -> str: """`name`, a leg's path or a ref's name, as ONE word of a command line - printed for a person to paste into a POSIX shell (#932). + printed for a person to paste into a POSIX shell (#932, #961). WHERE THE TEXT PRINTS AS IT IS it is `spelled(shlex.quote(name))`, as every such word was: an ordinary name, and a printable one that is not - ASCII, are byte for byte what they were (`_prints_as_it_is`). + ASCII where this stdout writes the bytes git is handed, are byte for + byte what they were (`_prints_as_it_is`). + + ONE CHANGE: a word that begins with `=` is quoted, as + `repo_shape.shell_quoted` quotes it (#486). zsh expands a bare `=ls` to + `/usr/bin/ls` and stops a line at `=x` with `zsh: x not found`, where + dash and bash read both as they are; `=` anywhere else is itself to all + three. `shlex.quote` leaves a word bare only where it holds no `'`. WHAT WAS WRONG. `spelled` writes what a terminal must not be handed as an escape (#391), and a shell reads that text as the characters it is. @@ -808,15 +823,23 @@ def _paste_word(name: str) -> str: ASCII, so it is the same on any stdout. It is `sh`'s dialect, as `shlex.quote` already was. - A NAME WITH NO SUCH FORM keeps `spelled`'s text, as it was: one whose - bytes do not read back as it (`os.fsdecode` of them is another name: a - lone surrogate with no byte, or a Windows code page that replaces), and - one that ENDS IN A LINE FEED, which `$(...)` strips, so the word would - name another path. A ref's name is neither: git refuses a control - character in one, and `_try_git` reads its bytes back. So it is a leg's - path, and one with no bytes is no directory a leg is moved in; a path - that ends in a line feed, which only a hand-made manifest holds, is the - one name left with no pasteable word. + A NAME WITH NO SUCH FORM keeps `spelled`'s text, as it was: + - one `os.fsencode` cannot write, a lone surrogate with no byte or a + character the filesystem encoding lacks; + - one whose bytes `$(...)` does not give back, so the word would name + another path: it strips a trailing line feed and drops a NUL; + - on Windows, one whose bytes do not read back as it (`os.fsdecode` of + them is another name: a code page that replaces). A command is not + handed these bytes there, so the read-back is what is asked. + + On POSIX the read-back is not asked (#961): git IS handed these bytes, + so a name `os.fsdecode` would read as another has its word, as + `sp\\udcc3\\udca9c` has `"$(printf -- 'sp\\303\\251c')"`, the bytes of the + directory `spéc`. A ref's name is none of the three: git refuses a + control character in one, and `_try_git` reads its bytes back. So it is + a leg's path, and one with no bytes is no directory a leg is moved in; + a path that ends in a line feed, which only a hand-made manifest holds, + is the one name left with no pasteable word. A COPY, as `inherited_child_environment` is: a root re-copies its pinned files one row at a time, so this file may meet a `repo_shape.py` older @@ -824,13 +847,16 @@ def _paste_word(name: str) -> str: `tests/test_bootstrap_pasteable_lines.py` runs each form it writes. """ quoted = shlex.quote(name) + if quoted.startswith("="): + quoted = f"'{quoted}'" if _prints_as_it_is(quoted): return quoted try: raw = os.fsencode(name) except UnicodeEncodeError: return spelled(quoted) - if os.fsdecode(raw) != name or raw.endswith(b"\n"): + if raw.endswith(b"\n") or b"\0" in raw or ( + os.name == "nt" and os.fsdecode(raw) != name): return spelled(quoted) escaped = "".join(chr(byte) if byte in PRINTF_BARE else f"\\{byte:03o}" for byte in raw) diff --git a/tests/test_bootstrap_after_bump.py b/tests/test_bootstrap_after_bump.py index fa802e53..381566d9 100644 --- a/tests/test_bootstrap_after_bump.py +++ b/tests/test_bootstrap_after_bump.py @@ -534,14 +534,17 @@ def printed() -> str: def test_the_branch_and_its_upstream_are_spelled_in_the_move_back(root): - """The line that moves a branch back names the branch, its upstream and - a command with both in it; each is spelled (#391). The state is built - by hand: a commit past the pin, a remote-tracking ref at it, the branch - tracking that ref, and the leg detached at the pin. Named with the byte - 0xff since #715, as the test above is, and read from `tracking_branch:`; - the line is asked as the escape it must print, not as `spelled` makes - it. Its upstream is read as the system decodes it since #824 (#710), and - HEAD is read here as bytes, which git's answer is.""" + """The line that moves a branch back names the branch and its upstream, + and holds a command with both in it. Each is spelled where the line + names it (#391); the command is not spelled: `_paste_word` words each + for a shell (#932), and a byte that is not UTF-8 is a `printf` that + prints it again. The state is built by hand: a commit past the pin, a + remote-tracking ref at it, the branch tracking that ref, and the leg + detached at the pin. Named with the byte 0xff since #715, as the test + above is, and read from `tracking_branch:`; the line is asked as the + escape it must print, not as `spelled` makes it. Its upstream is read + as the system decodes it since #824 (#710), and HEAD is read here as + bytes, which git's answer is.""" needs_a_byte_name(root.parent) manifest = root / "project.yaml" text = manifest.read_bytes() diff --git a/tests/test_bootstrap_pasteable_lines.py b/tests/test_bootstrap_pasteable_lines.py index 9efa73b4..cf34946e 100644 --- a/tests/test_bootstrap_pasteable_lines.py +++ b/tests/test_bootstrap_pasteable_lines.py @@ -11,17 +11,25 @@ on #824: for an upstream `origin/a<0xff>b` the line named no ref, and git answered `not something we can merge`. -THE RULE NOW (`_paste_word`). A name that `spelled` has nothing to escape in, -and this stdout can write, is the word it was, byte for byte: `shlex.quote`. -Any other is `"$(printf -- '')"`, which `sh` turns back into -the bytes of the name. A name that has no such word (one that ends in a line -feed, which `$(...)` strips, and one `os.fsencode` cannot write) keeps what +THE RULE NOW (`_paste_word`). A name that `spelled` has nothing to escape in +is the word it was, byte for byte (`shlex.quote`), where this stdout can +write it as the bytes git is handed: an ASCII name on any stdout, and one +that is not ASCII where the stdout's codec and the filesystem's give the +same bytes for it. A word that begins with `=` is quoted, as zsh expands a +bare one (#961). Any other is `"$(printf -- '')"`, which `sh` +turns back into the bytes `os.fsencode` gives the name. A name that has no +such word (one that ends in a line feed, which `$(...)` strips, one with a +NUL, which it drops, and one `os.fsencode` cannot write) keeps what `spelled` made of it. -EACH TEST BELOW RUNS WHAT WAS PRINTED, AS PRINTED: the command in the line, -through `/bin/sh -c` from the root, and asks where the branch is, or the -operation, afterwards. Asked of the words themselves it would hold only what -the helper says of itself, which the first fix would have changed with it. +THE TESTS THAT TAKE A `root` RUN WHAT WAS PRINTED, AS PRINTED: the command +in the line, through a shell's `-c` from the root, and ask where the branch +is, or the operation, afterwards. Asked of the words themselves it would +hold only what the helper says of itself, which the first fix would have +changed with it. The tests that take `bootstrap_module` ask `_paste_word` +for its words, in process, and where a shell's reading is the question run +the word through that shell; the stdout and the filesystem encoding they +vary are stubs, as `LegacyWindowsOs` is. The state is the one `tests/test_bootstrap_non_utf8_branch.py` builds (a `main` past the pin, with no commit of its own, tracking a remote-tracking @@ -43,6 +51,7 @@ import pytest from conftest import IDENTITY_EMAIL, IDENTITY_NAME, REPO, git, run_script +from repo_shape import shell_quoted from test_bootstrap_after_bump import ( # noqa: F401 MAIN, bootstrapped, head_ref, rev, root, skip_validators, ) @@ -90,11 +99,45 @@ r'''"$(printf -- '-sp\377ec')"''', id="leading-dash"), pytest.param("sp%s\udcffec", r"sp%s\udcffec", r'''"$(printf -- 'sp\045s\377ec')"''', id="percent"), + # Two escapes that are the bytes of a character (`spéc`, U+DCC3 U+DCA9): + # git is handed those bytes, and a `printf` prints them (#961). + pytest.param("sp\udcc3\udca9c", r"sp\udcc3\udca9c", + r'''"$(printf -- 'sp\303\251c')"''', id="UTF-8-as-bytes"), + # zsh expands a bare `=spec` to the path of a command (#961). + pytest.param("=spec", "=spec", "'=spec'", id="leading-equals"), ] SHELLS = [pytest.param(shell, id=shell.rsplit("/", 1)[1]) for shell in ("/bin/sh", "/bin/dash", "/bin/bash", "/bin/zsh")] +class Stdout: + """The one thing `_prints_as_it_is` reads of `sys.stdout`: its codec.""" + + def __init__(self, encoding: str): + self.encoding = encoding + + +class Latin1Os: + """`os` where the filesystem encoding is latin-1: git is handed `é` as + the byte 0xe9, and each byte reads back as the character of that number. + Only what `bootstrap.py` reads of it differs; everything else is this + system's `os`, as `LegacyWindowsOs`'s is.""" + + @staticmethod + def fsencode(name: str) -> bytes: + return name.encode("latin-1", "surrogateescape") + + @staticmethod + def fsdecode(raw: bytes) -> str: + return raw.decode("latin-1", "surrogateescape") + + def __getattr__(self, attr: str): + return getattr(os, attr) + + +LATIN1_FS = Latin1Os() + + def pasted(root: Path, command: str, shell: str = "/bin/sh") -> None: """`command`, as step (a) printed it, run from the root by `shell -c` as a person pastes it, with an editor that keeps the message git offers and @@ -113,6 +156,18 @@ def pasted(root: Path, command: str, shell: str = "/bin/sh") -> None: "replace")) +def printed_by(shell: str, word: str) -> bytes: + """What `shell` prints for `printf %s `: the bytes it reads the + word as. Skips where this system has no such shell.""" + if not os.path.exists(shell): + pytest.skip(f"this system has no {shell}") + proc = subprocess.run([shell, "-c", f"printf %s {word}"], + capture_output=True, check=False) + assert proc.returncode == 0, word + "\n" + proc.stderr.decode( + "utf-8", "replace") + return proc.stdout + + def mounted_at(root: Path, path: str) -> Path: """The spec leg moved to `path`, in the tree and in the manifest, as a hand-written manifest holds it (`tests/test_bootstrap_after_bump.py`'s @@ -206,7 +261,9 @@ def test_the_move_back_command_is_pasteable_whatever_the_legs_path_is_named( and `project.yaml` can hold it), the command names that directory when run as printed; so with a leading `-`, which `printf` would read as an option without its `--`, and a `%`, which it would read as a conversion - without its escape.""" + without its escape. A path made of the escapes of a character's bytes + (`spc`, the directory `spéc`) is that directory too, and + one that begins with `=` is quoted, which zsh would expand (#961).""" pin, ahead, lines = moved_back_lines(root, tmp_path, path=path) command = f"git -C {word} merge --ff-only origin/main" @@ -218,6 +275,19 @@ def test_the_move_back_command_is_pasteable_whatever_the_legs_path_is_named( assert rev(leg, MAIN) == ahead +@pytest.mark.parametrize("shell", SHELLS) +def test_a_leg_path_that_begins_with_an_equals_sign_is_pasteable_in_a_shell( + root, tmp_path, shell): + """THE ISSUE (#961): `git -C =spec merge ...` is read by zsh as the path + of a command called `spec`, and stops at `zsh: spec not found`. Quoted, + the printed command brings `main` forward in each shell, zsh included + (dash and bash read the bare word too, which is why only zsh failed).""" + pin, ahead, lines = moved_back_lines(root, tmp_path, path="=spec") + + pasted(root, command_of(lines[0]), shell) + assert rev(root / "=spec", MAIN) == ahead + + # --- `_say_in_progress`'s two commands -------------------------------------- @pytest.mark.parametrize("which, done", [ @@ -257,7 +327,7 @@ def test_the_commands_that_end_an_operation_are_pasteable_whatever_the_path( ("my spec", "'my spec'"), ("it's", "'it'\"'\"'s'"), ("", "''"), - ("=x", "=x"), + ("a=b", "a=b"), pytest.param("origin/café", "'origin/café'", id="printable-UTF-8"), pytest.param("日本", "'日本'", id="CJK"), pytest.param("a\u00a0b", "'a\u00a0b'", id="no-break-space"), @@ -266,11 +336,42 @@ def test_a_name_with_nothing_to_escape_is_the_word_it_was( bootstrap_module, name, word): """Byte for byte `spelled(shlex.quote(name))`, which every such word was: no `printf`, no new quoting. A U+00A0 is no escape to `spelled`, - and prints as it is.""" + and prints as it is, and an `=` that does not begin the word is itself + to every shell.""" assert bootstrap_module._paste_word(name) == word assert word == bootstrap_module.spelled(shlex.quote(name)) +@pytest.mark.parametrize("name, word", [ + ("=x", "'=x'"), + ("=ls", "'=ls'"), + ("=", "'='"), + ("==x", "'==x'"), + ("=%", "'=%'"), + ("=a b", "'=a b'"), + ("origin/=x", "origin/=x"), +], ids=lambda value: None if value else "empty") +def test_a_word_that_begins_with_an_equals_sign_is_quoted( + bootstrap_module, name, word): + """`shell_quoted` quotes a word that begins with `=` (#486), and so does + `_paste_word` (#961), once: a word `shlex.quote` quotes already is not + quoted again, and an `=` inside a name is left as it was. The word is + `shell_quoted`'s, as the doctor and `family.py` word theirs.""" + assert bootstrap_module._paste_word(name) == word + assert word == shell_quoted(name, "posix") + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("name", ["=x", "=ls", "=", "==x", "=%", "=a b"]) +def test_a_word_that_begins_with_an_equals_sign_reads_back_in_every_shell( + bootstrap_module, shell, name): + """zsh expands a bare `=ls` to `/usr/bin/ls` and stops at `=x`, so what + is asked is the shell: the word prints the name, in dash, bash and zsh + alike (#961).""" + word = bootstrap_module._paste_word(name) + assert printed_by(shell, word) == name.encode("ascii") + + @pytest.mark.parametrize("upstream, word", [ pytest.param("main", "origin/main", id="ASCII"), pytest.param("it's", "'origin/it'\"'\"'s'", id="a-quote"), @@ -292,24 +393,36 @@ def test_the_move_back_line_of_a_printable_upstream_is_the_one_it_was( assert rev(root / "spec", MAIN) == ahead -# --- a stdout that cannot write the name ------------------------------------ +# --- a stdout that does not write the bytes git is handed ------------------- @NEEDS_A_UTF8_PROCESS -def test_a_stdout_that_cannot_write_the_name_gets_the_printf_word( - root, tmp_path): +@pytest.mark.parametrize("encoding, held", [ + pytest.param("ascii", r"origin/caf\xe9", id="cannot-write-it"), + pytest.param("latin-1", "origin/café", id="writes-other-bytes"), +]) +def test_a_stdout_of_another_codec_than_the_names_gets_the_printf_word( + root, tmp_path, encoding, held): """`PYTHONIOENCODING=ascii` writes `é` as `\\xe9`, which no shell - reads back as the name, so the command holds its bytes as `printf` - does; the prose names the upstream as the stdout writes it.""" + reads back as the name, and `latin-1` writes it as the byte 0xe9, where + git was handed 0xc3 0xa9 (#961): pasted, the quoted word named no ref. + So the command holds the bytes git is handed, as `printf` does; the + prose names the upstream as the stdout writes it, read back here in that + codec.""" leg = root / "spec" pin, ahead = past_the_pin(leg, "café", "main") - result = run_script(root / "scripts" / "bootstrap.py", "--skip-validators", - cwd=root, env={"PYTHONIOENCODING": "ascii"}) - assert result.returncode == 0, result.stderr + result.stdout + result = subprocess.run( + [sys.executable, str(root / "scripts" / "bootstrap.py"), + "--skip-validators"], cwd=str(root), stdin=subprocess.DEVNULL, + env={**os.environ, "PYTHONIOENCODING": encoding}, + capture_output=True, check=False) + printed = result.stdout.decode(encoding) + assert result.returncode == 0, printed + result.stderr.decode( + encoding, "replace") word = r'''"$(printf -- 'origin/caf\303\251')"''' command = f"git -C spec merge --ff-only {word}" - line = moved_back_line("spec", ahead, pin, r"origin/caf\xe9", command) - assert line in result.stdout.split("\n"), result.stdout + line = moved_back_line("spec", ahead, pin, held, command) + assert line in printed.split("\n"), printed pasted(root, command_of(line)) assert rev(leg, MAIN) == ahead @@ -328,6 +441,10 @@ def test_a_stdout_that_cannot_write_the_name_gets_the_printf_word( ("a\nb", r'''"$(printf -- 'a\012b')"'''), ("dé\udcff", r'''"$(printf -- 'd\303\251\377')"'''), ("=x\udcff", r'''"$(printf -- '=x\377')"'''), + # Escapes that are the bytes of a character, which `os.fsdecode` reads + # as that character: git is handed those bytes, so they have a word. + ("sp\udcc3\udca9c", r'''"$(printf -- 'sp\303\251c')"'''), + ("caf\udcc3\udca9\udcff", r'''"$(printf -- 'caf\303\251\377')"'''), # An escape is ALWAYS three digits: a digit that follows is the name's. ("a\x1b7\udcff", r'''"$(printf -- 'a\0337\377')"'''), ("\t1\udcff", r'''"$(printf -- '\0111\377')"'''), @@ -337,20 +454,22 @@ def test_the_printf_word_prints_the_bytes_of_the_name( """The word for each byte class, exactly, and what `/bin/sh` makes of it is the bytes `os.fsencode` gives the name: a byte outside the safe set is a three-digit octal escape, `%` and the backslash and the apostrophe - included, and a newline in the middle of a name is kept.""" + included, and a newline in the middle of a name is kept. So are the + escapes of a character's bytes (#961): `os.fsdecode` of them is that + character, which is no reason to keep the escape text.""" assert bootstrap_module._paste_word(name) == word - out = subprocess.run(["/bin/sh", "-c", f"printf %s {word}"], - capture_output=True, check=True).stdout - assert out == os.fsencode(name) + assert printed_by("/bin/sh", word) == os.fsencode(name) -@pytest.mark.parametrize("name", ["a\udcffb\n", "sp\n", "\ud83d"], - ids=["byte-then-LF", "ends-in-LF", "lone-surrogate"]) +@pytest.mark.parametrize("name", ["a\udcffb\n", "sp\n", "\ud83d", "a\x00b"], + ids=["byte-then-LF", "ends-in-LF", "lone-surrogate", + "NUL"]) def test_a_name_with_no_printf_word_keeps_the_text_it_had( bootstrap_module, name): - """`$(...)` strips a trailing line feed, so a word for such a name would - be another path, and a lone surrogate has no byte at all: both keep what - `spelled(shlex.quote(name))` made of them, as before, and are never the + """`$(...)` strips a trailing line feed and drops a NUL, so a word for + such a name would be another path (the NUL's was `ab` in dash and bash, + #961), and a lone surrogate has no byte at all: each keeps what + `spelled(shlex.quote(name))` made of it, as before, and is never the stripped word.""" assert bootstrap_module._paste_word(name) == bootstrap_module.spelled( shlex.quote(name)) @@ -369,6 +488,71 @@ def test_a_name_the_codec_would_not_read_back_keeps_the_text_it_had( shlex.quote(name)) +@pytest.mark.parametrize("encoding, filesystem, word", [ + pytest.param("utf-8", None, "'origin/café'", id="both-UTF-8"), + pytest.param("latin-1", None, r'''"$(printf -- 'origin/caf\303\251')"''', + id="stdout-latin-1"), + pytest.param("cp1252", None, r'''"$(printf -- 'origin/caf\303\251')"''', + id="stdout-cp1252"), + pytest.param("utf-8", LATIN1_FS, r'''"$(printf -- 'origin/caf\351')"''', + id="filesystem-latin-1"), + pytest.param("latin-1", LATIN1_FS, "'origin/café'", id="both-latin-1"), + pytest.param("cp1252", LATIN1_FS, "'origin/café'", + id="two-codecs-one-byte-for-this-name"), +]) +def test_a_name_that_is_not_ascii_is_quoted_where_both_codecs_give_its_bytes( + bootstrap_module, monkeypatch, encoding, filesystem, word): + """A quoted `é` is what the terminal shows, and a paste of it is the + bytes the stdout's codec made of it: only where those are the bytes + `os.fsencode` gives the name, the ones git is handed, does the paste name + the same. Under a latin-1 stdout and a UTF-8 filesystem (a stdout the + locale does not set, `PYTHONIOENCODING`, a terminal emulator that is + not UTF-8) the word was `'origin/café'`, pasted as 0xe9 where the ref is + 0xc3 0xa9; under a latin-1 filesystem and a UTF-8 stdout it was the + reverse (#961). `cp1252` and `latin-1` are two codecs and one byte for + `é`, and that is all the name asks. The filesystem encoding is the + shim's `os.fsencode`, as `LegacyWindowsOs`'s is.""" + monkeypatch.setattr(sys, "stdout", Stdout(encoding)) + if filesystem is not None: + monkeypatch.setattr(bootstrap_module, "os", filesystem) + assert bootstrap_module._paste_word("origin/café") == word + + +@pytest.mark.parametrize("encoding", ["latin-1", "ascii", "utf-16"]) +def test_an_ascii_name_is_the_word_it_was_on_any_stdout( + bootstrap_module, monkeypatch, encoding): + """An ASCII name shows as itself on every stdout that can write it, so + the filesystem's codec is not asked of it (#961): `origin/main` stays + bare, under a UTF-16 stdout too, whose bytes are not git's.""" + monkeypatch.setattr(sys, "stdout", Stdout(encoding)) + monkeypatch.setattr(bootstrap_module, "os", LATIN1_FS) + assert bootstrap_module._paste_word("origin/main") == "origin/main" + + +def test_a_stdout_with_no_such_codec_gets_the_printf_word( + bootstrap_module, monkeypatch): + """A stream a caller replaced may name any encoding, and `str.encode` + answers an unknown one with a `LookupError`, which is no + `UnicodeEncodeError`: no codec holds the name, so it gets the `printf` + word, which is ASCII, and nothing raises (#961).""" + monkeypatch.setattr(sys, "stdout", Stdout("no-such-codec")) + assert bootstrap_module._paste_word("origin/main") == ( + r'''"$(printf -- 'origin/main')"''') + + +def test_the_printf_word_is_the_bytes_the_filesystem_encoding_gives( + bootstrap_module, monkeypatch): + """Git is handed `os.fsencode`'s bytes, so the word prints those: under + a latin-1 filesystem `é` is the byte 0xe9, and the UTF-8 bytes of the + character (`name.encode("utf-8", "surrogateescape")`) would name another + path, `café`. Run through `/bin/sh`, the word prints what the shim's + `fsencode` gives (#961).""" + monkeypatch.setattr(bootstrap_module, "os", LATIN1_FS) + word = bootstrap_module._paste_word("café\t") + assert word == r'''"$(printf -- 'caf\351\011')"''' + assert printed_by("/bin/sh", word) == b"caf\xe9\t" + + def test_a_stdout_that_is_not_there_is_asked_as_utf8( bootstrap_module, monkeypatch): """`pythonw` has no stdout (`sys.stdout` is None): the word is the one From b420171f4648b192918891e8b20d794be0dfad91 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 19:13:26 +0000 Subject: [PATCH 4/7] validate-manifest.py and bootstrap.py: a naming policy this user cannot read is named relative to the root with the remedy, each validator is opened before any runs, and step (c) names a register under an absolute leg path whole and heads one whose stat was refused `register, if one is there` (#958 fix round 1, #430 slice D) On lane 4's RV958 review of #958 at 0ef875c0: - M1: step (c)'s header goes through `_named`, so a register under a leg whose `path:` is absolute is named whole, where `relative_to` ended the run in a `ValueError` traceback, readable or not. - N1: a register whose `stat` was refused is headed `register, if one is there:` (`_a_register` now returns the label), so the line does not say that one is there. - S1: `NamingPolicy.load` goes through `_load`, so a policy in a `contracts/` this user cannot search is `yaml-unreadable` naming `contracts/repository-naming.yaml`, with the remedy, not the absolute path alone. - N3: `_unreadable_file` spells the path it names (`_shown`, a copy of validate-pins.py's). - S2: `_a_validator` opens each validator after its `stat`, before any runs, as bump-leg.py's `_validators_readable` does (#934); one at mode 000 is `validator-unreadable` by name, not the child's own `can't open file` line. - S3 and N5: tests for `_is_a`'s `ValueError` clause in both files, for the spelling in `_named` on both of its branches, for the I/O-error cause at every site that reads it, for a directory where a file goes, and for the out-of-root naming in both files. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- templates/assembly-root/scripts/bootstrap.py | 85 ++-- .../scripts/validate-manifest.py | 41 +- tests/test_assembly_root_unreadable_paths.py | 381 +++++++++++++++--- 3 files changed, 413 insertions(+), 94 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 3ce1c612..33d75ef5 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -462,6 +462,24 @@ def _load_manifest(root: Path, path: Path): raise +def _a_validator(root: Path, path: Path) -> bool: + """Whether step (b) runs the validator at `path`: a file this user can + open, or `validator-unreadable` naming it (`_unreadable`) where its + `stat` or its open was refused (#430). Opened, and closed unread, as + `bump-leg.py`'s `_validators_readable` opens each (#934), since the + child has to read it: one at mode 000 in a `scripts/` this user can + search reached the child, and the run said Python's own `can't open + file` line and `bootstrap REFUSED`, with nothing to put it right.""" + if not _is_file(root, path, "validator-unreadable", VALIDATOR_READ): + return False + try: + path.open("rb").close() + except OSError as exc: + raise _unreadable("validator-unreadable", root, path, exc, + VALIDATOR_READ) from exc + return True + + class _Unreadable(Exception): """A name in a leg that step (a) must look at and whose `stat` the operating system refused (#430): `path`, and its `OSError`, `error`. @@ -1625,26 +1643,36 @@ def _cached_tip(root: Path) -> str | None: return None -def _register_paths(root: Path, legs: list[dict]) -> list[Path]: +def _register_paths(root: Path, legs: list[dict]) -> list[tuple[Path, str]]: + """Each register step (c) reads, with what its line calls it + (`_a_register`).""" candidates = [root / REGISTER_RELPATH] for leg in legs: path = str(leg.get("path") or "") if path and path != ".": candidates.append(root / path / REGISTER_RELPATH) - return [c for c in candidates if _a_register(c)] + called = [(c, _a_register(c)) for c in candidates] + return [(c, label) for c, label in called if label is not None] -def _a_register(path: Path) -> bool: - """Whether step (c) takes `path` for a register: a file, or a name whose - `stat` the operating system refused (#430), which may be one and which - `_load_register_rows` then names as unreadable. `Path.is_file()` raised - that refusal out of step (c), which never fails, up to Python 3.13, a - traceback, and from 3.14 answered False, so the run said `authority is - not wallet-carried in this org` of a register it could not read.""" +#: What step (c) calls a register whose `stat` the operating system refused +#: (#430): a directory on the way to it cannot be searched, so whether one +#: is there is not known, and the line does not say that one is. +MAYBE_A_REGISTER = "register, if one is there" + + +def _a_register(path: Path) -> str | None: + """What step (c) calls `path`: `register` where it is a file, + `MAYBE_A_REGISTER` where its `stat` the operating system refused (#430), + which `_load_register_rows` then names as unreadable, and `None` where + no register is there. `Path.is_file()` raised that refusal out of step + (c), which never fails, up to Python 3.13, a traceback, and from 3.14 + answered False, so the run said `authority is not wallet-carried in + this org` of a register it could not read.""" try: - return _is_a(path, stat.S_ISREG) + return "register" if _is_a(path, stat.S_ISREG) else None except OSError: - return True + return MAYBE_A_REGISTER def _row_objects(row: dict) -> list[str]: @@ -1731,18 +1759,20 @@ def _row_report_line(row: dict, repositories: set[str], def _report_one_register(register_path: Path, root: Path, - repositories: set[str], prefixes: list[str]) -> None: + repositories: set[str], prefixes: list[str], + label: str = "register") -> None: """One register file, start to finish: read it, print each row that names this project, and the "no rows"/"no row names…" line when there is nothing to show. Split from `read_authority` for #132 so its loop over every register - file found is a single call per file. + file found is a single call per file. Its first line is `label`, which + `_a_register` chose, and the path as `_named` names it (#430): relative + to the root, POSIX on every platform, as git and the reader spell it, + and whole where a leg's `path:` is absolute, which `relative_to` ended + in a `ValueError`, a traceback from the step that never fails. """ - # POSIX, on every platform: this line names a file in the repository, - # and git and the reader both spell it that way. - print(" register: " - f"{spelled(register_path.relative_to(root).as_posix())}") + print(f" {label}: {_named(root, register_path)}") rows = _load_register_rows(register_path, root) if rows is None: return @@ -1774,8 +1804,9 @@ def read_authority(root: Path, legs: list[dict]) -> None: if leg.get("repository")} prefixes = [str(leg.get("path")) for leg in legs if leg.get("path") and leg.get("path") != "."] - for register_path in found: - _report_one_register(register_path, root, repositories, prefixes) + for register_path, label in found: + _report_one_register(register_path, root, repositories, prefixes, + label) print(" Grants are read here for REPORTING only. This command confers " "nothing and enforces nothing; a required check in the repository " "that owns the object is what confers.") @@ -1867,12 +1898,12 @@ def _run_neutral_validators(root: Path, manifest: dict | None, validator that failed, in the order they ran: `_end_of_run` says which were findings and which refusals (#764). - Each validator is looked for BEFORE ANY RUNS, and one whose `stat` the - operating system refuses is `validator-unreadable`, naming it (#430): - only one that is not there is SKIPPED. `Path.is_file()` raised that - refusal out of the run up to Python 3.13, a traceback, and from 3.14 - answered False, so a `scripts/` this user cannot search SKIPPED all - three and the run said `bootstrap ok`. + Each validator is looked for BEFORE ANY RUNS, and one whose `stat` or + open the operating system refuses is `validator-unreadable`, naming it + (`_a_validator`, #430): only one that is not there is SKIPPED. + `Path.is_file()` raised a refused `stat` out of the run up to Python + 3.13, a traceback, and from 3.14 answered False, so a `scripts/` this + user cannot search SKIPPED all three and the run said `bootstrap ok`. """ if skip: print(" skipped (--skip-validators)") @@ -1883,9 +1914,7 @@ def _run_neutral_validators(root: Path, manifest: dict | None, "governed for that.") return [] failed = [] - present = [(label, argv_, _is_file(root, root / argv_[0], - "validator-unreadable", - VALIDATOR_READ)) + present = [(label, argv_, _a_validator(root, root / argv_[0])) for label, argv_ in VALIDATORS] for label, argv_, there in present: if not there: diff --git a/templates/assembly-root/scripts/validate-manifest.py b/templates/assembly-root/scripts/validate-manifest.py index 663c9a64..2e53a533 100755 --- a/templates/assembly-root/scripts/validate-manifest.py +++ b/templates/assembly-root/scripts/validate-manifest.py @@ -158,21 +158,31 @@ def _is_a(path: Path, kind: Callable[[int], bool]) -> bool: return kind(mode) +def _shown(typed: str) -> str: + """A path as a refusal names it: as it is, unless a character in it is + not printable (`str.isprintable`), and then as Python's `repr` spells + it, `'x\\x1b[31mRED'`. `validate-pins.py`'s `_shown`, written here + rather than imported, as `_is_a` is (#370, #610). Printed raw, an ESC + in the name reached the terminal as the control it is.""" + return typed if typed.isprintable() else repr(typed) + + def _unreadable_file(root: Path, path: Path, exc: OSError) -> Refusal: """`yaml-unreadable` for a file this validator must read or find and the operating system will not let it (#430): its `stat` refused, or its open. `validate-pins.py`'s `_unreadable_file`, in its words. NAMED ONCE, RELATIVE TO THE ROOT, POSIX on every platform, or whole - where it is not under the root. THE ADVICE FOLLOWS THE CAUSE: a - permission (`PermissionError`, which is `EACCES` and `EPERM`) is put - right by the mode or the owner, and any other cause is `OTHER_REASON`, - which is not told to `chmod`. + where it is not under the root (a `--policy` given elsewhere), and + spelled (`_shown`). THE ADVICE FOLLOWS THE CAUSE: a permission + (`PermissionError`, which is `EACCES` and `EPERM`) is put right by the + mode or the owner, and any other cause is `OTHER_REASON`, which is not + told to `chmod`. """ try: - named = path.relative_to(root).as_posix() + named = _shown(path.relative_to(root).as_posix()) except ValueError: - named = str(path) + named = _shown(str(path)) advice = ( "Make what this user may not open readable by this user: `chmod u+r` " "on a file, `chmod u+rx` on a directory (and on every directory above " @@ -196,18 +206,21 @@ def _is_file(root: Path, path: Path) -> bool: raise _unreadable_file(root, path, exc) from exc -def _load(root: Path, path: Path): - """`load_yaml(path)`, with a file this user cannot OPEN named as one - whose `stat` was refused is (`_unreadable_file`, #430), so `project.yaml` - at mode 000 and one in a folder nobody may search say one line. +def _load(root: Path, path: Path, read: Callable = load_yaml): + """`read(path)`, `load_yaml` or `NamingPolicy.load`, with a file this + user cannot OPEN named as one whose `stat` was refused is + (`_unreadable_file`, #430), so `project.yaml` at mode 000 and one in a + folder nobody may search say one line, and so does the naming policy + in a `contracts/` this user cannot search, which was named whole, with + nothing to put it right. Read off the refusal's cause, as `validate-pins.py`'s `_load` reads it: `load_yaml` has raised its refusal `from` the `OSError` since it was written, so a `repo_shape.py` older than this file is read the same way. - Any other refusal (a file that is not UTF-8, or does not parse) keeps - its words.""" + Any other refusal (a file that is not UTF-8, or does not parse, or a + policy of the wrong kind) keeps its words.""" try: - return load_yaml(path) + return read(path) except Refusal as exc: if isinstance(exc.__cause__, OSError): raise _unreadable_file(root, path, @@ -1208,7 +1221,7 @@ def main(argv: list[str] | None = None) -> int: if not isinstance(manifest, dict): raise Refusal("manifest-unreadable", f"{manifest_path}: not a mapping") policy_path = args.policy or (root / "contracts" / "repository-naming.yaml") - policy = NamingPolicy.load(policy_path) + policy = _load(root, policy_path, NamingPolicy.load) # Inside the `try`: a declared referent's pin file this user cannot # reach is a check that cannot run, so it is refused, exit 2, and no # finding is said over it (#430). diff --git a/tests/test_assembly_root_unreadable_paths.py b/tests/test_assembly_root_unreadable_paths.py index 82c21179..16581ca8 100644 --- a/tests/test_assembly_root_unreadable_paths.py +++ b/tests/test_assembly_root_unreadable_paths.py @@ -25,19 +25,32 @@ absent, and the leg was put on its tracking branch, where the pick's `--continue` goes on. +And at #958's first head, 0ef875c0, lane 4's review (RV958) measured: + + * `validate-manifest.py`, a naming policy in a `contracts/` it cannot + search: named whole, with nothing to put it right, on every Python. + * `bootstrap.py`, a validator at mode 000 in a `scripts/` it can search: + the child's own `can't open file` line, and `bootstrap REFUSED`. + * `bootstrap.py`, a register under a leg whose `path:` is absolute: a + `ValueError` traceback in step (c), readable or not. + * `bootstrap.py`, a register whose `stat` was refused: a `register:` line, + which says that one is there. + THE RULE NOW, lane 1's for every slice. The path is `stat`ed: ENOENT, ENOTDIR, EBADF and ELOOP (and Windows' 21, 123 and 1921) mean absent, and any other `OSError` is named `: cannot be read ()`, in `validate-pins.py`'s words, with its cure: `chmod u+r` a file and `chmod u+rx` a directory and every one above it, or, for another cause, its `OTHER_REASON`. - * A `project.yaml`, a validator or a declared referent's pin file is a - refusal, exit 2, and a `project.yaml` this user cannot OPEN says the same - line as one it cannot reach. + * A `project.yaml`, the naming policy, a validator or a declared + referent's pin file is a refusal, exit 2, and one this user cannot OPEN + says the same line as one it cannot reach. * A leg's `.git`, or an operation's marker in its git dir, is a line in step (a), and the leg is left as it is; a register is a line in step (c), and nothing is claimed from it. Neither step fails the run, as - neither did before: the validators refuse such a leg. + neither did before: the validators refuse such a leg. A register whose + `stat` was refused is `register, if one is there:`, and a name outside + the root is named whole. THE HARNESS. Each refusal is asked twice and must say the same both times: as a project runs the tool, with the mode taken away (POSIX, and not as @@ -91,7 +104,14 @@ #: The pin file `as_a_descendant_by_chain` makes the root hold: its chain's #: first link's. PIN = "contracts/openxdox-pin.yaml" +POLICY = "contracts/repository-naming.yaml" REGISTER = "governance/review-authority/register.yaml" +#: A register with one row naming the code leg's repository. +ROWS = ("rows:\n - holder: agent:x\n act: review\n" + " target_repo: testorg/Atlas-code\n state: active\n") +#: What step (c) calls a register whose `stat` was refused (RV958's N1). +MAYBE = "register, if one is there" +DASH = "\u2014" GRANTS = (" Grants are read here for REPORTING only. This command confers " "nothing and enforces nothing; a required check in the repository " "that owns the object is what confers.") @@ -113,17 +133,57 @@ def refused(code: str, named: str, reads: str, def left_as_it_is(named: str, reason: str = "Permission denied", - advice: str = PERMISSION) -> list[str]: - """Step (a)'s two lines for the spec leg, where `named` cannot be read.""" - return [f" [spec] spec: {named}: cannot be read ({reason}), so the leg " - "is left as it is", f" {advice}{AGAIN}"] + advice: str = PERMISSION, + where: str = "[spec] spec") -> list[str]: + """Step (a)'s two lines for the leg `where` names (the spec leg), where + `named` cannot be read.""" + return [f" {where}: {named}: cannot be read ({reason}), so the leg is " + "left as it is", f" {advice}{AGAIN}"] + + +def nothing_claimed(named: str, label: str = MAYBE, + reason: str = "Permission denied", + advice: str = PERMISSION) -> list[str]: + """Step (c)'s lines for a register at `named` this user cannot read: + `label` is `register` where its `stat` worked and only its open was + refused.""" + return [f" {label}: {named}", + f" unreadable, so nothing is claimed from it: {named}: cannot " + f"be read ({reason})", f" {advice}{AGAIN}"] -def nothing_claimed(named: str) -> list[str]: - """Step (c)'s lines for a register at `named` this user cannot read.""" - return [f" register: {named}", - f" unreadable, so nothing is claimed from it: {named}: cannot " - "be read (Permission denied)", f" {PERMISSION}{AGAIN}"] +def not_initialized(shown: str) -> str: + """Step (a)'s line for a spec leg at `shown` that is not there.""" + return (f" [spec] {shown}: NOT INITIALIZED. Run `git submodule update " + f"--init {shown}` {DASH} the validators below will refuse until " + "you do.") + + +def refusing(monkeypatch, name: str, failure: OSError | None, + method: str = "stat") -> str: + """The `stat` of each path called `name` refused as Python 3.14 answers + it (`refused_stat`) where `failure` is None, else `Path.` + raising `failure`; returns the reason the refusal says.""" + if failure is None: + refused_stat(monkeypatch, name) + return "Permission denied" + failing(monkeypatch, method, {name: failure}) + return failure.strerror + + +def leg_path(root: Path, role: str, typed: str) -> str: + """The `role` leg's `path:` set to `typed`, a character that does not + print written as its YAML escape; returns `typed` as `bootstrap.py` + spells it, which, for the ASCII names these tests give, is that + escape.""" + manifest = root / "project.yaml" + text = manifest.read_text(encoding="utf-8") + old = f' path: "{role}"' + assert text.count(old) == 1, text + escaped = typed.encode("unicode_escape").decode("ascii") + manifest.write_text(text.replace(old, f' path: "{escaped}"'), + encoding="utf-8") + return escaped def placed(root: Path, role: str) -> str: @@ -132,6 +192,16 @@ def placed(root: Path, role: str) -> str: return f" [{role}] {role}: on main at {pin[:12]} (branch tip == pin)" +def up_to_step_b(root: Path) -> str: + """The whole of stdout of a run refused before step (b) runs any + validator, both legs placed by step (a).""" + return "\n".join([ + f"bootstrap: Atlas ({root.resolve()})", "", + "(a) legs on tracking branches, pins untouched", + placed(root, "spec"), placed(root, "code"), "", + "(b) neutral validators", ""]) + + def skipping_validators(root: Path, step_a: list[str], step_c: list[str]) -> str: """The whole of stdout of `bootstrap.py --skip-validators` in `root`.""" @@ -233,6 +303,73 @@ def test_a_manifest_whose_stat_is_refused_is_the_same_refusal( assert seen.out == "" +def test_a_manifest_that_is_a_directory_is_no_manifest_to_either_tool( + boot, project): + """THE OTHER HALF, which the base answered the same: a directory where + `project.yaml` goes is no manifest, never a file that cannot be read. + The gate refuses `manifest-missing`, and `bootstrap.py` reads a project + that has not elected the schema.""" + manifest = project / "project.yaml" + manifest.unlink() + manifest.mkdir() + result = validate(project) + assert result.stderr == ( + f"REFUSED manifest-missing: {project.resolve() / 'project.yaml'} " + "does not exist. A project that has elected this schema declares it " + "in `project.yaml`; a project that has not, does not run this " + "validator.\n"), result.stderr + assert result.returncode == 2 + loaded = boot._load_root_and_manifest(SimpleNamespace(root=project)) + assert loaded[1] is None + + +# --- validate-manifest.py: the naming policy --------------------------------- + +@CANNOT_RUN_AS_ROOT +def test_a_naming_policy_the_gate_cannot_reach_is_refused_by_name(project): + """`contracts/` at mode 000 and no `--policy`, the case a project meets: + the base named the policy whole, with nothing to put it right, on every + Python (RV958's S1).""" + with at_mode_000(project, "contracts"): + result = validate(project) + assert result.stderr == refused("yaml-unreadable", POLICY, + MANIFEST_READS), result.stderr + assert result.returncode == 2, result.stdout + + +@pytest.mark.parametrize("given, failure, advice", [ + pytest.param(False, os_error(errno.EIO), OTHER, id="io-error"), + pytest.param(True, PermissionError(errno.EACCES, "Permission denied"), + PERMISSION, id="given-outside-the-root-with-an-esc"), +]) +def test_a_naming_policy_the_gate_cannot_open_is_named_on_every_python( + manifest_gate, project, tmp_path, monkeypatch, capsys, given, + failure, advice): + """In-process, as root and on every platform. An I/O error is no + permission, and is not told to `chmod`. A `--policy` outside the root is + named whole, and spelled as `repr` spells a name that does not print, as + `validate-pins.py` spells one (RV958's N3).""" + policy = tmp_path / "loc\x1bked" / "repository-naming.yaml" + given_args = ["--policy", str(policy)] if given else [] + failing(monkeypatch, "read_text", {"repository-naming.yaml": failure}) + assert manifest_gate.main(["--root", str(project), *given_args]) == 2 + assert capsys.readouterr().err == refused( + "yaml-unreadable", repr(str(policy)) if given else POLICY, + MANIFEST_READS, failure.strerror, advice) + + +def test_a_name_that_does_not_print_is_spelled_under_the_root_too( + manifest_gate, tmp_path): + """The other branch of `_unreadable_file`'s naming, under the root, + where the names this gate reads are fixed or a referent's: spelled, as + outside it.""" + refusal = manifest_gate._unreadable_file( + tmp_path, tmp_path / "loc\x1bked" / "x.yaml", + PermissionError(errno.EACCES, "Permission denied")) + assert refusal.detail == ( + "'loc\\x1bked/x.yaml': cannot be read (Permission denied)") + + # --- validate-manifest.py: a declared referent's pin file -------------------- @CANNOT_RUN_AS_ROOT @@ -273,12 +410,8 @@ def test_a_referent_pin_whose_stat_is_refused_is_refused_on_every_python( missing; an I/O error is no permission, and is not told to `chmod`.""" held_by_chain(project) clear_ambient_pin_sources(monkeypatch) - if failure is None: - refused_stat(monkeypatch, "openxdox-pin.yaml") - else: - failing(monkeypatch, "stat", {"openxdox-pin.yaml": failure}) + reason = refusing(monkeypatch, "openxdox-pin.yaml", failure) assert manifest_gate.main(["--root", str(project)]) == 2 - reason = "Permission denied" if failure is None else failure.strerror assert capsys.readouterr().err == refused( "yaml-unreadable", PIN, MANIFEST_READS, reason, advice) @@ -321,13 +454,21 @@ def test_a_manifest_bootstrap_cannot_reach_or_open_is_refused_first(project, assert still_detached(project, "spec") -def test_a_manifest_bootstrap_cannot_stat_is_the_same_refusal(boot, project, - monkeypatch): - refused_stat(monkeypatch, "project.yaml") +@pytest.mark.parametrize("method, failure, advice", [ + pytest.param("stat", None, PERMISSION, id="stat-refused"), + pytest.param("stat", os_error(errno.EIO), OTHER, id="stat-io-error"), + pytest.param("read_text", os_error(errno.EIO), OTHER, id="open-io-error"), +]) +def test_a_manifest_bootstrap_cannot_stat_or_open_is_the_same_refusal( + boot, project, monkeypatch, method, failure, advice): + """On every Python: the `stat` refused as 3.14 answers it, and an I/O + error from the `stat` or the open, which is no permission and is not + told to `chmod`.""" + reason = refusing(monkeypatch, "project.yaml", failure, method) with pytest.raises(boot.Refusal) as caught: boot._load_root_and_manifest(SimpleNamespace(root=project)) assert f"{caught.value}\n" == refused("yaml-unreadable", "project.yaml", - BOOTSTRAP_READS) + BOOTSTRAP_READS, reason, advice) # --- bootstrap.py: the validators -------------------------------------------- @@ -346,38 +487,62 @@ def test_a_scripts_folder_bootstrap_cannot_search_is_refused_not_skipped( "validator-unreadable", "scripts/validate-repository-naming.py", VALIDATOR_READS), result.stderr assert result.returncode == 2 - assert result.stdout == "\n".join([ - f"bootstrap: Atlas ({project.resolve()})", "", - "(a) legs on tracking branches, pins untouched", - placed(project, "spec"), placed(project, "code"), "", - "(b) neutral validators", ""]) + assert result.stdout == up_to_step_b(project) +@CANNOT_RUN_AS_ROOT +def test_a_validator_at_mode_000_is_refused_before_any_runs(project): + """Its `stat` works and the child cannot open it: the base ran the two + before it, and the child said Python's own `can't open file` line, with + nothing to put it right (RV958's S2). It is refused by name, as + `bump-leg.py` refuses it (#934), and none runs.""" + with at_mode_000(project, "scripts/validate-pins.py"): + result = bootstrap(project) + assert result.stderr == refused( + "validator-unreadable", "scripts/validate-pins.py", + VALIDATOR_READS), result.stderr + assert result.returncode == 2 + assert result.stdout == up_to_step_b(project) + + +@pytest.mark.parametrize("method, failure, advice", [ + pytest.param("stat", None, PERMISSION, id="stat-refused"), + pytest.param("stat", os_error(errno.EIO), OTHER, id="stat-io-error"), + pytest.param("open", os_error(errno.EIO), OTHER, id="open-io-error"), +]) def test_a_validator_whose_stat_is_refused_stops_step_b_before_any_runs( - boot, project, monkeypatch, capsys): + boot, project, monkeypatch, capsys, method, failure, advice): """On every Python, the second validator: the base SKIPPED it and ran - the other two. Nothing is run or said before the refusal.""" + the other two, or, where only its open failed, ran it, and the child + said why. Nothing is run or said before the refusal.""" manifest = boot.load_yaml(project / "project.yaml") - refused_stat(monkeypatch, "validate-manifest.py") + reason = refusing(monkeypatch, "validate-manifest.py", failure, method) with pytest.raises(boot.Refusal) as caught: boot._run_neutral_validators(project, manifest, False) assert f"{caught.value}\n" == refused( "validator-unreadable", "scripts/validate-manifest.py", - VALIDATOR_READS) + VALIDATOR_READS, reason, advice) assert capsys.readouterr().out == "" +@pytest.mark.parametrize("in_its_place", [False, True], + ids=["taken-away", "a-directory-in-its-place"]) def test_a_validator_that_is_not_there_is_still_skipped(boot, project, - monkeypatch, capsys): + monkeypatch, capsys, + in_its_place): """THE OTHER HALF, which the base answered the same. The validators are stood in for: what is asked is which of them step (b) runs, and a - pinned copy taken away is `validate-pins.py`'s own finding.""" + pinned copy taken away is `validate-pins.py`'s own finding. A directory + is no validator, and is not opened.""" monkeypatch.delenv("SHAPE_UPSTREAM_PATH", raising=False) ran = [] monkeypatch.setattr(boot, "_run", lambda root, argv: ran.append(argv[0]) or 0) manifest = boot.load_yaml(project / "project.yaml") - (project / "scripts" / "validate-repository-naming.py").unlink() + naming = project / "scripts" / "validate-repository-naming.py" + naming.unlink() + if in_its_place: + naming.mkdir() assert boot._run_neutral_validators(project, manifest, False) == [] assert ran == ["scripts/validate-manifest.py", "scripts/validate-pins.py"] assert capsys.readouterr().out == ( @@ -413,17 +578,67 @@ def test_a_leg_bootstrap_cannot_search_is_left_as_it_is(project): ]) def test_a_leg_whose_stat_is_refused_is_not_uninitialized( boot, project, monkeypatch, capsys, failure, advice): - if failure is None: - refused_stat(monkeypatch, ".git") - else: - failing(monkeypatch, "stat", {".git": failure}) + reason = refusing(monkeypatch, ".git", failure) boot.checkout_tracking_branch(project.resolve(), {"role": "spec", "path": "spec"}, "main") - reason = "Permission denied" if failure is None else failure.strerror assert capsys.readouterr().out.splitlines() == left_as_it_is( "spec/.git", reason, advice) +@CANNOT_RUN_AS_ROOT +@pytest.mark.parametrize("outside", [False, True], + ids=["inside-the-root", "outside-the-root"]) +def test_a_leg_in_a_folder_nobody_may_search_is_named_and_spelled(project, + outside): + """The code leg's `path:` holds an ESC and is in a folder at mode 000, + inside the root, or outside it as an absolute path, which the gate finds + (`manifest-leg-path`). Each line spells the name (#391); one outside the + root is named whole, where the base's step (c) ended in a `ValueError` + traceback from `relative_to` (RV958's M1 and S3); and step (c) does not + say that a register is there (RV958's N1).""" + folder = (project.parent if outside else project) / "loc\x1bked" + folder.mkdir() + shown = leg_path(project, "code", (folder / "code").as_posix() if outside + else "loc\x1bked/code") + with mode_of(folder.parent, folder.name, 0): + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [placed(project, "spec"), + *left_as_it_is(f"{shown}/.git", where=f"[code] {shown}")], + [*nothing_claimed(f"{shown}/{REGISTER}"), GRANTS]), result.stdout + assert result.returncode == 0, result.stderr + + +def test_a_register_under_a_leg_path_outside_the_root_is_read(project): + """A register this user can read, under a leg whose `path:` is + absolute: the base's step (c) ended in the same `ValueError` traceback + (RV958's M1). It is named whole, and its row is read.""" + spec = project.parent / "elsewhere" / "spec" + (spec / REGISTER).parent.mkdir(parents=True) + (spec / REGISTER).write_text(ROWS) + shown = leg_path(project, "spec", spec.as_posix()) + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [not_initialized(shown), placed(project, "code")], + [f" register: {spec / REGISTER}", + " agent:x \u00b7 review \u00b7 testorg/Atlas-code \u00b7 active " + "\u00b7 expires -", GRANTS]), result.stdout + assert result.returncode == 0, result.stderr + + +def test_a_leg_path_holding_a_nul_is_not_initialized(project): + """Passes on the base too, and is here to keep it: no file name holds a + NUL, so no leg is there, as `exists()` answered, and step (c) finds no + register under it. `_is_a` without its `ValueError` clause ended the run + in `ValueError: embedded null byte` (RV958's S3).""" + shown = leg_path(project, "spec", "sp\x00ec") + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [not_initialized(shown), placed(project, "code")], + DEGRADE), result.stdout + assert result.returncode == 0, result.stderr + + @CANNOT_RUN_AS_ROOT def test_an_operation_bootstrap_cannot_see_keeps_the_leg_where_it_is( project): @@ -466,35 +681,73 @@ def test_a_marker_whose_lstat_is_refused_keeps_the_leg_on_every_python( # --- bootstrap.py: a register ------------------------------------------------ +def a_register(root: Path) -> Path: + """A register in the root, holding `ROWS`.""" + register = root / REGISTER + register.parent.mkdir(parents=True) + register.write_text(ROWS) + return register + + @CANNOT_RUN_AS_ROOT -@pytest.mark.parametrize("locked", ["governance", REGISTER], - ids=["a-folder-nobody-may-enter", "at-mode-000"]) +@pytest.mark.parametrize("locked, label", [ + pytest.param("governance", MAYBE, id="a-folder-nobody-may-enter"), + pytest.param(REGISTER, "register", id="at-mode-000"), +]) def test_a_register_bootstrap_cannot_read_is_named_not_absent(project, - locked): + locked, label): """The base's 3.12 ended in a traceback in step (c), and its 3.14 said `authority is not wallet-carried in this org`; at mode 000 it named the absolute path. Step (c) names it, claims nothing from it, and the run - goes on as it does for a register that does not parse.""" - register = project / REGISTER - register.parent.mkdir(parents=True) - register.write_text("rows:\n - holder: agent:x\n act: review\n" - " target_repo: testorg/Atlas-code\n") + goes on as it does for a register that does not parse. Only where its + `stat` worked does the line say that a register is there.""" + a_register(project) with at_mode_000(project, locked): result = bootstrap(project, "--skip-validators") assert result.stdout == skipping_validators( project, [placed(project, "spec"), placed(project, "code")], - [*nothing_claimed(REGISTER), GRANTS]), result.stdout + [*nothing_claimed(REGISTER, label), GRANTS]), result.stdout assert result.returncode == 0 +@pytest.mark.parametrize("failure", [None, os_error(errno.EIO)], + ids=["permission", "io-error"]) def test_a_register_whose_stat_is_refused_is_not_dropped(boot, project, - monkeypatch): + monkeypatch, + failure): """On every Python: 3.14's `is_file()` dropped it, and step (c) said the - org has no wallet-carried authority.""" - refused_stat(monkeypatch, "register.yaml") + org has no wallet-carried authority. Nor is one dropped for an I/O + error.""" + refusing(monkeypatch, "register.yaml", failure) legs = [{"role": "spec", "path": "spec"}] assert boot._register_paths(project, legs) == [ - project / REGISTER, project / "spec" / REGISTER] + (project / REGISTER, MAYBE), (project / "spec" / REGISTER, MAYBE)] + + +@pytest.mark.parametrize("directory", [False, True], + ids=["a-file", "a-directory"]) +def test_only_a_file_is_a_register_where_the_stat_works(boot, project, + directory): + """THE OTHER HALF, which the base answered the same: a file is a + register, and a directory at its name is none.""" + if directory: + (project / REGISTER).mkdir(parents=True) + else: + a_register(project) + assert boot._register_paths(project, []) == ( + [] if directory else [(project / REGISTER, "register")]) + + +def test_a_register_that_cannot_be_opened_claims_nothing_on_every_python( + boot, project, monkeypatch, capsys): + """In-process, as root and on every platform: an I/O error from the + open, which is no permission and is not told to `chmod`.""" + a_register(project) + failing(monkeypatch, "read_text", {"register.yaml": os_error(errno.EIO)}) + boot.read_authority(project, []) + assert capsys.readouterr().out.splitlines() == [ + *nothing_claimed(REGISTER, "register", os.strerror(errno.EIO), OTHER), + GRANTS] # --- the one errno set (slice Z holds every copy to it) ---------------------- @@ -546,3 +799,27 @@ def test_every_other_refusal_is_raised(copy_of_is_a, monkeypatch, tmp_path, with pytest.raises(OSError) as caught: copy_of_is_a(path, stat.S_ISREG) assert caught.value is error + + +@pytest.mark.parametrize("name", ["a\x00b", "a\ud83db"], + ids=["a-nul", "a-lone-surrogate"]) +def test_a_name_no_file_can_have_is_absent(copy_of_is_a, tmp_path, name): + """A NUL, or a lone surrogate the file system's encoding cannot write + (where it has one): no file has such a name, so nothing is there, as + `is_file()` answered. Without the `ValueError` clause the run ended in + its traceback (RV958's S3).""" + assert copy_of_is_a(tmp_path / name, stat.S_ISREG) is False + + +@pytest.mark.parametrize("make, link", [ + pytest.param(Path.mkdir, False, id="a-directory"), + pytest.param(lambda path: path.symlink_to("nowhere"), True, + id="a-link-to-nothing-asked-of-the-name", marks=POSIX_LINKS), +]) +def test_anything_at_a_name_in_a_leg_is_found(boot, tmp_path, make, link): + """A leg's `.git` that is a directory (a leg cloned on its own) is + there, and so is an operation's marker that is a link to nothing, as + `os.path.lexists` answered.""" + asked = tmp_path / "asked" + make(asked) + assert boot._found(asked, link=link) is True From 7e87e05fa832215ab852ee8a87db7f36dc9dd6da Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 19 Oct 2026 20:43:24 +0000 Subject: [PATCH 5/7] tests/test_adopt_reencoded_files.py: the copies-refused case asks git under `*.py working-tree-encoding=UTF-16`, which refuses every pinned copy whatever its size, not UTF-16LE-BOM, which refuses only an odd byte count (#958) CI's `tests` job failed at cf8d5667 (run 39268984740) on `test_check_finds_what_the_split_would_store_otherwise[copies-refused-by-an-encoding]`: `check` said the source's rules store the 5 pinned shape copies as other bytes, where the case expected git to refuse them. Git refuses `working-tree-encoding=UTF-16LE-BOM` only for a file of an odd byte count; an even one it decodes as UTF-16LE and stores as other bytes (measured with git 2.43.0 and 2.47.1). Main's copies include two of odd size, `validate-manifest.py` and `bootstrap.py`; this branch made both even, so all five are even and the case drew `check`'s other finding. `check` is right both ways; the case's expectation turned on the copies' sizes. `UTF-16` with no byte order requires a BOM, which no pinned copy has, so git refuses every copy at any size: the case now pins that refusal and passes on main's copies and on this branch's. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_reencoded_files.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_adopt_reencoded_files.py b/tests/test_adopt_reencoded_files.py index 899a9aeb..8735534b 100644 --- a/tests/test_adopt_reencoded_files.py +++ b/tests/test_adopt_reencoded_files.py @@ -57,7 +57,12 @@ CRLF = b"one\r\ntwo\r\n" TEXT_EOL_LF = b"* text eol=lf\n" TXT_ENCODING = b"*.txt working-tree-encoding=UTF-16LE\n" -PY_UTF16_BOM = b"*.py working-tree-encoding=UTF-16LE-BOM\n" +#: An encoding git refuses for every pinned copy, whatever its size: `UTF-16` +#: with no byte order needs a BOM, and no copy has one. `UTF-16LE-BOM`, the +#: rule #530 measured, refuses only a file of an odd byte count and stores +#: an even one as other bytes, so which finding the copies drew turned on +#: their sizes: all five even (#958) made it `COPIES_GROUP`'s. +PY_UTF16 = b"*.py working-tree-encoding=UTF-16\n" NOT_UTF8 = b"caf\xe9\n" EXPANDED_ID = b"$Id: deadbeef $\n" #: A filter this machine defines: its clean step uppercases. @@ -157,8 +162,10 @@ def encoded_attributes() -> dict: pytest.param({ROOT_ATTRIBUTES: f"/scripts/*.py filter={UPPER}\n".encode()}, {}, UPPER_CLEAN, [(COPIES_GROUP, "5 pinned shape copies")], id="copies-alone-under-a-filter"), - pytest.param({ROOT_ATTRIBUTES: PY_UTF16_BOM}, {}, (), - [(REFUSED_GROUP, "from UTF-16LE-BOM to UTF-8`)")], + pytest.param({ROOT_ATTRIBUTES: PY_UTF16}, {}, (), + [(REFUSED_GROUP, "pinned shape copies the split writes " + "(git said `fatal: BOM is required in " + "'scripts/")], id="copies-refused-by-an-encoding"), pytest.param({ROOT_ATTRIBUTES: b"* working-tree-encoding=UTF-16LE\n" b".gitattributes -working-tree-encoding\n"}, From 2a2bc8f9f55f173817edfd6a9953a66c343b5811 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 19 Oct 2026 22:51:35 +0000 Subject: [PATCH 6/7] bootstrap.py: the NOT INITIALIZED line names a leg's path as one shell word after a `--`, and the Windows stdout check asks the UTF-8 git is handed, so `--init my spec` is not two pathspecs and `=spec` is not expanded by zsh (#961) #973 (#961) left one line of rv932's list for #958 (#430 slice D), which moves it into `_initialized`: `git submodule update --init {spelled(path)}`, the command the NOT INITIALIZED line gives a person to paste. This commit is that line, on #958's head (7e87e05f), and takes rv973's three NITs. - The path is `_paste_word(path)`, after a `--`. `my spec` was two pathspecs, `=spec` was expanded by zsh, a byte that is not UTF-8 was its escape text, which names no path, and `-spec` was read as an option. - `_handed_to_git` is the bytes the stdout is compared with: `os.fsencode`'s on POSIX, UTF-8 on Windows, where git is handed UTF-16 and stores UTF-8. Under PYTHONLEGACYWINDOWSFSENCODING=1 `os.fsencode` is the code page's, and a UTF-8 stdout got the printf word of those bytes for `origin/caf` + e-acute, where the quoted word was right (rv973's NIT 1). - The module paragraph lists the names `os.fsencode` cannot write among those with no word (NIT 3), and the new command among the ones it prints. - A unit row pins a name the filesystem encoding cannot write, which kills the mutant that asks the comparison outside its `try` (NIT 2). - Tests: `not_initialized` in `tests/test_assembly_root_unreadable_paths.py` takes the word, and six rows run bootstrap with a path that needs quoting. `tests/test_bootstrap_spelling.py`'s NOT INITIALIZED row asks for the word. The pasteable file empties a leg (`git submodule deinit`) and pastes the printed command into a shell. A pinned copy: every project's `bootstrap.py` takes an `upstream-changed` row, and re-pins it. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- templates/assembly-root/scripts/bootstrap.py | 50 ++++-- tests/test_assembly_root_unreadable_paths.py | 45 ++++- tests/test_bootstrap_pasteable_lines.py | 174 ++++++++++++++++++- tests/test_bootstrap_spelling.py | 21 ++- 4 files changed, 256 insertions(+), 34 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index 4e6d5d28..d7b350b5 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -67,16 +67,19 @@ too, the line gives git's words. Every name git prints back is read as bytes and decoded so that it reaches git again as the bytes it was (#710). The commands it prints for a person to paste (the - `merge --ff-only` that brings a moved-back branch forward, and the - `--continue` and `--abort` of an operation in progress) name the - leg's path and the upstream as ONE shell word each: quoted (a word - that begins with `=` too, which zsh would expand), and, where the - name holds a byte that is not UTF-8, a character a terminal must - not be handed, or a character this stdout cannot write, or writes - as other bytes than git is handed, a `printf` that prints those - bytes again, so a paste names what is on disk and not its escape - (#932, #961). A name no `printf` can print, one that ends in a line - feed or holds a NUL, is printed spelled, as it was. Advancing a pin + `merge --ff-only` that brings a moved-back branch forward, the + `--continue` and `--abort` of an operation in progress, and the + `git submodule update --init --` of a leg that is not there) name + the leg's path, and the upstream where there is one, as ONE shell + word each: quoted (a word that begins with `=` too, which zsh would + expand), and, where the name holds a byte that is not UTF-8, a + character a terminal must not be handed, or a character this stdout + cannot write, or writes as other bytes than git is handed, a + `printf` that prints those bytes again, so a paste names what is on + disk and not its escape (#932, #961). A name no `printf` can print, + one that ends in a line feed, holds a NUL or has a character + `os.fsencode` cannot write, is printed spelled, as it was. + Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -532,7 +535,12 @@ def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: one, or `_say_unreadable` where the `stat` of its `.git` is refused (#430). `Path.exists()` raised that refusal out of the run up to Python 3.13, a traceback, and from 3.14 answered False, so an initialized leg - in a mount this user cannot search was NOT INITIALIZED.""" + in a mount this user cannot search was NOT INITIALIZED. + + The command is for a person to paste, so the leg's path is ONE word of + it, as `_paste_word` words it, after a `--`: `my spec` was two + pathspecs, `=spec` was expanded by zsh, a byte that is not UTF-8 was + its escape text and `-spec` was an option (#961).""" try: there = _found(sub / ".git") except _Unreadable as unread: @@ -540,7 +548,7 @@ def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: return False if not there: print(f" {where}: NOT INITIALIZED. Run " - f"`git submodule update --init {spelled(path)}` — the " + f"`git submodule update --init -- {_paste_word(path)}` — the " f"validators below will refuse until you do.") return there @@ -1223,18 +1231,30 @@ def _sequence_in_progress(gitdir: Path) -> tuple[str, str, str] | None: return SEQUENCER_TODO, "a cherry-pick", "cherry-pick" +def _handed_to_git(text: str) -> bytes: + """The bytes git is handed for `text` as a name (#961): `os.fsencode`'s + on POSIX, which is what `subprocess` passes. On Windows git is handed + UTF-16 and stores UTF-8, which is what Git Bash reads a paste as, so + they are UTF-8 there, and not what `os.fsencode` makes of the text + under `PYTHONLEGACYWINDOWSFSENCODING=1` (`mbcs`, a character the code + page lacks replaced).""" + if os.name == "nt": + return text.encode("utf-8", "surrogatepass") + return os.fsencode(text) + + def _prints_as_it_is(text: str) -> bool: """Whether `print` writes `text` as it is, and a paste of that gives git the bytes it is handed (#932, #961): `spelled` has no escape to make in it, this stdout holds every character of it and, where the text is not - ASCII, writes the same bytes as `os.fsencode`, so what the terminal - shows is the text and a paste of it names the same.""" + ASCII, writes the bytes git is handed (`_handed_to_git`), so what the + terminal shows is the text and a paste of it names the same.""" if spelled(text) != text: return False encoding = getattr(sys.stdout, "encoding", None) or "utf-8" try: written = text.encode(encoding) - return text.isascii() or written == os.fsencode(text) + return text.isascii() or written == _handed_to_git(text) except (LookupError, UnicodeEncodeError): return False diff --git a/tests/test_assembly_root_unreadable_paths.py b/tests/test_assembly_root_unreadable_paths.py index 16581ca8..170ad10c 100644 --- a/tests/test_assembly_root_unreadable_paths.py +++ b/tests/test_assembly_root_unreadable_paths.py @@ -74,6 +74,9 @@ from conftest import (NO_BYTECODE, REPO, clear_ambient_pin_sources, git, load_without_bytecode, run_script) +from test_bootstrap_non_utf8_branch import ( + NEEDS_A_UTF8_PROCESS, needs_the_byte, +) from test_bump_leg_unopenable import refused_stat from test_manifest_validator import as_a_descendant_by_chain, link_tree from test_validate_pins_named_and_spelled import ( @@ -152,11 +155,14 @@ def nothing_claimed(named: str, label: str = MAYBE, f"be read ({reason})", f" {advice}{AGAIN}"] -def not_initialized(shown: str) -> str: - """Step (a)'s line for a spec leg at `shown` that is not there.""" +def not_initialized(shown: str, word: str | None = None) -> str: + """Step (a)'s line for a spec leg at `shown` that is not there. The + command that makes it one holds the path as ONE shell word, `word` + (`shown` itself where the name needs no quoting), after a `--`, which + keeps a path that begins with `-` from being read as an option (#961).""" return (f" [spec] {shown}: NOT INITIALIZED. Run `git submodule update " - f"--init {shown}` {DASH} the validators below will refuse until " - "you do.") + f"--init -- {shown if word is None else word}` {DASH} the " + "validators below will refuse until you do.") def refusing(monkeypatch, name: str, failure: OSError | None, @@ -634,7 +640,36 @@ def test_a_leg_path_holding_a_nul_is_not_initialized(project): shown = leg_path(project, "spec", "sp\x00ec") result = bootstrap(project, "--skip-validators") assert result.stdout == skipping_validators( - project, [not_initialized(shown), placed(project, "code")], + project, [not_initialized(shown, f"'{shown}'"), + placed(project, "code")], + DEGRADE), result.stdout + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("typed, word", [ + pytest.param("my spec", "'my spec'", id="space"), + pytest.param("=spec", "'=spec'", id="leading-equals"), + pytest.param("-spec", "-spec", id="leading-dash"), + pytest.param("sp\x1bec", r'''"$(printf -- 'sp\033ec')"''', id="escape"), + pytest.param("sp\u2028ec", r'''"$(printf -- 'sp\342\200\250ec')"''', + id="line-separator", marks=NEEDS_A_UTF8_PROCESS), + pytest.param("sp\udcffec", r'''"$(printf -- 'sp\377ec')"''', + id="byte-0xff"), +]) +def test_the_command_that_initializes_a_leg_holds_its_path_as_one_word( + project, tmp_path, typed, word): + """THE ISSUE (#961, rv932's SHOULD 3): the line printed + `--init my spec` (two pathspecs), `--init =spec` (zsh expands it), + `--init sp\\x1bec` and `--init sp\\udcffec` (the escape text, which + names no path) and `--init -spec` (an option). The path is one word, + `_paste_word`'s, after a `--`; `tests/test_bootstrap_pasteable_lines.py` + pastes each into a shell.""" + if "\udcff" in typed: + needs_the_byte(tmp_path) + shown = leg_path(project, "spec", typed) + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [not_initialized(shown, word), placed(project, "code")], DEGRADE), result.stdout assert result.returncode == 0, result.stderr diff --git a/tests/test_bootstrap_pasteable_lines.py b/tests/test_bootstrap_pasteable_lines.py index cf34946e..5edda90c 100644 --- a/tests/test_bootstrap_pasteable_lines.py +++ b/tests/test_bootstrap_pasteable_lines.py @@ -4,7 +4,9 @@ `move_back_to_pin` prints `git -C merge --ff-only `, and `_say_in_progress` a leg's `git -C --continue` and `--abort`. -Both went through `spelled(shlex.quote(...))`. `spelled` writes a byte that +Both went through `spelled(shlex.quote(...))`, and `_initialized` (#961) put +`spelled(path)` in `git submodule update --init `, for a leg that is not +there, with no quote and no `--`. `spelled` writes a byte that is not UTF-8 as #391's `\\udcXX` text, and a control character, a line separator and a bidirectional control as the escape Python spells it, and a shell reads `'origin/a\\udcffb'` as the characters it is. RV824 measured it @@ -20,7 +22,10 @@ turns back into the bytes `os.fsencode` gives the name. A name that has no such word (one that ends in a line feed, which `$(...)` strips, one with a NUL, which it drops, and one `os.fsencode` cannot write) keeps what -`spelled` made of it. +`spelled` made of it. The leg's path in `git submodule update --init -- ` +is such a word too, after a `--`, which keeps a path that begins with `-` +from being read as an option. The bytes git is handed are `os.fsencode`'s on +POSIX and UTF-8 on Windows, where git is handed UTF-16 and stores UTF-8. THE TESTS THAT TAKE A `root` RUN WHAT WAS PRINTED, AS PRINTED: the command in the line, through a shell's `-c` from the root, and ask where the branch @@ -105,6 +110,9 @@ r'''"$(printf -- 'sp\303\251c')"''', id="UTF-8-as-bytes"), # zsh expands a bare `=spec` to the path of a command (#961). pytest.param("=spec", "=spec", "'=spec'", id="leading-equals"), + pytest.param("my spec", "my spec", "'my spec'", id="space"), + # `git -C -spec` takes the path; a `--init -spec` is an option (#961). + pytest.param("-spec", "-spec", "-spec", id="leading-dash-ASCII"), ] SHELLS = [pytest.param(shell, id=shell.rsplit("/", 1)[1]) for shell in ("/bin/sh", "/bin/dash", "/bin/bash", "/bin/zsh")] @@ -136,6 +144,36 @@ def __getattr__(self, attr: str): LATIN1_FS = Latin1Os() +#: The `printf` word of `origin/café` as git is handed it under UTF-8. +CAFE_AS_BYTES = r'''"$(printf -- 'origin/caf\303\251')"''' + + +class WindowsOs: + """`os` on Windows in its default mode: `nt`, and a UTF-8 `fsencode`, + which this system's is. Only `name` differs from this system's `os`.""" + + name = "nt" + + def __getattr__(self, attr: str): + return getattr(os, attr) + + +class LegacyMbcsOs(WindowsOs): + """`os` on Windows under `PYTHONLEGACYWINDOWSFSENCODING=1`: `fsencode` + is the code page's (`cp1252` here) with `replace`, so it is not the bytes + git is handed, which are UTF-8, as `LegacyWindowsOs` is for `fsdecode`.""" + + @staticmethod + def fsencode(name: str) -> bytes: + return name.encode("cp1252", "replace") + + @staticmethod + def fsdecode(raw: bytes) -> str: + return raw.decode("cp1252", "replace") + + +WINDOWS_FS = WindowsOs() +LEGACY_MBCS_FS = LegacyMbcsOs() def pasted(root: Path, command: str, shell: str = "/bin/sh") -> None: @@ -156,6 +194,32 @@ def pasted(root: Path, command: str, shell: str = "/bin/sh") -> None: "replace")) +def emptied_leg_line(root: Path, tmp_path: Path, path: str + ) -> tuple[Path, str]: + """The spec leg mounted at `path` and emptied (`git submodule deinit`), + so that step (a) finds it not there: its directory, and the line step + (a) prints for it (#961).""" + if "\udcff" in path: + needs_the_byte(tmp_path) + leg = mounted_at(root, path) + # Bytes: git's own line names the path, which may be the byte 0xff. + emptied = subprocess.run(["git", "submodule", "deinit", "-f", "--", path], + cwd=str(root), capture_output=True, check=False) + assert emptied.returncode == 0, emptied.stderr.decode("utf-8", "replace") + assert not os.path.lexists(leg / ".git"), "the leg is emptied" + return leg, not_initialized_line(root) + + +def not_initialized_line(root: Path) -> str: + """The one line of step (a) that says a leg is NOT INITIALIZED.""" + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + lines = [line for line in step_a(result.stdout) + if "NOT INITIALIZED" in line] + assert len(lines) == 1, result.stdout + return lines[0] + + def printed_by(shell: str, word: str) -> bytes: """What `shell` prints for `printf %s `: the bytes it reads the word as. Skips where this system has no such shell.""" @@ -319,6 +383,61 @@ def test_the_commands_that_end_an_operation_are_pasteable_whatever_the_path( check=False).returncode == 1, f"the merge is {done}" +# --- the command that initializes a leg that is not there ------------------- + +@pytest.mark.parametrize("path, label, word", [ + row for row in PATHS if not row.values[0].startswith("-")]) +def test_the_initializing_command_is_pasteable_whatever_the_legs_path_is( + root, tmp_path, path, label, word): + """THE ISSUE (#961, rv932's SHOULD 3): `_initialized` printed `git + submodule update --init {spelled(path)}`, bare, which was two pathspecs + for `my spec`, expanded by zsh for `=spec`, an option for `-spec` and the + escape text for a byte that is not UTF-8. The leg is emptied (`git + submodule deinit`) and step (a) says it is not there; the command it + prints, run as printed from the root, puts the leg back at its pin: + its `.git` is there again, and its HEAD is the commit the root's index + records (the move of the leg is staged, not committed). A path that + begins with `-` is the next test's, as git holds no submodule there.""" + leg, line = emptied_leg_line(root, tmp_path, path) + + assert line == (f" [spec] {label}: NOT INITIALIZED. Run `git submodule " + f"update --init -- {word}` \u2014 the validators below " + "will refuse until you do."), line + pasted(root, command_of(line)) + assert os.path.lexists(leg / ".git"), "the leg is initialized again" + pin = git("rev-parse", f":{path}", cwd=root).stdout.strip() + assert rev(leg) == pin + + +def test_the_initializing_command_reads_a_path_that_begins_with_a_dash( + root): + """Git ignores a `submodule..path` that begins with `-` (it warns + that it may be read as an option), so no command can initialize a leg + there. What `--` changes is what git says of it: that it found no + submodule at the PATH `-spec` (in words that differ by version), where + `--init -spec` is an option (#961).""" + leg = mounted_at(root, "-spec") + (leg / ".git").unlink() + line = not_initialized_line(root) + + assert command_of(line) == "git submodule update --init -- -spec", line + proc = subprocess.run(["/bin/sh", "-c", command_of(line)], cwd=str(root), + capture_output=True, check=False) + assert b"path '-spec'" in proc.stderr, proc.stderr + + +@pytest.mark.parametrize("shell", SHELLS) +def test_the_initializing_command_of_an_equals_sign_path_pastes_in_a_shell( + root, tmp_path, shell): + """A leg at `=spec`: zsh expands the bare word of `--init =spec` to the + path of a command called `spec` and stops, so the word is quoted, and + the command puts the leg back in each shell (#961).""" + leg, line = emptied_leg_line(root, tmp_path, "=spec") + + pasted(root, command_of(line), shell) + assert os.path.lexists(leg / ".git"), "the leg is initialized again" + + # --- every ordinary line is what it was ------------------------------------- @pytest.mark.parametrize("name, word", [ @@ -490,10 +609,8 @@ def test_a_name_the_codec_would_not_read_back_keeps_the_text_it_had( @pytest.mark.parametrize("encoding, filesystem, word", [ pytest.param("utf-8", None, "'origin/café'", id="both-UTF-8"), - pytest.param("latin-1", None, r'''"$(printf -- 'origin/caf\303\251')"''', - id="stdout-latin-1"), - pytest.param("cp1252", None, r'''"$(printf -- 'origin/caf\303\251')"''', - id="stdout-cp1252"), + pytest.param("latin-1", None, CAFE_AS_BYTES, id="stdout-latin-1"), + pytest.param("cp1252", None, CAFE_AS_BYTES, id="stdout-cp1252"), pytest.param("utf-8", LATIN1_FS, r'''"$(printf -- 'origin/caf\351')"''', id="filesystem-latin-1"), pytest.param("latin-1", LATIN1_FS, "'origin/café'", id="both-latin-1"), @@ -529,6 +646,51 @@ def test_an_ascii_name_is_the_word_it_was_on_any_stdout( assert bootstrap_module._paste_word("origin/main") == "origin/main" +@pytest.mark.parametrize("encoding, filesystem, word", [ + pytest.param("utf-8", LEGACY_MBCS_FS, "'origin/café'", + id="legacy-mode-UTF-8-stdout"), + pytest.param("cp1252", WINDOWS_FS, CAFE_AS_BYTES, + id="default-mode-cp1252-stdout"), + pytest.param("utf-8", WINDOWS_FS, "'origin/café'", + id="default-mode-UTF-8-stdout"), +]) +def test_on_windows_the_bytes_git_is_handed_are_utf_8( + bootstrap_module, monkeypatch, encoding, filesystem, word): + """git for Windows is handed UTF-16 and stores UTF-8, and Git Bash reads + a paste as UTF-8, so the stdout is compared with the UTF-8 of the name + and not with `os.fsencode`: under `PYTHONLEGACYWINDOWSFSENCODING=1` that + is `mbcs`, and a UTF-8 stdout got `origin/caf\\351`, a name Git Bash does + not hold, where the quoted word was right (rv973's NIT 1, #961). In + the default mode the two are the same bytes. The `printf` word of a name + a legacy-mode stdout cannot write is still made of `os.fsencode`'s + bytes, and no row here says it is right.""" + monkeypatch.setattr(sys, "stdout", Stdout(encoding)) + monkeypatch.setattr(bootstrap_module, "os", filesystem) + assert bootstrap_module._paste_word("origin/café") == word + + +def test_the_euro_sign_is_quoted_on_a_utf_8_stdout_under_the_legacy_mode( + bootstrap_module, monkeypatch): + """The euro sign is one byte in `cp1252`, 0x80, and three in UTF-8: the + `printf` word of `os.fsencode`'s bytes was `\\2005` for `5`, which + is no name Git Bash holds (rv973's NIT 1, #961).""" + monkeypatch.setattr(sys, "stdout", Stdout("utf-8")) + monkeypatch.setattr(bootstrap_module, "os", LEGACY_MBCS_FS) + assert bootstrap_module._paste_word("\u20ac5") == "'\u20ac5'" + + +def test_a_name_the_filesystem_encoding_cannot_write_keeps_the_text_it_had( + bootstrap_module, monkeypatch): + """`日本` is printable and this stdout writes it, but a latin-1 filesystem + has no bytes for it, so git cannot be handed it: the word is the text it + was, and the comparison of the bytes, asked inside `_prints_as_it_is`'s + `try`, raises nothing (rv973's NIT 2: moved out of it, `os.fsencode` + raised `UnicodeEncodeError`).""" + monkeypatch.setattr(sys, "stdout", Stdout("utf-8")) + monkeypatch.setattr(bootstrap_module, "os", LATIN1_FS) + assert bootstrap_module._paste_word("日本") == "'日本'" + + def test_a_stdout_with_no_such_codec_gets_the_printf_word( bootstrap_module, monkeypatch): """A stream a caller replaced may name any encoding, and `str.encode` diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index ef4ec397..312802d9 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -374,23 +374,28 @@ def test_a_legs_role_is_spelled(project, tmp_path): assert "\x1b" not in out, out -@pytest.mark.parametrize("path, spelling", [ - (r'"sp\x1bec"', r"sp\x1bec"), - (r'"sp\ud83dec"', r"sp\ud83dec"), - (r'"sp\u2028ec"', r"sp\u2028ec"), +@pytest.mark.parametrize("path, spelling, word", [ + (r'"sp\x1bec"', r"sp\x1bec", r'''"$(printf -- 'sp\033ec')"'''), + (r'"sp\ud83dec"', r"sp\ud83dec", r"'sp\ud83dec'"), + (r'"sp\u2028ec"', r"sp\u2028ec", + r'''"$(printf -- 'sp\342\200\250ec')"'''), ]) def test_a_legs_path_is_spelled_where_it_is_not_initialized( - project, tmp_path, path, spelling): + project, tmp_path, path, spelling, word): """A path the manifest holds that is no directory: step (a) says so, and says what to run, before any validator has had its say. A lone surrogate - here raised from the `print`.""" + here raised from the `print`. The line names the leg spelled; the + command to paste holds the path as `_paste_word` words it (#961): the + bytes as a `printf`, and a lone surrogate, which has none, as its + escape text in quotes. The child runs in UTF-8 mode, so that its + filesystem encoding is the one the words are made of.""" edit(project, "project.yaml", ' path: "spec"', f" path: {path}") code, out, err = run(bootstrap_of(project), "--skip-validators", encoding="utf-8:strict", tmp_path=tmp_path, - cwd=project) + cwd=project, changes={"PYTHONUTF8": "1"}) assert code == 0 and "Traceback" not in err, out + err assert (f" [spec] {spelling}: NOT INITIALIZED. Run `git submodule update " - f"--init {spelling}` {DASH} the validators below will refuse " + f"--init -- {word}` {DASH} the validators below will refuse " "until you do.") in out.split("\n"), out assert not {"\x1b", "\u2028"} & set(out), out From e23137c7a34a33dc7850627b321c065b5eb0341d Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Tue, 20 Oct 2026 00:16:19 +0000 Subject: [PATCH 7/7] bootstrap.py: the NOT INITIALIZED command puts `./` before a path that begins with `:`, and on Windows the `printf` word is made of UTF-8, so `--init -- :spec` names the leg and a legacy code page prints no other path (#961) rv973's re-check of 2a2bc8f9 (#961 staged on #958) found it READY with four NITs, and this takes them. - NIT 1. git reads a word that begins with `:` as pathspec magic, even after the `--`, so `--init -- :spec` matched nothing and the leg stayed empty, in every shell and on git 2.43 and 2.47.1. `_initialized` words `./:spec` instead: `./` goes before the path, and the whole is the one word. - NIT 2. `_paste_word` made its bytes with `os.fsencode`, which under PYTHONLEGACYWINDOWSFSENCODING=1 is the code page's, so a cp1252 stdout got the `printf` word of `\351` for `origin/caf` + e-acute: another path to Git Bash, which reads a paste as UTF-8. It takes the bytes from `_handed_to_git` now, which is strict UTF-8 on Windows, and the read-back of `os.fsdecode` is gone: a lone surrogate has no UTF-8 bytes, so it keeps its text, as the read-back kept it. - NIT 3. A path with a space at either end gets a row, which kills `_paste_word(path.strip())`. The old composite assert in the spelling test is two asserts. Tests: the pasteable file pastes `:spec` through sh, dash, bash and zsh and re-initializes the leg, and checks the Windows words; the unreadable-paths file has rows for a `:`, for a space at either end, and for `./` inside the one word. A pinned copy: every project's `bootstrap.py` takes an `upstream-changed` row, and re-pins it. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- templates/assembly-root/scripts/bootstrap.py | 49 ++++++----- tests/test_assembly_root_unreadable_paths.py | 9 +++ tests/test_bootstrap_pasteable_lines.py | 85 ++++++++++++++++---- tests/test_bootstrap_spelling.py | 3 +- 4 files changed, 107 insertions(+), 39 deletions(-) diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index d7b350b5..4c7da9bd 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -76,10 +76,12 @@ character a terminal must not be handed, or a character this stdout cannot write, or writes as other bytes than git is handed, a `printf` that prints those bytes again, so a paste names what is on - disk and not its escape (#932, #961). A name no `printf` can print, - one that ends in a line feed, holds a NUL or has a character - `os.fsencode` cannot write, is printed spelled, as it was. - Advancing a pin + disk and not its escape (#932, #961). The `--init` of a path that + begins with `:`, which git reads as pathspec magic even after the + `--`, names `./:spec`. A name no `printf` can print, one that ends + in a line feed, holds a NUL or has a character git cannot be handed + (a lone surrogate, or one `os.fsencode` cannot write), is printed + spelled, as it was. Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -540,15 +542,18 @@ def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: The command is for a person to paste, so the leg's path is ONE word of it, as `_paste_word` words it, after a `--`: `my spec` was two pathspecs, `=spec` was expanded by zsh, a byte that is not UTF-8 was - its escape text and `-spec` was an option (#961).""" + its escape text and `-spec` was an option. The path is a pathspec, so + one that begins with `:` is read as magic even after the `--` + (`:spec` matched nothing): `./` before it makes it the path (#961).""" try: there = _found(sub / ".git") except _Unreadable as unread: _say_unreadable(where, root, unread) return False if not there: + word = _paste_word(f"./{path}" if path.startswith(":") else path) print(f" {where}: NOT INITIALIZED. Run " - f"`git submodule update --init -- {_paste_word(path)}` — the " + f"`git submodule update --init -- {word}` — the " f"validators below will refuse until you do.") return there @@ -1237,9 +1242,11 @@ def _handed_to_git(text: str) -> bytes: UTF-16 and stores UTF-8, which is what Git Bash reads a paste as, so they are UTF-8 there, and not what `os.fsencode` makes of the text under `PYTHONLEGACYWINDOWSFSENCODING=1` (`mbcs`, a character the code - page lacks replaced).""" + page lacks replaced). A name git cannot be handed raises + `UnicodeEncodeError`: a lone surrogate on Windows, which UTF-8 has no + bytes for, and on POSIX one the filesystem encoding cannot write.""" if os.name == "nt": - return text.encode("utf-8", "surrogatepass") + return text.encode("utf-8") return os.fsencode(text) @@ -1282,8 +1289,8 @@ def _paste_word(name: str) -> str: something we can merge`. A path with a tab, an ESC or a U+2028 in it, or one this stdout cannot write, was printed as its escape too. - ANY OTHER NAME is written as the bytes `os.fsencode` gives it, in a - command substitution that prints them again: + ANY OTHER NAME is written as the bytes git is handed (`_handed_to_git`), + in a command substitution that prints them again: `"$(printf -- 'origin/a\\377b')"`. Each byte outside `PRINTF_BARE` is a three-digit octal escape, which the `printf` of dash, bash and zsh reads, and a dash before 0.5.13 has no `$'...'` for. `--` keeps a name @@ -1293,18 +1300,17 @@ def _paste_word(name: str) -> str: `shlex.quote` already was. A NAME WITH NO SUCH FORM keeps `spelled`'s text, as it was: - - one `os.fsencode` cannot write, a lone surrogate with no byte or a - character the filesystem encoding lacks; + - one git cannot be handed, which `_handed_to_git` raises on: a lone + surrogate with no byte, or on POSIX a character the filesystem + encoding lacks; - one whose bytes `$(...)` does not give back, so the word would name - another path: it strips a trailing line feed and drops a NUL; - - on Windows, one whose bytes do not read back as it (`os.fsdecode` of - them is another name: a code page that replaces). A command is not - handed these bytes there, so the read-back is what is asked. + another path: it strips a trailing line feed and drops a NUL. - On POSIX the read-back is not asked (#961): git IS handed these bytes, - so a name `os.fsdecode` would read as another has its word, as + The bytes are not read back as the name (#961): git IS handed them, so + a name `os.fsdecode` would read as another has its word, as `sp\\udcc3\\udca9c` has `"$(printf -- 'sp\\303\\251c')"`, the bytes of the - directory `spéc`. A ref's name is none of the three: git refuses a + directory `spéc`. On Windows they are UTF-8 whatever the code page, + which Git Bash reads a paste as. A ref's name is neither: git refuses a control character in one, and `_try_git` reads its bytes back. So it is a leg's path, and one with no bytes is no directory a leg is moved in; a path that ends in a line feed, which only a hand-made manifest holds, @@ -1321,11 +1327,10 @@ def _paste_word(name: str) -> str: if _prints_as_it_is(quoted): return quoted try: - raw = os.fsencode(name) + raw = _handed_to_git(name) except UnicodeEncodeError: return spelled(quoted) - if raw.endswith(b"\n") or b"\0" in raw or ( - os.name == "nt" and os.fsdecode(raw) != name): + if raw.endswith(b"\n") or b"\0" in raw: return spelled(quoted) escaped = "".join(chr(byte) if byte in PRINTF_BARE else f"\\{byte:03o}" for byte in raw) diff --git a/tests/test_assembly_root_unreadable_paths.py b/tests/test_assembly_root_unreadable_paths.py index 170ad10c..02fb1722 100644 --- a/tests/test_assembly_root_unreadable_paths.py +++ b/tests/test_assembly_root_unreadable_paths.py @@ -650,6 +650,15 @@ def test_a_leg_path_holding_a_nul_is_not_initialized(project): pytest.param("my spec", "'my spec'", id="space"), pytest.param("=spec", "'=spec'", id="leading-equals"), pytest.param("-spec", "-spec", id="leading-dash"), + # Git reads a leading `:` as pathspec magic, even after the `--`. + pytest.param(":spec", "./:spec", id="leading-colon"), + # `./` goes before the path, inside the one word: `./':my spec'` is the + # same name to a shell, and is not the word. + pytest.param(":my spec", "'./:my spec'", id="leading-colon-and-space"), + pytest.param(":sp\udcffec", r'''"$(printf -- './:sp\377ec')"''', + id="leading-colon-and-byte-0xff"), + pytest.param("spec ", "'spec '", id="trailing-space"), + pytest.param(" spec", "' spec'", id="leading-space"), pytest.param("sp\x1bec", r'''"$(printf -- 'sp\033ec')"''', id="escape"), pytest.param("sp\u2028ec", r'''"$(printf -- 'sp\342\200\250ec')"''', id="line-separator", marks=NEEDS_A_UTF8_PROCESS), diff --git a/tests/test_bootstrap_pasteable_lines.py b/tests/test_bootstrap_pasteable_lines.py index 5edda90c..ccf8bdb6 100644 --- a/tests/test_bootstrap_pasteable_lines.py +++ b/tests/test_bootstrap_pasteable_lines.py @@ -24,8 +24,11 @@ NUL, which it drops, and one `os.fsencode` cannot write) keeps what `spelled` made of it. The leg's path in `git submodule update --init -- ` is such a word too, after a `--`, which keeps a path that begins with `-` -from being read as an option. The bytes git is handed are `os.fsencode`'s on -POSIX and UTF-8 on Windows, where git is handed UTF-16 and stores UTF-8. +from being read as an option; a path that begins with `:` is `./:spec` there, +as git reads a leading `:` as pathspec magic even after the `--`. The bytes +git is handed are `os.fsencode`'s on POSIX and UTF-8 on Windows, where git is +handed UTF-16 and stores UTF-8, both in the comparison with the stdout and in +the `printf` word. THE TESTS THAT TAKE A `root` RUN WHAT WAS PRINTED, AS PRINTED: the command in the line, through a shell's `-c` from the root, and ask where the branch @@ -111,6 +114,9 @@ # zsh expands a bare `=spec` to the path of a command (#961). pytest.param("=spec", "=spec", "'=spec'", id="leading-equals"), pytest.param("my spec", "my spec", "'my spec'", id="space"), + # A name is not stripped: its spaces are part of the path (rv973's R-H). + pytest.param("spec ", "spec ", "'spec '", id="trailing-space"), + pytest.param(" spec", " spec", "' spec'", id="leading-space"), # `git -C -spec` takes the path; a `--init -spec` is an option (#961). pytest.param("-spec", "-spec", "-spec", id="leading-dash-ASCII"), ] @@ -202,9 +208,13 @@ def emptied_leg_line(root: Path, tmp_path: Path, path: str if "\udcff" in path: needs_the_byte(tmp_path) leg = mounted_at(root, path) - # Bytes: git's own line names the path, which may be the byte 0xff. - emptied = subprocess.run(["git", "submodule", "deinit", "-f", "--", path], - cwd=str(root), capture_output=True, check=False) + # `./` before a path that begins with `:`, which is pathspec magic even + # after the `--`. Bytes: git's own line names the path, which may be the + # byte 0xff. + pathspec = f"./{path}" if path.startswith(":") else path + emptied = subprocess.run( + ["git", "submodule", "deinit", "-f", "--", pathspec], + cwd=str(root), capture_output=True, check=False) assert emptied.returncode == 0, emptied.stderr.decode("utf-8", "replace") assert not os.path.lexists(leg / ".git"), "the leg is emptied" return leg, not_initialized_line(root) @@ -426,6 +436,21 @@ def test_the_initializing_command_reads_a_path_that_begins_with_a_dash( assert b"path '-spec'" in proc.stderr, proc.stderr +@pytest.mark.parametrize("shell", SHELLS) +def test_the_initializing_command_names_a_path_that_begins_with_a_colon( + root, tmp_path, shell): + """Git reads a word that begins with `:` as pathspec magic, even after + the `--`: `--init -- :spec` is `pathspec ':spec' did not match any + file(s) known to git`, and the leg stays empty. `./` before the path + makes it the path, and the printed command puts the leg back in each + shell (rv973's NIT 1, #961).""" + leg, line = emptied_leg_line(root, tmp_path, ":spec") + + assert command_of(line) == "git submodule update --init -- ./:spec", line + pasted(root, command_of(line), shell) + assert os.path.lexists(leg / ".git"), "the leg is initialized again" + + @pytest.mark.parametrize("shell", SHELLS) def test_the_initializing_command_of_an_equals_sign_path_pastes_in_a_shell( root, tmp_path, shell): @@ -594,14 +619,21 @@ def test_a_name_with_no_printf_word_keeps_the_text_it_had( shlex.quote(name)) -def test_a_name_the_codec_would_not_read_back_keeps_the_text_it_had( - bootstrap_module, monkeypatch): - """The word is the bytes `os.fsencode` gives the name, so it is the name - only where `os.fsdecode` gives the name back from them. Under Windows' - legacy code page (`LegacyWindowsOs`: `fsdecode` replaces what the page - lacks) it does not, and a word would name another path: the name keeps - what `spelled(shlex.quote(name))` made of it.""" - monkeypatch.setattr(bootstrap_module, "os", LegacyWindowsOs()) +@pytest.mark.parametrize("filesystem", [ + pytest.param(LegacyWindowsOs(), id="fsdecode-replaces"), + pytest.param(WINDOWS_FS, id="default-mode"), + pytest.param(LEGACY_MBCS_FS, id="legacy-mode"), +]) +def test_on_windows_a_lone_surrogate_has_no_bytes_and_keeps_the_text_it_had( + bootstrap_module, monkeypatch, filesystem): + """Windows hands git UTF-16 and takes UTF-8 back, and UTF-8 has no bytes + for a lone surrogate, so there is nothing for a `printf` to print: the + name keeps what `spelled(shlex.quote(name))` made of it. It was the + read-back of `os.fsdecode` that kept it (`LegacyWindowsOs`, #944's shim: + it replaces what the page lacks), and under the default mode `os.fsencode` + gave a surrogate three junk bytes, which the read-back let through + (rv973's NIT 2, #961).""" + monkeypatch.setattr(bootstrap_module, "os", filesystem) name = "a\udcffb" assert bootstrap_module._paste_word(name) == bootstrap_module.spelled( shlex.quote(name)) @@ -661,14 +693,35 @@ def test_on_windows_the_bytes_git_is_handed_are_utf_8( and not with `os.fsencode`: under `PYTHONLEGACYWINDOWSFSENCODING=1` that is `mbcs`, and a UTF-8 stdout got `origin/caf\\351`, a name Git Bash does not hold, where the quoted word was right (rv973's NIT 1, #961). In - the default mode the two are the same bytes. The `printf` word of a name - a legacy-mode stdout cannot write is still made of `os.fsencode`'s - bytes, and no row here says it is right.""" + the default mode the two are the same bytes.""" monkeypatch.setattr(sys, "stdout", Stdout(encoding)) monkeypatch.setattr(bootstrap_module, "os", filesystem) assert bootstrap_module._paste_word("origin/café") == word +@pytest.mark.parametrize("filesystem, name, word", [ + pytest.param(WINDOWS_FS, "origin/café", CAFE_AS_BYTES, id="default-mode"), + pytest.param(LEGACY_MBCS_FS, "origin/café", CAFE_AS_BYTES, + id="legacy-mode"), + pytest.param(LEGACY_MBCS_FS, "\u20ac5", + r'''"$(printf -- '\342\202\2545')"''', id="legacy-mode-euro"), + pytest.param(LEGACY_MBCS_FS, "日本", + r'''"$(printf -- '\346\227\245\346\234\254')"''', + id="legacy-mode-no-such-character-in-the-code-page"), +]) +def test_on_windows_the_printf_word_is_made_of_utf_8( + bootstrap_module, monkeypatch, filesystem, name, word): + """A stdout that cannot write the name (`ascii`) gets the `printf` word, + and Git Bash reads what it prints as UTF-8: so those are the bytes, and + not what `os.fsencode` makes of the name under + `PYTHONLEGACYWINDOWSFSENCODING=1`, which is `mbcs`: `\\351` for e-acute, + `\\2005` for the euro sign, and `?` for a character the code page + lacks (rv973's NIT 2, #961).""" + monkeypatch.setattr(sys, "stdout", Stdout("ascii")) + monkeypatch.setattr(bootstrap_module, "os", filesystem) + assert bootstrap_module._paste_word(name) == word + + def test_the_euro_sign_is_quoted_on_a_utf_8_stdout_under_the_legacy_mode( bootstrap_module, monkeypatch): """The euro sign is one byte in `cp1252`, 0x80, and three in UTF-8: the diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index 312802d9..a6f43cbc 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -393,7 +393,8 @@ def test_a_legs_path_is_spelled_where_it_is_not_initialized( code, out, err = run(bootstrap_of(project), "--skip-validators", encoding="utf-8:strict", tmp_path=tmp_path, cwd=project, changes={"PYTHONUTF8": "1"}) - assert code == 0 and "Traceback" not in err, out + err + assert code == 0, out + err + assert "Traceback" not in err, out + err assert (f" [spec] {spelling}: NOT INITIALIZED. Run `git submodule update " f"--init -- {word}` {DASH} the validators below will refuse " "until you do.") in out.split("\n"), out