diff --git a/templates/assembly-root/AGENTS-shape.md b/templates/assembly-root/AGENTS-shape.md index cb23566e..f410515d 100644 --- a/templates/assembly-root/AGENTS-shape.md +++ b/templates/assembly-root/AGENTS-shape.md @@ -97,8 +97,8 @@ make park commit, push and RECORD every open feature, so another make resume recreate the parked features here, from that record ``` -`make bootstrap` places each leg on its tracking branch AT the pin where that -loses no commit, and leaves one detached at any other commit, saying why. The +`make bootstrap` places a leg on its tracking branch AT the pin only where that +loses no commit, and otherwise leaves its checkout where it is, saying why. The line `authority is not wallet-carried in this org` is a report, not a fault. **`make park` BEFORE the human leaves a workstation; on the next one, clone, diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index c70284c6..f92f8047 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -28,7 +28,17 @@ branch is at the pin, behind it, ahead of it or not there at all, and whatever the branch or `origin` holds: a commit made there is on no branch, and a checkout would leave it only in the reflog, so bootstrap - says where the leg is instead. A branch no command can be given (a NUL, + says where the leg is instead, and so is one whose HEAD is on a ref + outside `refs/heads/`, where a commit lands on that ref (#704). A leg + with a merge, a cherry-pick, a revert, a rebase or an `am` in progress + is not moved at all: a checkout ends the first three, carrying what + they staged onto the branch, and leaves the last two to go on over + it, as it does a cherry-pick or a revert of several commits whose + stop was committed by hand (its `sequencer/todo` is still there), so + the line names the operation and the commands that finish or abort + it (#704). A move git would make over an IGNORED file at a path + the pin tracks is refused, in git's words, as one over an untracked + file is. A branch no command can be given (a NUL, or a lone surrogate this system cannot encode) is never tried: the leg is left as it is and the line says why (#455); step (b)'s manifest validator names the key when the manifest holds it. Advancing a pin @@ -51,7 +61,9 @@ wallet-carried authority is not misconfigured, and a bootstrap that failed there would have made the layout load-bearing again. - (d) EXITS 0 unless a validator failed. + (d) EXITS 0 unless a validator failed, and 2 where step (a) or the + upstream notice cannot start git, refused `git-not-runnable` as the + validators refuse it (#694). SCHEMA-NEUTRAL BY CONTRACT. A one-repository project runs the same command: step (a) is a no-op where there are no submodules, and steps (c) and (d) are @@ -68,6 +80,8 @@ import shlex import subprocess import sys +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -96,6 +110,27 @@ NOTHING_ELSE = (" nothing else is tried; the recorded pin is " "authoritative. Deal with what git said and run bootstrap " "again.") +#: What a leg's git dir holds while git has an operation in progress there, +#: each with what step (a) calls it and the command that ends it, asked in +#: this order (#704): the two rebases first, as git's own `status` asks +#: them, since a pick a rebase stops on is the rebase's to finish or abort. +IN_PROGRESS = ( + ("rebase-apply/applying", "an `am`", "am"), + ("rebase-apply", "a rebase", "rebase"), + ("rebase-merge", "a rebase", "rebase"), + ("MERGE_HEAD", "a merge", "merge"), + ("CHERRY_PICK_HEAD", "a cherry-pick", "cherry-pick"), + ("REVERT_HEAD", "a revert", "revert"), +) +#: What a cherry-pick or a revert of SEVERAL commits leaves while it is not +#: done, asked after `IN_PROGRESS`: its stop committed by hand takes the +#: head file away and leaves this, whose first word says which it is. +SEQUENCER_TODO = "sequencer/todo" +#: The remedy of the `git-not-runnable` refusal (#694). +GIT_NOT_RUNNABLE = ( + "Remediation: put git on PATH and run this again. Step (a) reads each " + "leg's pin and moves the leg through git, and the validators read the " + "pins through git, so without it nothing here can be placed or checked.") REGISTER_RELPATH = Path("governance") / "review-authority" / "register.yaml" VALIDATORS = ( ("naming", ["scripts/validate-repository-naming.py", "--project", @@ -122,6 +157,32 @@ def _try_git(args: list[str], cwd: Path) -> str | None: return None +@contextmanager +def _git_runs(unread: str) -> Iterator[None]: + """`git-not-runnable`, exit 2, where git could not be STARTED in the + block (#694), with `unread` saying what was being done. + + `git_out` and `_git_says` raise `OSError` then, and nothing here caught + it: `make bootstrap` with no git on PATH ended in a `FileNotFoundError` + traceback, exit 1, which is `bootstrap FAILED`'s code. The code and the + words are the validators'. Which `OSError` is git's is read as + `validate-pins.py` reads it, by NAME: one whose `filename` is `git`, or + a nameless `FileNotFoundError` or `PermissionError`, which is what + Windows raises for a program it cannot start. An error that names a + directory, which `subprocess` raises for a `cwd` it cannot enter, is + not git's, and goes on as it did. + """ + try: + yield + except OSError as exc: + if exc.filename != "git" and (exc.filename is not None or not ( + isinstance(exc, (FileNotFoundError, PermissionError)))): + raise + raise Refusal("git-not-runnable", + f"{unread}: git could not be run " + f"({exc.strerror or exc})", GIT_NOT_RUNNABLE) from exc + + def _argument_fault(branch: str) -> str | None: """What stops `branch` being an argument to a command, or `None` (#455). @@ -206,9 +267,21 @@ def _git_says(args: list[str], cwd: Path) -> list[str] | None: def _head_of(sub: Path) -> tuple[str | None, str | None]: """The commit the leg has checked out, and the ref HEAD is on (`None` - when it is detached).""" + when it is detached). + + The ref is read as BYTES and decoded as this system decodes a name it + is given (`os.fsdecode`), so a branch whose name is not UTF-8 reads + back as the `refs/heads/` built from the same name. `git_out`'s + strict decoding raised `UnicodeDecodeError` over it: a second run with + `--branch` such a name ended in a traceback in `place_at_pin`, and the + check of where `checkout -b` left HEAD would end the first (#704).""" + try: + ref = git_out(["symbolic-ref", "--quiet", "HEAD"], cwd=sub, + binary=True) + except Refusal: + ref = None return (_try_git(["rev-parse", "--verify", "--quiet", "HEAD"], sub), - _try_git(["symbolic-ref", "--quiet", "HEAD"], sub)) + None if ref is None else os.fsdecode(ref.rstrip(b"\r\n"))) def _is_ancestor(sub: Path, commit: str | None, of: str | None) -> bool: @@ -263,8 +336,12 @@ def _check_out(sub: Path, branch: str, post-checkout hook's exit status is the checkout's own, so a hook that fails after HEAD has moved makes git exit non-zero over a checkout that happened, which is not a refusal. + + `--no-overwrite-ignore` (#704): git's default overwrites an IGNORED + file at a path the branch tracks, silently, where it refuses an + untracked one; with it, git refuses both, in the same words. """ - why = _git_says(["checkout", "-q", branch], sub) + why = _git_says(["checkout", "-q", "--no-overwrite-ignore", branch], sub) if _head_of(sub) == (pin, f"refs/heads/{branch}"): return True, why return False, why or [ @@ -279,12 +356,10 @@ def _move_and_check_out(sub: Path, branch: str, From the pin the branch is by then the commit already checked out, so no file in the leg changes, a local edit included. From another branch the checkout changes files, as `place_at_pin`'s does, and git refuses - it where a local change to a file that differs, or an untracked file, is - in the way: the branch has moved by then, and HEAD has not. It does not - refuse an IGNORED file at a path the pin tracks: git overwrites that, - as it does for `place_at_pin` and the on-branch `merge --ff-only`, and - it ends a merge in progress on the branch HEAD leaves (#458's to rule - on, #603). + it where a local change to a file that differs, or an untracked or + IGNORED file (#704), is in the way: the branch has moved by then, and + HEAD has not. A leg with an operation in progress is never brought + here: `checkout_tracking_branch` refuses it first (#704). Returns whether the branch moved, whether HEAD is on it at the pin, and git's lines (`None` when both steps exited 0). The checkout runs only @@ -345,14 +420,16 @@ def _place_of(current: str | None) -> str: def _say_left(head: str | None, current: str, named: str) -> None: - """The line under `fast-forwarded` when the leg started on ANOTHER - branch, `current`, and is on the tracking branch `named` now (#603). + """The line under `fast-forwarded` (#603), or under `on ` or + `created` (#704), when the leg started on ANOTHER branch, `current`, + and is on the tracking branch `named` now. The person is told, since their next commit lands on `named`. P9 - (`place_at_pin`) says nothing of the branch it leaves; that is #458's. - `head` is `None` for a branch with no commit yet, which has no commit to - keep, so it is said so. Its own function, so that `fast_forward_to_pin` - stays within Sonar's cognitive-complexity limit (S3776). + (`place_at_pin`) used to say nothing of the branch it left, where T7 + (`fast_forward_to_pin`) said it for the same move (#704). `head` is + `None` for a branch with no commit yet, which has no commit to keep, so + it is said so. Its own function, so that `fast_forward_to_pin` stays + within Sonar's cognitive-complexity limit (S3776). """ kept = ("a branch with no commit yet" if head is None else "which keeps its commits") @@ -373,14 +450,90 @@ def _say_neither(label: str, state: str, head: str | None, """The checkout is not one a move starts from: `state` says what the branch is to the pin, and the line names where the leg stands instead. Nothing moves. A commit made on a leg detached off the pin is on no - branch, so checking the branch out would leave it only in the reflog. - A leg on another branch is not said here (#603): that branch holds its - commits, so the leg is moved off it.""" + branch, so checking the branch out would leave it only in the reflog, + and one made on a HEAD on a ref outside `refs/heads/` lands on that ref + (#704). A leg on another branch is not said here (#603): that branch + holds its commits, so the leg is moved off it.""" print(f" {label}: {state}, and the checkout is at {_short(head)} " f"({_place_of(current)}), which is neither") _say_keep() +def _on_a_branch(current: str | None) -> bool: + """Whether HEAD is on a branch, `refs/heads/`, one with no commit + yet included: the one checkout besides the pin a move starts from.""" + return current is not None and current.startswith("refs/heads/") + + +def _in_progress(sub: Path) -> tuple[str, str, str] | None: + """The `IN_PROGRESS` row of the operation git has in progress in the + leg, or `None` (#704). + + Asked of the leg's own git dir, which `git rev-parse --git-dir` names + (a submodule's is under the root's `.git/modules/`, a linked + worktree's under the leg's `worktrees/`, and each operation's state is + that worktree's own). Read as bytes and decoded as the system decodes a + path, so no name of a directory can fail the read. A leg git names no + git dir for has nothing to read here, and the move after this says what + git says of it. + """ + try: + said = git_out(["rev-parse", "--git-dir"], cwd=sub, binary=True) + except Refusal: + return None + gitdir = sub / os.fsdecode(said.rstrip(b"\r\n")) + for row in IN_PROGRESS: + if os.path.lexists(gitdir / row[0]): + return row + return _sequence_in_progress(gitdir) + + +def _sequence_in_progress(gitdir: Path) -> tuple[str, str, str] | None: + """A cherry-pick or a revert of several commits that is not done, or + `None` (rv704's S1 on #704). + + `git cherry-pick A B` stops on a conflict in A, the person resolves it + and commits by hand: `CHERRY_PICK_HEAD` is gone, and `git status` still + says `Cherry-pick currently in progress.` and to run `git cherry-pick + --continue`, which picks B onto wherever HEAD is by then. A move onto + the tracking branch put B there. `sequencer/todo` is what is left, and + its first word is `revert` for a revert and `pick` for a cherry-pick; + a file that cannot be read is taken as the cherry-pick, the commoner.""" + todo = gitdir / SEQUENCER_TODO + if not os.path.lexists(todo): + return None + try: + first = todo.read_bytes().split(None, 1)[:1] + except OSError: + first = [] + if first == [b"revert"]: + return SEQUENCER_TODO, "a revert", "revert" + return SEQUENCER_TODO, "a cherry-pick", "cherry-pick" + + +def _say_in_progress(label: str, path: str, + row: tuple[str, str, str]) -> None: + """A leg with an operation in progress (#704): what is in progress, + that nothing moved, and the two commands that end it, the leg's path + shell-quoted, then spelled, as `move_back_to_pin` prints its command. + + No move is made from here. A checkout of the tracking branch, even from + HEAD on it already or detached at the pin, ends a merge, a cherry-pick + or a revert, taking away the head file `--continue` reads and carrying + the staged result onto the branch, and it puts a rebase or an `am` on + the branch, which goes on from there, as a cherry-pick or a revert of + several commits does once its stop is committed by hand (its + `sequencer/todo`); `merge --ff-only` refuses some of them and not + others. It is the operation's own to finish or abort, so every such leg + is refused alike, before any path of step (a) is chosen.""" + marker, doing, verb = row + command = f"git -C {spelled(shlex.quote(path))} {verb}" + print(f" {label}: {doing} is in progress in the leg ({marker} is in its " + "git dir), so nothing moved") + print(f" finish it (`{command} --continue`) or abort it " + f"(`{command} --abort`), then run bootstrap again.") + + def place_at_pin(sub: Path, label: str, branch: str, pin: str) -> None: """Step (a) for a leg whose tracking branch IS the pin: check it out, and say `on ` only when HEAD is on it afterwards. @@ -397,11 +550,16 @@ def place_at_pin(sub: Path, label: str, branch: str, pin: str) -> None: bootstrap, checked the branch out and printed `on at ` over a commit that was then on no branch, held by HEAD's reflog alone. Detached AT the pin, or already on the branch, the checkout is as it was. - On ANOTHER branch it is as it was too: that branch holds its commits. + On ANOTHER branch it is as it was too: that branch holds its commits, + and the line under `on ` now says the leg left it, as T7's does + (#704). A HEAD on a ref outside `refs/heads/` (`git symbolic-ref HEAD + refs/remotes/origin/main`) is left where it is, as T7 leaves it: a + commit made there lands on that ref, and this placed the leg while T7 + said `which is neither` of the same HEAD (#704). """ named = spelled(branch) head, current = _head_of(sub) - if current is None and head != pin: + if not (_on_a_branch(current) or (current is None and head == pin)): _say_neither(label, f"branch {named} is at the pin {_short(pin)}", head, current) return @@ -412,6 +570,8 @@ def place_at_pin(sub: Path, label: str, branch: str, pin: str) -> None: f"{_short(pin)}, and git refused to check it out:", why) return print(f" {label}: on {named} at {_short(pin)} (branch tip == pin)") + if current not in (None, f"refs/heads/{branch}"): + _say_left(head, current, named) _say_after(why, moved=not already) @@ -428,7 +588,8 @@ def fast_forward_to_pin(sub: Path, label: str, branch: str, local: str, - HEAD on the branch itself: `merge --ff-only`, which moves the working tree too, and moves nothing when a local change to a file the move - changes, or an untracked file, is in the way. `merge.autostash` is + changes, or an untracked file, is in the way, or an ignored one + (`--no-overwrite-ignore`, #704). `merge.autostash` is off for it: with it on, git stashes the change, moves the branch, applies the stash back into conflicts and exits 0, and the leg was left unmerged under a line saying it had been fast-forwarded. `-c`, @@ -443,33 +604,36 @@ def fast_forward_to_pin(sub: Path, label: str, branch: str, local: str, this checkout when it is AT the pin; leaving the leg for a branch BEHIND the pin was a difference nobody could explain (#458). The checkout changes files from here, so git can refuse it over a local - edit or an untracked file, and `_move_and_check_out` says what is - left then. A HEAD on a branch with no commit yet takes this path too, - as it does in `place_at_pin`. When the checkout is made, one line - under the `fast-forwarded` one says the leg left that branch, and - that it keeps its commits, since the next commit made in the leg is - on the tracking branch now. NO COMMIT IS LOST, BUT TWO THINGS ARE NOT - REFUSED, as in `place_at_pin`, in the missing-branch path and, for - the first, in the on-branch `merge --ff-only`: git overwrites an - IGNORED file at a path the pin tracks, and a merge in progress on the - other branch is ended, its `MERGE_HEAD` removed and its staged result - carried onto the tracking branch. Both are #458's to rule on. + edit or an untracked or ignored file, and `_move_and_check_out` says + what is left then. A HEAD on a branch with no commit yet takes this + path too, as it does in `place_at_pin`. When the checkout is made, + one line under the `fast-forwarded` one says the leg left that + branch, and that it keeps its commits, since the next commit made in + the leg is on the tracking branch now. NO COMMIT IS LOST, AND NOTHING + ELSE A PERSON HAS (#704): git used to overwrite an IGNORED file at a + path the pin tracks, here, in `place_at_pin`, in the missing-branch + path and in the on-branch `merge --ff-only`, and a checkout ended a + merge in progress on the other branch, its `MERGE_HEAD` removed and + its staged result carried onto the tracking branch. Every move now + refuses the first, and no leg with an operation in progress reaches + here (`_say_in_progress`). A LEG DETACHED OFF THE PIN IS LEFT WHERE IT IS: a commit made on the detached leg after the bump is on no branch, and checking the branch out would leave it only in the reflog. That is said, with what to do, - and nothing moves. + and nothing moves. So is a HEAD on a ref outside `refs/heads/`, which + `place_at_pin` now leaves too (#704). `label` is the leg as `checkout_tracking_branch` prints it, already spelled; the branch names and git's own lines are spelled here (#391). """ named = spelled(branch) head, current = _head_of(sub) - other_branch = (current is not None and current.startswith("refs/heads/") + other_branch = (_on_a_branch(current) and current != f"refs/heads/{branch}") if current == f"refs/heads/{branch}": why = _git_says(["-c", "merge.autostash=false", "merge", "--ff-only", - "-q", pin], sub) + "--no-overwrite-ignore", "-q", pin], sub) moved = placed = why is None elif other_branch or (current is None and head == pin): moved, placed, why = _move_and_check_out(sub, branch, pin) @@ -564,8 +728,77 @@ def _stays(sub: Path, branch: str, pin: str) -> str: return f"at {_short(head)} ({_place_of(current)})" +def create_at_pin(sub: Path, label: str, branch: str, pin: str, + remote: str | None) -> None: + """Step (a) for a leg whose tracking branch IS NOT THERE: `checkout -b + `, from HEAD detached at the pin or on a branch, a branch + with no commit yet included. + + A checkout git refuses is said in git's own words under the line that + says the branch could not be made (#704), with `--no-overwrite-ignore`, + so an IGNORED file at a path the pin tracks is refused as an untracked + one is, where git overwrote it silently. The line used to say `leaving + detached` of a leg on another branch, too; it says where the leg stays. + A HEAD on a ref outside `refs/heads/` is left where it is, as the + branch-at-the-pin and the behind-branch paths leave it (#704). + + The checkout is judged by WHERE HEAD IS AFTERWARDS, as `_check_out` + judges one (#533; rv704's S3): a post-checkout hook that fails after + the branch is made makes git exit non-zero, and the line said `could + not create` over a leg on the branch at the pin. It says `created`, + and git's words follow it (`_say_after`). From another branch, the line + under `created` says the leg left it, as P9's and T7's do (rv704's S2). + """ + named = spelled(branch) + head, current = _head_of(sub) + if head is not None and current is None and head != pin: + # `checkout -b ` from a leg detached anywhere else + # leaves the commit made there on no branch (#533), as placing a + # branch that exists would. The branch is not there to check + # out, so the way out names the pin alone. A HEAD git cannot + # read (the leg's gitdir is gone) is no commit to keep: the + # checkout below fails, and says it could not. + print(f" {label}: branch {named} does not exist, and the " + f"checkout is at {_short(head)} (detached), which is not " + f"the pin {_short(pin)}") + _say_keep("the pin") + return + if current is not None and not _on_a_branch(current): + print(f" {label}: branch {named} does not exist, and the checkout " + f"is at {_short(head)} ({_place_of(current)}), which is not a " + "branch") + _say_keep("the pin") + return + why = _git_says(["checkout", "-q", "--no-overwrite-ignore", "-b", branch, + pin], sub) + if _head_of(sub) != (pin, f"refs/heads/{branch}"): + stays = "detached" if current is None else f"it {_place_of(current)}" + _say_refused(f" {label}: could not create branch {branch!r} at " + f"the pin {_short(pin)}; leaving {stays}", why or [ + "`git checkout -b` exited 0, and HEAD is not on the " + "branch at the pin"]) + return + if remote is not None: + _try_git(["branch", "-q", f"--set-upstream-to=origin/{branch}", + branch], sub) + state = "at the pin" + if remote is not None and remote != pin: + state = (f"at the pin; origin/{named} is {_short(remote)}, which " + f"the pin deliberately does NOT follow") + print(f" {label}: branch {named} created at " + f"{_short(pin)} — {state}") + if current not in (None, f"refs/heads/{branch}"): + _say_left(head, current, named) + _say_after(why) + + def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: - """Step (a) for one leg. Prints; never raises; never moves the pin.""" + """Step (a) for one leg. Prints; never moves the pin; and never raises, + but an `OSError` where git cannot be started (#694), which + `_checkout_all_tracking_branches` turns into `git-not-runnable`. + + A leg with an operation in progress is refused before any path is + chosen (`_say_in_progress`, #704), so no move below meets one.""" path = str(leg.get("path") or "") sub = root / path # What is PRINTED of the leg is spelled (#391): `role`, `path` and `branch` @@ -586,37 +819,16 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None: f"refs/heads/{branch}"], sub) remote = _try_git(["rev-parse", "--verify", "--quiet", f"refs/remotes/origin/{branch}"], sub) + doing = _in_progress(sub) + if doing is not None: + _say_in_progress(where, path, doing) + return if local is None: if pin is None: print(f" {where}: no gitlink recorded; leaving as is") - return - head, current = _head_of(sub) - if head is not None and current is None and head != pin: - # `checkout -b ` from a leg detached anywhere else - # leaves the commit made there on no branch (#533), as placing a - # branch that exists would. The branch is not there to check - # out, so the way out names the pin alone. A HEAD git cannot - # read (the leg's gitdir is gone) is no commit to keep: the - # checkout below fails, and says it could not. - print(f" {where}: branch {named} does not exist, and the " - f"checkout is at {_short(head)} (detached), which is not " - f"the pin {_short(pin)}") - _say_keep("the pin") - return - if _try_git(["checkout", "-q", "-b", branch, pin], sub) is None: - print(f" {where}: could not create branch {branch!r} at " - f"the pin {_short(pin)}; leaving detached") - return - if remote is not None: - _try_git(["branch", "-q", f"--set-upstream-to=origin/{branch}", - branch], sub) - state = "at the pin" - if remote is not None and remote != pin: - state = (f"at the pin; origin/{named} is {_short(remote)}, which " - f"the pin deliberately does NOT follow") - print(f" {where}: branch {named} created at " - f"{_short(pin)} — {state}") + else: + create_at_pin(sub, where, branch, pin, remote) return if local == pin: @@ -647,7 +859,9 @@ def shape_upstream_notice(root: Path) -> None: be on this machine — a clone at `SHAPE_UPSTREAM_PATH`, or a tip somebody cached at `.shape-upstream-tip` — saying so costs nothing and closes the gap that had two projects updated by hand. Where it is not, this is silent: - an absent answer is not a finding, and it never fails the bootstrap. + an absent answer is not a finding, and it never fails the bootstrap. A + clone named there that git cannot be STARTED to read is no absent + answer but a machine with no git, refused `git-not-runnable` (#694). """ pin_path = root / "contracts" / "shape-pin.yaml" if not pin_path.is_file(): @@ -662,7 +876,9 @@ def shape_upstream_notice(root: Path) -> None: tip = None upstream = os.environ.get(SHAPE_UPSTREAM_ENV) if upstream and Path(upstream).is_dir(): - tip = _try_git(["rev-parse", "HEAD"], Path(upstream)) + with _git_runs(f"the clone {SHAPE_UPSTREAM_ENV} names, " + f"{spelled(upstream)}, could not be read"): + tip = _try_git(["rev-parse", "HEAD"], Path(upstream)) if tip is None: cache = root / SHAPE_UPSTREAM_CACHE if cache.is_file(): @@ -831,7 +1047,9 @@ def _checkout_all_tracking_branches(root: Path, sub_legs: list[dict], branch_arg: str | None) -> None: """Step (a): place every submodule leg on its tracking branch. - Split from `main` for #132. + Split from `main` for #132. Git that cannot be started is + `git-not-runnable`, naming the leg it met it at (#694): one fact about + the machine, so no leg after it is tried. """ if not sub_legs: print(" no submodule legs declared. A one-repository project runs " @@ -839,7 +1057,11 @@ def _checkout_all_tracking_branches(root: Path, sub_legs: list[dict], return branch = branch_arg or (manifest or {}).get("tracking_branch") or "main" for leg in sub_legs: - checkout_tracking_branch(root, leg, str(leg.get("branch") or branch)) + with _git_runs(f"the {spelled(leg.get('role'))} leg at " + f"{spelled(leg.get('path') or '')} could not be " + "placed on its tracking branch"): + checkout_tracking_branch(root, leg, + str(leg.get("branch") or branch)) def _run_neutral_validators(root: Path, manifest: dict | None, @@ -893,11 +1115,19 @@ def main(argv: list[str] | None = None) -> int: name = (manifest or {}).get("name") or root.name print(f"bootstrap: {spelled(name)} ({spelled(root)})") - print("\n(a) legs on tracking branches, pins untouched") - _checkout_all_tracking_branches(root, sub_legs, manifest, args.branch) + try: + print("\n(a) legs on tracking branches, pins untouched") + _checkout_all_tracking_branches(root, sub_legs, manifest, args.branch) - print("\n(b) neutral validators") - failed = _run_neutral_validators(root, manifest, args.skip_validators) + print("\n(b) neutral validators") + failed = _run_neutral_validators(root, manifest, args.skip_validators) + except Refusal as exc: + # Only `git-not-runnable` reaches here (#694): nothing else in the + # two steps raises one. Stdout first, so a log of both streams + # reads in the order the run went. + sys.stdout.flush() + print(str(exc), file=sys.stderr) + return 2 print("\n(c) review authority") read_authority(root, legs) diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index 6dce353f..f69b81dd 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -58,8 +58,9 @@ the line says to write its section back. (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project - and knows how to bootstrap itself — its legs onto their tracking - branches at their pinned commits, its own validators, its own + and knows how to bootstrap itself — it puts its legs on their + tracking branches at their pins where that loses no commit, and says + why where it does not; its own validators; its own review-authority readout. This command does not reimplement any of that; it walks the members `family.yaml` names and calls theirs. `--members-make validate` runs `make validate` instead, which is what @@ -69,6 +70,9 @@ checked out that no credential would check out. Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). + Where (a) cannot start git, the run STOPS there, exit 2, refused + `git-not-runnable` as `validate-family.py` refuses it (#694): each + member's own bootstrap runs git too, so (b) would only say it again. THE FAMILY CONFERS NOTHING and neither does this command. It fetches, it delegates, and it says what it could not reach. @@ -133,7 +137,8 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for the run to end FAILED on (#648); every member and leg is said to be - fetched only where there is none and git exited 0.""" + fetched only where there is none and git exited 0. Raises the + `git-not-runnable` refusal where git cannot be started (#694).""" source, token = credential() args = ["git"] if token: @@ -143,8 +148,15 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: args += ["submodule", "update", "--init", "--recursive"] print(f" credential source: {source}") sys.stdout.flush() - proc = subprocess.run(args, cwd=str(root), capture_output=True, text=True, - encoding="utf-8", errors="replace", check=False) + try: + proc = subprocess.run(args, cwd=str(root), capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=False) + except OSError as exc: + refusal = git_not_started(exc) + if refusal is None: + raise + raise refusal from exc if proc.returncode != 0: # The token can appear in git's own error text (it is in the URL it # tried), so the output is scrubbed before it is printed. @@ -171,6 +183,31 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: for row, _, _ in left] +def git_not_started(exc: OSError) -> Refusal | None: + """The `git-not-runnable` refusal for an `OSError` that says git could + not be STARTED (#694), else `None`, and the error goes on as it did. + + Nothing caught one: `make bootstrap` with no git on PATH ended (a) in a + `FileNotFoundError` traceback, exit 1, the code of `family bootstrap + FAILED`, though nothing had failed. Git's is told by its NAME, as + `validate-pins.py` tells it: a `filename` of `git`, or none at all on a + `FileNotFoundError` or a `PermissionError`, as Windows raises them; one + naming a directory is not git's. The code, and the remedy's first + sentence, are `validate-family.py`'s.""" + nameless = exc.filename is None and isinstance( + exc, (FileNotFoundError, PermissionError)) + if exc.filename != "git" and not nameless: + return None + return Refusal( + "git-not-runnable", + f"the members could not be fetched: git could not be run " + f"({exc.strerror or exc})", + "Remediation: put git on PATH and run this again. Step (a) fetches " + "every member through git, and each member's own bootstrap places " + "its legs through git too, so without it nothing here can be " + "fetched or bootstrapped.") + + def cures(left: list[tuple[dict, str, str]], stopper: str | None, rows: list[dict], modules: Gitmodules) -> str: """What (a)'s line says cures the rows `left` names, one sentence for @@ -513,7 +550,11 @@ def main(argv: list[str] | None = None) -> int: if args.no_fetch: print(" skipped (--no-fetch)") else: - failed += fetch_members(root, rows) + try: + failed += fetch_members(root, rows) + except Refusal as exc: + print(str(exc), file=sys.stderr) + return 2 print(f"\n(b) each member's own `make {spelled(args.members_make)}`") if args.skip_members: diff --git a/tests/test_bootstrap_after_bump.py b/tests/test_bootstrap_after_bump.py index 0acbdb59..8c368a89 100644 --- a/tests/test_bootstrap_after_bump.py +++ b/tests/test_bootstrap_after_bump.py @@ -57,16 +57,15 @@ the tracking branch to the pin and the checkout puts the leg on it, as from the pin; the other branch is not touched. One line under the `fast-forwarded` one says the leg left that branch and that it keeps its commits (or has none yet), -since the next commit made in the leg is on the tracking branch now; no other -start prints it, and nor does a refusal. From another branch the checkout -changes files, so git can refuse it over a local edit or an untracked file, -which a checkout from the pin never meets: the branch is then left at the pin -and HEAD where it was, and the next run meets the branch-at-the-pin state. Two -things are NOT refused, as in the branch-at-the-pin and missing-branch cases -(#458's to rule on): an IGNORED file at a path the pin tracks is overwritten, -and a merge in progress on the other branch is ended, its `MERGE_HEAD` removed -and its staged result carried onto `main`. A leg detached at a commit that is -not the pin is still left, whatever it holds. +since the next commit made in the leg is on the tracking branch now; the +branch-at-the-pin path prints it under `on main` too (#704), and no other start +prints it, nor does a refusal. From another branch the checkout changes files, +so git can refuse it over a local edit or an untracked file, which a checkout +from the pin never meets: the branch is then left at the pin and HEAD where it +was, and the next run meets the branch-at-the-pin state. An IGNORED file in the +way and an operation in progress are refused too, on every path (#704): +`tests/test_bootstrap_leg_moves.py`. A leg detached at a commit that is not the +pin is still left, whatever it holds. EACH TEST GOES THE WAY A PERSON DOES: a scaffold into bare repositories on disk, cloned recursively and bootstrapped, so each leg is on `main` at its @@ -882,7 +881,9 @@ def test_a_leg_on_another_branch_with_its_own_branch_at_the_pin_is_as_it_was( is made. `fast_forward_to_pin` did not do the same for a branch BEHIND the pin until #603 (D1, option A of its checklist triage) made the two one rule: the T7 tests at the end of this module are this test's twins - for a branch behind the pin.""" + for a branch behind the pin. The line under `on main` says the leg left + `elsewhere`, as T7's does under `fast-forwarded`; it said nothing of the + branch, and the next commit was on `main` unannounced (#704).""" leg, new = branch_at_the_new_pin(root, bootstrapped["remotes"], tmp_path, "advance-other-branch") git("checkout", "-q", "-b", "elsewhere", cwd=leg) @@ -890,8 +891,8 @@ def test_a_leg_on_another_branch_with_its_own_branch_at_the_pin_is_as_it_was( result = bootstrap(root) assert result.returncode == 0, result.stderr + result.stdout - assert f" [spec] spec: on main at {new[:12]} (branch tip == pin)\n" in ( - result.stdout), result.stdout + assert (f" [spec] spec: on main at {new[:12]} (branch tip == pin)\n" + + left_branch()) in result.stdout, result.stdout assert head_ref(leg) == MAIN and rev(leg) == new assert rev(leg, "refs/heads/elsewhere") != new, "its commit is on it" @@ -1291,9 +1292,8 @@ def test_a_checkout_off_another_branch_git_refuses_leaves_main_at_the_pin( git("checkout", "-q", "--", EDITED, cwd=leg) again = skip_validators(root) - assert f" [spec] spec: on main at {new[:12]} (branch tip == pin)\n" in ( - again.stdout), again.stdout - assert LEFT not in again.stdout, "P9 says nothing of the branch it left" + assert (f" [spec] spec: on main at {new[:12]} (branch tip == pin)\n" + + left_branch()) in again.stdout, "P9 says the branch it left" assert head_ref(leg) == MAIN and rev(leg) == new @@ -1334,8 +1334,8 @@ def test_a_leg_on_a_branch_with_no_commit_yet_is_placed_as_on_any_branch( """HEAD on a branch with NO COMMIT YET (`git switch --orphan`): there is no commit to keep, and `place_at_pin` checks `main` out from it, so a `main` behind the pin does the same (#603). The four lines side by side: - free, `main` goes to the pin and the leg onto it, T7 says the leg left a - branch with no commit yet and P9 says nothing of it; with an untracked + free, `main` goes to the pin and the leg onto it, and T7 and P9 both say + the leg left a branch with no commit yet (P9 since #704); with an untracked file in the way, git refuses the checkout, HEAD stays on the unborn branch, `main` is at the pin and nothing is said of the leg leaving.""" old = rev(root / "spec") @@ -1368,11 +1368,9 @@ def test_a_leg_on_a_branch_with_no_commit_yet_is_placed_as_on_any_branch( assert (leg / ADDED).read_bytes() == b"mine\n" assert LEFT not in result.stdout else: - assert ((fast_forwarded(old, new) + left_branch("orphan", True) - if where == "behind" else + assert ((fast_forwarded(old, new) if where == "behind" else f" [spec] spec: on main at {new[:12]} (branch tip == pin)\n") - in result.stdout), result.stdout - assert (LEFT in result.stdout) == (where == "behind") + + left_branch("orphan", True) in result.stdout), result.stdout assert head_ref(leg) == MAIN and rev(leg) == new assert git("status", "--porcelain", cwd=leg).stdout == "" diff --git a/tests/test_bootstrap_leg_moves.py b/tests/test_bootstrap_leg_moves.py new file mode 100644 index 00000000..b0a950a9 --- /dev/null +++ b/tests/test_bootstrap_leg_moves.py @@ -0,0 +1,603 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`make bootstrap` moves a leg only where nothing the person has is lost, and +reads every HEAD one way (#704). + +THE ISSUE'S LINES, each measured on lane 5's #682 head (8b39cd19, git 2.43.0) +before this change, in a scaffold's recursive clone bootstrapped once: + +- LINE 1. Every move that changes files overwrote an IGNORED file at a path + the pin tracks, silently: the checkout of the branch AT the pin from another + branch (P9, `place_at_pin`), the checkout of the branch moved from BEHIND the + pin (T7, `fast_forward_to_pin`), the `merge --ff-only` of a branch behind the + pin with HEAD on it, and the `checkout -b` of a branch that is not there. A + `README.md` the leg's `info/exclude` lists, holding the person's notes, was + the pin's `README.md` after each, under `on main at `. Each now carries + `--no-overwrite-ignore`, so git refuses it as it refuses an untracked file, + in its own words, and the leg stays where it is. +- LINE 2. A merge, a cherry-pick or a revert in progress on the branch HEAD + left was ENDED by each of those moves: its head file (`MERGE_HEAD`, + `CHERRY_PICK_HEAD`, `REVERT_HEAD`) removed and what it had staged carried + onto `main`. A merge was ended from HEAD detached at the pin, from HEAD + already on `main` at the pin (a checkout that moves nothing) and by the move + back of a branch past the pin, too; a rebase or an `am` stopped there was + put on `main`, to go on over it. Brett Heap's R1 (2026-10-16T13:07:34Z): a + leg in the way of a move REFUSES. A leg whose git dir holds any of the + five markers is refused before any path is chosen, naming the operation, + saying nothing moved, and giving the commands that finish or abort it. +- LINE 5. A HEAD that is a symbolic ref outside `refs/heads/` (`git + symbolic-ref HEAD refs/remotes/origin/main`) was placed by P9 and the + missing-branch path and left by T7 (`which is neither`). The issue's + recommended answer: every path leaves it, since a commit made there lands + on that ref. +- THE REVIEW'S THREE (rv704 on #745, at 4e2b211b). S1: a cherry-pick or a + revert of several commits whose stop was committed by hand leaves only + `sequencer/todo`, and was moved onto `main`, where its `--continue` then + committed; it is refused as in progress. S2: the missing-branch path + says the branch it left, as P9 and T7 do. S3: it judges its `checkout + -b` by where HEAD is, so a failing post-checkout hook is not called a + branch that could not be made. + +Lines 3 and 4 are elsewhere: P9's line naming the branch it left is in +`tests/test_bootstrap_after_bump.py` (its other-branch tests, now reading +the line), and line 4 is the narrowed sentence in `AGENTS-shape.md` (R2). + +Every state is made the way a person makes it, with git's own commands in the +leg, and every way out is run as printed, from the root. +""" + +from __future__ import annotations + +import itertools +import os +import shlex +import subprocess +from pathlib import Path + +import pytest + +from conftest import IDENTITY_CONFIG, git, run_script +from test_bootstrap_after_bump import ( # noqa: F401 + ADDED, EDITED, MAIN, NOTHING_ELSE, bootstrapped, bumped, + failing_post_checkout, head_ref, left_branch, moved_and_refused, rev, + root, skip_validators, +) + +#: The pin's tracked file the branch `other` deletes, so that checking the +#: tracking branch out from `other` writes it. +IN_THE_WAY = "README.md" +#: What the person keeps in the ignored file. +MINE = b"MY LOCAL NOTES\n" +#: The three paths of the issue, by the state of `main` beside the pin. +PATHS = pytest.mark.parametrize("path", ["at", "behind", "missing"], + ids=["P9 at the pin", "T7 behind the pin", + "missing branch"]) +#: A branch name per `bumped` call: each pushes to the module's spec remote. +#: Not `tmp_path.name`, which a passed test's removal hands to the next one. +ADVANCES = itertools.count() + + +def keep(options: str) -> str: + """The line under a checkout step (a) will not move off.""" + return (" not moving the checkout off it. Keep any commit made " + f"there on a branch, check out {options} in the leg, and run " + "bootstrap again.\n") + + +def tracking(root: Path, remotes: Path, tmp_path: Path, path: str, + touch: str | None = None) -> tuple[str, str]: + """`main` as `path` names it, with the leg left DETACHED at the pin: + AT the pin (P9: the bootstrapped clone's own state), BEHIND it (T7: a + bump, `bumped`, with `touch` as it takes it) or NOT THERE (the missing- + branch path). Returns the pin and the commit a branch made beside + `main` starts from: `main`'s own, or the pin where there is none.""" + leg = root / "spec" + if path == "behind": + start = rev(leg, MAIN) + pin = bumped(root, remotes, tmp_path, f"moves-{next(ADVANCES)}", + touch) + return pin, start + pin = rev(leg) + git("checkout", "-q", "--detach", pin, cwd=leg) + if path == "missing": + git("branch", "-q", "-D", "main", cwd=leg) + return pin, pin + + +def main_of(leg: Path) -> str | None: + """`main`'s commit, or None where the leg has no `main`.""" + return git("rev-parse", "--verify", "--quiet", MAIN, cwd=leg, + check=False).stdout.strip() or None + + +def ignore(leg: Path, name: str) -> None: + """`name` listed in the leg's own `info/exclude`, a person's ignore + that no commit carries, and the person's bytes written there.""" + exclude = leg / git("rev-parse", "--git-path", "info/exclude", + cwd=leg).stdout.strip() + listed = exclude.read_bytes() if exclude.is_file() else b"" + exclude.parent.mkdir(exist_ok=True) + exclude.write_bytes(listed + name.encode("utf-8") + b"\n") + (leg / name).write_bytes(MINE) + + +def snapshot(leg: Path) -> tuple: + """Where the leg is: HEAD's ref and commit, `main`, and the status.""" + return (head_ref(leg), rev(leg), main_of(leg), + git("status", "--porcelain", cwd=leg).stdout) + + +def placed_once_cleared(root: Path, pin: str) -> str: + """The way out works: a run after it puts the leg on `main` AT the + pin. Returns what that run printed.""" + again = skip_validators(root) + assert again.returncode == 0, again.stderr + again.stdout + assert head_ref(root / "spec") == MAIN, again.stdout + assert rev(root / "spec") == pin, again.stdout + return again.stdout + + +def typed(root: Path, command: str) -> None: + """`command`, a `git -C ...` step (a) printed, run from the root as a + person types it, under the suite's identity, with an editor that keeps + the message git offers.""" + argv = shlex.split(command) + assert argv[0] == "git", argv + proc = subprocess.run(["git", *IDENTITY_CONFIG, *argv[1:]], + cwd=str(root), capture_output=True, text=True, + env={**os.environ, "GIT_EDITOR": "true"}, + check=False) + assert proc.returncode == 0, proc.stderr + proc.stdout + + +# --- line 1: an ignored file in the way is refused on every path ------------ + +def refused_first_line(path: str, pin: str, start: str) -> str: + """What step (a) prints first over a move git refused, by path.""" + if path == "at": + return (f" [spec] spec: branch main is at the pin {pin[:12]}, and " + "git refused to check it out:\n") + if path == "behind": + return moved_and_refused(start, pin) + return (f" [spec] spec: could not create branch 'main' at the pin " + f"{pin[:12]}; leaving it on other\n") + + +@PATHS +def test_an_ignored_file_a_move_off_another_branch_writes_is_refused( + root, bootstrapped, tmp_path, path): + """HEAD on `other`, a branch that deleted `README.md`, which the pin + tracks, and the person's own `README.md` ignored there. The checkout + of `main` wrote the pin's over it (P9, the issue's own measure; T7; the + missing branch's `checkout -b`). Git now refuses it in its own words, + the file is the person's still, and HEAD stays on `other`. T7's + `branch -f` has moved `main` to the pin by then, as over an untracked + file, and says so; the next run meets P9's state.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, path) + leg = root / "spec" + git("checkout", "-q", "-b", "other", start, cwd=leg) + git("rm", "-q", "--", IN_THE_WAY, cwd=leg) + git("commit", "-q", "-m", "other keeps no README", cwd=leg) + ignore(leg, IN_THE_WAY) + + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + assert refused_first_line(path, pin, start) in result.stdout, ( + result.stdout) + assert f" {IN_THE_WAY}\n" in result.stdout, "git names the file" + assert NOTHING_ELSE in result.stdout + assert (leg / IN_THE_WAY).read_bytes() == MINE, "the file is the person's" + assert head_ref(leg) == "refs/heads/other", "HEAD stays on `other`" + assert main_of(leg) == {"at": pin, "behind": pin, "missing": None}[path] + + (leg / IN_THE_WAY).unlink() + again = placed_once_cleared(root, pin) + assert left_branch("other") in again, "it says the leg left `other`" + + +def test_an_ignored_file_the_fast_forward_writes_is_refused( + root, bootstrapped, tmp_path): + """T7 with HEAD ON `main` behind the pin: `merge --ff-only` wrote the + pin's `NEW-CONTRACT.md` over the person's ignored one and said + `fast-forwarded`. It now refuses, and `main` stays where it was.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, "behind") + leg = root / "spec" + git("checkout", "-q", "main", cwd=leg) + ignore(leg, ADDED) + + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + assert (f" [spec] spec: branch main tip {start[:12]} is behind the PIN " + f"{pin[:12]}, and git refused to move it:\n") in result.stdout, ( + result.stdout) + assert f" {ADDED}\n" in result.stdout, "git names the file" + assert NOTHING_ELSE in result.stdout + assert (leg / ADDED).read_bytes() == MINE + assert head_ref(leg) == MAIN + assert rev(leg) == start, "`main` is not moved" + + (leg / ADDED).unlink() + placed_once_cleared(root, pin) + + +# --- line 2: an operation in progress is refused before any move ------------ + +def theirs(leg: Path, start: str) -> None: + """A branch `theirs` from `start`, one commit adding `b.txt`; HEAD is + left on it.""" + git("checkout", "-q", "-b", "theirs", start, cwd=leg) + (leg / "b.txt").write_bytes(b"b\n") + git("add", "--", "b.txt", cwd=leg) + git("commit", "-q", "-m", "theirs", cwd=leg) + + +def merge_theirs(leg: Path) -> None: + """`theirs` merged where HEAD is, `--no-commit --no-ff`: `MERGE_HEAD`, + and `b.txt` staged.""" + git("merge", "-q", "--no-commit", "--no-ff", "theirs", cwd=leg) + + +def merging(leg: Path, start: str) -> None: + """`other` from `start`, merging `theirs`.""" + theirs(leg, start) + git("checkout", "-q", "-b", "other", start, cwd=leg) + merge_theirs(leg) + + +def cherry_picking(leg: Path, start: str) -> None: + """`other` from `start`, picking `theirs`'s second commit, which changes + the README its first one wrote: a conflict, resolved and staged, so + `CHERRY_PICK_HEAD` is there and nothing is unmerged.""" + git("checkout", "-q", "-b", "theirs", start, cwd=leg) + for text in (b"first\n", b"second\n"): + (leg / EDITED).write_bytes(text) + git("commit", "-q", "-am", text.decode().strip(), cwd=leg) + pick = rev(leg) + git("checkout", "-q", "-b", "other", start, cwd=leg) + assert git("cherry-pick", pick, cwd=leg, check=False).returncode != 0 + (leg / EDITED).write_bytes(b"resolved\n") + git("add", "--", EDITED, cwd=leg) + + +def reverting(leg: Path, start: str) -> None: + """`other` from `start` with a commit adding `y.txt`, reverted with + `--no-commit`: `REVERT_HEAD`, and the deletion staged.""" + git("checkout", "-q", "-b", "other", start, cwd=leg) + (leg / "y.txt").write_bytes(b"y\n") + git("add", "--", "y.txt", cwd=leg) + git("commit", "-q", "-m", "y", cwd=leg) + git("revert", "--no-commit", "HEAD", cwd=leg) + + +#: Each operation: what makes it, the marker, what the line calls it, and +#: the git command that ends it. +OPERATIONS = { + "merge": (merging, "MERGE_HEAD", "a merge", "merge"), + "cherry-pick": (cherry_picking, "CHERRY_PICK_HEAD", "a cherry-pick", + "cherry-pick"), + "revert": (reverting, "REVERT_HEAD", "a revert", "revert"), + "rebase": (None, "rebase-merge", "a rebase", "rebase"), + "rebase --apply": (None, "rebase-apply", "a rebase", "rebase"), + "am": (None, "rebase-apply/applying", "an `am`", "am"), + "cherry-pick sequence": (None, "sequencer/todo", "a cherry-pick", + "cherry-pick"), + "revert sequence": (None, "sequencer/todo", "a revert", "revert"), +} + + +def in_progress(operation: str) -> str: + """The two lines step (a) prints for a leg with `operation` in progress + (#704), byte for byte.""" + _, marker, doing, verb = OPERATIONS[operation] + return (f" [spec] spec: {doing} is in progress in the leg ({marker} is " + "in its git dir), so nothing moved\n" + f" finish it (`git -C spec {verb} --continue`) or abort " + f"it (`git -C spec {verb} --abort`), then run bootstrap again.\n") + + +def refuses(root: Path, operation: str) -> None: + """Bootstrap refuses the leg as `in_progress` says and nothing moves: + HEAD, `main`, the index and the operation's marker are as they were.""" + leg = root / "spec" + marker = OPERATIONS[operation][1] + gitdir = leg / git("rev-parse", "--git-dir", cwd=leg).stdout.strip() + before = snapshot(leg) + assert (gitdir / marker).exists(), f"{operation} is in progress" + + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + assert in_progress(operation) in result.stdout, result.stdout + assert snapshot(leg) == before, "nothing moved" + assert (gitdir / marker).exists(), f"the {operation} goes on" + + +def refused_until_aborted(root: Path, operation: str, pin: str) -> None: + """`refuses`, then the `--abort` the line names, run as printed from the + root, and a run after it places the leg at the pin.""" + refuses(root, operation) + # The `--abort` of the second line, which is in the output as it is in + # `in_progress`: the fourth span of backticks there. + abort = in_progress(operation).splitlines()[1].split("`")[3] + assert abort.endswith(" --abort"), abort + git(*shlex.split(abort)[1:], cwd=root) + placed_once_cleared(root, pin) + + +@PATHS +@pytest.mark.parametrize("operation", ["merge", "cherry-pick", "revert"]) +def test_an_operation_in_progress_on_the_branch_a_move_leaves_is_refused( + root, bootstrapped, tmp_path, path, operation): + """HEAD on `other` with a merge, a cherry-pick or a revert in progress + there, on each of the three paths that check `main` out from another + branch. Each move ENDED it: the head file gone, the staged result on + `main`, and `on main` or `fast-forwarded` or `created` said over it.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, path) + OPERATIONS[operation][0](root / "spec", start) + refused_until_aborted(root, operation, pin) + + +def detached_at_the_pin(root: Path, remotes: Path, tmp_path: Path) -> str: + """HEAD detached AT the pin, `main` there too (P9 from the pin).""" + return tracking(root, remotes, tmp_path, "at")[0] + + +def on_main_at_the_pin(root: Path, remotes: Path, tmp_path: Path) -> str: + """HEAD ON `main` AT the pin: the leg already placed, where a checkout + of `main` moves nothing and still ends the merge.""" + pin = rev(root / "spec") + assert head_ref(root / "spec") == MAIN + return pin + + +def past_the_pin(root: Path, remotes: Path, tmp_path: Path) -> str: + """`main` at `origin/main` one commit PAST the pin, HEAD detached at the + pin: `move_back_to_pin`'s state, a fresh clone of a leg that moved on.""" + leg = root / "spec" + pin = rev(leg) + ahead = git("commit-tree", rev(leg, f"{pin}^{{tree}}"), "-p", pin, "-m", + "the remote moved on", cwd=leg).stdout.strip() + git("update-ref", "refs/remotes/origin/main", ahead, cwd=leg) + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-f", "main", ahead, cwd=leg) + return pin + + +@pytest.mark.parametrize("state", [detached_at_the_pin, on_main_at_the_pin, + past_the_pin], + ids=["detached at the pin", "on main at the pin", + "main past the pin"]) +def test_a_merge_in_progress_where_no_branch_is_left_is_not_ended_either( + root, bootstrapped, tmp_path, state): + """A merge started where HEAD already is: detached at the pin, on `main` + at the pin, and detached at the pin under a `main` the move back takes + to the pin. The checkout ended it from all three, though none leaves a + branch: git's checkout drops `MERGE_HEAD` whenever it runs.""" + pin = state(root, bootstrapped["remotes"], tmp_path) + leg = root / "spec" + where = head_ref(leg) + theirs(leg, pin) + git("checkout", "-q", *(["main"] if where == MAIN else ["--detach", pin]), + cwd=leg) + merge_theirs(leg) + refused_until_aborted(root, "merge", pin) + + +def rebased(leg: Path, pin: str, start: str, *backend: str) -> None: + """`mine` from `start`, changing the README the pin changes too, rebased + onto the pin: it stops on the conflict with HEAD detached AT the pin, + resolved and staged.""" + git("checkout", "-q", "-b", "mine", start, cwd=leg) + (leg / EDITED).write_bytes(b"mine\n") + git("commit", "-q", "-am", "mine", cwd=leg) + assert git("rebase", "-q", *backend, pin, cwd=leg, + check=False).returncode != 0 + assert head_ref(leg) is None, "the rebase detached HEAD" + assert rev(leg) == pin, "at the pin, onto which it rebases" + (leg / EDITED).write_bytes(b"resolved\n") + git("add", "--", EDITED, cwd=leg) + + +def applied(leg: Path, pin: str, start: str, tmp_path: Path) -> None: + """`other` at the pin, applying with `am -3` a patch made beside it that + changes the README the pin changes: it stops on the conflict, resolved + and staged, with HEAD on `other`.""" + git("checkout", "-q", "-b", "theirs", start, cwd=leg) + (leg / EDITED).write_bytes(b"theirs\n") + git("commit", "-q", "-am", "theirs", cwd=leg) + patch = tmp_path / "theirs.patch" + patch.write_bytes(git("format-patch", "-1", "--stdout", "HEAD", + cwd=leg).stdout.encode("utf-8")) + git("checkout", "-q", "-b", "other", pin, cwd=leg) + assert git("am", "-q", "-3", str(patch), cwd=leg, + check=False).returncode != 0 + (leg / EDITED).write_bytes(b"resolved\n") + git("add", "--", EDITED, cwd=leg) + + +@pytest.mark.parametrize("operation", ["rebase", "rebase --apply", "am"]) +def test_a_rebase_or_an_am_stopped_in_the_leg_is_not_moved_onto_main( + root, bootstrapped, tmp_path, operation): + """A rebase stopped with HEAD detached AT the pin (either backend), and + an `am` stopped on another branch, each over a conflict resolved and + staged. The checkout of `main` was made and the operation was left in + progress over it, its `--continue` then committing onto `main`. The pin + changes the README (`touch`), and `main` is moved to it, as a pull in + the leg would: the branch-at-the-pin state.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, "behind", + touch=EDITED) + leg = root / "spec" + git("branch", "-q", "-f", "main", pin, cwd=leg) + if operation == "am": + applied(leg, pin, start, tmp_path) + else: + rebased(leg, pin, start, *(["--apply"] if "apply" in operation + else [])) + refused_until_aborted(root, operation, pin) + + +def picking_two(leg: Path, start: str) -> None: + """`other` from `start` with its own README, then `git cherry-pick + theirs~1 theirs`: the first pick, which changes the README too, stops + on the conflict; the second adds `z.txt`.""" + git("checkout", "-q", "-b", "theirs", start, cwd=leg) + (leg / EDITED).write_bytes(b"theirs\n") + git("commit", "-q", "-am", "theirs edits README", cwd=leg) + (leg / "z.txt").write_bytes(b"z\n") + git("add", "--", "z.txt", cwd=leg) + git("commit", "-q", "-m", "theirs adds z", cwd=leg) + git("checkout", "-q", "-b", "other", start, cwd=leg) + (leg / EDITED).write_bytes(b"other\n") + git("commit", "-q", "-am", "other edits README", cwd=leg) + assert git("cherry-pick", "theirs~1", "theirs", cwd=leg, + check=False).returncode != 0 + + +def reverting_two(leg: Path, start: str) -> None: + """`other` from `start` with three commits (the README to `a`, `z.txt` + added, the README to `c`), then `git revert `: + the first revert stops on the README conflict; the second takes + `z.txt` out.""" + git("checkout", "-q", "-b", "other", start, cwd=leg) + (leg / EDITED).write_bytes(b"a\n") + git("commit", "-q", "-am", "README a", cwd=leg) + (leg / "z.txt").write_bytes(b"z\n") + git("add", "--", "z.txt", cwd=leg) + git("commit", "-q", "-m", "adds z", cwd=leg) + (leg / EDITED).write_bytes(b"c\n") + git("commit", "-q", "-am", "README c", cwd=leg) + assert git("revert", "HEAD~2", "HEAD~1", cwd=leg, + check=False).returncode != 0 + + +#: Each sequence: what starts it, and the subject of `other`'s last commit +#: once its `--continue` is done, with whether that leaves `z.txt` there. +SEQUENCES = { + "cherry-pick sequence": (picking_two, "theirs adds z", True), + "revert sequence": (reverting_two, 'Revert "adds z"', False), +} + + +@pytest.mark.parametrize("operation", sorted(SEQUENCES)) +def test_a_sequence_whose_stop_was_committed_by_hand_is_refused( + root, bootstrapped, tmp_path, operation): + """rv704's S1: a cherry-pick or a revert of TWO commits stops on the + first, which the person resolves and commits by hand. Its head file is + gone, and `git status` still says it is in progress, with its + `--continue`. Bootstrap moved the leg onto `main`, and that + `--continue` then put the second commit there. It is refused by its + `sequencer/todo` now, and the `--continue` it prints, typed from the + root, finishes the sequence on `other`.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, "at") + leg = root / "spec" + make, subject, has_z = SEQUENCES[operation] + make(leg, start) + (leg / EDITED).write_bytes(b"resolved\n") + git("add", "--", EDITED, cwd=leg) + git("commit", "-q", "--no-edit", cwd=leg) + gitdir = leg / git("rev-parse", "--git-dir", cwd=leg).stdout.strip() + assert not (gitdir / "CHERRY_PICK_HEAD").exists(), "the stop is committed" + assert not (gitdir / "REVERT_HEAD").exists(), "the stop is committed" + + refuses(root, operation) + + typed(root, in_progress(operation).splitlines()[1].split("`")[1]) + assert head_ref(leg) == "refs/heads/other", "the sequence ends on `other`" + assert git("log", "-1", "--format=%s", cwd=leg).stdout.strip() == subject + assert (leg / "z.txt").exists() is has_z + assert main_of(leg) == pin, "`main` is not moved" + placed_once_cleared(root, pin) + + +@pytest.mark.parametrize("start_on", ["detached", "other"]) +def test_a_branch_made_under_a_failing_hook_is_said_to_be_made( + root, bootstrapped, tmp_path, start_on): + """rv704's S3: with no `main`, a post-checkout hook that fails after + `checkout -b main ` made the branch and put HEAD on it makes git + exit non-zero. The line said `could not create branch 'main' ...; + leaving ...` and `nothing else is tried` over a leg on `main` at the + pin. It is judged by where HEAD is, as a checkout of a branch that + exists is (#533): `created`, the line saying the leg left `other` + where it was there (S2), and the hook's words after them.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, "missing") + leg = root / "spec" + left = "" + if start_on == "other": + git("checkout", "-q", "-b", "other", start, cwd=leg) + left = left_branch("other") + + result = run_script(root / "scripts" / "bootstrap.py", "--skip-validators", + cwd=root, env=failing_post_checkout(tmp_path)) + assert result.returncode == 0, result.stderr + result.stdout + assert (f" [spec] spec: branch main created at {pin[:12]} — at the " + "pin\n" + left + " checked out, and then git said:\n" + " post-checkout hook failed\n") in result.stdout, ( + result.stdout) + assert "could not create" not in result.stdout + assert NOTHING_ELSE not in result.stdout + assert head_ref(leg) == MAIN + assert rev(leg) == pin + + +@pytest.mark.skipif(os.name == "nt", reason="U+DCFF is three bytes there, " + "and step (a) does not try such a branch at all") +def test_a_branch_whose_name_is_not_utf8_is_read_back_on_every_run( + root, tmp_path): + """`--branch` a name holding U+DCFF, the byte 0xff POSIX spells a name + that is not UTF-8 with, run twice. The first run makes the branch, and + the check of where `checkout -b` left HEAD (S3) reads HEAD back; the + second places the leg on it. Each read `symbolic-ref` decoded as UTF-8, + and on base the second run ended in a `UnicodeDecodeError` traceback. + Made on a disk that takes the byte, skipped on one that does not.""" + try: + (tmp_path / "probe-\udcff").write_bytes(b"") + except (OSError, ValueError): + pytest.skip("this disk takes no file name with the byte 0xff") + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "--detach", pin, cwd=leg) + for said in (rf" [spec] spec: branch a\udcffb created at {pin[:12]} " + "— at the pin\n", + rf" [spec] spec: on a\udcffb at {pin[:12]} (branch tip == " + "pin)\n"): + result = run_script(root / "scripts" / "bootstrap.py", + "--skip-validators", "--branch", "a\udcffb", + cwd=root) + assert result.returncode == 0, result.stderr + result.stdout + assert "Traceback" not in result.stderr, result.stderr + assert said in result.stdout, result.stdout + head = subprocess.run(["git", "symbolic-ref", "--quiet", "HEAD"], + cwd=str(leg), capture_output=True, + check=True).stdout + assert head == b"refs/heads/a\xffb\n", head + + +# --- line 5: a HEAD on a ref outside refs/heads/ is left by every path ------ + +@PATHS +def test_a_head_on_a_remote_tracking_ref_is_left_where_it_is( + root, bootstrapped, tmp_path, path): + """`git symbolic-ref HEAD refs/remotes/origin/main`, at `main`'s commit + or the pin's where there is no `main`. P9 placed the leg (`on main`) and + the missing-branch path made `main` there, while T7 said `which is + neither` of the same HEAD. Every path leaves it now, as T7 did.""" + pin, start = tracking(root, bootstrapped["remotes"], tmp_path, path) + leg = root / "spec" + git("update-ref", "refs/remotes/origin/main", start, cwd=leg) + git("symbolic-ref", "HEAD", "refs/remotes/origin/main", cwd=leg) + before = snapshot(leg) + place = f"at {start[:12]} (on refs/remotes/origin/main)" + expected = { + "at": (f" [spec] spec: branch main is at the pin {pin[:12]}, and the " + f"checkout is {place}, which is neither\n" + + keep("the pin or the branch")), + "behind": (f" [spec] spec: branch main tip {start[:12]} is behind " + f"the PIN {pin[:12]}, and the checkout is {place}, which " + "is neither\n" + keep("the pin or the branch")), + "missing": (f" [spec] spec: branch main does not exist, and the " + f"checkout is {place}, which is not a branch\n" + + keep("the pin")), + }[path] + + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + assert expected in result.stdout, result.stdout + assert snapshot(leg) == before, "nothing moved" diff --git a/tests/test_bootstrap_without_git.py b/tests/test_bootstrap_without_git.py new file mode 100644 index 00000000..4f2985dd --- /dev/null +++ b/tests/test_bootstrap_without_git.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Both roots' `bootstrap.py` with no git to run refuse `git-not-runnable`, +exit 2, as the validators beside them do (#694). + +THE DEFECT, measured on lane 5's #682 head (8b39cd19) with `PATH` set to an +empty directory and the interpreter named by its path: the family root's +`make bootstrap` printed `(a) members and their legs`, `credential source: +none`, then a `FileNotFoundError: [Errno 2] No such file or directory: 'git'` +traceback, exit 1, which is the code of `family bootstrap FAILED`, though +nothing had failed. The assembly root's ended the same way in step (a), at +the first leg's `recorded_gitlink`, with `--skip-validators` too; and in a +clone whose legs are not initialized, where step (a) runs no git, its +offline upstream notice did, after `validate-pins.py` had refused +`git-not-runnable` itself. + +THE RULE NOW. Each refuses `git-not-runnable`, the validators' code, with +their first remedy sentence (`put git on PATH and run this again`), exit 2, +and no traceback. The run STOPS there: every later step runs git too, so +going on would only say it again. Which `OSError` is git's is read by NAME, +as `validate-pins.py` reads it: `filename` `git`, or none on a +`FileNotFoundError` or a `PermissionError`, which is how Windows raises +them; an error naming a directory goes on as it did. + +The end-to-end runs are a person's: a scaffold's recursive clone, a plain +clone, and a family holder, each run with no git on `PATH`. The rule's other +errors are asked of each root's own function, in this process. +""" + +from __future__ import annotations + +import errno +import os +from pathlib import Path + +import pytest + +from conftest import (PROJECT, REPO, clone_just_pushed, load_without_bytecode, + run_script) +from test_family import family, holder # noqa: F401 + +#: What `subprocess` says when there is no git to start, here. +NO_GIT = ("The system cannot find the file specified" if os.name == "nt" + else os.strerror(errno.ENOENT)) +#: The assembly root's remedy, after the validators' first sentence. +ASSEMBLY_REMEDY = ( + "Remediation: put git on PATH and run this again. Step (a) reads each " + "leg's pin and moves the leg through git, and the validators read the " + "pins through git, so without it nothing here can be placed or checked.") +#: The family root's. +FAMILY_REMEDY = ( + "Remediation: put git on PATH and run this again. Step (a) fetches every " + "member through git, and each member's own bootstrap places its legs " + "through git too, so without it nothing here can be fetched or " + "bootstrapped.") + + +def without_git(root: Path, tmp_path: Path, *extra: str, + env: dict | None = None): + """`scripts/bootstrap.py` in `root` with `PATH` an empty directory: no + git, and no `gh` either. The interpreter is `sys.executable`, named by + its path.""" + empty = tmp_path / "no-git" + empty.mkdir(exist_ok=True) + return run_script(root / "scripts" / "bootstrap.py", *extra, cwd=root, + env={**(env or {}), "PATH": str(empty)}) + + +def refused(result, *lines: str) -> None: + """Exit 2, no traceback, and stderr ending on `lines`.""" + assert result.returncode == 2, result.stderr + result.stdout + assert "Traceback" not in result.stderr, result.stderr + said = result.stderr.splitlines() + assert said[-len(lines):] == list(lines), result.stderr + + +@pytest.mark.parametrize("extra", [(), ("--skip-validators",)], + ids=["bootstrap", "skip-validators"]) +def test_an_assembly_root_with_no_git_refuses_in_step_a(project, tmp_path, + extra): + """THE ISSUE'S ASSEMBLY-ROOT CASE, run: a recursive clone. The first leg + is where git is first needed, and the refusal names it; no leg after it + is tried and no step after (a) runs.""" + result = without_git(project, tmp_path, *extra) + refused(result, + "REFUSED git-not-runnable: the spec leg at spec could not be " + f"placed on its tracking branch: git could not be run ({NO_GIT})", + ASSEMBLY_REMEDY) + assert result.stdout.endswith( + "\n(a) legs on tracking branches, pins untouched\n"), result.stdout + + +def test_the_offline_upstream_notice_with_no_git_refuses_too(scaffolded, + tmp_path): + """A PLAIN clone: both legs NOT INITIALIZED, so step (a) runs no git, and + `SHAPE_UPSTREAM_PATH` names a directory, so the notice after the + validators runs one. `validate-pins.py` refuses `git-not-runnable` + itself; the notice then ended the run in a traceback. It refuses, and + the review-authority step is not reached.""" + clone = tmp_path / PROJECT + made = clone_just_pushed(scaffolded["remotes"] / f"{PROJECT}.git", clone, + recurse=False) + assert made.returncode == 0, made.stderr + upstream = tmp_path / "upstream" + upstream.mkdir() + + result = without_git(clone, tmp_path, + env={"SHAPE_UPSTREAM_PATH": str(upstream)}) + refused(result, + "REFUSED git-not-runnable: the clone SHAPE_UPSTREAM_PATH names, " + f"{upstream}, could not be read: git could not be run ({NO_GIT})", + ASSEMBLY_REMEDY) + assert " [spec] spec: NOT INITIALIZED." in result.stdout, result.stdout + assert "(c) review authority" not in result.stdout + + +@pytest.mark.parametrize("extra", [(), ("--skip-members",)], + ids=["bootstrap", "skip-members"]) +def test_a_family_holder_with_no_git_refuses_at_its_fetch(holder, tmp_path, + extra): + """THE ISSUE'S FAMILY-ROOT CASE, run. Step (a)'s `git submodule update` + is where git is first needed; the run stops there, since each member's + own bootstrap would only refuse the same way (#694's question for + #648).""" + result = without_git(holder, tmp_path, *extra) + refused(result, + "REFUSED git-not-runnable: the members could not be fetched: git " + f"could not be run ({NO_GIT})", FAMILY_REMEDY) + assert result.stdout.endswith( + "\n(a) members and their legs\n credential source: none\n"), ( + result.stdout) + + +# --- which `OSError` is git's: each root's own function --------------------- + +@pytest.fixture(scope="module", params=["assembly-root", "family-root"]) +def root_module(request): + """Each root's `bootstrap.py`, loaded by path under a name of its own.""" + return load_without_bytecode( + REPO / "templates" / request.param / "scripts" / "bootstrap.py", + f"bootstrap_without_git_{request.param.replace('-', '_')}") + + +def made_of(module, error: OSError): + """What `module` makes of `error` raised where it starts git: the + `Refusal` it raises instead, or the error itself, raised on.""" + if hasattr(module, "_git_runs"): + try: + with module._git_runs("unread"): + raise error + except Exception as raised: # noqa: BLE001 - what is asked + return raised + return module.git_not_started(error) or error + + +@pytest.mark.parametrize("error", [ + FileNotFoundError(errno.ENOENT, "No such file or directory", "git"), + PermissionError(errno.EACCES, "Permission denied", "git"), + OSError(errno.ENOEXEC, "Exec format error", "git"), + FileNotFoundError(2, "The system cannot find the file specified"), +], ids=["no git", "git not executable", "git no program", "windows"]) +def test_each_way_git_cannot_start_is_git_not_runnable(root_module, error): + made = made_of(root_module, error) + assert made.code == "git-not-runnable", made + assert made.detail.endswith( + f": git could not be run ({error.strerror})"), made.detail + + +@pytest.mark.parametrize("error", [ + FileNotFoundError(errno.ENOENT, "No such file or directory", "/nowhere"), + PermissionError(errno.EACCES, "Permission denied", "/a/leg"), + OSError(errno.EIO, "Input/output error"), +], ids=["a directory not there", "a directory not entered", "nameless"]) +def test_an_error_that_is_not_gits_goes_on_as_it_did(root_module, error): + assert made_of(root_module, error) is error