diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index e46d4331..3a8f8e9c 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -17,8 +17,12 @@ called this) first, then a `SHAPE_LEGS_TOKEN` PAT, then nothing at all. Whichever it finds is used ONLY through a `git -c url.<...>.insteadOf=<...>` rewrite for the duration of the one command, - covering both the HTTPS and SSH spellings a submodule URL may carry. It - is never written into `.git/config`, never persisted, never printed. + covering both the HTTPS and SSH spellings a submodule URL may carry, + and of the two fetches made again where that command dies (below). It + is never written into `.git/config`, never persisted, never printed, + and never traced: each of those commands runs with every `GIT_TRACE*` + variable taken out of its environment and trace2 turned off + (`untraced`, #996), since a trace writes a command's arguments. With no credential this is a plain `git submodule update --init --recursive`, exactly as it was for a family whose members are public. @@ -49,21 +53,29 @@ found` there said `ok`; - at a member whose store still lacks the commit this holder records for it where git's fetch there reached the member's - remote, which git's own trace of the run says, by the fetch's - exit code (`fetched_in`): its `not our ref` said `ok` too. A - fetch git could not make, as for want of a credential, leaves - the store as short, so the store alone tells nothing. + remote, by the fetch's exit code: git's own trace of the run + says it (`fetched_in`), or, with `SHAPE_LEGS_TOKEN` or + `SHAPE_LEGS_APP_TOKEN` set, whose rewrite a trace would write, + so that git's update is never traced, the two fetches git's + update makes there, made again in this process, untraced, say it + (`refetched`, #996): its `not our ref` said `ok` too, and with a + token set it still did, with the advice to set the token that + was set (lane 5's SHOULD 4 on #954). A fetch git could not make, + as for want of a credential, leaves the store as short, so the + store alone tells nothing. A leg in a member is asked as a member is, and a gitlink git's update passes without a look, its `update` `none` or the gitlink inactive, is passed (lane 5's SHOULD 1 on #954). Where neither tells, an exit 128 is read as before, the advice and `ok` where no row is left not checked out but for want of a credential, and the - run names only where git's init in this holder stopped. A run with - `SHAPE_LEGS_TOKEN` or `SHAPE_LEGS_APP_TOKEN` set is never traced, - since the trace would write that token, so the second is told only - where neither is set; with one set, B4 still gives the advice to - set the token that is set, and `ok` (lane 1's SHOULD 4 on #889, - lane 5's SHOULD 4 on #954, a known limit). + run names only where git's init in this holder stopped. + + Where git's update ends on a signal, read from its exit code alone, + `128 + N` where a git it ran died of signal N and `-N` where it was + itself killed, the run fails naming the signal and the code, with + no advice (`ended_on`, #998): no credential crashes git or kills + it. It said `ok` there, with the advice, beside the 139 of git + 2.43.0 crashing on a member's own `.gitmodules`. "every member and leg fetched" is said only where git exited 0 AND every row's member is checked out (#648). A member left NOT CHECKED @@ -141,8 +153,9 @@ gitlink this holder records with no url or at an unmerged `.gitmodules` (#879), or found where git's update died past this holder's own records, in a member's own index or at a member's - commit no fetch brought (#892); an exit 128 none of those shows is - no failure here, as (a) says. Any + commit no fetch brought (#892), or found that git's update ended + on a signal (#998); an exit 128 none of those shows is no failure + here, as (a) says. Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). @@ -173,6 +186,7 @@ import re import shlex import shutil +import signal import stat import subprocess import sys @@ -220,12 +234,30 @@ #: that fails ends git's update 1, before it checks out or enters any #: member, and a checkout that refuses is passed with `git checkout`'s own #: 1, so no other code is asked past this holder's records -#: (`stopped_below`, #892). +#: (`stopped_below`, #892). A code past it that a signal gives is read +#: apart (`ended_on`, #998). GIT_DIED = 128 +#: What (a)'s line says where git's update ended on a signal (`ended_on`, +#: #998): a git it ran crashed, or it was killed, and no credential does +#: either. +UPDATE_KILLED = "UPDATE KILLED" #: The trace2 target git's update is run with where no credential is in #: use, a directory, so each git it starts writes its events to a file of #: its own there (`fetched_in`, #892). TRACE_ENV = "GIT_TRACE2_EVENT" +#: How the name of each variable that asks git for a trace begins +#: (`GIT_TRACE`, `GIT_TRACE2_EVENT`, `GIT_TRACE_CURL` and the rest): each is +#: taken out of the environment of a git given the token (`untraced`, #996). +TRACE_PREFIX = "GIT_TRACE" +#: trace2's three targets, each set to `0` for a git given the token: git +#: reads a target the system or the global config names (`trace2. +#: eventTarget`) where its variable is absent or empty, and `0` turns it +#: off (`untraced`, #996). +TRACE2_OFF = ("GIT_TRACE2", "GIT_TRACE2_EVENT", "GIT_TRACE2_PERF") +#: What `git submodule` sets to `0` for every git it starts, so a url from +#: a `.gitmodules` is given no transport git gives only a person's own +#: (`user`); the fetches made again are given it too (`refetched`, #996). +FROM_USER_ENV = "GIT_PROTOCOL_FROM_USER" #: The whys a `.gitmodules` git read only up to a line it cannot parse, or #: up to a key with no value it stops at, may be the cause of, since what #: it did not read is past that line (`Gitmodules.cut`, #856). @@ -388,10 +420,15 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: records with no url, it is for that gitlink (`updated_none`, #879). Its line says no member was updated, or, where `rows` names none, nothing was (lane 1's NIT 5 on #889). Where none of that is so and - git's update died past this holder's own records, in a member's own + git's update ended on a signal (`ended_on`, #998), the one failure + line is for git's update, and its line names the signal and the code; + where it died past this holder's own records, in a member's own index or at a member's commit no fetch brought (`stopped_below`, #892), the one failure line is for the gitlink git stopped at, and - its line says the update stopped there. + its line says the update stopped there. Whether a fetch there reached + its remote is git's own trace's answer, or, where a token is in use + and git's update was not traced, that of the two fetches made again + with the token's rewrite (`fetch_reached`, #996). Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for @@ -403,23 +440,24 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: row's mount cannot be asked about, after git's own words (`unfetched`, #430).""" source, token = credential() - args = ["git"] + rewrite: list[str] = [] if token: for target in REWRITE_TARGETS: - args += ["-c", f"url.https://x-access-token:{token}@github.com/" - f".insteadOf={target}"] - args += ["submodule", "update", "--init", "--recursive"] + rewrite += ["-c", f"url.https://x-access-token:{token}@github.com/" + f".insteadOf={target}"] + args = ["git", *rewrite, "submodule", "update", "--init", "--recursive"] print(f" credential source: {source}") sys.stdout.flush() proc, fetched = run_update(args, root, traced=not token) left, stopper, modules = unfetched(root, rows, proc, source, token) whole = updated_none(stopper, modules) + signum = ended_on(proc.returncode) below = (None if left or whole or proc.returncode != GIT_DIED - else stopped_below(root, fetched)) + else stopped_below(root, fetch_reached(fetched, rewrite))) if proc.returncode != 0: fetch_incomplete(proc, source, token, credential_advice(bool(left), stopper, modules, - below)) + below, signum)) if left: print(" NOT CHECKED OUT, and not for want of a credential: " + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" @@ -434,6 +472,10 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: file=sys.stderr) return [f"{spelled(stopper)} (git stopped at it)" if whole == STOPPED else f"{GITMODULES} (git cannot read it)"] + if signum is not None: + print(f" {UPDATE_KILLED}, and not for want of a credential: " + + killed_said(signum, proc.returncode), file=sys.stderr) + return [f"git's update (ended on signal {signum})"] if below is not None: print(f" {UPDATE_STOPPED}, and not for want of a credential: " + below_said(below), file=sys.stderr) @@ -454,7 +496,11 @@ def run_update(args: list[str], root: Path, traced: bool (`CREDENTIAL_ENV`): trace2 writes each command's arguments, and the rewrite that carries that token is one, as is the url a remote helper is given with it, which git 2.43 writes unredacted (2.47.1 redacts the - url, not the argument). A credential the person's own git config gives, + url, not the argument). Where `traced` says no, git's update runs in + `untraced()`'s environment, so no trace the person asked for writes + that token either: with `GIT_TRACE2_EVENT` set in this run's own + environment, git wrote the rewrite into that file, the token in it + (#996). A credential the person's own git config gives, a url rewrite in a config file or in `GIT_CONFIG_COUNT`, is read by the gits the update starts and can reach the trace files too (lane 5's NIT 1 on #954). The trace is a directory of the run's own, made with @@ -468,7 +514,8 @@ def run_update(args: list[str], root: Path, traced: bool trace = Path(tempfile.mkdtemp(prefix="bootstrap-trace-")) except OSError: trace = None - env = None if trace is None else {**os.environ, TRACE_ENV: str(trace)} + env = (untraced() if not traced else None if trace is None + else {**os.environ, TRACE_ENV: str(trace)}) try: proc = started(args, root, env) return proc, None if trace is None else fetched_in(trace) @@ -558,6 +605,80 @@ def same_place(path: str | Path) -> str: return os.path.normcase(os.path.realpath(path)) +#: How the walk asks, of a checkout whose store lacks the commit recorded +#: for it and that commit, whether a fetch there reached its remote and +#: did not bring it (B4, `entered`): `fetch_reached` makes one. +Reached = Callable[[Path, str], bool] + + +def fetch_reached(fetched: frozenset[str] | None, rewrite: list[str] + ) -> Reached: + """How the walk asks whether a fetch git's update made in a checkout + reached its remote (`entered`). Where `rewrite`, the token's `-c`, is + given, git's update was not traced, and the two fetches it makes there + are made again (`refetched`, #996); else the trace answers, by whether + `fetched` holds that checkout's work tree (`fetched_in`, #892), and + no fetch reached any remote where the run was not traced (`fetched` + None).""" + if rewrite: + return lambda checkout, commit: refetched(checkout, commit, rewrite) + return lambda checkout, commit: (fetched is not None + and same_place(checkout) in fetched) + + +def refetched(checkout: Path, commit: str, rewrite: list[str]) -> bool: + """Whether a fetch in `checkout` reaches its remote and leaves its + store without `commit` (#996): the two fetches git's update makes + there (`run_update_procedure` in git 2.43's submodule--helper.c), + made again in this process, since a run with a token in use is not + traced (`run_update`). `git fetch origin`, then, where the store still + lacks `commit`, `git fetch origin `, each with `rewrite`, the + token's `-c`, in its argv, as git's update was given it, and in + `untraced()`'s environment, so neither writes the token anywhere; what + they print is neither read nor shown. Each runs with + `GIT_PROTOCOL_FROM_USER=0`, as `git submodule` runs every git it + starts, so a url a member's config took from a `.gitmodules` is given + only the transports git's update gives it: a plain fetch took the + `file` transport, which git's update refused (`protocol.file.allow` + is `user`), and brought a commit git's update could not (measured, + git 2.43.0). + + TOLD BY THE EXIT CODE AND THE STORE, never git's words: B4 where the + first exits 0, so reached the remote, and the store still lacks + `commit` after the second. A first that fails, as where the token + does not reach that remote, is the credential's, as git's own was; + and where either brings the commit, git's update did not die for want + of it as far as this tells, so nothing is said. Asked only of a + checkout the walk reaches whose store lacks its commit after git's + update exited 128 with a token in use, where the walk ends, so it + costs two fetches at most.""" + env = {**untraced(), FROM_USER_ENV: "0"} + first = subprocess.run(["git", *rewrite, "fetch", "origin"], + cwd=str(checkout), capture_output=True, + check=False, env=env) + if first.returncode != 0 or commit_held(checkout, commit): + return False + subprocess.run(["git", *rewrite, "fetch", "origin", commit], + cwd=str(checkout), capture_output=True, check=False, + env=env) + return not commit_held(checkout, commit) + + +def untraced() -> dict[str, str]: + """This run's environment for a git given the token in its argv + (#996): every variable whose name begins `GIT_TRACE` (`TRACE_PREFIX`) + taken out, which on Windows `os.environ` spells in upper case + whatever case it was set in, and trace2's targets set to `0` + (`TRACE2_OFF`), which turns off one the system or the global config + names too. A trace writes a command's arguments, the token's rewrite + among them, and the url a remote helper is given, the token in it, to + a file or a socket.""" + env = {name: value for name, value in os.environ.items() + if not name.startswith(TRACE_PREFIX)} + env.update(dict.fromkeys(TRACE2_OFF, "0")) + return env + + def unfetched(root: Path, rows: list[dict], proc: subprocess.CompletedProcess, source: str, token: str ) -> tuple[list[tuple[dict, str, str]], str | None, @@ -592,7 +713,8 @@ def fetch_incomplete(proc: subprocess.CompletedProcess, source: str, def credential_advice(denied: bool, stopper: str | None, - modules: Gitmodules, below: Below | None = None) -> str: + modules: Gitmodules, below: Below | None = None, + signum: int | None = None) -> str: """(a)'s advice to set a credential, or nothing where git asked for none (#839): where its init died at a gitlink it has no url for (`stopper`), or it read nothing from `.gitmodules`, a merge leaving it unmerged, a @@ -602,12 +724,15 @@ def credential_advice(denied: bool, stopper: str | None, Nor where git's update died past this holder's own records at what no credential cures (`below`, `stopped_below`, #892): a gitlink a member gives no url, or a member's commit a fetch that reached its remote did - not bring. Anywhere else a fetch git made failed, a member's, or a - leg's in a member checked out, and the advice stands; where a row is - said to be no credential's beside it (`denied`), it is for the members - that line does not name.""" - if (stopper is not None or below is not None or modules.unmerged - or modules.refused or modules.unreadable): + not bring. Nor where it ended on a signal (`signum`, `ended_on`, + #998): a crash or a kill is no want of a credential, and a fetch git + could not make ends git's update there with 128, so an update that + ends `128 + N` met none. Anywhere else a fetch git made failed, a + member's, or a leg's in a member checked out, and the advice stands; + where a row is said to be no credential's beside it (`denied`), it is + for the members that line does not name.""" + if (stopper is not None or below is not None or signum is not None + or modules.unmerged or modules.refused or modules.unreadable): return "" which = ("the members the next line does not name" if denied else "the members") @@ -617,6 +742,44 @@ def credential_advice(denied: bool, stopper: str | None, "bootstrapped below.") +def ended_on(returncode: int) -> int | None: + """The signal git's update ended on, or None (#998). TOLD BY THE EXIT + CODE ALONE, never git's words, as `stopped_below` tells a 128: + - `128 + N`, where a git it ran died of signal N, as `run_command` + gives a child's signal, and git's update goes on past that gitlink + and ends with that code (measured, git 2.43.0: `submodule--helper + died of signal 11` on a member's own `.gitmodules` giving a `branch` + no value, exit 139); + - `-N`, where git itself was killed by signal N, as Python gives it. + Not 128, git's `die`, which `stopped_below` asks about, nor a code from + `128 + signal.NSIG` on, which numbers no signal this system has, as + 255, which git exits with where a command of its returns -1: those are + read as before, the advice and `ok` where nothing else is told.""" + if returncode < 0: + return -returncode + if GIT_DIED < returncode < GIT_DIED + signal.NSIG: + return returncode - GIT_DIED + return None + + +def killed_said(signum: int, returncode: int) -> str: + """(a)'s line, after its lead, where git's update ended on `signum` + with `returncode` (`ended_on`, #998): the signal by its number and, + where this system names it, its name, then the code; a code past 128 + is a git it ran that died of it, whose own words git printed above, + and a negative one is git itself killed.""" + try: + name = f"{signal.Signals(signum).name}, " + except ValueError: + name = "" + said = (f"git's update ended on signal {signum} ({name}exit " + f"{returncode}), ") + if returncode < 0: + return said + "which killed it." + return (said + "which a git it ran died of. Git's own words above say " + "which git and where.") + + def git_not_started(exc: OSError) -> Refusal | None: """The `git-not-runnable` refusal for an `OSError` that says git could not be STARTED (#694), else `None`, and the error goes on as it did. @@ -1317,8 +1480,7 @@ class Below(NamedTuple): UNTOLD = Below("", ()) -def stopped_below(root: Path, fetched: frozenset[str] | None - ) -> Below | None: +def stopped_below(root: Path, reached: Reached) -> Below | None: """Where git's update, which died (`GIT_DIED`) where `unfetchable` left no row and nothing in this holder's own records stopped it, died past them (#892), or None where that is not told. @@ -1339,14 +1501,15 @@ def stopped_below(root: Path, fetched: frozenset[str] | None - one whose `.git` the operating system will not `stat`, or that git cannot be run in, is untold (`checkout_stop`, lane 5's SHOULD 2); - one whose store lacks its recorded commit (`commit_held`) is the - stop where a fetch git made there reached the remote (`fetched` - holds its work tree, `fetched_in`), and untold where none did, as - for want of a credential, or where the run was not traced - (`fetched` None). Git asks whether a ref reaches the commit - (`is_tip_reachable`) before it fetches, but it dies there only - where its fetch of the commit fails, and a fetch of a commit the - store holds is answered from the store (measured, git 2.43.0: exit - 0), so a commit held and reached by no ref is never the stop; + stop where a fetch git made there reached the remote (`reached`, + `fetch_reached`: git's own trace, `fetched_in`, or, with a token in + use, the two fetches made again, `refetched`, #996), and untold + where none did, as for want of a credential, or where the run was + not traced and no token was in use. Git asks whether a ref reaches + the commit (`is_tip_reachable`) before it fetches, but it dies there + only where its fetch of the commit fails, and a fetch of a commit + the store holds is answered from the store (measured, git 2.43.0: + exit 0), so a commit held and reached by no ref is never the stop; - one at another commit than the one recorded, which git did not check out there, is untold, nothing in it asked: its `git checkout` there refused to overwrite a change, exit 1, which git's update passes, or @@ -1362,11 +1525,11 @@ def stopped_below(root: Path, fetched: frozenset[str] | None other than whole, a merge leaving it unmerged, a value git dies on, a line it cannot parse, it is untold: git stops there other than at a gitlink with no url.""" - found = walked(root, fetched, ()) + found = walked(root, reached, ()) return found if found is not None and found.path else None -def walked(root: Path, fetched: frozenset[str] | None, +def walked(root: Path, reached: Reached, owners: tuple[str, ...]) -> Below | None: """`stopped_below` asked of the checkout `root`, which `owners` lead to from this holder (none for this holder, whose own records @@ -1385,7 +1548,7 @@ def walked(root: Path, fetched: frozenset[str] | None, if not updated_there(root, modules, config, path): continue found = checkout_stop(root / path, prefix + path, commit, owners, - fetched) + reached) if found is not None: return found return None @@ -1419,7 +1582,7 @@ def updated_there(root: Path, modules: Gitmodules, def checkout_stop(checkout: Path, where: str, commit: str, owners: tuple[str, ...], - fetched: frozenset[str] | None) -> Below | None: + reached: Reached) -> Below | None: """`walked`'s answer for the gitlink at `where`, recorded at `commit` by the last of `owners` (this holder where there is none), whose checkout is `checkout`: None where its `.git` is not there, asked as @@ -1431,25 +1594,25 @@ def checkout_stop(checkout: Path, where: str, commit: str, try: if not _is_a(checkout / ".git", _anything): return None - return entered(checkout, where, commit, owners, fetched) + return entered(checkout, where, commit, owners, reached) except OSError: return UNTOLD def entered(checkout: Path, where: str, commit: str, owners: tuple[str, ...], - fetched: frozenset[str] | None) -> Below | None: + reached: Reached) -> Below | None: """`checkout_stop`'s answer for a checkout whose `.git` is there: B4 - where its store lacks `commit` and a fetch git made in it reached the - remote, `UNTOLD` where none did or where it is at another commit, and - else what `walked` finds in it.""" + where its store lacks `commit` and a fetch in it reached the remote + and did not bring it (`reached`), `UNTOLD` where none did or where it + is at another commit, and else what `walked` finds in it.""" if not commit_held(checkout, commit): - if fetched is not None and same_place(checkout) in fetched: + if reached(checkout, commit): return Below(where, owners, commit) return UNTOLD if head_commit(checkout) != commit: return UNTOLD - return walked(checkout, fetched, (*owners, where)) + return walked(checkout, reached, (*owners, where)) def commit_held(checkout: Path, commit: str) -> bool: diff --git a/tests/test_family_bootstrap_stops_named.py b/tests/test_family_bootstrap_stops_named.py index aab2be76..31d1229c 100644 --- a/tests/test_family_bootstrap_stops_named.py +++ b/tests/test_family_bootstrap_stops_named.py @@ -21,7 +21,10 @@ WHAT TELLS THEM FROM A MISSING CREDENTIAL, never git's words: for B5, the member's own index, read as the holder's is (`init_stop`); for B4, a commit the member's store lacks after the update, where git's own trace -of the run says its fetch there exited 0. The store alone tells +of the run says its fetch there exited 0, or, with a token in use, whose +rewrite a trace would write, where `git fetch origin` made again there, +untraced, exits 0 and the commit is still absent after `git fetch origin +` (#996). The store alone tells nothing: a member checked out before the run whose pin moved to a commit its remote holds, fetched with no credential, is as short of it, which was measured, and that is still the credential's skip, a control here, @@ -30,17 +33,32 @@ WHERE NEITHER TELLS, the run reads as before, the advice and ok, and a row pins each: a member's own `.gitmodules` git cannot read, or reads only up to a bad line; a stray git never came to, since a clone that failed ended -its update 1; B4 with a credential in use, which is never traced; a member -git did not check out at its pin, refused (exit 1) or dead on an -`index.lock` (128), since git's update may have ended there, even at a -commit its store holds and no ref reaches, which git's fetch does not stop -at; a leg this user cannot search; and a member whose fetch reached -another member's remote but not its own. A gitlink git's update skips +its update 1; a member git did not check out at its pin, refused (exit 1) +or dead on an `index.lock` (128), since git's update may have ended there, +even at a commit its store holds and no ref reaches, which git's fetch +does not stop at; a leg this user cannot search; and a member whose fetch +reached another member's remote but not its own. A gitlink git's update skips (`update = none` in the member's config or its `.gitmodules`, or `active = false`) is passed as git passes it, and a stop past it is named. B5 and B4 a level down, in a leg, are named up to the holder, and B5 is named before a leg git never fetched. +GIT'S UPDATE ENDED ON A SIGNAL (#998), lane 5's P3 on #954: IRRS's own +`.gitmodules` gives its `code` leg a `branch` with no value, git 2.43.0's +update crashes in IRRS (`submodule--helper died of signal 11`), goes on +and exits 139, and bootstrap said `ok` with the credential advice. It +names the signal and the code now, gives no advice, and fails. P3 is +skipped where git does not crash; a wrapper `git` that answers (a)'s +update with 139, or is killed by SIGSEGV, holds both shapes on every git. +A code no signal gives, 128 or 255, reads as before. + +A CREDENTIAL IN USE (#996): B4 with `SHAPE_LEGS_TOKEN` or +`SHAPE_LEGS_APP_TOKEN` set said the advice to set the token that was set, +and ok. It is named now, and the token is in no output and no file under +TMPDIR, with every trace the person may ask for on; a wrapper `git` logs +the two fetches made again, their argv and their trace variables. With +the token's transport refused, the run is still the credential's skip. + THE SAME FAMILY, OFFLINE RULES AND HELPERS as `tests/test_family_holder_remedies_compose.py` (#879). A missing credential is git's refusal of the file transport to the fixture's local remotes, as @@ -53,7 +71,11 @@ from __future__ import annotations +import itertools +import os import re +import shlex +import shutil from pathlib import Path import pytest @@ -83,6 +105,22 @@ #: records. LEAD = " UPDATE STOPPED, and not for want of a credential: " +#: How it begins where git's update ended on a signal (#998). +KILLED_LEAD = " UPDATE KILLED, and not for want of a credential: " + +#: What it says of a signal a git that git's update ran died of, which +#: git's code past 128 gives, and of one that killed git itself, which +#: Python gives as a negative code. +DIED_OF = ("git's update ended on signal {signum} ({name}, exit {code}), " + "which a git it ran died of. Git's own words above say which git " + "and where.") +KILLED_IT = ("git's update ended on signal {signum} ({name}, exit {code}), " + "which killed it.") + +#: The failure a run whose update ended on a signal ends on. +KILLED_FAILED = ("family bootstrap FAILED: git's update (ended on signal " + "{signum})") + #: What it says of a gitlink an `owner`'s own index records with no url #: (B5), and what cures it, up to the gitlink `above` records for `owner`. STRAY_SAID = ( @@ -121,9 +159,9 @@ def said_after_the_fetch(result, source: str = "none") -> list[str]: credential's `source`, as `after_fetch_line` reads them, the line this change adds included.""" ours = [line for line in result.stderr.splitlines() - if line.startswith((*OURS, LEAD))] + if line.startswith((*OURS, LEAD, KILLED_LEAD))] assert ours, result.stderr - assert re.fullmatch(r" FETCH INCOMPLETE \(git exit \d+\) using " + assert re.fullmatch(r" FETCH INCOMPLETE \(git exit -?\d+\) using " rf"credential source '{source}':", ours[0]), ( result.stderr) return ours[1:] @@ -175,6 +213,10 @@ def b5(root: Path, env: dict, family) -> None: bumped(root, env, family, stray_in(root / ROW, family, env)) +#: A number for each commit `unpushed` makes in this session. +MADE = itertools.count(1) + + def unpushed(family, env: dict, tmp_path: Path, remote: str = OTHER) -> tuple[Path, str]: """A clone of the fixture's `remote` (OTHER's unless said) of the @@ -192,11 +234,19 @@ def unpushed(family, env: dict, tmp_path: Path, and the walk never ran. CI, which runs these rows less than a second apart, failed the refused checkout's row so at 4f97fedb and the unreadable leg's at 938a83ef; a commit date fixed by - `GIT_COMMITTER_DATE` and `GIT_AUTHOR_DATE` fails both on git 2.43.0.""" + `GIT_COMMITTER_DATE` and `GIT_AUTHOR_DATE` fails both on git 2.43.0. + + AND A NUMBER OF ITS OWN (`MADE`), since the clone's path alone is not + one: under `tmp_path_retention_policy=failed` pytest removes a passed + row's directory, and gives the next row whose name begins with the + same 30 characters the same one. Two rows of #996 so named, + `test_with_a_credential_in_use_0` both, made one commit in the same + second, and the second row's commit was on a branch of the remote.""" clone = tmp_path / f"unpushed-{remote}" raw(tmp_path, env, "clone", "-q", "--", str(family["remotes"] / f"{remote}.git"), str(clone)) - return clone, committed(clone, env, f"Made in {clone} and never pushed") + return clone, committed(clone, env, f"Made in {clone} and never pushed " + f"({next(MADE)})") def moved_by_hand(root: Path, env: dict, project: str, commit: str) -> None: @@ -734,6 +784,141 @@ def test_a_member_s_gitmodules_read_up_to_a_bad_line_is_read_as_before( assert git_update(holder, env).returncode == 128 +# --- git's update ended on a signal (#998) ---------------------------------- + +def test_p3_a_member_s_gitmodules_git_crashes_on_fails_naming_the_signal( + family, holder, offline, tmp_path): + """Lane 5's P3 on #954: IRRS's own `.gitmodules` gives its `code` leg + a `branch` with no value, committed there and recorded by hand, as + `family.py bump`'s own git reads that file too. Git 2.43.0's update + crashes in IRRS (`submodule--helper died of signal 11`), goes on to + IRSS, and exits 139; bootstrap said ok with the credential advice. It + names the signal and the code now, gives no advice, and fails on git's + update. Skipped where git does not crash so: from 2.43.1 git refuses + that line as it refuses a `url` with none, and dies (128).""" + env = cloning(offline, tmp_path, family) + member = holder / ROW + data = (member / GITMODULES).read_bytes() + line = (chr(9) + "path = code" + chr(10)).encode() + assert data.count(line) == 1, data + bare = (chr(9) + "branch" + chr(10)).encode() + (member / GITMODULES).write_bytes(data.replace(line, line + bare)) + raw(member, env, "add", "--", GITMODULES) + moved_by_hand(holder, env, MEMBER, + committed(member, env, "A branch with no value")) + + result = bootstrap(holder, env, "--skip-members") + update = git_update(holder, env) + + if update.returncode != 139: + pytest.skip(f"this git's update exits {update.returncode} here, " + "where git 2.43.0's crashes (exit 139)") + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ + KILLED_LEAD + DIED_OF.format(signum=11, name="SIGSEGV", code=139), + KILLED_FAILED.format(signum=11)] + + +#: A wrapper `git` here is a POSIX shell script, which Windows does not run +#: by its `#!` line. +POSIX_WRAPPER = pytest.mark.skipif( + os.name == "nt", reason="a wrapper `git` here is a POSIX shell script, " + "which Windows does not execute by its `#!` line") + + +#: The update (a) runs and the fetches the walk runs, as the wrapper's +#: `case` matches them among the commands bootstrap runs itself. +UPDATE_CASE = '*" submodule update --init --recursive "*' +FETCH_CASE = '*" fetch "*' + + +def wrapped(env: dict, tmp_path: Path, action: str = ":", + log: Path | None = None) -> dict: + """`env` with a wrapper `git` first on PATH, which runs `action`, a + shell command, for the `git submodule update --init --recursive` that + (a) runs, and the real git, by its absolute path, for every other + command, bootstrap's own and the walk's; the gits git starts find its + own first on their PATH. With `log`, it first writes a line there for + that update and for each fetch: the command's arguments, a token in + them masked as ``, then the `GIT_TRACE*` variables and the + `GIT_PROTOCOL_FROM_USER` its environment holds.""" + real = shutil.which("git", path=env["PATH"]) + assert real, env["PATH"] + shim = tmp_path / "wrapper" + shim.mkdir() + script = shim / "git" + logged = [] if log is None else [ + 'case " $* " in', f"{UPDATE_CASE}|{FETCH_CASE})", + 'args=$(printf "%s" "$*" | sed ' + '"s/x-access-token:[^@]*@/x-access-token:@/g")', + 'traces=$(env | grep -E "^(GIT_TRACE|GIT_PROTOCOL_FROM_USER)" ' + '| sort | paste -s -d " " -)', + f'echo "$args | $traces" >> {shlex.quote(str(log))} ;;', "esac"] + script.write_bytes(chr(10).join([ + "#!/bin/sh", *logged, + 'case " $* " in', f"{UPDATE_CASE}) {action} ;;", "esac", + f'exec {shlex.quote(real)} "$@"', ""]).encode()) + script.chmod(0o755) + return {**env, "PATH": str(shim) + os.pathsep + env["PATH"]} + + +#: `(what the wrapper answers (a)'s update with, what (a)'s line says of +#: it)`: 139, the code git's update ends with where a git it ran died of +#: SIGSEGV, as git 2.43.0's in P3; and the wrapper killed by SIGSEGV itself, +#: with no core file, whose code Python gives as -11. +KILLED_BY = { + "exit-139": ("exit 139", + DIED_OF.format(signum=11, name="SIGSEGV", code=139)), + "sigsegv": ("ulimit -c 0; kill -s SEGV $$", + KILLED_IT.format(signum=11, name="SIGSEGV", code=-11)), +} + + +@POSIX_WRAPPER +@pytest.mark.parametrize("action, said", KILLED_BY.values(), ids=KILLED_BY) +def test_an_update_ended_on_a_signal_fails_naming_it(family, holder, offline, + tmp_path, action, said): + """#998's two shapes on any git, every member checked out at its pin: + git's update ended on SIGSEGV, a git it ran dead of it (139) or git + itself killed by it (-11). Bootstrap said ok with the credential + advice for both. It names the signal and the code now, gives no + advice, and fails on git's update. The credential's own 128 keeps the + advice and ok (`test_the_credential_s_skip_is_as_it_was`).""" + env = cloning(offline, tmp_path, family) + + result = bootstrap(holder, wrapped(env, tmp_path, action), + "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [KILLED_LEAD + said, + KILLED_FAILED.format(signum=11)] + + +#: What the wrapper answers (a)'s update with, where no signal gives the +#: code: git's own `die`, 128, which the walk past the holder's records +#: asks, and finds nothing in with every member checked out at its pin; +#: and 255, past every signal's `128 + N`, which git exits with where a +#: command of its returns -1. +NO_SIGNAL = {"exit-128": "exit 128", "exit-255": "exit 255"} + + +@POSIX_WRAPPER +@pytest.mark.parametrize("action", NO_SIGNAL.values(), ids=NO_SIGNAL) +def test_a_code_no_signal_gives_is_read_as_before(family, holder, offline, + tmp_path, action): + """No misfire: where git's update exits with a code no signal gives, + nothing is said of a signal, and the run reads as before, the advice + and ok.""" + env = cloning(offline, tmp_path, family) + + result = bootstrap(holder, wrapped(env, tmp_path, action), + "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert result.stdout.endswith("\nfamily bootstrap ok\n"), result.stdout + + # --- a credential in use ----------------------------------------------------- def test_with_a_credential_in_use_b5_is_named_and_the_token_is_not( @@ -755,16 +940,212 @@ def test_with_a_credential_in_use_b5_is_named_and_the_token_is_not( assert TOKEN not in result.stdout + result.stderr -def test_with_a_credential_in_use_b4_is_read_as_before(family, holder, - offline, tmp_path): - """What it cannot tell: a run with a credential in use is not traced, - since trace2 writes a command's arguments and the credential is one, so - B4 is the advice and ok, as before.""" +#: The two variables a credential arrives in, the App's first. +CREDENTIALS = ("SHAPE_LEGS_APP_TOKEN", "SHAPE_LEGS_TOKEN") + +#: The trace files `traced_by_the_caller` asks for that a run writes where +#: every remote is a local path: GIT_TRACE_CURL's is written for http only. +TRACES = ("trace.txt", "trace2-event.json", "trace2-normal.txt", + "trace2-perf.txt") + + +def traced_by_the_caller(env: dict, offline, tmp_path: Path) -> dict: + """`env` with every trace a person may ask for written under the run's + TMPDIR: `GIT_TRACE`, `GIT_TRACE2_EVENT` and `GIT_TRACE_CURL` in the + environment, and trace2's normal and perf targets in the global + config, a copy of the one `env` names with `trace2.normalTarget` and + `trace2.perfTarget` added.""" + temp = offline.temp + config = tmp_path / "traced-gitconfig" + config.write_bytes( + Path(env["GIT_CONFIG_GLOBAL"]).read_bytes() + chr(10).join([ + "[trace2]", + f'{chr(9)}normalTarget = "{(temp / TRACES[2]).as_posix()}"', + f'{chr(9)}perfTarget = "{(temp / TRACES[3]).as_posix()}"', + ""]).encode()) + return {**env, "GIT_CONFIG_GLOBAL": str(config), + "GIT_TRACE": str(temp / TRACES[0]), + "GIT_TRACE2_EVENT": str(temp / TRACES[1]), + "GIT_TRACE_CURL": str(temp / "trace-curl.txt")} + + +def files_holding(temp: Path, text: str) -> list[str]: + """Every file under `temp` whose bytes hold `text`, as a path relative + to it.""" + return [path.relative_to(temp).as_posix() + for path in sorted(temp.rglob("*")) + if path.is_file() and text.encode() in path.read_bytes()] + + +@pytest.mark.parametrize("variable", CREDENTIALS) +def test_with_a_credential_in_use_b4_is_named_and_the_token_is_not( + family, holder, offline, tmp_path, variable): + """#996, lane 5's SHOULD 4 on #954: B4 at IRSS with a token in + `variable`. Git's update, which carries the token's rewrite in its + argv, is not traced, and the run gave the advice to set the token that + was set, and ok. The two fetches git's update makes in IRSS are made + again, untraced: the first reaches the remote, the commit is still + absent after the second, and B4 is named, exit 1, with no advice. + Every trace the person asked for is on, the environment's and the + global config's, and the walk's own gits, which carry no token, write + each; the token is in none of them, nor in any other file under + TMPDIR, nor in the output. Before, git's update wrote it into + `GIT_TRACE2_EVENT`'s file.""" + env = {**cloning(offline, tmp_path, family), variable: TOKEN} + commit = b4(holder, env, family, tmp_path) + + result = bootstrap(holder, traced_by_the_caller(env, offline, tmp_path), + "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result, variable) == [ + LEAD + ABSENT_SAID.format(path=OTHER_ROW, commit=commit, + owner="this holder", tail=BUMP_NAMED), + STOP_FAILED.format(path=OTHER_ROW)] + assert TOKEN not in result.stdout + result.stderr + assert files_holding(offline.temp, TOKEN) == [] + assert [name for name in TRACES + if (offline.temp / name).is_file()] == list(TRACES) + + +#: The states the credential's skip is asked in with a token in use: +#: IRSS's pin moved on to a commit its remote holds, and B4 at IRSS. +WITH_A_TOKEN_REFUSED = {"pin-moved-on": pin_moved_on, "b4": b4} + + +@pytest.mark.parametrize("make", WITH_A_TOKEN_REFUSED.values(), + ids=WITH_A_TOKEN_REFUSED) +def test_with_a_credential_in_use_the_credential_s_skip_is_as_it_was( + family, holder, offline, tmp_path, make): + """No misfire with a token in use (#996): IRSS's pin moved on, or B4 + at IRSS, and a remote the token's transport cannot reach, the file + transport refused. Git's update dies at IRSS (128), and the first + fetch made again there fails as git's own did, since it is given only + the transports git's update gives a url (`GIT_PROTOCOL_FROM_USER=0`), + so it is the credential's skip: the advice and ok, as before, and the + token is in no output and no file under TMPDIR.""" + env = cloning(offline, tmp_path, family) + make(holder, env, family, tmp_path) + without = {**isolated(offline, tmp_path), "SHAPE_LEGS_TOKEN": TOKEN} + + result = bootstrap(holder, + traced_by_the_caller(without, offline, tmp_path), + "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result, "SHAPE_LEGS_TOKEN") == [ALONE] + assert result.stdout.endswith("\nfamily bootstrap ok\n"), result.stdout + assert TOKEN not in result.stdout + result.stderr + assert files_holding(offline.temp, TOKEN) == [] + assert git_update(holder, without).returncode == 128 + + +#: The token's two `-c`s, as the wrapper's log masks them, and the trace +#: variables a git given the token runs with: trace2's three targets off, +#: and no other; and for each fetch made again, `GIT_PROTOCOL_FROM_USER` +#: too, as `git submodule` gives every git it starts. +REWRITE_LOGGED = " ".join( + f"-c url.https://x-access-token:@github.com/.insteadOf={target}" + for target in ("https://github.com/", "git@github.com:")) +UNTRACED = "GIT_TRACE2=0 GIT_TRACE2_EVENT=0 GIT_TRACE2_PERF=0" +REFETCHED = f"GIT_PROTOCOL_FROM_USER=0 {UNTRACED}" +UPDATE_ARGS = "submodule update --init --recursive" + + +def logged(log: Path) -> list[str]: + """The wrapper's log, a line for each command it logged.""" + return log.read_bytes().decode().splitlines() + + +@POSIX_WRAPPER +def test_with_a_credential_in_use_the_two_fetches_are_made_again_untraced( + family, holder, offline, tmp_path): + """#996's fetches, as a wrapper `git` logs the commands bootstrap runs + itself, B4 at IRSS with a token in use: git's update, then `git fetch + origin` and `git fetch origin ` in IRSS, as git's update makes + them there, each with the token's rewrite in its argv, as the update + was given it, and with `GIT_PROTOCOL_FROM_USER=0`. Every trace the + person may ask for is on, and each of the three runs with no + `GIT_TRACE*` variable but trace2's three targets set to 0.""" env = {**cloning(offline, tmp_path, family), "SHAPE_LEGS_TOKEN": TOKEN} + commit = b4(holder, env, family, tmp_path) + log = tmp_path / "git-calls.log" + traced = traced_by_the_caller(env, offline, tmp_path) + + result = bootstrap(holder, wrapped(traced, tmp_path, log=log), + "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert logged(log) == [ + f"{REWRITE_LOGGED} {UPDATE_ARGS} | {UNTRACED}", + f"{REWRITE_LOGGED} fetch origin | {REFETCHED}", + f"{REWRITE_LOGGED} fetch origin {commit} | {REFETCHED}"] + + +@POSIX_WRAPPER +def test_with_no_credential_the_walk_makes_no_fetch_of_its_own( + family, holder, offline, tmp_path): + """With no token, B4 is told by git's own trace of its update (#892), + which the run asks for in a directory of its own under TMPDIR: the + walk makes no fetch of its own (#996), and B4 is named.""" + env = cloning(offline, tmp_path, family) b4(holder, env, family, tmp_path) + log = tmp_path / "git-calls.log" + + result = bootstrap(holder, wrapped(env, tmp_path, log=log), + "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + (update,) = logged(log) + traced = f"{UPDATE_ARGS} | GIT_TRACE2_EVENT={offline.temp}" + assert re.fullmatch(re.escape(traced + os.sep + "bootstrap-trace-") + + r"\S+", update), update + + +def pinned_off_branch(holder: Path, env: dict, family, + tmp_path: Path) -> Path: + """IRSS's gitlink and pin moved on to a commit its remote holds at a + ref that is no branch, `refs/pins/`: `git fetch origin`, which + fetches its branches, does not bring it, and `git fetch origin + ` does, the remote advertising that ref.""" + clone, commit = unpushed(family, env, tmp_path) + raw(clone, env, "push", "-q", str(family["remotes"] / f"{OTHER}.git"), + f"{commit}:refs/pins/{commit[:12]}") + moved_by_hand(holder, env, OTHER, commit) + return holder - result = bootstrap(holder, env, "--skip-members") + +#: `(how IRSS's pin moved on, whether the second fetch is made)`: to a +#: commit on a branch of its remote, which the first fetch made again +#: brings, and to one at a ref that is no branch, which only the second +#: brings. +BROUGHT_BY = {"first-fetch": (pin_moved_on, False), + "second-fetch": (pinned_off_branch, True)} + + +@POSIX_WRAPPER +@pytest.mark.parametrize("make, second", BROUGHT_BY.values(), ids=BROUGHT_BY) +def test_with_a_credential_in_use_a_commit_a_fetch_brings_is_untold( + family, holder, offline, tmp_path, make, second): + """Where a fetch made again brings the commit, nothing is said (#996): + git's update dies before it fetches in IRSS, as the wrapper stands in + for by answering it 128 having fetched nothing, and IRSS's pin moved + on to a commit its remote holds. Where `git fetch origin` brings it, + `git fetch origin ` is not made; where only that second fetch + brings it, it is made, and the commit is no B4. The run reads as + before, the advice and ok.""" + env = {**cloning(offline, tmp_path, family), "SHAPE_LEGS_TOKEN": TOKEN} + make(holder, env, family, tmp_path) + commit = pinned(holder, OTHER) + log = tmp_path / "git-calls.log" + + result = bootstrap(holder, wrapped(env, tmp_path, "exit 128", log), + "--skip-members") assert result.returncode == 0, result.stdout + result.stderr assert said_after_the_fetch(result, "SHAPE_LEGS_TOKEN") == [ALONE] - assert TOKEN not in result.stdout + result.stderr + assert logged(log) == [ + f"{REWRITE_LOGGED} {UPDATE_ARGS} | {UNTRACED}", + f"{REWRITE_LOGGED} fetch origin | {REFETCHED}", + *([f"{REWRITE_LOGGED} fetch origin {commit} | {REFETCHED}"] + if second else [])]