diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index e60be90c..80736343 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -37,12 +37,33 @@ as below, with no advice (#879). It said `family bootstrap ok` there, beside git's exit 128, as a run with every member checked out passes (#648), and gave the advice where git stopped at a - gitlink. This holder's records are the test, so an exit 128 they do - not show is read as before, the advice and `ok` where every member - is checked out: git stopped inside a member, at a gitlink in that - member's own index with no url, or at a gitlink of this holder's at - a commit no remote holds (`not our ref`) (lane 1's SHOULD 4 on - #889). + gitlink. + + So it fails where git's update died (exit 128) past this holder's + own records, in the two places that are told from a missing + credential there, each asked as git asks it, in git's order, and + never of git's words (`stopped_below`, #892): + - at a gitlink in a member's own index that git has no url for, + read in the member checked out at the commit this holder records + as this holder's own index is read (`init_stop`): git's `No url + found` there said `ok`; + - at a member whose store still lacks the commit this holder + records for it where git's fetch there reached the member's + remote, which git's own trace of the run says, by the fetch's + exit code (`fetched_in`): its `not our ref` said `ok` too. A + fetch git could not make, as for want of a credential, leaves + the store as short, so the store alone tells nothing. + A leg in a member is asked as a member is, and a gitlink git's + update passes without a look, its `update` `none` or the gitlink + inactive, is passed (lane 5's SHOULD 1 on #954). Where neither + tells, an exit 128 is read as before, the advice and `ok` where no + row is left not checked out but for want of a credential, and the + run names only where git's init in this holder stopped. A run with + `SHAPE_LEGS_TOKEN` or `SHAPE_LEGS_APP_TOKEN` set is never traced, + since the trace would write that token, so the second is told only + where neither is set; with one set, B4 still gives the advice to + set the token that is set, and `ok` (lane 1's SHOULD 4 on #889, + lane 5's SHOULD 4 on #954, a known limit). "every member and leg fetched" is said only where git exited 0 AND every row's member is checked out (#648). A member left NOT CHECKED @@ -116,8 +137,10 @@ value in `.gitmodules` that git cannot read (#768), or found that, with every member checked out, git's init in this holder died at a gitlink this holder records with no url or at an unmerged - `.gitmodules` (#879); an exit 128 this holder's records do not show - is no failure here, as (a) says. Any + `.gitmodules` (#879), or found where git's update died past this + holder's own records, in a member's own index or at a member's + commit no fetch brought (#892); an exit 128 none of those shows is + no failure here, as (a) says. Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). @@ -143,6 +166,7 @@ import argparse import errno +import json import os import re import shlex @@ -150,6 +174,7 @@ import stat import subprocess import sys +import tempfile from collections import Counter from collections.abc import Callable from pathlib import Path @@ -184,6 +209,21 @@ #: names any row: under "0 member(s)" it said no member was updated (lane #: 1's NIT 5 on #889). NONE_UPDATED = {True: "NO MEMBER UPDATED", False: "NOTHING UPDATED"} +#: What (a)'s line says where git's update died past this holder's own +#: records (`stopped_below`, #892): git updated what came before the stop +#: in its order, so it is no `NONE_UPDATED`. +UPDATE_STOPPED = "UPDATE STOPPED" +#: The exit code of git's update where it died (`die`), which `Failed to +#: recurse into submodule path` passes up from a member unchanged. A clone +#: that fails ends git's update 1, before it checks out or enters any +#: member, and a checkout that refuses is passed with `git checkout`'s own +#: 1, so no other code is asked past this holder's records +#: (`stopped_below`, #892). +GIT_DIED = 128 +#: The trace2 target git's update is run with where no credential is in +#: use, a directory, so each git it starts writes its events to a file of +#: its own there (`fetched_in`, #892). +TRACE_ENV = "GIT_TRACE2_EVENT" #: The whys a `.gitmodules` git read only up to a line it cannot parse, or #: up to a key with no value it stops at, may be the cause of, since what #: it did not read is past that line (`Gitmodules.cut`, #856). @@ -345,7 +385,11 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: #879); where its init in this holder died at a gitlink this holder records with no url, it is for that gitlink (`updated_none`, #879). Its line says no member was updated, or, where `rows` names none, - nothing was (lane 1's NIT 5 on #889). + nothing was (lane 1's NIT 5 on #889). Where none of that is so and + git's update died past this holder's own records, in a member's own + index or at a member's commit no fetch brought (`stopped_below`, + #892), the one failure line is for the gitlink git stopped at, and + its line says the update stopped there. Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for @@ -365,19 +409,15 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: args += ["submodule", "update", "--init", "--recursive"] print(f" credential source: {source}") sys.stdout.flush() - try: - proc = subprocess.run(args, cwd=str(root), capture_output=True, - text=True, encoding="utf-8", errors="replace", - check=False) - except OSError as exc: - refusal = git_not_started(exc) - if refusal is None: - raise - raise refusal from exc + proc, fetched = run_update(args, root, traced=not token) left, stopper, modules = unfetched(root, rows, proc, source, token) + whole = updated_none(stopper, modules) + below = (None if left or whole or proc.returncode != GIT_DIED + else stopped_below(root, fetched)) if proc.returncode != 0: fetch_incomplete(proc, source, token, - credential_advice(bool(left), stopper, modules)) + credential_advice(bool(left), stopper, modules, + below)) if left: print(" NOT CHECKED OUT, and not for want of a credential: " + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" @@ -385,7 +425,6 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: + ". " + cures(left, stopper, rows, modules), file=sys.stderr) return [f"{spelled(row.get('project') or '?')} (not checked out)" for row, _, _ in left] - whole = updated_none(stopper, modules) if whole is not None: print(f" {NONE_UPDATED[bool(rows)]}, and not for want of a " "credential: " + reason(whole, stopper, modules) + ". " @@ -393,11 +432,130 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: file=sys.stderr) return [f"{spelled(stopper)} (git stopped at it)" if whole == STOPPED else f"{GITMODULES} (git cannot read it)"] + if below is not None: + print(f" {UPDATE_STOPPED}, and not for want of a credential: " + + below_said(below), file=sys.stderr) + return [f"{spelled(below.path)} (git stopped at it)"] if proc.returncode == 0: print(" every member and leg fetched") return [] +def run_update(args: list[str], root: Path, traced: bool + ) -> tuple[subprocess.CompletedProcess, frozenset[str] | None]: + """(a)'s `git submodule update`, run as `args` say in `root`, and the + work trees in which a fetch it made reached its remote (`fetched_in`, + #892), or None where the run was not traced. + + TRACED ONLY WHERE `traced` SAYS, which `fetch_members` says where + neither `SHAPE_LEGS_TOKEN` nor `SHAPE_LEGS_APP_TOKEN` is set + (`CREDENTIAL_ENV`): trace2 writes each command's arguments, and the + rewrite that carries that token is one, as is the url a remote helper + is given with it, which git 2.43 writes unredacted (2.47.1 redacts the + url, not the argument). A credential the person's own git config gives, + a url rewrite in a config file or in `GIT_CONFIG_COUNT`, is read by the + gits the update starts and can reach the trace files too (lane 5's + NIT 1 on #954). The trace is a directory of the run's own, made with + `tempfile.mkdtemp`, readable by its owner alone, and removed whatever + the run does, so only a run killed before it ends leaves it behind; + one that cannot be made leaves the run untraced. Raises the + `git-not-runnable` refusal where git cannot be started (#694).""" + trace = None + if traced: + try: + trace = Path(tempfile.mkdtemp(prefix="bootstrap-trace-")) + except OSError: + trace = None + env = None if trace is None else {**os.environ, TRACE_ENV: str(trace)} + try: + proc = started(args, root, env) + return proc, None if trace is None else fetched_in(trace) + finally: + if trace is not None: + shutil.rmtree(trace, ignore_errors=True) + + +def started(args: list[str], root: Path, env: dict[str, str] | None + ) -> subprocess.CompletedProcess: + """`args` run in `root` under `env` (None for this process's own), or + the `git-not-runnable` refusal where git cannot be started (#694).""" + try: + return subprocess.run(args, cwd=str(root), capture_output=True, + text=True, encoding="utf-8", errors="replace", + check=False, env=env) + except OSError as exc: + refusal = git_not_started(exc) + if refusal is None: + raise + raise refusal from exc + + +def fetched_in(trace: Path) -> frozenset[str]: + """Each work tree (`same_place`) in which a `git fetch` git's update + made exited 0, so reached the remote it fetched from, read from the + trace2 events in `trace`, a file for each git the update started + (#892). + + TOLD BY THE EXIT CODE, NEVER GIT'S WORDS, as `gitmodules_listed` + tells a listing (#856): git's event format is JSON, a line an event, + and a process's `cmd_name` event names its command, its `exit` event + gives its code, and its `def_repo` event for repository 1 gives the + work tree it runs in (a `git submodule` process opens each member as + another repository, 2 and up, which are not its own). Measured, git + 2.43.0: where a member's commit is on no branch of its remote, git's + `fetch` there exits 0 and its fetch of the commit 128 (`not our ref`); + where git cannot reach the remote, as for want of a credential, both + exit 128. A line that is no event is skipped, and a git with no trace2 + (before 2.22) writes nothing, so nothing is said to be fetched.""" + runs: dict[str, dict] = {} + for event in trace_events(trace): + run = runs.setdefault(str(event.get("sid")), {}) + kind = event.get("event") + if kind == "cmd_name": + run["name"] = event.get("name") + elif kind == "def_repo" and event.get("repo") == 1: + run["worktree"] = event.get("worktree") + elif kind == "exit": + run["code"] = event.get("code") + return frozenset(same_place(run["worktree"]) for run in runs.values() + if run.get("name") == "fetch" and run.get("code") == 0 + and isinstance(run.get("worktree"), str)) + + +def trace_events(trace: Path) -> list[dict]: + """Every trace2 event in the files in `trace`, in the order of their + names, then of their lines; a file that cannot be read, or a line that + is no event, is skipped.""" + try: + names = sorted(os.listdir(trace)) + except OSError: + return [] + events: list[dict] = [] + for name in names: + try: + text = (trace / name).read_text(encoding="utf-8", + errors="replace") + except OSError: + continue + events += filter(None, map(trace_event, text.splitlines())) + return events + + +def trace_event(line: str) -> dict | None: + """One trace2 event, or None where `line` is not one.""" + try: + event = json.loads(line) + except ValueError: + return None + return event if isinstance(event, dict) else None + + +def same_place(path: str | Path) -> str: + """`path` as one spelling for one directory: its real path, in the + case the disk compares names in where it folds case.""" + return os.path.normcase(os.path.realpath(path)) + + def unfetched(root: Path, rows: list[dict], proc: subprocess.CompletedProcess, source: str, token: str ) -> tuple[list[tuple[dict, str, str]], str | None, @@ -432,19 +590,22 @@ def fetch_incomplete(proc: subprocess.CompletedProcess, source: str, def credential_advice(denied: bool, stopper: str | None, - modules: Gitmodules) -> str: + modules: Gitmodules, below: Below | None = None) -> str: """(a)'s advice to set a credential, or nothing where git asked for none (#839): where its init died at a gitlink it has no url for (`stopper`), or it read nothing from `.gitmodules`, a merge leaving it unmerged, a value git dies on or a file it cannot parse, it fetched nothing. Every row (a) then leaves not checked out is said to be no credential's, and the advice beside that was its denial (w768's candidate 1 on #819). - Anywhere else a fetch git made failed, a member's, or a leg's in a - member checked out, and the advice stands; where a row is said to be no - credential's beside it (`denied`), it is for the members that line does - not name.""" - if (stopper is not None or modules.unmerged or modules.refused - or modules.unreadable): + Nor where git's update died past this holder's own records at what no + credential cures (`below`, `stopped_below`, #892): a gitlink a member + gives no url, or a member's commit a fetch that reached its remote did + not bring. Anywhere else a fetch git made failed, a member's, or a + leg's in a member checked out, and the advice stands; where a row is + said to be no credential's beside it (`denied`), it is for the members + that line does not name.""" + if (stopper is not None or below is not None or modules.unmerged + or modules.refused or modules.unreadable): return "" which = ("the members the next line does not name" if denied else "the members") @@ -662,7 +823,9 @@ class Gitmodules(NamedTuple): file, or nothing. `cut_key` is the `submodule..` with no value that cut it, where that is what git stopped at (`ends_blob_reading`), a line `git config --list` takes, and empty - where a line it cannot parse did.""" + where a line it cannot parse did. `updates` is `{name: update}`, the + `update` git takes from each section, where any gives one, for the walk + past this holder's records (`updated_there`, #892).""" given: Counter read: dict[str, str] @@ -672,6 +835,7 @@ class Gitmodules(NamedTuple): unreadable: bool = False cut: str = "" cut_key: str = "" + updates: dict[str, str] | None = None def gitmodules_sections(root: Path) -> Gitmodules: @@ -841,11 +1005,13 @@ def read_sections(entries: list[tuple[str, str, str | None, bool]] option`); where two sections give one path, the later in the file decides, as a later value in one section replaces an earlier, and a section that gives a path again lets go of the one it gave, even where - a later section gave that one too: git then reads no section there.""" + a later section gave that one too: git then reads no section there. + `updates` keeps each section's last `update`, as git keeps it (#892).""" sections: dict[str, dict[str, str]] = {} read: dict[str, str] = {} held: dict[str, str] = {} urls: dict[str, str] = {} + updates: dict[str, str] = {} for name, key, value, ignored in entries: sections.setdefault(name, {})[key] = "" if value is None else value if ignored or value is None or value.startswith("-"): @@ -857,8 +1023,10 @@ def read_sections(entries: list[tuple[str, str, str | None, bool]] read[value] = name elif key == "url": urls[name] = value + elif key == "update": + updates[name] = value return Gitmodules(Counter(keys["path"] for keys in sections.values() - if "path" in keys), read, urls) + if "path" in keys), read, urls, updates=updates) def refused_value(root: Path, key: str, value: str | None) -> bool: @@ -965,22 +1133,36 @@ def init_stop(root: Path, modules: Gitmodules, section git reads gives a path. A gitlink git has no url for (`has_url`) stops it. So a stray case variant or a member mounted by hand stops a plain clone's update, and leaves a recursive clone's to - the credential. Read nothing, it answers None.""" + the credential. Read nothing, it answers None. Asked of a member's + checkout, with the member's own `.gitmodules` and config, it is where + git's init in that member stops (`stopped_below`, #892).""" + for path, _ in init_asks(root, modules, config): + if not has_url(modules, config, path): + return path + return None + + +def init_asks(root: Path, modules: Gitmodules, + config: dict[str, list[str | None]]) -> list[tuple[str, str]]: + """`(path, commit)` of each gitlink git's init asks about, in index + order (`init_stop`): every gitlink the index records, but where the + config sets `submodule.active` only those git counts active + (`active`). Empty where the index cannot be read.""" try: index = git_out(["ls-files", "-s", "-z"], cwd=root, binary=True) except Refusal: - return None + return [] + asked = [] for record in index.split(b"\0"): meta, _, entry = record.partition(b"\t") - if meta.split()[:1] != [GITLINK_MODE]: + fields = meta.split() + if fields[:1] != [GITLINK_MODE]: continue path = entry.decode("utf-8", "surrogateescape") - if ("submodule.active" in config - and not active(root, modules, config, path)): - continue - if not has_url(modules, config, path): - return path - return None + if ("submodule.active" not in config + or active(root, modules, config, path)): + asked.append((path, fields[1].decode("ascii", "replace"))) + return asked def active(root: Path, modules: Gitmodules, @@ -1114,6 +1296,224 @@ def updated_none(stopper: str | None, modules: Gitmodules) -> str | None: return file_reason(modules) or (None if stopper is None else STOPPED) +class Below(NamedTuple): + """Where git's update died past this holder's own records (#892), as + `stopped_below` finds it. `path` is the gitlink git stopped at, from + this holder; `owners` the checkouts from this holder's member down to + the one whose own index records `path`, none where this holder's + does; `commit` the commit recorded at `path`, where git's fetch there + reached the remote and did not bring it (B4), and empty where `path` + is a gitlink its owner's index records with no url (B5).""" + + path: str + owners: tuple[str, ...] + commit: str = "" + + +#: What a checkout answers where git's update stopped at it and why is +#: not told (`walked`): nothing past it is asked, and nothing is said. +UNTOLD = Below("", ()) + + +def stopped_below(root: Path, fetched: frozenset[str] | None + ) -> Below | None: + """Where git's update, which died (`GIT_DIED`) where `unfetchable` + left no row and nothing in this holder's own records stopped it, died + past them (#892), or None where that is not told. + + ASKED AS GIT 2.43 WALKS IT, never of git's words. Git updates the + gitlinks its init asks about in index order, each in turn, and in + each one checked out at the commit recorded for it, runs the same + update, whose init asks every gitlink of that checkout's own index + first; it dies at the first gitlink it has no url for, or at the + first commit a fetch did not bring, and that ends every update above + it (`Failed to recurse`, exit 128). So each gitlink is asked in that + order (`walked`): + - one git's update passes without a look (`updated_there`), its + update strategy `none` or the gitlink inactive, is passed, and + nothing in it asked: git prints `Skipping submodule` and goes on + (lane 5's SHOULD 1 on #954); + - one not checked out is passed, as git passes a clone that failed; + - one whose `.git` the operating system will not `stat`, or that git + cannot be run in, is untold (`checkout_stop`, lane 5's SHOULD 2); + - one whose store lacks its recorded commit (`commit_held`) is the + stop where a fetch git made there reached the remote (`fetched` + holds its work tree, `fetched_in`), and untold where none did, as + for want of a credential, or where the run was not traced + (`fetched` None). Git asks whether a ref reaches the commit + (`is_tip_reachable`) before it fetches, but it dies there only + where its fetch of the commit fails, and a fetch of a commit the + store holds is answered from the store (measured, git 2.43.0: exit + 0), so a commit held and reached by no ref is never the stop; + - one at another commit than the one recorded, which git did not check + out there, is untold, nothing in it asked: its `git checkout` there + refused to overwrite a change, exit 1, which git's update passes, or + died, exit 128, as on an `index.lock` left in its store, which ends + git's update there, or a rebase or a merge `update` asks for failed, + and which it was is not told, so no stop past it is said where git + may never have come to it (measured, git 2.43.0: where it was + passed, an `index.lock` in IRRS named B5's stray in IRSS); + - and one checked out at it is asked as this holder is, its own + `.gitmodules` and config read as `gitmodules_sections` and + `submodule_config` read this holder's, and `init_stop` asked of its + index, then each of its gitlinks. Where git reads that `.gitmodules` + other than whole, a merge leaving it unmerged, a value git dies on, + a line it cannot parse, it is untold: git stops there other than at + a gitlink with no url.""" + found = walked(root, fetched, ()) + return found if found is not None and found.path else None + + +def walked(root: Path, fetched: frozenset[str] | None, + owners: tuple[str, ...]) -> Below | None: + """`stopped_below` asked of the checkout `root`, which `owners` lead + to from this holder (none for this holder, whose own records + `unfetchable` asked): the stop, `UNTOLD`, or None where nothing there + stops git's update.""" + modules = gitmodules_sections(root) + config = submodule_config(root) + if owners: + if file_reason(modules) is not None or modules.cut: + return UNTOLD + stop = init_stop(root, modules, config) + if stop is not None: + return Below(f"{owners[-1]}/{stop}", owners) + prefix = f"{owners[-1]}/" if owners else "" + for path, commit in init_asks(root, modules, config): + if not updated_there(root, modules, config, path): + continue + found = checkout_stop(root / path, prefix + path, commit, owners, + fetched) + if found is not None: + return found + return None + + +def updated_there(root: Path, modules: Gitmodules, + config: dict[str, list[str | None]], path: str) -> bool: + """Whether git's update, its init done, goes on to the gitlink at + `path`, as `prepare_to_clone_next_submodule` in git 2.43's + submodule--helper.c asks before it clones, fetches or enters it + (`walked`, lane 5's SHOULD 1 on #954). Not where no section git reads + gives that path. Not where its update strategy is `none`: the last + `submodule..update` this repository's config gives, else the + section's own `update` (`Gitmodules.updates`), which git's init copies + into the config where it holds none. And not where git counts it + inactive (`is_submodule_active`): where the config sets + `submodule.active`, `init_asks` asked `active` already; else its own + `submodule..active`, where a config scope above the one git's + init writes `true` in keeps it false (`git_bool`).""" + name = modules.read.get(path) + if name is None: + return False + values = config.get(f"submodule.{name}.update") + update = (values[-1] if values and values[-1] is not None + else (modules.updates or {}).get(name)) + if update == "none": + return False + own = config.get(f"submodule.{name}.active") + return not own or git_bool(root, own[-1]) is not False + + +def checkout_stop(checkout: Path, where: str, commit: str, + owners: tuple[str, ...], + fetched: frozenset[str] | None) -> Below | None: + """`walked`'s answer for the gitlink at `where`, recorded at `commit` + by the last of `owners` (this holder where there is none), whose + checkout is `checkout`: None where its `.git` is not there, asked as + `checked_out` asks a member's (`_is_a`, #430); `UNTOLD` where the + operating system refused that `stat`, or refused a git run in it or + below it (lane 5's SHOULD 2 on #954: `Path.exists` raised that + refusal, a traceback, before Python 3.14, and from 3.14 reads it as no + checkout); else `entered`'s answer.""" + try: + if not _is_a(checkout / ".git", _anything): + return None + return entered(checkout, where, commit, owners, fetched) + except OSError: + return UNTOLD + + +def entered(checkout: Path, where: str, commit: str, + owners: tuple[str, ...], + fetched: frozenset[str] | None) -> Below | None: + """`checkout_stop`'s answer for a checkout whose `.git` is there: B4 + where its store lacks `commit` and a fetch git made in it reached the + remote, `UNTOLD` where none did or where it is at another commit, and + else what `walked` finds in it.""" + if not commit_held(checkout, commit): + if fetched is not None and same_place(checkout) in fetched: + return Below(where, owners, commit) + return UNTOLD + if head_commit(checkout) != commit: + return UNTOLD + return walked(checkout, fetched, (*owners, where)) + + +def commit_held(checkout: Path, commit: str) -> bool: + """Whether the store `checkout` reads holds `commit`: `git cat-file -e + ^{commit}` exits 0. Not whether a ref reaches it, as git's + update asks before it fetches (`is_tip_reachable`): git's fetch of a + commit its store holds is answered from the store, so its update does + not die for want of one (measured, git 2.43.0), and a held commit no + ref reaches, whose checkout died, was named B4 (#954's round 1).""" + proc = subprocess.run(["git", "cat-file", "-e", f"{commit}^{{commit}}"], + cwd=str(checkout), capture_output=True, + check=False) + return proc.returncode == 0 + + +def head_commit(checkout: Path) -> str | None: + """The commit `checkout`'s HEAD is at, or None where it is at none.""" + try: + return git_out(["rev-parse", "--verify", "--quiet", "HEAD^{commit}"], + cwd=checkout) + except Refusal: + return None + + +def below_said(below: Below) -> str: + """(a)'s line, after its lead, for where git's update died past this + holder's records (`stopped_below`, #892): where git stopped and why, + then what cures it, which is the repository's that records the + gitlink, the last of `below.owners` or this holder. + + - A gitlink a member's own index records with no url (B5): no clone of + that member can check it out. `git -C rm --cached` takes it + out of that index, or a section gives it its url; committed and + pushed there where its HEAD records it, the gitlink that records the + member moves to that commit: this holder's with its pin (`family.py + bump`), and a leg's in its member, up to this holder. + - A commit no fetch brought (B4): pushed to that remote from a clone + that holds it, or the gitlink moved to one the remote holds, which + `validate-family.py` names, with the `bump` that moves the pin too, + where this holder records it, and up to this holder where a member + does.""" + owner = below.owners[-1] if below.owners else "this holder" + path, commit = spelled(below.path), below.commit + if commit: + tail = (f": `{PYTHON} scripts/validate-family.py` names the `bump` " + "that moves it with its pin." if not below.owners else + ", and so on up to this holder.") + return (f"git stopped at {path}, whose fetch reached its remote and " + f"did not bring {commit}, the commit {spelled(owner)} " + f"records there. Push {commit} to that remote from a clone " + f"that holds it, or move the gitlink {spelled(owner)} " + f"records there to a commit that remote holds{tail}") + above = below.owners[-2] if len(below.owners) > 1 else "this holder" + removal = shlex.join(["git", "-C", owner, "rm", "--cached", "--", + below.path[len(owner) + 1:]]) + tail = (", with its pin (`family.py bump`)." if len(below.owners) == 1 + else ", and so on up to this holder.") + return (f"git stopped at {path}, a gitlink {spelled(owner)}'s own index " + f"records with no url. No clone of {spelled(owner)} can check it " + f"out: `{spelled(removal)}` takes it out of that index, or a " + f"section of its {GITMODULES} can give it its path and url; " + f"where its HEAD records it, commit that in {spelled(owner)} and " + f"push it, then move the gitlink {spelled(above)} records for " + f"{spelled(owner)} to that commit{tail}") + + def members_of(root: Path) -> tuple[str, list[dict]]: """`(name:, the member rows)` out of `family.yaml`, or a refusal. diff --git a/tests/test_family_bootstrap_stops_named.py b/tests/test_family_bootstrap_stops_named.py new file mode 100644 index 00000000..aab2be76 --- /dev/null +++ b/tests/test_family_bootstrap_stops_named.py @@ -0,0 +1,770 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The family holder's pinned `scripts/bootstrap.py`: `family bootstrap ok` +beside git's exit 128 where git's update died past the holder's own +records (#892). + +THE ISSUE'S TWO LINES, each reproduced on #889's round-1 head (3b15de03) +before this change, on git 2.43.0, in the holder and in a plain clone: + +- B5. A gitlink committed in `members/IRRS`'s own index that its own + `.gitmodules` gives no url, at the commit the holder records for IRRS. + Git's update said `No url found for submodule path + 'members/IRRS/vendor/nested'` and exited 128, and `bootstrap.py + --skip-members` gave the credential advice and `family bootstrap ok`, + exit 0. It fails now, naming that gitlink and its cure, with no advice. +- B4. IRSS's gitlink and pin moved by hand to a commit no branch of its + remote holds. Git's fetch there reached the remote, its fetch of the + commit said `not our ref`, and its update exited 128; bootstrap said + `ok` with the advice (#648's credential skip). It fails now, naming the + member and the commit, with no advice. + +WHAT TELLS THEM FROM A MISSING CREDENTIAL, never git's words: for B5, the +member's own index, read as the holder's is (`init_stop`); for B4, a +commit the member's store lacks after the update, where git's own trace +of the run says its fetch there exited 0. The store alone tells +nothing: a member checked out before the run whose pin moved to a commit +its remote holds, fetched with no credential, is as short of it, which +was measured, and that is still the credential's skip, a control here, +beside a member checked out whole whose own nested gitlink has its url. + +WHERE NEITHER TELLS, the run reads as before, the advice and ok, and a row +pins each: a member's own `.gitmodules` git cannot read, or reads only up +to a bad line; a stray git never came to, since a clone that failed ended +its update 1; B4 with a credential in use, which is never traced; a member +git did not check out at its pin, refused (exit 1) or dead on an +`index.lock` (128), since git's update may have ended there, even at a +commit its store holds and no ref reaches, which git's fetch does not stop +at; a leg this user cannot search; and a member whose fetch reached +another member's remote but not its own. A gitlink git's update skips +(`update = none` in the member's config or its `.gitmodules`, or `active = +false`) is passed as git passes it, and a stop past it is named. B5 and B4 +a level down, in a leg, are named up to the holder, and B5 is named before +a leg git never fetched. + +THE SAME FAMILY, OFFLINE RULES AND HELPERS as +`tests/test_family_holder_remedies_compose.py` (#879). A missing credential +is git's refusal of the file transport to the fixture's local remotes, as +in `tests/test_family_holder_bootstrap_texts.py` (#839). A commit a test +makes in a member is pushed to a branch of the member's remote named for +it, so a plain clone fetches it, and the remote's `main` is not moved. A +commit made to be pushed nowhere names the clone it is made in, so no +other row's push puts it on a remote (`unpushed`). +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from conftest import rmtree, run_script +from repo_shape import PYTHON +from test_family import FAMILY, SPARE, family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_bump_leg_unopenable import made_unopenable, needs_a_mode +from test_family_bootstrap_gate_order import ( + bootstrap, cloned, type_line, with_config, +) +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( + GITMODULES, assert_passes_in_every_mode, commit_all, committed_gitmodules, + pinned, +) +from test_family_holder_bootstrap_texts import ALONE, OURS +from test_family_holder_remedies_compose import ( + OTHER_ROW, STOP_FAILED, cloning, valued, +) +from test_family_member_path import MEMBER, OTHER, raw +from test_validate_family_typed_states import ROW +from test_validate_family_unnamed_mounts import spare_commits, stage_gitlink + +#: How (a)'s line begins where git's update died past the holder's own +#: records. +LEAD = " UPDATE STOPPED, and not for want of a credential: " + +#: What it says of a gitlink an `owner`'s own index records with no url +#: (B5), and what cures it, up to the gitlink `above` records for `owner`. +STRAY_SAID = ( + "git stopped at {path}, a gitlink {owner}'s own index records with no " + "url. No clone of {owner} can check it out: `git -C {owner} rm --cached " + "-- {rel}` takes it out of that index, or a section of its .gitmodules " + "can give it its path and url; where its HEAD records it, commit that in " + "{owner} and push it, then move the gitlink {above} records for {owner} " + "to that commit{tail}") + +#: What it says of a commit `owner` records that git's fetch, which +#: reached the remote, did not bring (B4), and what cures it. +ABSENT_SAID = ( + "git stopped at {path}, whose fetch reached its remote and did not bring " + "{commit}, the commit {owner} records there. Push {commit} to that remote " + "from a clone that holds it, or move the gitlink {owner} records there to " + "a commit that remote holds{tail}") + +#: How each ends where the holder records the gitlink, and where a member +#: does. +WITH_ITS_PIN = ", with its pin (`family.py bump`)." +BUMP_NAMED = (f": `{PYTHON} scripts/validate-family.py` names the `bump` that " + "moves it with its pin.") +UP_TO_THE_HOLDER = ", and so on up to this holder." + +#: The gitlink a member's own index records with no url (B5). +NESTED = "vendor/nested" + +#: A token for the runs with a credential in use: the fixture's members are +#: local paths, which its rewrite of GitHub's urls never reaches. +TOKEN = "w892-a-token-no-forge-takes" + + +def said_after_the_fetch(result, source: str = "none") -> list[str]: + """Bootstrap's own stderr lines after its fetch line, which names the + credential's `source`, as `after_fetch_line` reads them, the line this + change adds included.""" + ours = [line for line in result.stderr.splitlines() + if line.startswith((*OURS, LEAD))] + assert ours, result.stderr + assert re.fullmatch(r" FETCH INCOMPLETE \(git exit \d+\) using " + rf"credential source '{source}':", ours[0]), ( + result.stderr) + return ours[1:] + + +def git_update(root: Path, env: dict): + """Git's own `submodule update --init --recursive` in `root`, as (a) + runs it, its exit code unchecked.""" + return raw(root, env, "submodule", "update", "--init", "--recursive", + check=False) + + +def committed(checkout: Path, env: dict, message: str) -> str: + """What is staged in `checkout` committed, an empty commit where + nothing is. Returns the commit.""" + raw(checkout, env, "commit", "-q", "--allow-empty", "-m", message) + return raw(checkout, env, "rev-parse", "HEAD").stdout.decode( + "ascii").strip() + + +def pushed(checkout: Path, env: dict, remote: Path, commit: str) -> None: + """`commit` pushed from `checkout` to a branch of `remote` named for + it.""" + raw(checkout, env, "push", "-q", str(remote), + f"{commit}:refs/heads/w892/{commit[:12]}") + + +def bumped(root: Path, env: dict, family, commit: str) -> None: + """MEMBER's `commit`, made in the holder's mount, pushed to its remote + and recorded with `family.py bump`, which moves the gitlink, the pin and + the digest in one commit.""" + pushed(root / ROW, env, family["remotes"] / f"{MEMBER}.git", commit) + result = run_script(FAMILY, "bump", "--family-root", str(root), + "--member", MEMBER, "--to", commit, env=env) + assert result.returncode == 0, result.stdout + result.stderr + + +def stray_in(checkout: Path, family, env: dict) -> str: + """B5's gitlink: SPARE's newer commit recorded at `NESTED` in + `checkout`'s own index, which its `.gitmodules` gives no section, and + committed there. Returns the commit.""" + stage_gitlink(checkout, NESTED, spare_commits(family)[1]) + return committed(checkout, env, "A gitlink its .gitmodules gives no url") + + +def b5(root: Path, env: dict, family) -> None: + """B5 in the holder: MEMBER moved to a commit of its own whose index + records the stray, by `family.py bump`, so the gate passes.""" + bumped(root, env, family, stray_in(root / ROW, family, env)) + + +def unpushed(family, env: dict, tmp_path: Path, + remote: str = OTHER) -> tuple[Path, str]: + """A clone of the fixture's `remote` (OTHER's unless said) of the + test's own and an empty commit made on its `main` there and never + pushed: no store the holder reaches holds it, and its tree is that + `main`'s. Returns both. + + ITS MESSAGE NAMES THE CLONE, a path no other row's clone has, so no + other row makes the same commit. Under one message, every row's had + the same parent, tree, author and message, and git dates a commit to + the second, so two rows that made one in the same second made the + same commit. The remotes are the module's: where the earlier row had + pushed it (`pin_moved_on`, B4's cure), the later row's B4 commit was + on the remote, git's fetch brought it, its update exited 1, not 128, + and the walk never ran. CI, which runs these rows less than a second + apart, failed the refused checkout's row so at 4f97fedb and the + unreadable leg's at 938a83ef; a commit date fixed by + `GIT_COMMITTER_DATE` and `GIT_AUTHOR_DATE` fails both on git 2.43.0.""" + clone = tmp_path / f"unpushed-{remote}" + raw(tmp_path, env, "clone", "-q", "--", + str(family["remotes"] / f"{remote}.git"), str(clone)) + return clone, committed(clone, env, f"Made in {clone} and never pushed") + + +def moved_by_hand(root: Path, env: dict, project: str, commit: str) -> None: + """`project`'s gitlink and pin moved to `commit` by hand and committed, + as no `family.py bump` moves them to a commit no branch of the member's + remote reaches (`member-to-local-only`, #613).""" + pin = pinned(root, project) + raw(root, env, "update-index", "--cacheinfo", + f"160000,{commit},members/{project}") + manifest = root / "family.yaml" + text = manifest.read_text(encoding="utf-8") + assert text.count(pin) == 1, "fixture drift: the row's pin line" + manifest.write_text(text.replace(pin, commit), encoding="utf-8") + raw(root, env, "add", "--", "family.yaml") + commit_all(root, env, f"{project} moved by hand") + + +def b4(root: Path, env: dict, family, tmp_path: Path) -> str: + """B4 in the holder: OTHER's gitlink and pin at `unpushed`'s commit. + Returns the commit.""" + _, commit = unpushed(family, env, tmp_path) + moved_by_hand(root, env, OTHER, commit) + return commit + + +def in_place(holder: Path, env: dict, family, tmp_path: Path) -> Path: + """The holder itself, every member checked out.""" + return holder + + +def plain_clone(holder: Path, env: dict, family, tmp_path: Path) -> Path: + """A plain clone of the holder, no member checked out.""" + return cloned(holder, env, family, tmp_path, recurse=False) + + +#: Where each state is asked: the holder, and a plain clone of it. +WHERE = {"holder": in_place, "plain-clone": plain_clone} + + +# --- B5: a gitlink a member's own index records with no url ----------------- + +@pytest.mark.parametrize("where", WHERE.values(), ids=WHERE) +def test_b5_a_gitlink_a_member_gives_no_url_fails_naming_it( + family, holder, offline, tmp_path, where): + """B5: git's update died at `members/IRRS/vendor/nested` (exit 128), + and bootstrap said ok with the credential advice. It names the gitlink, + the member whose index records it and the cure, gives no advice, and + fails on that gitlink; git's own update exits 128 there, as it says.""" + env = cloning(offline, tmp_path, family) + b5(holder, env, family) + root = where(holder, env, family, tmp_path) + + result = bootstrap(root, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ + LEAD + STRAY_SAID.format(path=f"{ROW}/{NESTED}", owner=ROW, + rel=NESTED, above="this holder", + tail=WITH_ITS_PIN), + STOP_FAILED.format(path=f"{ROW}/{NESTED}")] + assert git_update(root, env).returncode == 128 + + +def test_b5_s_cure_typed_as_printed_ends_ok(family, holder, offline, + tmp_path): + """B5's cure: the `git -C` line typed as printed takes the gitlink out + of the member's index; committed there, pushed, and recorded with + `family.py bump`, the run checks every member and leg out and ends ok, + and the gate passes in every mode.""" + env = cloning(offline, tmp_path, family) + b5(holder, env, family) + result = bootstrap(holder, env, "--skip-members") + (line,) = re.findall(r"`(git -C [^`]*)`", result.stderr) + + type_line(holder, env, line) + bumped(holder, env, family, committed(holder / ROW, env, + "The gitlink taken out")) + + booted = bootstrap(holder, env, "--skip-members") + assert booted.returncode == 0, booted.stdout + booted.stderr + assert " every member and leg fetched" in booted.stdout.splitlines() + assert_passes_in_every_mode(holder, 2) + + +def test_b5_in_a_leg_is_named_up_to_the_holder(family, holder, offline, + tmp_path): + """B5 a level down: the gitlink in the own index of IRRS's `code` leg, + which IRRS records and the holder records IRRS at. Git's update died + there, and the line names it, the leg whose index records it, and the + cure up to the holder.""" + env = cloning(offline, tmp_path, family) + assert git_update(holder, env).returncode == 0 + leg = holder / ROW / "code" + made = stray_in(leg, family, env) + pushed(leg, env, family["remotes"] / f"{MEMBER}-code.git", made) + raw(holder / ROW, env, "add", "--", "code") + bumped(holder, env, family, committed(holder / ROW, env, "The leg moved")) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ + LEAD + STRAY_SAID.format(path=f"{ROW}/code/{NESTED}", + owner=f"{ROW}/code", rel=NESTED, above=ROW, + tail=UP_TO_THE_HOLDER), + STOP_FAILED.format(path=f"{ROW}/code/{NESTED}")] + + +# --- B4: a member's commit git's fetch did not bring ------------------------- + +@pytest.mark.parametrize("where", WHERE.values(), ids=WHERE) +def test_b4_a_commit_the_fetch_did_not_bring_fails_naming_it( + family, holder, offline, tmp_path, where): + """B4: git's fetch in IRSS reached its remote and did not bring the + commit the holder records, and its update exited 128; bootstrap said ok + with the credential advice. It names the member, the commit and the + cure, gives no advice, and fails on the member; git's own update exits + 128 there, and the run leaves no trace directory behind.""" + env = cloning(offline, tmp_path, family) + commit = b4(holder, env, family, tmp_path) + root = where(holder, env, family, tmp_path) + + result = bootstrap(root, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ + LEAD + ABSENT_SAID.format(path=OTHER_ROW, commit=commit, + owner="this holder", tail=BUMP_NAMED), + STOP_FAILED.format(path=OTHER_ROW)] + assert git_update(root, env).returncode == 128 + assert list(offline.temp.glob("bootstrap-trace-*")) == [] + + +def test_b4_s_cures_as_printed(family, holder, offline, tmp_path): + """B4's cures: the gate names the `bump` that moves the gitlink and the + pin, as the line says; and the commit pushed from the clone that holds + it, the run checks every member and leg out and ends ok, and the gate + passes in every mode.""" + env = cloning(offline, tmp_path, family) + clone, commit = unpushed(family, env, tmp_path) + moved_by_hand(holder, env, OTHER, commit) + + gate = validate(holder) + + assert gate.returncode == 2, gate.stdout + gate.stderr + assert "member-commit-unresolvable" in gate.stderr + assert "scripts/family.py bump --family-root " in gate.stderr + pushed(clone, env, family["remotes"] / f"{OTHER}.git", commit) + booted = bootstrap(holder, env, "--skip-members") + assert booted.returncode == 0, booted.stdout + booted.stderr + assert " every member and leg fetched" in booted.stdout.splitlines() + assert_passes_in_every_mode(holder, 2) + + +def leg_moved(root: Path, env: dict, family, tmp_path: Path, *, + stray: bool) -> str: + """Every member and leg checked out by git's update; then IRRS's `code` + leg recorded in IRRS's own index at `unpushed`'s commit of the leg's + remote, with B5's stray beside it where `stray` says, committed in + IRRS and recorded with `family.py bump`. Returns the leg's commit.""" + assert git_update(root, env).returncode == 0 + _, commit = unpushed(family, env, tmp_path, f"{MEMBER}-code") + member = root / ROW + raw(member, env, "update-index", "--cacheinfo", f"160000,{commit},code") + bumped(root, env, family, stray_in(member, family, env) if stray + else committed(member, env, "Its code leg moved by hand")) + return commit + + +def test_b4_in_a_leg_is_named_up_to_the_holder(family, holder, offline, + tmp_path): + """B4 a level down (lane 5's P2 and R1 on #954): IRRS's `code` leg + recorded in IRRS at a commit no branch of the leg's remote holds. Git's + fetch there reached the remote and did not bring it, and its update + exited 128. The whole line is named, the member that records the leg, + and the cure up to the holder.""" + env = cloning(offline, tmp_path, family) + commit = leg_moved(holder, env, family, tmp_path, stray=False) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ + LEAD + ABSENT_SAID.format(path=f"{ROW}/code", commit=commit, + owner=ROW, tail=UP_TO_THE_HOLDER), + STOP_FAILED.format(path=f"{ROW}/code")] + assert git_update(holder, env).returncode == 128 + + +def test_b5_is_named_before_a_leg_git_never_fetched(family, holder, + offline, tmp_path): + """Git's order (lane 5's R6 on #954): B5's stray in IRRS's own index, + and IRRS's `code` leg recorded beside it at a commit its store lacks. + Git's init in IRRS dies at the stray before it updates any leg, so the + leg's commit was never fetched, and B5 is what is named.""" + env = cloning(offline, tmp_path, family) + leg_moved(holder, env, family, tmp_path, stray=True) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ + LEAD + STRAY_SAID.format(path=f"{ROW}/{NESTED}", owner=ROW, + rel=NESTED, above="this holder", + tail=WITH_ITS_PIN), + STOP_FAILED.format(path=f"{ROW}/{NESTED}")] + + +# --- the credential's skip, as it was ---------------------------------------- + +def pin_moved_on(holder: Path, env: dict, family, tmp_path: Path) -> Path: + """IRSS's gitlink and pin moved on to a commit a branch of its remote + holds and its store does not, as pulling another's `family.py bump` + leaves a holder whose members were checked out before.""" + clone, commit = unpushed(family, env, tmp_path) + pushed(clone, env, family["remotes"] / f"{OTHER}.git", commit) + moved_by_hand(holder, env, OTHER, commit) + return holder + + +def beside_a_member_checked_out_whole(holder: Path, env: dict, family, + tmp_path: Path) -> Path: + """IRRS given a nested gitlink WITH its section and url, SPARE mounted + by `git submodule add` and recorded with `family.py bump`, every member + and leg then checked out whole by git's update, and IRSS's pin then + moved on (`pin_moved_on`).""" + raw(holder / ROW, env, "submodule", "add", "-q", "--", + str(family["remotes"] / f"{SPARE}.git"), NESTED) + bumped(holder, env, family, committed(holder / ROW, env, + "A nested member with its url")) + assert git_update(holder, env).returncode == 0 + assert (holder / ROW / NESTED / "project.yaml").is_file() + return pin_moved_on(holder, env, family, tmp_path) + + +#: `(the state, git's update's exit code in it with no credential)`: every +#: member left unchecked out, a checked-out member whose pin moved on, and +#: that beside a member checked out whole. +CREDENTIAL_S = { + "members-not-checked-out": (plain_clone, 1), + "pin-moved-on": (pin_moved_on, 128), + "beside-a-member-checked-out-whole": (beside_a_member_checked_out_whole, + 128), +} + + +@pytest.mark.parametrize("make, code", CREDENTIAL_S.values(), + ids=CREDENTIAL_S) +def test_the_credential_s_skip_is_as_it_was(family, holder, offline, + tmp_path, make, code): + """No misfire: where git cannot reach the members' remotes, as for want + of a credential, the store lacks the commit as in B4, but no fetch git + made reached a remote, and every gitlink a member's index records has + its url. The advice stands alone, and the run ends ok, as before. With + the credential, the same run checks every member and leg out.""" + env = cloning(offline, tmp_path, family) + root = make(holder, env, family, tmp_path) + without = isolated(offline, tmp_path) + + result = bootstrap(root, without, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert result.stdout.endswith("\nfamily bootstrap ok\n"), result.stdout + assert git_update(root, without).returncode == code + fetched = bootstrap(root, env, "--skip-members") + assert fetched.returncode == 0, fetched.stdout + fetched.stderr + assert " every member and leg fetched" in fetched.stdout.splitlines() + + +def test_a_stray_git_never_came_to_is_not_said(family, holder, offline, + tmp_path): + """No misfire: B5 in IRRS, and IRSS's store gone, so git's update must + clone IRSS first, which it cannot for want of a credential. A clone + that fails ends git's update 1 before it checks out or enters any + member, so it never came to the stray: the advice stands and the run + ends ok, as before.""" + env = cloning(offline, tmp_path, family) + b5(holder, env, family) + raw(holder, env, "submodule", "deinit", "-q", "-f", "--", OTHER_ROW) + rmtree(holder / ".git" / "modules" / OTHER_ROW) + without = isolated(offline, tmp_path) + + result = bootstrap(holder, without, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert git_update(holder, without).returncode == 1 + + +def test_a_member_s_gitmodules_git_cannot_read_is_read_as_before( + family, holder, offline, tmp_path): + """Where neither check tells: IRRS's own `.gitmodules` gives its `code` + leg `update = None`, a value git dies on as it reads the file (#768), + committed there and recorded by hand, since `family.py bump`'s own git + dies reading it too. Git's update dies in IRRS (exit 128), but at no + gitlink with no url and no commit a fetch did not bring, so nothing + past the holder's records is said: the run reads as before, the advice + and ok.""" + env = cloning(offline, tmp_path, family) + member = holder / ROW + (member / ".gitmodules").write_bytes( + valued((member / ".gitmodules").read_bytes(), "code")) + raw(member, env, "add", "--", ".gitmodules") + moved_by_hand(holder, env, MEMBER, + committed(member, env, "A value git dies on")) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert git_update(holder, env).returncode == 128 + + +def refused_checkout(root: Path, env: dict, family, tmp_path: Path) -> None: + """IRRS's checkout moved by hand to a commit of its own that changes + `NOTES.md` and records B5's gitlink, with `NOTES.md` then edited and + not committed, while the holder still records the pin; and B4 at IRSS. + Git's checkout of the pin in IRRS refuses to overwrite the edit (`git + checkout`'s own exit 1, which its update passes), and git dies at IRSS + (exit 128).""" + member = root / ROW + (member / "NOTES.md").write_bytes(b"moved on by hand" + chr(10).encode()) + raw(member, env, "add", "--", "NOTES.md") + stray_in(member, family, env) + (member / "NOTES.md").write_bytes(b"edited" + chr(10).encode()) + b4(root, env, family, tmp_path) + + +def locked_checkout(root: Path, env: dict, family, tmp_path: Path) -> None: + """IRRS's gitlink and pin moved by hand to its checkout's HEAD~1, which + its store holds, with an `index.lock` left in that store; and B5's + stray committed in IRSS's own index, which the holder records. Git's + checkout in IRRS dies (`Unable to create ... index.lock`, exit 128), + which ends its update there, before it comes to IRSS.""" + member = root / ROW + older = raw(member, env, "rev-parse", "HEAD~1").stdout.decode().strip() + moved_by_hand(root, env, MEMBER, older) + locked_in(member, env) + moved_by_hand(root, env, OTHER, stray_in(root / OTHER_ROW, family, env)) + + +def held_and_locked(root: Path, env: dict, family, tmp_path: Path) -> None: + """IRRS's gitlink and pin moved by hand to a commit its store holds + and no ref of it reaches, made there with `git commit-tree` and pushed + nowhere, with an `index.lock` left in that store. Git's fetch there + reaches the remote, its fetch of the commit is answered from the store + (exit 0), and its checkout dies (exit 128): no branch of the remote + holds the commit, as in B4, but git did not stop for want of it.""" + member = root / ROW + held = raw(member, env, "commit-tree", "HEAD^{tree}", "-p", "HEAD", "-m", + "Held, and no ref reaches it").stdout.decode().strip() + moved_by_hand(root, env, MEMBER, held) + locked_in(member, env) + + +def locked_in(checkout: Path, env: dict) -> None: + """An empty `index.lock` left in the store `checkout` reads.""" + lock = raw(checkout, env, "rev-parse", "--git-path", + "index.lock").stdout.decode().strip() + (checkout / lock).write_bytes(b"") + + +#: A checkout git did not make at the commit recorded for it, its own +#: refusal (exit 1, which git's update passes) or its death (exit 128, which +#: ends git's update), with a stop past it that the walk named before round +#: 1: B4 at IRSS, and B5 at IRSS, which git never came to; and a death at a +#: commit the store holds and no ref reaches, which the walk named B4. +NOT_CHECKED_OUT_THERE = {"refused": refused_checkout, + "index-lock": locked_checkout, + "held-commit-index-lock": held_and_locked} + + +@pytest.mark.parametrize("make", NOT_CHECKED_OUT_THERE.values(), + ids=NOT_CHECKED_OUT_THERE) +def test_a_member_git_did_not_check_out_is_untold(family, holder, offline, + tmp_path, make): + """A checkout at another commit than the one recorded is untold, and + nothing past it is asked: whether git's checkout there refused, which + git passes, or died, which ends its update, is not told, and with an + `index.lock` in IRRS round 0's walk named B5's stray in IRSS, where git + never came (measured for lane 5's index.lock case on #954), and, at a + commit IRRS's store holds and no ref reaches, B4 at IRRS, whose fetch + git's update did not die of. The run reads as before #954, the advice + and ok, while git's update exits 128.""" + env = cloning(offline, tmp_path, family) + make(holder, env, family, tmp_path) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert git_update(holder, env).returncode == 128 + + +def update_none_in_config(root: Path, env: dict) -> None: + """`update = none` for IRRS in the holder's own config (lane 5's P1).""" + raw(root, env, "config", f"submodule.{ROW}.update", "none") + + +def update_none_in_gitmodules(root: Path, env: dict) -> None: + """`update = none` in IRRS's section of HEAD's `.gitmodules`, which + git's init copies into the config, committed.""" + data = raw(root, env, "show", f"HEAD:{GITMODULES}").stdout + line = f"\tpath = {ROW}\n".encode() + assert data.count(line) == 1, data + committed_gitmodules(root, env, + data.replace(line, line + b"\tupdate = none\n")) + + +def as_it_is(root: Path, env: dict) -> None: + """Nothing more in the holder.""" + + +def as_given(env: dict) -> dict: + """`env` as it is.""" + return env + + +def inactive(env: dict) -> dict: + """`env` with `submodule.members/IRRS.active` false in git's command + scope, above the local config git's init writes `true` in.""" + return with_config(env, f"submodule.{ROW}.active", "false") + + +#: `(the state, the environment it is read in)`: IRRS's update strategy +#: `none`, in the holder's config or its `.gitmodules`, and IRRS inactive. +SKIPPED_BY_GIT = { + "update-none-in-config": (update_none_in_config, as_given), + "update-none-in-gitmodules": (update_none_in_gitmodules, as_given), + "inactive": (as_it_is, inactive), +} + + +@pytest.mark.parametrize("make, scoped", SKIPPED_BY_GIT.values(), + ids=SKIPPED_BY_GIT) +def test_a_member_git_s_update_skips_is_passed(family, holder, offline, + tmp_path, make, scoped): + """Lane 5's SHOULD 1 on #954 (P1): B5's stray in IRRS, which git's + update skips without a look (`Skipping submodule`), and IRSS's pin + moved on, with no credential. Git dies at IRSS for want of the + credential (exit 128). The walk entered IRRS all the same and failed + on the stray with no advice; it passes IRRS now, so the run is the + credential's skip, the advice and ok, as before #954. With the + credential, git skips IRRS, checks IRSS out and exits 0, and the run + says every member and leg was fetched: git never comes to the stray.""" + env = cloning(offline, tmp_path, family) + b5(holder, env, family) + pin_moved_on(holder, env, family, tmp_path) + make(holder, env) + without = scoped(isolated(offline, tmp_path)) + + result = bootstrap(holder, without, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert git_update(holder, without).returncode == 128 + fetched = bootstrap(holder, scoped(env), "--skip-members") + assert fetched.returncode == 0, fetched.stdout + fetched.stderr + assert " every member and leg fetched" in fetched.stdout.splitlines() + + +@needs_a_mode +def test_a_leg_this_user_cannot_search_is_untold(family, holder, offline, + tmp_path): + """Lane 5's SHOULD 2 on #954 (P7 and P9): IRRS's `code` leg made mode + 000, and B4 at IRSS. The walk's `stat` of the leg's `.git` is refused, + which ended the run in a `PermissionError` traceback; that leg is + untold now, nothing past it asked, and the run says what it said before + #954, the advice and ok. Git's update in IRRS cannot clone the leg + and exits 1 (`Failed to clone 'code' a second time, aborting`); the + holder's update passes that, as it passes a checkout that refused, + and dies at IRSS's B4 (exit 128), so the walk meets the leg. CI's + exit 1 here at 938a83ef, read then as git 2.55.0 ending its update at + the leg, was B4's commit already on the remote (`unpushed`): git + 2.43.0 ends with the same two lines, and exits 1, where it is.""" + env = cloning(offline, tmp_path, family) + b4(holder, env, family, tmp_path) + + with made_unopenable(holder / ROW, "code"): + result = bootstrap(holder, env, "--skip-members") + update = git_update(holder, env) + + assert result.returncode == 0, result.stdout + result.stderr + assert "Traceback" not in result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert update.returncode == 128 + + +def test_a_fetch_that_reached_another_member_s_remote_tells_nothing( + family, holder, offline, tmp_path): + """The fetch THERE (lane 5's R7 on #954): IRRS's pin moved on to a + commit its remote holds, whose fetch reaches that remote, and B4's + commit at IRSS, whose `origin` names no repository, so its fetch never + reaches one. IRSS is untold, whatever IRRS's fetch reached: the advice + and ok, as before #954.""" + env = cloning(offline, tmp_path, family) + clone, commit = unpushed(family, env, tmp_path, MEMBER) + pushed(clone, env, family["remotes"] / f"{MEMBER}.git", commit) + moved_by_hand(holder, env, MEMBER, commit) + b4(holder, env, family, tmp_path) + raw(holder / OTHER_ROW, env, "remote", "set-url", "origin", + str(tmp_path / "no-repository-here.git")) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert git_update(holder, env).returncode == 128 + + +def test_a_member_s_gitmodules_read_up_to_a_bad_line_is_read_as_before( + family, holder, offline, tmp_path): + """Where neither check tells (lane 5's R11 on #954): IRRS's own + `.gitmodules` with a line git cannot parse before `code`'s section, + committed there and recorded by hand, and the work tree's file then + deleted. Git reads IRRS's index's file up to that line and dies at + `code` (exit 128), whose section is past it, so a section written back + would be a second one: nothing past the holder's records is said, and + the run reads as before, the advice and ok.""" + env = cloning(offline, tmp_path, family) + member = holder / ROW + data = (member / GITMODULES).read_bytes() + second = data.index(b"[submodule", 1) + bad = (chr(9) + 'branch = "unterminated' + chr(10)).encode("ascii") + committed_gitmodules(member, env, data[:second] + bad + data[second:]) + made = raw(member, env, "rev-parse", "HEAD").stdout.decode().strip() + moved_by_hand(holder, env, MEMBER, made) + (member / GITMODULES).unlink() + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result) == [ALONE] + assert git_update(holder, env).returncode == 128 + + +# --- a credential in use ----------------------------------------------------- + +def test_with_a_credential_in_use_b5_is_named_and_the_token_is_not( + family, holder, offline, tmp_path): + """B5 is read from the member's own index, which needs no trace, so it + is named with a credential in use too, and the token is printed + nowhere.""" + env = {**cloning(offline, tmp_path, family), "SHAPE_LEGS_TOKEN": TOKEN} + b5(holder, env, family) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert said_after_the_fetch(result, "SHAPE_LEGS_TOKEN") == [ + LEAD + STRAY_SAID.format(path=f"{ROW}/{NESTED}", owner=ROW, + rel=NESTED, above="this holder", + tail=WITH_ITS_PIN), + STOP_FAILED.format(path=f"{ROW}/{NESTED}")] + assert TOKEN not in result.stdout + result.stderr + + +def test_with_a_credential_in_use_b4_is_read_as_before(family, holder, + offline, tmp_path): + """What it cannot tell: a run with a credential in use is not traced, + since trace2 writes a command's arguments and the credential is one, so + B4 is the advice and ok, as before.""" + env = {**cloning(offline, tmp_path, family), "SHAPE_LEGS_TOKEN": TOKEN} + b4(holder, env, family, tmp_path) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert said_after_the_fetch(result, "SHAPE_LEGS_TOKEN") == [ALONE] + assert TOKEN not in result.stdout + result.stderr