diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index f69b81dd..cf8ba415 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -40,14 +40,22 @@ for (a stray case variant, a member mounted by hand), since git's init dies at the first such gitlink it asks about, before it checks out any member. Which section and which url git reads is its own - parser's answer, not `git config`'s (`gitmodules_sections`), and - while a merge leaves `.gitmodules` unmerged in the index git reads - none at all: every such member is said to be that, and the cure is - the file resolved and added. Git exits 0 over the second where a - recursive clone ran first, and 128 (`No url found for submodule - path`) in a plain clone; a missing credential exits non-zero too, - so what the holder records is the test, never git's words. The line - says what cures each (`cures`): + parser's answer, not `git config`'s (`gitmodules_sections`), from + the work tree's file or, where the work tree has none, the index's, + then HEAD's, and while a merge leaves `.gitmodules` unmerged in the + index git reads none at all: every such member is said to be that, + and the cure is the file resolved and added. Where the file gives a + submodule a value git dies on as it reads it (`refused_value`, #768), + every such member is said to be that, naming each key and its value, + and the cure is the key given a value git takes or its line taken + out; where every member was checked out the run fails all the same, + since git updated none of them. A work tree file git cannot parse at + all, a conflict marker or an open quote left in it, is said and + failed on the same way, and its cure is the file mended by hand. Git + exits 0 over the second where a recursive clone ran first, and 128 + (`No url found for submodule path`) in a plain clone; a missing + credential exits non-zero too, so what the holder records is the + test, never git's words. The line says what cures each (`cures`): `validate-family.py` names the commit for the first two; for the fourth it prints the `git submodule update --init --checkout` that checks the member out whatever the clone chose. For the third and @@ -67,7 +75,8 @@ the family Makefile's `validate` target uses. (c) EXITS 0 unless a member's own command failed, or (a) left a member not - checked out that no credential would check out. Any other member that + checked out that no credential would check out, or found a line or a + value in `.gitmodules` that git cannot read (#768). Any other member that is not checked out is REPORTED and skipped, not a failure: it is the state a missing credential leaves, and it is already reported in (a). Where (a) cannot start git, the run STOPS there, exit 2, refused @@ -82,6 +91,7 @@ import argparse import os +import re import shlex import shutil import subprocess @@ -104,14 +114,44 @@ #: Why (a) left a row's member NOT CHECKED OUT where no credential would #: check it out (#648), in the order `unfetchable` asks. `STOPPED` is -#: filled with the gitlink git's init died at (`init_stop`). +#: filled with the gitlink git's init died at (`init_stop`), and `REFUSED` +#: with each entry of `.gitmodules` git refused (`reason`). `UNREADABLE` is +#: a work tree file git cannot parse at all (#768). UNMERGED = f"git reads no {GITMODULES} while a merge leaves it unmerged" +REFUSED = f"git cannot read {GITMODULES}: {{}}" +UNREADABLE = f"git cannot parse {GITMODULES}" NO_GITLINK = "no gitlink is recorded at that path" NO_SECTION = f"no section of {GITMODULES} gives that path" NO_URL = "git has no url for that path" EXITED_0 = "git exited 0 and checked nothing out there" STOPPED = "git stopped at {}, a gitlink it has no url for" +#: What git 2.43 dies on in a `.gitmodules` it reads (#768), as +#: `parse_config` in its submodule-config.c reads a submodule's key: these +#: keys with no value (a `branch` with none crashes it, `BRANCH_READ_SINCE`), +#: and an `update` other than these four. A `shallow` must be a boolean and +#: a `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). +NO_VALUE_REFUSED = frozenset({"path", "url", "update", "ignore", "branch"}) +#: The first git that reads a `branch` with no value as it reads a `url` +#: with none, `missing value` and `bad config line`, which dies in the work +#: tree's file and ends the reading of the index's or HEAD's, a blob: 2.43.1, +#: and 2.44.0 on its main line, by their submodule-config.c, and 2.47.1 as +#: measured. A git before it crashes reading any `.gitmodules` that gives +#: one, a blob too: 2.43.0 as measured (exit 139), and 2.42.4 and 2.39.5 by +#: their source, which has the same `xstrdup` of no value +#: (`crashes_on_branch`). +BRANCH_READ_SINCE = (2, 43, 1) +#: `git version 2.43.0`, `git version 2.45.1.windows.1`, `git version +#: 2.39.3 (Apple Git-146)`: three numbers, since the third decides here. +GIT_VERSION_RE = re.compile(r"git version (\d+)\.(\d+)\.(\d+)") +UPDATE_VALUES = ("none", "checkout", "rebase", "merge") +#: What git takes for a key whose value it refused, as (a)'s cure says it. +TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", + "shallow": "a boolean, `true` or `false`", + "fetchrecursesubmodules": "a boolean or `on-demand`", + "ignore": "one of `untracked`, `dirty`, `all` or `none`", + "path": "a path", "url": "a url", "branch": "a branch name"} + #: The two environment variables a leg/member credential may arrive in, in the #: order the `validate` workflow resolves them. An App-minted installation #: token is short-lived and per-owner; the PAT is the standing fallback. @@ -134,6 +174,11 @@ def credential() -> tuple[str, str]: def fetch_members(root: Path, rows: list[dict]) -> list[str]: """(a) `git submodule update --init --recursive`, authenticated or not. + Where every member is checked out and git died on a value in + `.gitmodules`, or on a work tree file it cannot parse, so it updated + none of them, the one failure line it returns is for `.gitmodules` + (#768). + Returns a failure line for each of `rows` whose member it leaves not checked out where no credential would check it out (`unfetchable`), for the run to end FAILED on (#648); every member and leg is said to be @@ -172,11 +217,15 @@ def fetch_members(root: Path, rows: list[dict]) -> list[str]: left, stopper, modules = unfetchable(root, rows, proc.returncode == 0) if left: print(" NOT CHECKED OUT, and not for want of a credential: " - + ", ".join(f"{spelled(path)} (" - + (STOPPED.format(spelled(stopper)) - if why == STOPPED else why) + ")" + + ", ".join(f"{spelled(path)} ({reason(why, stopper, modules)})" for _, path, why in left) + ". " + cures(left, stopper, rows, modules), file=sys.stderr) + elif modules.refused or modules.unreadable: + why = REFUSED if modules.refused else UNREADABLE + print(" NO MEMBER UPDATED, and not for want of a credential: " + + reason(why, stopper, modules) + ". " + + cures(left, stopper, rows, modules), file=sys.stderr) + return [f"{GITMODULES} (git cannot read it)"] elif proc.returncode == 0: print(" every member and leg fetched") return [f"{spelled(row.get('project') or '?')} (not checked out)" @@ -215,6 +264,12 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, - A `.gitmodules` a merge left unmerged in the index, which git reads nothing from (`Gitmodules.unmerged`): the file resolved and added. + - A `.gitmodules` that gives a value git dies on (`Gitmodules.refused`, + #768): each key given a value git takes (`TAKES`), or its line taken + out, and the file committed, with `validate-family.py --no-members`, + whose `gitmodules-value-refused` prints the commands. + - A work tree `.gitmodules` git cannot parse (`Gitmodules.unreadable`): + the file mended by hand and committed. - A gitlink or a section the holder lacks at the row's path: `validate-family.py`, which names it and the commit that cures it, `member-gitlink-absent` (with the move, for a gitlink recorded only @@ -237,10 +292,7 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, clone chose. `--init` with the path registers a member a subset clone left out, and `--checkout` overrides an `update = none`.""" whys = {why for _, _, why in left} - said = [] - if UNMERGED in whys: - said.append(f"Resolve the merge in {GITMODULES} and `git add " - f"{GITMODULES}`, then run this again.") + said = gitmodules_cures(whys, modules) if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") @@ -280,6 +332,45 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, return " ".join(said) +def gitmodules_cures(whys: set[str], modules: Gitmodules) -> list[str]: + """`cures`' sentences for a `.gitmodules` git reads nothing from, each + in the order `cures` says them (#648, #768): a merge left unmerged, + for a row `whys` names so; each value git dies on; a work tree file + git cannot parse.""" + said = [] + if UNMERGED in whys: + said.append(f"Resolve the merge in {GITMODULES} and `git add " + f"{GITMODULES}`, then run this again.") + if modules.refused: + given = "; ".join( + f"give `{spelled(key)}` a value git takes, " + f"{TAKES[key.rpartition('.')[2]]}, or take that line out" + for key in dict.fromkeys(key for key, _ in modules.refused)) + said.append(f"{given[:1].upper()}{given[1:]}, and commit " + f"{GITMODULES}. `{PYTHON} " + "scripts/validate-family.py --no-members` prints the " + "commands.") + if modules.unreadable: + said.append(f"Mend {GITMODULES} by hand (`git config -f " + f"{GITMODULES} --list` says where it breaks), and " + "commit it.") + return said + + +def reason(why: str, stopper: str | None, modules: Gitmodules) -> str: + """`why` as (a)'s line says it: `STOPPED` names the gitlink git stopped + at, and `REFUSED` each entry of `.gitmodules` git refused, as "`` + is ``", or "has no value" (#768).""" + if why == STOPPED: + return STOPPED.format(spelled(stopper)) + if why == REFUSED: + return REFUSED.format(" and ".join( + f"`{spelled(key)}` " + ("has no value" if value is None + else f"is `{spelled(value)}`") + for key, value in modules.refused)) + return why + + class Gitmodules(NamedTuple): """`.gitmodules` as (a) asks it (#648), read by `gitmodules_sections`. @@ -289,92 +380,229 @@ class Gitmodules(NamedTuple): `{path: name}`, the section git reads for each path, and `urls` is `{name: url}`, the url git takes from each, as git's own parser has them. `unmerged` says a merge left the file unmerged in the index, and - then git reads nothing from it, and the three are empty.""" + `refused` holds each `(submodule.., value)` git dies on as + it reads the file, None for a key with no value (#768): git reads + nothing from it then, and the three are empty. `unreadable` says the + work tree's file is one `git config --list` cannot parse, which git + dies reading: the three are empty then too.""" given: Counter read: dict[str, str] urls: dict[str, str] unmerged: bool = False + refused: tuple[tuple[str, str | None], ...] = () + unreadable: bool = False def gitmodules_sections(root: Path) -> Gitmodules: """`.gitmodules` as `git submodule update` reads it: the work tree's - file, which a fresh clone takes from HEAD. Nothing where there is no - such file, or one git cannot read: git has no url for any submodule - then either. Nothing, and `unmerged`, where the index holds the file - only at a merge's conflict stages (`git ls-files -u`): git reads no + file, which a fresh clone takes from HEAD, or, where the work tree has + none, as a deletion not yet committed or a sparse checkout leaves it, + the index's, then HEAD's (`gitmodules_source`, #768). Nothing where + none holds one, or git cannot read it: git has no url for any + submodule then either. Nothing, and `unreadable`, where that is the + work tree's file and `git config --list` cannot parse it: git dies + reading it (`bad config line`), before it checks out any member. + Nothing, and `unmerged`, where the index holds the file only at a + merge's conflict stages (`git ls-files -s`): git reads no `.gitmodules` at all then, whatever the work tree holds (`repo_read_gitmodules`), and its init dies at the first gitlink. + Nothing, and `refused`, where git dies on an entry as it reads the + file (`refused_value`, #768), before it checks out any member. - `read` and `urls` keep git 2.43's own rules (submodule-config.c), which - drop what `git config --list` keeps. Entry by entry, in file order: - - - a section whose name is empty or has a `..` component, split on `/` - and on a backslash, is ignored whole (git warns `ignoring suspicious - submodule name`); - - a `path` or `url` that begins with `-` is dropped, and the value - before it stands (`may be interpreted as a command-line option`); - - where two sections give one path, the later in the file decides, as - a later value in one section replaces an earlier. A section that - gives a path again lets go of the one it gave, even where a later - section gave that one too: git then reads no section there.""" + `read` and `urls` keep git's own rules (submodule-config.c), which + drop what `git config --list` keeps (`gitmodules_entries`, + `read_sections`).""" nothing = Gitmodules(Counter(), {}, {}) + source = gitmodules_source(root) + if source is None: + return nothing._replace(unmerged=True) try: - stages = git_out(["ls-files", "-u", "-z", "--", GITMODULES], - cwd=root, binary=True) + listed = git_out(["config", "-z", *source, "--list"], cwd=root, + binary=True) except Refusal: - stages = b"" - if any(record.partition(b"\t")[2] == GITMODULES.encode() - for record in stages.split(b"\0")): - return nothing._replace(unmerged=True) - if not os.path.lexists(root / GITMODULES): - return nothing + return nothing._replace(unreadable=os.path.isfile(root / GITMODULES)) + entries = gitmodules_entries(root, listed, source[0] == "--blob") + refused = tuple(dict.fromkeys( + (f"submodule.{name}.{key}", value) + for name, key, value, ignored in entries + if not ignored and refused_value(root, key, value))) + if refused: + return nothing._replace(refused=refused) + return read_sections(entries) + + +def gitmodules_source(root: Path) -> list[str] | None: + """What `git config` is given to read the `.gitmodules` git reads + (`config_from_gitmodules`, #768): `--file` for the work tree's, where + there is one, else `--blob` for the index's at stage 0, else HEAD's. + None where the index holds it only at a merge's conflict stages.""" try: - listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], + stages = git_out(["ls-files", "-s", "-z", "--", GITMODULES], cwd=root, binary=True) except Refusal: - return nothing - sections: dict[str, dict[str, str]] = {} - read: dict[str, str] = {} - held: dict[str, str] = {} - urls: dict[str, str] = {} + stages = b"" + source = ["--blob", f"HEAD:{GITMODULES}"] + for record in stages.split(b"\0"): + meta, _, name = record.partition(b"\t") + fields = meta.split() + if name != GITMODULES.encode() or len(fields) != 3: + continue + if fields[2] != b"0": + return None + source = ["--blob", fields[1].decode("ascii")] + if os.path.lexists(root / GITMODULES): + return ["--file", GITMODULES] + return source + + +def gitmodules_entries(root: Path, listed: bytes, blob: bool + ) -> list[tuple[str, str, str | None, bool]]: + """`(name, key, value, ignored)` for each `submodule..` that + `git config -z --list` printed in `listed`, in file order, as git's + submodule reader meets them: None for a key with no value, and + `ignored` for a section git ignores whole, whose name is empty or has + a `..` component, split on `/` and on a backslash (git warns `ignoring + suspicious submodule name`), a value git would die on included. In the + index's or HEAD's file, a `blob`, git reads nothing after a key it + refuses with no value (`ends_blob_reading`).""" + entries: list[tuple[str, str, str | None, bool]] = [] for record in listed.split(b"\0"): - key, _, value = record.decode("utf-8", - "surrogateescape").partition("\n") + key, valued, value = record.decode("utf-8", + "surrogateescape").partition("\n") section, _, variable = key.rpartition(".") if not section.startswith("submodule."): continue name = section[len("submodule."):] - sections.setdefault(name, {})[variable] = value - if (not name or ".." in name.replace("\\", "/").split("/") - or value.startswith("-")): + ignored = not name or ".." in name.replace("\\", "/").split("/") + if blob and not ignored and not valued and ends_blob_reading( + root, variable): + break + entries.append((name, variable, value if valued else None, ignored)) + return entries + + +def ends_blob_reading(root: Path, key: str) -> bool: + """Whether a `key` with no value ends git's reading of the index's or + HEAD's `.gitmodules`, a blob, where `bad config line` is an error and + no die (#768): each key git refuses with no value, but a `branch` only + where git does not crash on it (`crashes_on_branch`); before 2.43.1 + git crashes on it there too, and `refused_value` names it.""" + return key in NO_VALUE_REFUSED and (key != "branch" + or not crashes_on_branch(root)) + + +def read_sections(entries: list[tuple[str, str, str | None, bool]] + ) -> Gitmodules: + """`Gitmodules` of `entries` (`gitmodules_entries`) where git refuses + none. `given` counts every section that gives a path, an ignored one + too. `read` and `urls` keep git's rules: a section git ignores gives + neither; a `path` or `url` that begins with `-` is dropped, and the + value before it stands (`may be interpreted as a command-line + option`); where two sections give one path, the later in the file + decides, as a later value in one section replaces an earlier, and a + section that gives a path again lets go of the one it gave, even where + a later section gave that one too: git then reads no section there.""" + sections: dict[str, dict[str, str]] = {} + read: dict[str, str] = {} + held: dict[str, str] = {} + urls: dict[str, str] = {} + for name, key, value, ignored in entries: + sections.setdefault(name, {})[key] = "" if value is None else value + if ignored or value is None or value.startswith("-"): continue - if variable == "path": + if key == "path": if name in held: read.pop(held[name], None) held[name] = value read[value] = name - elif variable == "url": + elif key == "url": urls[name] = value return Gitmodules(Counter(keys["path"] for keys in sections.values() if "path" in keys), read, urls) -def submodule_config(root: Path) -> set[str]: +def refused_value(root: Path, key: str, value: str | None) -> bool: + """Whether git 2.43 dies reading a `.gitmodules` that gives a submodule + `value` for `key`, None for no value (#768), as `parse_config` in its + submodule-config.c reads the key: a `path`, `url`, `update` or + `ignore` with no value (`bad config line`), and a `branch` with none + (it crashes, and git 2.43.1 and later refuse it as they refuse a `url` + with none); an `update` other than `none`, `checkout`, `rebase` or + `merge`, in that case, a `!command` included (`invalid value`); a + `shallow` that is no boolean (`bad boolean config value`); and a + `fetchrecursesubmodules` that is neither a boolean nor `on-demand`. An + `ignore` it does not know is a warning, and any other key takes any + value.""" + if value is None: + return key in NO_VALUE_REFUSED + if key == "update": + return value not in UPDATE_VALUES + if key == "shallow" or (key == "fetchrecursesubmodules" + and value != "on-demand"): + return git_bool(root, value) is None + return False + + +def crashes_on_branch(root: Path, version: str | None = None) -> bool: + """Whether this git crashes reading a `.gitmodules` that gives a + submodule a `branch` with no value (#768): every git before 2.43.1 + does, in the index's or HEAD's file too, where a later git reads that + line as the end of the file (`BRANCH_READ_SINCE`). `version` is what + `git version` printed, for a test to give; None asks this git. A git + whose version cannot be read is taken for one that crashes: the run + then fails naming the key, a cure true on every git, rather than pass + a member git never asked for as the missing credential's.""" + if version is None: + try: + version = git_out(["version"], cwd=root) + except Refusal: + return True + found = GIT_VERSION_RE.match(version) + return (found is None + or tuple(map(int, found.groups())) < BRANCH_READ_SINCE) + + +def git_bool(root: Path, value: str | None) -> bool | None: + """`value` read as a boolean by git itself, which reads one so for + every key (`git config --type=bool`, its `git_parse_maybe_bool`): + `true`, `yes`, `on` in any case, a number other than 0, or no value at + all are true, and `false`, `no`, `off`, 0 and an empty value false. + None where git takes it for no boolean, and dies. It is asked as the + `--default` of a key looked up in an empty file, so no `probe.value` + in any of the person's own config scopes is read in its place.""" + if value is None: + return True + try: + answer = git_out(["config", "--file", os.devnull, "--type=bool", + f"--default={value}", "--get", "probe.value"], + cwd=root) + except Refusal: + return None + return answer == "true" + + +def submodule_config(root: Path) -> dict[str, list[str | None]]: """Every `submodule.` key this repository's git config sets, in every - scope git reads, as git's init asks them: `submodule.active`, and - `submodule..url` as an earlier init writes it. None where git - lists none.""" + scope git reads, with each value it gives, None for no value, as git's + init asks them: `submodule.active` and its patterns, a section's own + `submodule..active` (#768), and `submodule..url` as an + earlier init writes it. Empty where git lists none.""" try: config = git_out(["config", "-z", "--get-regexp", r"^submodule\."], cwd=root, binary=True) except Refusal: - return set() - return {record.decode("utf-8", "surrogateescape").partition("\n")[0] - for record in config.split(b"\0")} + return {} + found: dict[str, list[str | None]] = {} + for record in filter(None, config.split(b"\0")): + key, valued, value = record.decode("utf-8", + "surrogateescape").partition("\n") + found.setdefault(key, []).append(value if valued else None) + return found -def has_url(modules: Gitmodules, config: set[str], path: str) -> bool: +def has_url(modules: Gitmodules, config: dict[str, list[str | None]], + path: str) -> bool: """Whether git's init finds a url for the gitlink at `path`: in the section git reads for that path (`Gitmodules.read`), or in the git config under that section's name, as an earlier init writes it. Where @@ -384,8 +612,8 @@ def has_url(modules: Gitmodules, config: set[str], path: str) -> bool: or f"submodule.{name}.url" in config) -def init_stop(root: Path, modules: Gitmodules, config: set[str] - ) -> str | None: +def init_stop(root: Path, modules: Gitmodules, + config: dict[str, list[str | None]]) -> str | None: """The gitlink `git submodule update --init` dies at, or None: the first in the index, in index order, that its init step asks about and has no url for (`No url found for submodule path`, exit 128), before it @@ -394,11 +622,11 @@ def init_stop(root: Path, modules: Gitmodules, config: set[str] ASKED AS GIT 2.43 ASKS IT, of the index, `modules` and `config`, this repository's git config. Every gitlink the index records is asked, but where the config sets `submodule.active`, as a recursive clone sets it - (`.`), only those a section git reads gives a path: git counts no other - active. A gitlink git has no url for (`has_url`) stops it. So a stray - case variant or a member mounted by hand stops a plain clone's update, - and leaves a recursive clone's to the credential. Read nothing, it - answers None.""" + (`.`), only those git counts active (`active`), and never one no + section git reads gives a path. A gitlink git has no url for + (`has_url`) stops it. So a stray case variant or a member mounted by + hand stops a plain clone's update, and leaves a recursive clone's to + the credential. Read nothing, it answers None.""" try: index = git_out(["ls-files", "-s", "-z"], cwd=root, binary=True) except Refusal: @@ -408,13 +636,54 @@ def init_stop(root: Path, modules: Gitmodules, config: set[str] if meta.split()[:1] != [GITLINK_MODE]: continue path = entry.decode("utf-8", "surrogateescape") - if path not in modules.read and "submodule.active" in config: + if ("submodule.active" in config + and not active(root, modules, config, path)): continue if not has_url(modules, config, path): return path return None +def active(root: Path, modules: Gitmodules, + config: dict[str, list[str | None]], path: str) -> bool: + """Whether git's init counts the gitlink at `path` active where the + config sets `submodule.active` (`is_tree_submodule_active` in git + 2.43's submodule.c): never where no section git reads gives that path; + else as the section's own `submodule..active` says, its last + value read as a boolean (`git_bool`), where the config sets one that + is (#768); else where one of `submodule.active`'s patterns names it + (`pattern_names`).""" + name = modules.read.get(path) + if name is None: + return False + own = config.get(f"submodule.{name}.active") + flag = git_bool(root, own[-1]) if own else None + if flag is not None: + return flag + return any(pattern_names(pattern, path) + for pattern in config["submodule.active"]) + + +def pattern_names(pattern: str | None, path: str) -> bool: + """Whether a `submodule.active` pattern names `path` (#768). A plain + path, as `git clone --recurse-submodules=` writes one, names + itself and every path under it, as git's pathspec does, and `.` names + every path. A pattern with pathspec magic (a leading `:`), a glob + character or a backslash, or with an empty, `.` or `..` component past + a leading `./`, is read as naming every path, as every pattern was + before: this does not match those as git does.""" + if (pattern is None or pattern.startswith(":") + or any(char in pattern for char in "*?[\\")): + return True + while pattern.startswith("./"): + pattern = pattern[2:] + pattern = pattern.rstrip("/") + if pattern in ("", ".") or any(part in ("", ".", "..") + for part in pattern.split("/")): + return True + return path == pattern or path.startswith(pattern + "/") + + def unfetchable(root: Path, rows: list[dict], exited_0: bool ) -> tuple[list[tuple[dict, str, str]], str | None, Gitmodules]: @@ -425,7 +694,9 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool Asked of this holder's own records, never of git's words: any row at all while a merge leaves `.gitmodules` unmerged in the index, since git - then reads no section and asks for no credential; a row at + then reads no section and asks for no credential, or while the file + gives a value git dies on (`Gitmodules.refused`, #768), or is a work + tree file git cannot parse (`Gitmodules.unreadable`); a row at whose path the holder records no gitlink (`recorded_gitlink`, as `validate-family.py`'s `member-gitlink-absent` reads one), or whose gitlink no section of `.gitmodules` gives a path (as its @@ -436,33 +707,51 @@ def unfetchable(root: Path, rows: list[dict], exited_0: bool died at a gitlink it has no url for, any row at all, since git then checked out no member and asked for no credential. Any other row not checked out is the missing credential's, and stays the skip it always - was. A path two rows give is said once.""" + was. A path two rows give is said once. Where every row's member is + checked out and git did not exit 0, `.gitmodules` is read all the + same, for `fetch_members` to ask `refused` of.""" left: dict[str, dict] = {} for row in rows: if not (member_path(root, row) / ".git").exists(): left.setdefault(row_path(row), row) - if not left: + if not left and exited_0: return [], None, Gitmodules(Counter(), {}, {}) modules = gitmodules_sections(root) + if not left: + return [], None, modules config = submodule_config(root) stopper = None if exited_0 else init_stop(root, modules, config) found: list[tuple[dict, str, str]] = [] for path, row in left.items(): - if modules.unmerged: - found.append((row, path, UNMERGED)) - elif recorded_gitlink(root, path) is None: - found.append((row, path, NO_GITLINK)) - elif not modules.given[path]: - found.append((row, path, NO_SECTION)) - elif not has_url(modules, config, path): - found.append((row, path, NO_URL)) - elif exited_0: - found.append((row, path, EXITED_0)) - elif stopper is not None: - found.append((row, path, STOPPED)) + why = row_reason(root, path, modules, config, exited_0, stopper) + if why is not None: + found.append((row, path, why)) return found, stopper, modules +def row_reason(root: Path, path: str, modules: Gitmodules, + config: dict[str, list[str | None]], exited_0: bool, + stopper: str | None) -> str | None: + """Why `unfetchable` says the member at `path`, not checked out, is + none a credential would check out (#648, #768), in the order it asks, + or None where it is the missing credential's.""" + if modules.unmerged: + return UNMERGED + if modules.refused: + return REFUSED + if modules.unreadable: + return UNREADABLE + if recorded_gitlink(root, path) is None: + return NO_GITLINK + if not modules.given[path]: + return NO_SECTION + if not has_url(modules, config, path): + return NO_URL + if exited_0: + return EXITED_0 + return None if stopper is None else STOPPED + + def members_of(root: Path) -> tuple[str, list[dict]]: manifest_path = root / MANIFEST if not manifest_path.is_file(): diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index d551e4a4..d95a8636 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -62,6 +62,15 @@ clone fetches that path from only one of them, so the other is a url the next edit can make the one. It reads that file alone, so `--no-members` asks it too. + - that no section of `.gitmodules` gives a value git dies on as it reads + the file, before `git submodule update` checks out any member, the + `gitmodules-value-refused` finding (#768): a `path`, `url`, `update`, + `ignore` or `branch` with no value, an `update` other than `none`, + `checkout`, `rebase` or `merge`, a `shallow` that is no boolean, or a + `fetchRecurseSubmodules` that is neither a boolean nor `on-demand`. It + reads the work tree's file and HEAD's, so `--no-members` asks it too, + and where it is found no member that is not checked out is refused as + uninitialized: git checked none out. - the REVERSE of every member check (#361): that each gitlink this repository records in `members/` (the index at stage 0, else HEAD), and each path a `.gitmodules` section gives, is a path some row names. A @@ -106,10 +115,12 @@ 1 a FINDING: drift a reviewer can act on 2 a REFUSAL: the question could not be asked — no manifest, an uninitialized member (but for one no section of HEAD's `.gitmodules` - gives, or gives a url, or whose removal is staged: those are - `member-gitmodules-absent`, `member-gitmodules-url-absent` and - `member-gitlink-removal-staged`, exit 1), an unreadable pin, no git - to run. An unanswerable question is never an implicit pass. + gives, or gives a url, or whose removal is staged, and any while + `.gitmodules` gives a value git dies on: those are + `member-gitmodules-absent`, `member-gitmodules-url-absent`, + `member-gitlink-removal-staged` and `gitmodules-value-refused`, + exit 1), an unreadable pin, no git to run. An unanswerable question + is never an implicit pass. """ from __future__ import annotations @@ -128,7 +139,8 @@ from repo_shape import ( # noqa: E402 COMMIT_RE, PROJECT_ID_RE, PYTHON, SHA256_RE, TREE_DIGEST_DEFINITION, VISIBILITY_CHOICES, NamingPolicy, Refusal, file_sha256, find_repo_root, - git_out, load_yaml, recorded_gitlink, repo_basename, tree_digest, + git_out, load_yaml, recorded_gitlink, repo_basename, spelled, + tree_digest, ) DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") @@ -1263,12 +1275,20 @@ def _gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: not read: the missing git is one fact about the machine, and every check here that meets it says so under one code. """ + return _sections_listed(_gitmodules_listed(root)) + + +def _gitmodules_listed(root: Path) -> bytes: + """What `git config -z --file .gitmodules --list` prints of the work + tree's file, or nothing where there is none, read as + `_gitmodules_sections` says: what it and `_check_gitmodules_values` + read (#768).""" if not os.path.lexists(root / GITMODULES): - return {} + return b"" with _git_runs(f"{GITMODULES} could not be read as git reads it"): try: - listed = git_out(["config", "-z", "--file", GITMODULES, "--list"], - cwd=root, binary=True) + return git_out(["config", "-z", "--file", GITMODULES, "--list"], + cwd=root, binary=True) except Refusal as exc: raise Refusal( "gitmodules-unreadable", @@ -1278,7 +1298,6 @@ def _gitmodules_sections(root: Path) -> dict[str, dict[str, str]]: "holder's submodules can be cloned from it as it stands. Mend " f"the file by hand (`git config -f {GITMODULES} --list` says " "where it breaks), then run this again.") from exc - return _sections_listed(listed) def _sections_listed(listed: bytes) -> dict[str, dict[str, str]]: @@ -1311,6 +1330,14 @@ def _head_gitmodules(root: Path no pathspec, as `_member_gitlinks` lists HEAD. One git cannot read is `gitmodules-unreadable`, as the work tree's is. """ + head = _head_listed(root) + return None if head is None else (head[0], _sections_listed(head[1])) + + +def _head_listed(root: Path) -> tuple[bool, bytes] | None: + """`(whether HEAD holds a .gitmodules, what `git config -z --blob + --list` prints of it)`, read as `_head_gitmodules` says: + what it and `_check_gitmodules_values` read (#768).""" with _git_runs(f"HEAD's {GITMODULES} could not be read as git reads it"): try: tree = git_out(["ls-tree", "-z", "HEAD"], cwd=root, binary=True) @@ -1322,10 +1349,10 @@ def _head_gitmodules(root: Path if name == GITMODULES.encode() and meta.split()[1:2] == [b"blob"]), None) if blob is None: - return False, {} + return False, b"" try: - listed = git_out(["config", "-z", "--blob", blob, "--list"], - cwd=root, binary=True) + return True, git_out(["config", "-z", "--blob", blob, "--list"], + cwd=root, binary=True) except Refusal as exc: raise Refusal( "gitmodules-unreadable", @@ -1335,7 +1362,214 @@ def _head_gitmodules(root: Path "holder's submodules can be cloned from it. Mend the file by " f"hand (`git config -f {GITMODULES} --list` says where it " "breaks), commit it, then run this again.") from exc - return True, _sections_listed(listed) + + +#: Where `gitmodules-value-refused` says a value is, by whether the work +#: tree's file and HEAD's give it (#768). +GIVEN_WHERE = {(True, True): "here and in HEAD, which a clone reads", + (True, False): "in the work tree, not in HEAD", + (False, True): "in HEAD, which a clone reads, not in the " + "work tree"} + +#: What git takes for a `submodule..` whose value it dies on as +#: it reads `.gitmodules` (#768), as `gitmodules-value-refused` says it. +GIT_TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", + "shallow": "a boolean, `true` or `false`", + "fetchrecursesubmodules": "a boolean or `on-demand`", + "ignore": "one of `untracked`, `dirty`, `all` or `none`", + "path": "a path", "url": "a url", "branch": "a branch name"} + + +def _check_gitmodules_values(root: Path, members: list, + report: Report) -> tuple[bool, bool]: + """No section of `.gitmodules` gives a value git dies on as it reads + the file, the `gitmodules-value-refused` finding (#768). Returns + whether the work tree's file gives one, and whether the work tree's + or HEAD's does. + + Git dies reading such a file, before `git submodule update` checks + out any member, so `make bootstrap` checks none out either. In + a plain clone the members modes refused `member-uninitialized` for the + first row, naming that same update, and `--no-members` said `family + ok`. The rules are `parse_config`'s in git's submodule-config.c + (`_values_refused`). The work tree's file is read, which git reads + here, and HEAD's, which a clone reads, as `member-gitmodules-absent` + reads it: no member needs to be checked out, so every mode asks it. + One finding for each key, naming each value git dies on there. + + The remedy is by key, one command to a line, on the work tree's file, + as `gitmodules-path-unnamed` takes a key out: `--unset-all` takes out + every line of it, since git reads a section without one, and + `.gitmodules` is then committed where HEAD's gives it too. A member's + own section gets its `path` or `url` written back, as it last gave + one, or else as `family.py add` writes it, named after its path, with + `https://github.com/` and the row's `repository:` (`_written_back`). + Where HEAD's file alone gives the value, the work tree's is what to + commit, and where the work tree has no `.gitmodules`, the first such + finding puts HEAD's back first.""" + present = os.path.lexists(root / GITMODULES) + here = _submodule_entries(_gitmodules_listed(root)) + head = _head_listed(root) + there = _submodule_entries(head[1]) if head else [] + in_work = _values_refused(root, here) + in_head = _values_refused(root, there) + rows = {f"{DEFAULT_MEMBERS_DIR}/{row['project']}": row.get("repository") + for row in members if isinstance(row, dict) + and isinstance(row.get("project"), str) and row["project"]} + put_back = present + for name, key in dict.fromkeys([*in_work, *in_head]): + said = _value_said(name, key, in_work, in_head) + if present and (name, key) not in in_work: + report.finding("gitmodules-value-refused", + f"{said} Stage and commit the work tree's " + f"{GITMODULES}, which does not give it.") + continue + back = _written_back(name, key, here if present else there, rows) + report.finding("gitmodules-value-refused", _value_taken_out( + said, f"submodule.{name}.{key}", back, put_back=put_back, + committed=(name, key) in in_head)) + put_back = True + return bool(in_work), bool(in_work or in_head) + + +def _value_said(name: str, key: str, + in_work: dict[tuple[str, str], list[str | None]], + in_head: dict[tuple[str, str], list[str | None]]) -> str: + """`gitmodules-value-refused` up to its remedy, for the key `key` of + the section `name` (#768): each value the work tree's file and HEAD's + give it, where, and what git takes. Where HEAD's alone gives one, the + file git reads here may not, so it says git dies on it in a clone, + which reads HEAD's.""" + full = f"submodule.{name}.{key}" + values = dict.fromkeys([*in_work.get((name, key), []), + *in_head.get((name, key), [])]) + where = (name, key) in in_work, (name, key) in in_head + dies = ("so in a clone neither `git submodule update` nor make " + "bootstrap checks out any member" if where == (False, True) + else "before `git submodule update` checks out any member, so " + "neither that nor make bootstrap checks one out") + return (f"{GITMODULES} gives `{spelled(full)}` " + " and ".join( + "no value" if value is None else f"the value `{spelled(value)}`" + for value in values) + f" {GIVEN_WHERE[where]}, and git dies " + f"reading a file that does, {dies}. Git takes {GIT_TAKES[key]} for " + "that key.") + + +def _value_taken_out(said: str, full: str, back: str | None, *, + put_back: bool, committed: bool) -> str: + """`gitmodules-value-refused` whole, `said` (`_value_said`) and its + commands, one to a line, on the work tree's file (#768): HEAD's put + back first where the work tree has none and no finding before put it + back (`put_back` False), the key `full` taken out, and `back` written + in, where it is given, then the commit where HEAD's gives the value + (`committed`).""" + key = full.rpartition(".")[2] + lines = [] if put_back else [f"git checkout HEAD -- {GITMODULES}"] + lines.append(f"git config -f {GITMODULES} --unset-all " + f"{shlex.quote(full)}") + if back is not None: + lines.append(f"git config -f {GITMODULES} {shlex.quote(full)} " + f"{back}") + return (f"{said} " + + ("" if put_back else f"The work tree holds no {GITMODULES}, " + "so the first line below puts HEAD's back. ") + + "Take the key out" + + (f" and write its {key} back" if back is not None else "") + + " by hand, in a POSIX shell" + + (f", then stage and commit {GITMODULES}" if committed else "") + + ":" + "".join(f"\n {line}" for line in lines)) + + +def _submodule_entries(listed: bytes) -> list[tuple[str, str, str | None]]: + """`(name, key, value)` for each `submodule..` in what `git + config -z --list` printed, in file order, None for a key with no value + (#768). A section git ignores whole, whose name is empty or has a `..` + component, split on `/` and on a backslash, gives none.""" + entries: list[tuple[str, str, str | None]] = [] + for record in listed.split(b"\0"): + key, valued, value = record.decode("utf-8", + "surrogateescape").partition("\n") + section, _, variable = key.rpartition(".") + name = section[len("submodule."):] + if (section.startswith("submodule.") and name + and ".." not in name.replace("\\", "/").split("/")): + entries.append((name, variable, value if valued else None)) + return entries + + +def _values_refused(root: Path, entries: list[tuple[str, str, str | None]] + ) -> dict[tuple[str, str], list[str | None]]: + """`{(name, key): [each value]}` of `entries` (`_submodule_entries`) + that git dies on as it reads `.gitmodules` (#768), as `parse_config` + in its submodule-config.c reads a key: a `path`, `url`, `update` or + `ignore` with no value (`bad config line`), a `branch` with none (git + before 2.43.1 crashes on it, and a later one refuses it as it refuses + a `url` with none), an `update` other than `none`, `checkout`, + `rebase` or `merge`, in that case, a `!command` included (`invalid + value`), a `shallow` that is no boolean, and a `fetchrecursesubmodules` + that is neither a boolean nor `on-demand`, as git itself reads a + boolean (`git config --type=bool`). An `ignore` git does not know is + a warning, and any other key takes any value.""" + refused: dict[tuple[str, str], list[str | None]] = {} + for name, key, value in entries: + if value is None: + dies = key in {"path", "url", "update", "ignore", "branch"} + elif key == "update": + dies = value not in {"none", "checkout", "rebase", "merge"} + else: + dies = (key == "shallow" or (key == "fetchrecursesubmodules" + and value != "on-demand") + ) and not _git_takes_bool(root, value) + if dies: + refused.setdefault((name, key), []).append(value) + return refused + + +def _git_takes_bool(root: Path, value: str) -> bool: + """Whether git reads `value` as a boolean, asked of git itself (`git + config --type=bool`), which reads `shallow` and + `fetchRecurseSubmodules` so (#768). It is asked as the `--default` of + a key looked up in an empty file, so no `probe.value` in any of the + person's own config scopes is read in its place.""" + with _git_runs(f"a value in {GITMODULES} could not be read as git " + "reads a boolean"): + try: + git_out(["config", "--file", os.devnull, "--type=bool", + f"--default={value}", "--get", "probe.value"], + cwd=root) + except Refusal: + return False + return True + + +def _written_back(name: str, key: str, + entries: list[tuple[str, str, str | None]], + rows: dict[str, object]) -> str | None: + """What `gitmodules-value-refused` writes back for the `path` or `url` + of the section `name` once its key is out (#768), POSIX-quoted, or None + for any other key, and for a section whose path no row's + `members/` is: the value the section last gave it that git + takes, or else, as `family.py add` writes a section named after its + path, that path, or `https://github.com/` and the row's `repository:`, + `` where that is not `/`. The section's path is the one + it last gave, or its name where it gave none.""" + if key not in {"path", "url"}: + return None + gave = {wanted: [value for section, variable, value in entries + if section == name and variable == wanted and value + and not value.startswith("-")] + for wanted in ("path", "url")} + path = gave["path"][-1] if gave["path"] else name + if path not in rows: + return None + if gave[key]: + return shlex.quote(gave[key][-1]) + if key == "path": + return shlex.quote(path) + repository = rows[path] + return (shlex.quote(f"https://github.com/{repository}.git") + if isinstance(repository, str) and QUALIFIED_RE.match(repository) + else "") #: Where git's submodule commands read `.gitmodules` from when the work @@ -2250,10 +2484,25 @@ def _check_members(root: Path, members: list, no_members: bool, never reached. Where git's init would stop at a gitlink it has no url for, `member-uninitialized` says so before its update, with the cure (`_init_stop_said`, #762). + + A VALUE GIT DIES ON IN `.gitmodules` (#768) is asked first of the + checks that read that file, and where the work tree's file gives one, + `gitmodules-path-twice` and `gitmodules-path-unnamed` read nothing from + it, as git reads nothing: no command they would print for a section + then meets one of its own lines. Where + either file gives one, git checks out no member, so a member that is + not checked out is asked nothing, as one `member-gitmodules-absent` + names: the finding fails the run, where `member-uninitialized` named + the update that dies, and `_init_stop_said`, which would name a stop + git never reaches there, is not asked. """ _check_member_names_unique(members, report) _check_member_ids_unique(root, members, report) sections = _gitmodules_sections(root) + as_read = sections + refused_here, refused = _check_gitmodules_values(root, members, report) + if refused_here: + sections = {} _check_gitmodules_paths(sections, report) named = _row_paths(members) gitlinks = _member_gitlinks(root) @@ -2263,7 +2512,7 @@ def _check_members(root: Path, members: list, no_members: bool, unregistered = _check_gitlinks_registered(root, members, gitlinks, report) unregistered |= _check_removals_staged(root, members, gitlinks, removed, - sections, report) + as_read, report) if no_members: print(f" -- {len(members)} member(s) SKIPPED (--no-members): " "their gitlinks, digests and project ids are not checked " @@ -2280,11 +2529,14 @@ def _check_members(root: Path, members: list, no_members: bool, if asked is not None: gitlinked.append(asked) stop = (_init_stop_said(root, sections, named) - if any(asked.path not in unregistered - and not (root / asked.path / ".git").exists() - for asked in gitlinked) else "") + if not refused and any( + asked.path not in unregistered + and not (root / asked.path / ".git").exists() + for asked in gitlinked) else "") for asked in gitlinked: - _check_member_checkout(root, asked, report, unregistered, stop) + _check_member_checkout(root, asked, report, + set(gitlinks) if refused else unregistered, + stop) if not members: report.note("no members yet: a family with none is empty, not " "wrong") diff --git a/tests/test_family_gitmodules_values.py b/tests/test_family_gitmodules_values.py new file mode 100644 index 00000000..3a6610ac --- /dev/null +++ b/tests/test_family_gitmodules_values.py @@ -0,0 +1,897 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A `.gitmodules` value git dies on as it reads the file, in a family +holder's own bootstrap and gate, and where git reads the file from (#768). + +THE ISSUE'S THREE LINES, each reproduced on #682's head f74f640c first: + +- Bootstrap said `family bootstrap ok`, exit 0, where git dies reading + `.gitmodules` before it checks out any member: an `update = None` (the + typo of `update = none`), an `update = !true`, a `url`, `update`, + `ignore` or `branch` with no value (git 2.43.0 crashes on the last, + where 2.43.1 and later refuse it as the others), a `shallow = maybe`, a + `fetchRecurseSubmodules = bogus`. It names every row with the key and + its value now, and fails, where every member was checked out too. +- The family gate named none of them: members checked and `--pins` + refused `member-uninitialized`, whose remedy is the update that dies, + and `--no-members` said `family ok`. Every mode finds + `gitmodules-value-refused` now, read from the work tree's file and + HEAD's, and its commands take the key out. +- What #682 left approximate: where the work tree has no `.gitmodules`, + git reads the index's, then HEAD's; a section's own + `submodule..active` decides before `submodule.active`'s patterns; + and a plain path pattern names only itself and what is under it. + +THE SAME FAMILY, OFFLINE RULES AND STATES as +`test_family_bootstrap_gate_order.py` (#648), whose helpers are imported: +a plain clone of the holder, `git submodule update` given the fixture's +local members (`fetching`) where a credential is said to be there and not +given them where it is not. Every printed command is typed as printed. +""" + +from __future__ import annotations + +import re +import sys + +import pytest + +from conftest import REPO, load_without_bytecode, run_script +from repo_shape import PYTHON +from test_family import ORG, family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import ( + CHECKOUT_CURE, DUPLICATE, EXITED_0, FETCHED, NO_URL, NOT_ALL_FETCHED, + SECTION_CURE, STOPPED, after_first_line, bootstrap, cloned, edited_clone, + failed_on, fetching, not_fetched, type_line, url_edited, with_config, + without_url, +) +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( + GITMODULES, assert_passes_in_every_mode, commit_all, committed_gitmodules, + pinned, +) +from test_family_member_path import MEMBER, OTHER, raw +from test_family_tracking_branch import on_main # noqa: F401 +from test_validate_family_typed_states import ROW, removal_staged +from test_validate_family_unnamed_mounts import commands, findings + +#: The issue's table, `update = None` first, then `branch` with no value: +#: each a line added to OTHER's section after its url, as rv648 (p11) and +#: w648c (p12) added it, the key git lists for it, and the value, None for +#: none. Git's update dies on each, exit 128, but for `branch` with no +#: value, which git 2.43.0 crashes on (exit 139, w768's probe) and 2.43.1 +#: and later refuse as the others (exit 128, CI's 2.55.0): the tool's +#: answer is the test, never how git dies. +REFUSED_LINES = { + "update-None": ("update = None", "update", "None"), + "update-command": ("update = !true", "update", "!true"), + "url-no-value": ("url", "url", None), + "update-no-value": ("update", "update", None), + "ignore-no-value": ("ignore", "ignore", None), + "shallow-maybe": ("shallow = maybe", "shallow", "maybe"), + "fetch-bogus": ("fetchRecurseSubmodules = bogus", + "fetchrecursesubmodules", "bogus"), + "branch-no-value": ("branch", "branch", None), +} + +#: What both tools say git takes for each key. +TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", + "shallow": "a boolean, `true` or `false`", + "fetchrecursesubmodules": "a boolean or `on-demand`", + "ignore": "one of `untracked`, `dirty`, `all` or `none`", + "path": "a path", "url": "a url", "branch": "a branch name"} + +#: Where the finding says the value is. +HERE_AND_HEAD = "here and in HEAD, which a clone reads" +WORK_TREE_ONLY = "in the work tree, not in HEAD" +HEAD_ONLY = "in HEAD, which a clone reads, not in the work tree" + + +def with_line(line: str): + """An edit giving OTHER's section `line` after its url.""" + return lambda data: url_edited( + data, OTHER, lambda url: url + f"\t{line}\n".encode()) + + +def full(key: str) -> str: + return f"submodule.members/{OTHER}.{key}" + + +def refused_why(key: str, value: str | None) -> str: + """Why (a) says each row was not checked out.""" + return (f"git cannot read .gitmodules: `{full(key)}` " + + ("has no value" if value is None else f"is `{value}`")) + + +def value_cure(key: str) -> str: + """What (a) says cures it.""" + return (f"Give `{full(key)}` a value git takes, {TAKES[key]}, or take " + f"that line out, and commit .gitmodules. `{PYTHON} " + "scripts/validate-family.py --no-members` prints the commands.") + + +def said(key: str, value: str | None, where: str) -> str: + """`gitmodules-value-refused` up to its remedy. Where HEAD's file + alone gives the value, git dies on it in a clone, which reads HEAD's, + and that is what it says (rv819's NIT 8).""" + given = "no value" if value is None else f"the value `{value}`" + dies = ("so in a clone neither `git submodule update` nor make " + "bootstrap checks out any member" if where == HEAD_ONLY + else "before `git submodule update` checks out any member, so " + "neither that nor make bootstrap checks one out") + return (f"FINDING gitmodules-value-refused: .gitmodules gives " + f"`{full(key)}` {given} {where}, and git dies reading a file " + f"that does, {dies}. Git takes {TAKES[key]} for that key. ") + + +def taken_out(key: str, value: str | None, *, back: str | None = None, + where: str = HERE_AND_HEAD, put_back: bool = False) -> str: + """`gitmodules-value-refused` WHOLE, with the commands that take the key + out of the work tree's file, after the one that puts HEAD's back where + `put_back`, and write `back` in where it is given.""" + lines = (["git checkout HEAD -- .gitmodules"] if put_back else []) + [ + f"git config -f .gitmodules --unset-all {full(key)}"] + if back is not None: + lines.append(f"git config -f .gitmodules {full(key)} {back}") + return (said(key, value, where) + + ("The work tree holds no .gitmodules, so the first line below " + "puts HEAD's back. " if put_back else "") + + "Take the key out" + + (f" and write its {key} back" if back is not None else "") + + " by hand, in a POSIX shell" + + ("" if where == WORK_TREE_ONLY + else ", then stage and commit .gitmodules") + + ":" + "".join(f"\n {line}" for line in lines)) + + +def url_of(data: bytes) -> str: + """The url `family.py add` wrote in OTHER's section of `data`.""" + (url,) = re.findall(rb'\[submodule "members/' + OTHER.encode() + + rb'"\]\r?\n(?:\t[^\n]*\n)*?\turl = ([^\r\n]*)', data) + return url.decode("utf-8") + + +def assert_the_credential_s_skip(clone, env) -> None: + """Bootstrap with no credential for the members: git fails on their + fetch, not on `.gitmodules`, so they are skipped, and the run is ok.""" + result = bootstrap(clone, env, "--skip-members") + assert result.returncode == 0, result.stdout + result.stderr + assert "FETCH INCOMPLETE" in result.stderr + assert "not for want of a credential" not in result.stderr + + +def typed_and_committed(root, env, finding: str) -> None: + """Every command `finding` prints, typed as printed, then `.gitmodules` + staged and committed, as the finding says.""" + for command in commands(finding): + type_line(root, env, command) + raw(root, env, "add", "--", GITMODULES) + commit_all(root, env, "The key taken out as printed") + + +def found_in_every_mode_and_typed(root, env, expected: str) -> None: + """The gate finds `expected` alone, exit 1, in all three modes, and + its commands are typed as printed and committed.""" + for extra in ((), ("--pins",), ("--no-members",)): + gate = validate(root, *extra) + assert gate.returncode == 1, (extra, gate.stdout + gate.stderr) + assert findings(gate) == [expected], extra + typed_and_committed(root, env, expected) + + +def assert_a_good_holder_s_run(root, env) -> None: + """Bootstrap checks every member out and is a good holder's, and the + gate passes in every mode.""" + again = bootstrap(root, env, "--skip-members") + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + assert_passes_in_every_mode(root, 2) + + +# --- lines 1 and 2: bootstrap and the gate ---------------------------------- + +@pytest.mark.parametrize("line, key, value", REFUSED_LINES.values(), + ids=REFUSED_LINES) +def test_a_value_git_dies_on_fails_bootstrap_and_every_mode_of_the_gate( + family, holder, offline, tmp_path, line, key, value): + """THE ISSUE'S LINES 1 AND 2. The line committed, a plain clone with the + credential: git's update dies and checks nothing out, and bootstrap + said ok. It names both rows, the key and its value, and fails. The + gate refused `member-uninitialized` members checked, with the update + that dies for remedy, and said `family ok` under `--no-members`; it + finds `gitmodules-value-refused` in all three modes now. Typed as + printed and committed, which writes back the url a `url` line took + with it, the file is the one `family.py add` wrote, and the same run + checks both members out and passes the gate in every mode.""" + env = fetching(isolated(offline, tmp_path)) + clone, data = edited_clone(holder, env, family, tmp_path, with_line(line)) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, update.stderr + assert not (clone / "members" / MEMBER / ".git").exists(), ( + "premise: git checked nothing out") + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert after_first_line(result) == NOT_ALL_FETCHED + why = refused_why(key, value) + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure(key)), + failed_on(MEMBER, OTHER)], result.stderr + found_in_every_mode_and_typed( + clone, env, taken_out(key, value, + back=url_of(data) if key == "url" else None)) + assert raw(clone, env, "show", f"HEAD:{GITMODULES}").stdout == data + assert_a_good_holder_s_run(clone, env) + + +#: Values git takes, which no line or finding names (the issue's no-misfire +#: cases but `update = none`, whose run is the exit-0 skip as before, and +#: each other `update` git takes, rv819's finding 4). +TAKEN = {"update-checkout": "update = checkout", + "update-rebase": "update = rebase", + "update-merge": "update = merge", + "shallow-true": "shallow = true", + "fetch-on-demand": "fetchRecurseSubmodules = on-demand", + "ignore-bogus": "ignore = bogus"} + + +@pytest.mark.parametrize("line", TAKEN.values(), ids=TAKEN) +def test_a_value_git_takes_is_a_good_holder_s_run(family, holder, offline, + tmp_path, line): + """No misfire. Git takes each (an `ignore` it does not know is a + warning only). With no credential, git fails on the members' fetch and + they are the credential's: skipped, and ok. With it, the plain clone's + update checks both members out, bootstrap is a good holder's, and the + gate passes in every mode, in the clone and in the holder.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, with_line(line)) + assert_the_credential_s_skip(clone, env) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_first_line(result) == FETCHED + assert result.stderr == "" + assert_passes_in_every_mode(clone, 2) + assert_passes_in_every_mode(holder, 2) + + +def test_update_none_is_still_the_exit_0_skip(family, holder, offline, + tmp_path): + """No misfire. `update = none` is a value git takes: git exits 0 and + skips OTHER, and bootstrap names it with the update that checks it out, + as before (#648's N1), and no line or finding says git cannot read the + file.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + with_line("update = none")) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines() == [ + not_fetched((f"members/{OTHER}", EXITED_0), + cure=CHECKOUT_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)] + assert validate(clone, "--no-members").returncode == 0 + assert_passes_in_every_mode(holder, 2) + + +#: A section under a name git ignores whole, its `update = None` included +#: (`ignoring suspicious submodule name`): no row names its path. Git splits +#: the name on a backslash too, on every system (`check_submodule_name`); +#: in the file it is escaped, as git config escapes a subsection name. +IGNORED = b'[submodule "../x"]\n\tpath = x\n\tupdate = None\n' +IGNORED_NAMES = {"slash": IGNORED, + "backslash": IGNORED.replace(b"../x", b"..\\\\x")} + + +@pytest.mark.parametrize("ignored", IGNORED_NAMES.values(), ids=IGNORED_NAMES) +def test_a_value_in_a_section_git_ignores_whole_is_no_finding( + family, holder, offline, tmp_path, ignored): + """No misfire. A section whose name has a `..` component is ignored + whole by git, a value it would die on included. With no credential, + the members are the credential's: skipped, and ok. With it, the plain + clone's update warns and checks both members out, bootstrap is a good + holder's, and the gate names the section as one no row names, not as a + value git refuses.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: data + ignored) + assert_the_credential_s_skip(clone, env) + + result = bootstrap(clone, fetching(env), "--skip-members") + + assert result.returncode == 0, result.stdout + result.stderr + assert after_first_line(result) == FETCHED + gate = validate(clone, "--no-members") + assert [found.split(":")[0] for found in findings(gate)] == [ + "FINDING gitmodules-path-unnamed"], gate.stderr + + +def test_a_holder_with_every_member_checked_out_fails_on_it_too( + holder, offline, tmp_path): + """Every member checked out, as in the holder itself, and `update = + None` committed: git dies before it updates any of them, so no row is + left to name, and bootstrap said ok over git's `fatal:` line. It says + no member was updated, and why, and fails on `.gitmodules`; typed as + the gate prints it and committed, the run is a good holder's.""" + env = fetching(isolated(offline, tmp_path)) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, with_line("update = None")(data)) + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + " NO MEMBER UPDATED, and not for want of a credential: " + f"{refused_why('update', 'None')}. {value_cure('update')}", + "family bootstrap FAILED: .gitmodules (git cannot read it)"] + gate = validate(holder) + assert gate.returncode == 1, gate.stdout + gate.stderr + assert findings(gate) == [taken_out("update", "None")] + typed_and_committed(holder, env, findings(gate)[0]) + + again = bootstrap(holder, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +#: A `.gitmodules` git cannot parse at all, the line added at its end: a +#: merge's conflict marker left in it, and a quote left open (rv819's +#: finding 3). `git config --list` and git's update both die on it. +DAMAGED = {"conflict-marker": b"<<<<<<< HEAD\n", + "open-quote": b'\tupdate = "none\n'} + +#: What (a) says of such a file, and what cures it. +UNPARSED = "git cannot parse .gitmodules" +MEND_CURE = ("Mend .gitmodules by hand (`git config -f .gitmodules --list` " + "says where it breaks), and commit it.") + + +@pytest.mark.parametrize("damage", DAMAGED.values(), ids=DAMAGED) +def test_a_gitmodules_git_cannot_parse_fails_with_every_member_checked_out( + holder, offline, tmp_path, damage): + """Every member checked out, as in the holder itself, and a + `.gitmodules` git cannot parse committed: git's update dies before it + updates any member, and bootstrap said `family bootstrap ok`, where a + value git refuses failed the run. It says no member was updated, and + why, and fails on `.gitmodules`; mended and committed, the run is a + good holder's.""" + env = fetching(isolated(offline, tmp_path)) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, data + damage) + update = raw(holder, env, "submodule", "update", "--init", "--recursive", + check=False) + assert b"bad config line" in update.stderr, update.stderr + + result = bootstrap(holder, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + " NO MEMBER UPDATED, and not for want of a credential: " + f"{UNPARSED}. {MEND_CURE}", + "family bootstrap FAILED: .gitmodules (git cannot read it)"] + committed_gitmodules(holder, env, data) + + again = bootstrap(holder, env, "--skip-members") + + assert again.returncode == 0, again.stdout + again.stderr + assert after_first_line(again) == FETCHED + + +def test_a_gitmodules_git_cannot_parse_is_named_for_every_row( + family, holder, offline, tmp_path): + """A plain clone of a holder whose committed `.gitmodules` git cannot + parse, with the credential: git's update dies and checks nothing out. + Bootstrap said no section gave either path, with the validator for + cure; it says git cannot parse the file, and how to mend it.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: data + DAMAGED["conflict-marker"]) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", UNPARSED), + (f"members/{OTHER}", UNPARSED), cure=MEND_CURE), + failed_on(MEMBER, OTHER)], result.stderr + + +#: The work tree's `.gitmodules` lost and not committed, its index entry +#: left (git reads that) or staged out too (git reads HEAD's). +def lose_work_tree_file(root, env) -> None: + (root / GITMODULES).unlink() + + +def lose_work_tree_and_index_files(root, env) -> None: + raw(root, env, "rm", "-q", "--cached", "--", GITMODULES) + (root / GITMODULES).unlink() + + +LOST = {"index-read": lose_work_tree_file, + "head-read": lose_work_tree_and_index_files} + + +@pytest.mark.parametrize("lose", LOST.values(), ids=LOST) +def test_where_the_work_tree_has_no_gitmodules_git_reads_the_index_s_or_head_s( + family, holder, offline, tmp_path, lose): + """THE ISSUE'S LINE 3, its first item. A plain clone whose work tree + lost `.gitmodules`, not committed, with no credential: git reads the + index's file, or HEAD's where the index lost it too, and fails on the + members' fetch. Bootstrap read the work tree's alone, said "no section + of .gitmodules gives that path" for each, and failed; the members are + the credential's now, skipped, and the run ends ok.""" + env = isolated(offline, tmp_path) + clone = cloned(holder, env, family, tmp_path, recurse=False) + lose(clone, env) + + assert_the_credential_s_skip(clone, env) + + +def test_a_key_with_no_value_ends_git_s_reading_of_the_index_s_file( + family, holder, offline, tmp_path): + """No misfire. A `url` with no value, the last line of the index's + file, which git reads where the work tree has none: in a blob git + calls the line bad and stops reading there, an error and no die, so it + reads every section before it and fails on the fetch for want of a + credential. The members are the credential's: skipped, and ok.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, with_line("url")) + lose_work_tree_file(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert b"bad config line" in update.stderr, update.stderr + + assert_the_credential_s_skip(clone, env) + + +@pytest.mark.parametrize("lose", LOST.values(), ids=LOST) +def test_a_branch_with_no_value_in_a_blob_is_read_as_this_git_reads_it( + family, holder, offline, tmp_path, lose): + """A `branch` with no value, the last line of the index's file, or + HEAD's, which git reads where the work tree has none, and no + credential. Git 2.43.1 and later read it as a `url` with none, `bad + config line`, an error in a blob and no die: they read every section + before it and fail on the fetch for want of a credential, and + bootstrap said git cannot read the file and failed; the members are + the credential's now, skipped, and ok. Git 2.43.0 crashes on it there + too and checks nothing out, and bootstrap names it in every row, the + key and no value, and fails, as in the work tree's file.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, with_line("branch")) + lose(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, "premise: no credential, or a crash" + if b"bad config line" in update.stderr: + assert_the_credential_s_skip(clone, env) + return + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = refused_why("branch", None) + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure("branch")), + failed_on(MEMBER, OTHER)], result.stderr + + +def test_the_index_s_file_is_read_where_it_differs_from_head_s( + family, holder, offline, tmp_path): + """THE ISSUE'S LINE 3: git reads the index's `.gitmodules` before + HEAD's. A plain clone with the credential, `update = None` staged and + not committed, and the work tree's file lost: git reads the index's + and dies, HEAD's being clean, and bootstrap names it in every row + (rv819's finding 4).""" + env = fetching(isolated(offline, tmp_path)) + clone = cloned(holder, env, family, tmp_path, recurse=False) + data = raw(clone, env, "show", f"HEAD:{GITMODULES}").stdout + (clone / GITMODULES).write_bytes(with_line("update = None")(data)) + raw(clone, env, "add", "--", GITMODULES) + lose_work_tree_file(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, update.stderr + assert not (clone / "members" / MEMBER / ".git").exists(), ( + "premise: git checked nothing out") + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = refused_why("update", "None") + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure("update")), + failed_on(MEMBER, OTHER)], result.stderr + + +def test_a_key_with_no_value_in_a_section_git_ignores_reads_on( + family, holder, offline, tmp_path): + """No misfire. A section git ignores whole, first in the index's file, + gives `url` with no value: git warns and reads on, as in the work + tree's file, so it reads both members' sections and fails on the fetch + for want of a credential. The members are the credential's: skipped, + and ok (rv819's finding 4).""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone( + holder, env, family, tmp_path, + lambda data: b'[submodule "../x"]\n\tpath = x\n\turl\n' + data) + lose_work_tree_file(clone, env) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert b"bad config line" not in update.stderr, update.stderr + + assert_the_credential_s_skip(clone, env) + + +#: `git version` lines, and whether `crashes_on_branch` says that git +#: crashes on a `branch` with no value: before 2.43.1, and where no version +#: can be read. +VERSIONS = { + "2.43.0": ("git version 2.43.0", True), + "2.43.1": ("git version 2.43.1", False), + "2.42.4": ("git version 2.42.4", True), + "apple-2.39.3": ("git version 2.39.3 (Apple Git-146)", True), + "windows-2.43.0": ("git version 2.43.0.windows.1", True), + "windows-2.45.1": ("git version 2.45.1.windows.1", False), + "2.55.0": ("git version 2.55.0", False), + "3.0.0": ("git version 3.0.0", False), + "unread": ("git version unknown", True), +} + + +@pytest.mark.parametrize("version, crashes", VERSIONS.values(), + ids=VERSIONS) +def test_which_git_crashes_on_a_branch_with_no_value(monkeypatch, tmp_path, + version, crashes): + """Which git the family root's bootstrap takes for one that crashes + on a `branch` with no value, where it reads the index's or HEAD's + file: every git before 2.43.1, and one whose version it cannot read. + The test above meets the git it runs on; this meets each side of the + line. Loading it puts its own folder on `sys.path`, which is put + back.""" + monkeypatch.setattr(sys, "path", list(sys.path)) + module = load_without_bytecode( + REPO / "templates" / "family-root" / "scripts" / "bootstrap.py", + "gitmodules_values_bootstrap") + + assert module.crashes_on_branch(tmp_path, version) is crashes + + +def test_a_value_git_dies_on_in_the_index_s_file_is_named_and_mended( + family, holder, offline, tmp_path): + """`update = None` committed, and the work tree's file lost, not + committed: git reads the index's and dies all the same, and bootstrap + names it. The gate reads HEAD's, and, the work tree holding none, its + first line puts HEAD's back before the key is taken out. Typed as + printed and committed, the same run checks both members out and the + gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + clone, _ = edited_clone(holder, env, family, tmp_path, + with_line("update = None")) + lose_work_tree_file(clone, env) + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + why = refused_why("update", "None") + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{MEMBER}", why), (f"members/{OTHER}", why), + cure=value_cure("update")), + failed_on(MEMBER, OTHER)], result.stderr + found_in_every_mode_and_typed( + clone, env, + taken_out("update", "None", where=HEAD_ONLY, put_back=True)) + assert_a_good_holder_s_run(clone, env) + + +def test_two_keys_with_no_gitmodules_in_the_work_tree_put_head_s_back_once( + holder, offline, tmp_path): + """Two values git dies on committed, and the work tree's file lost: one + finding for each key, and only the first puts HEAD's back, since a + second `git checkout` would put back the key the first took out. Typed + as printed and committed, the holder passes in every mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + # Each line goes in right after the url, so `update` comes first. + committed_gitmodules(holder, env, with_line("update = None")( + with_line("shallow = maybe")(data))) + lose_work_tree_file(holder, env) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [ + taken_out("update", "None", where=HEAD_ONLY, put_back=True), + taken_out("shallow", "maybe", where=HEAD_ONLY)] + for finding in findings(result): + for command in commands(finding): + type_line(holder, env, command) + raw(holder, env, "add", "--", GITMODULES) + commit_all(holder, env, "Both keys taken out as printed") + assert raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout == data + assert_passes_in_every_mode(holder, 2) + + +# --- line 3: submodule..active and submodule.active's patterns --------- + +def test_a_member_whose_own_active_is_false_is_not_where_git_stops( + family, holder, offline, tmp_path): + """THE ISSUE'S LINE 3, its second item. OTHER's section lost its url, + committed; in a plain clone `submodule.active` is `.`, as a recursive + clone sets it, so every member is in the subset, and OTHER's own + `submodule.members/

.active` is set twice, true then false, of which + git reads the last. Git asks that first, inits + MEMBER alone and fails on its fetch for want of a credential, and stops + nowhere; bootstrap said git stopped at OTHER. MEMBER is the + credential's now, and OTHER, which git has no url for, is named with + the section written back.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: without_url(data, OTHER)) + raw(clone, env, "config", "submodule.active", ".") + for flag in ("true", "false"): + raw(clone, env, "config", "--add", + f"submodule.members/{OTHER}.active", flag) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert update.returncode != 0, "premise: the fetch failed" + assert b"No url found" not in update.stderr, update.stderr + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched((f"members/{OTHER}", NO_URL), + cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(OTHER)], result.stderr + + +#: `submodule.active` patterns: `(the patterns, whether git's init then +#: asks OTHER and dies, (a)'s rows, the rows it fails on)`. A plain path, +#: however spelt, names MEMBER alone, as `git clone +#: --recurse-submodules=` writes it; a leading directory names both; +#: the start of OTHER's path that is no directory of it names nothing. +STOPS_AT_OTHER = (f"members/{MEMBER}", STOPPED.format(path=f"members/{OTHER}")) +PATTERNS = { + "a-path": ((f"members/{MEMBER}",), False, + ((f"members/{OTHER}", NO_URL),), (OTHER,)), + "a-path-spelt-another-way": ((f"./members/{MEMBER}/",), False, + ((f"members/{OTHER}", NO_URL),), (OTHER,)), + "a-leading-directory": (("members",), True, + (STOPS_AT_OTHER, (f"members/{OTHER}", NO_URL)), + (MEMBER, OTHER)), + "a-string-prefix": ((f"members/{MEMBER}", f"members/{OTHER[:-1]}"), + False, ((f"members/{OTHER}", NO_URL),), (OTHER,)), +} + + +@pytest.mark.parametrize("patterns, stops, left, failed", PATTERNS.values(), + ids=PATTERNS) +def test_a_subset_s_pattern_says_whether_git_stops_at_a_member_outside_it( + family, holder, offline, tmp_path, patterns, stops, left, failed): + """THE ISSUE'S LINE 3, its third item. OTHER's section lost its url, + committed, and the clone's subset is `patterns`, with no credential. + Where the pattern leaves OTHER out, git's init skips it and fails on + MEMBER's fetch, and bootstrap said git stopped at OTHER; MEMBER is the + credential's now. Where it names OTHER, git does die there, and that + is still said.""" + env = isolated(offline, tmp_path) + clone, _ = edited_clone(holder, env, family, tmp_path, + lambda data: without_url(data, OTHER)) + for pattern in patterns: + raw(clone, env, "config", "--add", "submodule.active", pattern) + update = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert (f"No url found for submodule path 'members/{OTHER}'".encode() + in update.stderr) is stops, update.stderr + + result = bootstrap(clone, env, "--skip-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert result.stderr.splitlines()[-2:] == [ + not_fetched(*left, cure=SECTION_CURE.format(path=f"members/{OTHER}")), + failed_on(*failed)], result.stderr + + +# --- the gate: which file gives it, and what is written back ----------------- + +def mended_not_committed(root, env, data: bytes) -> None: + """`update = None` committed, the work tree's file mended by hand.""" + committed_gitmodules(root, env, with_line("update = None")(data)) + (root / GITMODULES).write_bytes(data) + + +def edited_not_committed(root, env, data: bytes) -> None: + """`update = None` in the work tree's file alone.""" + (root / GITMODULES).write_bytes(with_line("update = None")(data)) + + +WHERE = { + "head-only": (mended_not_committed, + said("update", "None", HEAD_ONLY) + "Stage and commit the " + "work tree's .gitmodules, which does not give it."), + "work-tree-only": (edited_not_committed, + taken_out("update", "None", where=WORK_TREE_ONLY)), +} + + +@pytest.mark.parametrize("set_up, expected", WHERE.values(), ids=WHERE) +def test_the_gate_says_which_file_gives_it(holder, offline, tmp_path, set_up, + expected): + """In the holder, members checked out: HEAD's file alone gives the + value, the work tree's mended and not committed, and the finding says + to commit that, printing no command; or the work tree's alone, which + git reads here and a clone does not, and the command takes it out with + nothing to commit. Done as it says, the holder passes in every mode.""" + env = isolated(offline, tmp_path) + set_up(holder, env, raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [expected] + for command in commands(expected): + type_line(holder, env, command) + raw(holder, env, "add", "--", GITMODULES) + raw(holder, env, "commit", "-q", "--allow-empty", "-m", "As it says") + assert_passes_in_every_mode(holder, 2) + + +def test_a_url_no_row_gives_one_for_is_written_back_as_a_placeholder( + holder, offline, tmp_path): + """OTHER's row gives no `/` repository, and its section's + only `url` line has no value, both committed: there is no url as + `family.py add` writes one, so the finding writes back `` for a + person to fill in, as `member-gitmodules-absent` does (rv819's finding + 4).""" + env = isolated(offline, tmp_path) + rows = (holder / "family.yaml").read_bytes() + qualified = f"repository: {ORG}/{OTHER}".encode() + assert rows.count(qualified) == 1, rows + (holder / "family.yaml").write_bytes( + rows.replace(qualified, f"repository: {OTHER}".encode())) + raw(holder, env, "add", "--", "family.yaml") + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, replaced("url")(data)) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result)[0] == taken_out("url", None, back="") + + +def test_a_value_head_s_file_alone_gives_leaves_the_work_tree_s_read( + holder, offline, tmp_path): + """HEAD's file alone gives `update = None`, and the work tree's, + mended and not committed, gives OTHER's path in a second section. Git + reads the work tree's here, so `gitmodules-path-twice` still reads it + and names the two: only a value in the work tree's file stops that + reading (rv819's finding 4).""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, with_line("update = None")(data)) + (holder / GITMODULES).write_bytes(data + DUPLICATE) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + assert [found.split(":")[0] for found in findings(result)] == [ + "FINDING gitmodules-value-refused", "FINDING gitmodules-path-twice"] + + +#: OTHER's `url` or `path` line replaced by the key with no value, so the +#: section gives none git takes: `(edit, key, the value written back, the +#: findings after it)`. The url is `family.py add`'s for the row; the path +#: is the section's name. With no path, no section of HEAD's file gives +#: OTHER's, which `member-gitmodules-absent` says too, and its line writes +#: the same path again. +def replaced(key: str, before: bytes = b""): + """An edit replacing OTHER's `key` line with `before`, then the key + with no value.""" + pattern = re.compile(rb"(\[submodule \"members/" + OTHER.encode() + + rb"\"\]\r?\n(?:\t[^\n]*\n)*?)\t" + key.encode() + + rb" = [^\n]*\n") + return lambda data: pattern.sub( + lambda found: found.group(1) + before + f"\t{key}\n".encode(), data, + count=1) + + +#: `(edit, key, the value written back, the findings after it)`. A url git +#: drops, one that begins with `-`, is no url to write back (rv819's +#: finding 4). +WRITTEN_BACK = { + "url": (replaced("url"), "url", + f"https://github.com/InkRouter/{OTHER}.git", ()), + "path": (replaced("path"), "path", f"members/{OTHER}", + ("member-gitmodules-absent",)), + "url-after-a-dashed-one": ( + replaced("url", b"\turl = -bad\n"), "url", + f"https://github.com/InkRouter/{OTHER}.git", ()), +} + + +@pytest.mark.parametrize("edit, key, back, beside", WRITTEN_BACK.values(), + ids=WRITTEN_BACK) +def test_a_url_or_path_left_with_no_value_is_written_back_as_add_writes_it( + holder, offline, tmp_path, edit, key, back, beside): + """The section's only `url` or `path` line has no value, committed: + taking the key out would leave the member none, so the finding writes + it back after, as `family.py add` writes it, never a url git drops. + The work tree's file gives the value too, so no other check of it is + asked, and no `gitmodules-path-unnamed` takes the member's section out + by key. Typed as printed and committed, the holder passes in every + mode.""" + env = isolated(offline, tmp_path) + data = raw(holder, env, "show", f"HEAD:{GITMODULES}").stdout + committed_gitmodules(holder, env, edit(data)) + + result = validate(holder, "--no-members") + + assert result.returncode == 1, result.stdout + result.stderr + found = findings(result) + assert found[0] == taken_out(key, None, back=back) + assert [code.split(":")[0][len("FINDING "):] + for code in found[1:]] == list(beside) + for finding in found: + for command in commands(finding): + type_line(holder, env, command) + raw(holder, env, "add", "--", GITMODULES) + commit_all(holder, env, "Written back as printed") + assert_passes_in_every_mode(holder, 2) + + + +def test_a_probe_value_in_the_person_s_own_config_changes_no_answer( + family, holder, offline, tmp_path): + """Both tools ask git whether a value is a boolean through a key of + their own, `probe.value`. One the person's own config gives, here no + boolean, made both refuse `shallow = true` (rv819's NIT 9): bootstrap + named every row and failed, and the gate found + `gitmodules-value-refused`. Neither reads it now: the members are the + credential's, skipped, and ok, and the gate passes.""" + env = with_config(isolated(offline, tmp_path), "probe.value", "notabool") + clone, _ = edited_clone(holder, env, family, tmp_path, + with_line("shallow = true")) + + assert_the_credential_s_skip(clone, env) + gate = run_script(clone / "scripts" / "validate-family.py", + "--no-members", cwd=clone, env=env) + assert gate.returncode == 0, gate.stdout + gate.stderr + + +# --- beside #762's staged removal -------------------------------------------- + +def test_a_staged_removal_beside_a_refused_value_puts_no_section_back( + on_main, offline, tmp_path): + """#768 BESIDE #762, the point rv820 asked the second PR to land + (resolution point 3 of the merge with main `b9b70692`). The work tree's + file gives OTHER's section `update = None`, which git dies on, and + `git rm --cached` has staged MEMBER's gitlink out, taking no section + from `.gitmodules`. The gate reads no section from a file it refuses + (`sections` is emptied), and the finding for the removal read that as + a section `git rm` took out: it printed `git checkout HEAD -- + .gitmodules` and said it put the section back. It is handed the + sections as read, so it prints the reset alone, in every mode.""" + env = isolated(offline, tmp_path) + commit = pinned(on_main, MEMBER) + raw(on_main, env, "rm", "-q", "--cached", "--", ROW) + edited_not_committed( + on_main, env, raw(on_main, env, "show", f"HEAD:{GITMODULES}").stdout) + expected = [taken_out("update", "None", where=WORK_TREE_ONLY), + removal_staged(on_main, commit, "git-rm-cached")] + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(on_main, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert findings(result) == expected, extra + staged = findings(result)[1] + assert commands(staged) == [f"git reset -q -- {ROW}"], extra + assert f"git checkout HEAD -- {GITMODULES}" not in staged, extra diff --git a/tests/test_validate_family_without_git.py b/tests/test_validate_family_without_git.py index 1ca0e9ca..3d5ad928 100644 --- a/tests/test_validate_family_without_git.py +++ b/tests/test_validate_family_without_git.py @@ -121,18 +121,20 @@ def test_with_no_gitmodules_the_gitlink_listing_is_named(holder, monkeypatch, capsys, extra): """THE ISSUE'S RESIDUE: a holder with member rows and its `.gitmodules` gone from the working tree, where main still ended in the traceback in - the default and `--pins` modes. The first check to run git is now #361's - listing of `members/`, which every mode runs, `--no-members` included: - where it passed before, having run no git at all, it now refuses.""" + the default and `--pins` modes. The first check to run git was #361's + listing of `members/`; since #768 it is the reading of HEAD's + `.gitmodules` for a value git dies on, which every mode runs, + `--no-members` included: where it passed before, having run no git at + all, it now refuses.""" (holder / ".gitmodules").unlink() module = holder_validator(holder, monkeypatch) without_git(module, monkeypatch, validator=True, helpers=True) code, err = run(module, holder, capsys, *extra) assert code == 2, err assert err == [ - "REFUSED git-not-runnable: the gitlinks this repository records in " - "members/ could not be listed: git could not be run (No such file " - "or directory)", REMEDIATION] + "REFUSED git-not-runnable: HEAD's .gitmodules could not be read as " + "git reads it: git could not be run (No such file or directory)", + REMEDIATION] @MEMBER_MODES