diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 41da2127..d551e4a4 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -43,11 +43,18 @@ path the row may not have is a fact about the member (#367). A row whose gitlink is recorded only at a case variant of its path is refused naming that variant, the index move that puts it back, and what to do - with a directory the variant leaves on disk (#611). The first is asked - of every row before the other two are asked of any (#648), so a later - row's gitlink refusal is never hidden behind an earlier row's - `member-uninitialized`, as a plain clone whose `git submodule update` - died at that gitlink left it. + with a directory the variant leaves on disk (#611); with `--sparse` + where a sparse checkout keeps a variant off disk, an entry the index + holds at or under the row's path moved aside first, and the lines that + make `.gitmodules` give the row's path where no section of HEAD's does, + as after a `git mv` (#762). The first is asked of every row before the + other two are asked of any (#648), so a later row's gitlink refusal is + never hidden behind an earlier row's `member-uninitialized`, as a plain + clone whose `git submodule update` died at that gitlink left it. Where + git's init would stop at a gitlink it has no url for, that refusal + names it and bootstrap's cure for it before its update, and a pinned + commit the member's store lacks names the fetch that brings it, or the + `bump` for one no remote holds (#762). And, across the rows, that no two record ONE `id`: that is one project pinned twice, however each row spelled its repository (#206). This one reads `family.yaml` alone, so `--no-members` asks it too. @@ -69,7 +76,9 @@ that records it at that path, as the row's own refusal does members checked (#648). A gitlink whose removal is staged and not committed, which HEAD still records, says to commit it, and prints no `git rm`, - which would fail (#681). + which would fail (#681). One no section of HEAD's `.gitmodules` gives + a url is said to be checked out nowhere by a recursive clone, and to + stop a plain clone's `git submodule update --init` (#762). - that a section of HEAD's `.gitmodules` gives the path of every row's gitlink, the `member-gitmodules-absent` finding (#611, #407's decision 3). A recursive clone reads HEAD's file, so a gitlink no section of it @@ -82,7 +91,12 @@ add` staged and not committed is this finding for its own member; a section taken out and staged, not committed, is no finding; and a file put back and staged, not committed, in a holder whose HEAD lost it is - still this finding for every member. + still this finding for every member. A section that gives the path and + no url is `member-gitmodules-url-absent`, read the same way (#762). + - that no row's own gitlink has its removal staged and not committed, + the `member-gitlink-removal-staged` finding (#762): HEAD still records + it, and the next commit un-registers the member under its row. It + needs no member checked out, so `--no-members` asks it too. - the COPIED shape files, each against its `sha256` row in `contracts/shape-pin.yaml`. Editing a copy in place is drift and the exit is to carry it upstream, never to update the digest. @@ -92,9 +106,10 @@ 1 a FINDING: drift a reviewer can act on 2 a REFUSAL: the question could not be asked — no manifest, an uninitialized member (but for one no section of HEAD's `.gitmodules` - gives: that is `member-gitmodules-absent`, exit 1), an unreadable - pin, no git to run. An unanswerable question is never an implicit - pass. + gives, or gives a url, or whose removal is staged: those are + `member-gitmodules-absent`, `member-gitmodules-url-absent` and + `member-gitlink-removal-staged`, exit 1), an unreadable pin, no git + to run. An unanswerable question is never an implicit pass. """ from __future__ import annotations @@ -455,26 +470,32 @@ def _pin_commit(pin: dict, project: str, report: Report) -> str: def _check_member_gitlink(root: Path, path: str, commit: str, project: str, repository, gitlinks: dict[str, str], - report: Report, rule: str) -> None: + report: Report, rule: str, + removed: frozenset[str] = frozenset()) -> None: """The submodule gitlink this repository records at `path`, against the pin — THE LOCKSTEP RULE. Split from `_check_member` for #137. Raises `Refusal` when there is no gitlink at all: a row naming a member this repository does not record as a submodule (`_gitlink_put_back` says what to do). Where `gitlinks` (`_member_gitlinks`) holds one at a case - variant of `path`, the refusal names that instead (#611).""" + variant of `path`, the refusal names that instead (#611). A gitlink + whose removal is staged (`removed`) is HEAD's, which the next commit + drops: `member-gitlink-removal-staged` names it, and nothing is said of + it here (#762).""" gitlink = recorded_gitlink(root, path) if gitlink is None: variants = sorted(other for other in gitlinks if other != path and other.casefold() == path.casefold()) if variants: - raise _gitlink_in_another_case(path, commit, project, variants, - gitlinks) + raise _gitlink_in_another_case(root, path, commit, project, + variants, gitlinks, repository) raise Refusal( "member-gitlink-absent", f"no gitlink (mode 160000) recorded at {path!r}. {project} is " f"named in {MANIFEST} but this repository does not record it as a " "submodule.", _gitlink_put_back(root, path, commit, repository)) + if path in removed: + return if gitlink != commit: report.finding( "member-gitlink-mismatch", @@ -524,9 +545,10 @@ def _gitlink_put_back(root: Path, path: str, commit: str, f"{shlex.quote(f'160000,{commit},{path}')}{lines}") -def _gitlink_in_another_case(path: str, commit: str, project: str, - variants: list[str], - gitlinks: dict[str, str]) -> Refusal: +def _gitlink_in_another_case(root: Path, path: str, commit: str, + project: str, variants: list[str], + gitlinks: dict[str, str], + repository) -> Refusal: """`member-gitlink-absent` for a row whose gitlink this repository records only at a case variant of its path (#611, #363's item 3), as a holder committed in #363's state does: `Members/

`, under a row and a @@ -547,19 +569,25 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, member; these take it out and record it together. `--ignore-unmatch`, since a variant whose removal is staged and not yet committed is still named here, from HEAD, and the plain `git rm` of it failed (`pathspec - ... did not match`). - - WHAT IS LEFT ON DISK. Where nothing is checked out at the row's path, - as in a fresh clone, `git submodule update --init --recursive` checks - the member out there once the move is committed. Where the disk holds - a variant apart from the row's path, a directory there (the empty one a - fresh clone leaves, a plain one, or a checkout) stays behind, - untracked, and `family.py add` refuses it as - `family-members-dir-variant` (#574), so the text says to move it out of - the holder, by an `mv` with nothing to fill in, which a shell test - skips where the disk folds the two names into one directory, the - row's own, and where the directory is a checkout (#681). A checkout - is deleted by hand once nothing only it holds is needed. + ... did not match`). What else the move meets, `_move_context` reads + (#762): a variant a sparse checkout keeps off disk, an entry the index + holds at or under the row's path, and a `.gitmodules` that does not + give the row's path. + + WHAT IS LEFT ON DISK. Where the disk holds a variant apart from the + row's path, a directory there (the empty one a fresh clone leaves, a + plain one, or a checkout) stays behind, untracked, and `family.py add` + refuses it as `family-members-dir-variant` (#574), so the text says to + move it out of the holder, by an `mv` with nothing to fill in, which a + shell test skips where the disk folds the two names into one + directory, the row's own, and where the directory is a checkout + (#681). A checkout is deleted by hand once nothing only it holds is + needed. Where nothing is checked out at the row's path, as in a fresh + clone, `git submodule update --init --recursive` then checks the member + out there, LAST (#762): a checkout `git mv` moved to the variant names + the member's own store, and that update points the store at the row's + path, after which `git -C status`, which the text gives for + what only the checkout holds, reads the row's path instead. """ found = ", ".join(f"{variant} (at {gitlinks[variant]})" for variant in variants) @@ -571,14 +599,102 @@ def _gitlink_in_another_case(path: str, commit: str, project: str, f"directory with it: {found}. A recursive clone checks out nothing " f"at {path}.", "Remediation: the member is recorded in the wrong case. " - + _wrong_case_move(path, commit, variants)) + + _wrong_case_move(path, commit, variants, + _move_context(root, path, variants, repository))) + + +class MoveContext(NamedTuple): + """What the wrong-case move meets beside the variants it takes out + (#762), as `_move_context` reads it: `sparse`, the variants a sparse + checkout keeps off disk; `in_the_way`, every entry the index holds at + or under the row's path; and `section`, the lines that make + `.gitmodules` give the row's path, with what they do, or empty where a + section of HEAD's already gives it.""" + + sparse: tuple[str, ...] = () + in_the_way: tuple[str, ...] = () + section: str = "" + section_said: str = "" + + +def _move_context(root: Path, path: str, variants: list[str], + repository) -> MoveContext: + """What the index and HEAD's `.gitmodules` hold around the move of a + member recorded only at `variants` to its row's `path` (#762). + + `--sparse` (the issue's (f)): a variant a sparse checkout keeps off + disk (`git ls-files -t`'s `S`, as `family.py`'s `_skip_worktree` reads + it) is passed over by `git rm -r --cached --ignore-unmatch`, which + exits 0 and removes nothing, so `git update-index --add` then left two + gitlinks in the index. Measured on git 2.43, in a cone and a non-cone + checkout alike; `git rm --sparse` came with git 2.34 (`family.py`'s + `SPARSE_RM_GIT`). + + AN ENTRY IN THE WAY ((g)): `git update-index --add --cacheinfo` + replaced a file at the row's path without a word, and failed on a file + under it (`'members/

' appears as both a file and as a directory`, + exit 128) after the line before it had taken the variant out, which + left the member un-registered. Every entry at or under the path, as + git compares paths, but those a merge left unmerged: at the path the + move's second line replaces them, stages 1 to 3, as it replaces one at + stage 0; under it, it leaves them beside the gitlink, and the commit + after it refuses them (`Committing is not possible because you have + unmerged files`, exit 128), so neither is lost without a word (git 2.43 + and 2.47.1, rv820's NIT 5). The whole index, with no pathspec, as + `_member_gitlinks` lists it. + + THE SECTION ((e)): `git mv members/

Members/

` rewrites its + section's `path` to the variant, under the name it had, so the move + put the gitlink back and `member-gitmodules-absent` followed. Where no + section of HEAD's `.gitmodules`, which a clone reads, gives the row's + path, the section that gives a variant's is given the row's path, the + first variant's where two have one; and where none gives a variant's + either, the section is written as `family.py add` writes one + (`_section_lines`). Then `git add` stages the file. + """ + with _git_runs(f"the index entries at and around {path} could not be " + "listed"): + listed = git_out(["ls-files", "-t", "-z"], cwd=root, binary=True) + sparse: set[str] = set() + in_the_way: dict[str, None] = {} + for record in listed.split(b"\0"): + tag, name = record[:1], record[2:].decode("utf-8", "surrogateescape") + if tag == b"S" and name in variants: + sparse.add(name) + if tag != b"M" and (name == path or name.startswith(path + "/")): + in_the_way[name] = None + head = _head_gitmodules(root) + sections = head[1] if head is not None else {} + given = {keys["path"]: name for name, keys in sections.items() + if "path" in keys} + section = said = "" + if path not in given: + named = next((given[variant] for variant in variants + if variant in given), None) + if named is not None: + section = (f"\n git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{named}.path')} " + f"{shlex.quote(path)}") + said = (f"give {path} back to the section of HEAD's " + f"{GITMODULES} that gives the variant's path, as git mv " + "leaves it") + else: + section = _section_lines(path, repository, sections) + said = (f"write its section as family.py add writes one, since " + f"no section of HEAD's {GITMODULES} gives {path} or the " + "variant's path") + section += f"\n git add -- {GITMODULES}" + return MoveContext(tuple(sorted(sparse)), tuple(in_the_way), section, + said) -def _wrong_case_move(path: str, commit: str, variants: list[str]) -> str: +def _wrong_case_move(path: str, commit: str, variants: list[str], + context: MoveContext = MoveContext()) -> str: """The index move for a member recorded only at `variants`, case variants of its row's `path`, to `path` at the pinned `commit`, with what it leaves on disk: what `_gitlink_in_another_case` says, and `member-gitlink-unnamed` under `--no-members` (#648), in one text. + What else it meets is `context` (`_move_context`, #762). EACH `mv` IS GUARDED BY TWO TESTS A POSIX SHELL RUNS (#681, rv611c's N-1 and N-2), not by a system's name: this text is read in CI too, on @@ -612,12 +728,22 @@ def _wrong_case_move(path: str, commit: str, variants: list[str]) -> str: unpushed = " or ".join(f"`git -C {shlex.quote(variant)} log --oneline " "HEAD --branches --not --remotes`" for variant in variants) + flag = " --sparse" if context.sparse else "" return ( - "Move its gitlink to the row's path, at the commit the row pins, in " - "the index alone, which touches no file, with the two lines below, " - f"and commit that. Where nothing is checked out at {path}, as in a " - "fresh clone, `git submodule update --init --recursive` then checks " - "the member out there. A directory left at " + _entries_aside(path, context.in_the_way) + + ("Then move" if context.in_the_way else "Move") + + " its gitlink to the row's path, at the commit the row pins, in " + "the index alone, which touches no file, with the " + + ("first two lines below" if context.section + else "two lines below") + + (f" (the first with --sparse, which needs git 2.34 or later: a " + f"sparse checkout keeps {' and '.join(context.sparse)} off disk, " + "and without it git rm passes over what that keeps off disk, exit " + "0, and removes nothing)" if context.sparse else "") + + (f"; the lines after them {context.section_said}, and stage " + f"{GITMODULES}. Commit that." if context.section + else ", and commit that.") + + " A directory left at " f"{' or '.join(variants)} is the row's own where the disk folds " "case, as macOS's and Windows' usually do and a Linux mount can " "(WSL's /mnt/c). Where the disk holds the two apart it stays there, " @@ -636,20 +762,64 @@ def _wrong_case_move(path: str, commit: str, variants: list[str]) -> str: "holds is needed. A checkout whose .git is a directory, a clone of " "its own, holds its commits there and nowhere else, and " f"{unpushed} lists the ones not pushed: move it out of the holder " - "whole, by hand, rather than delete it. The two lines:\n" - f" git rm -r --cached --ignore-unmatch -- {taken_out}\n" + "whole, by hand, rather than delete it. Then, where nothing is " + f"checked out at {path}, as in a fresh clone, `git submodule update " + "--init --recursive` checks the member out there. Run it last: a " + "checkout git mv moved to a variant names the member's own store, " + f"which that update points at {path}, so {status} reads {path} " + "after it. The " + + ("lines" if context.section else "two lines") + ":\n" + f" git rm -r --cached{flag} --ignore-unmatch -- {taken_out}\n" " git update-index --add --cacheinfo " - f"{shlex.quote(f'160000,{commit},{path}')}") + f"{shlex.quote(f'160000,{commit},{path}')}{context.section}") + + +def _entries_aside(path: str, entries: tuple[str, ...]) -> str: + """What the wrong-case move says first where the index holds `entries` + at or under the row's `path` (#762's (g)), or nothing: they go aside + with a committed `git mv`, as `family.py`'s `_members_is_a_file` moves + a file at `members`, since the move's second line replaced one at the + path without a word and failed on one under it. `` is the + person's to choose, as there.""" + if not entries: + return "" + if entries == (path,): + held = (f"The index holds an entry at {path} that is no gitlink, " + "which the move's second line replaces without a word") + else: + held = ("The index holds " + + ", ".join(entries[:5]) + + (f" and {len(entries) - 5} more" if len(entries) > 5 + else "") + + f" under {path}, at which the move's second line stops " + "(appears as both a file and as a directory) after the first " + "has taken the variant out") + return ( + f"{held}. Move it aside first, `git mv -- {shlex.quote(path)} `, a new name outside {DEFAULT_MEMBERS_DIR}/, and commit that " + "move on its own. ") def _check_member_tree(member: Path, pin: dict, path: str, commit: str, - project: str, report: Report, rule: str) -> None: + project: str, report: Report, rule: str, + stop: str = "") -> None: """Is `member`'s own object store readable at `commit`, and does its digest match the pin? Split from `_check_member` for #137 — the two questions were already one comment block ("the digest and the identity") in the source. Raises `Refusal` when the tree cannot even be read: an uninitialized submodule, or a commit its checkout does not have. Presence is not identity and an unreadable surface must fail rather than degrade. + + `member-uninitialized` names `git submodule update --init --recursive`, + which dies where git's init stops at a gitlink it has no url for, before + it checks out any member (rv648's S1, H1 and U1): `stop` is then what + says so and cures it (`_init_stop_said`, #762). + + `member-commit-unresolvable` names the fetch of the pinned commit, which + a remote that holds it serves and one that does not answers `not our + ref`, and THE LOCKSTEP RULE's `bump` for a pin no remote holds (#762, + rv681's P2: on the remote, the fetch put it in the store and the gate + passed; on none, the fetch and `git submodule update` both exit 128). """ if not (member / ".git").exists(): raise Refusal( @@ -658,17 +828,24 @@ def _check_member_tree(member: Path, pin: dict, path: str, commit: str, "pinned digest nor the project it claims can be read. Presence is " "not identity and an unreadable surface must fail rather than " "degrade.", - "Remediation: `git submodule update --init --recursive`, or `make " - "bootstrap`, which resolves a credential for private members " - "first.") + f"Remediation: {stop}`git submodule update --init --recursive`, " + "or `make bootstrap`, which resolves a credential for private " + "members first.") try: git_out(["rev-parse", "--verify", "--quiet", f"{commit}^{{commit}}"], cwd=member) except Refusal as exc: + fetch = shlex.join(["git", "-C", path, "fetch", "origin", commit]) raise Refusal( "member-commit-unresolvable", f"{path}: the pinned commit {commit} is not in that member's " - f"object store ({exc.detail})") from exc + f"object store ({exc.detail})", + f"Remediation: where a remote of the member holds it, `{fetch}` " + "fetches it into that store, and this then reads it. Where none " + "does, that fetch fails `not our ref`, exit 128, as `git " + "submodule update` does, and the pin names a commit no clone can " + "fetch: move the gitlink and the pin to one a remote holds. " + + rule) from exc recorded = pin.get("tree_sha256") if not isinstance(recorded, str) or not SHA256_RE.match(recorded): @@ -709,7 +886,8 @@ class Gitlinked(NamedTuple): def _check_member(root: Path, row: dict, seen: dict, report: Report, - gitlinks: dict[str, str]) -> Gitlinked | None: + gitlinks: dict[str, str], + removed: frozenset[str] = frozenset()) -> Gitlinked | None: """One member up to its checkout: the row itself and the gitlink. The member, for `_check_member_checkout` to ask its digest and identity of, or None where nothing more is asked of it. @@ -747,12 +925,13 @@ def _check_member(root: Path, row: dict, seen: dict, report: Report, rule = _lockstep(root, project, str(row.get("id") or project)) with _git_runs(f"the gitlink recorded at {path} could not be read"): _check_member_gitlink(root, path, commit, project, - row.get("repository"), gitlinks, report, rule) + row.get("repository"), gitlinks, report, rule, + removed) return Gitlinked(row, project, path, pin, commit, rule) def _check_member_checkout(root: Path, asked: Gitlinked, report: Report, - unregistered: set[str]) -> None: + unregistered: set[str], stop: str = "") -> None: """One member's digest and identity, read in its checkout, asked once every row's gitlink has been (`_check_members`, #648). @@ -762,7 +941,11 @@ def _check_member_checkout(root: Path, asked: Gitlinked, report: Report, submodule update --init`, which exits 0 there and checks nothing out, and the refusal dropped the finding that says why: the state a fresh clone of a holder whose HEAD lost `.gitmodules` is in, and CI's. The - finding is made all the same, so the run still fails. + finding is made all the same, so the run still fails. So is one whose + section gives no url, and one whose removal is staged (#762), each the + finding of its own. `stop` is what `member-uninitialized` says first + where git's init would stop at a gitlink it has no url for + (`_init_stop_said`). """ member = root / asked.path if asked.path in unregistered and not (member / ".git").exists(): @@ -770,7 +953,7 @@ def _check_member_checkout(root: Path, asked: Gitlinked, report: Report, with _git_runs(f"{asked.project}'s pinned commit and its tree digest " f"could not be read in {asked.path}"): _check_member_tree(member, asked.pin, asked.path, asked.commit, - asked.project, report, asked.rule) + asked.project, report, asked.rule, stop) _check_member_identity(member, asked.row, asked.project, report) @@ -1155,6 +1338,44 @@ def _head_gitmodules(root: Path return True, _sections_listed(listed) +#: Where git's submodule commands read `.gitmodules` from when the work +#: tree's is gone from disk, in git's order (`config_from_gitmodules`): +#: `(the name git resolves, the line that puts the work tree's back from +#: it, whose it is)`. +GITMODULES_SOURCES = ( + (f":{GITMODULES}", f"git checkout -- {GITMODULES}", "the index's"), + (f"HEAD:{GITMODULES}", f"git checkout HEAD -- {GITMODULES}", + "HEAD's, as the index holds none"), +) + + +def _gitmodules_put_back(root: Path) -> tuple[str, str, str]: + """Where the work tree's `.gitmodules` is gone from disk, the one git's + submodule commands read in its place, as `(the name git resolves, the + line that puts the work tree's back from it, whose it is)`; three empty + strings where the work tree holds one, or nothing does (#762, rv820's + SHOULD 2). + + GIT'S ORDER (`GITMODULES_SOURCES`): `:.gitmodules`, the index's at stage + 0, then `HEAD:.gitmodules`, each resolved here as git resolves it, with + `git rev-parse --verify --quiet`. A file deleted and not staged, or + whose deletion is staged, leaves git every section it read before; a + cure that wrote one with `git config -f` made a file of that section + alone, which staged and committed dropped every other, so the line puts + the file back first. + """ + if os.path.lexists(root / GITMODULES): + return "", "", "" + with _git_runs(f"where git reads {GITMODULES} from could not be read"): + for spec, line, whose in GITMODULES_SOURCES: + try: + git_out(["rev-parse", "--verify", "--quiet", spec], cwd=root) + except Refusal: + continue + return spec, line, whose + return "", "", "" + + def _check_gitmodules_paths(sections: dict[str, dict[str, str]], report: Report) -> None: """No two sections of `.gitmodules` give ONE path (#300). @@ -1253,7 +1474,8 @@ def _staged_removals(root: Path, gitlinks: dict[str, str]) -> frozenset[str]: no entry at that path, at any stage. A `git rm` of one fails (`pathspec ... did not match any files`), so a finding about one says to commit the removal and prints none: it named the same `git rm` until the - commit. A path the index holds unmerged is a conflict, not a removal, + commit. A row's own is `member-gitlink-removal-staged` (#762). A path + the index holds unmerged is a conflict, not a removal, and is not one of these. The whole index, `-z`, as `_member_gitlinks` lists it.""" with _git_runs(f"the gitlinks this repository records in " @@ -1267,13 +1489,15 @@ def _staged_removals(root: Path, gitlinks: dict[str, str]) -> frozenset[str]: class RowPath(NamedTuple): """What a row names by the path it gives: `who` names the row in a finding, `canonical` is the one path its project may have, - `members/`, or None for a row with no `project:`, and `pin` is + `members/`, or None for a row with no `project:`, `pin` is the commit its `pin:` names, lowered, or None where that is no 40-hex - commit (#648).""" + commit (#648), and `repository` its `repository:`, for the section the + wrong-case move writes (#762).""" who: str canonical: str | None pin: str | None = None + repository: object = None def _row_paths(members: list) -> dict[str, RowPath]: @@ -1308,7 +1532,8 @@ def _row_paths(members: list) -> dict[str, RowPath]: commit = str(pin.get("commit") or "") if isinstance(pin, dict) else "" named.setdefault(path, RowPath( f"the row for {project}" if project else "a row with no project:", - canonical, commit.lower() if COMMIT_RE.match(commit) else None)) + canonical, commit.lower() if COMMIT_RE.match(commit) else None, + row.get("repository"))) return named @@ -1414,9 +1639,15 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], Every member check starts from a row, so a gitlink no row names was never asked about: a member mounted by hand with `git submodule add`, or - a row deleted by hand. A recursive clone of the holder fetches it all - the same, and so does `bootstrap.py`'s `git submodule update`, while - nothing pins, digests or identifies it. + a row deleted by hand. Where a section of HEAD's `.gitmodules` gives its + path a url, a recursive clone of the holder fetches it all the same, and + so does `bootstrap.py`'s `git submodule update`, while nothing pins, + digests or identifies it. Where none does, as for a stray case variant + or a gitlink recorded by hand, a recursive clone checks out nothing at + it, and in a plain clone `git submodule update --init` stops there + (`No url found for submodule path`, exit 128) before it checks out any + member: the text said a recursive clone fetched it (#762, w648b's S1, + S2, H1 and X1). EXACTLY, as git compares paths. A gitlink a row names in another case is still a finding, and says which of three it is (`_variant_case`). A @@ -1450,12 +1681,23 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], no `git rm` (#681). A hand mount's keeps the branch and the `add`. """ holds = set(gitlinks) + head = _head_gitmodules(root) if set(gitlinks) - set(named) else None + fetched = ({keys["path"] for keys in head[1].values() + if "path" in keys and "url" in keys} + if head is not None else set()) for path, commit in sorted(gitlinks.items()): if path in named: continue said = (f"{path} is a gitlink at {commit} that no row of members: " - "names, so a recursive clone of this holder fetches it and " - "nothing here pins, digests or identifies it.") + + ("names, so a recursive clone of this holder fetches it " + "and nothing here pins, digests or identifies it." + if path in fetched else + f"names and no section of HEAD's {GITMODULES} gives a " + "url, so a recursive clone of this holder checks out " + "nothing at it, in a plain clone git submodule update " + "--init stops there (No url found for submodule path, " + "exit 128) before it checks out any member, and nothing " + "here pins, digests or identifies it.")) variant = _case_variant(path, named) if variant is not None: row_path, row = variant @@ -1468,7 +1710,10 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], "member-gitlink-unnamed", f"{said}{_variant_said(row_path, row)} The member is " "recorded in the wrong case. " - + _wrong_case_move(row_path, row.pin, variants)) + + _wrong_case_move(row_path, row.pin, variants, + _move_context(root, row_path, + variants, + row.repository))) continue if kind != STRAY: report.finding( @@ -1678,12 +1923,20 @@ def _check_gitlinks_registered(root: Path, members: list, The remedy writes the section as `family.py add` writes it (`_section_lines`). + + A SECTION WITH NO URL is `member-gitmodules-url-absent`, made here for + the same rows from the same file, and returned with them (#762, rv648's + U1): one section gives the path and no url, so in a plain clone `git + submodule update --init` dies there (`No url found for submodule + path`, exit 128) before it checks out any member, and no mode named it. + A path more than one section gives is `gitmodules-path-twice`'s. """ head = _head_gitmodules(root) if head is None: return set() held, sections = head - given = {keys["path"] for keys in sections.values() if "path" in keys} + given = Counter(keys["path"] for keys in sections.values() + if "path" in keys) unregistered: set[str] = set() for row in members: project = row.get("project") if isinstance(row, dict) else None @@ -1691,7 +1944,12 @@ def _check_gitlinks_registered(root: Path, members: list, continue path = f"{DEFAULT_MEMBERS_DIR}/{project}" if (str(row.get("path") or path) != path or path in unregistered - or path not in gitlinks or path in given): + or path not in gitlinks): + continue + if path in given: + if given[path] == 1 and _url_absent(root, path, gitlinks[path], + row, sections, report): + unregistered.add(path) continue unregistered.add(path) lines = _section_lines(path, row.get("repository"), sections) @@ -1709,6 +1967,266 @@ def _check_gitlinks_registered(root: Path, members: list, return unregistered +def _url_absent(root: Path, path: str, commit: str, row: dict, + sections: dict[str, dict[str, str]], report: Report) -> bool: + """`member-gitmodules-url-absent` where the one section of HEAD's + `.gitmodules` (`sections`) that gives `path` gives no url (#762); True + where it was made. The url written is the one `family.py add` writes, + `https://github.com/` and the row's `repository:`, or `` where + that is not `/`, as `_section_lines` writes it. Where the + work tree's file is gone from disk, a line before it puts that back + (`_gitmodules_put_back`), since `git config -f` would make a file of + that one key, and staged it dropped every other section.""" + name = next(name for name, keys in sections.items() + if keys.get("path") == path) + if "url" in sections[name]: + return False + repository = row.get("repository") + url = (shlex.quote(f"https://github.com/{repository}.git") + if isinstance(repository, str) and QUALIFIED_RE.match(repository) + else "") + _, put_back, whose = _gitmodules_put_back(root) + report.finding( + "member-gitmodules-url-absent", + f"{path} is a gitlink at {commit} that a row of members: names, and " + f"the section of HEAD's {GITMODULES} that gives that path, " + f'"{name}", gives no url, so in a plain clone of this holder git ' + "submodule update --init stops there (No url found for submodule " + "path, exit 128) before it checks out any member, as make " + "bootstrap's does, and a recursive clone checks out nothing there. " + "Write its url back by hand, in a POSIX shell, as family.py add " + f"writes one, then stage and commit {GITMODULES}" + + (f", which is gone from the work tree: the first line puts back " + f"{whose}, so that every other section stays:\n {put_back}" + if put_back else ":") + + f"\n git config -f {GITMODULES} " + f"{shlex.quote(f'submodule.{name}.url')} {url}") + return True + + +def _check_removals_staged(root: Path, members: list, + gitlinks: dict[str, str], removed: frozenset[str], + worktree: dict[str, dict[str, str]], + report: Report) -> set[str]: + """A row's own gitlink whose removal is staged and not committed — the + `member-gitlink-removal-staged` finding (#762, w681's candidate 2). + Returns the paths it was made for, of which `_check_member` asks no + checkout that is not there. + + HEAD records the gitlink and the index holds nothing at its path + (`removed`, `_staged_removals`), so `recorded_gitlink` answers from + HEAD and the gate said `family ok`, members checked or not, while the + next commit un-registers the member under its row, and the gate then + refuses `member-gitlink-absent`: CI was the first to see it. It reads + the index first because the index is what the next commit records, and + that holds here too; a finding, not a refusal, since every question + the row asks can still be asked of HEAD. A clean checkout, CI's, never + holds one. + + BOTH WAYS OUT ARE TYPED (rv762's probes, git 2.43): to stay, `git reset + -q -- ` puts the gitlink back as HEAD records it; where a plain + `git rm` also took its section out of the work tree's `.gitmodules` + (`worktree`), `git checkout HEAD -- .gitmodules` puts that back, and + where it took the checkout, `git submodule update --init --recursive -- + ` checks it out again, since that update dies `No url found` + while the section is gone. To go, the same, then `family.py remove`, + which takes the row, the gitlink and the section out in one commit and + refuses a work tree's `.gitmodules` that is not the index's + (`gitmodules-unstaged`) and a staged removal alike, on a branch, as the + duplicate-id remedy names it. + """ + if not removed: + return set() + head = _head_gitmodules(root) + in_head = ({keys["path"] for keys in head[1].values() if "path" in keys} + if head is not None else set()) + in_worktree = {keys["path"] for keys in worktree.values() + if "path" in keys} + staged: set[str] = set() + for row in members: + project = row.get("project") if isinstance(row, dict) else None + if not isinstance(project, str) or not project: + continue + path = f"{DEFAULT_MEMBERS_DIR}/{project}" + if (str(row.get("path") or path) != path or path in staged + or path not in removed): + continue + staged.add(path) + lines = _put_back_lines(root, path, path in in_head + and path not in in_worktree) + report.finding("member-gitlink-removal-staged", _removal_staged_said( + root, row, project, path, gitlinks[path], lines)) + return staged + + +def _put_back_lines(root: Path, path: str, + section_taken: bool) -> list[tuple[str, str]]: + """The lines `member-gitlink-removal-staged` prints that keep the member + at `path`, each with what it puts back (#762): its gitlink; the section + a plain `git rm` took out of the work tree's `.gitmodules`, where + `section_taken` (HEAD's file gives the path and the work tree's does + not); and its checkout, where none is there.""" + lines = [(f"git reset -q -- {shlex.quote(path)}", "its gitlink")] + if section_taken: + lines.append((f"git checkout HEAD -- {GITMODULES}", + f"the {GITMODULES} section git rm took out, " + "putting that file back as HEAD has it and " + "dropping any other change to it")) + if not (root / path / ".git").exists(): + lines.append(("git submodule update --init --recursive -- " + f"{shlex.quote(path)}", "its checkout")) + return lines + + +def _removal_staged_said(root: Path, row: dict, project: str, path: str, + commit: str, lines: list[tuple[str, str]]) -> str: + """`member-gitlink-removal-staged`'s text for the row naming `project`, + whose gitlink at `path` is at `commit` in HEAD, with `lines` + (`_put_back_lines`) below it (#762).""" + member_id = str(row.get("id") or project) + remove = (f"{PYTHON} scripts/family.py remove --family-root " + f"{shlex.quote(str(root))} --member {shlex.quote(project)}") + commands = "\n ".join(line for line, _ in lines) + return ( + f"{path} is a gitlink at {commit} that a row of members: names and " + "HEAD records, and the index holds nothing at that path: its removal " + "is staged and not committed, so the next commit un-registers the " + "member under its row, and this gate then refuses " + "member-gitlink-absent. If the member is to stay, " + + _put_back_said([said for _, said in lines]) + + ", and nothing is left to commit. If it is to go, put it back the " + "same way, then take it out with family.py remove, from a checkout " + "of openRepoShape, which takes its row, its gitlink and its section " + "out in one commit, on a branch other than the holder's tracking " + "branch, which it refuses " + f"(`{_switch_line(root, 'remove', member_id)}` makes one): " + f"`{remove}`. {BRANCH_LEFT}\n {commands}") + + +def _put_back_said(put: list[str]) -> str: + """What the lines `member-gitlink-removal-staged` prints put back, one + to a line, in their order: the gitlink first (#762).""" + if len(put) == 1: + return f"the line below puts {put[0]} back as HEAD records it" + ordinals = ["the first", "the second", "the third"] + return ("the lines below put the member back as HEAD has it: " + + ", ".join(f"{ordinals[at]} {said}" + for at, said in enumerate(put))) + + +def _init_stop_said(root: Path, sections: dict[str, dict[str, str]], + named: dict[str, RowPath]) -> str: + """What `member-uninitialized` says before its update where `git + submodule update --init` would stop at a gitlink it has no url for, or + nothing where it would not (#762, rv648's S1, H1 and U1, and H2). + + GIT'S INIT DIES AT THE FIRST SUCH GITLINK, in index order, before it + checks out any member (`No url found for submodule path`, exit 128), so + the update the refusal named died the same way, at a stray case + variant, a member mounted by hand, a section with no url, or a gitlink + outside `members/`. Asked as the family bootstrap's `init_stop` asks it + (#648), more simply (`_init_stopper`): of the index, the `.gitmodules` + git's init reads and this holder's git config. Every gitlink, but where + the config sets `submodule.active`, as a recursive clone does, only + those a section gives a path; one whose section, the last that gives + its path, has no url, and the config none (`submodule..url`, as + an earlier init writes it), stops it. What bootstrap also keeps of + git's own parser (a name or a value git ignores, a `.gitmodules` a + merge left unmerged) is not read here; where nothing is found the plain + remedy stands. + + THE `.gitmodules` GIT READS is the work tree's (`sections`) where it is + on disk, and where it is gone from disk the index's, else HEAD's + (`_gitmodules_put_back`, rv820's SHOULD 2): read from the work tree + alone, a plain clone whose file was deleted and not staged was said to + stop at its first row, where git's update read the index's and checked + both members out. Where git reads the index's or HEAD's, the line that + puts the work tree's back comes first, so that the cure keeps every + other section, and `--no-members`, which reads the work tree's, names + what the cure says it names. + + THE CURE IS BOOTSTRAP'S FOR THE SAME GITLINK (`cures`), word for word + but for the path, printed as this file prints one (`_init_stop_cure`). + """ + spec, put_back, whose = _gitmodules_put_back(root) + with _git_runs("the gitlinks git submodule update asks could not be " + "listed"): + try: + if spec: + sections = _sections_listed(git_out( + ["config", "-z", "--blob", spec, "--list"], cwd=root, + binary=True)) + stopper = _init_stopper(root, sections) + except Refusal: + return "" + if stopper is None: + return "" + restore = (f"{GITMODULES} is gone from the work tree, so git reads " + f"{whose}: `{put_back}` puts it back first, so that what " + "follows keeps every other section. " if put_back else "") + return (f"`git submodule update --init` stops at {stopper}, a gitlink it " + "has no url for (No url found for submodule path, exit 128), " + f"before it checks out any member. {restore}" + f"{_init_stop_cure(stopper, sections, named)} Then ") + + +def _init_stopper(root: Path, + sections: dict[str, dict[str, str]]) -> str | None: + """The gitlink `git submodule update --init` stops at, the first in + index order, or None, as `_init_stop_said` asks it: of the index, + `sections` (the `.gitmodules` git's init reads) and this holder's git + config. A config git lists nothing of (`--get-regexp` exits 1) holds + none; an index git cannot list raises `Refusal`.""" + index = git_out(["ls-files", "-s", "-z"], cwd=root, binary=True) + try: + listed = git_out(["config", "-z", "--get-regexp", r"^submodule\."], + cwd=root, binary=True) + except Refusal: + listed = b"" + config = {record.decode("utf-8", "surrogateescape").partition("\n")[0] + for record in listed.split(b"\0")} + read = {keys["path"]: name for name, keys in sections.items() + if "path" in keys} + for record in index.split(b"\0"): + meta, _, entry = record.partition(b"\t") + if meta.split()[:1] != [GITLINK_MODE]: + continue + path = entry.decode("utf-8", "surrogateescape") + name = read.get(path) + if name is None and "submodule.active" in config: + continue + if name is None or ("url" not in sections[name] + and f"submodule.{name}.url" not in config): + return path + return None + + +def _init_stop_cure(stopper: str, sections: dict[str, dict[str, str]], + named: dict[str, RowPath]) -> str: + """Bootstrap's cure (`cures`) for `stopper`, the gitlink git's init + stops at, as `_init_stop_said` prints it: `--no-members` for a path two + of `sections` give or a gitlink in `members/` no row names, the section + written back for a row's own path (`named`), and `git rm --cached` for + a gitlink outside `members/`, which no mode here reads.""" + given = Counter(keys["path"] for keys in sections.values() + if "path" in keys) + if given[stopper] > 1: + return (f"`{PYTHON} scripts/validate-family.py --no-members` names " + f"each path more than one section of {GITMODULES} gives, and " + "how to take the others out.") + if stopper in named: + return (f"Write the section of {GITMODULES} for {stopper} back as " + "`family.py add` writes one, with its path and url, and " + f"commit {GITMODULES}.") + if stopper.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR: + return (f"`{PYTHON} scripts/validate-family.py --no-members` names " + f"each gitlink in {DEFAULT_MEMBERS_DIR}/ that no row names, " + "and the commit that cures it.") + removal = shlex.join(["git", "rm", "--cached", "--", stopper]) + return (f"No row names {stopper} and no clone can check it out: " + f"`{removal}` takes its gitlink out, then commit that.") + + def _check_members(root: Path, members: list, no_members: bool, report: Report) -> None: """Every row in `members:`, or the `--no-members` skip line. @@ -1717,8 +2235,9 @@ def _check_members(root: Path, members: list, no_members: bool, out comes first, so `--no-members` asks them too: the ones that read a file alone, `family.yaml`'s names (#331) and ids (#206) and the paths `.gitmodules` gives twice (#300), then the gitlinks and sections no row - names, which read the index and HEAD as well (#361), and the gitlinks - no section of HEAD's `.gitmodules` gives (#611). + names, which read the index and HEAD as well (#361), the gitlinks no + section of HEAD's `.gitmodules` gives (#611) or gives a url, and a + row's own whose removal is staged (#762). THEN EVERY ROW'S GITLINK, AND ONLY THEN ANY ROW'S CHECKOUT (#648). Each row used to be asked whole before the next was asked at all, so in a @@ -1728,7 +2247,9 @@ def _check_members(root: Path, members: list, no_members: bool, checks out any member, an earlier row was refused `member-uninitialized`, naming that same update, and the later row's `member-gitlink-absent`, which names the move that cures both, was - never reached. + never reached. Where git's init would stop at a gitlink it has no url + for, `member-uninitialized` says so before its update, with the cure + (`_init_stop_said`, #762). """ _check_member_names_unique(members, report) _check_member_ids_unique(root, members, report) @@ -1741,6 +2262,8 @@ def _check_members(root: Path, members: list, no_members: bool, _check_gitmodules_named(sections, gitlinks, named, removed, report) unregistered = _check_gitlinks_registered(root, members, gitlinks, report) + unregistered |= _check_removals_staged(root, members, gitlinks, removed, + sections, report) if no_members: print(f" -- {len(members)} member(s) SKIPPED (--no-members): " "their gitlinks, digests and project ids are not checked " @@ -1753,11 +2276,15 @@ def _check_members(root: Path, members: list, no_members: bool, report.finding("member-row", f"a member is not a mapping: {row!r}") continue - asked = _check_member(root, row, seen, report, gitlinks) + asked = _check_member(root, row, seen, report, gitlinks, removed) if asked is not None: gitlinked.append(asked) + stop = (_init_stop_said(root, sections, named) + if any(asked.path not in unregistered + and not (root / asked.path / ".git").exists() + for asked in gitlinked) else "") for asked in gitlinked: - _check_member_checkout(root, asked, report, unregistered) + _check_member_checkout(root, asked, report, unregistered, stop) if not members: report.note("no members yet: a family with none is empty, not " "wrong") diff --git a/tests/test_family_head_lost_gitmodules.py b/tests/test_family_head_lost_gitmodules.py index 2aff6080..50584dc9 100644 --- a/tests/test_family_head_lost_gitmodules.py +++ b/tests/test_family_head_lost_gitmodules.py @@ -659,9 +659,14 @@ def test_a_row_written_twice_is_one_finding_per_path(holder, offline, # --- 574/2: a gitlink recorded only at a case variant of the row's path ----- def variant_refused(path: str, commit: str, variants: list[str], - gitlinks: dict[str, str]) -> list[str]: + gitlinks: dict[str, str], *, sparse: tuple = (), + aside: str = "", section: tuple = (), + section_said: str = "") -> list[str]: """`member-gitlink-absent`'s lines for a row whose gitlink is recorded - only at `variants`.""" + only at `variants`. What the move meets beside them (#762): `sparse`, + the variants a sparse checkout keeps off disk; `aside`, the sentence + for an entry in the way; and `section`, the lines that make + `.gitmodules` give `path`, with `section_said`, what they do.""" project = path.split("/", 1)[1] found = ", ".join(f"{variant} (at {gitlinks[variant]})" for variant in variants) @@ -677,12 +682,19 @@ def variant_refused(path: str, commit: str, variants: list[str], "records its gitlink only at a case variant of that path, which " f"macOS and Windows hold as one directory with it: {found}. A " f"recursive clone checks out nothing at {path}.", - "Remediation: the member is recorded in the wrong case. Move its " - "gitlink to the row's path, at the commit the row pins, in the index " - "alone, which touches no file, with the two lines below, and commit " - f"that. Where nothing is checked out at {path}, as in a fresh clone, " - "`git submodule update --init --recursive` then checks the member " - f"out there. A directory left at {' or '.join(variants)} is the " + "Remediation: the member is recorded in the wrong case. " + + aside + ("Then move" if aside else "Move") + " its gitlink to the " + "row's path, at the commit the row pins, in the index alone, which " + "touches no file, with the " + + ("first two lines below" if section else "two lines below") + + (" (the first with --sparse, which needs git 2.34 or later: a " + f"sparse checkout keeps {' and '.join(sparse)} off disk, and " + "without it git rm passes over what that keeps off disk, exit 0, " + "and removes nothing)" if sparse else "") + + (f"; the lines after them {section_said}, and stage .gitmodules. " + "Commit that." if section else ", and commit that.") + + " A directory left at " + f"{' or '.join(variants)} is the " "row's own where the disk folds case, as macOS's and Windows' " "usually do and a Linux mount can (WSL's /mnt/c). Where the disk " "holds the two apart it stays there, untracked, and family.py add " @@ -700,9 +712,16 @@ def variant_refused(path: str, commit: str, variants: list[str], "nothing only it holds is needed. A checkout whose .git is a " "directory, a clone of its own, holds its commits there and nowhere " f"else, and {unpushed} lists the ones not pushed: move it out of the " - "holder whole, by hand, rather than delete it. The two lines:", - " git rm -r --cached --ignore-unmatch -- " + " ".join(variants), - f" git update-index --add --cacheinfo 160000,{commit},{path}"] + "holder whole, by hand, rather than delete it. Then, where nothing is " + f"checked out at {path}, as in a fresh clone, `git submodule update " + "--init --recursive` checks the member out there. Run it last: a " + "checkout git mv moved to a variant names the member's own store, " + f"which that update points at {path}, so {status} reads {path} after " + "it. The " + ("lines" if section else "two lines") + ":", + " git rm -r --cached" + (" --sparse" if sparse else "") + + " --ignore-unmatch -- " + " ".join(variants), + f" git update-index --add --cacheinfo 160000,{commit},{path}", + *(f" {line}" for line in section)] def refused_at(root: Path, commit: str, variants: list[str]) -> list[str]: diff --git a/tests/test_validate_family_remedies_typed.py b/tests/test_validate_family_remedies_typed.py index 87582356..559ea01b 100644 --- a/tests/test_validate_family_remedies_typed.py +++ b/tests/test_validate_family_remedies_typed.py @@ -63,9 +63,9 @@ from test_family_tracking_branch import on_main # noqa: F401 from test_validate_family_branch_step import BRANCH_LEFT from test_validate_family_unnamed_mounts import ( - GITLINK_REMEDY, GITLINK_SAID, VARIANT, codes, commands, findings, removal, - spare_commits, stage_gitlink, stage_unmerged_gitlink, take_out_as_printed, - with_section, + GITLINK_REMEDY, GITLINK_SAID, GITLINK_SAID_FETCHED, VARIANT, codes, + commands, findings, removal, spare_commits, stage_gitlink, + stage_unmerged_gitlink, take_out_as_printed, with_section, ) MEMBER = MEMBERS[0] @@ -142,7 +142,7 @@ def test_a_hand_mounts_staged_removal_is_committed_and_added_as_printed( assert again.returncode == 1, again.stdout + again.stderr staged, *section = findings(again) - assert staged == (GITLINK_SAID.format(path=HAND, commit=at) + assert staged == (GITLINK_SAID_FETCHED.format(path=HAND, commit=at) + STAGED_HAND_MOUNT + f"\n {switch}\n {add}") retyped = raw(holder, env, *shlex.split(rm)[1:], check=False) assert retyped.returncode == 128, retyped.stderr diff --git a/tests/test_validate_family_typed_states.py b/tests/test_validate_family_typed_states.py new file mode 100644 index 00000000..2cc4e722 --- /dev/null +++ b/tests/test_validate_family_typed_states.py @@ -0,0 +1,1083 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The holder gate's findings and refusals for a member recorded in the +wrong case, a gitlink git stops at, and a commit its checkout lacks, each +typed as printed in the state it names (#762). + +THE ISSUE'S LINES, each reproduced on #747's head (58d6c50f) before this +change: + +- (e) After `mkdir Members && git mv members/IRRS Members/IRRS`, committed, + the move back put the gitlink at `members/IRRS` and left the section's + `path` at `Members/IRRS`, as `git mv` rewrote it: a second round + (`member-gitmodules-absent`), then a third (`member-uninitialized`). The + move now gives the section the row's path back and stages it, and the + update that checks the member out comes last: before it, `git -C + Members/IRRS status` reads the checkout `git mv` moved there; after it, + which points the member's own store at `members/IRRS`, it does not. +- (f) A variant a sparse checkout keeps off disk: `git rm -r --cached + --ignore-unmatch` exited 0 and removed nothing, in a cone and a non-cone + checkout, and the index ended with two gitlinks. The line says + `--sparse` there, which needs git 2.34 or later. +- (g) A file at the row's path was replaced by the move's second line + without a word, and one under it stopped that line (`appears as both a + file and as a directory`, exit 128) after the first had taken the variant + out, which committed left the member un-registered. The entry goes aside + first with a committed `git mv`. +- A row's own `git rm --cached members/IRRS`, staged: `family ok` in every + mode, and `member-gitlink-absent` only after the commit. A plain `git rm` + was `member-uninitialized`, whose update dies `No url found`. It is + `member-gitlink-removal-staged` now, with the lines that put it back and + the `family.py remove` that takes it out. +- `member-commit-unresolvable` printed no remedy. It names the fetch of the + pinned commit, which a remote that holds it serves, on a branch or not, + and THE LOCKSTEP RULE's `bump` for a pin no remote holds, where the fetch + and `git submodule update` exit 128 `not our ref`. +- In a plain clone of a holder with a stray case variant (S1), a hand mount + with no section (H1), a section with no url (U1) or a gitlink outside + `members/` (H2), members checked and `--pins` refused + `member-uninitialized` with the update that dies at that gitlink. It + names the gitlink and bootstrap's own cure now, and U1 is + `member-gitmodules-url-absent` in every mode, where `--no-members` said + `family ok`. +- `member-gitlink-unnamed` said a recursive clone fetches a gitlink no + section gives a url; a recursive clone checks out nothing there. + +THE SAME FAMILY AND OFFLINE RULES as `test_family_head_lost_gitmodules.py` +(#611), whose helpers are imported. The folding disk is emulated as there, +`core.ignorecase=true` for every git the test types; a state that needs a +second directory at a case variant is skipped where the disk cannot hold +one (`case_sensitive_disk`). +""" + +from __future__ import annotations + +import itertools +import re +import shlex +from pathlib import Path + +import pytest + +from conftest import ( + IDENTITY_CONFIG, REPO, clone_just_pushed, git, rmtree, run_script, +) +from repo_shape import PYTHON, tree_digest +from test_family import ( # noqa: F401 + MEMBERS, NAME, ORG, SPARE, family, validate, +) +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import ( + DUPLICATE, OUTSIDE_CURE, SECTION_CURE, TWICE_CURE, UNNAMED_CURE, + UNNAMED_STOPS, cloned, fetching, type_line, without_url, +) +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( # noqa: F401 + DISKS, GUARDED_MOVE, assert_passes_in_every_mode, case_sensitive_disk, + commit_all, committed_gitmodules, github_to, on_disk, pinned, + recorded_at, typed_in_sh, variant_refused, without_section, +) +from test_family_member_path import raw +from test_family_tracking_branch import on_main # noqa: F401 +from test_validate_family_branch_step import ( + BRANCH_LEFT, RULE, rule_commands, switch_argv, +) +from test_validate_family_unnamed_mounts import ( + GITLINK_SAID, GITLINK_SAID_FETCHED, VARIANT, codes, commands, findings, + spare_commits, stage_gitlink, stage_unmerged_gitlink, +) + +MEMBER, OTHER = MEMBERS +ROW = f"members/{MEMBER}" +VARIANT_PATH = f"Members/{MEMBER}" + +#: What members mode's variant refusal says before the move it prints. +WRONG_CASE_LEAD = "Remediation: the member is recorded in the wrong case. " + + +def refused_lines(root: Path, *extra: str) -> list[str]: + """The gate in `root`, members checked unless `extra` says otherwise, + which must refuse: its stderr lines.""" + result = validate(root, *extra) + assert result.returncode == 2, result.stderr + result.stdout + return result.stderr.splitlines() + + +def unnamed_as_moved(variant: str, at: str, refused: list[str], *, + said: str = GITLINK_SAID) -> str: + """`member-gitlink-unnamed` WHOLE for MEMBER's gitlink at `variant`, + under `--no-members`: what every unnamed gitlink says, the row it is a + variant of, and members mode's refusal `refused` after its lead, word + for word, with its lines.""" + assert refused[1].startswith(WRONG_CASE_LEAD), refused[1] + return "\n".join([ + said.format(path=variant, commit=at) + + VARIANT.format(row_path=ROW, project=MEMBER) + + " The member is recorded in the wrong case. " + + refused[1][len(WRONG_CASE_LEAD):], *refused[2:]]) + + +def type_all(root: Path, env: dict, lines: list[str]) -> None: + """Every printed command line, typed as printed.""" + for line in lines: + type_line(root, env, line.strip()) + + +def refused_and_unnamed(root: Path, commit: str, + expected: list[str]) -> list[str]: + """The members-checked refusal in `root`, which must be `expected`, + and `--no-members`, whose one finding must be the same move as + `member-gitlink-unnamed` (`unnamed_as_moved`) for MEMBER's gitlink at + `commit`: the refusal's lines.""" + lines = refused_lines(root) + assert lines == expected + fallback = validate(root, "--no-members") + assert findings(fallback) == [unnamed_as_moved(VARIANT_PATH, commit, + expected)] + return lines + + +# --- (e): a `git mv` to a variant, moved back with its section --------------- + +#: What the move says its `.gitmodules` lines do, for a section `git mv` +#: rewrote and for none at all. +SECTION_GIVEN_BACK = (f"give {ROW} back to the section of HEAD's .gitmodules " + "that gives the variant's path, as git mv leaves it") +SECTION_WRITTEN = (f"write its section as family.py add writes one, since no " + f"section of HEAD's .gitmodules gives {ROW} or the " + "variant's path") + + +def git_moved(root: Path, env: dict) -> str: + """MEMBER's mount moved to `Members/IRRS` with `git mv`, its parent made + first, as git needs, and committed: the gitlink, the checkout and the + section's `path` all move. Returns the commit the gitlink names.""" + commit = pinned(root, MEMBER) + (root / "Members").mkdir() + raw(root, env, "mv", "--", ROW, VARIANT_PATH) + commit_all(root, env, f"git mv {ROW} {VARIANT_PATH}") + assert f"\tpath = {VARIANT_PATH}\n".encode() in raw( + root, env, "show", "HEAD:.gitmodules").stdout, "premise: git mv" + return commit + + +@DISKS +def test_a_git_mv_to_a_variant_is_moved_back_with_its_section( + family, holder, offline, tmp_path, case_sensitive_disk, ignorecase): + """THE ISSUE'S (e). The refusal, and the same move under `--no-members`, + print the section's path given back and staged, beside the two lines. + Typed as printed and committed, the guarded `mv` moves nothing (the + directory left is the member's checkout), the checkout is deleted by + hand as the text says, and the update then checks the member out at + its row's path: `git submodule update --init` exits 0, HEAD's section + gives `members/IRRS`, and the gate passes in every mode.""" + env = on_disk(fetching(isolated(offline, tmp_path)), ignorecase) + commit = git_moved(holder, env) + section = (f"git config -f .gitmodules submodule.{ROW}.path {ROW}", + "git add -- .gitmodules") + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}, section=section, + section_said=SECTION_GIVEN_BACK) + + lines = refused_lines(holder) + + assert lines == expected + fallback = validate(holder, "--no-members") + assert fallback.returncode == 1, fallback.stdout + fallback.stderr + assert findings(fallback)[0] == unnamed_as_moved( + VARIANT_PATH, commit, expected, said=GITLINK_SAID_FETCHED) + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + (move,) = GUARDED_MOVE.findall(lines[1]) + assert typed_in_sh(holder, move).returncode == 0 + assert (holder / VARIANT_PATH / ".git").is_file(), "premise: a checkout" + rmtree(holder / VARIANT_PATH) + updated = raw(holder, env, "submodule", "update", "--init", "--recursive", + check=False) + assert updated.returncode == 0, updated.stderr + assert (holder / ROW / "project.yaml").is_file() + assert f"\tpath = {ROW}\n".encode() in raw( + holder, env, "show", "HEAD:.gitmodules").stdout + assert_passes_in_every_mode(holder, 2) + + +def test_the_update_comes_last_where_git_mv_moved_the_checkout( + family, holder, offline, tmp_path, case_sensitive_disk): + """WHY THE UPDATE IS LAST. Work nobody committed in the checkout `git mv` + moved to `Members/IRRS`: after the lines and their commit, `git -C + Members/IRRS status`, which the text gives for what only that checkout + holds, lists it. An update run first points the member's store, which + that checkout names, at `members/IRRS`, and the same status then reads + `members/IRRS` and lists nothing: deleted then, the work was lost.""" + env = fetching(isolated(offline, tmp_path)) + git_moved(holder, env) + (holder / VARIANT_PATH / "WORK.md").write_bytes(b"not committed\n") + lines = refused_lines(holder) + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + (status,) = set(re.findall(r"`(git -C \S+ status)`", lines[1])) + + before = raw(holder, env, *shlex.split(status)[1:], "--porcelain").stdout + raw(holder, env, "submodule", "update", "--init", "--recursive") + after = raw(holder, env, *shlex.split(status)[1:], "--porcelain").stdout + + assert before == b"?? WORK.md\n" + assert after == b"" + + +def test_a_variant_no_section_gives_is_moved_with_its_section_written( + family, holder, offline, tmp_path): + """No section of HEAD's `.gitmodules` gives the row's path or the + variant's (#363's state, MEMBER's section taken out by hand): the move + writes the section as `family.py add` writes one, so its commit leaves + no `member-gitmodules-absent` behind, and the gate passes in every + mode.""" + env = fetching(isolated(offline, tmp_path)) + without_section(holder, env, MEMBER) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + section = (f"git config -f .gitmodules submodule.{ROW}.path {ROW}", + f"git config -f .gitmodules submodule.{ROW}.url " + f"https://github.com/{ORG}/{MEMBER}.git", + "git add -- .gitmodules") + + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}, section=section, + section_said=SECTION_WRITTEN) + + lines = refused_and_unnamed(holder, commit, expected) + + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + assert_passes_in_every_mode(holder, 2) + + +# --- (f): a variant a sparse checkout keeps off disk ------------------------- + +#: The two sparse checkouts that keep `Members/` off disk: a cone that holds +#: every directory the holder's gate reads, and non-cone patterns. +SPARSE = { + "cone": ("--cone", "members", "scripts", "contracts", ".github"), + "no-cone": ("--no-cone", "/*", "!/Members/"), +} + + +@pytest.mark.parametrize("rules", SPARSE.values(), ids=SPARSE) +def test_a_variant_a_sparse_checkout_keeps_off_disk_is_moved_with_sparse( + holder, offline, tmp_path, rules): + """THE ISSUE'S (f). `Members/IRRS` is skip-worktree, so the plain line + removes nothing, exit 0 (the premise, typed), and the next leaves two + gitlinks. The refusal, and `--no-members`, print `--sparse` and say + which git it needs; typed as printed and committed, the index records + the member once, at its row's path, and the gate passes in every + mode.""" + env = isolated(offline, tmp_path) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + raw(holder, env, "sparse-checkout", "set", *rules) + assert raw(holder, env, "ls-files", "-t", "--", VARIANT_PATH).stdout == ( + f"S {VARIANT_PATH}\n".encode()), "premise: kept off disk" + plain = raw(holder, env, "rm", "-r", "-q", "--cached", "--ignore-unmatch", + "--", VARIANT_PATH, check=False) + assert plain.returncode == 0, plain.stderr + assert raw(holder, env, "ls-files", "--", VARIANT_PATH).stdout, ( + "premise: the plain line removes nothing") + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}, + sparse=(VARIANT_PATH,)) + + lines = refused_and_unnamed(holder, commit, expected) + + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + listed = raw(holder, env, "ls-files", "-s").stdout.decode() + assert [line.split("\t")[1] for line in listed.splitlines() + if line.startswith("160000 ")] == [ROW, f"members/{OTHER}"] + assert_passes_in_every_mode(holder, 2) + + +def test_a_sparse_checkout_keeping_another_directory_off_disk_is_no_sparse( + holder, offline, tmp_path): + """rv820's SHOULD 3: a non-cone sparse checkout keeps `notes/`, which the + gate never reads, off disk, and the variant on it. The refusal, and + `--no-members`, print no `--sparse` and name nothing kept off disk; + typed as printed and committed, the index records the member once, at + its row's path, and the gate passes in every mode.""" + env = isolated(offline, tmp_path) + (holder / "notes").mkdir() + (holder / "notes" / "kept.md").write_bytes(b"kept off disk\n") + raw(holder, env, "add", "--", "notes/kept.md") + commit_all(holder, env, "A directory the gate never reads") + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + raw(holder, env, "sparse-checkout", "set", "--no-cone", "/*", "!/notes/") + assert raw(holder, env, "ls-files", "-t", "--", "notes/kept.md", + VARIANT_PATH).stdout == ( + f"H {VARIANT_PATH}\nS notes/kept.md\n".encode()), "premise" + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}) + + lines = refused_and_unnamed(holder, commit, expected) + + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + listed = raw(holder, env, "ls-files", "-s").stdout.decode() + assert [line.split("\t")[1] for line in listed.splitlines() + if line.startswith("160000 ")] == [ROW, f"members/{OTHER}"] + assert_passes_in_every_mode(holder, 2) + + +# --- (g): an entry the index holds at or under the row's path ---------------- + +#: A file at the row's path, one under it, and seven under it, of which the +#: text names five: `(the entries, what the text names under the path)`. +IN_THE_WAY = { + "at": ((ROW,), None), + "under": ((f"{ROW}/notes",), f"{ROW}/notes"), + "seven": (tuple(f"{ROW}/note{n}" for n in range(7)), + f"{ROW}/note0, {ROW}/note1, {ROW}/note2, {ROW}/note3, " + f"{ROW}/note4 and 2 more"), +} + + +@pytest.mark.parametrize("entries, named", IN_THE_WAY.values(), + ids=IN_THE_WAY) +def test_an_entry_in_the_way_is_moved_aside_first_as_printed( + family, holder, offline, tmp_path, entries, named): + """THE ISSUE'S (g). The checkout taken away and a file committed at + `members/IRRS`, or under it, beside the variant's gitlink. The refusal, + and `--no-members`, name the entry, or the first five of seven and how + many more (rv820's NIT 6), and the committed `git mv` that moves it + aside first. Typed as printed, `` filled in, then the two + lines and their commit, the update checks the member out, the files + are kept at their new name, and the gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + rmtree(holder / ROW) + for entry in entries: + (holder / entry).parent.mkdir(parents=True, exist_ok=True) + (holder / entry).write_bytes(b"a file\n") + raw(holder, env, "add", "--", entry) + commit_all(holder, env, "A file in the way") + aside = (f"The index holds an entry at {ROW} that is no gitlink, which " + "the move's second line replaces without a word" + if named is None else + f"The index holds {named} under {ROW}, at which the move's " + "second line stops (appears as both a file and as a directory) " + "after the first has taken the variant out") + ( + f". Move it aside first, `git mv -- {ROW} `, a new name " + "outside members/, and commit that move on its own. ") + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}, aside=aside) + + lines = refused_and_unnamed(holder, commit, expected) + + (moved,) = re.findall(r"`(git mv -- [^`]*)`", lines[1]) + type_line(holder, env, moved.replace("", f"{MEMBER}-aside")) + commit_all(holder, env, "Move the file aside, as printed") + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + raw(holder, env, "submodule", "update", "--init", "--recursive") + kept = raw(holder, env, "ls-tree", "-r", "--name-only", "HEAD", "--", + f"{MEMBER}-aside").stdout.decode().splitlines() + assert kept == [entry.replace(ROW, f"{MEMBER}-aside") + for entry in entries] + assert_passes_in_every_mode(holder, 2) + + +def test_an_entry_beside_the_rows_path_is_not_in_the_way( + holder, offline, tmp_path): + """rv820's SHOULD 3: a file committed at `members/IRRS-notes`, beside the + variant's gitlink, is not at or under `members/IRRS` as git compares + paths, so the refusal, and `--no-members`, move nothing aside. Typed as + printed and committed, the file stays where it is, and the gate passes + in every mode.""" + env = isolated(offline, tmp_path) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + beside = f"{ROW}-notes" + (holder / beside).write_bytes(b"a file\n") + raw(holder, env, "add", "--", beside) + commit_all(holder, env, "A file beside the row's path") + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}) + + lines = refused_and_unnamed(holder, commit, expected) + + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + assert raw(holder, env, "ls-files", "-s", "--", ROW).stdout == ( + f"160000 {commit} 0\t{ROW}\n".encode()) + assert raw(holder, env, "ls-files", "--", beside).stdout == ( + f"{beside}\n".encode()) + assert_passes_in_every_mode(holder, 2) + + +def test_a_gitlink_a_merge_left_unmerged_at_the_rows_path_is_replaced( + holder, offline, tmp_path): + """rv820's NIT 5: the row's path held at stages 1, 2 and 3 alone, as a + conflicted merge of the member's gitlink leaves it, beside the + variant's gitlink. The move's second line replaces the three stages, + so nothing is said to be in the way; typed as printed and committed, + the index records the member once, at stage 0, and the gate passes in + every mode.""" + env = isolated(offline, tmp_path) + commit = recorded_at(holder, env, MEMBER, VARIANT_PATH) + stage_unmerged_gitlink(holder, ROW, commit) + expected = variant_refused(ROW, commit, [VARIANT_PATH], + {VARIANT_PATH: commit}) + + lines = refused_lines(holder) + + assert lines == expected + type_all(holder, env, lines[2:]) + commit_all(holder, env, "Move the gitlink back, as printed") + assert raw(holder, env, "ls-files", "-s", "--", ROW).stdout == ( + f"160000 {commit} 0\t{ROW}\n".encode()) + assert_passes_in_every_mode(holder, 2) + + +# --- a row's own staged un-registration ------------------------------------ + +#: How the row's gitlink was taken out: with its section and checkout, or +#: from the index alone. +TAKEN = {"git-rm": ("rm", "-q", "--"), "git-rm-cached": ("rm", "-q", + "--cached", "--")} + +#: What the lines that put it back each put back, by how it was taken out. +PUT_BACK = { + "git-rm": ( + "the lines below put the member back as HEAD has it: the first its " + "gitlink, the second the .gitmodules section git rm took out, " + "putting that file back as HEAD has it and dropping any other change " + "to it, the third its checkout", + [f"git reset -q -- {ROW}", "git checkout HEAD -- .gitmodules", + f"git submodule update --init --recursive -- {ROW}"]), + "git-rm-cached": ( + "the line below puts its gitlink back as HEAD records it", + [f"git reset -q -- {ROW}"]), +} + + +def removal_staged(root: Path, commit: str, how: str) -> str: + """`member-gitlink-removal-staged` WHOLE for MEMBER's gitlink at + `commit`, taken out `how`, in `root`.""" + said, lines = PUT_BACK[how] + remove = (f"{PYTHON} scripts/family.py remove --family-root " + f"{shlex.quote(str(root))} --member {MEMBER}") + switch = shlex.join(switch_argv(root, "remove", "irrs")) + return "\n".join([ + f"FINDING member-gitlink-removal-staged: {ROW} is a gitlink at " + f"{commit} that a row of members: names and HEAD records, and the " + "index holds nothing at that path: its removal is staged and not " + "committed, so the next commit un-registers the member under its " + "row, and this gate then refuses member-gitlink-absent. If the " + f"member is to stay, {said}, and nothing is left to commit. If it is " + "to go, put it back the same way, then take it out with family.py " + "remove, from a checkout of openRepoShape, which takes its row, its " + "gitlink and its section out in one commit, on a branch other than " + "the holder's tracking branch, which it refuses " + f"(`{switch}` makes one): `{remove}`. {BRANCH_LEFT}", + *(f" {line}" for line in lines)]) + + +@pytest.mark.parametrize("how", TAKEN) +def test_a_rows_staged_removal_is_named_and_put_back_as_printed( + on_main, offline, tmp_path, how): + """THE ISSUE'S STAGED UN-REGISTRATION, with `git rm --cached` and with + a plain `git rm`, staged: `member-gitlink-removal-staged`, exit 1, in + every mode, where the gate said `family ok` (or, after a plain `git + rm`, refused `member-uninitialized` with an update that dies `No url + found`). Typed as printed, nothing is left to commit, and the gate + passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + commit = pinned(on_main, MEMBER) + raw(on_main, env, *TAKEN[how], ROW) + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(on_main, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert findings(result) == [removal_staged(on_main, commit, how)] + assert f" ok {ROW}: gitlink == pin" not in result.stdout, extra + + type_all(on_main, env, commands(findings(validate(on_main))[0])) + assert raw(on_main, env, "status", "--porcelain").stdout == b"" + assert_passes_in_every_mode(on_main, 2) + + +@pytest.mark.parametrize("how", TAKEN) +def test_a_rows_staged_removal_to_go_is_removed_as_printed( + on_main, offline, tmp_path, how): + """The other way out: put back as printed, then the `switch` and the + `family.py remove` it names, typed as printed from a checkout of + openRepoShape, take the member out in one commit, and the gate passes + in every mode with one member.""" + env = fetching(isolated(offline, tmp_path)) + raw(on_main, env, *TAKEN[how], ROW) + (finding,) = findings(validate(on_main)) + type_all(on_main, env, commands(finding)) + switch, remove = [command for command in re.findall(r"`([^`]*)`", finding) + if command.startswith(("git -C", PYTHON))] + git(*shlex.split(switch)[1:], cwd=on_main.parent) + argv = shlex.split(remove) + removed = run_script(REPO / argv[1], *argv[2:], cwd=REPO, env=env) + assert removed.returncode == 0, removed.stdout + removed.stderr + assert_passes_in_every_mode(on_main, 1) + + +def test_a_staged_removal_once_committed_is_member_gitlink_absent( + holder, offline, tmp_path): + """No misfire after the commit: the removal committed is no longer + staged, and the gate refuses `member-gitlink-absent`, as it did.""" + env = isolated(offline, tmp_path) + raw(holder, env, "rm", "-q", "--cached", "--", ROW) + commit_all(holder, env, "The removal committed") + + lines = refused_lines(holder) + + assert lines[0].startswith("REFUSED member-gitlink-absent: "), lines + assert "member-gitlink-removal-staged" not in "\n".join(lines) + + +def test_a_staged_removal_head_gives_no_section_for_puts_none_back( + on_main, offline, tmp_path): + """rv820's NIT 6: no section of HEAD's `.gitmodules` gives the row's + path (taken out by hand and committed), and the gitlink's removal is + staged. The finding prints the reset alone, and no `git checkout HEAD + -- .gitmodules`, which would put back no section.""" + env = isolated(offline, tmp_path) + without_section(on_main, env, MEMBER) + commit = pinned(on_main, MEMBER) + raw(on_main, env, *TAKEN["git-rm-cached"], ROW) + + staged = [found for found in findings(validate(on_main, "--no-members")) + if found.startswith("FINDING member-gitlink-removal-staged: ")] + + assert staged == [removal_staged(on_main, commit, "git-rm-cached")] + + +# --- member-commit-unresolvable --------------------------------------------- + +#: The dates every `commit_ahead` commit carries, so that the clock never +#: tells two of them apart: their bytes do. +AHEAD_DATES = {"GIT_AUTHOR_DATE": "2031-01-01T00:00:00+0000", + "GIT_COMMITTER_DATE": "2031-01-01T00:00:00+0000"} + +#: The number each `commit_ahead` commit writes into `AHEAD.md`. Not +#: `tmp_path.name`, for the reason `test_family_bootstrap_gate_order.py` +#: gives for its `BRANCHES`: a passed test's directory is removed under +#: `tmp_path_retention_policy=failed`, and the next test is given its +#: number. +AHEAD = itertools.count() + + +def commit_ahead(family, tmp_path: Path, ref: str | None) -> tuple[str, str]: + """A new commit of MEMBER, made in a scratch clone of its remote and + pushed to `ref` there, or nowhere where `ref` is None. Returns the + commit and its tree digest. + + ONE COMMIT PER TEST. The tests below share one remote (`family` is + module-scoped), and a commit with the same parent, tree, message, + identity and second IS the one an earlier test pushed: on CI (run + 39031276097) the no-remote test's commit, made in the same second as + the on-no-branch test's, was the commit that test had pushed, so its + fetch fetched it. `AHEAD.md` holds a number no other call writes + (`AHEAD`) and the dates are fixed (`AHEAD_DATES`), so in every run the + bytes keep two calls apart, never the clock.""" + scratch = tmp_path / "scratch" + git("clone", "-q", str(family["remotes"] / f"{MEMBER}.git"), str(scratch), + cwd=tmp_path) + (scratch / "AHEAD.md").write_bytes(f"ahead {next(AHEAD)}\n".encode()) + git("add", "--", "AHEAD.md", cwd=scratch) + raw(scratch, AHEAD_DATES, *IDENTITY_CONFIG, "commit", "-qm", "Ahead") + commit = git("rev-parse", "HEAD", cwd=scratch).stdout.strip() + if ref is not None: + git("push", "-q", "origin", f"HEAD:{ref}", cwd=scratch) + return commit, tree_digest(scratch, commit) + + +def pinned_by_hand(root: Path, env: dict, commit: str, digest: str) -> None: + """MEMBER's gitlink and pin moved to `commit`, its digest `digest`, + together and committed, with nothing fetched into its store.""" + old = pinned(root, MEMBER) + rows = (root / "family.yaml").read_bytes() + assert rows.count(old.encode()) == 1, "fixture drift: MEMBER's pin" + rows = rows.replace(old.encode(), commit.encode()) + found = re.compile(rb'tree_sha256: "?([0-9a-f]{64})"?').search( + rows, rows.index(commit.encode())) + rows = rows[:found.start(1)] + digest.encode() + rows[found.end(1):] + (root / "family.yaml").write_bytes(rows) + raw(root, env, "update-index", "--cacheinfo", f"160000,{commit},{ROW}") + raw(root, env, "add", "--", "family.yaml") + commit_all(root, env, "Pin moved, the store not fetched") + + +def unresolvable(root: Path, commit: str) -> str: + """`member-commit-unresolvable`'s remediation in `root`, its text up to + THE LOCKSTEP RULE pinned, which it returns whole.""" + lines = refused_lines(root) + assert lines[0].startswith( + f"REFUSED member-commit-unresolvable: {ROW}: the pinned commit " + f"{commit} is not in that member's object store ("), lines + assert lines[1].startswith( + "Remediation: where a remote of the member holds it, `git -C " + f"{ROW} fetch origin {commit}` fetches it into that store, and this " + "then reads it. Where none does, that fetch fails `not our ref`, exit " + "128, as `git submodule update` does, and the pin names a commit no " + "clone can fetch: move the gitlink and the pin to one a remote " + f"holds. {RULE}"), lines[1] + return "\n".join(lines[1:]) + + +@pytest.mark.parametrize("ref", ["refs/heads/w762-ahead", "refs/w762/ahead"], + ids=["on-a-branch", "on-no-branch"]) +def test_a_pin_a_remote_holds_is_fetched_as_printed(family, holder, offline, + tmp_path, ref): + """rv681's P2, the pin on the remote and not in the member's store: + refused, members checked and `--pins`, now with a remedy. Its fetch, + typed as printed, brings the commit whether a branch holds it or not, + and the gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + commit, digest = commit_ahead(family, tmp_path, ref) + branches = git("branch", "--format=%(refname)", "--contains", commit, + cwd=family["remotes"] / f"{MEMBER}.git").stdout.split() + assert branches == ([ref] if ref.startswith("refs/heads/") else []), ( + "premise: on that branch alone, or on none") + pinned_by_hand(holder, env, commit, digest) + remedy = unresolvable(holder, commit) + assert refused_lines(holder, "--pins")[1:] == remedy.splitlines() + (fetch,) = [command for command in re.findall(r"`([^`]*)`", remedy) + if command.startswith("git -C ") and " fetch " in command] + + type_line(holder, env, fetch) + + assert_passes_in_every_mode(holder, 2) + + +def test_a_pin_no_remote_holds_is_bumped_as_printed(family, on_main, offline, + tmp_path): + """rv681's P2, a pin no remote holds: the fetch the remedy names exits + 128 `not our ref`, as it says, and so does `git submodule update`. Its + rule's `switch` and `bump`, typed as printed from a checkout of + openRepoShape with a commit the remote holds as `--to`, put the gitlink + and the pin back in step, and the gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + commit, digest = commit_ahead(family, tmp_path, None) + on_remote = git("cat-file", "-e", commit, + cwd=family["remotes"] / f"{MEMBER}.git", check=False) + assert on_remote.returncode == 1, "premise: no remote holds it" + pinned_by_hand(on_main, env, commit, digest) + remedy = unresolvable(on_main, commit) + (fetch,) = [command for command in re.findall(r"`([^`]*)`", remedy) + if command.startswith("git -C ") and " fetch " in command] + fetched = raw(on_main, env, *shlex.split(fetch)[1:], check=False) + assert fetched.returncode == 128, fetched.stderr + assert b"not our ref" in fetched.stderr + update = raw(on_main, env, "submodule", "update", "--init", "--", ROW, + check=False) + assert update.returncode != 0, update.stderr + assert b"not our ref" in update.stderr + bump, switch = rule_commands(remedy) + held = git("rev-parse", "main", + cwd=family["remotes"] / f"{MEMBER}.git").stdout.strip() + + git(*shlex.split(switch)[1:], cwd=on_main.parent) + argv = shlex.split(bump.replace("", held)) + bumped = run_script(REPO / argv[1], *argv[2:], cwd=REPO, env=env) + + assert bumped.returncode == 0, bumped.stdout + bumped.stderr + assert_passes_in_every_mode(on_main, 2) + + +# --- S1, H1, U1 and H2: a plain clone whose update stops at another gitlink -- + +def url_less(holder: Path, env: dict, family) -> str: + """rv648's U1: OTHER's section with its url taken out, committed. + Returns OTHER's path, where git's init stops.""" + data = raw(holder, env, "show", "HEAD:.gitmodules").stdout + committed_gitmodules(holder, env, without_url(data, OTHER)) + return f"members/{OTHER}" + + +def outside(holder: Path, env: dict, family) -> str: + """rv648's H2: SPARE's gitlink recorded by hand at `vendor/`, + with no row and no section, committed. Returns its path.""" + stage_gitlink(holder, f"vendor/{SPARE}", spare_commits(family)[1]) + commit_all(holder, env, "A gitlink mounted by hand outside members/") + return f"vendor/{SPARE}" + + +def twice(holder: Path, env: dict, family) -> str: + """rv648's D1: a second section, `dup`, giving OTHER's path and no url, + after OTHER's own, committed: git reads the later. Returns OTHER's + path, where git's init stops.""" + data = raw(holder, env, "show", "HEAD:.gitmodules").stdout + committed_gitmodules(holder, env, data + DUPLICATE) + return f"members/{OTHER}" + + +#: `(set-up, the cure members mode prints for where git stops)`. +STOPS = { + "stray": (UNNAMED_STOPS["stray"], lambda path: UNNAMED_CURE), + "hand-mount": (UNNAMED_STOPS["hand-mount"], lambda path: UNNAMED_CURE), + "url-less": (url_less, lambda path: SECTION_CURE.format(path=path)), + "twice": (twice, lambda path: TWICE_CURE), + "outside": (outside, lambda path: OUTSIDE_CURE.format(path=path)), +} + +#: What `member-uninitialized` says, before and after the update it names. +UNINITIALIZED = ( + f"REFUSED member-uninitialized: {ROW} is not an initialized submodule " + "checkout, so neither the pinned digest nor the project it claims can be " + "read. Presence is not identity and an unreadable surface must fail " + "rather than degrade.") +UPDATE = ("`git submodule update --init --recursive`, or `make bootstrap`, " + "which resolves a credential for private members first.") + + +def cured_as_printed(clone: Path, env: dict, stopper: str, + remedy: str) -> None: + """What the refusal says cures `stopper`, done as it says: the + `--no-members` run it names and every `git` line that run prints for + the gitlink, its `.gitmodules` staged where a line writes it, or the + `git rm --cached` it names itself; then the commit.""" + if "--no-members" in remedy: + fallback = validate(clone, "--no-members") + assert fallback.returncode == 1, fallback.stdout + fallback.stderr + (finding,) = [found for found in findings(fallback) + if f" {stopper} " in found.split("\n", 1)[0]] + type_all(clone, env, [line for line in commands(finding) + if line.startswith("git ")]) + if finding.startswith("FINDING gitmodules-path-twice: "): + # Its words, `--unset-all submodule..` for every key + # of the section that is not the member's, filled in for `dup`. + assert "`dup` (no url)" in finding, finding + raw(clone, env, "config", "-f", ".gitmodules", "--unset-all", + "submodule.dup.path") + raw(clone, env, "add", "--", ".gitmodules") + elif "back as `family.py add` writes one" in remedy: + fallback = validate(clone, "--no-members") + (finding,) = findings(fallback) + assert finding.startswith( + "FINDING member-gitmodules-url-absent: "), finding + type_all(clone, env, commands(finding)) + raw(clone, env, "add", "--", ".gitmodules") + else: + (line,) = re.findall(r"`(git rm [^`]*)`", remedy) + type_line(clone, env, line) + commit_all(clone, env, "Cure where git stops, as printed") + + +@pytest.mark.parametrize("set_up, cure", STOPS.values(), ids=STOPS) +def test_a_plain_clone_stopped_at_another_gitlink_names_it_members_checked( + family, holder, offline, tmp_path, set_up, cure): + """rv648's S1, H1, U1 and H2 in a plain clone, after the update died + there (exit 128, `No url found for submodule path`). Members checked + and `--pins`, the first row's `member-uninitialized` named that same + update; it names where git stops and the cure bootstrap names for it, + word for word, before the update. Done as it says and committed, the + update checks both members out, and the gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + stopper = set_up(holder, env, family) + clone = cloned(holder, env, family, tmp_path, recurse=False) + died = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert died.returncode == 128, died.stderr + assert (f"No url found for submodule path '{stopper}'".encode() + in died.stderr) + + for extra in ((), ("--pins",)): + assert refused_lines(clone, *extra) == [ + UNINITIALIZED, + f"Remediation: `git submodule update --init` stops at {stopper}, " + "a gitlink it has no url for (No url found for submodule path, " + f"exit 128), before it checks out any member. {cure(stopper)} " + f"Then {UPDATE}"], extra + + cured_as_printed(clone, env, stopper, cure(stopper)) + raw(clone, env, *github_to(family["remotes"]), "submodule", "update", + "--init", "--recursive") + assert_passes_in_every_mode(clone, 2) + + +def test_a_holder_whose_update_stops_nowhere_keeps_the_plain_remedy( + family, holder, offline, tmp_path): + """No misfire: a plain clone of a good holder, nothing checked out yet, + is refused `member-uninitialized` with the update alone, which then + checks both members out.""" + env = fetching(isolated(offline, tmp_path)) + clone = cloned(holder, env, family, tmp_path, recurse=False) + + assert refused_lines(clone) == [UNINITIALIZED, f"Remediation: {UPDATE}"] + raw(clone, env, "submodule", "update", "--init", "--recursive") + assert_passes_in_every_mode(clone, 2) + + +def test_a_recursive_clone_not_fetched_keeps_the_plain_remedy( + family, holder, offline, tmp_path): + """No misfire: a recursive clone sets `submodule.active`, and git's init + then asks only the gitlinks a section gives a path, so a stray stops + nothing there. With the members not fetched (no credential, as + `protocol.file.allow=user` refuses them here), the refusal names the + update alone, which, with the credential, checks both out.""" + env = fetching(isolated(offline, tmp_path)) + UNNAMED_STOPS["stray"](holder, env, family) + remote = family["remotes"] / f"{NAME}.git" + raw(holder, env, "push", "-q", str(remote), "HEAD:refs/heads/w762/active") + clone = tmp_path / "clone" + made = clone_just_pushed(remote, clone, "w762/active", + config=["-c", "protocol.file.allow=user"]) + assert made.returncode != 0, "premise: the members were not fetched" + + assert refused_lines(clone) == [UNINITIALIZED, f"Remediation: {UPDATE}"] + raw(clone, env, "submodule", "update", "--init", "--recursive") + assert (clone / ROW / "project.yaml").is_file() + + +def test_a_url_an_earlier_init_registered_keeps_the_plain_remedy( + family, holder, offline, tmp_path): + """No misfire: `git submodule init` wrote each member's url into the + clone's git config, then the work tree's section for OTHER lost its + url. Git's init reads the url it wrote and stops nowhere, so the + refusal names the update alone, which checks both members out.""" + env = fetching(isolated(offline, tmp_path)) + clone = cloned(holder, env, family, tmp_path, recurse=False) + raw(clone, env, "submodule", "init") + (clone / ".gitmodules").write_bytes( + without_url((clone / ".gitmodules").read_bytes(), OTHER)) + + assert refused_lines(clone) == [UNINITIALIZED, f"Remediation: {UPDATE}"] + raw(clone, env, "submodule", "update", "--init", "--recursive") + assert (clone / f"members/{OTHER}" / "project.yaml").is_file() + + +#: How `.gitmodules` is gone from a work tree, and what git's submodule +#: commands then read in its place, with the line that puts it back: +#: deleted and not staged, and its deletion staged (rv820's SHOULD 2). +GONE = { + "deleted": ("the index's", "git checkout -- .gitmodules"), + "deletion-staged": ("HEAD's, as the index holds none", + "git checkout HEAD -- .gitmodules"), +} + + +def gone(root: Path, env: dict, how: str) -> None: + """`.gitmodules` taken out of `root`'s work tree, as `how` (`GONE`) + says.""" + if how == "deleted": + (root / ".gitmodules").unlink() + else: + raw(root, env, "rm", "-q", "--", ".gitmodules") + + +@pytest.mark.parametrize("how", GONE) +def test_a_plain_clone_whose_gitmodules_is_gone_keeps_the_plain_remedy( + family, holder, offline, tmp_path, how): + """rv820's SHOULD 2: a plain clone of a good holder whose `.gitmodules` + is gone from the work tree. Git reads the index's, or HEAD's, so its + update stops nowhere: the refusal names the update alone, where it + said git stopped at the first row and printed a cure that, written + with `git config -f`, made a file of that one section. The update then + checks both members out.""" + env = fetching(isolated(offline, tmp_path)) + clone = cloned(holder, env, family, tmp_path, recurse=False) + gone(clone, env, how) + + for extra in ((), ("--pins",)): + assert refused_lines(clone, *extra) == [ + UNINITIALIZED, f"Remediation: {UPDATE}"], extra + raw(clone, env, "submodule", "update", "--init", "--recursive") + assert (clone / ROW / "project.yaml").is_file() + assert (clone / f"members/{OTHER}" / "project.yaml").is_file() + + +@pytest.mark.parametrize("how", GONE) +def test_a_url_less_section_whose_gitmodules_is_gone_is_put_back_first( + family, holder, offline, tmp_path, how): + """rv820's SHOULD 2, where git does stop: rv648's U1 in a plain clone + whose `.gitmodules` is gone from the work tree. Git reads the index's, + or HEAD's, and stops at OTHER's section, which gives no url; the + refusal names that path, not the first row's, and the `git checkout` + that puts the file back before the cure. Done as it says and + committed, HEAD's `.gitmodules` keeps MEMBER's section, the update + checks both members out, and the gate passes in every mode.""" + env = fetching(isolated(offline, tmp_path)) + stopper = url_less(holder, env, family) + clone = cloned(holder, env, family, tmp_path, recurse=False) + gone(clone, env, how) + died = raw(clone, env, "submodule", "update", "--init", "--recursive", + check=False) + assert died.returncode == 128, died.stderr + assert (f"No url found for submodule path '{stopper}'".encode() + in died.stderr) + whose, put_back = GONE[how] + + for extra in ((), ("--pins",)): + assert refused_lines(clone, *extra) == [ + UNINITIALIZED, + f"Remediation: `git submodule update --init` stops at {stopper}, " + "a gitlink it has no url for (No url found for submodule path, " + "exit 128), before it checks out any member. .gitmodules is gone " + f"from the work tree, so git reads {whose}: `{put_back}` puts it " + "back first, so that what follows keeps every other section. " + f"{SECTION_CURE.format(path=stopper)} Then {UPDATE}"], extra + + type_line(clone, env, put_back) + cured_as_printed(clone, env, stopper, SECTION_CURE.format(path=stopper)) + kept = raw(clone, env, "show", "HEAD:.gitmodules").stdout + assert f'[submodule "{ROW}"]\n\tpath = {ROW}\n\turl = '.encode() in kept, ( + kept) + raw(clone, env, *github_to(family["remotes"]), "submodule", "update", + "--init", "--recursive") + assert_passes_in_every_mode(clone, 2) + + +#: U1's own finding, whole, for OTHER's section in this fixture. +URL_ABSENT = ( + f"FINDING member-gitmodules-url-absent: members/{OTHER} is a gitlink at " + "{commit} that a row of members: names, and the section of HEAD's " + f".gitmodules that gives that path, \"members/{OTHER}\", gives no url, " + "so in a plain clone of this holder git submodule update --init stops " + "there (No url found for submodule path, exit 128) before it checks out " + "any member, as make bootstrap's does, and a recursive clone checks out " + "nothing there. Write its url back by hand, in a POSIX shell, as " + "family.py add writes one, then stage and commit .gitmodules:\n" + f" git config -f .gitmodules submodule.members/{OTHER}.url " + f"https://github.com/{ORG}/{OTHER}.git") + + +@pytest.mark.parametrize("extra", [(), ("--pins",), ("--no-members",)], + ids=["members-checked", "pins", "no-members"]) +def test_a_section_with_no_url_is_a_finding_in_every_mode( + family, holder, offline, tmp_path, extra): + """U1 in the holder it was committed in, whose checkouts stay: no mode + named it (`--no-members` said `family ok`). It is its own finding in + every mode, exit 1; typed as printed, staged and committed, the gate + passes in every mode.""" + env = isolated(offline, tmp_path) + url_less(holder, env, family) + + result = validate(holder, *extra) + + assert result.returncode == 1, result.stdout + result.stderr + assert findings(result) == [URL_ABSENT.format( + commit=pinned(holder, OTHER))] + type_all(holder, env, commands(findings(result)[0])) + raw(holder, env, "add", "--", ".gitmodules") + commit_all(holder, env, "OTHER's url written back, as printed") + assert_passes_in_every_mode(holder, 2) + + +@pytest.mark.parametrize("how", GONE) +def test_a_section_with_no_url_whose_gitmodules_is_gone_is_put_back_first( + family, holder, offline, tmp_path, how): + """U1 in its holder, with `.gitmodules` gone from the work tree: the + finding's line, `git config -f`, would make a file of that one key, so + a line before it puts the file back from where git reads it (rv820's + SHOULD 2). In every mode; typed as printed, staged and committed, + HEAD's file keeps MEMBER's section, and the gate passes in every + mode.""" + env = isolated(offline, tmp_path) + url_less(holder, env, family) + gone(holder, env, how) + whose, put_back = GONE[how] + said = "then stage and commit .gitmodules:\n" + expected = URL_ABSENT.format(commit=pinned(holder, OTHER)) + assert expected.count(said) == 1 + expected = expected.replace( + said, "then stage and commit .gitmodules, which is gone from the " + f"work tree: the first line puts back {whose}, so that every other " + f"section stays:\n {put_back}\n") + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(holder, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert findings(result) == [expected], extra + + type_all(holder, env, commands(expected)) + raw(holder, env, "add", "--", ".gitmodules") + commit_all(holder, env, "OTHER's url written back, as printed") + assert f'[submodule "{ROW}"]'.encode() in raw( + holder, env, "show", "HEAD:.gitmodules").stdout + assert_passes_in_every_mode(holder, 2) + + +def test_a_recursive_clone_of_a_url_less_section_checks_out_nothing_there( + family, holder, offline, tmp_path): + """What U1's finding says of a recursive clone: nothing is checked out + at OTHER's path, and the clone's gate names the finding where it was.""" + env = fetching(isolated(offline, tmp_path)) + url_less(holder, env, family) + remote = family["remotes"] / f"{NAME}.git" + raw(holder, env, "push", "-q", str(remote), "HEAD:refs/heads/w762/u1") + clone = tmp_path / "clone" + clone_just_pushed(remote, clone, "w762/u1") + + assert not (clone / f"members/{OTHER}" / "project.yaml").exists() + fallback = validate(clone, "--no-members") + assert codes(fallback) == ["member-gitmodules-url-absent"] + + +# --- `member-gitlink-unnamed`: what a recursive clone does ------------------- + +@pytest.mark.parametrize("set_up", UNNAMED_STOPS.values(), ids=UNNAMED_STOPS) +def test_a_recursive_clone_of_a_gitlink_no_section_gives_checks_out_nothing( + family, holder, offline, tmp_path, set_up): + """w648b's S2 and X1: a recursive clone of a holder with a stray case + variant, or a hand mount with no section, checks out nothing at it (an + empty directory), and the clone's `--no-members` says so, where it said + a recursive clone fetches it. A plain clone's update stopping there is + the test above.""" + env = fetching(isolated(offline, tmp_path)) + stopper = set_up(holder, env, family) + commit = raw(holder, env, "rev-parse", + f"HEAD:{stopper}").stdout.decode("ascii").strip() + clone = cloned(holder, env, family, tmp_path, recurse=True) + + assert list((clone / stopper).iterdir()) == [] + fallback = validate(clone, "--no-members") + assert fallback.returncode == 1, fallback.stdout + fallback.stderr + (finding,) = findings(fallback) + assert finding.startswith(GITLINK_SAID.format(path=stopper, + commit=commit)), finding + + +def test_a_recursive_clone_of_a_hand_mount_with_its_section_fetches_it( + family, holder, offline, tmp_path): + """The other branch of the same sentence: a hand mount `git submodule + add` made, with its section and url, is fetched by a recursive clone, + and the clone's `--no-members` says it fetches it.""" + env = fetching(isolated(offline, tmp_path)) + hand = f"members/{SPARE}" + raw(holder, env, "submodule", "add", "-q", "--", + str(family["remotes"] / f"{SPARE}.git"), hand) + commit_all(holder, env, "Mount IRQS by hand") + commit = raw(holder, env, "rev-parse", + f"HEAD:{hand}").stdout.decode("ascii").strip() + clone = cloned(holder, env, family, tmp_path, recurse=True) + + assert (clone / hand / "project.yaml").is_file() + (finding,) = [found for found in findings(validate(clone, "--no-members")) + if found.startswith("FINDING member-gitlink-unnamed: ")] + assert finding.startswith(GITLINK_SAID_FETCHED.format(path=hand, + commit=commit)) + + +def test_a_hand_mount_whose_section_gives_no_url_is_checked_out_nowhere( + family, holder, offline, tmp_path): + """rv820's SHOULD 3: a hand mount `git submodule add` made, whose + section's url was then taken out and committed. A section gives its + path and none a url, so `--no-members` says a recursive clone checks + out nothing at it, as for one no section gives, and not that a + recursive clone fetches it.""" + env = fetching(isolated(offline, tmp_path)) + hand = f"members/{SPARE}" + raw(holder, env, "submodule", "add", "-q", "--", + str(family["remotes"] / f"{SPARE}.git"), hand) + raw(holder, env, "config", "-f", ".gitmodules", "--unset", + f"submodule.{hand}.url") + raw(holder, env, "add", "--", ".gitmodules") + commit_all(holder, env, "Mount IRQS by hand, its url taken out") + commit = raw(holder, env, "rev-parse", + f"HEAD:{hand}").stdout.decode("ascii").strip() + + (finding,) = [found for found in findings(validate(holder, "--no-members")) + if found.startswith("FINDING member-gitlink-unnamed: ")] + + assert finding.startswith(GITLINK_SAID.format(path=hand, + commit=commit)), finding diff --git a/tests/test_validate_family_unnamed_mounts.py b/tests/test_validate_family_unnamed_mounts.py index f9257bca..db0d4ed3 100644 --- a/tests/test_validate_family_unnamed_mounts.py +++ b/tests/test_validate_family_unnamed_mounts.py @@ -80,8 +80,19 @@ #: What every unnamed gitlink says after its path and commit, and what one #: that is NOT a case variant of a row's path says to do about it. A #: holder's CI matches on text, so it is pinned here and not rebuilt from the -#: validator's own strings. +#: validator's own strings. GITLINK_SAID is a gitlink no section of HEAD's +#: `.gitmodules` gives a url, which a recursive clone checks out nowhere and +#: a plain clone's `git submodule update --init` stops at (#762: it said a +#: recursive clone fetched it); GITLINK_SAID_FETCHED one a section gives a +#: url, which a recursive clone fetches. GITLINK_SAID = ( + "FINDING member-gitlink-unnamed: {path} is a gitlink at {commit} that " + "no row of members: names and no section of HEAD's .gitmodules gives a " + "url, so a recursive clone of this holder checks out nothing at it, in " + "a plain clone git submodule update --init stops there (No url found " + "for submodule path, exit 128) before it checks out any member, and " + "nothing here pins, digests or identifies it.") +GITLINK_SAID_FETCHED = ( "FINDING member-gitlink-unnamed: {path} is a gitlink at {commit} that " "no row of members: names, so a recursive clone of this holder fetches " "it and nothing here pins, digests or identifies it.") @@ -297,7 +308,7 @@ def test_a_member_mounted_by_hand_is_two_findings(family, holder, tmp_path, assert result.returncode == 1, result.stderr + result.stdout assert "REFUSED" not in result.stderr, result.stderr gitlink, section = findings(result) - assert gitlink.splitlines()[0] == GITLINK_SAID.format( + assert gitlink.splitlines()[0] == GITLINK_SAID_FETCHED.format( path=f"members/{SPARE}", commit=tip) + GITLINK_REMEDY assert commands(gitlink) == [ switch_for(holder, SPARE), @@ -434,7 +445,7 @@ def test_a_removed_row_is_named_by_its_gitlink_and_its_section(holder): assert result.returncode == 1, result.stderr + result.stdout assert codes(result) == ["member-gitlink-unnamed", "gitmodules-path-unnamed"] - assert findings(result)[0].startswith(GITLINK_SAID.format( + assert findings(result)[0].startswith(GITLINK_SAID_FETCHED.format( path="members/IRSS", commit=irss["pin"]["commit"]) + GITLINK_REMEDY) (holder / "family.yaml").write_bytes(text) assert_passes_in_every_mode(holder, 2) @@ -500,7 +511,7 @@ def test_a_row_hand_edited_into_a_case_variant_of_its_mount_is_the_edit( + ([] if extra == ("--no-members",) else ["member-path"])) said = dict(path="members/IRSS", project="IRSS", row_path="Members/IRSS") gitlink, section = findings(result)[:2] - assert gitlink == GITLINK_SAID.format( + assert gitlink == GITLINK_SAID_FETCHED.format( path="members/IRSS", commit=commit_) + ROW_EDITED.format( what="gitlink", **said) assert section == SECTION_SAID.format(