diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index f0ec8356..9e77c3ce 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -90,6 +90,19 @@ which would fail (#681). One no section of HEAD's `.gitmodules` gives a url is said to be checked out nowhere by a recursive clone, and to stop a plain clone's `git submodule update --init` (#762). + - that every gitlink OUTSIDE `members/`, which no row can name, is at a + path a section of `.gitmodules` gives, the `gitlink-unnamed` finding + (#878): a plain clone's `git submodule update --init` stops at one no + section gives (`No url found for submodule path`, exit 128) before it + checks out any member, as the holder's CI runs it, so it is taken out + of the index. One a section gives is that section's + `gitmodules-path-unnamed`, which takes the gitlink out of the index + first, then the section: taken out alone, the section left the + gitlink, and this gate passed it. Each `git rm --cached` comes after + `git add -- .gitmodules` where lines printed before it may leave that + file with changes not staged, or it holds some, since git takes no + gitlink out of the index then. Neither needs a member checked out, + so `--no-members` asks them too. - that a section of HEAD's `.gitmodules` gives the path of every row's gitlink, the `member-gitmodules-absent` finding (#611, #407's decision 3). A recursive clone reads HEAD's file, so a gitlink no section of it @@ -1745,10 +1758,21 @@ def _check_gitmodules_paths(sections: dict[str, dict[str, str]], f"inherits none of them, then commit {GITMODULES}.") +def _in_members(path: str) -> bool: + """Whether `path` is in `members/`, the directory spelt in any case + (#361): `Members/Zed` is in `members/` on macOS and Windows.""" + return path.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR + + def _member_gitlinks(root: Path) -> dict[str, str]: - """`{path: commit}` for every gitlink this repository records in - `members/`, the directory spelt in any case: what `_check_gitlinks_named` - asks of each (#361). + """`{path: commit}` for every gitlink this repository records: what + `_check_gitlinks_named` asks of each in `members/`, the directory spelt + in any case (#361, `_in_members`), and `_check_gitlinks_elsewhere` and + `_check_gitmodules_named` of each outside it (#878). It listed + `members/` alone, so no mode asked about a gitlink elsewhere, and + `gitmodules-path-unnamed`'s remedy for a section that gave one took + out the section and left the gitlink, at which a plain clone's `git + submodule update --init` stops. IN THE ORDER `recorded_gitlink` READS ONE: the index at stage 0 first, HEAD second. A path the index holds at stage 0 as a gitlink is answered @@ -1794,8 +1818,7 @@ def _member_gitlinks(root: Path) -> dict[str, str]: if stage_at is not None and fields[stage_at] != b"0": continue path = name.decode("utf-8", "surrogateescape") - if path.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR: - found[path] = fields[oid_at].decode("ascii").lower() + found[path] = fields[oid_at].decode("ascii").lower() return found @@ -2010,6 +2033,10 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], still a gitlink HEAD records, so it is still named, and its `git rm` typed again failed: it says to commit the removal instead and prints no `git rm` (#681). A hand mount's keeps the branch and the `add`. + + A gitlink outside `members/` is no member's mount, and no row can name + it: `_check_gitlinks_elsewhere` and `_check_gitmodules_named` ask + about it (#878). """ holds = set(gitlinks) head = _head_gitmodules(root) if set(gitlinks) - set(named) else None @@ -2017,7 +2044,7 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], if "path" in keys and "url" in keys} if head is not None else set()) for path, commit in sorted(gitlinks.items()): - if path in named: + if path in named or not _in_members(path): continue said = (f"{path} is a gitlink at {commit} that no row of members: " + ("names, so a recursive clone of this holder fetches it " @@ -2075,7 +2102,7 @@ def _check_gitlinks_named(root: Path, gitlinks: dict[str, str], def _second_path_out(root: Path, path: str, commit: str, - staged: bool) -> str: + staged: bool, stage_first: bool = False) -> str: """What a `STRAY` gitlink at `path`, a second path to a member's own directory, which `_member_gitlinks` reads at `commit`, says to do: take it out of the index alone, with `-f` where git refuses it without @@ -2085,20 +2112,148 @@ def _second_path_out(root: Path, path: str, commit: str, holds the variant, and HEAD the row's own gitlink, whose removal is staged (`member-gitlink-removal-staged`). A commit or an untracked file in the checkout `git mv` moved, and the plain `git rm --cached` refused - (w761's candidate 2 on #828, measured on the gate's state).""" + (w761's candidate 2 on #828, measured on the gate's state). + + `gitlink-unnamed` says the same of a gitlink outside `members/` that no + section gives (#878), after `git add -- .gitmodules` where + `stage_first` (`_stage_first`).""" if staged: return f"{STAGED_REMOVAL} Commit that removal." head = _forced(root, path, commit) + return ((f"{STAGE_SAID[0].upper()}{STAGE_SAID[1:]}take" if stage_first + else "Take") + + " it out of the index alone, which touches no file" + + _forced_said(head, "the second line" if stage_first + else "the line") + + ":\n " + (f"{STAGE_LINE}\n " if stage_first else "") + + _cached_removal(path, head)) + + +#: The line the `git rm --cached` of a gitlink outside `members/` comes +#: after where `.gitmodules` may hold changes not staged (`_stage_first`, +#: #878), and what the text says of it, before what it says of the `git rm`. +STAGE_LINE = f"git add -- {GITMODULES}" +STAGE_SAID = (f"stage {GITMODULES} with the first line, since git rm refuses " + "to take any gitlink out of the index, -f or not, while " + f"{GITMODULES} holds changes not staged (please stage your " + f"changes to {GITMODULES} or stash them to proceed, exit 128), " + "as lines printed above may leave it or as it stands, then ") + + +def _stage_first(root: Path, edited: bool) -> bool: + """Whether the `git rm --cached` that `gitlink-unnamed` and + `gitmodules-path-unnamed` print for a gitlink outside `members/` comes + after `git add -- .gitmodules` (`STAGE_LINE`, #878). + + GIT TAKES NO GITLINK OUT OF THE INDEX, `--cached` and `-f` or not, + while the work tree's `.gitmodules` holds changes the index does not + (`please stage your changes to .gitmodules or stash them to proceed`, + exit 128, `is_staging_gitmodules_ok` in git's rm.c), measured on git + 2.43 after a `gitmodules-value-refused` finding's `--unset-all` lines + (the order #879's C3 names for a stray's `git rm --cached`), after an + earlier unnamed section's lines, and over a hand edit not staged. So it + does where `edited`, where lines printed before it may write the file, + and where the file on disk differs from the index's as it stands: its + bytes hashed as `git add` would store them (`git hash-object`) against + the blob the index holds, as git compares them, and never by stat, + which a file written in the same second as the index may fool. The + commit the text names next commits what it stages, which the lines + before it say to commit. Not where the work tree holds no + `.gitmodules` and no line before it writes one, where `git add` would + stage its deletion, nor where the index holds none at stage 0: git + asks nothing of the file then.""" + if edited: + return True + if not os.path.lexists(root / GITMODULES): + return False + with _git_runs(f"whether {GITMODULES} holds changes not staged could not " + "be read"): + try: + staged = git_out(["rev-parse", "--verify", "--quiet", + f":{GITMODULES}"], cwd=root) + except Refusal: + return False + return git_out(["hash-object", "--", GITMODULES], cwd=root) != staged + + +def _forced_said(head: str | None, line: str) -> str: + """What the text says after "take it out of the index alone, which + touches no file" of the `git rm --cached` it prints as `line`, where + `_forced` answered `head` (#839): that the removal is committed, and + where the line takes `-f`, why, and what `-f` does (`FORCED_SAID`), + and that nothing is left to commit where HEAD records nothing.""" if head is None: - return ("Take it out of the index alone, which touches no file, and " - f"commit that:\n git rm --cached -- {shlex.quote(path)}") - return ("Take it out of the index alone, which touches no file" - + (", and commit that. The index stages it at a commit HEAD does " - "not record there" if head else - ": HEAD records nothing there, so nothing of it is left to " - "commit. The index stages it") - + f" over a checkout, so the line takes -f: {FORCED_SAID}:" - f"\n git rm --cached -f -- {shlex.quote(path)}") + return ", and commit that" + return ((", and commit that. The index stages it at a commit HEAD does " + "not record there" if head else + ": HEAD records nothing there, so nothing of it is left to " + "commit. The index stages it") + + f" over a checkout, so {line} takes -f: {FORCED_SAID}") + + +def _cached_removal(path: str, head: str | None) -> str: + """`git rm --cached -- `, with `-f` where `_forced` answered + `head` (#839).""" + return ("git rm --cached" + ("" if head is None else " -f") + + f" -- {shlex.quote(path)}") + + +def _check_gitlinks_elsewhere(root: Path, gitlinks: dict[str, str], + named: dict[str, RowPath], + removed: frozenset[str], + report: Report) -> None: + """Every gitlink OUTSIDE `members/` that no row names is at a path a + section of `.gitmodules` gives, the `gitlink-unnamed` finding (#878). + + NO ROW CAN NAME ONE, since a member's path is `members/` + (`member-path`), and NO CLONE CAN CHECK OUT ONE NO SECTION GIVES: a + plain clone's `git submodule update --init` stops there (`No url found + for submodule path`, exit 128) before it checks out any member, which + is how the holder's CI checks its members out before it runs the + member half of this gate, a recursive clone checks out nothing at it, + and `git submodule status` exits 128 (`no submodule mapping found`). + No mode asked about one: `_member_gitlinks` listed `members/` alone, so + `gitmodules-path-unnamed`'s `--unset-all` lines, typed as printed, + left this state and the gate passed it. Bootstrap's `init_stop` names + it, with its cure (`cures`), and so does this, in its words: `git rm + --cached`, which touches no file, with `-f` where git refuses it + without (`_forced`, #839), and the commit, or, where its removal is + staged already (`removed`), the commit alone (`_second_path_out`). + + THE WORK TREE'S `.gitmodules`, read whole, as `_check_gitmodules_named` + reads it where it gives no value git dies on: a gitlink a section + there gives is that section's finding, which takes the gitlink out + first, so each such gitlink is one finding, never two. Where the file + gives a value git dies on, `gitmodules-path-unnamed` reads none of + its sections, and a gitlink one of them gives is said once that is + mended. + + `git add -- .gitmodules` COMES FIRST where a `gitmodules-value-refused` + finding, printed before this one, may leave the file with changes not + staged, or it holds some as it stands (`_stage_first`): git takes no + gitlink out of the index then, and the `git rm` exited 128.""" + unnamed = [path for path in sorted(gitlinks) + if path not in named and not _in_members(path)] + if not unnamed: + return + given = {keys["path"] for keys in _gitmodules_sections(root).values() + if "path" in keys} + stage_first = _stage_first(root, + "gitmodules-value-refused" in report.codes) + for path in unnamed: + if path in given: + continue + report.finding( + "gitlink-unnamed", + f"{path} is a gitlink at {gitlinks[path]}, outside " + f"{DEFAULT_MEMBERS_DIR}/, that no section of {GITMODULES} gives, " + "so in a plain clone of this holder git submodule update --init " + "stops there (No url found for submodule path, exit 128) before " + "it checks out any member, and a recursive clone checks out " + f"nothing at it. No row names {path} and no clone can check it " + "out. " + + _second_path_out(root, path, gitlinks[path], path in removed, + stage_first)) def _mounted_by_hand(root: Path, path: str, project: str | None, @@ -2145,7 +2300,7 @@ def _mounted_by_hand(root: Path, path: str, project: str | None, f" git rm -- {shlex.quote(path)}\n{add}") -def _check_gitmodules_named(sections: dict[str, dict[str, str]], +def _check_gitmodules_named(root: Path, sections: dict[str, dict[str, str]], gitlinks: dict[str, str], named: dict[str, RowPath], removed: frozenset[str], report: Report) -> None: @@ -2175,10 +2330,24 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], sections give the path (`gitmodules-path-twice`) it takes out only the one git reads for it, the last in the file, so each of those keeps its own commands. + + A SECTION AT THE PATH OF A GITLINK OUTSIDE `members/` (#878), which no + row can name, and which the index holds, prints the `git rm --cached` + that takes the gitlink out first, and the commit, then its own lines + (`_gitlink_out_first`): its lines alone took out the section and left + the gitlink, which no section then gave, and the gate passed it while + a plain clone's `git submodule update --init` stopped there. Once for + each path, by the first section that gives it: the `git rm` typed + again fails. A gitlink whose removal is staged prints none, as above. + After a finding whose lines write `.gitmodules`, a value's or an + earlier section's (`edited`), the `git rm` comes after `git add -- + .gitmodules`, which it needs (`_stage_first`). """ holds = {keys["path"] for keys in sections.values() if "path" in keys} given = Counter(keys["path"] for keys in sections.values() if "path" in keys) + taken: set[str] = set() + edited = "gitmodules-value-refused" in report.codes for name, keys in sections.items(): path = keys.get("path") if path is None or path in named: @@ -2204,13 +2373,22 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], said += (" If its row was deleted or its path: edited by hand, " "put the row back as git last had it.") lead = "Otherwise take" - if path in gitlinks and path not in removed and given[path] == 1: + if (path in gitlinks and path not in removed and given[path] == 1 + and _in_members(path)): report.finding( "gitmodules-path-unnamed", f"{said} Otherwise the git rm printed above for the " f"gitlink at {path} takes this section out too, so no " "command is printed here.") continue + first = (_gitlink_out_first(root, name, path, gitlinks, removed, + taken, edited) + if variant is None else None) + then, line = f"then commit {GITMODULES}", "" + if first is not None: + first_said, line, then = first + said += first_said + lead = "Then, with the lines after it, take" removal = "".join( f"\n git config -f {GITMODULES} --unset-all " f"{shlex.quote(f'submodule.{name}.{key}')}" for key in keys) @@ -2218,7 +2396,64 @@ def _check_gitmodules_named(sections: dict[str, dict[str, str]], "gitmodules-path-unnamed", f"{said} {lead} the section out by hand, in a POSIX shell, every " "key it holds, so a submodule later given its name inherits none " - f"of them, then commit {GITMODULES}:{removal}") + f"of them, {then}:{line}{removal}") + edited = True + + +def _gitlink_out_first(root: Path, name: str, path: str, + gitlinks: dict[str, str], removed: frozenset[str], + taken: set[str], + edited: bool) -> tuple[str, str, str] | None: + """What `gitmodules-path-unnamed` adds for the section `name`, whose + `path` is a gitlink outside `members/` (#878), `(what it says, the + line it prints first, how it ends)`, or None where it adds nothing. + + THE GITLINK GOES FIRST, out of the index alone, with `git rm --cached`, + which touches no file and leaves the section, and the commit, as + `member-gitlink-unnamed` takes out a second path to a member; `-f` + where git refuses it without, with what `-f` does (`_forced`, + `FORCED_SAID`, #839). Taken out alone, the section left a gitlink no + section gives (`gitlink-unnamed`), at which a plain clone's `git + submodule update --init` stops (`No url found for submodule path`, + exit 128) before it checks out any member, and `git submodule status` + exits 128 (`no submodule mapping found`). It ends as the section's + finding does, `then commit .gitmodules`, but where HEAD's `.gitmodules` + holds no section of its name, as after a `git submodule add` not + committed: staged, the file is HEAD's again, and nothing of it is + left to commit. + + `git add -- .gitmodules` COMES FIRST where `edited`, where lines + printed before it may leave the file with changes not staged (a + `gitmodules-value-refused` finding's, an earlier unnamed section's), + or where it holds some as it stands (`_stage_first`): git takes no + gitlink out of the index then, and the `git rm` exited 128. + + None for a path in `members/`, whose gitlink has a finding of its own + (`member-gitlink-unnamed`), for one that is no gitlink, and one + whose removal is staged (`removed`), where a `git rm` fails (`pathspec + ... did not match`), and for one an earlier section's finding took out + (`taken`): the `git rm` typed again fails there too. A path it adds to + is added to `taken`.""" + if (path not in gitlinks or path in removed or _in_members(path) + or path in taken): + return None + taken.add(path) + head = _forced(root, path, gitlinks[path]) + in_head = _head_gitmodules(root) + then = (f"then commit {GITMODULES}" + if in_head is not None and name in in_head[1] else + f"then stage {GITMODULES}: HEAD's holds no section of that name, " + "so nothing of it is left to commit") + stage = _stage_first(root, edited) + nth = "the second line" if stage else "the first line" + return (" Otherwise take its gitlink out first, since a gitlink no " + "section gives stops a plain clone's git submodule update --init " + "(No url found for submodule path, exit 128) before it checks out " + "any member: " + (STAGE_SAID if stage else "") + + f"take it out of the index alone, which touches no file, with " + f"{nth}" + _forced_said(head, nth) + ".", + (f"\n {STAGE_LINE}" if stage else "") + + f"\n {_cached_removal(path, head)}", then) def _section_lines(path: str, repository, @@ -2588,7 +2823,8 @@ def _init_stop_cure(stopper: str, sections: dict[str, dict[str, str]], stops at, as `_init_stop_said` prints it: `--no-members` for a path two of `sections` give or a gitlink in `members/` no row names, the section written back for a row's own path (`named`), and `git rm --cached` for - a gitlink outside `members/`, which no mode here reads.""" + a gitlink outside `members/`, as `gitlink-unnamed` and + `gitmodules-path-unnamed` name it in every mode (#878).""" given = Counter(keys["path"] for keys in sections.values() if "path" in keys) if given[stopper] > 1: @@ -2599,7 +2835,7 @@ def _init_stop_cure(stopper: str, sections: dict[str, dict[str, str]], return (f"Write the section of {GITMODULES} for {stopper} back as " "`family.py add` writes one, with its path and url, and " f"commit {GITMODULES}.") - if stopper.split("/", 1)[0].casefold() == DEFAULT_MEMBERS_DIR: + if _in_members(stopper): return (f"`{PYTHON} scripts/validate-family.py --no-members` names " f"each gitlink in {DEFAULT_MEMBERS_DIR}/ that no row names, " "and the commit that cures it.") @@ -2667,7 +2903,8 @@ def _check_members(root: Path, members: list, no_members: bool, gitlinks = _member_gitlinks(root) removed = _staged_removals(root, gitlinks) _check_gitlinks_named(root, gitlinks, named, removed, report) - _check_gitmodules_named(sections, gitlinks, named, removed, report) + _check_gitlinks_elsewhere(root, gitlinks, named, removed, report) + _check_gitmodules_named(root, sections, gitlinks, named, removed, report) unregistered = _check_gitlinks_registered(root, members, gitlinks, report, refused) unregistered |= _check_removals_staged(root, members, gitlinks, removed, diff --git a/tests/test_family_bootstrap_gate_order.py b/tests/test_family_bootstrap_gate_order.py index fc3a7caa..6b687433 100644 --- a/tests/test_family_bootstrap_gate_order.py +++ b/tests/test_family_bootstrap_gate_order.py @@ -659,11 +659,11 @@ def test_a_hand_mount_outside_members_is_taken_out_of_the_index( family, holder, offline, tmp_path): """rv648's H2. SPARE's gitlink recorded by hand at `vendor/`, outside members/, with no row and no section, and committed: in a plain - clone git's init dies there before it checks out any member. No mode - of the validator reads a gitlink outside members/, and `--no-members` - says `family ok`, so the line names the `git rm --cached` that takes - the gitlink out. Typed as printed and committed, the same run checks - both members out, and the gate passes in every mode.""" + clone git's init dies there before it checks out any member. The line + names the `git rm --cached` that takes the gitlink out, and the + validator's `--no-members`, which said `family ok` there, names the same + line now, `gitlink-unnamed` (#878). Typed as printed and committed, the + same run checks both members out, and the gate passes in every mode.""" env = fetching(isolated(offline, tmp_path)) stopper = f"vendor/{SPARE}" stage_gitlink(holder, stopper, spare_commits(family)[1]) @@ -680,8 +680,11 @@ def test_a_hand_mount_outside_members_is_taken_out_of_the_index( failed_on(MEMBER, OTHER)], result.stderr assert f"No url found for submodule path '{stopper}'" in result.stderr fallback = validate(clone, "--no-members") - assert fallback.returncode == 0, "premise: it reads only members/" + assert fallback.returncode == 1, fallback.stdout + fallback.stderr (line,) = re.findall(r"`(git rm [^`]*)`", result.stderr) + (named,) = findings(fallback) + assert named.startswith(f"FINDING gitlink-unnamed: {stopper} "), named + assert commands(named) == [line], "the same line (#878)" type_line(clone, env, line) commit_all(clone, env, "The gitlink taken out as printed") diff --git a/tests/test_family_unnamed_gitlink.py b/tests/test_family_unnamed_gitlink.py new file mode 100644 index 00000000..6f922260 --- /dev/null +++ b/tests/test_family_unnamed_gitlink.py @@ -0,0 +1,545 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The family root's `validate-family.py`: a gitlink outside `members/` +that no row names, and the `gitmodules-path-unnamed` remedy that left one +(#878). + +THE ISSUE'S LINE, reproduced on #876's head (77f8a6c8) before this change: +`git submodule add tools/x`, committed, or IRSS added at +`Members/IRRS` and moved by hand with `git mv` to `renamed-1`, committed. +The gate exited 1 in every mode, `gitmodules-path-unnamed`, and printed +only the `--unset-all` lines that take the section out. Typed as printed, +with `.gitmodules` staged and committed, the gate exited 0 in every mode, +while `git submodule status` exited 128 (`no submodule mapping found`) +and a plain clone's `git submodule update --init --recursive` exited 128 +(`No url found for submodule path`) with no member checked out: the +gate's gitlinks were read in `members/` alone, so no mode asked about the +one the lines left. + +ITS THREE DIRECTIONS, each held here: + +- The section's finding prints the `git rm --cached` that takes its + gitlink out of the index first, and the commit, then its own lines. +- A gitlink outside `members/` that no section gives is a finding of its + own, `gitlink-unnamed`, whose `git rm --cached` is the one bootstrap's + `init_stop` cure names for it. +- Where the index stages that gitlink at a commit HEAD does not record + and a checkout is on disk at it, the `git rm` takes `-f`, with what + `-f` does, as #876's `_forced` gives it: git refused it there (`staged + content different from both the file and the HEAD`, exit 1). + +Each remedy is typed as printed, split as a POSIX shell splits it and run +with no shell, then the gate is asked in every mode; where HEAD changed, a +plain clone's `git submodule update --init --recursive` must check out +both members and `git submodule status` exit 0 there and in the holder. +WHERE LINES PRINTED BEFORE IT WRITE `.gitmodules` (a value finding's, in +the order #879's C3 names for a stray's `git rm --cached`, or an earlier +unnamed section's), or the file holds a hand edit not staged, git refused +#878's `git rm --cached` (`please stage your changes to .gitmodules or +stash them to proceed`, exit 128), so `git add -- .gitmodules` comes +directly before it there, and every line is typed in the order printed. +THE CONTROLS: the holder as `family.py` made it, a member under +`members/` with its section and its row, and a section with no gitlink, +or none in the index, which prints its own lines alone, as it did. + +THE SAME FAMILY, OFFLINE RULES AND HELPERS as +`tests/test_family_holder_bootstrap_texts.py` (#839) and +`tests/test_validate_family_unnamed_mounts.py` (#361). A checkout is git's +own, made by `git submodule add` from the fixture's local remotes; a state +with no checkout is an index record (`stage_gitlink`). The `git mv` form +needs `Members/IRRS` apart from `members/IRRS` on disk, and is skipped +where the disk folds case (`case_sensitive_disk`). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from conftest import git +from test_family import family, validate # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bootstrap_gate_order import cloned, fetching, type_line +from test_family_dry_run import isolated +from test_family_head_lost_gitmodules import ( # noqa: F401 + FORCED_SAID, assert_passes_in_every_mode, case_sensitive_disk, + commit_all, staged_and_committed, typed, +) +from test_family_holder_bootstrap_texts import WORK +from test_family_member_path import MEMBER, OTHER, raw +from test_validate_family_remedies_typed import STAGED +from test_validate_family_unnamed_mounts import ( + SECTION_REMEDY, SECTION_SAID, codes, commands, findings, removal, + spare_commits, stage_gitlink, with_section, +) + +#: The gitlink outside `members/` the issue mounts IRSS at, the case +#: variant of MEMBER's path it mounts it at for the `git mv`, and where +#: that moves it. +TOOLS = "tools/x" +VARIANT = f"Members/{MEMBER}" +RENAMED = "renamed-1" + +#: `gitlink-unnamed` up to what it says of its `git rm --cached`. A +#: holder's CI matches on text, so it is pinned here and not rebuilt from +#: the gate's own strings. +OUTSIDE_SAID = ( + "FINDING gitlink-unnamed: {path} is a gitlink at {commit}, outside " + "members/, that no section of .gitmodules gives, so in a plain clone of " + "this holder git submodule update --init stops there (No url found for " + "submodule path, exit 128) before it checks out any member, and a " + "recursive clone checks out nothing at it. No row names {path} and no " + "clone can check it out. ") + +#: What every text says first of the `git rm --cached` it prints. +TAKE_OUT = "take it out of the index alone, which touches no file" + +#: What it says first where lines printed before it may leave `.gitmodules` +#: with changes not staged, or it holds some, and the line it prints for +#: that: git takes no gitlink out of the index then (#879's C3). +STAGE_SAID = ("stage .gitmodules with the first line, since git rm refuses to " + "take any gitlink out of the index, -f or not, while " + ".gitmodules holds changes not staged (please stage your " + "changes to .gitmodules or stash them to proceed, exit 128), " + "as lines printed above may leave it or as it stands, then ") +STAGE_LINE = "\n git add -- .gitmodules" + +#: What it says next, by what `_forced` reads: the commit; or `-f`, where +#: the index stages the gitlink at a commit HEAD does not record there +#: (`FORCED_OVER`), or where HEAD records nothing (`FORCED_NO_HEAD`), over +#: a checkout. `{line}` names the line that takes it. +COMMIT_THAT = ", and commit that" +FORCED_OVER = (", and commit that. The index stages it at a commit HEAD " + "does not record there over a checkout, so {line} takes -f: " + + FORCED_SAID) +FORCED_NO_HEAD = (": HEAD records nothing there, so nothing of it is left to " + "commit. The index stages it over a checkout, so {line} " + "takes -f: " + FORCED_SAID) + +#: What `gitmodules-path-unnamed` says of a section no row names, before +#: what it says to do. +ROW_BACK = (" If its row was deleted or its path: edited by hand, put the " + "row back as git last had it.") + +#: What it says of the gitlink at a section's path outside `members/`, +#: which it takes out first, then of the section. +GITLINK_FIRST = ( + " Otherwise take its gitlink out first, since a gitlink no section " + "gives stops a plain clone's git submodule update --init (No url found " + "for submodule path, exit 128) before it checks out any member: ") +SECTION_THEN = (" Then, with the lines after it, take the section out by " + "hand, in a POSIX shell, every key it holds, so a submodule " + "later given its name inherits none of them, {then}:") + +#: How the section's text ends: the commit, or, where HEAD's `.gitmodules` +#: holds no section of its name, the file staged and nothing to commit. +THEN_COMMIT = "then commit .gitmodules" +THEN_STAGE = ("then stage .gitmodules: HEAD's holds no section of that name, " + "so nothing of it is left to commit") + + +def outside(path: str, commit: str, said: str, flag: str = "", *, + stage: bool = False) -> str: + """`gitlink-unnamed` whole for the gitlink at `path`, at `commit`: + `said` (`COMMIT_THAT`, `FORCED_OVER` or `FORCED_NO_HEAD`) of its `git + rm --cached`, which carries `flag`, after `git add -- .gitmodules` + where `stage`.""" + first = STAGE_SAID + TAKE_OUT if stage else TAKE_OUT + return (OUTSIDE_SAID.format(path=path, commit=commit) + + first[0].upper() + first[1:] + + said.format(line="the second line" if stage else "the line") + + ":" + (STAGE_LINE if stage else "") + + f"\n git rm --cached{flag} -- {path}") + + +def section_first(path: str, name: str, url: str, said: str, then: str, + flag: str = "", *, stage: bool = False) -> str: + """`gitmodules-path-unnamed` whole for the section `name`, giving `path` + and `url`, at a gitlink outside `members/`: its `git rm --cached`, + carrying `flag`, after `git add -- .gitmodules` where `stage`, of which + it says `said`, then its own lines, and `then`.""" + nth = "the second line" if stage else "the first line" + return (SECTION_SAID.format(path=path, name=name, url=f"url {url}") + + ROW_BACK + GITLINK_FIRST + (STAGE_SAID if stage else "") + + f"{TAKE_OUT}, with {nth}" + said.format(line=nth) + "." + + SECTION_THEN.format(then=then) + (STAGE_LINE if stage else "") + + f"\n git rm --cached{flag} -- {path}" + + "".join(f"\n {line}" for line in removal(name, "path", + "url"))) + + +def found_in_every_mode(root: Path, code: str) -> str: + """The one finding the gate prints, `code`'s, the same in every mode.""" + seen: set[str] = set() + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(root, *extra) + assert result.returncode == 1, (extra, result.stdout + result.stderr) + assert codes(result) == [code], (extra, result.stderr) + seen.update(findings(result)) + (finding,) = seen + return finding + + +def staged_at(root: Path, env: dict, path: str) -> str: + """The commit the index stages the gitlink at `path` at.""" + return raw(root, env, "rev-parse", f":{path}").stdout.decode( + "ascii").strip() + + +def section_url(root: Path, env: dict, name: str) -> str: + """The url the work tree's `.gitmodules` gives the section `name`.""" + return raw(root, env, "config", "-f", ".gitmodules", "--get", + f"submodule.{name}.url").stdout.decode("utf-8").strip() + + +def assert_a_plain_clone_checks_out_every_member(root: Path, env: dict, + family, tmp_path: Path + ) -> None: + """`git submodule status` exits 0 in `root`; a plain clone of its HEAD, + then `git submodule update --init --recursive`, exits 0 there with both + members checked out, and `git submodule status` exits 0 there too.""" + raw(root, env, "submodule", "status") + clone = cloned(root, env, family, tmp_path, recurse=False) + raw(clone, env, "submodule", "update", "--init", "--recursive") + for project in (MEMBER, OTHER): + assert (clone / "members" / project / ".git").exists(), project + raw(clone, env, "submodule", "status") + + +def submodule_added(root: Path, env: dict, family, path: str) -> None: + """IRSS mounted at `path` by `git submodule add`, not committed.""" + raw(root, env, "submodule", "add", "-q", "--", + str(family["remotes"] / "IRSS.git"), path) + + +# --- the issue's two states -------------------------------------------------- + +def at_tools(root: Path, env: dict, family) -> tuple[str, str]: + """`git submodule add tools/x`, committed. Returns the section's + name and the path it gives.""" + submodule_added(root, env, family, TOOLS) + commit_all(root, env, "IRSS mounted at tools/x by hand") + return TOOLS, TOOLS + + +def moved_to_renamed(root: Path, env: dict, family) -> tuple[str, str]: + """IRSS added at `Members/IRRS`, committed, then moved by hand with + `git mv` to `renamed-1`, committed: git gives its section that path + under the name it had.""" + submodule_added(root, env, family, VARIANT) + commit_all(root, env, "IRSS mounted at Members/IRRS by hand") + raw(root, env, "mv", "--", VARIANT, RENAMED) + commit_all(root, env, "Moved by hand to renamed-1") + return VARIANT, RENAMED + + +#: `(how the state is made, whether it needs a disk that keeps case)`. +FORMS = {"tools-x": (at_tools, False), + "git-mv-to-renamed-1": (moved_to_renamed, True)} + + +@pytest.mark.parametrize("form, keeps_case", FORMS.values(), ids=FORMS) +def test_a_section_s_remedy_takes_its_gitlink_out_first( + family, holder, offline, tmp_path, request, form, keeps_case): + """THE ISSUE'S LINE, both forms. The section's finding, in every mode, + prints the `git rm --cached` of its gitlink first, and the commit, then + the `--unset-all` lines it printed alone. Typed as printed, with + `.gitmodules` staged and committed, the gate passes in every mode, and + a plain clone checks out both members: the lines alone left the + gitlink, and its update exited 128 at it.""" + if keeps_case: + request.getfixturevalue("case_sensitive_disk") + env = fetching(isolated(offline, tmp_path)) + name, path = form(holder, env, family) + + finding = found_in_every_mode(holder, "gitmodules-path-unnamed") + + assert finding == section_first(path, name, + section_url(holder, env, name), + COMMIT_THAT, THEN_COMMIT) + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + + +def section_taken_out_alone(root: Path, env: dict, family) -> None: + """THE STATE THE OLD REMEDY LEFT: tools/x's section taken out by the + lines `gitmodules-path-unnamed` printed alone, staged and committed.""" + at_tools(root, env, family) + for line in removal(TOOLS, "path", "url"): + type_line(root, env, line) + staged_and_committed(root, env, "The section taken out alone") + + +def test_a_gitlink_outside_members_no_section_gives_is_a_finding( + family, holder, offline, tmp_path): + """THE NEW FINDING, in the state the old remedy left, where every mode + said `family ok` while a plain clone's update stopped at the gitlink. + It names it in every mode, with bootstrap's `git rm --cached`; typed as + printed and committed, the gate passes in every mode, and a plain + clone checks out both members.""" + env = fetching(isolated(offline, tmp_path)) + section_taken_out_alone(holder, env, family) + before = cloned(holder, env, family, tmp_path / "before", recurse=False) + stopped = raw(before, env, "submodule", "update", "--init", "--recursive", + check=False) + assert stopped.returncode == 128, "premise: a plain clone stops there" + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == outside(TOOLS, staged_at(holder, env, TOOLS), + COMMIT_THAT) + typed(holder, env, finding) + commit_all(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + + +# --- -f where git refuses the plain git rm --cached -------------------------- + +def added_not_committed(root: Path, env: dict, family, work) -> None: + """rv865b's (a): `git submodule add tools/x`, not committed, + with `work` in its checkout. HEAD records nothing there.""" + submodule_added(root, env, family, TOOLS) + work(root / TOOLS, env) + + +def moved_on(root: Path, env: dict, work) -> None: + """A commit in tools/x's checkout, staged in the holder, where HEAD + records the one before it, then `work` in that checkout.""" + raw(root / TOOLS, env, "commit", "-q", "--allow-empty", "-m", + "Moved on") + raw(root, env, "add", "--", TOOLS) + work(root / TOOLS, env) + + +def committed_then_moved_on(root: Path, env: dict, family, work) -> None: + at_tools(root, env, family) + moved_on(root, env, work) + + +def no_section_then_moved_on(root: Path, env: dict, family, work) -> None: + section_taken_out_alone(root, env, family) + moved_on(root, env, work) + + +#: `(how the state is made, the finding's code, what it says of its -f, +#: how the section's text ends, or None for the gitlink's own finding)`. +FORCED = { + "section-added-not-committed": (added_not_committed, + "gitmodules-path-unnamed", + FORCED_NO_HEAD, THEN_STAGE), + "section-committed-moved-on": (committed_then_moved_on, + "gitmodules-path-unnamed", FORCED_OVER, + THEN_COMMIT), + "no-section-moved-on": (no_section_then_moved_on, "gitlink-unnamed", + FORCED_OVER, None), +} + + +@pytest.mark.parametrize("work", WORK.values(), ids=WORK) +@pytest.mark.parametrize("make, code, said, then", FORCED.values(), + ids=FORCED) +def test_a_gitlink_staged_over_a_checkout_is_taken_out_with_f( + family, holder, offline, tmp_path, make, code, said, then, work): + """DIRECTION 3: the index stages tools/x at a commit HEAD does not + record there, or where HEAD records nothing, and a checkout is on disk + at it that has moved on or holds a file git does not track. The plain + `git rm --cached` refuses there, exit 1, and changes nothing; the line + takes `-f`, in every mode, and says what it does. Typed as printed, + then staged and committed as the text says, or staged where it says + nothing is left to commit, the gate passes in every mode, the checkout + is still on disk, and where HEAD changed a plain clone checks out both + members.""" + env = fetching(isolated(offline, tmp_path)) + make(holder, env, family, work) + refused = raw(holder, env, "rm", "-q", "--cached", "--", TOOLS, + check=False) + assert refused.returncode == 1, "premise: the plain git rm refuses" + + finding = found_in_every_mode(holder, code) + + assert finding == ( + outside(TOOLS, staged_at(holder, env, TOOLS), said, " -f") + if then is None else + section_first(TOOLS, TOOLS, section_url(holder, env, TOOLS), said, + then, " -f")) + typed(holder, env, finding) + if then == THEN_STAGE: + raw(holder, env, "add", "--", ".gitmodules") + raw(holder, env, "diff", "--cached", "--quiet") + else: + staged_and_committed(holder, env, "Typed as printed") + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + assert_passes_in_every_mode(holder, 2) + assert (holder / TOOLS / ".git").is_file(), "-f touched no file" + + +# --- after lines that write .gitmodules, git add first (#879's C3) ---------- + +def value_first(root: Path, env: dict, family) -> str: + """#879's C3 order, before this finding: MEMBER's section given + `update = None`, committed, beside the state the old remedy left, so + the value finding's `--unset-all` line prints first. Returns the text + the gitlink's finding prints whole.""" + section_taken_out_alone(root, env, family) + raw(root, env, "config", "-f", ".gitmodules", + f"submodule.members/{MEMBER}.update", "None") + staged_and_committed(root, env, "update = None") + return outside(TOOLS, staged_at(root, env, TOOLS), COMMIT_THAT, + stage=True) + + +def earlier_section_first(root: Path, env: dict, family) -> str: + """A section no row names, for tools/y, with no gitlink, committed + before tools/x is added: its `--unset-all` lines print first.""" + for key, value in (("path", "tools/y"), ("url", "/nowhere/y.git")): + raw(root, env, "config", "-f", ".gitmodules", + f"submodule.tools/y.{key}", value) + staged_and_committed(root, env, "A section for tools/y") + at_tools(root, env, family) + return section_first(TOOLS, TOOLS, section_url(root, env, TOOLS), + COMMIT_THAT, THEN_COMMIT, stage=True) + + +def hand_edit_first(root: Path, env: dict, family) -> str: + """tools/x added and committed, then `.gitmodules` edited by hand, not + staged.""" + at_tools(root, env, family) + gitmodules = root / ".gitmodules" + gitmodules.write_bytes(gitmodules.read_bytes() + b"# a hand edit\n") + return section_first(TOOLS, TOOLS, section_url(root, env, TOOLS), + COMMIT_THAT, THEN_COMMIT, stage=True) + + +#: `(how the state is made, which returns the text #878's finding prints +#: last, the codes printed in order)`: lines printed before #878's `git rm +#: --cached` write `.gitmodules`, or it holds a change not staged. +STAGE_FIRST = { + "value-finding-first": (value_first, ["gitmodules-value-refused", + "gitlink-unnamed"]), + "earlier-section-first": (earlier_section_first, + ["gitmodules-path-unnamed", + "gitmodules-path-unnamed"]), + "hand-edit-not-staged": (hand_edit_first, ["gitmodules-path-unnamed"]), +} + + +@pytest.mark.parametrize("make, printed", STAGE_FIRST.values(), + ids=STAGE_FIRST) +def test_after_lines_that_write_gitmodules_its_git_rm_stages_it_first( + family, holder, offline, tmp_path, make, printed): + """#879's C3, for #878's own line. Git takes no gitlink out of the + index, `-f` or not, while `.gitmodules` holds changes not staged + (`please stage your changes to .gitmodules or stash them to proceed`, + exit 128), as a value finding's `--unset-all` lines leave it, or an + earlier unnamed section's, or a hand edit. So `git add -- .gitmodules` + comes directly before #878's `git rm --cached`, in every mode, and says + why. Every line of every finding typed in the order printed, with + nothing between, then `.gitmodules` staged and committed: the gate + passes in every mode, and a plain clone checks out both members.""" + env = fetching(isolated(offline, tmp_path)) + expected = make(holder, env, family) + + for extra in ((), ("--pins",), ("--no-members",)): + result = validate(holder, *extra) + assert codes(result) == printed, (extra, result.stderr) + assert findings(result)[-1] == expected, extra + for finding in findings(result): + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed, in order") + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member(holder, env, family, + tmp_path) + + +# --- where no git rm is printed, or once ------------------------------------- + +def test_a_gitlink_outside_members_whose_removal_is_staged_says_commit_it( + family, holder): + """`gitlink-unnamed` for a gitlink HEAD records and the index no longer + holds: its `git rm` fails there (`pathspec ... did not match`), so it + prints none and says to commit the removal; committed, the gate passes + in every mode.""" + _, tip = spare_commits(family) + stage_gitlink(holder, TOOLS, tip) + git("commit", "-qm", "A gitlink recorded by hand at tools/x", + cwd=holder) + git("rm", "-q", "--cached", "--", TOOLS, cwd=holder) + + finding = found_in_every_mode(holder, "gitlink-unnamed") + + assert finding == (OUTSIDE_SAID.format(path=TOOLS, commit=tip) + STAGED + + " Commit that removal.") + git("commit", "-qm", "The removal committed", cwd=holder) + assert_passes_in_every_mode(holder, 2) + + +def test_two_sections_for_one_gitlink_outside_members_print_one_git_rm( + family, holder, offline, tmp_path): + """Two sections give tools/x (`gitmodules-path-twice`): the first + section's finding prints the gitlink's `git rm --cached`, and the + second its own lines alone, since that `git rm` typed again fails. + Every line typed as printed, `.gitmodules` staged and committed, the + gate passes in every mode.""" + _, tip = spare_commits(family) + stage_gitlink(holder, TOOLS, tip) + with_section(holder, tmp_path, TOOLS, TOOLS, "/nowhere/x.git") + with_section(holder, tmp_path, "second", TOOLS, "/nowhere/second.git") + + result = validate(holder, "--no-members") + + assert codes(result) == ["gitmodules-path-twice", + "gitmodules-path-unnamed", + "gitmodules-path-unnamed"], result.stderr + first, second = findings(result)[1:] + assert commands(first) == [f"git rm --cached -- {TOOLS}", + *removal(TOOLS, "path", "url")] + assert commands(second) == removal("second", "path", "url") + env = isolated(offline, tmp_path) + for finding in (first, second): + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) + + +# --- the controls ------------------------------------------------------------ + +def test_the_holder_as_made_passes_and_a_plain_clone_checks_it_out( + family, holder, offline, tmp_path): + """CONTROL: a member under `members/` with its section and its row, + two of them, as `init` and `add` made the holder.""" + assert_passes_in_every_mode(holder, 2) + assert_a_plain_clone_checks_out_every_member( + holder, fetching(isolated(offline, tmp_path)), family, tmp_path) + + +@pytest.mark.parametrize("recorded", [False, True], + ids=["no-gitlink", "gitlink-removal-staged"]) +def test_a_section_with_no_gitlink_in_the_index_prints_its_lines_alone( + family, holder, offline, tmp_path, recorded): + """CONTROL: a section no row names, at tools/x, where the index holds + no gitlink, prints what it printed before, its `--unset-all` lines + alone, in every mode: there is no gitlink, or one HEAD records whose + removal is staged, where a `git rm` fails. Typed as printed, staged + and committed, the gate passes in every mode.""" + env = isolated(offline, tmp_path) + if recorded: + stage_gitlink(holder, TOOLS, spare_commits(family)[1]) + with_section(holder, tmp_path, TOOLS, TOOLS, "/nowhere/x.git") + if recorded: + git("rm", "-q", "--cached", "--", TOOLS, cwd=holder) + + finding = found_in_every_mode(holder, "gitmodules-path-unnamed") + + assert finding == ( + SECTION_SAID.format(path=TOOLS, name=TOOLS, url="url /nowhere/x.git") + + SECTION_REMEDY + "".join(f"\n {line}" for line in removal( + TOOLS, "path", "url"))) + typed(holder, env, finding) + staged_and_committed(holder, env, "Typed as printed") + assert_passes_in_every_mode(holder, 2) diff --git a/tests/test_validate_family_unnamed_mounts.py b/tests/test_validate_family_unnamed_mounts.py index db0d4ed3..23defc41 100644 --- a/tests/test_validate_family_unnamed_mounts.py +++ b/tests/test_validate_family_unnamed_mounts.py @@ -421,16 +421,19 @@ def test_a_gitlink_is_read_from_the_index_first_and_from_head_second( def test_only_members_is_asked_about(family, holder): """`members/` in any case, and nothing beside it: a gitlink elsewhere - in the holder is not a member's mount, and no row could name it.""" + in the holder is not a member's mount, and no row could name it. One + no section gives is `gitlink-unnamed` instead, after this finding + (#878, `tests/test_family_unnamed_gitlink.py` types it).""" _, tip = spare_commits(family) stage_gitlink(holder, f"tools/{SPARE}", tip) stage_gitlink(holder, f"membersX/{SPARE}", tip) stage_gitlink(holder, f"MEMBERS/{SPARE}", tip) result = validate(holder, "--no-members") assert result.returncode == 1, result.stderr + result.stdout - (gitlink,) = findings(result) - assert gitlink.startswith(GITLINK_SAID.format(path=f"MEMBERS/{SPARE}", - commit=tip)) + assert codes(result) == ["member-gitlink-unnamed", "gitlink-unnamed", + "gitlink-unnamed"], result.stderr + assert findings(result)[0].startswith(GITLINK_SAID.format( + path=f"MEMBERS/{SPARE}", commit=tip)) def test_a_removed_row_is_named_by_its_gitlink_and_its_section(holder):