diff --git a/scripts/family.py b/scripts/family.py index 372e3b92..c594ecec 100755 --- a/scripts/family.py +++ b/scripts/family.py @@ -3419,30 +3419,44 @@ def _index_entries(root: Path, path: str) -> list[tuple[str, str]]: #: #363, #364). IN_INDEX = "in the index" IN_HEAD_ONLY = "in HEAD and no longer in the index" +#: A section of `.gitmodules` that gives a member another spelling of its +#: row's path, where HEAD records the member's gitlink at that path and the +#: index holds it nowhere: its removal staged (#761, rv685's candidate 1). +IN_GITMODULES = "in .gitmodules, its gitlink's removal staged" ON_DISK = "on disk and untracked" #: What the remedy does to a path `_members_spellings` found, in the order #: `_spellings_refusal` says them. What HEAD holds and the index no longer #: does is a pending change, committed first. What the index holds is renamed #: with `git mv` where it is on disk in its own spelling, with `git mv -#: --sparse` where a sparse checkout keeps it off disk, and otherwise taken -#: out of the index, which touches no file, as #318 takes out a variant of -#: the mount path. A `members/` renamed with `git mv` is renamed back with -#: it, whole, in two steps through a free name (#685). A member's own -#: gitlink, held only under another spelling of its row's path, is moved -#: back to that path in the index alone (#646). What is on disk and -#: untracked is moved out of the way, and a `members/` a hand renamed is -#: moved back. +#: --sparse` where the sparse rules leave it out, and otherwise taken out of +#: the index, which touches no file, as #318 takes out a variant of the +#: mount path. A `members/` renamed with `git mv` is renamed back with it, +#: whole, in two steps through a free name (#685). A member's own gitlink, +#: held only under another spelling of its row's path, is moved back to that +#: path in the index alone (#646), after what the index holds at or under +#: that path, or at `members` itself, is set aside (#761), with `git mv`, +#: `git mv --sparse` or `git rm -r --cached` as the same rule names what the +#: index holds (Codex's P2 1 on #828). What is on disk and untracked is +#: moved out of the way, but for a checkout, which a move would leave +#: pointing nowhere, and which is deleted by hand (#761); a `members/` a +#: hand renamed is moved back. COMMIT = "git commit" RENAME = "git mv" RENAME_SPARSE = "git mv --sparse" UNSTAGE = "git rm -r --cached" RENAME_BACK = "git mv back" +ASIDE = "git mv aside" +ASIDE_SPARSE = "git mv --sparse aside" +ASIDE_OUT = "git rm -r --cached aside" RESTORE = "git update-index" MOVE = "mv" +CHECKOUT = "by hand" MOVE_BACK = "mv back" -REMEDY_ORDER = (COMMIT, RENAME, RENAME_SPARSE, UNSTAGE, RENAME_BACK, RESTORE, - MOVE, MOVE_BACK) +REMEDY_ORDER = (COMMIT, RENAME, RENAME_SPARSE, UNSTAGE, RENAME_BACK, ASIDE, + ASIDE_SPARSE, ASIDE_OUT, RESTORE, MOVE, CHECKOUT, MOVE_BACK) +#: Each remedy that sets aside what is in the way of a move back (#761). +SET_ASIDE = (ASIDE, ASIDE_SPARSE, ASIDE_OUT) #: What a `--sparse` the remedy prints needs (#685). `git rm --sparse` came #: with git 2.34. `git mv` took `--sparse` in 2.34 too, but there it still @@ -3458,6 +3472,21 @@ def _index_entries(root: Path, path: str) -> list[tuple[str, str]]: #: is still off disk, and in a cone-mode sparse checkout `git rm --cached` #: then still removes nothing. An older git may differ (2.33's `mv.c` has #: no sparse code at all), but none was run. +#: +#: THE RULES, NOT THE BIT (#761, rv685's candidate 2). git 2.43 asks the +#: sparse-checkout rules of each path it updates, whatever its +#: `skip-worktree` bit says: an entry the rules leave out that is on disk +#: with the bit cleared is refused by a plain `git mv` (`exist outside of +#: your sparse-checkout definition`, exit 1), file or directory, and taken +#: by `git mv --sparse`; one off disk with the bit cleared is refused by +#: `git rm --cached` the same way and taken by `git rm --cached --sparse`; +#: and `git rm --cached --ignore-unmatch` passes over such a gitlink with +#: exit 0, as it does one marked. So each `--sparse` is printed where the +#: rules leave the entry out (`_sparse_rules_exclude`), as well as where it +#: is marked. A gitlink on disk as a directory is the one entry plain `git +#: mv` takes there, and `--sparse` takes it too. Measured on git 2.43. A git +#: that cannot answer `check-rules` leaves the bit alone to decide, as +#: before. SPARSE_MV_GIT = "which needs git 2.38 or later" SPARSE_RM_GIT = "which needs git 2.34 or later" @@ -3471,19 +3500,40 @@ class Spelled(NamedTuple): itself, or at or under a case variant of the mount path, as `_members_spellings` found it: `name` as git or the disk spells it, `kind` (`a file`, `a submodule`, `a symlink`, `a directory`), `where` - (`IN_INDEX`, `IN_HEAD_ONLY` or `ON_DISK`), `target`, the path its remedy - acts on, `mount`, whether it is at or under a case variant of the mount - path itself, `remedy`, which of `REMEDY_ORDER` acts on it, and - `committed`, whether HEAD holds it too, so that taking it out of the - index is a change to commit. + (`IN_INDEX`, `IN_HEAD_ONLY`, `IN_GITMODULES` or `ON_DISK`), `target`, + the path its remedy acts on, `mount`, whether it is at or under a case + variant of the mount path itself, `remedy`, which of `REMEDY_ORDER` acts + on it, and `committed`, whether HEAD holds it too, so that taking it out + of the index is a change to commit. `sparse` says, for `UNSTAGE`, that + the sparse rules leave it out, so that `git rm --cached` takes it only + with `--sparse` (#761). A MEMBER'S OWN GITLINK (`RESTORE`, #646) has its row's path as `target`, `committed` says whether the move leaves anything to commit, `recorded` is the commit it records, at which the move records it back, and - `sparse` whether a sparse checkout keeps it off disk (`skip-worktree`), - which `git rm --cached --ignore-unmatch` passes over without a word - unless it is given `--sparse` too; `recorded` is empty, and `sparse` - false, for every other remedy. + `sparse` whether a sparse checkout keeps it off disk (`skip-worktree`) + or its rules leave it out, either of which `git rm --cached + --ignore-unmatch` passes over without a word unless it is given + `--sparse` too; `recorded` is empty, and `sparse` false, for every other + remedy but `UNSTAGE` and `ASIDE_OUT`. `sections` names each section of + `.gitmodules` that gives the member another spelling of its row's path, + as a `git mv` of its mount leaves it, which the move points back at that + path (#761); empty for every other remedy. One found `IN_GITMODULES` is + a member HEAD records at its row's path and the index holds nowhere, + named by the path its section gives, and moved back at HEAD's commit. + `forced` says the removal takes `-f`: the index stages the variant at a + commit HEAD does not record there, and a checkout is on disk at it, + which `git rm --cached` refuses wherever that checkout has moved on or + holds a file it does not track (#761). + + WHAT IS SET ASIDE BEFORE THE MOVE BACK (`SET_ASIDE`, #761) is an entry + the index holds at or under a row's path, or at `members` itself, which + `git update-index` would replace or fail on: renamed with `git mv` + (`ASIDE`) where it is on disk as the index holds it, with `git mv + --sparse` (`ASIDE_SPARSE`) where a sparse checkout needs that, else + taken out of the index (`ASIDE_OUT`), `sparse` saying that its rules + leave it out, so that `git rm --cached` takes it only with `--sparse` + (Codex's P2 1 on #828); `target` is the entry itself. WHAT A `members/` RENAMED WITH `git mv` HOLDS (`RENAME_BACK`, #685) has that spelling as `target`, `committed` says whether renaming it back @@ -3504,6 +3554,8 @@ class Spelled(NamedTuple): sparse: bool = False through: str = "" put_back: bool = False + sections: tuple[str, ...] = () + forced: bool = False def _folded(name: str) -> str: @@ -3677,6 +3729,8 @@ def _disk_spellings( gitlink `members/`), it is `members/` a hand renamed (Codex's P2 on #574), and it is moved BACK; only a case variant of the mount path in it is moved out of the way first. + - An entry to be moved that is a directory holding a `.git` FILE is a + CHECKOUT, never moved (`_moved_or_kept`, #761). """ found: list[tuple[str, str, str]] = [] moved_back = False @@ -3684,7 +3738,8 @@ def _disk_spellings( if name in (canon, MEMBERS_DIR): if kind == "a directory": found.extend((f"{name}/{entry}", - _disk_kind(root / name / entry), MOVE) + _disk_kind(root / name / entry), + _moved_or_kept(root / name / entry)) for entry in entries or () if _folded(entry) == _folded(project) and entry != project) @@ -3697,7 +3752,8 @@ def _disk_spellings( and os.path.lexists(root / name / entry / ".git") for entry in entries) moved_back = moved_back or back - found.extend((f"{name}/{entry}", _disk_kind(root / name / entry), MOVE) + found.extend((f"{name}/{entry}", _disk_kind(root / name / entry), + _moved_or_kept(root / name / entry)) for entry in entries if not back or _folded(entry) == _folded(project)) if back: @@ -3705,6 +3761,29 @@ def _disk_spellings( return found +def _moved_or_kept(path: Path) -> str: + """`CHECKOUT` for a directory, not a link, that holds a `.git` FILE, a + checkout whose store that file names by a path relative to it, as `git + submodule add` and `git mv` write one; `MOVE` for anything else (#761). + + A PLAIN `mv` LEAVES SUCH A CHECKOUT POINTING NOWHERE (#681's N-1, + measured for the validator with plain git): moved out of the holder, + its `.git` file names `../../.git/modules/` from a place where + that is no store, and `git -C status` says `fatal: not a + git repository`, so `` cannot be filled in to keep it. Its + commits are in that store whatever becomes of it, so it is deleted by + hand once nothing only it holds is needed. A clone of its own, whose + `.git` is a directory, holds its commits there and is moved whole by + the `mv`, which keeps them.""" + try: + if stat.S_ISLNK(path.lstat().st_mode) or not path.is_dir(): + return MOVE + return CHECKOUT if stat.S_ISREG( + (path / ".git").lstat().st_mode) else MOVE + except OSError: + return MOVE + + def _tree_entries(root: Path, tree: str, recursive: bool = False) -> list[tuple[str, str, str]]: """`[(name, kind, object)]` of the tree object `tree`, by `git ls-tree @@ -3771,13 +3850,15 @@ def _free_name(root: Path, listed: dict[str, list]) -> str: def _renamed_with_git(root: Path, found: list[Spelled], - listed: dict[str, list], canon: str | None + listed: dict[str, list], canon: str | None, + excluded: frozenset[str] = frozenset() ) -> tuple[str, str, frozenset[str]] | None: """`(the spelling, the free name, the entries to put back first)` where `members/` was renamed to that spelling with `git mv`, and what - `_members_spellings` found in the index is that and nothing else - (#685), or None. `listed` is every index entry's records, and `canon` - is `_members_dir_on_disk`'s answer. + `_members_spellings` found in the index under it is that (#685), or + None. `listed` is every index entry's records, `canon` is + `_members_dir_on_disk`'s answer, and `excluded` the index entries the + sparse rules leave out (#761). `git mv` of `members/` renames each member's checkout, its gitlink and its section of `.gitmodules` (`path = Members/

`), staged or @@ -3792,12 +3873,21 @@ def _renamed_with_git(root: Path, found: list[Spelled], disk refuses. It is the remedy where all of these hold: - The index holds nothing at or under `members` itself. - - Everything the index holds under another spelling is under this one - spelling, none of it a case variant of the mount path, and some of it - is a member's own gitlink (`RESTORE`), which nothing at that spelling - itself can sit beside. + - Every member's own gitlink (`RESTORE`) the index holds under another + spelling is under this one spelling, which nothing at that spelling + itself can sit beside, and nothing the index holds under another + spelling is a case variant of the mount path. What it holds under a + THIRD spelling keeps its own remedy, which runs first, in + `REMEDY_ORDER` (#761, rv685's NIT 5): on Linux a `git mv` of it, or + its removal from the index, touches nothing under this spelling, and + on macOS and Windows, where the three are one directory, it is never + on disk in its own spelling (`_on_disk_as_named` reads the listing), + so it is taken out of the index, which touches no file, and its file + goes back with the rest. - HEAD holds no member's gitlink under another spelling that the index - no longer does, which the rename would leave out. + no longer does, and no section of `.gitmodules` gives one whose + removal is staged (`IN_GITMODULES`, #761), which the rename would + leave out, its section naming this spelling (rv685's candidate 1). - No `members/` is on this disk, or the one there is that spelling's own directory (`samefile`), as on macOS and Windows: a second directory would take the second step into it. @@ -3808,8 +3898,10 @@ def _renamed_with_git(root: Path, found: list[Spelled], leaves the holder half renamed (#685's review, MUST-B). - Every entry under it is at stage 0 (`git mv` says `source directory is empty` for an unmerged gitlink, and `conflicted` for a file), and - no sparse checkout keeps one off disk, which `git mv` refuses without - `--sparse`. + no sparse checkout keeps one off disk, nor do its rules leave one out + whatever its bit says, either of which `git mv` refuses without + `--sparse` (#761: `exist outside of your sparse-checkout definition`, + exit 1, measured on git 2.43 for an entry on disk, its bit cleared). - Every entry under it is on disk as the index holds it, as the disk resolves the name, or can be put back first: `git mv` of a directory says `bad source` for any entry under it that is not. A file is put @@ -3820,12 +3912,12 @@ def _renamed_with_git(root: Path, found: list[Spelled], for name in listed): return None held = [spelled for spelled in found if spelled.where == IN_INDEX] - spellings = {spelled.name.partition("/")[0] for spelled in held} + spellings = {spelled.name.partition("/")[0] for spelled in held + if spelled.remedy == RESTORE} if (len(spellings) != 1 - or not any(spelled.remedy == RESTORE for spelled in held) or any(spelled.mount for spelled in held) or any(spelled.remedy == RESTORE for spelled in found - if spelled.where == IN_HEAD_ONLY)): + if spelled.where in (IN_HEAD_ONLY, IN_GITMODULES))): return None (spelling,) = spellings if os.path.islink(root / spelling) or ( @@ -3845,7 +3937,7 @@ def _renamed_with_git(root: Path, found: list[Spelled], sections: set[str] | None = None put_back = set() for name in under: - if name in skipped: + if name in skipped or name in excluded: return None here = root / name gitlink = listed[name][0][0] == b"160000" @@ -3889,24 +3981,64 @@ def _members_spellings(root: Path, project: str, the row's path, at the commit it records there, by `git update-index`, which reads no disk, as `validate-family.py` gives the same move (#611). An unmerged one is recorded at its stage 2, and is no member's own where - it has no gitlink there. Where the index does hold something at or - under the row's path, a gitlink at a variant is a second one, which - taking out leaves the member where it is, and the move would replace a - file there or fail on one under it: it keeps the remedy below. + it has no gitlink there. Where the index holds a gitlink at or under + the row's path, a gitlink at a variant is a second one, which taking + out leaves the member where it is: it keeps the remedy below. + + WHAT IS IN THE WAY OF THE MOVE IS SET ASIDE FIRST (#761, #670's (g) and + (h)). `git update-index --add --cacheinfo` of the row's path replaces a + file the index holds there without a word, and fails on one under it, + or at `members` itself (`appears as both a file and as a directory`, + exit 128, after the removal took the variant out), and #574's remedy + takes the member's gitlink out instead. So where the index holds only + files and links at or under a row's path, each at stage 0, a gitlink at + a variant is still the member's own, and each of those entries is set + aside first (`SET_ASIDE`, `_set_aside`), as is a file or link the index + holds at `members` itself wherever a gitlink is moved back: renamed with + `git mv` where it is on disk as the index holds it, as + `_members_is_a_file` moves `members` itself, else taken out of the + index, each with `--sparse` where a sparse checkout needs it. WHOSE + GITLINK IT IS NEVER TURNS ON THE SPARSE CHECKOUT (Codex's P2 1 on #828): + a row whose file it marked or left out was dropped here, so the gitlink + kept #574's remedy, which took it away from its row. The other order + fails: measured, the move back first replaced the file at the row's + path, and the `git mv` of that path then took the member's gitlink + away. + + ITS SECTION OF `.gitmodules` IS POINTED BACK TOO (#761, #670's (e)). A + `git mv` of a member's mount (`git mv members/

Members/

`) moves + the section's `path` with it, which the move back of the gitlink leaves + naming the variant: `git submodule update` then says `No url found for + submodule path 'members/

'`. So each section that gives a member + another spelling of its row's path is named on the member's `RESTORE` + (`sections`), from the file `_submodule_paths` reads, and the move is a + change to commit where HEAD's section gives another path. + + A MEMBER WHOSE GITLINK'S REMOVAL IS STAGED (#761, rv685's candidate 1): + HEAD records it at its row's path, the index holds it nowhere, and a + section of `.gitmodules` gives it another spelling, as a `git mv + members Members` followed by `git rm --cached Members/

` leaves it. + Nothing in the index or HEAD is under another spelling for it, so the + rename back passed it by and left its section naming `Members/

`. It + is `IN_GITMODULES`, moved back at HEAD's commit, with its section, and + the rename back is not printed (`_renamed_with_git`). A `members/` RENAMED WITH `git mv` (#685), staged or committed, is renamed back with it, whole, in two steps through a free name (`RENAME_BACK`, `_renamed_with_git`), in place of the move back, which would leave each member's section of `.gitmodules` naming the other spelling; what the disk holds under that spelling goes back with it, so - it is not named. + it is not named. What the index holds under a third spelling keeps its + own remedy (#761). THE INDEX, the whole of it, as `_index_entries` reads it (#362). Each entry's remedy acts on `_spelled_target`'s path: renamed with `git mv` where that path is on disk in its own spelling (`_on_disk_as_named`) and is not `members/` itself under another spelling; renamed with `git mv - --sparse` where a sparse checkout keeps the entry off disk; else taken - out of the index, which touches no file. THEN HEAD, for what it holds + --sparse`, entry by entry, where a sparse checkout keeps the entry off + disk, or its rules leave it out whatever its bit says (#761); else taken + out of the index, which touches no file, with `--sparse` where the rules + leave it out. THEN HEAD, for what it holds that the index no longer does, a `git rm -r Members` not committed yet (#363): the add's own commit names three paths and nothing else, so it would keep that file beside the member, and with `core.ignorecase` set @@ -3929,18 +4061,14 @@ def _members_spellings(root: Path, project: str, if first == MEMBERS_DIR and rest) head = _head_spellings(root) in_head = {name: oid for name, _, oid in head} - # Each row's path the index holds nothing at or under, by its folded - # spelling: a gitlink at another spelling of it is that member's own. - # The move would replace a file there without a word, and a file under - # it would stop the move's second line after the first had taken the - # variant out (`appears as both a file and as a directory`; #646's - # review, N3). - strays = {_folded(row): row for row in rows - if not any(name == row or name.startswith(row + "/") - for name in indexed)} + gitlink = INDEX_ENTRY_KINDS[0o160000] + sparse = _SparseAsked(root, indexed) + marked, left_out = sparse.marked, sparse.left_out + strays, blocked, at_members = _rows_held(rows, indexed, listed) + owners = {**strays, **{_folded(row): row for row in blocked}} found: list[Spelled] = [] restored: list[str] = [] - sparse: set[str] | None = None + moved: set[str] = set() for name, mode in indexed.items(): spelled = _spelled_target(name, project, occupied) if spelled is None: @@ -3953,52 +4081,72 @@ def _members_spellings(root: Path, project: str, # member's own. ours = next((fields for fields in listed[name] if fields[2] in (b"0", b"2")), None) - own = (strays.get(_folded(name)) + own = (owners.get(_folded(name)) if ours is not None and ours[0] == b"160000" else None) if own is not None: # The move leaves nothing to commit only where HEAD holds no # such variant and records the gitlink at the row's path # already, at this commit: a move to the variant, staged alone. recorded = ours[1].decode("ascii") - if sparse is None: - sparse = _skip_worktree(root) - found.append(Spelled(_shown(name), kind, IN_INDEX, _shown(own), - mount, RESTORE, - name in in_head or in_head.get(own) - != recorded, recorded, name in sparse)) + moved.add(own) + found.append(Spelled( + _shown(name), kind, IN_INDEX, _shown(own), mount, RESTORE, + name in in_head or in_head.get(own) != recorded, recorded, + name in marked() or name in left_out(), + forced=in_head.get(name) != recorded + and os.path.lexists(root / name / ".git"))) continue + out = False if target != canon and _on_disk_as_named( root, target, target != name or mode == 0o160000): - remedy = RENAME + # On disk, and left out by the rules whatever its bit says: + # plain `git mv` refuses it (#761). + remedy = RENAME_SPARSE if name in left_out() else RENAME else: - if sparse is None: - sparse = _skip_worktree(root) - remedy = RENAME_SPARSE if name in sparse else UNSTAGE - target = name if name in sparse else target + remedy = RENAME_SPARSE if name in marked() else UNSTAGE + out = remedy == UNSTAGE and name in left_out() + target = name if remedy == RENAME_SPARSE else target found.append(Spelled(_shown(name), kind, IN_INDEX, _shown(target), - mount, remedy, name in in_head)) + mount, remedy, name in in_head, sparse=out)) for name, kind, oid in head: spelled = _spelled_target(name, project, occupied) if spelled is None or name in indexed: continue - own = (strays.get(_folded(name)) - if kind == INDEX_ENTRY_KINDS[0o160000] else None) + own = owners.get(_folded(name)) if kind == gitlink else None if own is not None: restored.append(name) + moved.add(own) found.append(Spelled(_shown(name), kind, IN_HEAD_ONLY, _shown(own), spelled[1], RESTORE, True, oid)) continue found.append(Spelled(_shown(name), kind, IN_HEAD_ONLY, _shown(spelled[0]), spelled[1], COMMIT, True)) - renamed = _renamed_with_git(root, found, listed, canon) + # A member HEAD records at its row's path, the index nowhere, and a + # section gives another spelling of (#761, rv685's candidate 1), in + # sorted order: `rows` is a set, so the text would follow its hash. + staged_out = [row for row in sorted(strays.values()) if row not in moved + and row in in_head and (row, gitlink, in_head[row]) in head] + staged, gone = _sections_restored(root, found, moved, staged_out, + in_head, project) + found.extend(staged) + # What the disk holds where its section says is the move's to name too. + restored.extend(gone) + found.extend(_in_the_way(root, indexed, head, moved, + (blocked, at_members), sparse)) + renamed = _renamed_with_git(root, found, listed, canon, left_out() + if any(spelled.remedy == RESTORE + for spelled in found) else frozenset()) if renamed is not None: spelling, through, missing = renamed gone = {_shown(name) for name in missing} found = [spelled._replace(target=spelling, remedy=RENAME_BACK, recorded="", sparse=False, through=through, - put_back=spelled.name in gone) - if spelled.where == IN_INDEX else spelled + put_back=spelled.name in gone, + sections=(), forced=False) + if spelled.where == IN_INDEX + and spelled.name.startswith(_shown(spelling) + "/") + else spelled for spelled in found] # What the disk holds at a gitlink moved back from HEAD is the move's to # name, as what it holds at an index entry is the index's, and what it @@ -4006,7 +4154,8 @@ def _members_spellings(root: Path, project: str, for path, kind, remedy in _disk_spellings(root, on_disk, project, canon, gitlinks): if any(name == path or path.startswith(name + "/") - or (remedy == MOVE and name.startswith(path + "/")) + or (remedy in (MOVE, CHECKOUT) + and name.startswith(path + "/")) for name in [*indexed, *restored, *([renamed[0]] if renamed else [])]): continue @@ -4015,6 +4164,220 @@ def _members_spellings(root: Path, project: str, return found +class _SparseAsked: + """What a sparse checkout marks (`skip-worktree`), and what its rules + leave out of the index entries under a spelling of `members` (#761), + each asked of git once, and only where it matters.""" + + def __init__(self, root: Path, indexed: dict[str, int]) -> None: + self.root = root + self.names = [name for name in indexed + if _folded(name.partition("/")[0]) == MEMBERS_DIR] + self._marked: frozenset[str] | None = None + self._out: frozenset[str] | None = None + + def marked(self) -> frozenset[str]: + """The entries a sparse checkout marks (`_skip_worktree`).""" + if self._marked is None: + self._marked = frozenset(_skip_worktree(self.root)) + return self._marked + + def left_out(self) -> frozenset[str]: + """The entries its rules leave out (`_sparse_rules_exclude`).""" + if self._out is None: + self._out = frozenset(_sparse_rules_exclude(self.root, + self.names)) + return self._out + + +def _in_the_way(root: Path, indexed: dict[str, int], + head: list[tuple[str, str, str]], moved: set[str], + held: tuple[dict[str, list[str]], str | None], + sparse: _SparseAsked) -> list[Spelled]: + """What is in the way of the move back of the gitlinks to the rows in + `moved`, set aside first (`_set_aside`, #761): what the index holds at + or under each of those rows, and a file or link at `members`, as + `_rows_held` found them (`held`), wherever a gitlink is moved back, and + nothing where none is. `head` is `_head_spellings`'s. HEAD's gitlink at + a row's path, which the move puts back, is no change for the removal of + what stands there to leave to commit.""" + blocked, at_members = held + if not moved: + return [] + names = [name for row in sorted(moved & blocked.keys()) + for name in blocked[row]] + ([at_members] if at_members else []) + if not names: + return [] + gitlink = INDEX_ENTRY_KINDS[0o160000] + kept = {name: oid for name, kind, oid in head + if kind != gitlink or name not in moved} + return _set_aside(root, indexed, kept, names, sparse.marked(), + sparse.left_out()) + + +def _rows_held(rows: frozenset[str], indexed: dict[str, int], + listed: dict[str, list] + ) -> tuple[dict[str, str], dict[str, list[str]], str | None]: + """`(strays, blocked, at_members)` for `_members_spellings`, from the + index's entries by mode (`indexed`) and their records (`listed`): + + - each row's path the index holds nothing at or under, by its folded + spelling, where a gitlink at another spelling of it is that member's + own (#646); + - each row's path the index holds files and links alone at or under, + each at stage 0, with those entries, which are set aside before the + move back (#761); + - `members` where the index holds a file or link there at stage 0, + which no `members/

` can sit beside, else None. + + Whatever a sparse checkout says of an entry in the way, the gitlink is + still the member's own: `_set_aside` gives the entry the `--sparse` git + needs (Codex's P2 1 on #828).""" + def merged_file(name: str) -> bool: + """A file or link the index holds at stage 0 alone.""" + return indexed[name] != 0o160000 and all( + fields[2] == b"0" for fields in listed[name]) + + strays: dict[str, str] = {} + blocked: dict[str, list[str]] = {} + for row in rows: + held = [name for name in indexed + if name == row or name.startswith(row + "/")] + if not held: + strays[_folded(row)] = row + elif all(merged_file(name) for name in held): + blocked[row] = held + at_members = (MEMBERS_DIR if MEMBERS_DIR in indexed + and merged_file(MEMBERS_DIR) else None) + return strays, blocked, at_members + + +def _sections_restored(root: Path, found: list[Spelled], moved: set[str], + staged_out: list[str], in_head: dict[str, str], + project: str) -> tuple[list[Spelled], list[str]]: + """Each section of `.gitmodules` that gives a member moved back another + spelling of its row's path, named on the member's `RESTORE` in `found` + (which this updates in place), and an `IN_GITMODULES` move for each row + in `staged_out`, whose gitlink HEAD records at its path and the index + holds nowhere, where a section gives it another spelling: returned, with + the path each such section gives, as git spells it (#761). `moved` is + the rows a `RESTORE` in `found` moves back to, which this adds those + rows to. + + The file is the one `_submodule_paths` reads, the work tree's where + there is one, which is the one `git config -f .gitmodules` edits; one + git cannot read names no section here, and is refused on its own once + the move is done (`gitmodules-unreadable`). HEAD's copy says whether + pointing a section back is a change to commit: it is where HEAD's + section gives another path, as after a committed `git mv`, and not + where only a staged one moved it.""" + if not moved and not staged_out: + return [], [] + try: + given = _submodule_paths(root) + except Refusal: + return [], [] + head = _head_section_paths(root) + + def giving(row: str) -> tuple[str, ...]: + return tuple(sorted( + name for name, (path, _, _) in given.items() + if path != row and _folded(path) == _folded(row))) + + added: list[Spelled] = [] + gone: list[str] = [] + for row in staged_out: + variants = sorted({given[name][0] for name in giving(row) + if _spelled_target(given[name][0], project) + is not None}) + if variants: + moved.add(row) + gone.extend(variants) + added.append(Spelled( + _shown(variants[0]), INDEX_ENTRY_KINDS[0o160000], + IN_GITMODULES, _shown(row), + _spelled_target(variants[0], project)[1], RESTORE, False, + in_head[row])) + for index, spelled in enumerate([*found, *added]): + if spelled.remedy != RESTORE: + continue + row = next(row for row in moved if _shown(row) == spelled.target) + names = giving(row) + updated = spelled._replace( + sections=names, committed=spelled.committed or any( + head.get(name) != row for name in names)) + if index < len(found): + found[index] = updated + else: + added[index - len(found)] = updated + return added, gone + + +def _head_section_paths(root: Path) -> dict[str, str]: + """`{name: path}` of every section of HEAD's `.gitmodules` that gives a + path, read as `_submodule_paths` reads a copy (`git config --blob`), or + none where HEAD names no commit, holds no `.gitmodules`, or holds one git + cannot read (#761).""" + try: + blob = git_out(["rev-parse", "-q", "--verify", f"HEAD:{GITMODULES}"], + cwd=root) + listed = git_out(["config", "-z", "--blob", blob, "--list"], + cwd=root, binary=True) + except Refusal: + return {} + found: dict[str, str] = {} + for record in listed.split(b"\0"): + key, _, value = record.decode("utf-8", + "surrogateescape").partition("\n") + section, _, variable = key.rpartition(".") + if section.startswith("submodule.") and variable == "path": + found[section[len("submodule."):]] = value + return found + + +def _set_aside(root: Path, indexed: dict[str, int], in_head: dict[str, str], + names: list[str], marked: frozenset[str], + left_out: frozenset[str]) -> list[Spelled]: + """One of `SET_ASIDE` for each of `names`, an entry the index holds at + or under a row's path that a gitlink is moved back to, or at `members` + itself (#761), by the rule `_members_spellings` names what the index + holds by; `in_head` is what HEAD holds that the move back does not put + back, `marked` what a sparse checkout marks (`skip-worktree`), and + `left_out` what its rules leave out. Renamed with `git mv` where it is + on disk as the index holds it, a file or link and no directory, or with + `git mv --sparse` where the rules leave it out; off disk, renamed with + `git mv --sparse` where it is marked, which keeps what it holds, and + else taken out of the index, which touches no file, with `--sparse` + where the rules leave it out. A rename leaves a change to commit, and a + removal one where `in_head` holds the entry. HEAD's gitlink at the row's + path is not in it: a file staged there over a staged `git mv` of the + mount and deleted from disk was said to leave its removal to commit, + and the commit printed failed with nothing to commit (found in #828's + fix round for Codex's P2s). + + EACH `--sparse` IS ONE GIT ASKS FOR (Codex's P2 1 on #828), measured on + git 2.43 and 2.47.1 for a file under a row's path. Marked and on disk, + plain `git mv` takes it. Marked and off disk, or left out by the rules + with its bit cleared and on disk, plain `git mv` and `git rm --cached` + refuse it (`exist outside of your sparse-checkout definition`, exit 1), + and `git mv --sparse` takes it. Left out and off disk, `git rm --cached` + refuses it the same way, `git mv` with or without `--sparse` says `bad + source` (exit 128), and `git rm --cached --sparse` takes it.""" + aside = [] + for name in names: + on = _on_disk_as_named(root, name, False) + if on: + remedy = ASIDE_SPARSE if name in left_out else ASIDE + else: + remedy = ASIDE_SPARSE if name in marked else ASIDE_OUT + aside.append(Spelled( + _shown(name), INDEX_ENTRY_KINDS.get(indexed[name], "a file"), + IN_INDEX, _shown(name), False, remedy, + remedy != ASIDE_OUT or name in in_head, + sparse=remedy == ASIDE_OUT and name in left_out)) + return aside + + #: How the work tree's `.gitmodules` stands beside the index's copy, as #: `_gitmodules_beside` answers it (#685). GITMODULES_MODIFIED = "modified" @@ -4047,7 +4410,7 @@ def _gitmodules_beside(root: Path) -> str: #: The order `_spellings_refusal` names its groups in. Its remedies run in #: `REMEDY_ORDER`: a pending change in HEAD is committed first. -SPELLED_ORDER = (IN_INDEX, IN_HEAD_ONLY, ON_DISK) +SPELLED_ORDER = (IN_INDEX, IN_HEAD_ONLY, IN_GITMODULES, ON_DISK) def _spellings_refusal(path: str, found: list[Spelled], @@ -4070,11 +4433,22 @@ def _spellings_refusal(path: str, found: list[Spelled], --cached` for what the index holds, and a commit; then, for a `members/` renamed with `git mv` (#685), what puts back an entry under it that the disk lacks, the `git mv` of it to a free name and of that - to `members`, and a commit; then the move of a member's own gitlink back - to its row's path, and a commit; then `mv` for what is on disk and - untracked, and `mv` back for a renamed `members/`. Every path is quoted + to `members`, and a commit; then what is in the way of the move back set + aside (#761), the move of a member's own gitlink back to its row's path + with its section of `.gitmodules`, and a commit; then `mv` for what is + on disk and untracked, a checkout there left to be deleted by hand + (#761), and `mv` back for a renamed `members/`. Every path is quoted for a POSIX shell (#532). A `--sparse` says the git it needs - (`SPARSE_MV_GIT`, `SPARSE_RM_GIT`, #685). + (`SPARSE_MV_GIT`, `SPARSE_RM_GIT`, #685), and is printed where the + sparse rules leave an entry out as well as where they mark it (#761). + + AN ENTRY IN THE WAY OF THE MOVE BACK (#761), at or under a row's path + or at `members` itself, which `git update-index` would replace or fail + on, is set aside first, in no commit of its own: the move's commit + takes both, so the history never holds the member's gitlink beside the + file, nor the file gone and the gitlink not back. Its `git mv --sparse` + and `git rm --cached --sparse` are said as the index's are (Codex's P2 + 1 on #828). THE WORK TREE'S `.gitmodules` (`gitmodules`, `_gitmodules_beside`'s answer, which `_checked_mount_path` passes only where the index holds a @@ -4098,20 +4472,43 @@ def _spellings_refusal(path: str, found: list[Spelled], without it git passes that one over, exit 0, and the index ends with two gitlinks; a `git update-index --add --cacheinfo` per row, at the commit the variant records, the first found where a row has two, the index's - before HEAD's; and the commit, where it leaves one to make. Its closing - sentences, after every other remedy, so that a `members/` moved back has - taken the checkout with it, are the validator's too: where nothing is - checked out at the row's path, as in a fresh clone, `git submodule - update --init --recursive` checks the member out there; on a - case-sensitive disk the directory left at the variant is moved out of - the holder, an `mv` with nothing to fill in, or this refusal names it - on the next run; on macOS and Windows it is the row's own directory, - and is left. The text is the same on every disk. + before HEAD's; a `git config -f .gitmodules submodule..path ` per section that gives another spelling of it, and `git add -- + .gitmodules` (#761); and the commit, where it leaves one to make. A + member whose removal is staged (`IN_GITMODULES`) is put back at HEAD's + commit in the same line. + + ITS CLOSING SENTENCES, after every other remedy, so that a `members/` + moved back has taken the checkout with it, are the validator's + (`_wrong_case_move`, #681), each `mv` guarded by tests a POSIX shell + runs (#761): `[ ! -e ] && [ ! -L ]`, true where + nothing is there, where the `mv` alone failed, exit 1 (`cannot stat`), + and stopped what was typed after it, as where a sparse checkout keeps + the variant off disk (Codex's P2 2 on #828), while a link there pointing + nowhere, which `-e` takes for nothing, is still moved; `[ -ef + ]`, true where the disk takes the two names for one + directory, the row's own; and `[ -e /.git ]`, true for a + checkout, which a move leaves pointing nowhere. So the text is the same + on every disk, it exits 0 wherever it is typed, and the shell, not a + sentence about the system, keeps the row's own directory in place. Then + where a checkout's commits are kept, and that it is deleted by hand + once nothing only it holds is needed; a clone of its own is moved out + whole, by hand. ONE ORDER DIFFERS FROM THE VALIDATOR'S, measured: the + update `git submodule update --init --recursive` comes LAST here. A + checkout at the variant that `git mv` of the mount made shares the + member's store, `.git/modules/members/

`, and the update, checking + the member out at the row's path, points that store's `core.worktree` + there: `git -C status` then printed the row's clean checkout + and not the variant's own uncommitted file. Read before the update, it + names that file, so the checkout is weighed and deleted first. """ mount = any(spelled.mount for spelled in found) groups = [] for where in SPELLED_ORDER: - here = [spelled for spelled in found if spelled.where == where] + # What is set aside is at a row's path or at `members`, no spelling + # of either: its part names it. + here = [spelled for spelled in found if spelled.where == where + and spelled.remedy not in SET_ASIDE] if not here: continue shown = ", ".join(f"`{spelled.name}` ({spelled.kind})" @@ -4122,11 +4519,6 @@ def _spellings_refusal(path: str, found: list[Spelled], if spelled.remedy == remedy}) for remedy in REMEDY_ORDER} - def each(command: str, targets: list[str]) -> str: - return " and ".join( - f"`{command} -- {shell_quoted(target, 'posix')} `" - for target in targets) - parts = [] if gitmodules == GITMODULES_MODIFIED: parts.append( @@ -4149,17 +4541,22 @@ def each(command: str, targets: list[str]) -> str: index = [] if by[RENAME]: index.append("rename what the index holds with `git mv`, entry by " - f"entry ({each(RENAME, by[RENAME])})") + f"entry ({_each(RENAME, by[RENAME])})") if by[RENAME_SPARSE]: - index.append("rename what the sparse checkout keeps off disk with " - f"`git mv --sparse`, {SPARSE_MV_GIT}, entry by entry " - f"({each(RENAME_SPARSE, by[RENAME_SPARSE])})") + index.append("rename what the sparse checkout leaves out with `git " + f"mv --sparse`, {SPARSE_MV_GIT}, entry by entry " + f"({_each(RENAME_SPARSE, by[RENAME_SPARSE])})") if by[UNSTAGE]: quoted = " ".join(shell_quoted(target, "posix") for target in by[UNSTAGE]) + # `--sparse` where the rules leave one out (#761), which `git rm + # --cached` otherwise refuses. + out = any(spelled.sparse for spelled in found + if spelled.remedy == UNSTAGE) index.append("take what has nothing on disk in that spelling out of " - f"the index, which touches no file (`{UNSTAGE} -- " - f"{quoted}`)") + f"the index, which touches no file (`{UNSTAGE}" + f"{' --sparse' if out else ''} -- {quoted}`" + + (f", {SPARSE_RM_GIT}" if out else "") + ")") # A removal from the index is a change to commit only where HEAD holds # what it removes, as it does once the pending change is committed; an # entry staged and never committed leaves nothing to commit. @@ -4199,52 +4596,27 @@ def each(command: str, targets: list[str]) -> str: f"{through} {MEMBERS_DIR}`)" + (", and commit the rename" if by[COMMIT] or any( spelled.committed for spelled in held) else "")) - # A MEMBER'S OWN GITLINK (#646), per row: every variant found, the - # index's before HEAD's, the first of which names the commit. - moved = {row: [spelled for spelled in found - if spelled.remedy == RESTORE and spelled.target == row] - for row in by[RESTORE]} - variants = sorted(spelled.name for held in moved.values() - for spelled in held) - closing: list[str] = [] - if moved: - pairs = " and ".join(f"`{name}` for the row {row}" for row in moved - for name in sorted(spelled.name - for spelled in moved[row])) - recorded = " and ".join( - "`git update-index --add --cacheinfo " - f"{shell_quoted(f'160000,{held[0].recorded},{row}', 'posix')}`" - for row, held in moved.items()) - taken_out = " ".join(shell_quoted(name, "posix") for name in variants) - # `--sparse` only where a variant needs it, so that everywhere else - # the line is the validator's word for word (#646's review, S1). - flag = " --sparse" if any(spelled.sparse for held in moved.values() - for spelled in held) else "" - parts.append( - "move each member's gitlink held only under another spelling of " - f"its row's path, {pairs}, to that path, at the commit it " - "records, in the index alone, which touches no file (`git rm -r " - f"--cached{flag} --ignore-unmatch -- {taken_out}`, " - + (f"{SPARSE_RM_GIT}, " if flag else "") + f"then {recorded})" - + (", and commit that" if by[COMMIT] or any( - spelled.committed for held in moved.values() - for spelled in held) else "")) - out = " and ".join( - f"`mv -- {shell_quoted(name, 'posix')} " - f"{shell_quoted('../' + name.replace('/', '-'), 'posix')}`" - for name in variants) - closing = [ - f"where nothing is checked out at {' or '.join(moved)}, as in a " - "fresh clone, `git submodule update --init --recursive` then " - "checks the member out there", - "on a case-sensitive disk, as Linux's usually is, a directory " - f"left at {' or '.join(variants)} stays there, untracked, and add " - "refuses it as family-members-dir-variant: move each one that is " - f"there out of the holder, {out}", - "on macOS and Windows it is the row's own directory, so leave it"] + # WHAT IS IN THE WAY OF THE MOVE BACK (#761), set aside first, in the + # move's own commit; then the move back itself, and its closing + # sentences, which come last. + aside = _set_aside_part(found, by) + if aside: + parts.append(aside) + moving, closing = _move_back_parts(found, by) + if moving: + parts.append(moving) if by[MOVE]: parts.append("move what is on disk and untracked out of the way " - f"({each(MOVE, by[MOVE])})") + f"({_each(MOVE, by[MOVE])})") + if by[CHECKOUT]: + named = " and ".join(f"`{target}`" for target in by[CHECKOUT]) + parts.append( + "delete by hand each checkout on disk and untracked, " + f"{named}, once nothing only it holds is needed, rather than " + "move it: its .git file names its store by a path relative to " + "it, which a move would leave pointing nowhere, and its commits " + "are kept in that store, so only what is not committed is in it " + "alone") for target in by[MOVE_BACK]: parts.append( f"move `{target}` back to {MEMBERS_DIR}/, since it holds " @@ -4267,6 +4639,188 @@ def each(command: str, targets: list[str]) -> str: + said[:1].upper() + said[1:] + "; then re-run.") +def _each(command: str, targets: list[str]) -> str: + """One printed ` -- ` for each of `targets`, + each quoted for a POSIX shell (#532).""" + return " and ".join( + f"`{command} -- {shell_quoted(target, 'posix')} `" + for target in targets) + + +def _set_aside_part(found: list[Spelled], by: dict[str, list[str]]) -> str: + """`_spellings_refusal`'s part that sets aside what is in the way of the + move back (#761), or empty; `by` is its targets by remedy. `git mv`, + `git mv --sparse` and `git rm -r --cached`, each `--sparse` said as the + index's own are (Codex's P2 1 on #828).""" + aside = [spelled for spelled in found if spelled.remedy in SET_ASIDE] + if not aside: + return "" + how = [] + if by[ASIDE]: + how.append("rename each that is on disk as the index holds it with " + f"`git mv` ({_each(RENAME, by[ASIDE])})") + if by[ASIDE_SPARSE]: + how.append("rename each that the sparse checkout leaves out with `git " + f"mv --sparse`, {SPARSE_MV_GIT} " + f"({_each(RENAME_SPARSE, by[ASIDE_SPARSE])})") + if by[ASIDE_OUT]: + quoted = " ".join(shell_quoted(target, "posix") + for target in by[ASIDE_OUT]) + out = any(spelled.sparse for spelled in aside + if spelled.remedy == ASIDE_OUT) + how.append("take each that is not on disk so out of the index, which " + f"touches no file (`{UNSTAGE}{' --sparse' if out else ''} " + f"-- {quoted}`" + (f", {SPARSE_RM_GIT}" if out else "") + + ")") + named = ", ".join(f"`{spelled.name}` ({spelled.kind})" + for spelled in aside) + return ("set aside first what the index holds in the way of the move of a " + f"member's gitlink below, {named}, since git's update-index " + "replaces a file at the path it records without a word and fails " + f"beside one under it or at {MEMBERS_DIR} itself, and taking the " + "gitlink out instead un-registers the member: " + + " and ".join(how)) + + +def _move_back_parts(found: list[Spelled], by: dict[str, list[str]] + ) -> tuple[str, list[str]]: + """`(the part, the closing sentences)` of `_spellings_refusal` that move + each member's own gitlink back to its row's path (#646, #761), or empty + ones; `by` is its targets by remedy. Per row, every variant found, the + index's before HEAD's, the first of which names the commit; the move is + committed where it, a pending change or what is set aside leaves a + change to commit.""" + moved = {row: [spelled for spelled in found + if spelled.remedy == RESTORE and spelled.target == row] + for row in by[RESTORE]} + if not moved: + return "", [] + variants = sorted(spelled.name for held in moved.values() + for spelled in held) + committed = bool(by[COMMIT]) or any( + spelled.committed for spelled in found + if spelled.remedy == RESTORE or spelled.remedy in SET_ASIDE) + rows = {spelled.name: row for row, held in moved.items() + for spelled in held} + return (_move_back_line(moved, variants, committed), + _closing_sentences(variants, rows, list(moved))) + + +def _move_back_line(moved: dict[str, list[Spelled]], variants: list[str], + committed: bool) -> str: + """The move back's own part, `moved` the variants by row: the variants + taken out of the index, each row's gitlink recorded at the commit named, + each section a `git mv` of a mount left naming a variant pointed back, + and the commit where `committed`.""" + pairs = " and ".join( + f"`{name}` for the row {row}" for row in moved + for name in sorted(spelled.name for spelled in moved[row] + if spelled.where != IN_GITMODULES)) + # A member whose removal is staged (#761), put back at HEAD's commit. + staged = " and ".join( + f"{row}, whose section gives `{spelled.name}`" + for row, held in moved.items() for spelled in held + if spelled.where == IN_GITMODULES) + recorded = " and ".join( + "`git update-index --add --cacheinfo " + f"{shell_quoted(f'160000,{held[0].recorded},{row}', 'posix')}`" + for row, held in moved.items()) + taken_out = " ".join(shell_quoted(name, "posix") for name in variants) + # `--sparse` only where a variant needs it, so that everywhere else the + # line is the validator's word for word (#646's review, S1), and `-f` + # likewise (#761): measured, `git rm --cached` refused a staged variant + # whose checkout held one commit, or one untracked file, of its own + # (`staged content different from both the file and the HEAD`, exit 1), + # and took it with `-f`, touching no file. + every = [spelled for held in moved.values() for spelled in held] + flag = " --sparse" if any(spelled.sparse for spelled in every) else "" + force = any(spelled.forced for spelled in every) + # Each section a `git mv` of a mount left naming the variant (#761). + pointed = sorted({(name, row) for row, held in moved.items() + for spelled in held for name in spelled.sections}) + sections = " and ".join( + f"`git config -f {GITMODULES} " + f"{shell_quoted(f'submodule.{name}.path', 'posix')} " + f"{shell_quoted(row, 'posix')}`" for name, row in pointed) + moves = [] + if pairs: + moves.append("move each member's gitlink held only under another " + f"spelling of its row's path, {pairs}, to that path, at " + "the commit it records") + if staged: + moves.append("put back at its row's path each member's gitlink whose " + f"removal is staged though a section of {GITMODULES} " + f"gives it another spelling, {staged}, at the commit " + "HEAD records") + return ( + ", and ".join(moves) + ", in the index alone, which touches no file" + + (f", and point each section of {GITMODULES} that gives one of them " + "another spelling of its row's path, as a `git mv` of its mount " + "leaves it, at that path" if pointed else "") + + f" (`git rm -r --cached{flag}{' -f' if force else ''} " + f"--ignore-unmatch -- {taken_out}`, " + + (f"{SPARSE_RM_GIT}, " if flag else "") + + ("whose -f takes out a staged gitlink whose checkout has moved on " + "or holds a file git does not track, which git rm otherwise " + "refuses, and with --cached touches no file, " if force else "") + + f"then {recorded}" + + (f", then {sections} and `git add -- {GITMODULES}`" if pointed + else "") + ")" + + (", and commit that" if committed else "")) + + +def _closing_sentences(variants: list[str], rows: dict[str, str], + moved: list[str]) -> list[str]: + """The move back's closing sentences (#681, #761), after every other + remedy: what a directory left at a variant is, its guarded `mv`, where a + checkout's commits are kept and that it is deleted by hand, a clone of + its own moved out whole, and the update, last. `rows` is each variant's + row path, and `moved` the rows.""" + # Nothing there, not even a link, moves nothing and exits 0 (Codex's P2 2 + # on #828): a plain `mv` of it failed and stopped the update. + guarded = " and ".join( + f"`[ ! -e {shell_quoted(name, 'posix')} ] && " + f"[ ! -L {shell_quoted(name, 'posix')} ] || " + f"[ {shell_quoted(name, 'posix')} -ef " + f"{shell_quoted(rows[name], 'posix')} ] || " + f"[ -e {shell_quoted(name + '/.git', 'posix')} ] || mv -- " + f"{shell_quoted(name, 'posix')} " + f"{shell_quoted('../' + name.replace('/', '-'), 'posix')}`" + for name in variants) + status = " or ".join(f"`git -C {shell_quoted(name, 'posix')} status`" + for name in variants) + unpushed = " or ".join( + f"`git -C {shell_quoted(name, 'posix')} log --oneline HEAD " + "--branches --not --remotes`" for name in variants) + return [ + f"a directory left at {' or '.join(variants)} is the row's own where " + "the disk folds case, as macOS's and Windows' usually do and a Linux " + "mount can (WSL's /mnt/c)", + "where the disk holds the two apart it stays there, untracked, and " + "add refuses it as family-members-dir-variant: move each one that is " + "there out of the holder, in a POSIX shell (Git Bash on Windows), " + f"{guarded}, which moves nothing where nothing is there, where the " + "two names are one directory or where it holds a .git entry, a " + "checkout", + "a checkout there that git submodule add or git mv made holds a .git " + "file naming its store by a path relative to it, which a move would " + "leave pointing nowhere: its commits are kept under this holder's " + ".git/modules/, in the directory its .git file names, whatever " + "becomes of the checkout, and only what is not committed is in the " + f"checkout alone ({status} shows it, except where a plain mv brought " + "the checkout to that path: git cannot open it there, so check its " + "files by hand), so delete such a checkout by hand once nothing only " + "it holds is needed, before the update below, which points a store " + "the row shares at the row's path", + "a checkout whose .git is a directory, a clone of its own, holds its " + f"commits there and nowhere else, and {unpushed} lists the ones not " + "pushed: move it out of the holder whole, by hand, rather than delete " + "it", + f"then, where nothing is checked out at {' or '.join(moved)}, as in a " + "fresh clone, `git submodule update --init --recursive` checks the " + "member out there"] + + def _submodule_paths( root: Path) -> dict[str, tuple[str, str | None, tuple[str, ...]]]: """`{name: (path, url, keys)}` of every submodule the holder's @@ -4364,7 +4918,13 @@ def _members_file_left_in_head(root: Path, path: str) -> Refusal | None: 128, measured on git 2.43), nor put back first with `git checkout -- members`, which would put the file back over such a directory and take the directory away. A link on disk is left to git's own dry run, which - refuses a path beyond one.""" + refuses a path beyond one. + + ONE A SPARSE CHECKOUT MARKS OR LEAVES OUT (#761, Codex's P2 1 on #828) + is off disk the same way, and the removal takes `--sparse`, which says + the git it needs: plain `git rm --cached` refuses it (`exist outside of + your sparse-checkout definition`, exit 1, measured on git 2.43 and + 2.47.1), as `_set_aside` names such an entry off disk.""" held = [kind for name, kind in _index_entries(root, MEMBERS_DIR) if name == MEMBERS_DIR] if any(kind != "a file" for kind in held) or ( @@ -4378,19 +4938,25 @@ def _members_file_left_in_head(root: Path, path: str) -> Refusal | None: for name, kind, _ in head): return None if held: + sparse = MEMBERS_DIR in _skip_worktree(root) or bool( + _sparse_rules_exclude(root, [MEMBERS_DIR])) return Refusal( "member-path-unaddable", f"git would add the mount path {path} and take `{MEMBERS_DIR}` " f"out of the holder with it: `{MEMBERS_DIR}` is a file HEAD and " - "the index hold and the work tree no longer does, its deletion " - "not staged, and the add's mount, putting a directory where the " - "file was, would take it out of the index, and its commit would " - f"take that deletion into itself (`D {MEMBERS_DIR}`). No git " - "command that writes was run and nothing was written.", + "the index hold and the work tree no longer does, " + + ("as the sparse checkout leaves it out" if sparse else + "its deletion not staged") + + ", and the add's mount, putting a directory where the file " + "was, would take it out of the index, and its commit would take " + f"that deletion into itself (`D {MEMBERS_DIR}`). No git command " + "that writes was run and nothing was written.", f"Remediation: every member is mounted under {MEMBERS_DIR}/, " "which must be a plain directory of the holder's own. Take the " "file out of the index, which touches no file (`git rm --cached " - f"-- {MEMBERS_DIR}`), and commit the removal on its own first " + + (f"--sparse -- {MEMBERS_DIR}`, {SPARSE_RM_GIT}" if sparse else + f"-- {MEMBERS_DIR}`") + + "), and commit the removal on its own first " "(`git commit`, with nothing else staged: a bare `git commit` " "takes every staged change, and `git status` lists them), since " f"the add's own commit names {GITMODULES}, {path} and {MANIFEST} " @@ -4501,7 +5067,10 @@ def _checked_mount_path(root: Path, project: str, path: str, spelling of `members/` alone. A member's own gitlink held only under another spelling of its row's path keeps that code, since the add is refused for it as for any path there, and its remedy moves it back to - the row's path instead (#646). + the row's path instead (#646), with its section of `.gitmodules` + where that names the other spelling too; and a section that does, + for a member whose gitlink's removal is staged, is named as found in + `.gitmodules` (#761). - A FILE NAMED `members` (#612, 574/5). Git's own dry run, below, does not refuse it (measured on git 2.43), and the mount's `git submodule add --force` then dies cloning into it (`could not create leading @@ -4609,14 +5178,21 @@ def _checked_mount_path(root: Path, project: str, path: str, spelled = _members_spellings(root, project, on_disk, rows) if spelled: # The work tree's `.gitmodules`, where a remedy runs `git mv` or `git - # rm` of a gitlink (#685). An unmerged copy is `gitmodules-unstaged`'s - # to name, with its own restore, first: git cannot rewrite it, nor - # read conflict markers, and a `git add` of it stages them. - beside = _gitmodules_beside(root) if any( + # rm` of a gitlink (#685), or points a section back with `git config + # -f` (#761). An unmerged copy is `gitmodules-unstaged`'s to name, + # with its own restore, first: git cannot rewrite it, nor read + # conflict markers, and a `git add` of it stages them. So is one a + # sparse checkout marks, as `gitmodules-sparse`, where a section is + # pointed back: `git config -f` would write a new file of that one + # line, which `git add` then refuses. + pointed = any(spelled_one.sections for spelled_one in spelled) + beside = _gitmodules_beside(root) if pointed or any( spelled_one.where == IN_INDEX and spelled_one.kind == INDEX_ENTRY_KINDS[0o160000] for spelled_one in spelled) else "" - if beside == GITMODULES_UNMERGED: + if beside == GITMODULES_UNMERGED or pointed and any( + tag.upper() == "S" for tag, _, _ + in _gitmodules_in_index(root)): _checked_gitmodules_as_staged(root, "add") beside = "" raise _spellings_refusal(path, spelled, beside) diff --git a/tests/test_family_add_member_variant.py b/tests/test_family_add_member_variant.py index 0b879ffb..c9eab9a9 100644 --- a/tests/test_family_add_member_variant.py +++ b/tests/test_family_add_member_variant.py @@ -18,11 +18,14 @@ disk" ruling, since `git mv` to the row's path fails or misplaces the gitlink: `git rm -r --cached --ignore-unmatch` of the variant, `git update-index --add --cacheinfo 160000,,members/IRRS` at the commit -the variant records, and a commit. Then, as the validator says, `git -submodule update --init --recursive` where nothing is checked out at the -row's path, an `mv` out of the holder of what a case-sensitive disk leaves at -the variant, and nothing to do on macOS and Windows, where that directory is -the row's own. +the variant records, and a commit. Then, as the validator says, an `mv` +out of the holder of what a case-sensitive disk leaves at the variant, +guarded by tests a POSIX shell runs so that it moves nothing, and exits 0, +where nothing is there (Codex's P2 2 on #828), and moves nothing where that +directory is the row's own (macOS and Windows) or a checkout, which is +deleted, or for a clone of its own moved, by hand; and last `git submodule +update --init --recursive` where nothing is checked out at the row's path +(#761). EVERY PRINTED COMMAND IS TYPED AS PRINTED (`typed_as_printed`), in each layout rv611b's review of #639 used, and the holder's gate then passes, the @@ -68,7 +71,19 @@ REMOVE = f"git rm -r --cached --ignore-unmatch -- {VARIANT}" REMOVE_SPARSE = f"git rm -r --cached --sparse --ignore-unmatch -- {VARIANT}" SUBMODULE = "git submodule update --init --recursive" -MOVE_OUT = f"mv -- {VARIANT} ../Members-{MEMBER}" + + +def guarded(name: str = VARIANT, row: str = ROW) -> str: + """The closing `mv` of a directory left at `name`, the variant of + `row`'s path, guarded by the tests a POSIX shell runs (#761, as + `validate-family.py` guards it, #681): it moves nothing where nothing is + there, not even a link pointing nowhere (Codex's P2 on #828), where the + two names are one directory, or where it holds a `.git` entry.""" + return (f"[ ! -e {name} ] && [ ! -L {name} ] || [ {name} -ef {row} ] || " + f"[ -e {name}/.git ] || mv -- {name} ../{name.replace('/', '-')}") + + +MOVE_OUT = guarded() def record(commit: str, row: str = ROW) -> str: @@ -76,34 +91,92 @@ def record(commit: str, row: str = ROW) -> str: return f"git update-index --add --cacheinfo 160000,{commit},{row}" -def moved_back(rows: dict, commit: bool = True, - sparse: bool = False) -> tuple[str, list[str]]: +def point_back(row: str = ROW, name: str | None = None) -> str: + """The line that points the section `name` (named after `row`, as `git + submodule add` names one) back at `row` (#761).""" + return f"git config -f .gitmodules submodule.{name or row}.path {row}" + + +def moved_back(rows: dict, commit: bool = True, sparse: bool = False, + sections: tuple = (), staged: dict | None = None, + forced: bool = False) -> tuple[str, list[str]]: """`(the part that moves each gitlink back, the closing parts)` of the remedy, for `rows`, `{row path: ([its variants], the commit named)}`; - `sparse` where a sparse checkout keeps a variant off disk.""" - variants = sorted(name for names, _ in rows.values() for name in names) + `sparse` where a sparse checkout keeps a variant off disk; `sections`, + the rows whose section is pointed back (#761); `staged`, `{row path: + (the path its section gives, HEAD's commit)}` for a member whose + removal is staged (#761); `forced` where the removal takes `-f` + (#761).""" + staged = staged or {} + every = {**{row: (names, at) for row, (names, at) in rows.items()}, + **{row: ([given], at) for row, (given, at) in staged.items()}} + variants = sorted(name for names, _ in every.values() for name in names) + row_of = {name: row for row, (names, _) in every.items() + for name in names} pairs = " and ".join(f"`{name}` for the row {row}" for row, (names, _) in sorted(rows.items()) for name in sorted(names)) + put = " and ".join(f"{row}, whose section gives `{given}`" + for row, (given, _) in sorted(staged.items())) lines = " and ".join(f"`{record(at, row)}`" - for row, (_, at) in sorted(rows.items())) - move = ("move each member's gitlink held only under another spelling of " - f"its row's path, {pairs}, to that path, at the commit it " - "records, in the index alone, which touches no file (`git rm -r " - f"--cached{' --sparse' if sparse else ''} --ignore-unmatch -- " + for row, (_, at) in sorted(every.items())) + said = [] + if pairs: + said.append("move each member's gitlink held only under another " + f"spelling of its row's path, {pairs}, to that path, at " + "the commit it records") + if put: + said.append("put back at its row's path each member's gitlink whose " + "removal is staged though a section of .gitmodules gives " + f"it another spelling, {put}, at the commit HEAD records") + pointed = " and ".join(f"`{point_back(row)}`" for row in sorted(sections)) + move = (", and ".join(said) + ", in the index alone, which touches no " + "file" + + (", and point each section of .gitmodules that gives one of " + "them another spelling of its row's path, as a `git mv` of " + "its mount leaves it, at that path" if sections else "") + + f" (`git rm -r --cached{' --sparse' if sparse else ''}" + f"{' -f' if forced else ''} --ignore-unmatch -- " f"{' '.join(variants)}`, " - + (f"{NEEDS_GIT_2_34}, " if sparse else "") + f"then {lines})" + + (f"{NEEDS_GIT_2_34}, " if sparse else "") + + ("whose -f takes out a staged gitlink whose checkout has moved " + "on or holds a file git does not track, which git rm " + "otherwise refuses, and with --cached touches no file, " + if forced else "") + f"then {lines}" + + (f", then {pointed} and `git add -- .gitmodules`" + if sections else "") + ")" + (", and commit that" if commit else "")) - moved = " and ".join(f"`mv -- {name} ../{name.replace('/', '-')}`" + moved = " and ".join(f"`{guarded(name, row_of[name])}`" for name in variants) + status = " or ".join(f"`git -C {name} status`" for name in variants) + unpushed = " or ".join(f"`git -C {name} log --oneline HEAD --branches " + "--not --remotes`" for name in variants) return move, [ - f"where nothing is checked out at {' or '.join(sorted(rows))}, as in " - f"a fresh clone, `{SUBMODULE}` then checks the member out there", - "on a case-sensitive disk, as Linux's usually is, a directory left " - f"at {' or '.join(variants)} stays there, untracked, and add refuses " - "it as family-members-dir-variant: move each one that is there out " - f"of the holder, {moved}", - "on macOS and Windows it is the row's own directory, so leave it"] + f"a directory left at {' or '.join(variants)} is the row's own where " + "the disk folds case, as macOS's and Windows' usually do and a Linux " + "mount can (WSL's /mnt/c)", + "where the disk holds the two apart it stays there, untracked, and " + "add refuses it as family-members-dir-variant: move each one that is " + "there out of the holder, in a POSIX shell (Git Bash on Windows), " + f"{moved}, which moves " + "nothing where nothing is there, where the two names are one " + "directory or where it holds a .git entry, a checkout", + "a checkout there that git submodule add or git mv made holds a .git " + "file naming its store by a path relative to it, which a move would " + "leave pointing nowhere: its commits are kept under this holder's " + ".git/modules/, in the directory its .git file names, whatever " + "becomes of the checkout, and only what is not committed is in the " + f"checkout alone ({status} shows it, except where a plain mv brought " + "the checkout to that path: git cannot open it there, so check its " + "files by hand), so delete such a checkout by hand once nothing only " + "it holds is needed, before the update below, which points a store " + "the row shares at the row's path", + "a checkout whose .git is a directory, a clone of its own, holds its " + f"commits there and nowhere else, and {unpushed} lists the ones not " + "pushed: move it out of the holder whole, by hand, rather than " + "delete it", + f"then, where nothing is checked out at {' or '.join(sorted(every))}, " + f"as in a fresh clone, `{SUBMODULE}` checks the member out there"] def remedy(commit: str) -> str: @@ -138,30 +211,34 @@ def variant(root: Path, row: str = ROW, spelled: str = VARIANT, return commit -def on_disk_as_named(root: Path, name: str) -> bool: - """Is `name` on disk in that very spelling, component by component?""" - here = root - for part in name.split("/"): - if not here.is_dir() or part not in os.listdir(here): - return False - here = here / part - return True +def typed_in_sh(root: Path, command: str) -> None: + """`command` typed into a POSIX shell in `root`, as printed: exit 0.""" + typed = subprocess.run(["sh", "-c", command], cwd=str(root), + capture_output=True, check=False) + assert typed.returncode == 0, (command, typed.stderr) -def typed_as_printed(root: Path, said: str) -> list[str]: +def typed_as_printed(root: Path, said: str, by_hand=None) -> list[str]: """EVERY COMMAND `said` PRINTS, TYPED AS PRINTED, in its order, and the commit each part asks for, as `type_remedy` types #574's: `` - filled in `renamed-N` for a `git mv` and `../moved-N` for a `mv`. Typed - where its sentence says so, and only there: `git submodule update` where - nothing is checked out at the row's path, and an `mv` with nothing to - fill in where its source is on disk in that spelling and the disk does - not fold case (on macOS and Windows the text says to leave it). A local - member is cloned with `protocol.file.allow`, the suite's one concession - to bare repositories as origins. Returns what was typed.""" - folds = disk_folds(root.parent) + filled in `renamed-N` for a `git mv` and `../moved-N` for a `mv`. `git + submodule update` is typed where its sentence says so, and only there, + where nothing is checked out at the row's path. Every other `mv`, and + each guarded one (#761), is typed into a POSIX shell AS PRINTED, whether + or not its source is there (Codex's P2 on #828: a guard that skipped an + absent source here hid an `mv` that failed, exit 1, where the variant is + not on disk), and must exit 0. `by_hand`, where given, is what a + person does by hand where the text says to (#761), run after the last + such part. A local member is cloned with `protocol.file.allow`, the + suite's one concession to bare repositories as origins. Returns what was + typed.""" typed: list[str] = [] count = {"git": 0, "mv": 0} - for part in said.split(" on every disk. ", 1)[1].split("; "): + parts = said.split(" on every disk. ", 1)[1].split("; ") + last_by_hand = max((index for index, part in enumerate(parts) + if "by hand" in part), + default=None) + for index, part in enumerate(parts): for command in re.findall(r"`([^`]*)`", part): argv = shlex.split(command) if command == "git commit": @@ -175,19 +252,18 @@ def typed_as_printed(root: Path, said: str) -> list[str]: elif "" in command: tool = argv[0] count[tool] += 1 - argv[-1] = (f"renamed-{count['git']}" if tool == "git" - else f"../moved-{count['mv']}") + argv = shlex.split(command.replace( + "", f"renamed-{count['git']}" if tool == "git" + else f"../moved-{count['mv']}")) if tool == "git": git(*argv[1:], cwd=root) else: os.rename(root / argv[2], root / argv[3]) command = shlex.join(argv) - elif argv[0] == "mv": - if folds or not on_disk_as_named(root, argv[2]): - continue - os.rename(root / argv[2], root / argv[3]) - elif argv[0] == "git" and (" -- " in command - or argv[1] == "update-index"): + elif argv[0] in ("mv", "["): + typed_in_sh(root, command) + elif argv[0] == "git" and (" -- " in command or argv[1] in ( + "update-index", "config")): git(*argv[1:], cwd=root) else: continue @@ -196,6 +272,8 @@ def typed_as_printed(root: Path, said: str) -> list[str]: part): git("commit", "-qm", "The remedy", cwd=root) typed.append("git commit") + if index == last_by_hand and by_hand is not None: + by_hand(root) return typed @@ -252,16 +330,28 @@ def removal_staged(root: Path, remotes: Path, commit: str) -> None: git("rm", "-q", "--cached", "--", VARIANT, cwd=root) +def clone_moved_out_by_hand(root: Path) -> None: + """What the text says of a clone of its own at the variant, done by + hand: the guarded `mv` moved nothing, since it holds a `.git` entry, and + the clone, whose `.git` is a directory, is moved out of the holder whole + (#761, as `validate-family.py` says it, #681).""" + assert (root / VARIANT / ".git").is_dir(), "premise: a clone of its own" + os.rename(root / VARIANT, root.parent / f"{MEMBER}-clone") + + #: `{layout: (how it is laid out after the move, whether it needs two #: directories differing only in case, where the refusal finds the gitlink, -#: what is typed after the move and its commit)}`. +#: what is typed after the move and its commit, what is done by hand)}`. LAYOUTS = { - "nothing on disk in that spelling": (nothing_there, False, IN_INDEX, []), + "nothing on disk in that spelling": (nothing_there, False, IN_INDEX, + [MOVE_OUT], None), "the only checkout at the variant": (checkout_at_the_variant, True, - IN_INDEX, [SUBMODULE, MOVE_OUT]), - "both directories": (both_directories, True, IN_INDEX, [MOVE_OUT]), + IN_INDEX, [MOVE_OUT, SUBMODULE], + clone_moved_out_by_hand), + "both directories": (both_directories, True, IN_INDEX, [MOVE_OUT], + clone_moved_out_by_hand), "the removal staged, not committed": (removal_staged, False, HEAD_ONLY, - []), + [MOVE_OUT], None), } @@ -274,8 +364,10 @@ def test_each_layout_moves_the_gitlink_back_as_printed(family, holder, remedy moves the member's gitlink back to its row's path, where #574's `git rm --cached`, `git mv` or `git commit` took it away. Typed as printed, the member is where its row says, at the commit it recorded, - the holder's gate passes, and the same `add` goes through.""" - lay_out, two_spellings, where, after = LAYOUTS[layout] + the holder's gate passes, and the same `add` goes through. A clone of + its own at the variant is kept by the guarded `mv` and moved out by + hand, as the text says, before the update (#761).""" + lay_out, two_spellings, where, after, by_hand = LAYOUTS[layout] if two_spellings and disk_folds(tmp_path): pytest.skip("two directories differing only in case need a disk " "that does not fold case") @@ -285,7 +377,7 @@ def test_each_layout_moves_the_gitlink_back_as_printed(family, holder, assert refused_untouched(offline, holder, family["remotes"]) == said assert said == (DIR_VARIANT + f"{where}: {VARIANT_SHOWN}", remedy(commit)) - assert typed_as_printed(holder, said[1]) == [ + assert typed_as_printed(holder, said[1], by_hand) == [ REMOVE, record(commit), "git commit", *after] registered(holder, family["root"], commit) gate_passes(holder) @@ -296,12 +388,12 @@ def test_a_fresh_clone_moves_the_gitlink_back_as_printed(family, holder, offline, tmp_path): """A FRESH RECURSIVE CLONE of a holder whose HEAD records the gitlink only at `Members/IRRS`: the clone leaves an empty `Members/IRRS` and - nothing at the row's path. The same text; typed as printed, `git - submodule update` checks the member out at its row's path, the empty - directory leaves the holder on a case-sensitive disk (on macOS and - Windows it is the row's own, and stays), and the clone's gate passes. - The branch is pushed to the fixture's remote under a name no other test - uses.""" + nothing at the row's path. The same text; typed as printed, the empty + directory leaves the holder on a case-sensitive disk by the guarded + `mv` (on macOS and Windows it is the row's own, and the shell's `-ef` + keeps it), `git submodule update` checks the member out at its row's + path, and the clone's gate passes. The branch is pushed to the + fixture's remote under a name no other test uses.""" commit = variant(holder) remote = family["remotes"] / f"{NAME}.git" branch = "w646/fresh-clone" @@ -312,8 +404,7 @@ def test_a_fresh_clone_moves_the_gitlink_back_as_printed(family, holder, said = refused_untouched(offline, clone, family["remotes"], "--dry-run") assert said == (in_the_index(VARIANT_SHOWN), remedy(commit)) assert typed_as_printed(clone, said[1]) == [ - REMOVE, record(commit), "git commit", SUBMODULE, - *([] if disk_folds(tmp_path) else [MOVE_OUT])] + REMOVE, record(commit), "git commit", MOVE_OUT, SUBMODULE] registered(clone, family["root"], commit) gate_passes(clone) added(offline, clone, family["remotes"]) @@ -346,7 +437,7 @@ def test_every_disk_gives_the_same_answer(family, holder, family_module, holder (`disk_as`), gives the one text the subprocess gives. The move reads the index and HEAD, never the disk, so the disk cannot change it.""" - lay_out, two_spellings, where, _ = LAYOUTS[layout] + lay_out, two_spellings, where, _, _ = LAYOUTS[layout] if two_spellings and disk_folds(tmp_path): pytest.skip("two directories differing only in case need a disk " "that does not fold case") @@ -388,7 +479,7 @@ def beside_a_file(root: Path, remotes: Path) -> tuple: remedy_of(index_part(unstage=["Members/README"]), move, *closing), ["git rm -r --cached -- Members/README", REMOVE, record(commit), - "git commit"], commit) + "git commit", MOVE_OUT], commit) def after_a_pending_change(root: Path, remotes: Path) -> tuple: @@ -403,7 +494,8 @@ def after_a_pending_change(root: Path, remotes: Path) -> tuple: return (in_the_index(VARIANT_SHOWN) + "; in HEAD and no longer in the " "index: `MEMBERS/old` (a file)", remedy_of(commit_first("MEMBERS/old"), move, *closing), - ["git commit", REMOVE, record(commit), "git commit"], commit) + ["git commit", REMOVE, record(commit), "git commit", MOVE_OUT], + commit) def nothing_to_commit(root: Path, remotes: Path) -> tuple: @@ -412,7 +504,7 @@ def nothing_to_commit(root: Path, remotes: Path) -> tuple: commit = staged_only(root) move, closing = moved_back({ROW: ([VARIANT], commit)}, commit=False) return (in_the_index(VARIANT_SHOWN), remedy_of(move, *closing), - [REMOVE, record(commit)], commit) + [REMOVE, record(commit), MOVE_OUT], commit) def members_renamed_by_hand(root: Path, remotes: Path) -> tuple: @@ -430,18 +522,19 @@ def members_renamed_by_hand(root: Path, remotes: Path) -> tuple: "`Members` (a directory)", remedy_of(move, MOVE_BACK, *closing), [REMOVE, record(commit), "git commit", "mv -- Members members", - SUBMODULE], commit) + MOVE_OUT, SUBMODULE], commit) def head_only_and_on_disk(root: Path, remotes: Path) -> tuple: """HEAD alone holds the gitlink at the variant, and the member's only - checkout is there: the directory is the move's to name, never a second - time as untracked.""" + checkout is there, a clone of its own: the directory is the move's to + name, never a second time as untracked, and it is moved out by hand + before the update (#761).""" commit = variant(root) git("rm", "-q", "--cached", "--", VARIANT, cwd=root) checkout_at_the_variant(root, remotes, commit) return (DIR_VARIANT + f"{HEAD_ONLY}: {VARIANT_SHOWN}", remedy(commit), - [REMOVE, record(commit), "git commit", SUBMODULE, MOVE_OUT], + [REMOVE, record(commit), "git commit", MOVE_OUT, SUBMODULE], commit) @@ -458,7 +551,7 @@ def head_holding_both(root: Path, remotes: Path) -> tuple: git("commit", "-qm", "Both spellings", cwd=root) git("rm", "-q", "--cached", "--", ROW, cwd=root) return (in_the_index(VARIANT_SHOWN), remedy(commit), - [REMOVE, record(commit), "git commit"], commit) + [REMOVE, record(commit), "git commit", MOVE_OUT], commit) def kept_off_disk(root: Path, remotes: Path) -> tuple: @@ -473,7 +566,7 @@ def kept_off_disk(root: Path, remotes: Path) -> tuple: git("update-index", "--skip-worktree", "--", VARIANT, cwd=root) move, closing = moved_back({ROW: ([VARIANT], commit)}, sparse=True) return (in_the_index(VARIANT_SHOWN), remedy_of(move, *closing), - [REMOVE_SPARSE, record(commit), "git commit"], commit) + [REMOVE_SPARSE, record(commit), "git commit", MOVE_OUT], commit) def file_blob(root: Path) -> str: @@ -510,22 +603,23 @@ def unmerged(root: Path, remotes: Path) -> tuple: git("rm", "-q", "--cached", "--", VARIANT, cwd=root) unmerged_at(root, {1: first, 2: commit, 3: first}) return (in_the_index(VARIANT_SHOWN), remedy(commit), - [REMOVE, record(commit), "git commit"], commit) + [REMOVE, record(commit), "git commit", MOVE_OUT], commit) #: `{state: (how it is made, from the holder and the remotes, returning #: (the refusal's two lines, what is typed, the member's commit), whether it -#: needs two directories differing only in case)}`. +#: needs two directories differing only in case, what is done by hand)}`. WITH_OTHERS = { - "beside a file under Members/": (beside_a_file, False), - "after a pending change in HEAD": (after_a_pending_change, False), - "with nothing left to commit": (nothing_to_commit, False), - "members/ renamed by hand": (members_renamed_by_hand, True), - "HEAD alone, the checkout at the variant": (head_only_and_on_disk, True), + "beside a file under Members/": (beside_a_file, False, None), + "after a pending change in HEAD": (after_a_pending_change, False, None), + "with nothing left to commit": (nothing_to_commit, False, None), + "members/ renamed by hand": (members_renamed_by_hand, True, None), + "HEAD alone, the checkout at the variant": (head_only_and_on_disk, True, + clone_moved_out_by_hand), "HEAD holding both, the row's removal staged": (head_holding_both, - False), - "kept off disk by a sparse checkout": (kept_off_disk, False), - "unmerged, with nothing in progress": (unmerged, False), + False, None), + "kept off disk by a sparse checkout": (kept_off_disk, False, None), + "unmerged, with nothing in progress": (unmerged, False, None), } @@ -539,14 +633,14 @@ def test_with_the_other_remedies_each_runs_in_its_order(family, holder, make, then the disk, and the move back's closing sentences last. Typed as printed, in that order, the member is where its row says, the gate passes and the same `add` goes through.""" - make, two_spellings = WITH_OTHERS[state] + make, two_spellings, by_hand = WITH_OTHERS[state] if two_spellings and disk_folds(tmp_path): pytest.skip("two directories differing only in case need a disk " "that does not fold case") first, said, typed, commit = make(holder, family["remotes"]) assert refused_untouched(offline, holder, family["remotes"]) == (first, said) - assert typed_as_printed(holder, said) == typed + assert typed_as_printed(holder, said, by_hand) == typed registered(holder, family["root"], commit) gate_passes(holder) added(offline, holder, family["remotes"]) @@ -566,7 +660,7 @@ def test_every_member_held_in_the_wrong_case_is_moved_back(family, holder, assert typed_as_printed(holder, said[1]) == [ f"git rm -r --cached --ignore-unmatch -- {VARIANT} {OTHER_VARIANT}", record(rows[ROW][1]), record(rows[OTHER_ROW][1], OTHER_ROW), - "git commit"] + "git commit", MOVE_OUT, guarded(OTHER_VARIANT, OTHER_ROW)] for row, (_, commit) in rows.items(): registered(holder, family["root"], commit, row) gate_passes(holder) @@ -667,7 +761,7 @@ def test_a_commit_the_row_does_not_pin_is_moved_not_bumped(family, holder, said = refused_untouched(offline, holder, family["remotes"]) assert said == (in_the_index(VARIANT_SHOWN), remedy(commit)) assert typed_as_printed(holder, said[1]) == [REMOVE, record(commit), - "git commit"] + "git commit", MOVE_OUT] registered(holder, family["root"], commit) gate = validate(holder) assert gate.returncode == 1, gate.stdout + gate.stderr @@ -688,7 +782,8 @@ def beside_a_file_at(path: str): """The variant's gitlink with a FILE at `path`, the row's path or under it, in the index: the move would replace the one, and stop on the other (`appears as both a file and as a directory`) once the variant was out - (#646's review, N3).""" + (#646's review, N3), so the file is set aside first, here out of the + index, since the disk holds the member's checkout there (#761).""" def make(root: Path) -> None: plant(root, "100644", path) plant(root, "160000", VARIANT) @@ -712,32 +807,34 @@ def unmerged_file_ours(root: Path) -> None: #: `{state: (how it is made after the row's gitlink leaves the index, what -#: `_members_spellings` gives the variant: where, target, remedy)}`. The +#: `_members_spellings` gives each path: where, target, remedy)}`. The #: index never holds the row's gitlink but in the first. OWN = { "a second gitlink beside the row's": ( lambda root: plant(root, "160000", VARIANT), - (IN_INDEX, VARIANT, "git rm -r --cached")), + [(IN_INDEX, VARIANT, "git rm -r --cached")]), "a file at the variant": ( lambda root: plant(root, "100644", VARIANT), - (IN_INDEX, VARIANT, "git rm -r --cached")), + [(IN_INDEX, VARIANT, "git rm -r --cached")]), "a gitlink under the variant": ( lambda root: plant(root, "160000", f"{VARIANT}/x"), - (IN_INDEX, VARIANT, "git rm -r --cached")), + [(IN_INDEX, VARIANT, "git rm -r --cached")]), "a file at the variant in HEAD alone": ( - head_file, (HEAD_ONLY, VARIANT, "git commit")), + head_file, [(HEAD_ONLY, VARIANT, "git commit")]), "a file at the row's path": ( - beside_a_file_at(ROW), (IN_INDEX, VARIANT, "git rm -r --cached")), + beside_a_file_at(ROW), [(IN_INDEX, ROW, "git update-index"), + (IN_INDEX, ROW, "git rm -r --cached aside")]), "a file under the row's path": ( beside_a_file_at(f"{ROW}/x"), - (IN_INDEX, VARIANT, "git rm -r --cached")), + [(IN_INDEX, ROW, "git update-index"), + (IN_INDEX, f"{ROW}/x", "git rm -r --cached aside")]), "unmerged, with no gitlink at stage 2": ( - unmerged_theirs, (IN_INDEX, VARIANT, "git rm -r --cached")), + unmerged_theirs, [(IN_INDEX, VARIANT, "git rm -r --cached")]), "unmerged, with a file at stage 2 and gitlinks at 1 and 3": ( - unmerged_file_ours, (IN_INDEX, VARIANT, "git rm -r --cached")), + unmerged_file_ours, [(IN_INDEX, VARIANT, "git rm -r --cached")]), "the member's own gitlink": ( lambda root: plant(root, "160000", VARIANT), - (IN_INDEX, ROW, "git update-index")), + [(IN_INDEX, ROW, "git update-index")]), } @@ -745,12 +842,12 @@ def unmerged_file_ours(root: Path) -> None: def test_only_the_members_own_gitlink_is_moved_back(holder, family_module, state): """WHICH PATH IS THE MEMBER'S OWN: a gitlink at a case variant of a - row's path, where the index holds nothing at or under that path. A - second gitlink beside the row's, a file at the variant, a gitlink under - it, a file at or under the row's path, or an unmerged variant with no - gitlink at stage 2 (none, or a file there, whatever stages 1 and 3 - hold) keeps #574's remedy.""" - make, (where, target, remedy_said) = OWN[state] + row's path, where the index holds nothing at or under that path but + files (#761), which are set aside first. A second gitlink beside the + row's, a file at the variant, a gitlink under it, or an unmerged variant + with no gitlink at stage 2 (none, or a file there, whatever stages 1 + and 3 hold) keeps #574's remedy.""" + make, said = OWN[state] if state != "a second gitlink beside the row's": git("rm", "-q", "--cached", "--", ROW, cwd=holder) make(holder) @@ -758,7 +855,7 @@ def test_only_the_members_own_gitlink_is_moved_back(holder, family_module, assert [(spelled.where, spelled.target, spelled.remedy) for spelled in family_module._members_spellings( holder, SPARE, family_module._members_on_disk(holder), - rows)] == [(where, target, remedy_said)] + rows)] == said def test_the_rows_that_have_a_path_of_their_own(family_module): diff --git a/tests/test_family_add_move_back_sections.py b/tests/test_family_add_move_back_sections.py new file mode 100644 index 00000000..ecbb906c --- /dev/null +++ b/tests/test_family_add_move_back_sections.py @@ -0,0 +1,1053 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`family.py add`'s move back of a member's gitlink held in the wrong case +(#646) and rename back of a `members/` renamed with `git mv` (#685), typed as +printed in each state git leaves them in (#761). + +1. (e) `git mv members/IRRS Members/IRRS` moves the section's `path` too. The + move back put the gitlink back and left the section naming the variant: + `git submodule update` said `No url found for submodule path + 'members/IRRS'`. Each such section is pointed back now (`git config -f + .gitmodules submodule..path `, `git add -- .gitmodules`), + in the move's commit where HEAD's section moved too. +2. (g) A file at or under the row's path beside the variant's gitlink kept + #574's remedy, which took the member's gitlink away. It is set aside + first now, with `git mv` where it is on disk, then the gitlink moved back. +3. (h) A file or link at `members` itself: each `git update-index` failed + `appears as both a file and as a directory`. It is set aside first too. +4. The plain `mv` of the variant's directory moved a live checkout out of + the holder, its `.git` file left pointing nowhere. The `mv` is guarded as + `validate-family.py` guards it (#681), the text says where the commits + are and to delete the checkout by hand, before the update, and a checkout + `add` finds on disk later is never given a plain `mv`. +5. rv685's three: a member whose gitlink's removal is staged beside a staged + `git mv members Members` is moved back with its section, not passed by; a + file under a third spelling keeps its own remedy beside the rename back; + and an entry the sparse rules leave out, its `skip-worktree` bit cleared, + is given `--sparse`. +6. Codex's two P2s on #828: what is in the way of the move back is the + member's own to set aside whatever a sparse checkout says of it, with + `--sparse` where it needs one, and the closing `mv` moves nothing, and + exits 0, where nothing is at the variant. +7. rv828's review: a `members` file HEAD holds is #684's to name first, + its removal with `--sparse` where a sparse checkout needs it, and the + rename back follows; a clone of its own, or a link, found on disk keeps + its `mv`; and nothing but a file or link in a move's way is set aside. + +States that need two directories differing only in case, or `git mv` into +one, are made on a disk that does not fold case alone; what a folding disk +answers is asked in process (`disk_as`). +""" + +from __future__ import annotations + +import os +import re +import shlex +import subprocess +from pathlib import Path + +import pytest + +from conftest import cloned_just_pushed, git, rmtree +from test_family import NAME, SPARE, family # noqa: F401 +from test_family_add_member_variant import ( + HEAD_ONLY, MOVE_OUT, OTHER_ROW, OTHER_VARIANT, REMOVE, REMOVE_SPARSE, ROW, + SUBMODULE, VARIANT, VARIANT_SHOWN, dry_run_in_process, guarded, + in_the_index, moved_back, point_back, record, refused_in_process, + registered, remedy, typed_as_printed, variant, +) +from test_family_add_refusals import ( # noqa: F401 + family_module, holder, offline, plant, +) +from test_family_add_spellings_remedies import ( + BOTH, folding_skip, renamed_back, renamed_with_git, two_steps, +) +from test_family_head_lost_gitmodules import assert_passes_in_every_mode +from test_family_holder_states import DELETED_LINE, DELETED_REMEDY +from test_family_members_case_variants import ( + DIR_VARIANT, NEEDS_GIT_2_34, NEEDS_GIT_2_38, added, disk_as, disk_folds, + git_version, index_part, refused_untouched, remedy_of, +) +from test_family_members_case_variants import move as untracked_moved +from test_family_refuse_up_front import needs_check_rules + +#: Both sections pointed back. +SECTIONS = (ROW, OTHER_ROW) +#: The two gitlinks under `Members/` taken out in one line, with `-f` where +#: a staged one has its checkout on disk. +REMOVE_BOTH = (f"git rm -r --cached --ignore-unmatch -- {VARIANT} " + f"{OTHER_VARIANT}") +REMOVE_BOTH_FORCED = REMOVE_BOTH.replace("--cached", "--cached -f") +REMOVE_FORCED = REMOVE.replace("--cached", "--cached -f") +POINTED = [point_back(), point_back(OTHER_ROW), "git add -- .gitmodules"] + + +def mount_moved(root: Path, committed: bool = True) -> str: + """`git mv members/IRRS Members/IRRS`, which moves the checkout, the + gitlink and the section's `path`, and its commit where `committed`. + Returns the commit the gitlink records.""" + commit = git("rev-parse", f"HEAD:{ROW}", cwd=root).stdout.strip() + (root / "Members").mkdir(exist_ok=True) + git("mv", "members/IRRS", VARIANT, cwd=root) + if committed: + git("commit", "-qm", "git mv members/IRRS Members/IRRS", cwd=root) + return commit + + +def work_there(root: Path, name: str = VARIANT) -> str: + """A commit of its own made in the checkout at `name`, and a file not + committed, written as BYTES. Returns the commit.""" + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", + "--allow-empty", "-m", "Work there", cwd=root / name) + (root / name / "WIP.txt").write_bytes(b"not committed\n") + return git("rev-parse", "HEAD", cwd=root / name).stdout.strip() + + +def deleted_by_hand(*names: str): + """What the text says of each checkout at `names`, done by hand, before + the update: `git -C status` first names the file not committed + there (the update would point the store at the row's path, and it then + names none), then the checkout is deleted.""" + def by_hand(root: Path) -> None: + for name in names: + if not (root / name / ".git").is_file(): + continue + status = git("-C", name, "status", "--porcelain", cwd=root) + assert "?? WIP.txt" in status.stdout.splitlines(), status.stdout + rmtree(root / name) + return by_hand + + +def kept_in_the_store(root: Path, commit: str, row: str = ROW) -> None: + """`commit` is in the member's store, `.git/modules/`, the + checkout it was made in deleted.""" + store = root / ".git" / "modules" / row + kept = subprocess.run(["git", f"--git-dir={store}", "cat-file", "-e", + commit], capture_output=True, check=False) + assert kept.returncode == 0, kept.stderr + + +def gate_then_add(offline, root: Path, remotes: Path) -> None: + """The holder's gate in every mode, the same `add`, the gate again.""" + assert_passes_in_every_mode(root, 2) + added(offline, root, remotes) + assert_passes_in_every_mode(root, 3) + + +# --- 1. (e) a section git mv moved ------------------------------------------- + +@pytest.mark.parametrize("committed", [True, False], + ids=["committed", "staged"]) +def test_a_mount_moved_with_git_mv_is_moved_back_with_its_section( + family, holder, offline, tmp_path, committed): + """(e) `git mv members/IRRS Members/IRRS`, the member's commit and a + file not committed in its checkout there. Typed as printed: the gitlink + and the section are back at `members/IRRS` (HEAD's `.gitmodules` byte + for byte the holder's), the commit only where HEAD's section moved + (none after a staged move, where `git commit` would fail), the removal + with `-f` where the move is staged, which `git rm --cached` refused + beside that checkout's work, the guarded `mv` keeps the checkout, `git + -C Members/IRRS status` names the file not committed, the checkout is + deleted by hand, and `git submodule update --init` checks the member + out at its row's path with the commit still in its store. The gate + passes and the same `add` goes through.""" + folding_skip(tmp_path) + commit = mount_moved(holder, committed) + work = work_there(holder) + move, closing = moved_back({ROW: ([VARIANT], commit)}, commit=committed, + sections=(ROW,), forced=not committed) + said = refused_untouched(offline, holder, family["remotes"], "--dry-run") + assert said == (in_the_index(VARIANT_SHOWN), remedy_of(move, *closing)) + assert typed_as_printed(holder, said[1], deleted_by_hand(VARIANT)) == [ + REMOVE if committed else REMOVE_FORCED, record(commit), point_back(), + "git add -- .gitmodules", *(["git commit"] if committed else []), + MOVE_OUT, SUBMODULE] + registered(holder, family["root"], commit) + assert (holder / ROW / "project.yaml").is_file() + kept_in_the_store(holder, work) + gate_then_add(offline, holder, family["remotes"]) + + +def test_a_fresh_clone_of_it_is_moved_back_with_its_section( + family, holder, offline, tmp_path): + """(e), committed and pushed, and a fresh recursive clone: its section + gives `Members/IRRS`, so the member is checked out there. The same + text; typed as printed, with that checkout deleted by hand, the clone's + gate passes and the same `add` goes through.""" + folding_skip(tmp_path) + commit = mount_moved(holder) + remote = family["remotes"] / f"{NAME}.git" + branch = "w761/mount-moved" + git("push", "-q", str(remote), f"HEAD:refs/heads/{branch}", cwd=holder) + clone = cloned_just_pushed(remote, tmp_path / "clone" / NAME, branch) + assert (clone / VARIANT / ".git").is_file(), "premise: checked out there" + move, closing = moved_back({ROW: ([VARIANT], commit)}, sections=(ROW,)) + said = refused_untouched(offline, clone, family["remotes"], "--dry-run") + assert said == (in_the_index(VARIANT_SHOWN), remedy_of(move, *closing)) + assert typed_as_printed(clone, said[1], + lambda root: rmtree(root / VARIANT)) == [ + REMOVE, record(commit), *POINTED[::2], "git commit", MOVE_OUT, + SUBMODULE] + registered(clone, family["root"], commit) + gate_then_add(offline, clone, family["remotes"]) + + +@pytest.mark.parametrize("folding", [False, True], ids=["linux", "folding"]) +def test_every_disk_points_the_section_back(family, holder, family_module, + offline, monkeypatch, capsys, + tmp_path, folding): + """(e) asked in process of a disk that folds case and of one that does + not: one text, the section line on both.""" + folding_skip(tmp_path) + commit = mount_moved(holder) + disk_as(monkeypatch, holder, folding) + move, closing = moved_back({ROW: ([VARIANT], commit)}, sections=(ROW,)) + assert dry_run_in_process(family_module, monkeypatch, offline, holder, + family["remotes"], capsys) == ( + in_the_index(VARIANT_SHOWN), remedy_of(move, *closing)) + + +def test_a_sparse_gitmodules_is_named_first_where_a_section_is_pointed_back( + family, holder, offline, tmp_path): + """(e), and a `.gitmodules` a sparse checkout marks: `git config -f` + would write a new file of the one line, which `git add` refuses, so + `gitmodules-sparse` names it first, with the holder as it was.""" + folding_skip(tmp_path) + mount_moved(holder) + git("update-index", "--skip-worktree", "--", ".gitmodules", cwd=holder) + first, _ = refused_untouched(offline, holder, family["remotes"]) + assert first.startswith("REFUSED gitmodules-sparse: "), first + + +# --- 2. (g) and 3. (h): what is in the way, set aside first ------------------ + +def set_aside(entries: list[str], kinds: dict, renamed=(), out=(), + sparse=(), out_sparse: bool = False) -> str: + """The part of the remedy that sets `entries` aside: `renamed` with `git + mv`, `sparse` with `git mv --sparse`, `out` taken out of the index, with + `--sparse` where `out_sparse` (Codex's P2 on #828).""" + how = [] + if renamed: + how.append("rename each that is on disk as the index holds it with " + "`git mv` (" + " and ".join(f"`git mv -- {name} `" + for name in renamed) + ")") + if sparse: + how.append("rename each that the sparse checkout leaves out with `git " + f"mv --sparse`, {NEEDS_GIT_2_38} (" + + " and ".join(f"`git mv --sparse -- {name} `" + for name in sparse) + ")") + if out: + how.append("take each that is not on disk so out of the index, which " + "touches no file (`git rm -r --cached" + f"{' --sparse' if out_sparse else ''} -- {' '.join(out)}`" + + (f", {NEEDS_GIT_2_34}" if out_sparse else "") + ")") + named = ", ".join(f"`{name}` ({kinds.get(name, 'a file')})" + for name in entries) + return ("set aside first what the index holds in the way of the move of " + f"a member's gitlink below, {named}, since git's update-index " + "replaces a file at the path it records without a word and fails " + "beside one under it or at members itself, and taking the " + "gitlink out instead un-registers the member: " + + " and ".join(how)) + + +def file_committed(root: Path, name: str) -> bytes: + """`name` written as BYTES and committed. Returns the bytes.""" + data = b"our own notes\n" + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_bytes(data) + git("add", "--", name, cwd=root) + git("commit", "-qm", f"A file at {name}", cwd=root) + return data + + +@pytest.mark.parametrize("entry", [ROW, f"{ROW}/notes"], + ids=["at the row's path", "under it"]) +def test_a_file_at_or_under_the_rows_path_is_set_aside_first( + family, holder, offline, tmp_path, entry): + """(g) `git mv members/IRRS Members/IRRS`, then a file committed at the + row's path or under it. #574's `git mv -- Members/IRRS ` took + the member away from its row. Typed as printed: the file renamed with + `git mv`, then the move back and the section in the same commit, the + checkout deleted by hand, the update; the file's bytes are at its new + name in HEAD, the member is where its row says, the gate passes and the + same `add` goes through.""" + folding_skip(tmp_path) + commit = mount_moved(holder) + data = file_committed(holder, entry) + move, closing = moved_back({ROW: ([VARIANT], commit)}, sections=(ROW,)) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (in_the_index(VARIANT_SHOWN), + remedy_of(set_aside([entry], {}, renamed=[entry]), move, + *closing)) + assert typed_as_printed(holder, said[1], + lambda root: rmtree(root / VARIANT)) == [ + f"git mv -- {entry} renamed-1", REMOVE, record(commit), *POINTED[::2], + "git commit", MOVE_OUT, SUBMODULE] + assert git("show", "HEAD:renamed-1", cwd=holder).stdout.encode() == data + registered(holder, family["root"], commit) + gate_then_add(offline, holder, family["remotes"]) + + +def test_a_file_where_the_disk_holds_the_checkout_is_taken_out_first( + family, holder, offline): + """(g) #646's state, the member's checkout still at `members/IRRS`, and + a file the index holds there, which `git mv` cannot take: it is taken + out of the index, which touches no file, then the gitlink is moved + back over the checkout.""" + commit = variant(holder) + plant(holder, "100644", ROW) + move, closing = moved_back({ROW: ([VARIANT], commit)}) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (in_the_index(VARIANT_SHOWN), + remedy_of(set_aside([ROW], {}, out=[ROW]), move, + *closing)) + assert typed_as_printed(holder, said[1]) == [ + f"git rm -r --cached -- {ROW}", REMOVE, record(commit), "git commit", + MOVE_OUT] + registered(holder, family["root"], commit) + gate_then_add(offline, holder, family["remotes"]) + + +def members_staged(root: Path) -> None: + """A file `members` staged and never committed, so HEAD holds none and + #684's refusal (`_members_file_left_in_head`) passes it by.""" + (root / "members").write_bytes(b"our own notes\n") + git("add", "--", "members", cwd=root) + + +def members_staged_deleted(root: Path) -> tuple: + """That file deleted from disk: taken out of the index.""" + members_staged(root) + (root / "members").unlink() + return set_aside(["members"], {}, out=["members"]), \ + "git rm -r --cached -- members" + + +def members_a_link(root: Path) -> tuple: + """A link `members -> Members` committed: renamed with `git mv`.""" + if os.name == "nt": + pytest.skip("a symlink needs a privilege Windows runners do not grant") + os.symlink("Members", root / "members", target_is_directory=True) + git("add", "--", "members", cwd=root) + git("commit", "-qm", "A link members -> Members", cwd=root) + return (set_aside(["members"], {"members": "a symlink"}, + renamed=["members"]), "git mv -- members renamed-1") + + +def members_staged_marked(root: Path) -> tuple: + """That file marked `skip-worktree` and kept off disk, as a sparse + checkout keeps it: it was left where it was, so each `git update-index` + still failed beside it. Renamed with `git mv --sparse`, which keeps what + it holds, as the index's own rule names such an entry (Codex's P2 1 on + #828).""" + if git_version() < (2, 38): + pytest.skip("`git mv --sparse` of one entry kept off disk needs git " + "2.38 or later") + members_staged(root) + git("update-index", "--skip-worktree", "--", "members", cwd=root) + (root / "members").unlink() + return (set_aside(["members"], {}, sparse=["members"]), + "git mv --sparse -- members renamed-1") + + +@pytest.mark.parametrize("make", [members_staged_deleted, members_a_link, + members_staged_marked], + ids=["a file staged and deleted from disk", "a link", + "a file staged and kept off disk by a sparse " + "checkout"]) +def test_an_entry_at_members_itself_is_set_aside_first(family, holder, + offline, tmp_path, + make): + """(h) `git mv members Members`, committed, and the index holds a link at + `members`, or a file HEAD does not hold, deleted from disk or kept off + it by a sparse checkout: each `git update-index` failed `appears as both + a file and as a directory`, exit 128. Typed as printed: the entry set + aside, both gitlinks and sections moved back in one commit, both + checkouts deleted by hand, the update; the gate passes and the same + `add` goes through. A file HEAD holds is #684's to name first, below.""" + folding_skip(tmp_path) + commits = renamed_with_git(holder, True) + aside, typed_first = make(holder) + move, closing = moved_back({ROW: ([VARIANT], commits[ROW]), + OTHER_ROW: ([OTHER_VARIANT], + commits[OTHER_ROW])}, + sections=SECTIONS) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (BOTH, remedy_of(aside, move, *closing)) + assert typed_as_printed(holder, said[1], lambda root: [ + rmtree(root / name) for name in (VARIANT, OTHER_VARIANT)]) == [ + typed_first, REMOVE_BOTH, record(commits[ROW]), + record(commits[OTHER_ROW], OTHER_ROW), *POINTED, "git commit", + MOVE_OUT, guarded(OTHER_VARIANT, OTHER_ROW), SUBMODULE] + for row, commit in commits.items(): + registered(holder, family["root"], commit, row) + gate_then_add(offline, holder, family["remotes"]) + + +#: #684's refusal of a `members` file HEAD and the index hold, kept off disk +#: by a sparse checkout: its removal takes `--sparse` (Codex's P2 1 on #828). +SPARSE_LINE = DELETED_LINE.replace("its deletion not staged", + "as the sparse checkout leaves it out") +SPARSE_REMEDY = DELETED_REMEDY.replace( + "(`git rm --cached -- members`)", + f"(`git rm --cached --sparse -- members`, {NEEDS_GIT_2_34})") + + +def head_members_deleted(root: Path) -> tuple: + """Deleted from disk, the deletion not staged: #684's own state.""" + (root / "members").unlink() + return DELETED_LINE, DELETED_REMEDY + + +def head_members_marked(root: Path) -> tuple: + """Marked `skip-worktree` and kept off disk, as a sparse checkout keeps + it: plain `git rm --cached` refuses it, exit 1.""" + git("update-index", "--skip-worktree", "--", "members", cwd=root) + (root / "members").unlink() + return SPARSE_LINE, SPARSE_REMEDY + + +def head_members_left_out(root: Path) -> tuple: + """Left out by non-cone rules that leave out the file `members` alone + (`/members/` keeps the directory in), its bit cleared and off disk: + plain `git rm --cached` refuses it, exit 1.""" + git("sparse-checkout", "set", "--no-cone", "/*", "!/members", + "/members/", cwd=root) + bit_cleared(root, "members", False, False) + return SPARSE_LINE, SPARSE_REMEDY + + +def typed_684(root: Path, remedy: str) -> list[str]: + """#684's remedy, typed as printed: its `git rm --cached` line, then the + commit it asks for, on its own.""" + (line,) = [command for command in re.findall(r"`([^`]*)`", remedy) + if command.startswith("git rm ")] + git(*shlex.split(line)[1:], cwd=root) + git("commit", "-qm", "The removal, on its own", cwd=root) + return [line, "git commit"] + + +#: `{state: how the committed file `members` is left off disk}`. +HEAD_MEMBERS = { + "deleted from disk": head_members_deleted, + "kept off disk by a sparse checkout": head_members_marked, + "left out by the sparse rules": head_members_left_out, +} + + +@pytest.mark.parametrize("state", [ + pytest.param(state, marks=needs_check_rules) if "rules" in state + else state for state in HEAD_MEMBERS]) +def test_a_members_file_head_holds_is_taken_out_then_renamed_back( + family, holder, offline, tmp_path, state): + """(h) once main holds #684 (rv828's MUST 1): `git mv members Members`, + committed, then a file `members` committed and left off disk. #684's + `member-path-unaddable` names it before `_members_spellings` is asked, + so the set-aside is never reached. Its `git rm --cached -- members` + typed as printed refused the file a sparse checkout marks or leaves out + (`exist outside of your sparse-checkout definition`, exit 1), and now + takes `--sparse` there. Typed as printed, the removal and its commit, + then the rename back the next `add` prints: both members are where their + rows say, the gate passes and the same `add` goes through.""" + folding_skip(tmp_path) + commits = renamed_with_git(holder, True) + file_committed(holder, "members") + line, remedy = HEAD_MEMBERS[state](holder) + assert refused_untouched(offline, holder, family["remotes"]) == (line, + remedy) + assert typed_684(holder, remedy) == [ + re.search(r"`(git rm [^`]*)`", remedy).group(1), "git commit"] + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (BOTH, remedy_of(renamed_back(True))) + assert typed_as_printed(holder, said[1]) == [*two_steps(), "git commit"] + for row, commit in commits.items(): + registered(holder, family["root"], commit, row) + gate_then_add(offline, holder, family["remotes"]) + + +# --- 4. a live checkout is never moved by a plain mv ------------------------ + +def test_a_checkout_left_at_the_variant_is_named_to_delete_by_hand( + family, holder, offline, tmp_path): + """(e) typed as printed with the hand step skipped: the update then + points the member's store at its row's path, and `git -C Members/IRRS + status` names nothing, though a file not committed is there (why the + text puts the hand step first). The next `add` names the checkout to + delete by hand, with no `mv`; deleted, the commit is still in the store, + and the gate and `add` pass.""" + folding_skip(tmp_path) + commit = mount_moved(holder) + work = work_there(holder) + said = refused_untouched(offline, holder, family["remotes"]) + typed_as_printed(holder, said[1]) + assert (holder / VARIANT / "WIP.txt").is_file() + assert git("-C", VARIANT, "status", "--porcelain", + cwd=holder).stdout == "" + first, again = refused_untouched(offline, holder, family["remotes"]) + assert first == DIR_VARIANT + (f"on disk and untracked: `{VARIANT}` (a " + "directory)") + assert again == remedy_of( + f"delete by hand each checkout on disk and untracked, `{VARIANT}`, " + "once nothing only it holds is needed, rather than move it: its .git " + "file names its store by a path relative to it, which a move would " + "leave pointing nowhere, and its commits are kept in that store, so " + "only what is not committed is in it alone") + assert re.findall(r"`mv ", again) == [] + assert typed_as_printed(holder, again, + lambda root: rmtree(root / VARIANT)) == [] + registered(holder, family["root"], commit) + kept_in_the_store(holder, work) + gate_then_add(offline, holder, family["remotes"]) + + +def a_clone_of_its_own(root: Path, remotes: Path) -> str: + """An untracked clone of a member at `Members/X`, its `.git` a + directory, which holds its commits: the `mv` moves it whole.""" + cloned_just_pushed(remotes / "IRRS.git", root / "Members" / "X", + recurse=False) + return "a directory" + + +def a_link_to_a_checkout(root: Path, remotes: Path) -> str: + """A link `Members/X -> ../members/IRRS`, the member's own checkout: a + link is moved, never followed to the checkout it names.""" + if os.name == "nt": + pytest.skip("a symlink needs a privilege Windows runners do not grant") + (root / "Members").mkdir() + os.symlink("../members/IRRS", root / "Members" / "X", + target_is_directory=True) + return "a symlink" + + +#: `{state: how it is laid out, returning what the refusal calls it}`. +MOVED_WHOLE = {"a clone of its own": a_clone_of_its_own, + "a link to a checkout": a_link_to_a_checkout} + + +@pytest.mark.parametrize("state", list(MOVED_WHOLE)) +def test_what_is_on_disk_and_no_checkout_keeps_its_mv(family, holder, + offline, tmp_path, + state): + """rv828's SHOULD 2, `_moved_or_kept`: only a directory, not a link, + holding a `.git` FILE is a checkout to delete by hand. An untracked + clone of its own at `Members/X`, or a link there to the member's + checkout, keeps #574's `mv`. Typed as printed, the `mv` moves it out of + the holder whole, and the same `add` goes through.""" + if disk_folds(tmp_path): + pytest.skip("`Members/X` beside `members/` needs a disk that does " + "not fold case") + kind = MOVED_WHOLE[state](holder, family["remotes"]) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (DIR_VARIANT + f"on disk and untracked: `Members/X` " + f"({kind})", remedy_of(untracked_moved("Members/X"))) + assert typed_as_printed(holder, said[1]) == [ + "mv -- Members/X ../moved-1"] + assert os.path.lexists(holder.parent / "moved-1") + added(offline, holder, family["remotes"]) + + +# --- 5. rv685's three -------------------------------------------------------- + +@pytest.mark.parametrize("committed", [False, True], + ids=["staged", "committed"]) +def test_a_members_removal_staged_beside_the_rename_is_moved_back( + family, holder, offline, tmp_path, committed): + """rv685's candidate 1: `git mv members Members`, then `git rm --cached + Members/IRSS`. Staged, HEAD records `members/IRSS` and the index holds + it nowhere: nothing named it, the rename back left its section at + `Members/IRSS`, `add` committed that, and the gate refused. Now it is + named, put back at HEAD's commit beside IRRS's move, with both sections + and no commit, which nothing would be left to make. Committed, HEAD + holds `Members/IRSS`: the move back from HEAD, with both sections and + the commit. Typed as printed, both members are where their rows say, + the gate passes and the same `add` goes through.""" + folding_skip(tmp_path) + commits = renamed_with_git(holder, committed) + git("rm", "-q", "--cached", "--", OTHER_VARIANT, cwd=holder) + rows = {ROW: ([VARIANT], commits[ROW])} + staged = {} + if committed: + rows[OTHER_ROW] = ([OTHER_VARIANT], commits[OTHER_ROW]) + where = "in HEAD and no longer in the index" + else: + staged = {OTHER_ROW: (OTHER_VARIANT, commits[OTHER_ROW])} + where = "in .gitmodules, its gitlink's removal staged" + move, closing = moved_back(rows, commit=committed, sections=SECTIONS, + staged=staged, forced=not committed) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (in_the_index(VARIANT_SHOWN) + + f"; {where}: `{OTHER_VARIANT}` (a submodule)", + remedy_of(move, *closing)) + assert typed_as_printed(holder, said[1], lambda root: [ + rmtree(root / name) for name in (VARIANT, OTHER_VARIANT)]) == [ + REMOVE_BOTH if committed else REMOVE_BOTH_FORCED, + record(commits[ROW]), record(commits[OTHER_ROW], OTHER_ROW), + *POINTED, *(["git commit"] if committed else []), MOVE_OUT, + guarded(OTHER_VARIANT, OTHER_ROW), SUBMODULE] + for row, commit in commits.items(): + registered(holder, family["root"], commit, row) + gate_then_add(offline, holder, family["remotes"]) + + +def test_a_third_spelling_keeps_its_own_remedy_beside_the_rename_back( + family, holder, offline, tmp_path): + """rv685's NIT 5: `git mv members Members`, committed, and a file + committed at `MEMBERS/notes`. The rename back was withheld for the move + back, whose section gap (e) then left. Now the file keeps its own + remedy, run first, and the rename back follows. Typed as printed, both + members are where their rows say, the gate passes and the same `add` + goes through.""" + folding_skip(tmp_path) + commits = renamed_with_git(holder, True) + file_committed(holder, "MEMBERS/notes") + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (DIR_VARIANT + "in the index: `MEMBERS/notes` (a file), " + f"{VARIANT_SHOWN}, `{OTHER_VARIANT}` (a submodule)", + remedy_of(index_part(rename=["MEMBERS/notes"], + commit="the rename"), + renamed_back(True))) + assert typed_as_printed(holder, said[1]) == [ + "git mv -- MEMBERS/notes renamed-1", "git commit", *two_steps(), + "git commit"] + for row, commit in commits.items(): + registered(holder, family["root"], commit, row) + gate_then_add(offline, holder, family["remotes"]) + + +@pytest.mark.parametrize("folding", [False, True], ids=["linux", "folding"]) +def test_every_disk_renames_back_beside_a_third_spelling( + family, holder, family_module, offline, monkeypatch, capsys, + tmp_path, folding): + """The same, the third spelling in the index alone, as a disk that folds + case holds it, where the three spellings are one directory and + `MEMBERS` is not listed: taken out of the index, which touches no file, + then the rename back, on both disks.""" + folding_skip(tmp_path) + renamed_with_git(holder, True) + plant(holder, "100644", "MEMBERS/notes") + disk_as(monkeypatch, holder, folding) + assert dry_run_in_process(family_module, monkeypatch, offline, holder, + family["remotes"], capsys) == ( + DIR_VARIANT + "in the index: `MEMBERS/notes` (a file), " + f"{VARIANT_SHOWN}, `{OTHER_VARIANT}` (a submodule)", + remedy_of(index_part(unstage=["MEMBERS/notes"]), renamed_back(True))) + + +def sparse_cone(root: Path) -> None: + """A cone-mode sparse checkout keeping `members/` and every directory + the gate reads, which leaves out `Members/`.""" + if git_version() < (2, 38): + pytest.skip("`git mv --sparse` of one entry needs git 2.38 or later") + git("sparse-checkout", "init", "--cone", cwd=root) + git("sparse-checkout", "set", "members", "contracts", "scripts", + ".github", cwd=root) + + +def bit_cleared(root: Path, name: str, on_disk: bool, directory: bool) -> None: + """`name`'s `skip-worktree` bit cleared, and it put back on disk, a + directory or a file written as BYTES, where `on_disk`.""" + if on_disk and directory: + (root / name).mkdir(parents=True, exist_ok=True) + elif on_disk: + (root / name).parent.mkdir(parents=True, exist_ok=True) + (root / name).write_bytes(b"our own notes\n") + git("update-index", "--no-skip-worktree", "--", name, cwd=root) + assert git("ls-files", "-t", "--", name, cwd=root).stdout.startswith("H ") + + +def readme_on_disk(root: Path) -> tuple: + """`Members/README` on disk: plain `git mv` refuses it.""" + file_committed(root, "Members/README") + sparse_cone(root) + bit_cleared(root, "Members/README", True, False) + return (DIR_VARIANT + "in the index: `Members/README` (a file)", + remedy_of(index_part(sparse=["Members/README"], + commit="the rename")), + ["git mv --sparse -- Members/README renamed-1", "git commit"]) + + +def readme_off_disk(root: Path) -> tuple: + """`Members/README` off disk: plain `git rm --cached` refuses it.""" + file_committed(root, "Members/README") + sparse_cone(root) + bit_cleared(root, "Members/README", False, False) + return (DIR_VARIANT + "in the index: `Members/README` (a file)", + remedy_of("take what has nothing on disk in that spelling out of " + "the index, which touches no file (`git rm -r --cached " + "--sparse -- Members/README`, which needs git 2.34 or " + "later), and commit the removal"), + ["git rm -r --cached --sparse -- Members/README", "git commit"]) + + +def own_gitlink_on_disk(root: Path) -> tuple: + """#646's state, `Members/IRRS` an empty directory on disk: plain `git + rm --cached --ignore-unmatch` passes it over, exit 0, and the move left + two gitlinks.""" + commit = variant(root) + sparse_cone(root) + bit_cleared(root, VARIANT, True, True) + move, closing = moved_back({ROW: ([VARIANT], commit)}, sparse=True) + return (in_the_index(VARIANT_SHOWN), remedy_of(move, *closing), + [REMOVE_SPARSE, record(commit), "git commit", MOVE_OUT]) + + +SPARSE = { + "a file on disk": readme_on_disk, + "a file off disk": readme_off_disk, + "a member's own gitlink on disk": own_gitlink_on_disk, +} + + +@needs_check_rules +@pytest.mark.parametrize("state", list(SPARSE)) +def test_an_entry_the_sparse_rules_leave_out_takes_sparse_whatever_its_bit( + family, holder, offline, state): + """rv685's candidate 2: a cone-mode sparse checkout leaves `Members/` + out, and an entry there has its `skip-worktree` bit cleared. Plain `git + mv` (on disk) and `git rm --cached` (off disk) refuse it, exit 1, `exist + outside of your sparse-checkout definition`, and the move back's removal + passes it over. Each is given `--sparse` and the git it needs; typed as + printed, the gate passes and the same `add` goes through.""" + first, said, typed = SPARSE[state](holder) + assert refused_untouched(offline, holder, family["remotes"]) == (first, + said) + assert typed_as_printed(holder, said) == typed + gate_then_add(offline, holder, family["remotes"]) + + +@needs_check_rules +def test_a_gitlink_the_sparse_rules_leave_out_is_renamed_with_sparse( + family, holder, offline): + """rv685's candidate 2, on a gitlink, as the issue asks (`Members/X`, on + disk as a directory, its bit cleared): plain `git mv` would take it, and + `git mv --sparse` takes it too, so the one rule holds; typed as printed, + the same `add` goes through.""" + commit = git("rev-parse", f"HEAD:{ROW}", cwd=holder).stdout.strip() + git("update-index", "--add", "--cacheinfo", f"160000,{commit},Members/X", + cwd=holder) + git("commit", "-qm", "A gitlink Members/X", cwd=holder) + sparse_cone(holder) + bit_cleared(holder, "Members/X", True, True) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (DIR_VARIANT + "in the index: `Members/X` (a submodule)", + remedy_of(index_part(sparse=["Members/X"], + commit="the rename"))) + assert typed_as_printed(holder, said[1]) == [ + "git mv --sparse -- Members/X renamed-1", "git commit"] + result = offline.add(holder, SPARE, remotes=family["remotes"]) + assert result.returncode == 0, result.stdout + result.stderr + + +@needs_check_rules +def test_the_rename_back_is_withheld_where_the_rules_leave_an_entry_out( + holder, family_module, tmp_path): + """The rename back's first `git mv` refuses a directory holding an entry + the sparse rules leave out, its bit cleared (exit 1, measured), so the + move back is printed, its removal with `--sparse`.""" + folding_skip(tmp_path) + renamed_with_git(holder, True) + sparse_cone(holder) + for name in (VARIANT, OTHER_VARIANT): + bit_cleared(holder, name, True, True) + _, said = refused_in_process(family_module, holder) + assert "git mv -- Members members.renaming" not in said + assert f"`git rm -r --cached --sparse --ignore-unmatch -- {VARIANT} " \ + f"{OTHER_VARIANT}`, {NEEDS_GIT_2_34}, " in said + + +# --- what each rule turns on ------------------------------------------------- + +def test_a_section_head_moved_is_a_change_to_commit(holder, family_module, + tmp_path): + """HEAD records the gitlink at its row's path and its section at + `Members/IRRS`, as #646's move back left it typed after a committed + `git mv` of the mount, and the index stages the gitlink at the variant + again: the move alone leaves nothing to commit, but pointing the + section back does, so the commit is printed.""" + folding_skip(tmp_path) + commit = mount_moved(holder) + git("rm", "-q", "--cached", "--", VARIANT, cwd=holder) + git("update-index", "--add", "--cacheinfo", f"160000,{commit},{ROW}", + cwd=holder) + git("commit", "-qm", "#646's move back, its section left", cwd=holder) + git("rm", "-q", "--cached", "--", ROW, cwd=holder) + git("update-index", "--add", "--cacheinfo", f"160000,{commit},{VARIANT}", + cwd=holder) + move, closing = moved_back({ROW: ([VARIANT], commit)}, sections=(ROW,), + forced=True) + assert refused_in_process(family_module, holder)[1] == remedy_of( + move, *closing) + + +def kept_off(root: Path) -> None: + """The new file at the row's path marked `skip-worktree` and kept off + disk, as a sparse checkout keeps it.""" + if git_version() < (2, 38): + pytest.skip("`git mv --sparse` of one entry kept off disk needs git " + "2.38 or later") + git("update-index", "--skip-worktree", "--", ROW, cwd=root) + (root / ROW).unlink() + + +def deleted(root: Path) -> None: + """The new file at the row's path deleted from disk, not marked.""" + (root / ROW).unlink() + + +#: `{state: (what is done to the new file once it is staged, the set-aside +#: part's arguments, the line typed for it, whether a commit is printed)}`. +STAGED_ASIDE = { + "on disk": (None, {"renamed": [ROW]}, f"git mv -- {ROW} renamed-1", + True), + "kept off disk by a sparse checkout": ( + kept_off, {"sparse": [ROW]}, f"git mv --sparse -- {ROW} renamed-1", + True), + "deleted from disk": (deleted, {"out": [ROW]}, + f"git rm -r --cached -- {ROW}", False), +} + + +@pytest.mark.parametrize("state", list(STAGED_ASIDE)) +def test_what_is_set_aside_is_committed_with_the_move(family, holder, + offline, tmp_path, + state): + """(g) staged: `git mv members/IRRS Members/IRRS` and a new file at the + row's path, neither committed, so the move back alone leaves nothing to + commit. + - On disk, the file renamed aside would be left staged, so the commit + is printed. + - Kept off disk by a sparse checkout, it is renamed with `git mv + --sparse`, which leaves a change to commit all the same (Codex's P2 1 + on #828). + - Deleted from disk, it is taken out of the index. HEAD records the + member's gitlink at that path, which the move puts back, so nothing + is left to commit and no commit is printed: one was, and `git commit` + failed with nothing to commit (found in #828's first fix round). + Typed as printed, HEAD holds the file at its new name where it was + renamed, the member is where its row says, the gate passes and the same + `add` goes through.""" + left, how, typed_first, committed = STAGED_ASIDE[state] + folding_skip(tmp_path) + commit = mount_moved(holder, committed=False) + (holder / ROW).write_bytes(b"our own notes\n") + git("add", "--", ROW, cwd=holder) + if left is not None: + left(holder) + move, closing = moved_back({ROW: ([VARIANT], commit)}, commit=committed, + sections=(ROW,), forced=True) + said = refused_untouched(offline, holder, family["remotes"]) + assert said[1] == remedy_of(set_aside([ROW], {}, **how), move, *closing) + assert typed_as_printed(holder, said[1], + lambda root: rmtree(root / VARIANT)) == [ + typed_first, REMOVE_FORCED, record(commit), *POINTED[::2], + *(["git commit"] if committed else []), MOVE_OUT, SUBMODULE] + if committed: + assert git("show", "HEAD:renamed-1", cwd=holder).stdout == ( + "our own notes\n") + registered(holder, family["root"], commit) + gate_then_add(offline, holder, family["remotes"]) + + +def test_both_removals_staged_with_no_gitmodules_on_disk(family, holder, + offline, tmp_path): + """A staged `git mv members Members`, both gitlinks then taken out of the + index, and `.gitmodules` deleted from the work tree: nothing the index + holds is a gitlink, yet both sections are pointed back, so the copy the + index holds is put back first, where `git config -f` would otherwise + write a new file of one line. Typed as printed, both members are where + their rows say, the gate passes and the same `add` goes through.""" + folding_skip(tmp_path) + commits = renamed_with_git(holder) + git("rm", "-q", "--cached", "--", VARIANT, OTHER_VARIANT, cwd=holder) + (holder / ".gitmodules").unlink() + move, closing = moved_back({}, commit=False, sections=SECTIONS, staged={ + ROW: (VARIANT, commits[ROW]), + OTHER_ROW: (OTHER_VARIANT, commits[OTHER_ROW])}) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == ( + DIR_VARIANT + "in .gitmodules, its gitlink's removal staged: " + f"{VARIANT_SHOWN}, `{OTHER_VARIANT}` (a submodule)", + remedy_of("put back the copy of .gitmodules the index holds, which " + "the work tree lacks, since git edits each member's section " + "there (`git checkout -- .gitmodules`)", move, *closing)) + assert typed_as_printed(holder, said[1], lambda root: [ + rmtree(root / name) for name in (VARIANT, OTHER_VARIANT)]) == [ + "git checkout -- .gitmodules", REMOVE_BOTH, record(commits[ROW]), + record(commits[OTHER_ROW], OTHER_ROW), *POINTED, MOVE_OUT, + guarded(OTHER_VARIANT, OTHER_ROW), SUBMODULE] + for row, commit in commits.items(): + registered(holder, family["root"], commit, row) + gate_then_add(offline, holder, family["remotes"]) + + +def no_gitlink_moved(root: Path) -> None: + """Every member's gitlink taken out of the index, and a file planted at + `members`: nothing is moved back, so nothing is in a move's way.""" + git("rm", "-r", "-q", "--cached", "--", "members", cwd=root) + plant(root, "100644", "members") + + +def a_gitlink_at_members(root: Path) -> None: + """`git mv members Members`, committed, and a gitlink planted at + `members`: the move back moves both members' gitlinks, and the gitlink + is no file or link to set aside.""" + renamed_with_git(root, True) + plant(root, "160000", "members") + + +#: `{state: how it is made}`, each with nothing to set aside. +NOTHING_ASIDE = {"no gitlink moved back": no_gitlink_moved, + "a gitlink at members": a_gitlink_at_members} + + +@pytest.mark.parametrize("state", list(NOTHING_ASIDE)) +def test_only_a_file_or_link_in_a_moves_way_is_set_aside(holder, + family_module, + tmp_path, state): + """rv828's NIT 5: what is set aside is a file or link, stage 0, at + `members` or at or under a row's path, and only where a gitlink is moved + back. A file at `members` with no gitlink to move back is in no move's + way, and a gitlink at `members` is never taken out of the index, which + would un-register what it records.""" + if state == "a gitlink at members": + folding_skip(tmp_path) + NOTHING_ASIDE[state](holder) + rows = family_module._rows_paths(family_module.Family.open(holder).members) + found = family_module._members_spellings( + holder, SPARE, family_module._members_on_disk(holder), rows) + assert [spelled.name for spelled in found + if spelled.remedy in family_module.SET_ASIDE] == [] + + +def test_a_checkout_under_an_index_entry_is_the_indexs_to_name( + holder, family_module): + """A checkout on disk at `Members/X` (a `.git` file) under which the + index holds `Members/X/notes`: the index's remedy names that path, and + the checkout is not named a second time from the disk.""" + (holder / "Members" / "X").mkdir(parents=True) + (holder / "Members" / "X" / ".git").write_bytes( + b"gitdir: ../../.git/modules/x\n") + plant(holder, "100644", "Members/X/notes") + found = family_module._members_spellings( + holder, SPARE, family_module._members_on_disk(holder), frozenset()) + assert [(spelled.where, spelled.target) for spelled in found] == [ + ("in the index", "Members/X")] + + +# --- 6. Codex's two P2s on #828 ---------------------------------------------- + +#: A file committed under the row's path, in the way of the move back. +NOTES = f"{ROW}/notes" + + +def notes_marked(on_disk: bool): + """`members/IRRS/notes` marked `skip-worktree`, as a sparse checkout + marks it, and kept off disk unless `on_disk`.""" + def make(root: Path) -> None: + if not on_disk and git_version() < (2, 38): + pytest.skip("`git mv --sparse` of one entry kept off disk needs " + "git 2.38 or later") + git("update-index", "--skip-worktree", "--", NOTES, cwd=root) + if not on_disk: + (root / NOTES).unlink() + return make + + +def notes_left_out(on_disk: bool): + """Non-cone sparse rules that leave out `members/IRRS/notes` alone, its + `skip-worktree` bit cleared, on disk where `on_disk`.""" + def make(root: Path) -> None: + git("sparse-checkout", "set", "--no-cone", "/*", f"!/{NOTES}", + cwd=root) + bit_cleared(root, NOTES, on_disk, False) + return make + + +#: `{state: (how it is made, the set-aside part's arguments, the line typed +#: for it)}`. +SPARSE_ASIDE = { + "marked, on disk": (notes_marked(True), {"renamed": [NOTES]}, + f"git mv -- {NOTES} renamed-1"), + "marked, off disk": (notes_marked(False), {"sparse": [NOTES]}, + f"git mv --sparse -- {NOTES} renamed-1"), + "left out by the rules, on disk": ( + notes_left_out(True), {"sparse": [NOTES]}, + f"git mv --sparse -- {NOTES} renamed-1"), + "left out by the rules, off disk": ( + notes_left_out(False), {"out": [NOTES], "out_sparse": True}, + f"git rm -r --cached --sparse -- {NOTES}"), +} + + +@pytest.mark.parametrize("state", [ + pytest.param(state, marks=needs_check_rules) if "rules" in state + else state for state in SPARSE_ASIDE]) +def test_a_blocker_a_sparse_checkout_marks_or_leaves_out_is_set_aside_too( + family, holder, offline, tmp_path, state): + """Codex's P2 1 on #828: `git mv members/IRRS Members/IRRS`, committed, + and a file committed under the row's path, which a sparse checkout + marks or its rules leave out. Such a row was dropped from the rows to + set aside before the member's own gitlink was asked for, so + `Members/IRRS` kept #574's `git mv -- Members/IRRS `, which + takes the member's gitlink and section away from its row. Now the + gitlink is moved back, and the file is set aside as the index's own + rule names it: plain `git mv` where it is marked and on disk, `git mv + --sparse` where it is marked and off disk or left out and on disk, and + `git rm -r --cached --sparse` where it is left out and off disk; plain + `git mv` and `git rm --cached` refuse each of the last three (measured + on git 2.43 and 2.47.1). Typed as printed, the file's bytes are at its + new name where it is renamed, the member is where its row says, the + gate passes and the same `add` goes through.""" + folding_skip(tmp_path) + make, how, typed_first = SPARSE_ASIDE[state] + commit = mount_moved(holder) + data = file_committed(holder, NOTES) + make(holder) + move, closing = moved_back({ROW: ([VARIANT], commit)}, sections=(ROW,)) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (in_the_index(VARIANT_SHOWN), + remedy_of(set_aside([NOTES], {}, **how), move, *closing)) + assert typed_as_printed(holder, said[1], + lambda root: rmtree(root / VARIANT)) == [ + typed_first, REMOVE, record(commit), *POINTED[::2], "git commit", + MOVE_OUT, SUBMODULE] + if typed_first.startswith("git mv"): + assert git("show", "HEAD:renamed-1", + cwd=holder).stdout.encode() == data + registered(holder, family["root"], commit) + gate_then_add(offline, holder, family["remotes"]) + + +def test_a_link_pointing_nowhere_at_the_variant_is_still_moved_out( + family, holder, offline, tmp_path): + """Codex's P2 2 on #828: the closing `mv` failed, exit 1, where nothing + is at the variant, so its guard now begins `[ ! -e ] && [ ! + -L ]`, and every typing of a state with nothing there types + it, as printed, moving nothing. The second test keeps what the guard did + before for a link there pointing nowhere, which `-e` takes for nothing: + #646's state with the removal staged, so HEAD alone holds the variant, + and a link `Members/IRRS -> nowhere`. (With the variant's gitlink still + in the index, git 2.45.1 and later refuse the link itself: `git status` + says `expected submodule path 'Members/IRRS' not to be a symbolic + link`, measured on git 2.47.1.) Typed as printed, the link is moved out + of the holder, the member is where its row says, the gate passes and + the same `add` goes through.""" + if os.name == "nt": + pytest.skip("a symlink needs a privilege Windows runners do not grant") + if disk_folds(tmp_path): + pytest.skip("a link at the variant beside the row's directory needs a " + "disk that does not fold case") + commit = variant(holder) + git("rm", "-q", "--cached", "--", VARIANT, cwd=holder) + (holder / "Members").mkdir() + os.symlink("nowhere", holder / VARIANT) + said = refused_untouched(offline, holder, family["remotes"]) + assert said == (DIR_VARIANT + f"{HEAD_ONLY}: {VARIANT_SHOWN}", + remedy(commit)) + assert typed_as_printed(holder, said[1]) == [ + REMOVE, record(commit), "git commit", MOVE_OUT] + assert not os.path.lexists(holder / VARIANT) + assert os.readlink(holder.parent / "Members-IRRS") == "nowhere" + registered(holder, family["root"], commit) + gate_then_add(offline, holder, family["remotes"]) diff --git a/tests/test_family_add_spellings_remedies.py b/tests/test_family_add_spellings_remedies.py index ba8e95a1..a64039ab 100644 --- a/tests/test_family_add_spellings_remedies.py +++ b/tests/test_family_add_spellings_remedies.py @@ -41,7 +41,7 @@ from conftest import appended_to_command_scope, cloned_just_pushed, git, rmtree from test_family import NAME, SPARE, family # noqa: F401 from test_family_add_member_variant import ( - OTHER_ROW, OTHER_VARIANT, REMOVE, ROW, VARIANT, VARIANT_SHOWN, + MOVE_OUT, OTHER_ROW, OTHER_VARIANT, REMOVE, ROW, VARIANT, VARIANT_SHOWN, dry_run_in_process, in_the_index, moved_back, record, refused_in_process, registered, typed_as_printed, variant, ) @@ -375,7 +375,8 @@ def test_a_folding_disk_that_received_it_renames_it_back( pull leaves it on a disk that folds case. There `Members` is that very directory, and the two steps are printed. On Linux `Members` is not on disk at all, a second spelling `git mv` cannot find, and #646's move - back is printed, as before.""" + back is printed, as before, pointing each section, which the rename + moved to `Members/`, back at its row's path (#761).""" folding_skip(tmp_path) commits = renamed_with_git(holder) os.rename(holder / "Members", holder / "members") @@ -383,7 +384,7 @@ def test_a_folding_disk_that_received_it_renames_it_back( move, closing = moved_back({ROW: ([VARIANT], commits[ROW]), OTHER_ROW: ([OTHER_VARIANT], commits[OTHER_ROW])}, - commit=False) + commit=False, sections=(ROW, OTHER_ROW)) assert dry_run_in_process(family_module, monkeypatch, offline, holder, family["remotes"], capsys) == ( BOTH, remedy_of(renamed_back(False)) if folding @@ -477,8 +478,6 @@ def it_a_link_outside(root: Path) -> None: lambda root: plant(root, "100644", "members/notes")), "a file at members in the index": ( lambda root: plant(root, "100644", "members")), - "another spelling in the index too": ( - lambda root: plant(root, "100644", "MEMBERS/notes")), "a case variant of the mount path under it": ( lambda root: plant(root, "100644", f"Members/{SPARE}")), "an entry under it a sparse checkout keeps off disk": ( @@ -500,9 +499,10 @@ def test_only_members_renamed_whole_is_renamed_back(holder, family_module, tmp_path, state): """The rename back is printed only where it types clean and leaves nothing out: no `members/` on disk but that spelling's own directory, - nothing in the index at or under `members` itself or under a second - spelling, no case variant of the mount path in it, nothing under it - kept off disk or unmerged, no member's gitlink that HEAD alone holds + nothing in the index at or under `members` itself, no member's gitlink + under a second spelling (a third spelling's own entries keep their own + remedy, #761), no case variant of the mount path in it, nothing under + it kept off disk or unmerged, no member's gitlink that HEAD alone holds there, and every entry under it on disk as the index holds it or able to be put back. Elsewhere each path keeps its own remedy (the move back of a member's own gitlink, or #574's), never `git mv back`.""" @@ -611,7 +611,8 @@ def test_a_missing_gitmodules_is_put_back_before_the_move_back( assert said == (in_the_index(VARIANT_SHOWN), remedy_of(RESTORE_FIRST, move, *closing)) assert typed_as_printed(holder, said[1]) == [ - "git checkout -- .gitmodules", REMOVE, record(commit), "git commit"] + "git checkout -- .gitmodules", REMOVE, record(commit), "git commit", + MOVE_OUT] registered(holder, family["root"], commit) assert_passes_in_every_mode(holder, 2) added(offline, holder, family["remotes"]) diff --git a/tests/test_family_members_case_variants.py b/tests/test_family_members_case_variants.py index b66cf9fc..028726c7 100644 --- a/tests/test_family_members_case_variants.py +++ b/tests/test_family_members_case_variants.py @@ -109,7 +109,7 @@ def index_part(*, rename=(), sparse=(), unstage=(), commit="") -> str: said.append("rename what the index holds with `git mv`, entry by " f"entry ({each('git mv', rename)})") if sparse: - said.append("rename what the sparse checkout keeps off disk with " + said.append("rename what the sparse checkout leaves out with " f"`git mv --sparse`, {NEEDS_GIT_2_38}, entry by entry " f"({each('git mv --sparse', sparse)})") if unstage: @@ -457,7 +457,7 @@ def index_info(line: bytes) -> None: ("Members/\N{REPLACEMENT CHARACTER}", "a file", "in HEAD and no longer in the index", "Members/\N{REPLACEMENT CHARACTER}", False, "git commit", True, "", - False, "", False)] + False, "", False, (), False)] # --- the disk (#364) ---------------------------------------------------------