diff --git a/scripts/family.py b/scripts/family.py index f7c45368..a3eb0e08 100755 --- a/scripts/family.py +++ b/scripts/family.py @@ -154,6 +154,7 @@ import sys import tempfile from pathlib import Path +from typing import NamedTuple SHAPE_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(SHAPE_ROOT / "scripts")) @@ -1935,6 +1936,473 @@ def _index_entries(root: Path, path: str) -> list[tuple[str, str]]: return entries +#: `str.translate`'s table for folding ASCII letters and nothing else, as +#: `_index_entries` folds a name's bytes and as git's own `icase` folds. +ASCII_FOLD = {upper: upper + 32 for upper in range(ord("A"), ord("Z") + 1)} + +#: Where `_members_spellings` found a path, as its refusal says it (#362, +#: #363, #364). +IN_INDEX = "in the index" +IN_HEAD_ONLY = "in HEAD and no longer in the index" +ON_DISK = "on disk and untracked" + +#: What the remedy does to a path `_members_spellings` found, in the order +#: `_spellings_refusal` says them. What HEAD holds and the index no longer +#: does is a pending change, committed first. What the index holds is renamed +#: with `git mv` where it is on disk in its own spelling, with `git mv +#: --sparse` where a sparse checkout keeps it off disk, and otherwise taken +#: out of the index, which touches no file, as #318 takes out a variant of +#: the mount path. What is on disk and untracked is moved out of the way, and +#: a `members/` a hand renamed is moved back. +COMMIT = "git commit" +RENAME = "git mv" +RENAME_SPARSE = "git mv --sparse" +UNSTAGE = "git rm -r --cached" +MOVE = "mv" +MOVE_BACK = "mv back" +REMEDY_ORDER = (COMMIT, RENAME, RENAME_SPARSE, UNSTAGE, MOVE, MOVE_BACK) + + +class Spelled(NamedTuple): + """A path at or under a spelling of `members/` other than `members/` + itself, or at or under a case variant of the mount path, as + `_members_spellings` found it: `name` as git or the disk spells it, + `kind` (`a file`, `a submodule`, `a symlink`, `a directory`), `where` + (`IN_INDEX`, `IN_HEAD_ONLY` or `ON_DISK`), `target`, the path its remedy + acts on, `mount`, whether it is at or under a case variant of the mount + path itself, `remedy`, which of `REMEDY_ORDER` acts on it, and + `committed`, whether HEAD holds it too, so that taking it out of the + index is a change to commit.""" + + name: str + kind: str + where: str + target: str + mount: bool + remedy: str + committed: bool = False + + +def _folded(name: str) -> str: + """`name` with ASCII case folded, as git's `icase` folds it.""" + return name.translate(ASCII_FOLD) + + +def _shown(name: str) -> str: + """`name`, read from git or a listing with `surrogateescape`, as a + refusal shows it: a byte that is not UTF-8 as U+FFFD, never as a lone + surrogate a stream may not encode.""" + return name.encode("utf-8", "surrogateescape").decode("utf-8", "replace") + + +def _spelled_target(name: str, project: str, + occupied: frozenset[str] = frozenset() + ) -> tuple[str, bool] | None: + """`(the path a remedy acts on, whether it is at or under a case variant + of members/)` for a path `name`, `/`-separated as git spells + it, or None when nothing about `name` is a case variant. + + ONE ENTRY OF THE OTHER SPELLING AT A TIME, NEVER THAT SPELLING ITSELF + (#574's review, B1). On macOS and Windows `Members` and `members` are one + directory, holding the members' own checkouts, so a remedy that renamed + `Members` would take every member with it. Under a spelling of `members` + other than `members` itself the remedy acts on the entry directly below + it (`Members/README`, `Members/docs`, `Members/IRQS`), or on the spelling + only where that is the entry, a file or link named `Members`. Where that + entry folds to a name `members/` itself tracks (`occupied`, folded), it + is that name's directory on such a disk, so the remedy acts on `name` + alone. Under `members/` itself only a case variant of the mount path is + one (`members/irqs`), and its remedy acts on that. The mount path itself, + `members/` exactly, is no variant: what is there is + `member-path-taken`'s first two questions.""" + first, _, rest = name.partition("/") + if _folded(first) != MEMBERS_DIR: + return None + second = rest.partition("/")[0] + mount = bool(rest) and _folded(second) == _folded(project) + if first != MEMBERS_DIR: + if not rest: + return first, False + if not mount and _folded(second) in occupied: + return name, False + return f"{first}/{second}", mount + if mount and second != project: + return f"{first}/{second}", True + return None + + +def _disk_kind(path: Path) -> str: + """What is at `path`, not following a link, as a refusal names it.""" + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + return "a symlink" + return "a directory" if stat.S_ISDIR(mode) else "a file" + + +def _members_on_disk(root: Path) -> dict[str, tuple[str, list[str] | None]]: + """`{spelling: (kind, its entries)}` for every spelling of `members` the + holder's root holds on disk, `members` itself included, READ BY LISTING + DIRECTORIES (#364). `kind` is `_disk_kind`'s; the entries are the + directory's own names, through a link, none for a file, and None for a + directory that cannot be listed. + + `os.path.lexists(root / path)` was the question, and it answers as the + disk resolves a name: exactly on Linux, in any case on macOS and + Windows. So an untracked `Members/` was refused there and the + member mounted beside it on Linux. A listing hands back each name in the + spelling the disk keeps, on every disk: the root is listed for every + spelling of `members`, ASCII case folded, and each of those for its + entries, one level at a time and never through a pathspec. + `_members_dir_on_disk` then says which of them, if any, is `members/` + itself on this disk. + """ + found: dict[str, tuple[str, list[str] | None]] = {} + for name in sorted(os.listdir(root)): + if _folded(name) != MEMBERS_DIR: + continue + top = root / name + entries: list[str] | None = [] + if top.is_dir(): + try: + entries = sorted(os.listdir(top)) + except OSError: + entries = None + found[name] = (_disk_kind(top), entries) + return found + + +def _members_dir_on_disk( + root: Path, on_disk: dict[str, tuple[str, list[str] | None]] + ) -> str | None: + """The spelling under which the holder's root keeps `members/` ITSELF + on this disk, from `_members_on_disk`'s listing: `members` where it is + listed; else the one directory listed under another spelling that IS + `members/` here (#574's review, B1, and Codex's P2 there); else None. + + THE ALIAS IS ASKED, NEVER GUESSED. A checkout on macOS or Windows of a + holder that held `Members/README` beside its members makes ONE directory + for both, spelled `Members` because that entry sorts first, and it holds + the members' own checkouts. There `members` resolves to it: + `os.path.isdir` of `members` is true and `os.path.samefile` names the + same directory. On Linux `members` is not there at all, whatever + `Members` holds (a `members/` a hand renamed, a human's own folder), so + that `Members` is a variant like any other. A link is never the alias: + it is named, and what it points at is not entered.""" + if MEMBERS_DIR in on_disk: + return MEMBERS_DIR + canonical = root / MEMBERS_DIR + for name, (kind, _) in on_disk.items(): + try: + if (kind == "a directory" and os.path.isdir(canonical) + and os.path.samefile(canonical, root / name)): + return name + except OSError: + continue + return None + + +def _on_disk_as_named(root: Path, name: str, directory: bool) -> bool: + """Is `name` on disk IN THAT SPELLING, a directory where `directory` + says so and no directory where it does not? Each component is looked up + in its parent's listing, exactly, so a disk that folds case cannot + answer for another entry: on macOS an index entry `Members` (a file) is + not the holder's `members/` directory, and `git mv` of it would move + that directory and every member in it. A link is no directory here.""" + here = root + for part in name.split("/"): + try: + if part not in os.listdir(here): + return False + except OSError: + return False + here = here / part + return stat.S_ISDIR(here.lstat().st_mode) == directory + + +def _skip_worktree(root: Path) -> set[str]: + """Every index entry a sparse checkout keeps off disk (`git ls-files + -t`'s `S`), by its name as `_index_records` decodes it. `git mv` of one + says `not under version control`, and `git rm --cached` refuses it as + outside the sparse-checkout definition; `git mv --sparse` renames it.""" + listed = git_out(["ls-files", "-t", "-z"], cwd=root, binary=True) + return {record[2:].decode("utf-8", "surrogateescape") + for record in listed.split(b"\0") if record.startswith(b"S ")} + + +def _disk_spellings( + root: Path, on_disk: dict[str, tuple[str, list[str] | None]], + project: str, canon: str | None, + gitlinks: set[str]) -> list[tuple[str, str, str]]: + """`[(path, kind, remedy)]` of what the disk holds at or under a + spelling of `members/` other than `members/` itself, or at a case + variant of the mount path, from `_members_on_disk`'s listing; `canon` is + `_members_dir_on_disk`'s answer, and `gitlinks` every gitlink the index + holds. + + - In `members/` ITSELF, under whichever spelling this disk keeps it, + each entry that is a case variant of `` (`members/irqs`, + `Members/irqs` where `Members` is that directory), moved. + - A spelling that is not a directory, a link to one included, or a + directory nobody may list, is named and moved itself: what is behind a + link is not entered, and `members` itself as a link is not judged + here (git's own dry run refuses it, `member-path-unaddable`). + - A directory under another spelling, entry by entry, each moved; an + EMPTY one holds nothing a clone or a disk could meet, and is what a + `git mv` of its last entry leaves behind, so it is not named. + - Where no `members/` is on disk at all and such a directory holds a + member's checkout (`//.git`, where the index holds the + gitlink `members/`), it is `members/` a hand renamed (Codex's P2 + on #574), and it is moved BACK; only a case variant of the mount path + in it is moved out of the way first. + """ + found: list[tuple[str, str, str]] = [] + moved_back = False + for name, (kind, entries) in on_disk.items(): + if name in (canon, MEMBERS_DIR): + if kind == "a directory": + found.extend((f"{name}/{entry}", + _disk_kind(root / name / entry), MOVE) + for entry in entries or () + if _folded(entry) == _folded(project) + and entry != project) + continue + if kind != "a directory" or entries is None: + found.append((name, kind, MOVE)) + continue + back = canon is None and not moved_back and any( + f"{MEMBERS_DIR}/{entry}" in gitlinks + and os.path.lexists(root / name / entry / ".git") + for entry in entries) + moved_back = moved_back or back + found.extend((f"{name}/{entry}", _disk_kind(root / name / entry), MOVE) + for entry in entries + if not back or _folded(entry) == _folded(project)) + if back: + found.append((name, kind, MOVE_BACK)) + return found + + +def _tree_entries(root: Path, tree: str, + recursive: bool = False) -> list[tuple[str, str, str]]: + """`[(name, kind, object)]` of the tree object `tree`, by `git ls-tree + -z` with NO PATH, so no pathspec setting in the environment can narrow + what is read. `kind` is `a tree` or, as `INDEX_ENTRY_KINDS` names an + index entry, `a submodule`, `a symlink` or `a file`. `recursive` lists + every file and gitlink below `tree`, and no tree.""" + listed = git_out(["ls-tree", *(["-r"] if recursive else []), "-z", tree], + cwd=root, binary=True) + entries = [] + for record in listed.split(b"\0"): + if not record: + continue + meta, _, name = record.partition(b"\t") + mode, kind, oid = meta.split() + entries.append((name.decode("utf-8", "surrogateescape"), + "a tree" if kind == b"tree" else + INDEX_ENTRY_KINDS.get(int(mode, 8), "a file"), + oid.decode("ascii"))) + return entries + + +def _head_spellings(root: Path) -> list[tuple[str, str]]: + """`[(path, kind)]` of every file and gitlink HEAD holds at or under a + spelling of `members` (`members/` itself included), or none in a holder + with NO COMMIT YET (#363). + + HEAD's root tree is listed, and each entry there whose name folds to + `members` is listed through by its object, so nothing else of HEAD is + read and no pathspec is used. `-q --verify HEAD^{tree}` answers nothing + at all, without a word, where HEAD names no commit yet, as + `MountRollback._head` reads it.""" + try: + tree = git_out(["rev-parse", "-q", "--verify", "HEAD^{tree}"], + cwd=root) + except Refusal: + return [] + held: list[tuple[str, str]] = [] + for name, kind, oid in _tree_entries(root, tree): + if _folded(name) != MEMBERS_DIR: + continue + if kind != "a tree": + held.append((name, kind)) + continue + held.extend((f"{name}/{below}", what) + for below, what, _ in _tree_entries(root, oid, True)) + return held + + +def _members_spellings(root: Path, project: str, + on_disk: dict[str, tuple[str, list[str] | None]] + ) -> list[Spelled]: + """Every path the holder has at or under a spelling of `members/` other + than `members/` itself, or at or under a case variant of the mount path + `members/`: in the index, in HEAD and no longer in the index, + and on disk untracked (#362, #363, #364), each with the remedy that acts + on it. `on_disk` is `_members_on_disk`'s listing. + + THE INDEX, the whole of it, as `_index_entries` reads it (#362). Each + entry's remedy acts on `_spelled_target`'s path: renamed with `git mv` + where that path is on disk in its own spelling (`_on_disk_as_named`) and + is not `members/` itself under another spelling; renamed with `git mv + --sparse` where a sparse checkout keeps the entry off disk; else taken + out of the index, which touches no file. THEN HEAD, for what it holds + that the index no longer does, a `git rm -r Members` not committed yet + (#363): the add's own commit names three paths and nothing else, so it + would keep that file beside the member, and with `core.ignorecase` set + git records the member's gitlink in the spelling HEAD has, where a fresh + clone cannot find it. THEN THE DISK, by `_disk_spellings` (#364): a path + the index holds itself, or holds anything above, or, for a path moved + out of the way, anything under, is the index's to name, so no two + remedies act on one path. + """ + canon = _members_dir_on_disk(root, on_disk) + indexed: dict[str, int] = { + name.decode("utf-8", "surrogateescape"): int(records[0].split()[0], 8) + for name, records in _index_records(root).items()} + gitlinks = {name for name, mode in indexed.items() if mode == 0o160000} + occupied = frozenset( + _folded(rest.partition("/")[0]) for first, _, rest + in (name.partition("/") for name in indexed) + if first == MEMBERS_DIR and rest) + head = _head_spellings(root) + in_head = {name for name, _ in head} + found: list[Spelled] = [] + sparse: set[str] | None = None + for name, mode in indexed.items(): + spelled = _spelled_target(name, project, occupied) + if spelled is None: + continue + target, mount = spelled + kind = INDEX_ENTRY_KINDS.get(mode, "a file") + if target != canon and _on_disk_as_named( + root, target, target != name or mode == 0o160000): + remedy = RENAME + else: + if sparse is None: + sparse = _skip_worktree(root) + remedy = RENAME_SPARSE if name in sparse else UNSTAGE + target = name if name in sparse else target + found.append(Spelled(_shown(name), kind, IN_INDEX, _shown(target), + mount, remedy, name in in_head)) + for name, kind in head: + spelled = _spelled_target(name, project, occupied) + if spelled is not None and name not in indexed: + found.append(Spelled(_shown(name), kind, IN_HEAD_ONLY, + _shown(spelled[0]), spelled[1], COMMIT, + True)) + for path, kind, remedy in _disk_spellings(root, on_disk, project, canon, + gitlinks): + if any(name == path or path.startswith(name + "/") + or (remedy == MOVE and name.startswith(path + "/")) + for name in indexed): + continue + found.append(Spelled(_shown(path), kind, ON_DISK, _shown(path), + _spelled_target(path, project)[1], remedy)) + return found + + +#: The order `_spellings_refusal` names its groups in. Its remedies run in +#: `REMEDY_ORDER`: a pending change in HEAD is committed first. +SPELLED_ORDER = (IN_INDEX, IN_HEAD_ONLY, ON_DISK) + + +def _spellings_refusal(path: str, found: list[Spelled]) -> Refusal: + """The refusal for what `_members_spellings` found: every path named in + its exact spelling, at most five in each place, with where it was found, + and every remedy (Brett Heap's ruling of 2026-10-14, "Refuse on every + disk", #362, #363, #364). + + `member-path-taken` where any is at or under a case variant of the mount + path, as #318 refuses one in the index; `family-members-dir-variant` + where every one is under a spelling of `members/` alone: a fault of the + holder's `members/` directory (`members_dir`, as `family.yaml` names + it), which no member can be added past. + + THE REMEDIES RUN IN `REMEDY_ORDER`, and each command names one entry, so + no command takes another's path from under it: `git commit` first for + what HEAD holds and the index no longer does, since the add's own commit + names three paths; then `git mv`, `git mv --sparse` and `git rm -r + --cached` for what the index holds, and a commit; then `mv` for what is + on disk and untracked, and `mv` back for a renamed `members/`. Every + path is quoted for a POSIX shell (#532). + """ + mount = any(spelled.mount for spelled in found) + groups = [] + for where in SPELLED_ORDER: + here = [spelled for spelled in found if spelled.where == where] + if not here: + continue + shown = ", ".join(f"`{spelled.name}` ({spelled.kind})" + for spelled in here[:5]) + more = f" and {len(here) - 5} more" if len(here) > 5 else "" + groups.append(f"{where}: {shown}{more}") + by = {remedy: sorted({spelled.target for spelled in found + if spelled.remedy == remedy}) + for remedy in REMEDY_ORDER} + + def each(command: str, targets: list[str]) -> str: + return " and ".join( + f"`{command} -- {shell_quoted(target, 'posix')} `" + for target in targets) + + parts = [] + if by[COMMIT]: + named = ", ".join(f"`{target}`" for target in by[COMMIT]) + parts.append( + "commit the pending change first (`git commit`), since the add's " + f"own commit names {GITMODULES}, {path} and {MANIFEST} alone: " + f"HEAD would keep {named} beside the member, or, with " + "`core.ignorecase` set, record the member's gitlink in that " + "spelling") + index = [] + if by[RENAME]: + index.append("rename what the index holds with `git mv`, entry by " + f"entry ({each(RENAME, by[RENAME])})") + if by[RENAME_SPARSE]: + index.append("rename what the sparse checkout keeps off disk with " + "`git mv --sparse`, entry by entry " + f"({each(RENAME_SPARSE, by[RENAME_SPARSE])})") + if by[UNSTAGE]: + quoted = " ".join(shell_quoted(target, "posix") + for target in by[UNSTAGE]) + index.append("take what has nothing on disk in that spelling out of " + f"the index, which touches no file (`{UNSTAGE} -- " + f"{quoted}`)") + # A removal from the index is a change to commit only where HEAD holds + # what it removes, as it does once the pending change is committed; an + # entry staged and never committed leaves nothing to commit. + removed = bool(by[UNSTAGE]) and (bool(by[COMMIT]) or any( + spelled.committed for spelled in found if spelled.remedy == UNSTAGE)) + renamed = bool(by[RENAME] or by[RENAME_SPARSE]) + if renamed or removed: + index[-1] += ", and commit " + ( + "both" if renamed and removed else + "the rename" if renamed else "the removal") + if index: + parts.append(", ".join(index)) + if by[MOVE]: + parts.append("move what is on disk and untracked out of the way " + f"({each(MOVE, by[MOVE])})") + for target in by[MOVE_BACK]: + parts.append( + f"move `{target}` back to {MEMBERS_DIR}/, since it holds " + f"checkouts of members git tracks under {MEMBERS_DIR}/ (`mv -- " + f"{shell_quoted(target, 'posix')} {MEMBERS_DIR}`)") + said = "; ".join(parts) + return Refusal( + "member-path-taken" if mount else "family-members-dir-variant", + (f"the family root holds a case variant of {path}, the same path on " + "a case-insensitive disk, though no `members:` row names it" + if mount else + f"the family root holds a path at or under another spelling of " + f"{MEMBERS_DIR}/, which a case-insensitive disk takes for " + f"{MEMBERS_DIR}/ itself") + + " — " + "; ".join(groups), + f"Remediation: every member is mounted under {MEMBERS_DIR}/, and a " + "holder keeps that directory in one spelling on every disk. " + + said[:1].upper() + said[1:] + "; then re-run.") + + def _submodule_paths( root: Path) -> dict[str, tuple[str, str | None, tuple[str, ...]]]: """`{name: (path, url, keys)}` of every submodule the holder's @@ -2011,11 +2479,36 @@ def _checked_mount_path(root: Path, project: str, path: str) -> None: `_mount_and_pin`), and `--force` turns them off or moves them to after the clone. So each is asked here first, the way git asks it, and named: - - SOMETHING AT THE PATH. On disk, by `lexists`, so a dangling link counts - as well as a directory a half-finished add or a hand left there. In - the INDEX, any entry at the path or under it, with nothing on disk: - given `--force`, git accepts a gitlink there and replaces it with the - member's, and whatever the human had staged is gone without a word. + - SOMETHING AT THE PATH. On disk, by a listing of `members/` in which + the path's own spelling is found (`_members_on_disk`, #364), so a + dangling link counts as well as a directory a half-finished add or a + hand left there; and by `lexists` too, last of all, for a directory no + listing could read. `members/` is listed under whichever spelling this + disk keeps it (`_members_dir_on_disk`): on macOS and Windows that can + be `Members`, one directory with `members/`, and the path in it is + there all the same — unless the index holds it under that other + spelling, which the next two questions name. In the INDEX, any entry + at the path or under it, in any case, with nothing on disk in its own + spelling: given `--force`, git accepts a gitlink there and replaces it + with the member's, and whatever the human had staged is gone without + a word. That takes it out of the index (#318); a variant that IS on + disk is renamed with a committed `git mv` instead, below. + - ANOTHER SPELLING OF `members/` OR OF THE PATH, ANYWHERE (#362, #363, + #364): in the index, in HEAD and no longer in the index, or on disk + untracked, `_members_spellings` reads it and `_spellings_refusal` + names it, with a remedy for each entry and never for the other + spelling's own directory, which on macOS and Windows holds the + members. Brett Heap ruled on 2026-10-14, "Refuse on every disk": + macOS and Windows hold `Members/` and `members/` as one directory, and + with `core.ignorecase` set git files a new member's gitlink in the + spelling it already holds, which the add's commit, naming the path, + then cannot record (#362) or records where `.gitmodules` and + `family.yaml` do not say (#363). So it is refused on all three, Linux + included, where a `Members/` of a holder's own is harmless until a + clone of it is made on the other two; the cost is that such a holder + renames it first. `member-path-taken` where a path is a case variant + of the mount path, `family-members-dir-variant` where it is under a + spelling of `members/` alone. - GIT'S OWN DRY RUN, `git add --dry-run --ignore-missing` of the path — the check `--force` skips — made here with `-f`, so an ignore rule passes it and nothing else does. It refuses a path beyond a symbolic @@ -2043,15 +2536,25 @@ def _checked_mount_path(root: Path, project: str, path: str) -> None: decide whose without asking. The path is compared exactly; git reads `.gitmodules` that way too. """ - if os.path.lexists(root / path): - raise Refusal( - "member-path-taken", - f"{path} already exists in the family root but no `members:` row " - "names it", - "Remediation: this is a half-finished add or a hand-made " - "directory. Remove it (`git rm -r` if it is tracked), then re-run.") + taken = Refusal( + "member-path-taken", + f"{path} already exists in the family root but no `members:` row " + "names it", + "Remediation: this is a half-finished add or a hand-made " + "directory. Remove it (`git rm -r` if it is tracked), then re-run.") + on_disk = _members_on_disk(root) + canon = _members_dir_on_disk(root, on_disk) held = _index_entries(root, path) - if held: + # Each spelling of the mount path the index holds other than its own, + # and whether git tracks a directory there. + variants = {(name[:len(path)], + len(name) > len(path) or kind == "a submodule") + for name, kind in held if name[:len(path)] != path} + if canon is not None and project in (on_disk[canon][1] or ()) and ( + canon == MEMBERS_DIR or not variants): + raise taken + if held and not any(_on_disk_as_named(root, variant, directory) + for variant, directory in variants): at_it = [entry for entry in held if entry[0].lower() == path.lower()] name, kind = (at_it or held)[0] more = f" and {len(held) - 1} more" if len(held) > 1 else "" @@ -2090,6 +2593,13 @@ def _checked_mount_path(root: Path, project: str, path: str) -> None: f"spelling other than the {path} that {GITMODULES} and " f"{MANIFEST} say, or both; then ") + "re-run.") + spelled = _members_spellings(root, project, on_disk) + if spelled: + raise _spellings_refusal(path, spelled) + # A DIRECTORY NO LISTING COULD READ hides nothing: `lexists` asks the + # path itself, as it always did, after every listing has answered. + if os.path.lexists(root / path): + raise taken try: run(["git", "--literal-pathspecs", "add", "--dry-run", "--ignore-missing", "-f", "--no-warn-embedded-repo", "--", path], @@ -2557,11 +3067,14 @@ class MountRollback: back what it recorded, by its exact name. git can stage under a name no step asks for: with `core.ignorecase` set, a mount whose directory the index already holds in another case is staged in that case, and the - commit's pathspec then matches nothing. So the WHOLE index is recorded - too, and `undo` compares it after its steps. Only when the two agree - does the sentence say the holder is as it was. When they do not, it - names what differs and sends the human to `git ls-files -s`, and the one - command that changes anything, `git rm -r --cached`, touches no file. + commit's pathspec then matches nothing. `_checked_mount_path` refuses + such an index before the mount now (#362); this stays behind it, for + anything else git stages under a name no step names. So the WHOLE index + is recorded too, and `undo` compares it after its steps. Only when the + two agree does the sentence say the holder is as it was. When they do + not, it names what differs and sends the human to `git ls-files -s`, + and the one command that changes anything, `git rm -r --cached`, + touches no file. """ def __init__(self, root: Path, path: str) -> None: diff --git a/tests/test_family_add_refusals.py b/tests/test_family_add_refusals.py index 35615529..7a037a25 100644 --- a/tests/test_family_add_refusals.py +++ b/tests/test_family_add_refusals.py @@ -1419,11 +1419,15 @@ def test_a_case_variant_of_the_mount_path_in_the_index_is_refused_first( def test_a_gitlink_git_stages_under_another_spelling_is_named_not_hidden( - family, holder, offline): + family, holder, offline, family_module, monkeypatch, capsys): """THE CLOSING SENTENCE SAYS "EXACTLY AS IT WAS" ONLY FOR WHAT IT COMPARED (#302). The index holds `Members/README`, a case variant of - `members/` and not of the mount path, so `member-path-taken` lets it - by: it is not at `members/IRQS` or under it in any spelling. With + `members/` and not of the mount path. #318 let it by, by design; Brett + Heap's ruling of 2026-10-14 ("Refuse on every disk") reverses that, and + `add` refuses it before the member is read (#362, + `test_family_members_case_variants.py`). That refusal is switched off + here, in process, because what this asks is what the unwind says when + git stages under a name no step asks for all the same. With `core.ignorecase` set, git stages the new gitlink in the spelling the index already gives the directory, `Members/IRQS`, and the commit, whose pathspec is `members/IRQS`, matches nothing. The unwind puts back every @@ -1436,14 +1440,18 @@ def test_a_gitlink_git_stages_under_another_spelling_is_named_not_hidden( tip = git("rev-parse", "main", cwd=family["remotes"] / f"{SPARE}.git").stdout.strip() before, index = snapshot(holder), index_bytes(holder) + monkeypatch.setattr(family_module, "_members_spellings", + lambda root, project, on_disk: []) - result = offline.add(holder, SPARE, remotes=family["remotes"]) - assert result.returncode == 2, result.stdout + result.stderr - assert result.stdout == (f"checking {ORG}/{SPARE} before mounting it\n" - f"mounting {ORG}/{SPARE} at {SPARE_MOUNT}\n") + code = add_in_process(family_module, monkeypatch, offline, holder, SPARE, + family["remotes"]) + out, err = capsys.readouterr() + assert code == 2, out + err + assert out == (f"checking {ORG}/{SPARE} before mounting it\n" + f"mounting {ORG}/{SPARE} at {SPARE_MOUNT}\n") # The step that failed, and not the command's own spelling, which is the # commit's business and changes with it. - lines = result.stderr.splitlines() + lines = err.splitlines() assert lines[0] == (f"REFUSED committing {SPARE} to the family. THE " "EXACT COMMAND WAS:") assert lines[-1] == INDEX_MOVED.format( @@ -1527,7 +1535,17 @@ def test_the_index_is_asked_at_the_mount_path_in_any_case_and_literally( never what a glob would add: `members/IRQ*` is a name, so it selects itself in any case, where `git --icase-pathspecs` would read it as a glob and select `members/IRQS`, `members/IRQSX/b` and `MEMBERS/irqs/a` - besides. (The entries are planted, with nothing on disk.)""" + besides. (The entries are planted, with nothing on disk.) + + A VARIANT OF `members/` ALONE IS NO LONGER LET BY. #318 pinned it here + as not refused, by design; Brett Heap's ruling of 2026-10-14 ("Refuse + on every disk") reverses that (#362). It is still not `_index_entries`'s + to select, since it is not at the mount path, and `_members_spellings` + names it instead, with every other path under a spelling of `members` + that is not `members` itself, and each case variant of the mount path + under `members/` itself, by the entry its remedy acts on, never the + other spelling's directory (#574's review, B1). With nothing on disk, + that remedy takes it out of the index, as #318's does.""" for name in ("MEMBERS/irqs/a", SPARE_MOUNT, f"{SPARE_MOUNT}X/b", "Members/README", "members/IRQ*/c", "members/irq*"): plant(holder, "100644", name) @@ -1537,6 +1555,14 @@ def test_the_index_is_asked_at_the_mount_path_in_any_case_and_literally( (f"members/{SPARE.lower()}/d", "a submodule")] assert family_module._index_entries(holder, "members/IRQ*") == [ ("members/IRQ*/c", "a file"), ("members/irq*", "a file")] + found = family_module._members_spellings( + holder, SPARE, family_module._members_on_disk(holder)) + assert [(spelled.name, spelled.target, spelled.mount, spelled.remedy) + for spelled in found] == [ + ("MEMBERS/irqs/a", "MEMBERS/irqs", True, "git rm -r --cached"), + ("Members/README", "Members/README", False, "git rm -r --cached"), + (f"members/{SPARE.lower()}/d", f"members/{SPARE.lower()}", True, + "git rm -r --cached")] def test_a_conflicted_entry_at_the_mount_path_is_counted_once_per_stage( diff --git a/tests/test_family_members_case_variants.py b/tests/test_family_members_case_variants.py new file mode 100644 index 00000000..fb2af770 --- /dev/null +++ b/tests/test_family_members_case_variants.py @@ -0,0 +1,956 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`family.py add` refuses a case variant of `members/` on every disk, before +the member is read (#362, #363, #364). + +BRETT HEAP'S RULING OF 2026-10-14, "Refuse on every disk": "Refuse before +reading the member, naming the entries and a committed `git mv` as the +remedy, with one rule for index, HEAD and disk. Linux, macOS and Windows give +the same answer, matching #318's any-case rule for the mount path. Cost: a +Linux holder with its own `Members/` folder must rename it first." + +THREE PLACES, ONE RULE. Before it, a path under `Members/` (any spelling of +`members` but `members` itself) was let through by design (#318), and: + +- in the INDEX, with `core.ignorecase` set, git staged the new gitlink as + `Members/

`, the add's commit matched nothing, and the remedy its closing + sentence printed failed the same way every time (#362); +- in HEAD and no longer in the index, a `git rm -r Members` not committed + yet, `add` committed the gitlink as `Members/

` with exit 0 while + `.gitmodules` and `family.yaml` said `members/

`, and every fresh clone + left the member empty (#363); +- on DISK, untracked, `_checked_mount_path`'s exact `lexists` refused + `Members/

` on macOS and Windows and mounted the member beside it on + Linux (#364). + +Each is refused now, naming every path in its own spelling and where it was +found, `member-path-taken` where it is a case variant of the mount path and +`family-members-dir-variant` where it is under a spelling of `members/` +alone, and the holder is left exactly as it was. EVERY PRINTED REMEDY IS +TYPED AS PRINTED here (`type_remedy`), and the same `add` then goes through. + +ONE ENTRY AT A TIME, NEVER THE OTHER SPELLING'S DIRECTORY (#574's review, +B1). On macOS and Windows a `Members/` and `members/` are ONE directory, and +it holds the members' own checkouts: a `git mv` of `Members` would take +them all. So the remedy renames, removes or moves the entries under it, and +the text is the same on all three systems. Where that one directory is +spelled `Members` on such a disk, `members/` itself is it +(`_members_dir_on_disk`, asked with `isdir` and `samefile`), so what it holds +is no variant; on Linux a `Members/` is always another directory (Codex's P2 +on #574). + +EVERY TEST RUNS ON ALL THREE SYSTEMS. An index or HEAD variant is planted +with `update-index --cacheinfo`, with nothing on disk. A disk state is made +in a holder with no `members/` yet (`unborn_holder`), or by renaming the +holder's `members/` in place, so no test makes two directories whose names +differ only in case. What the disk answers is either the same everywhere, +asserted for the disk the test runs on (`disk_folds`), or asked in process +of a disk that folds case and of one that does not (`disk_as`), each +emulated below the holder alone, so both answers are asserted on every OS. +""" + +from __future__ import annotations + +import os +import re +import shlex +import shutil +import subprocess +from pathlib import Path + +import pytest + +from conftest import git +from test_family import SPARE, family # noqa: F401 +from test_family_add_refusals import ( # noqa: F401 + family_module, holder, index_bytes, offline, plant, refusal, snapshot, + unborn_holder, +) + +#: `add`'s mount path for the spare member. +MOUNT = f"members/{SPARE}" + +#: The first sentence of each refusal, before ` — ` and the places it names. +DIR_VARIANT = ( + "REFUSED family-members-dir-variant: the family root holds a path at or " + "under another spelling of members/, which a case-insensitive disk takes " + "for members/ itself — ") +MOUNT_VARIANT = ( + f"REFUSED member-path-taken: the family root holds a case variant of " + f"{MOUNT}, the same path on a case-insensitive disk, though no " + "`members:` row names it — ") + +#: The mount path itself on disk, as it was always refused. +TAKEN = (f"REFUSED member-path-taken: {MOUNT} already exists in the family " + "root but no `members:` row names it", + "Remediation: this is a half-finished add or a hand-made directory. " + "Remove it (`git rm -r` if it is tracked), then re-run.") + +#: The part of the remedy for a `members/` a hand renamed `Members/`. +MOVE_BACK = ("move `Members` back to members/, since it holds checkouts of " + "members git tracks under members/ (`mv -- Members members`)") + + +def each(command: str, targets) -> str: + """One printed ` -- ` for each target.""" + return " and ".join(f"`{command} -- {target} `" + for target in targets) + + +def index_part(*, rename=(), sparse=(), unstage=(), commit="") -> str: + """The part of the remedy for what the index holds.""" + said = [] + if rename: + said.append("rename what the index holds with `git mv`, entry by " + f"entry ({each('git mv', rename)})") + if sparse: + said.append("rename what the sparse checkout keeps off disk with " + "`git mv --sparse`, entry by entry " + f"({each('git mv --sparse', sparse)})") + if unstage: + said.append("take what has nothing on disk in that spelling out of " + "the index, which touches no file (`git rm -r --cached -- " + + " ".join(unstage) + "`)") + return ", ".join(said) + (f", and commit {commit}" if commit else "") + + +def commit_first(*targets: str) -> str: + """The part for what HEAD holds and the index no longer does.""" + named = ", ".join(f"`{target}`" for target in targets) + return ("commit the pending change first (`git commit`), since the add's " + f"own commit names .gitmodules, {MOUNT} and family.yaml alone: " + f"HEAD would keep {named} beside the member, or, with " + "`core.ignorecase` set, record the member's gitlink in that " + "spelling") + + +def move(*targets: str) -> str: + """The part for what is on disk and untracked.""" + return ("move what is on disk and untracked out of the way " + f"({each('mv', targets)})") + + +def remedy_of(*parts: str) -> str: + """The remedy line: its parts, in the order they are to be run, between + what every remedy begins and ends with.""" + said = "; ".join(parts) + return ("Remediation: every member is mounted under members/, and a " + "holder keeps that directory in one spelling on every disk. " + + said[0].upper() + said[1:] + "; then re-run.") + + +def refused_untouched(offline, root, remotes, *extra): + """`add` of the spare member, refused before anything about the member + is printed or read, with the holder exactly as it was: the snapshot, the + index byte for byte, no `gh` call and no throwaway clone left. Returns + the refusal's two lines.""" + before, index = snapshot(root), index_bytes(root) + result = offline.add(root, SPARE, *extra, remotes=remotes) + assert result.returncode == 2, result.stdout + result.stderr + assert result.stdout == "" + assert index_bytes(root) == index + assert snapshot(root) == before + offline.assert_left_nothing() + return refusal(result.stderr) + + +def type_remedy(root: Path, remedy: str) -> list[str]: + """EVERY COMMAND THE REMEDY PRINTS, TYPED AS PRINTED, in its order, and + the commit each part asks for: the printed `git commit`, and one where a + part ends "and commit ...". Only `` is filled in: `renamed-N` + for a `git mv`, which is committed and stays, and `../moved-N` for a + `mv`, so what is untracked leaves the holder. Returns what was typed.""" + typed: list[str] = [] + count = {"git": 0, "mv": 0} + for part in remedy.split(" on every disk. ", 1)[1].split("; "): + for command in re.findall(r"`([^`]*)`", part): + if command == "git commit": + git("commit", "-qm", "The pending change", cwd=root) + elif " -- " in command: + if "" in command: + tool = command.partition(" ")[0] + count[tool] += 1 + command = command.replace( + "", f"renamed-{count['git']}" + if tool == "git" else f"../moved-{count['mv']}") + argv = shlex.split(command) + if argv[0] == "git": + git(*argv[1:], cwd=root) + else: + os.rename(root / argv[2], root / argv[3]) + else: + continue + typed.append(command) + if re.search(r", and commit (the rename|the removal|both)$", part): + git("commit", "-qm", "The remedy", cwd=root) + typed.append("git commit") + return typed + + +def mounted_alone(root) -> None: + """After the add: HEAD holds the spare member's gitlink at its mount + path and nothing at or under any other spelling of `members/` but the + members it held before, and the working tree is clean.""" + held = [line for line in git("ls-tree", "-r", "--name-only", "HEAD", + cwd=root).stdout.splitlines() + if line.split("/", 1)[0].lower() == "members"] + assert [name for name in held if not name.startswith("members/")] == [] + assert MOUNT in held + assert git("ls-tree", "HEAD", MOUNT, cwd=root).stdout.startswith( + "160000 commit ") + assert git("status", "--porcelain", cwd=root).stdout == "" + + +def added(offline, root, remotes) -> None: + """The same `add` again, through, and the member mounted alone.""" + again = offline.add(root, SPARE, remotes=remotes) + assert again.returncode == 0, again.stdout + again.stderr + mounted_alone(root) + offline.assert_left_nothing() + + +def disk_folds(directory: Path) -> bool: + """Does the disk `directory` is on fold case, as macOS's and Windows' + do by default? Asked with one directory, so none is ever made twice.""" + probe = directory / "Case-Probe" + probe.mkdir() + try: + return os.path.isdir(directory / "case-probe") + finally: + probe.rmdir() + + +def commit_bytes(root: Path, made: str) -> None: + """`made` written as BYTES, and committed with `family.yaml`.""" + (root / made).parent.mkdir(parents=True, exist_ok=True) + (root / made).write_bytes(b"our own notes\n") + git("add", "--", "family.yaml", made, cwd=root) + git("commit", "-qm", f"Notes at {made}", cwd=root) + + +# --- the index (#362) -------------------------------------------------------- + +@pytest.mark.parametrize("ignorecase", ["true", "false"]) +@pytest.mark.parametrize("name, target", [ + ("Members/README", "Members/README"), + ("MEMBERS/notes/a", "MEMBERS/notes"), + ("Members", "Members"), + # `irrs` is what `members/IRRS` holds on a disk that folds case: the + # remedy names the entry itself, never that directory. + ("Members/irrs/a", "Members/irrs/a"), +]) +def test_an_index_entry_with_nothing_on_disk_is_taken_out_of_the_index( + family, holder, offline, name, target, ignorecase): + """THE ISSUE'S STATE (#362): the index holds a path under `Members/`, or + `Members` itself, and nothing is on disk in that spelling. With + `core.ignorecase` set git stages the gitlink as `Members/IRQS` and the + commit matches nothing; without it the add commits and leaves `Members/` + beside the member. Refused both ways, before the member is read, and + `--dry-run` refuses it alike. `git mv` of an entry with nothing on disk + fails (`bad source`), so it is taken out of the index, touching no file, + as #318 takes out a variant of the mount path; staged and never + committed, it leaves nothing to commit. Typed as printed, it lets the + same `add` through.""" + git("config", "--local", "core.ignorecase", ignorecase, cwd=holder) + plant(holder, "100644", name) + said = (DIR_VARIANT + f"in the index: `{name}` (a file)", + remedy_of(index_part(unstage=[target]))) + assert refused_untouched(offline, holder, family["remotes"], + "--dry-run") == said + assert refused_untouched(offline, holder, family["remotes"]) == said + assert type_remedy(holder, said[1]) == [ + f"git rm -r --cached -- {target}"] + added(offline, holder, family["remotes"]) + + +#: What a holder with no `members/` yet commits, its file on disk, as +#: `(the code's first sentence, what the index names, the path renamed, +#: where the file is after the rename)`. +TRACKED = { + "Members/README": (DIR_VARIANT, "Members/README", "renamed-1"), + "Members": (DIR_VARIANT, "Members", "renamed-1"), + "Members/docs/notes": (DIR_VARIANT, "Members/docs", "renamed-1/notes"), + # #318's state with its file on disk: one round, not three. + f"Members/{SPARE}/README": (MOUNT_VARIANT, f"Members/{SPARE}", + "renamed-1/README"), +} + + +@pytest.mark.parametrize("made", list(TRACKED)) +def test_a_tracked_path_on_disk_takes_one_committed_git_mv(family, offline, + tmp_path, made): + """THE STATE ON A REAL DISK: a path under `Members/`, or a file + `Members`, committed with its file on disk. Brett Heap's ruling names + "a committed `git mv`" as the remedy, and this is it: the entry directly + under the other spelling is renamed, never that spelling's directory, + and the rename committed. One round does it, #318's tracked + `Members/IRQS/...` included, which used to take `git rm --cached`, a + commit, then a `mv` (#574's review, S3). On macOS and Windows `Members` + is `members/` itself here, and the text is the same. The `git mv` leaves + an empty `Members/` on Linux, which holds nothing and is not named.""" + first, target, renamed = TRACKED[made] + root = unborn_holder(tmp_path) + commit_bytes(root, made) + said, remedy = refused_untouched(offline, root, family["remotes"]) + assert said == first + f"in the index: `{made}` (a file)" + assert remedy == remedy_of(index_part(rename=[target], + commit="the rename")) + assert type_remedy(root, remedy) == [f"git mv -- {target} renamed-1", + "git commit"] + added(offline, root, family["remotes"]) + assert git("show", f"HEAD:{renamed}", cwd=root).stdout == ( + "our own notes\n") + + +def test_a_tracked_file_deleted_from_disk_is_taken_out_and_committed( + family, offline, tmp_path): + """` D Members/README`, what an `rm -rf Members` of a committed file + leaves (#574's review, S2): `git mv` says `bad source`, so the entry is + taken out of the index and, since HEAD holds it, that removal is + committed. The empty `Members/` an `rm` of the file leaves on Linux + holds nothing, and is not named.""" + root = unborn_holder(tmp_path) + commit_bytes(root, "Members/README") + (root / "Members" / "README").unlink() + said, remedy = refused_untouched(offline, root, family["remotes"]) + assert said == DIR_VARIANT + "in the index: `Members/README` (a file)" + assert remedy == remedy_of(index_part(unstage=["Members/README"], + commit="the removal")) + assert type_remedy(root, remedy) == [ + "git rm -r --cached -- Members/README", "git commit"] + added(offline, root, family["remotes"]) + + +def git_version() -> tuple[int, int]: + """This host's git, as `(major, minor)`.""" + found = re.match(r"git version (\d+)\.(\d+)", + git("--version", cwd=Path.cwd()).stdout) + return int(found.group(1)), int(found.group(2)) + + +@pytest.mark.skipif(git_version() < (2, 34), + reason="`git mv --sparse` came with git 2.34") +def test_an_entry_a_sparse_checkout_keeps_off_disk_takes_git_mv_sparse( + family, holder, offline): + """A SPARSE CHECKOUT (#574's review, S2): an entry the checkout keeps + off disk (`skip-worktree`) is refused by `git mv` as `not under version + control` and by `git rm --cached` as outside the sparse-checkout + definition. `git mv --sparse` renames it, keeping what it holds, and the + rename is committed.""" + plant(holder, "100644", "Members/README") + git("update-index", "--skip-worktree", "--", "Members/README", cwd=holder) + said, remedy = refused_untouched(offline, holder, family["remotes"]) + assert said == DIR_VARIANT + "in the index: `Members/README` (a file)" + assert remedy == remedy_of(index_part(sparse=["Members/README"], + commit="the rename")) + assert type_remedy(holder, remedy) == [ + "git mv --sparse -- Members/README renamed-1", "git commit"] + added(offline, holder, family["remotes"]) + + +def test_every_place_is_named_and_the_pending_change_is_committed_first( + family, holder, offline): + """TWO PLACES AT ONCE: the index holds `Members/README`, staged, and + HEAD holds `MEMBERS/old`, which the index no longer does. Both are + named, each where it was found, and the pending change is committed + first, since that commit is what takes `MEMBERS/old` out of HEAD. It + commits `Members/README` too, so its removal from the index is a change + to commit after it.""" + plant(holder, "100644", "MEMBERS/old") + git("commit", "-qm", "Track MEMBERS/old", cwd=holder) + git("rm", "-q", "--cached", "--", "MEMBERS/old", cwd=holder) + plant(holder, "100644", "Members/README") + said, remedy = refused_untouched(offline, holder, family["remotes"]) + assert said == (DIR_VARIANT + "in the index: `Members/README` (a file); " + "in HEAD and no longer in the index: `MEMBERS/old` (a " + "file)") + assert remedy == remedy_of( + commit_first("MEMBERS/old"), + index_part(unstage=["Members/README"], commit="the removal")) + assert type_remedy(holder, remedy) == [ + "git commit", "git rm -r --cached -- Members/README", "git commit"] + added(offline, holder, family["remotes"]) + + +def test_more_than_five_paths_in_one_place_are_counted(family, holder, + offline): + """Five names in each place at most, then a count, as the unwind's + sentence names an index that moved; the remedy still names every + entry it acts on.""" + for n in range(1, 8): + plant(holder, "100644", f"Members/{n}") + shown = ", ".join(f"`Members/{n}` (a file)" for n in range(1, 6)) + assert refused_untouched(offline, holder, family["remotes"]) == ( + DIR_VARIANT + f"in the index: {shown} and 2 more", + remedy_of(index_part(unstage=[f"Members/{n}" for n in range(1, 8)]))) + + +# --- HEAD, and no longer the index (#363) ------------------------------------ + +#: What HEAD holds and the index no longer does, as `(mode, name, the code's +#: first sentence, kind)`. +HEAD_ONLY = { + "a file under Members/": ("100644", "Members/README.md", DIR_VARIANT, + "a file"), + "a variant of the mount path": ("160000", f"members/{SPARE.lower()}", + MOUNT_VARIANT, "a submodule"), +} + + +@pytest.mark.parametrize("ignorecase", ["true", "false"]) +@pytest.mark.parametrize("state", list(HEAD_ONLY)) +def test_what_head_holds_and_the_index_no_longer_does_is_committed_first( + family, holder, offline, state, ignorecase): + """THE ISSUE'S STATE (#363): HEAD holds a path under `Members/`, and a + removal of it is staged and not committed. `add` used to commit the + gitlink as `Members/IRQS` with exit 0, `.gitmodules` and `family.yaml` + saying `members/IRQS`, and a fresh clone of the pushed holder left the + member empty. A case variant of the mount path itself in HEAD is the + same fault under `member-path-taken`. Refused on every disk, before the + member is read; committing the removal, as the remedy says, lets the + same `add` through with the member under `members/` alone.""" + mode, name, first, kind = HEAD_ONLY[state] + git("config", "--local", "core.ignorecase", ignorecase, cwd=holder) + plant(holder, mode, name) + git("commit", "-qm", f"Track {name}", cwd=holder) + git("rm", "-q", "--cached", "--", name, cwd=holder) + said, remedy = refused_untouched(offline, holder, family["remotes"]) + assert said == first + f"in HEAD and no longer in the index: `{name}` " \ + f"({kind})" + assert remedy == remedy_of(commit_first(name)) + assert type_remedy(holder, remedy) == ["git commit"] + added(offline, holder, family["remotes"]) + + +def test_a_name_head_holds_that_is_not_utf8_is_shown_replaced( + holder, family_module): + """A name in HEAD that is not UTF-8 is shown as the index's names are, + with U+FFFD, never as a lone surrogate a stream may not encode. Planted + through `update-index --index-info`, BYTES on its stdin, committed by + `write-tree` and `commit-tree`, which never look for it on disk, and + taken out of the index the same way, so no argument needs those + bytes.""" + oid = subprocess.run(["git", "hash-object", "-w", "--stdin"], + cwd=str(holder), input=b"a file\n", + capture_output=True, check=True).stdout.strip() + + def index_info(line: bytes) -> None: + subprocess.run(["git", "-c", "core.protectNTFS=false", "update-index", + "--index-info"], cwd=str(holder), input=line, + capture_output=True, check=True) + + index_info(b"100644 " + oid + b"\tMembers/\xff\n") + tree = git("write-tree", cwd=holder).stdout.strip() + commit = git("commit-tree", tree, "-p", "HEAD", "-m", + "A name that is not UTF-8", cwd=holder).stdout.strip() + git("update-ref", "HEAD", commit, cwd=holder) + index_info(b"0 " + b"0" * 40 + b"\tMembers/\xff\n") + assert family_module._members_spellings( + holder, SPARE, family_module._members_on_disk(holder)) == [ + ("Members/\N{REPLACEMENT CHARACTER}", "a file", + "in HEAD and no longer in the index", + "Members/\N{REPLACEMENT CHARACTER}", False, "git commit", True)] + + +# --- the disk (#364) --------------------------------------------------------- + +#: What is on disk and untracked in a holder with no `members/` yet, the +#: same on every disk, as `(the file made, the refusal, its remedy)`. +ON_DISK = { + "a variant of the mount path under members/": ( + f"members/{SPARE.lower()}/notes.txt", + MOUNT_VARIANT + "on disk and untracked: " + f"`members/{SPARE.lower()}` (a directory)", + move(f"members/{SPARE.lower()}")), + "a file named Members": ( + "Members", DIR_VARIANT + "on disk and untracked: `Members` (a file)", + move("Members")), +} + + +@pytest.mark.parametrize("state", list(ON_DISK)) +def test_an_untracked_path_on_disk_is_moved_out_of_the_way(family, offline, + tmp_path, state): + """THE ISSUE'S STATE (#364), where every disk holds the same thing: a + case variant of the mount path under `members/` itself, and a file + named `Members`, which no disk takes for a directory. The disk is listed, + one level at a time, so each OS sees the name in the spelling it keeps + and says the same thing. The printed `mv`, typed, lets the same `add` + through. The holder has no commit yet, so HEAD is unborn.""" + made, said, part = ON_DISK[state] + root = unborn_holder(tmp_path) + (root / made).parent.mkdir(parents=True, exist_ok=True) + (root / made).write_bytes(b"our own notes\n") + assert refused_untouched(offline, root, family["remotes"]) == ( + said, remedy_of(part)) + assert (root / made).read_bytes() == b"our own notes\n" + assert type_remedy(root, remedy_of(part)) == ["mv -- {} ../moved-1".format( + made.partition("/notes.txt")[0])] + added(offline, root, family["remotes"]) + + +#: An untracked directory spelled `Members` in a holder with no `members/` +#: yet: `members/` itself on a disk that folds case, another directory on +#: Linux. `(the file made, Linux's refusal and remedy, the folding disk's)`; +#: None where the folding disk lets the add through. +OTHER_SPELLING = { + "a variant of the mount path under Members/": ( + f"Members/{SPARE}/notes.txt", + (MOUNT_VARIANT + f"on disk and untracked: `Members/{SPARE}` (a " + "directory)", remedy_of(move(f"Members/{SPARE}"))), + TAKEN), + "a file under Members/": ( + "Members/README.md", + (DIR_VARIANT + "on disk and untracked: `Members/README.md` (a file)", + remedy_of(move("Members/README.md"))), + None), +} + + +@pytest.mark.parametrize("state", list(OTHER_SPELLING)) +def test_an_untracked_members_directory_is_moved_entry_by_entry_on_linux( + family, offline, tmp_path, state): + """THE ISSUE'S STATE (#364) under `Members/`, which the disk decides. On + Linux a `Members/` is never `members/`: each entry in it is named and + moved, never the directory, and the empty directory left is not named. + On macOS and Windows that `Members` IS `members/`: a `Members/IRQS` is + the mount path itself, `already exists` as it always was, and a + `Members/README.md` is a file in `members/`, as harmless as it is under + that spelling on Linux. Each answer, done as it says, lets the same + `add` through.""" + made, linux, folding = OTHER_SPELLING[state] + root = unborn_holder(tmp_path) + (root / made).parent.mkdir(parents=True) + (root / made).write_bytes(b"our own notes\n") + folds = disk_folds(root) + said = folding if folds else linux + if said is not None: + assert refused_untouched(offline, root, family["remotes"]) == said + if said is TAKEN: + shutil.rmtree(root / "Members" / SPARE) + elif said is not None: + assert len(type_remedy(root, said[1])) == 1 + result = offline.add(root, SPARE, remotes=family["remotes"]) + assert result.returncode == 0, result.stdout + result.stderr + assert git("ls-tree", "-r", "--name-only", "HEAD", + cwd=root).stdout.split() == [".gitmodules", "family.yaml", + MOUNT] + assert os.path.exists(root / made) is (folds and said is None) + + +@pytest.mark.skipif(os.name == "nt", reason="a symlink needs a privilege " + "Windows runners do not grant") +def test_a_spelling_that_is_a_link_is_named_and_not_entered(family, offline, + tmp_path): + """A `Members` that is a LINK to a directory holding `IRQS` is named + as the link it is and moved, link and all, and what it points at is + not entered: no disk takes a link for `members/` itself.""" + elsewhere = tmp_path / "elsewhere" + (elsewhere / SPARE).mkdir(parents=True) + root = unborn_holder(tmp_path) + os.symlink(elsewhere, root / "Members", target_is_directory=True) + said, remedy = refused_untouched(offline, root, family["remotes"]) + assert said == DIR_VARIANT + "on disk and untracked: `Members` (a symlink)" + assert remedy == remedy_of(move("Members")) + assert type_remedy(root, remedy) == ["mv -- Members ../moved-1"] + added(offline, root, family["remotes"]) + assert (elsewhere / SPARE).is_dir() + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="a directory nobody may list needs POSIX " + "permissions, which root does not obey") +def test_another_spelling_nobody_may_list_is_named_and_moved_whole( + family, offline, tmp_path): + """A `Members/` this user may not list cannot be named entry by entry, + and is not empty for all anyone can tell: on Linux it is named, and + moved, whole. (On a disk that folds case that `Members` is `members/` + itself, which nothing names.)""" + if disk_folds(tmp_path): + pytest.skip("on a disk that folds case this Members is members/") + root = unborn_holder(tmp_path) + (root / "Members").mkdir() + (root / "Members" / "notes.txt").write_bytes(b"our own notes\n") + (root / "Members").chmod(0o300) + try: + said, remedy = refused_untouched(offline, root, family["remotes"]) + assert type_remedy(root, remedy) == ["mv -- Members ../moved-1"] + finally: + for left in (root / "Members", tmp_path / "moved-1"): + if left.is_dir(): + left.chmod(0o755) + assert said == DIR_VARIANT + "on disk and untracked: `Members` (a " \ + "directory)" + assert remedy == remedy_of(move("Members")) + added(offline, root, family["remotes"]) + + +@pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, + reason="a directory nobody may list needs POSIX " + "permissions, which root does not obey") +def test_a_members_directory_nobody_may_list_still_hides_nothing( + family_module, tmp_path): + """`lexists` IS STILL ASKED, LAST. A `members/` this user may enter and + not list hides `members/IRQS` from every listing, and the path itself + still answers: refused `already exists`, as it always was.""" + root = unborn_holder(tmp_path) + (root / MOUNT).mkdir(parents=True) + (root / "members").chmod(0o300) + try: + with pytest.raises(family_module.Refusal) as caught: + family_module._checked_mount_path(root, SPARE, MOUNT) + finally: + (root / "members").chmod(0o755) + assert (caught.value.code, caught.value.detail) == ( + "member-path-taken", f"{MOUNT} already exists in the family root but " + "no `members:` row names it") + + +# --- one directory or two: what each disk answers ---------------------------- + +def disk_as(monkeypatch, root: Path, folding: bool) -> None: + """`os.path.isdir`, `os.path.samefile` and `os.path.lexists` answer, + BELOW `root` ONLY, as a disk that folds case answers them (macOS's and + Windows' by default) or as one that does not (Linux's), whatever disk + the test runs on: each component is found in its parent's listing, + exactly, or, folding, in any case. Listings stay exact; both kinds of + disk list a name in the spelling they keep.""" + real_isdir, real_samefile, real_lexists = ( + os.path.isdir, os.path.samefile, os.path.lexists) + + def resolved(path) -> Path | None: + path = Path(path) + try: + parts = path.relative_to(root).parts + except ValueError: + return path + here = root + for part in parts: + try: + names = os.listdir(here) + except OSError: + return path + same = [name for name in names if name == part + or (folding and name.lower() == part.lower())] + if not same: + return None + here = here / same[0] + return here + + def samefile(first, second) -> bool: + one, other = resolved(first), resolved(second) + if one is None or other is None: + raise FileNotFoundError(first if one is None else second) + return real_samefile(one, other) + + monkeypatch.setattr(os.path, "isdir", + lambda path: resolved(path) is not None + and real_isdir(resolved(path))) + monkeypatch.setattr(os.path, "samefile", samefile) + monkeypatch.setattr(os.path, "lexists", + lambda path: resolved(path) is not None + and real_lexists(resolved(path))) + + +def renamed_members(root: Path) -> None: + """The holder's `members/` renamed `Members/` in place, the members' + checkouts in it: what a case-insensitive checkout of a holder that held + `Members/README` beside them makes, and, on Linux, Codex's state, a + `members/` a hand renamed with nothing staged.""" + os.rename(root / "members", root / "Members") + + +def with_file(made: str, staged: bool = False): + """A layout: `members/` renamed, and `made` written there, as BYTES, + planted in the index too where `staged`.""" + def lay_out(root: Path) -> None: + renamed_members(root) + (root / made).parent.mkdir(parents=True, exist_ok=True) + (root / made).write_bytes(b"our own notes\n") + if staged: + plant(root, "100644", made) + return lay_out + + +def human_members(root: Path) -> None: + """`members/` gone from disk, and a human's own `Members/` holding a + `README.md` and a directory named like the member IRRS, with no + checkout in it.""" + shutil.rmtree(root / "members") + (root / "Members" / "IRRS").mkdir(parents=True) + (root / "Members" / "IRRS" / "notes").write_bytes(b"our own notes\n") + (root / "Members" / "README.md").write_bytes(b"our own notes\n") + + +def gitlink_named_members(root: Path) -> None: + """`members/` renamed, and a gitlink `Members` in the index: on a disk + that folds case, the directory that holds the members. (No Linux + checkout makes it: a gitlink there would be its own directory.)""" + renamed_members(root) + plant(root, "160000", "Members") + + +def file_named_members_in_the_index(root: Path) -> None: + """A file `Members` in the index, nothing of it on disk, beside the + holder's `members/`, which a disk that folds case resolves it to.""" + plant(root, "100644", "Members") + + +def variant_in_members(root: Path) -> None: + """An untracked `members/irqs`, under `members/` itself.""" + (root / "members" / SPARE.lower()).mkdir() + (root / "members" / SPARE.lower() / "x").write_bytes(b"x\n") + + +def linked_members(root: Path) -> None: + """`members/` gone, and a LINK `Members` to a directory outside the + holder: a disk that folds case resolves `members` through it, and it is + still never `members/` itself, but a link, named and moved.""" + if os.name == "nt": + pytest.skip("a symlink needs a privilege Windows runners do not grant") + shutil.rmtree(root / "members") + (root.parent / "elsewhere").mkdir() + os.symlink(root.parent / "elsewhere", root / "Members", + target_is_directory=True) + + +def every_index_remedy(root: Path) -> None: + """`members/` renamed, and three entries under `Members/` committed: a + gitlink `X` with its directory on disk, renamed; a file `README` where + the disk holds a directory, taken out of the index, which HEAD holds, so + the removal is committed too; and `docs/a`, which a sparse checkout + keeps off disk, renamed with `--sparse` by the entry itself.""" + renamed_members(root) + (root / "Members" / "README").mkdir() + (root / "Members" / "README" / "inner").write_bytes(b"x\n") + (root / "Members" / "X").mkdir() + plant(root, "100644", "Members/README") + plant(root, "160000", "Members/X") + plant(root, "100644", "Members/docs/a") + git("update-index", "--skip-worktree", "--", "Members/docs/a", cwd=root) + git("commit", "-qm", "Entries under Members/", cwd=root) + + +def gitlink_over_a_human_members(root: Path) -> None: + """`members/` gone, a human's `Members/notes.txt`, and a gitlink + `Members` in the index: what is under that entry is the index's to + name, never a second time on disk.""" + shutil.rmtree(root / "members") + (root / "Members").mkdir() + (root / "Members" / "notes.txt").write_bytes(b"our own notes\n") + plant(root, "160000", "Members") + + +IN_MEMBERS_README = DIR_VARIANT + "in the index: `Members/README` (a file)" +IN_MEMBERS_IRQS = (MOUNT_VARIANT + f"in the index: `Members/{SPARE}/README` " + "(a file)") +UNDER_MEMBERS_IRQS = f"on disk and untracked: `Members/{SPARE.lower()}` (a " \ + "directory)" +MOVED_BACK_TOO = "; on disk and untracked: `Members` (a directory)" +LINKED = (DIR_VARIANT + "on disk and untracked: `Members` (a symlink)", + remedy_of(move("Members"))) +EVERY_INDEX = (DIR_VARIANT + "in the index: `Members/README` (a file), " + "`Members/X` (a submodule), `Members/docs/a` (a file)") +EVERY_INDEX_REMEDY = index_part( + rename=["Members/X"], sparse=["Members/docs/a"], + unstage=["Members/README"], commit="both") +IRQS_FILE = MOUNT_VARIANT + f"in the index: `Members/{SPARE}` (a file)" +IRQS_FILE_REMEDY = index_part(rename=[f"Members/{SPARE}"], commit="the rename") +GITLINK_MEMBERS = DIR_VARIANT + "in the index: `Members` (a submodule)" + +#: `{layout: (how it is made, Linux's answer, the folding disk's answer)}`, +#: each answer the refusal's two lines, None where the check passes, and +#: `...` for a layout that disk never holds. +LAYOUTS = { + "members/ renamed Members (Codex's P2)": ( + renamed_members, + (DIR_VARIANT + "on disk and untracked: `Members` (a directory)", + remedy_of(MOVE_BACK)), + None), + "and Members/README in the index (#362, B1)": ( + with_file("Members/README", staged=True), + (IN_MEMBERS_README + "; on disk and untracked: `Members` (a " + "directory)", + remedy_of(index_part(rename=["Members/README"], commit="the rename"), + MOVE_BACK)), + (IN_MEMBERS_README, + remedy_of(index_part(rename=["Members/README"], + commit="the rename")))), + "and an untracked Members/IRQS": ( + with_file(f"Members/{SPARE}/leftover"), + (MOUNT_VARIANT + f"on disk and untracked: `Members/{SPARE}` (a " + "directory), `Members` (a directory)", + remedy_of(move(f"Members/{SPARE}"), MOVE_BACK)), + TAKEN), + "and Members/IRQS/README in the index (#318)": ( + with_file(f"Members/{SPARE}/README", staged=True), + (IN_MEMBERS_IRQS + "; on disk and untracked: `Members` (a directory)", + remedy_of(index_part(rename=[f"Members/{SPARE}"], + commit="the rename"), MOVE_BACK)), + (IN_MEMBERS_IRQS, + remedy_of(index_part(rename=[f"Members/{SPARE}"], + commit="the rename")))), + "and an untracked Members/irqs": ( + with_file(f"Members/{SPARE.lower()}/x"), + (MOUNT_VARIANT + UNDER_MEMBERS_IRQS + ", `Members` (a directory)", + remedy_of(move(f"Members/{SPARE.lower()}"), MOVE_BACK)), + (MOUNT_VARIANT + UNDER_MEMBERS_IRQS, + remedy_of(move(f"Members/{SPARE.lower()}")))), + "and a gitlink named Members in the index": ( + gitlink_named_members, ..., + (DIR_VARIANT + "in the index: `Members` (a submodule)", + remedy_of(index_part(unstage=["Members"])))), + "a human's own Members/, no members/": ( + human_members, + (DIR_VARIANT + "on disk and untracked: `Members/IRRS` (a directory), " + "`Members/README.md` (a file)", + remedy_of(move("Members/IRRS", "Members/README.md"))), + None), + "a file Members in the index, beside members/": ( + file_named_members_in_the_index, + (DIR_VARIANT + "in the index: `Members` (a file)", + remedy_of(index_part(unstage=["Members"]))), + (DIR_VARIANT + "in the index: `Members` (a file)", + remedy_of(index_part(unstage=["Members"])))), + "an untracked members/irqs": ( + variant_in_members, + (MOUNT_VARIANT + f"on disk and untracked: `members/{SPARE.lower()}` " + "(a directory)", remedy_of(move(f"members/{SPARE.lower()}"))), + (MOUNT_VARIANT + f"on disk and untracked: `members/{SPARE.lower()}` " + "(a directory)", remedy_of(move(f"members/{SPARE.lower()}")))), + "a link Members, no members/": (linked_members, LINKED, LINKED), + "and every remedy the index takes": ( + every_index_remedy, + (EVERY_INDEX + MOVED_BACK_TOO, + remedy_of(EVERY_INDEX_REMEDY, MOVE_BACK)), + (EVERY_INDEX, remedy_of(EVERY_INDEX_REMEDY))), + "and a file Members/IRQS in the index and on disk": ( + with_file(f"Members/{SPARE}", staged=True), + (IRQS_FILE + MOVED_BACK_TOO, remedy_of(IRQS_FILE_REMEDY, MOVE_BACK)), + (IRQS_FILE, remedy_of(IRQS_FILE_REMEDY))), + "a gitlink Members over a human's Members/": ( + gitlink_over_a_human_members, + (GITLINK_MEMBERS, remedy_of(index_part(rename=["Members"], + commit="the rename"))), + (GITLINK_MEMBERS, remedy_of(index_part(unstage=["Members"])))), +} + + +@pytest.mark.parametrize("layout, folding", [ + pytest.param(layout, folding, + id=f"{layout}-{'folding' if folding else 'linux'}") + for layout, answers in LAYOUTS.items() + for folding in (False, True) if answers[1 + folding] is not ...]) +def test_each_disk_answers_for_the_directory_it_holds( + holder, family_module, monkeypatch, layout, folding): + """ONE LAYOUT ON DISK, TWO DISKS, asked in process (#574's review, B1, + and Codex's P2). Where the holder keeps its members in one directory + spelled `Members` and no `members` is listed, a disk that folds case + answers `members` with that directory (`isdir`, then `samefile`): it is + `members/` itself, so an exact `IRQS` in it is `already exists`, a + variant is named under that spelling, and only the index's own entries + under `Members/` are renamed, one by one, never the directory. On Linux + no `members` is there at all: the same `Members` is a `members/` a hand + renamed, moved back, and what is under it is named. A `Members` in the + index that IS that directory is never renamed, and an index entry is + never taken to be on disk because the disk folds its name onto + something else.""" + lay_out, linux, on_folding = LAYOUTS[layout] + said = on_folding if folding else linux + lay_out(holder) + disk_as(monkeypatch, holder, folding) + if said is None: + assert family_module._checked_mount_path(holder, SPARE, MOUNT) is None + return + with pytest.raises(family_module.Refusal) as caught: + family_module._checked_mount_path(holder, SPARE, MOUNT) + assert tuple(str(caught.value).splitlines()) == said + + +#: `_disk_spellings` over a listing handed in, as `(the listing, the +#: spelling that is members/ on that disk, the gitlinks the index holds, +#: what is named and how)`. Every entry listed holds a `.git`. +MOVED_BACK = { + "once, though two spellings hold checkouts": ( + {"MEMBERS": ["IRRS"], "Members": ["IRSS"]}, None, + {"members/IRRS", "members/IRSS"}, + [("MEMBERS", "mv back"), ("Members/IRSS", "mv")]), + "never where members/ is on disk": ( + {"Members": ["IRRS"], "members": []}, "members", {"members/IRRS"}, + [("Members/IRRS", "mv")]), + "never for a .git no gitlink names": ( + {"Members": ["notes"]}, None, {"members/IRRS"}, + [("Members/notes", "mv")]), +} + + +@pytest.mark.parametrize("case", list(MOVED_BACK)) +def test_only_a_spelling_that_is_members_renamed_is_moved_back( + family_module, monkeypatch, tmp_path, case): + """THE MOVE BACK (Codex's P2) is for ONE spelling, where no `members/` + is on disk, holding a checkout whose gitlink the index holds under + `members/`. Two spellings side by side exist only on a disk that does + not fold case, and no test makes them, so the listing is handed in and + nothing is read from a disk.""" + listing, canon, gitlinks, named = MOVED_BACK[case] + monkeypatch.setattr(os.path, "lexists", + lambda path: Path(path).name == ".git") + monkeypatch.setattr(family_module, "_disk_kind", + lambda path: "a directory") + found = family_module._disk_spellings( + tmp_path, {name: ("a directory", entries) + for name, entries in listing.items()}, + SPARE, canon, gitlinks) + assert [(path, remedy) for path, _, remedy in found] == named + + +def test_the_printed_git_mv_leaves_the_members_where_they_are( + holder, family_module, monkeypatch): + """THE BLOCKER (#574's review, B1), typed: on a disk that folds case, + `Members/README` beside the members in one directory `Members`. The + remedy renames `Members/README` alone; typed as printed, the members' + checkouts are still in that directory, and the check passes.""" + with_file("Members/README", staged=True)(holder) + checkouts = sorted(os.listdir(holder / "Members")) + disk_as(monkeypatch, holder, True) + with pytest.raises(family_module.Refusal) as caught: + family_module._checked_mount_path(holder, SPARE, MOUNT) + assert type_remedy(holder, str(caught.value).splitlines()[1]) == [ + "git mv -- Members/README renamed-1", "git commit"] + assert sorted(os.listdir(holder / "Members")) == [ + name for name in checkouts if name != "README"] + assert (holder / "Members" / "IRRS" / ".git").exists() + assert family_module._checked_mount_path(holder, SPARE, MOUNT) is None + + +def test_a_members_directory_renamed_by_hand_is_moved_back(family, holder, + offline, tmp_path): + """CODEX'S P2 ON #574, ON THE DISK THE TEST RUNS ON. The holder's + `members/` renamed `Members` with nothing staged. On Linux that left + `members/` nowhere on disk, and the add mounted the member in a new + `members/` beside `Members/`, splitting the holder: refused now, and + the printed `mv` puts the directory back. On macOS and Windows the + rename changes a spelling and nothing else, the directory is still + `members/`, and the add goes through as it always did.""" + renamed_members(holder) + if not disk_folds(tmp_path): + said, remedy = refused_untouched(offline, holder, family["remotes"]) + assert said == DIR_VARIANT + "on disk and untracked: `Members` (a " \ + "directory)" + assert remedy == remedy_of(MOVE_BACK) + assert type_remedy(holder, remedy) == ["mv -- Members members"] + added(offline, holder, family["remotes"]) + + +# --- the control ------------------------------------------------------------- + +@pytest.mark.parametrize("ignorecase", ["true", "false"]) +def test_a_holder_with_one_spelling_of_members_adds_a_member( + family, holder, offline, ignorecase): + """THE CONTROL: the holder `init` and `add` made, `members/` in one + spelling everywhere, adds a member whatever `core.ignorecase` says.""" + git("config", "--local", "core.ignorecase", ignorecase, cwd=holder) + result = offline.add(holder, SPARE, remotes=family["remotes"]) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stderr == "" + mounted_alone(holder) + offline.assert_left_nothing()