diff --git a/adopt-project.py b/adopt-project.py index 5fac9479..7e72553e 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -1884,6 +1884,31 @@ def _is_there(path: Path) -> bool: return _is_a(path, lambda _mode: True) +def _is_itself_a(path: str | bytes, kind: Callable[[int], bool]) -> bool: + """Whether `path` is itself there and of `kind` (`stat.S_ISLNK`, + `stat.S_ISDIR`), asked with `os.lstat`, which follows no symlink, as + `_is_a` asks `stat` (#430): nothing there is False, and the `OSError` + of an `lstat` the operating system REFUSED is raised, never read as + nothing there. NOT `Path.is_symlink()` or `is_dir()`, which from 3.14 + answer False to ANY `OSError`. `path` is a `str` or the bytes + `_path_argument` hands git, so that no locale is in the way (#527).""" + try: + mode = os.lstat(path).st_mode + except OSError as exc: + if _absent(exc): + return False + raise + except ValueError: + # a name the operating system cannot take at all (a NUL in it) + return False + return kind(mode) + + +def _is_a_symlink(path: str | bytes) -> bool: + """Whether `path` is itself a symlink (`_is_itself_a`).""" + return _is_itself_a(path, stat.S_ISLNK) + + def _os_reason(exc: OSError) -> str: """The operating system's own words for `exc` (`Permission denied`), or the exception's name where it gave none, as `validate-pins.py`'s @@ -9267,7 +9292,14 @@ def _unsafe_path_findings(plan: Plan) -> tuple[list[str], list[str]]: # set kept as it was in the repository made. Twins in the code leg were # adopted so in 2 runs, and in 3 others, for #1189's review, `execute` # stopped after both legs were pushed, mounting them, at `git rm -r` ("the -# following files have local modifications"). +# following files have local modifications"), until #1183. Measured again +# for #1237's review with this finding taken out, 1 run each: twins kept in +# one leg (`A` and `a` in the code leg, and in the spec leg; `D/x` and +# `d/x`; `F/x` and `f`; `src/A` and `src/a`) were adopted; in the root, +# `A` and `a` were adopted, `F/x` beside `f` was still `adopt-lost` after +# both legs were pushed, and `D/x` beside `d/x` met `will not add file +# alias` in `check` (`plan-file-reencoded`). So the finding names the +# root's `adopt-lost`, and no `git rm`. # # So `check` names them, on every disk, as #520 names a branch in another # case: one plan is then not good on one machine and lost on another. @@ -9280,8 +9312,10 @@ def _unsafe_path_findings(plan: Plan) -> tuple[list[str], list[str]]: # path git refuses is `plan-path-git-refuses`'s alone, and a path no entry # answers once is another finding's. Twins the plan sends to DIFFERENT # repositories, and a twin it drops beside one it keeps, are not asked -# either, though `execute` on a disk that ignores case loses them too, as -# `adopt-lost` after both legs are pushed (#1183). +# either: no repository the split makes holds two of them, and `execute` +# mounts the legs so that a disk that ignores case loses none of them +# (`_split_case_twins`, #1183), where it lost them as `adopt-lost`, or +# stopped at `git rm`, after both legs were pushed. # # A set of ONE NAME whose paths hold one blob in one mode loses nothing on # such a disk: the one path it checks out holds what each path holds. In a @@ -9314,10 +9348,11 @@ def _unsafe_path_findings(plan: Plan) -> tuple[list[str], list[str]]: #: as `plan-case-twins` names it, in the order it is asked. TWIN_REPOSITORIES = {"root": "the assembly root", "spec": "the spec leg", "code": "the code leg"} -#: What a plan holding such twins is told to do, once after every finding. -#: It offers no answer that sends the twins of a set into different -#: repositories, or drops one: on a disk that ignores case `execute` loses -#: those too, after both legs are pushed (#1183). +#: What a plan holding such twins is told to do, once after every finding: +#: a rename in the source. It was written while an answer that sends the +#: twins of a set into different repositories, or drops all of a set but +#: one, was lost by `execute` on a disk that ignores case too, after both +#: legs were pushed; `_mount_the_legs` keeps those since #1183. CASE_TWINS_REMEDIATION = ( "Remediation: rename paths in the source until no two that one " "repository would hold differ only in case -- each twin of a set but " @@ -9414,8 +9449,8 @@ def _case_twin_findings(plan: Plan, tree: list, "macOS or Windows disk does by default, holds each set as one " "name: a clone made there checks out only one path of each set, " "and `execute` could stop there before any leg exists, at git's " - "`will not add file alias`, or after both legs are pushed, as " - "`adopt-lost` or at `git rm` mounting the legs, so `execute` " + "`will not add file alias`, or, for a set in the assembly root, " + "after both legs are pushed, as `adopt-lost`, so `execute` " "refuses them on every disk.") return found, [CASE_TWINS_REMEDIATION] if found else [], notes @@ -14329,22 +14364,229 @@ def _refuse_an_unmountable_path(assembly: Path, path: str) -> None: #: still a pathspec, where `*` and `?` are wildcards and a leading `:` is #: magic -- `git rm -- ':!keep.md'` removes every other path in the tree. GIT_RM = ("git", "--literal-pathspecs", "rm", "-r", "-q", "--") - - -def _git_rm(assembly: Path, path: str) -> None: +#: `GIT_RM` with `-f`, for a path at or above a case twin (#1183): see +#: `_split_case_twins`. `-f` skips git's question whether the file on the +#: disk still holds what the index holds, and that question was also what +#: kept git from removing a file through a symlink: given a path beyond a +#: symlink the disk holds on the way to it, `git rm -f` unlinks the file +#: the symlink leads to, wherever that is (r1237). The path itself counts +#: too where the index holds a gitlink there: git removes the folder at a +#: gitlink's path with all it holds, and follows a symlink there, so `git +#: rm -f` of a gitlink whose path is a symlink empties the folder it leads +#: to (r1237b). So it is never given either: `_taken_beyond_a_symlink` +#: takes each such path first. +GIT_RM_FORCED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "-f", "--") +#: `GIT_RM` with `--cached`, for a path the index holds beyond a symlink +#: the disk holds on the way to it, or a gitlink whose path the disk holds +#: as no folder: git takes it out of the index and touches nothing on the +#: disk (r1237, r1237b). See `_taken_beyond_a_symlink`. +GIT_RM_CACHED = ("git", "--literal-pathspecs", "rm", "-r", "-q", "--cached", + "--") + + +def _git_rm(assembly: Path, path: str, forced: bool = False, + cached: bool = False) -> None: """`git rm` of `path`, an entry's path as the plan holds it, in the assembly: what `run` does, with `path` handed to git as the bytes the - tree holds (`_path_argument`, #527). + tree holds (`_path_argument`, #527). `forced` is `GIT_RM_FORCED`, for a + path a case twin is at or under (#1183), and `cached` `GIT_RM_CACHED`, + for one the index holds beyond a symlink (r1237), or a gitlink whose + path the disk holds as no folder (r1237b). `run` takes only text (`check_program`), so this asks `check_program` of the command as the person reads it, and a git that refuses raises `CommandFailed` with that command, as `run` raises it: what `execute` reports after both legs exist is unchanged. """ - command = [*GIT_RM, path] + removal = GIT_RM_CACHED if cached else GIT_RM_FORCED if forced \ + else GIT_RM + command = [*removal, path] check_program(command) - proc = subprocess.run([*GIT_RM, _path_argument(path)], cwd=str(assembly), - capture_output=True, check=False) + proc = subprocess.run([*removal, _path_argument(path)], + cwd=str(assembly), capture_output=True, check=False) + if proc.returncode != 0: + raise CommandFailed(command, assembly, proc.returncode, + (proc.stderr + proc.stdout).decode("utf-8", + "replace")) + + +#: `git checkout-index`, writing each path it reads from its input, each +#: ended by a NUL, from the index over whatever the disk holds there. +CHECKOUT_INDEX = ("git", "checkout-index", "-f", "-z", "--stdin") + + +def _at_or_under(path: str, top: str) -> bool: + """Whether `path` is `top` or a path under it, both as git spells them: + case counts, as it does in the index.""" + return path == top or path.startswith(top + "/") + + +def _index_modes(assembly: Path) -> dict[str, str]: + """Each path the assembly's index holds, in its order, as `_git_path` + spells it, and its mode (`GITLINK_MODE` for a gitlink): `git ls-files + -s -z`, which writes nothing, read as `_index_entries` reads it.""" + raw = git_out(["ls-files", "-s", "-z"], cwd=assembly, binary=True) + modes = {} + for record in raw.split(b"\x00"): + head, _, name = record.partition(b"\t") + if name: + modes[_git_path(name)] = head.split()[0].decode("ascii") + return modes + + +def _split_case_twins(held: list[str], + removed: list[str]) -> list[list[str]]: + """Each set of case twins of `held`, the paths the assembly's index + holds (`_index_modes`, `_case_twin_sets`), that has a path at or under + one of `removed`, the paths `_mount_the_legs` takes out of the root: a + set the plan sends into different repositories, or drops a path of + (#1183); and a set it keeps whole in one leg, which reaches the mount + only as one of one blob in one mode, the NOTE `check` prints for it + (`_one_blob_twins`), every other being `plan-case-twins`. The paths of + such a set are removed from the root with `-f` too, and none is written + back, since the root keeps none of them (#1237's review, NIT 1). + + A DISK THAT IGNORES CASE HOLDS EACH SUCH SET AS ONE FILE. Measured for + #1183 with git 2.43, on a FUSE disk that ignores case and keeps it: the + split's clone checks out one file per set, holding the bytes of the twin + git wrote last, in the index's order (`a` after `A`, `d/x` after `D/x`), + git saying `the following paths have collided`. The mount's `git rm` of + a twin then meets that file: + - where the twin that goes sorts last (`A` kept in the root, `a` sent to + the code leg or dropped, and `F/x` kept beside `f`), `git rm` deletes + the one file both names share, `git add -A` stages the kept twin's + deletion, and `execute` stopped after both legs and the split were + pushed, as `adopt-lost`; + - where it sorts first (`A` sent to a leg or dropped, `a` kept), `git + rm` refused, `the following file has local modifications: A`, and + `execute` stopped mounting the legs after both were pushed: also with + BOTH twins dropped, or one sent to each leg. + So each path such a set has at or under it is removed with `-f`, and + the paths of these sets that the root keeps are written back from the + index after the last `git rm`. + + The list is asked of the index, which holds the source commit's tree, + once, and on every disk: on a disk that keeps case, the same commands + rewrite the bytes each kept twin already holds. A source with no such + set, as nearly every one is, gets the `git rm` of each path it always + got and nothing written back: that listing, which writes nothing, is + the one command added.""" + return [named for named in _case_twin_sets(held) + if any(_at_or_under(twin, top) + for twin in named for top in removed)] + + +def _on_the_disk(assembly: Path, path: str) -> str | bytes: + """`path`, as git spells it, in the assembly, as `os.lstat` is handed + it: the bytes the tree holds (`_path_argument`), so that no locale is + in the way (#527), or the text on Windows.""" + here = _path_argument(path) + return (os.path.join(os.fsencode(assembly), here) + if isinstance(here, bytes) else os.path.join(assembly, here)) + + +def _beyond_a_symlink(assembly: Path, path: str) -> str | None: + """The first folder on the way to `path`, a path the assembly's index + holds, from the top, that the disk holds as a symlink, as git spells + it; None where there is none (r1237). Each folder is asked with + `lstat` (`_is_a_symlink`, `_on_the_disk`), and none beyond the first + symlink, which would be asked in the folder it leads to. `path` itself + is not asked here: `git rm` of a file's or a symlink's entry unlinks + what is at the path, never what a symlink there leads to. A gitlink's + entry is another matter, and `_taken_beyond_a_symlink` asks a + gitlink's own path (r1237b).""" + parts = path.split("/") + for end in range(1, len(parts)): + above = "/".join(parts[:end]) + if _is_a_symlink(_on_the_disk(assembly, above)): + return above + return None + + +def _taken_beyond_a_symlink(assembly: Path, top: str, + held: dict[str, str]) -> bool: + """`git rm --cached` (`GIT_RM_CACHED`) of each path of `held`, the + paths the assembly's index holds and their modes (`_index_modes`), at + or under `top` that the disk holds beyond a symlink + (`_beyond_a_symlink`), asked before `top` is removed with `-f`: of the + symlink's own path where that is at or under `top`, which takes each + path beyond it at once, else of the path; and of each gitlink there + whose own path the disk holds as no folder (`_is_itself_a`): a + symlink, a file, or nothing. Whether that took every path `held` at or + under `top`, leaving none for `-f`. + + WHERE A FOLDER ON THE WAY IS A SYMLINK (#1237's review, r1237). A disk + that ignores case holds a folder `D/` and a symlink `d` as one name, + and the split's clone writes `d` last, so `D` is that symlink. Measured + with git 2.43 on a FUSE disk that ignores case, for a plan that sends + `D/` to the code leg, or drops it, and keeps `d` in the root, `check` + printing `plan ok`: `git rm -f -- D` unlinked `D/x` through the + symlink, the file `x` in the folder it leads to, outside the work + directory for `d` leading to an absolute path or to `../../victim`, + and `execute` printed `adoption verified`; for `d` leading to `E`, a + folder the root keeps, `E/x` was gone, and `execute` stopped as + `adopt-lost` after both legs and the split were pushed. Without `-f`, + git refused `D/x` as a local modification. So such a path is taken out + of the index alone, and no file on the disk is touched: git counts a + path beyond a symlink as gone from the work tree, `git add -A` does not + follow a symlink into a folder, and the symlink is the twin the root + keeps, written back from the index. The root's tree is the plan's, as + is each leg's, which `git filter-repo` made from the objects. + + WHERE THE INDEX HOLDS A GITLINK (#1237's second review, r1237b). `git + rm` of a gitlink's entry is no unlink: git removes the folder at its + path with everything in it, and opens that path as a folder, following + a symlink there. Measured with git 2.43, for a gitlink `D` the disk + holds as a symlink to a folder outside, `git rm -f -- D` emptied that + folder and then said `fatal: could not remove 'D'`; on a FUSE disk that + ignores case, for a plan that drops the gitlink `D` and `.gitmodules` + and keeps `d`, a symlink, in the root (or drops the folder `M/` that + holds both), `check` printed `plan ok` and `execute` emptied the folder + `d` leads to after both legs were pushed. For a gitlink the disk holds + as a file, or below one (`M` a file, `M/D` the gitlink), `git rm -f` + stopped at the same `could not remove`, and deleted nothing. So the + gitlink's own path is asked too, once no folder on the way is a + symlink, and where it is no folder it is taken out of the index alone: + `git rm --cached` touched nothing there in each case. A folder there, + the empty one a clone writes for a gitlink, is removed with `-f` as + before. + + REFUSING THERE WAS REJECTED: it would come after both legs are pushed, + for a plan that is good on every disk, and a re-run then meets + `leg-remote-exists`.""" + under = [path for path in held if _at_or_under(path, top)] + taken: list[str] = [] + for path in under: + if any(_at_or_under(path, gone) for gone in taken): + continue + link = _beyond_a_symlink(assembly, path) + if link is not None: + gone = link if _at_or_under(link, top) else path + elif held[path] == GITLINK_MODE and not _is_itself_a( + _on_the_disk(assembly, path), stat.S_ISDIR): + gone = path + else: + continue + _git_rm(assembly, gone, cached=True) + taken.append(gone) + return bool(taken) and all(any(_at_or_under(path, gone) + for gone in taken) for path in under) + + +def _write_from_the_index(assembly: Path, paths: list[str]) -> None: + """`git checkout-index` of `paths`, which the assembly's index holds, + over whatever the disk holds at each (#1183): handed to git on its + input as the bytes the tree holds, each `_git_path` undone, so that no + locale and no length of a command line is in the way. + + `CommandFailed` where git refuses, naming the command as `_git_rm` + names its own; what git said names the path.""" + command = list(CHECKOUT_INDEX) + check_program(command) + proc = subprocess.run( + command, cwd=str(assembly), capture_output=True, check=False, + input=b"".join(path.encode("utf-8", "surrogateescape") + b"\x00" + for path in paths)) if proc.returncode != 0: raise CommandFailed(command, assembly, proc.returncode, (proc.stderr + proc.stdout).decode("utf-8", @@ -14361,13 +14603,31 @@ def _mount_the_legs(assembly: Path, work_root: Path, names: dict, urls: dict, Each path is the source tree's, so `_git_rm` hands it to git as the bytes the tree holds (#527). The mount paths are `checked_value`s, ASCII. + + CASE TWINS THE PLAN SPLITS (#1183): a path with a twin at or under it + is removed with `-f`, and each twin the root keeps is written back from + the index after the last `git rm` (`_split_case_twins`), so that a disk + that ignores case loses none of them. Before that `-f`, each path the + index holds at or under it beyond a symlink the disk holds on the way, + and each gitlink there whose own path the disk holds as no folder, is + taken out of the index alone (`_taken_beyond_a_symlink`, r1237, + r1237b), so that `-f` removes nothing through a symlink. """ removed = sorted((p.rstrip("/") for p in (paths_for["spec"] + paths_for["code"] + paths_for["drop"])), key=lambda p: (p != GITMODULES, p)) + held = _index_modes(assembly) + split = [twin for named in _split_case_twins(list(held), removed) + for twin in named] for path in removed: - _git_rm(assembly, path) + forced = any(_at_or_under(twin, path) for twin in split) + if not (forced and _taken_beyond_a_symlink(assembly, path, held)): + _git_rm(assembly, path, forced) + kept = [twin for twin in split + if not any(_at_or_under(twin, path) for path in removed)] + if kept: + _write_from_the_index(assembly, kept) # The plan moved (or dropped) the source's own .gitmodules, so its deletion # is staged, and `git submodule add` refuses to write into a file the index # says is going away. Give the assembly a fresh, EMPTY, staged one for its diff --git a/tests/test_adopt_check_throwaway_repos.py b/tests/test_adopt_check_throwaway_repos.py index 4999e37c..deaa069b 100644 --- a/tests/test_adopt_check_throwaway_repos.py +++ b/tests/test_adopt_check_throwaway_repos.py @@ -46,14 +46,37 @@ of 4 fresh sources. Now `plan-case-twins` names each set the plan keeps in one repository, the root or a leg, on every disk, and `execute` refuses it before any leg exists. + +#1183: twins the plan sends into different repositories, or drops one of, +are no finding, and `execute` lost them on such a disk. Measured with git +2.43 on 4d40ce9f: the split's clone checks out one file per set, holding the +bytes of the twin git wrote last, and the mount's `git rm` of the twin that +goes deleted that file (`adopt-lost` for the twin the root keeps) or refused +it as a local modification (`REFUSED mounting the legs`), after both legs +were pushed. Now the mount removes such a path with `-f` and writes each +twin the root keeps back from the index. The rows that drive the mount make +that one file on any disk, as hard links; the rows that run `execute` whole +skip on a disk that keeps case, but for one control. + +`-f` also skipped what kept git from removing through a symlink. Where the +clone left a folder as its twin, a symlink to a folder outside the work +directory, `git rm -f` unlinked the file in that folder (r1237); where it +left a gitlink so, `git rm -f`, which removes the folder at a gitlink's +path with all it holds, emptied that folder (r1237b). Now each path the +index holds beyond a symlink on the way, and each gitlink whose own path +the disk holds as no folder (a symlink, a file, or nothing), leaves the +index alone (`git rm --cached`), and nothing on the disk is touched. The +rows make that symlink, or that file, by hand on a disk that keeps case. """ from __future__ import annotations import importlib.util +import json import os import shutil import subprocess +import sys import tempfile from argparse import Namespace from pathlib import Path @@ -62,8 +85,8 @@ import pytest from conftest import (ADOPT, WINDOWS_SKIP, appended_to_command_scope, - findings, git, planned, refused_before_any_leg, - run_script) + findings, git, make_source_repo, planned, + refused_before_any_leg, run_script) from test_adopt_nested_attributes import (KEEP_GITHUB, ROOT, ROOT_ATTRIBUTES, SOURCE, YML_CRLF, adoption_plan, committed_raw, github) @@ -72,6 +95,11 @@ from test_adopt_root_symlinks import links_as_links # noqa: F401 from test_adopt_root_symlinks import unwritable_work # noqa: F401 from test_adopt_root_symlinks import planned as planned_with_links +#: #527's child, `_mount_the_legs` under `LC_ALL=C PYTHONUTF8=0`, and the +#: environment it runs in. +from test_adopt_git_rm_bytes import MOUNT_IN_THE_C_LOCALE, c_locale_or_skip +#: The one skip for a mode that root reads through and Windows ignores. +from test_validate_pins_unreadable_workflow import CANNOT_RUN_AS_ROOT #: The two findings #910 adds, and the codes of `execute`'s refusals. PATH_GIT_REFUSES = "plan-path-git-refuses" @@ -500,9 +528,10 @@ def test_check_names_each_twin_git_refuses_and_each_set_it_holds(tmp_path): # --------------------------------------------------------------------------- #: The finding #1161 adds, the code of `execute`'s refusal, how its -#: remediation begins, and the whole of it: a rename in the source, and no -#: answer that sends the twins into different repositories, which a disk -#: that ignores case loses too (#1183, r1189's MUST 1). +#: remediation begins, and the whole of it: a rename in the source. It +#: offers no answer that sends the twins into different repositories +#: (r1189's MUST 1), written while a disk that ignores case lost those too; +#: `execute` keeps them since #1183 (the rows under #1183 below). PLAN_CASE_TWINS = "plan-case-twins" TWINS_REMEDY = "Remediation: rename paths in the source until no two" TWINS_REMEDIATION = ( @@ -526,8 +555,8 @@ def twins_kept(sets: str, where: str, paths: str, named: str) -> str: "does by default, holds each set as one name: a clone made there " "checks out only one path of each set, and `execute` could stop " "there before any leg exists, at git's `will not add file " - "alias`, or after both legs are pushed, as `adopt-lost` or at " - "`git rm` mounting the legs, so `execute` refuses them on every " + "alias`, or, for a set in the assembly root, after both legs are " + "pushed, as `adopt-lost`, so `execute` refuses them on every " "disk.") @@ -851,9 +880,9 @@ def test_paths_no_one_repository_holds_as_one_name_are_no_finding( """Folders in another case merge into one whose names differ, and no path is lost. Twins the plan sends to different repositories, and a twin it drops beside one it keeps, are not this finding, which asks one - repository at a time; on a disk that ignores case `execute` loses them - too, as `adopt-lost` after both legs are pushed (r1189's SHOULD 3, - #1183).""" + repository at a time (r1189's SHOULD 3). On a disk that ignores case + `execute` lost them, as `adopt-lost` after both legs were pushed, until + #1183: it keeps them now (the rows under #1183 below).""" source = a_source(tmp_path) with_paths(source, paths, own_blobs=True) assert findings(planned(source, tmp_path, answers)) == [] @@ -932,6 +961,732 @@ def test_on_a_disk_that_ignores_case_check_names_the_twins_too(tmp_path): twins_kept("1 set", ROOT_TWINS, "2 paths", "`A` and `a`")] +# --------------------------------------------------------------------------- +# #1183: case twins the plan sends into different repositories, or drops +# --------------------------------------------------------------------------- + +#: The branch `_commit_the_split` checks the assembly out on. +SPLIT_BRANCH = "adopt/three-repo-shape" +#: The two legs as `_mount_the_legs` finds them in the work directory. +LEG_NAMES = {"spec": "Northwind-spec", "code": "Northwind-code"} +#: What the mount adds to the root's index: its two gitlinks and the file +#: that registers them. +MOUNTED = (".gitmodules", "spec", "code") +#: `git rm` of one path, as `_mount_the_legs` runs it, and with `-f`. +REMOVED_AS = ["git", "--literal-pathspecs", "rm", "-r", "-q", "--"] +FORCED_AS = ["git", "--literal-pathspecs", "rm", "-r", "-q", "-f", "--"] +#: How the mount writes the twins the root keeps back from the index. +WRITTEN_AS = ["git", "checkout-index", "-f", "-z", "--stdin"] + + +def at_or_under(path: str, top: str) -> bool: + return path == top or path.startswith(top + "/") + + +def z_rows(repository: Path, *args: str) -> list[str]: + """The rows `git ` prints in `repository`, each ended by a NUL, + decoded as `_git_path` decodes a path: a byte that is not UTF-8 kept as + a surrogate, which `git`'s text would refuse.""" + raw = subprocess.run(["git", *args], cwd=str(repository), + capture_output=True, check=True).stdout + return [row.decode("utf-8", "surrogateescape") + for row in raw.split(b"\0") if row] + + +def an_assembly(base: Path, paths: tuple[str, ...], + add=with_paths) -> tuple[Path, dict]: + """`(assembly, blobs)`: `a_source` whose commit gains `paths` by `add`, + each with a blob of its own, cloned into `base/work` as + `_commit_the_split` clones it, with no template and the split branch + checked out at that commit, beside the two legs; `blobs` holds each + path of that commit's tree and its blob.""" + source = a_source(base) + add(source, paths, own_blobs=True) + commit = git("rev-parse", "HEAD", cwd=source).stdout.strip() + work = base / "work" + assembly = work / "Northwind" + git("clone", "-q", "--template=", str(source), str(assembly), cwd=base) + git("checkout", "-q", "-B", SPLIT_BRANCH, commit, cwd=assembly) + for role, name in LEG_NAMES.items(): + make_source_repo(work / name, tree={"file.txt": role}, edits=()) + return assembly, {row.split("\t", 1)[1]: row.split()[2] + for row in z_rows(source, "ls-tree", "-r", "-z", + commit)} + + +def as_one_file(assembly: Path, sets: tuple[tuple[str, ...], ...]) -> None: + """What a clone on a disk that ignores case checks out of each of + `sets`, made on a disk that keeps case: ONE FILE, holding the bytes of + the twin git wrote last, the last of the set in the index's order, as + #1183's trace measured. Each other twin is made a hard link to that + twin's file, so the twins share one file as they do there. Where the + clone's own disk ignores case, git did this itself, and nothing is + done.""" + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + return + for named in sets: + *others, last = named + for twin in others: + (assembly / twin).unlink() + os.link(assembly / last, assembly / twin) + + +def mounted(adopter, base: Path, assembly: Path, moved: dict) -> None: + """`_mount_the_legs` of `assembly`, `moved` its `paths_for`, then the + `git add -A -- .` that `_stage_the_split` begins with.""" + work = base / "work" + adopter._mount_the_legs( + assembly, work, LEG_NAMES, + {role: str(work / name) for role, name in LEG_NAMES.items()}, + {"spec": [], "code": [], "drop": [], **moved}, "spec", "code") + git("add", "-A", "--", ".", cwd=assembly) + + +def staged(assembly: Path) -> dict: + """Each path the assembly's index holds but the mount's own, and its + object.""" + held = {row.split("\t", 1)[1]: row.split()[1] + for row in z_rows(assembly, "ls-files", "-s", "-z")} + return {path: oid for path, oid in held.items() if path not in MOUNTED} + + +#: `(the paths a source commit gains, each with a blob of its own, the sets +#: of them a disk that ignores case holds as one, each in the index's +#: order, and the paths the plan sends to a leg or drops, by `leg:`)`. +#: Measured on 4d40ce9f on such a disk, `execute` stopped after both legs +#: were pushed: as `adopt-lost` for the twin the root keeps where the twin +#: that goes sorts last (`across`, `dropped`, `a-folder-kept-beside-a-file`), +#: and `REFUSED mounting the legs` at `git rm` where it sorts first. +SPLIT_TWINS = [ + pytest.param(("A", "a", "D/x", "d/x"), (("A", "a"), ("D/x", "d/x")), + {"code": ["a", "d/"]}, id="across"), + pytest.param(("A", "a", "D/x", "d/x"), (("A", "a"), ("D/x", "d/x")), + {"code": ["A", "D/"]}, id="across-the-other-way"), + pytest.param(("A", "a"), (("A", "a"),), {"drop": ["a"]}, id="dropped"), + pytest.param(("A", "a"), (("A", "a"),), {"drop": ["A", "a"]}, + id="both-dropped"), + pytest.param(("A", "a"), (("A", "a"),), {"spec": ["A"], "code": ["a"]}, + id="one-to-each-leg"), + pytest.param(("F/x", "f"), (("F/x", "f"),), {"code": ["f"]}, + id="a-folder-kept-beside-a-file"), +] + + +@pytest.mark.parametrize(("paths", "sets", "moved"), SPLIT_TWINS) +def test_the_mount_keeps_each_twin_the_root_keeps_with_its_own_blob( + adopter, tmp_path, paths, sets, moved): + """The twins share one file, as on a disk that ignores case. On + 4d40ce9f the mount's `git rm` refused that file as a local + modification where the twin that goes sorts first, and elsewhere the + root kept the other twin's bytes (on such a disk, none: the one file + was gone). Now the root's index holds each path the plan keeps there, + each with its own blob, and no path the plan moved or dropped.""" + assembly, blobs = an_assembly(tmp_path, paths) + as_one_file(assembly, sets) + mounted(adopter, tmp_path, assembly, moved) + removed = [path.rstrip("/") for leg in moved.values() for path in leg] + assert staged(assembly) == { + path: oid for path, oid in blobs.items() + if not any(at_or_under(path, top) for top in removed)} + + +def git_commands_of_the_mount(adopter, monkeypatch, base: Path, + assembly: Path, moved: dict) -> list: + """`(command, input)` of each `git rm` and `git checkout-index` that + `mounted` runs, in order, each command as text.""" + ran = [] + real = subprocess.run + + def recorded(command, *args, **named): + words = [os.fsdecode(word) for word in command] + if "rm" in words[:3] or "checkout-index" in words[:2]: + ran.append((words, named.get("input"))) + return real(command, *args, **named) + + monkeypatch.setattr(subprocess, "run", recorded) + mounted(adopter, base, assembly, moved) + monkeypatch.undo() + return ran + + +def test_a_source_with_no_twin_set_split_is_mounted_as_before( + adopter, monkeypatch, tmp_path): + """`AB` and `ab` are twins the root keeps both of, and `a` goes to the + code leg: no set has a path at or under a path that goes (`a` is no + folder above `ab`), so each path is removed as on 4d40ce9f, without + `-f`, and nothing is written back from the index.""" + assembly, _ = an_assembly(tmp_path, ("AB", "a", "ab", "b/x")) + assert git_commands_of_the_mount( + adopter, monkeypatch, tmp_path, assembly, + {"code": ["a"], "drop": ["b/"]}) == [ + ([*REMOVED_AS, "a"], None), ([*REMOVED_AS, "b"], None)] + + +def test_only_a_path_a_twin_is_at_or_under_is_removed_with_f( + adopter, monkeypatch, tmp_path): + """`a` and `b` go to the code leg, and `A`, the twin of `a`, stays in + the root: `a` alone is removed with `-f`, and `A` alone is written back + from the index, after the last `git rm`. On 4d40ce9f neither was.""" + assembly, _ = an_assembly(tmp_path, ("A", "a", "b")) + assert git_commands_of_the_mount( + adopter, monkeypatch, tmp_path, assembly, {"code": ["a", "b"]}) == [ + ([*FORCED_AS, "a"], None), ([*REMOVED_AS, "b"], None), + (WRITTEN_AS, b"A\0")] + + +def test_a_path_git_cannot_write_back_is_refused_naming_the_command( + adopter, tmp_path): + """`_write_from_the_index` of a path the index does not hold: git + refuses it, and the mount raises `CommandFailed`, one of + `LATE_FAILURES`, naming the command and carrying what git said, as + `_git_rm` does.""" + assembly, _ = an_assembly(tmp_path, ("A",)) + with pytest.raises(adopter.CommandFailed) as caught: + adopter._write_from_the_index(assembly, ["nowhere"]) + assert caught.value.args_list == WRITTEN_AS + assert caught.value.code != 0 + assert "nowhere" in caught.value.output + + +def test_a_forced_git_rm_git_refuses_is_refused_naming_it_with_f( + adopter, tmp_path): + """`_git_rm` with `forced` of a path the index does not hold: git + refuses it, and the mount raises `CommandFailed` naming the command it + ran, with `-f`, as `execute` prints it after both legs exist.""" + assembly, _ = an_assembly(tmp_path, ("A",)) + with pytest.raises(adopter.CommandFailed) as caught: + adopter._git_rm(assembly, "nowhere", True) + assert caught.value.args_list == [*FORCED_AS, "nowhere"] + assert caught.value.code != 0 + assert "nowhere" in caught.value.output + + +#: Case twins whose names hold the byte E9, which is not UTF-8, each spelled +#: as `_git_path` spells it: that byte as the surrogate U+DCE9. +NOT_UTF8_TWINS = ("caf\udce9A", "caf\udce9a") + + +def with_names_not_utf8(repository: Path, paths: tuple[str, ...], + own_blobs: bool = True) -> None: + """`with_paths` for names that are not UTF-8, which it hands `git + mktree` as text: each of `paths` goes into the index as the bytes it + spells, with a blob of its own holding those bytes and a newline, and + the index is committed on `main`.""" + assert own_blobs + for path in paths: + name = path.encode("utf-8", "surrogateescape") + blob = blob_of(repository, name + b"\n") + subprocess.run(["git", "update-index", "--add", "--cacheinfo", + b"100644," + blob.encode("ascii") + b"," + name], + cwd=str(repository), check=True) + git("commit", "-qm", "Add paths", cwd=repository) + + +def test_twins_whose_names_are_not_utf_8_keep_their_own_blobs( + adopter, tmp_path): + """`cafA` kept in the root and `cafa` sent to the code leg, the + byte E9 not UTF-8, sharing one file: the mount reads the index's names + as bytes, and hands the twin it writes back to git as the bytes the + tree holds, so the root's index holds that twin with its own blob. A + listing decoded, or a name encoded, as strict UTF-8 raises a + `UnicodeError` there, after both legs are pushed.""" + try: + os.close(os.open(os.fsencode(tmp_path) + b"/caf\xe9", + os.O_CREAT | os.O_WRONLY, 0o644)) + except (OSError, ValueError): + pytest.skip("this disk, or this platform, takes no file name that " + "is not UTF-8") + kept, moved = NOT_UTF8_TWINS + assembly, blobs = an_assembly(tmp_path, NOT_UTF8_TWINS, + with_names_not_utf8) + as_one_file(assembly, (NOT_UTF8_TWINS,)) + mounted(adopter, tmp_path, assembly, {"code": [moved]}) + assert kept in blobs + assert staged(assembly) == {path: oid for path, oid in blobs.items() + if path != moved} + + +#: `git rm` of one path from the index alone, as the mount takes a path the +#: disk holds beyond a symlink (r1237). +CACHED_AS = ["git", "--literal-pathspecs", "rm", "-r", "-q", "--cached", + "--"] + + +def with_a_link(target: str): + """An `add` for `an_assembly`: each of `paths` goes into the source + commit with a blob of its own, as `with_paths` writes it, but a path + whose last name is `d`, which is a symlink leading to `target`.""" + def add(repository: Path, paths: tuple[str, ...], + own_blobs: bool = True) -> None: + assert own_blobs + for path in paths: + mode, data = (("120000", target.encode()) if path.split("/")[-1] + == "d" else ("100644", f"{path}\n".encode())) + git("update-index", "--add", "--cacheinfo", + f"{mode},{blob_of(repository, data)},{path}", cwd=repository) + git("commit", "-qm", "Add paths", cwd=repository) + return add + + +def as_the_link(assembly: Path, folder: str, link: str) -> None: + """What a clone on a disk that ignores case checks out of the folder + `folder` and its twin `link`, a symlink that sorts after it: the + symlink alone, which both names reach, as r1237 measured. Made on a + disk that keeps case: the folder gives way to a symlink leading where + `link` leads. Where the clone's own disk ignores case, git did this + itself, and nothing is done.""" + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + return + for child in (assembly / folder).iterdir(): + child.unlink() + (assembly / folder).rmdir() + os.symlink(os.readlink(assembly / link), assembly / folder) + + +def what_it_holds(folder: Path) -> dict: + return {child.name: child.read_bytes() for child in folder.iterdir()} + + +#: `(the paths a source commit gains, a path named `d` among them the +#: symlink, where it leads, `{victim}` being a folder outside the assembly, +#: the paths the plan sends to a leg or drops, the `git rm` and `git +#: checkout-index` the mount runs)`. On c9f0cc18 the mount ran `git rm -f` +#: of the folder `D` (or `src`) and unlinked `x` in the folder the symlink +#: leads to. +BEYOND_A_LINK = [ + pytest.param(("D/x", "d"), "{victim}", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-link-out-of-the-assembly"), + pytest.param(("D/x", "d"), "../../victim", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-relative-link-out-of-the-assembly"), + pytest.param(("D/x", "d"), "{victim}", {"drop": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="the-folder-dropped"), + pytest.param(("D/x", "E/x", "d"), "E", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-link-to-a-folder-the-root-keeps"), + pytest.param(("src/D/x", "src/d"), "{victim}", {"drop": ["src/"]}, + [([*CACHED_AS, "src/D"], None), ([*FORCED_AS, "src"], None)], + id="both-in-a-dropped-folder-beside-other-files"), + pytest.param(("D/x", "D/y", "d"), "{victim}", {"code": ["D/"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="two-files-beyond-the-link-taken-at-once"), + pytest.param(("D/x", "D/y", "d"), "{victim}", + {"code": ["D/x"], "drop": ["D/y"]}, + [([*CACHED_AS, "D/x"], None), ([*CACHED_AS, "D/y"], None), + (WRITTEN_AS, b"d\0")], + id="entries-below-the-link-taken-path-by-path"), +] + + +@pytest.mark.parametrize(("paths", "target", "moved", "commands"), + BEYOND_A_LINK) +def test_no_file_is_removed_through_a_symlink_a_case_twin_left( + adopter, monkeypatch, tmp_path, paths, target, moved, commands): + """`D/x` goes, `d`, a symlink to a folder, stays in the root (or goes + with it), and the clone left `D` as that symlink, as on a disk that + ignores case. On c9f0cc18 the mount's `git rm -f -- D` unlinked `x` in + the folder the symlink leads to, outside the work directory where it + leads there, and `execute` printed `adoption verified` (r1237); without + `-f`, git refused it. Now each path beyond the symlink leaves the index + alone, `git rm --cached`, and the folder holds what it held; `git add + -A` stages nothing beyond the symlink, and the root's index holds each + path the plan keeps there with its own blob.""" + victim = tmp_path / "victim" + victim.mkdir() + (victim / "x").write_bytes(b"victim\n") + (victim / "keep").write_bytes(b"keep\n") + assembly, blobs = an_assembly( + tmp_path, paths, with_a_link(target.format(victim=victim))) + link = next(path for path in paths if path.split("/")[-1] == "d") + folder = link[:-1] + "D" + leads_to = assembly / target if target == "E" else victim + held = what_it_holds(leads_to) + as_the_link(assembly, folder, link) + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, moved) + assert what_it_holds(leads_to) == held + assert ran == commands + removed = [path.rstrip("/") for leg in moved.values() for path in leg] + left = staged(assembly) + # A disk that keeps case holds the symlink made in `folder`'s place + # beside `link`, and `git add -A` adds it there; on a disk that + # ignores case the two are one, `link`'s own. + assert left.pop(folder, blobs[link]) == blobs[link] + assert left == {path: oid for path, oid in blobs.items() + if not any(at_or_under(path, top) for top in removed)} + + +def test_a_folder_that_is_no_symlink_is_still_removed_with_f( + adopter, monkeypatch, tmp_path): + """The control: `D/x` goes and `d`, a symlink to a folder outside the + assembly, stays in the root, on a disk that keeps case, where `D` is + the folder the clone wrote. `D` is removed with `-f`, as on c9f0cc18: + `D/x` is gone from the disk and the index, the folder `d` leads to + holds what it held, and `d` is written back from the index.""" + victim = tmp_path / "victim" + victim.mkdir() + (victim / "x").write_bytes(b"victim\n") + assembly, blobs = an_assembly(tmp_path, ("D/x", "d"), + with_a_link(str(victim))) + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + pytest.skip("on a disk that ignores case the clone leaves `D` the " + "symlink `d`: the rows above") + assert (assembly / "D" / "x").read_bytes() == b"D/x\n" + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, {"code": ["D/"]}) + assert ran == [([*FORCED_AS, "D"], None), (WRITTEN_AS, b"d\0")] + assert not os.path.lexists(assembly / "D") + assert what_it_holds(victim) == {"x": b"victim\n"} + assert staged(assembly) == {path: oid for path, oid in blobs.items() + if path != "D/x"} + + +#: `(what the disk holds, each folder or symlink by its path, a path the +#: index holds, the folder `_beyond_a_symlink` names)`. A symlink leads to +#: `elsewhere`, a folder beside them; one made beyond a symlink is made in +#: the folder that symlink leads to. +ON_THE_WAY = [ + pytest.param({"D": "link"}, "D/x", "D", id="the-folder-a-symlink"), + pytest.param({"D": "folder", "D/E": "link"}, "D/E/x", "D/E", + id="a-folder-below-it-a-symlink"), + pytest.param({"D": "link", "D/E": "link"}, "D/E/x", "D", + id="a-symlink-beyond-the-first-symlink"), + pytest.param({"D": "folder"}, "D/x", None, id="a-folder"), + pytest.param({"d": "link"}, "d", None, id="the-path-itself-a-symlink"), + pytest.param({}, "nowhere/D/x", None, id="nothing-there"), + pytest.param({"F": "file"}, "F/G/x", None, id="a-file-on-the-way"), +] + + +@pytest.mark.parametrize(("layout", "path", "named"), ON_THE_WAY) +def test_the_first_symlink_on_the_way_is_named_and_nothing_else( + adopter, tmp_path, layout, path, named): + """Each folder on the way to the path is asked with `lstat`, from the + top: the first that is a symlink is named, and not a symlink beyond + it; the path itself, which `git rm` of a file's or a symlink's entry + unlinks as what it is, is not asked here (a gitlink's is, by + `_taken_beyond_a_symlink`: the rows under r1237b below), and a folder + that is not there, or a file, is no symlink.""" + (tmp_path / "elsewhere").mkdir() + assembly = tmp_path / "assembly" + assembly.mkdir() + for where, what in layout.items(): + if what == "link": + os.symlink(tmp_path / "elsewhere", assembly / where) + elif what == "folder": + (assembly / where).mkdir() + else: + (assembly / where).write_bytes(b"F\n") + assert adopter._beyond_a_symlink(assembly, path) == named + + +@CANNOT_RUN_AS_ROOT +def test_a_folder_on_the_way_nobody_may_search_is_raised_not_read_as_none( + adopter, tmp_path): + """A folder on the way that this user may not search: its `lstat` is + refused, and the `PermissionError` is raised, one of `LATE_FAILURES`, + so `execute` names it, as `_is_a` raises a refused `stat` (#430). Read + as no symlink, the path would be removed with `-f` unasked.""" + assembly = tmp_path / "assembly" + (assembly / "D" / "E").mkdir(parents=True) + (assembly / "D").chmod(0) + try: + with pytest.raises(PermissionError): + adopter._beyond_a_symlink(assembly, "D/E/x") + finally: + (assembly / "D").chmod(0o755) + assert issubclass(PermissionError, adopter.LATE_FAILURES) + + +@CANNOT_RUN_AS_ROOT +def test_nothing_beyond_the_first_symlink_on_the_way_is_asked(adopter, + tmp_path): + """`D` is a symlink to `elsewhere`, a folder this user may not search: + `D` is named, and nothing in `elsewhere` is asked. Asked past the first + symlink, `lstat` of `D/E` is refused there, and the `PermissionError` + would stop the mount after both legs are pushed (r1237b's R02).""" + elsewhere = tmp_path / "elsewhere" + (elsewhere / "E").mkdir(parents=True) + assembly = tmp_path / "assembly" + assembly.mkdir() + os.symlink(elsewhere, assembly / "D") + elsewhere.chmod(0) + try: + assert adopter._beyond_a_symlink(assembly, "D/E/x") == "D" + finally: + elsewhere.chmod(0o755) + + +# --------------------------------------------------------------------------- +# r1237b: a gitlink whose case twin is a symlink, and every path held asked +# --------------------------------------------------------------------------- + +#: A path's kind in `with_kinds`: a symlink leading to `{victim}`, a folder +#: outside the assembly, and a gitlink. +LINK_OUT = ("link", "{victim}") +GITLINK = ("gitlink", None) + + +def with_kinds(kinds: dict): + """An `add` for `an_assembly`: each of `paths` goes into the source + commit with a blob of its own, as `with_paths` writes it, but a path + `kinds` gives `("link", target)` is a symlink leading to `target`, and + one it gives `GITLINK` a gitlink to the source's first commit, which + the clone checks out as an empty folder.""" + def add(repository: Path, paths: tuple[str, ...], + own_blobs: bool = True) -> None: + assert own_blobs + first = git("rev-parse", "HEAD", cwd=repository).stdout.strip() + for path in paths: + kind, target = kinds.get(path, ("file", None)) + if kind == "gitlink": + mode, oid = "160000", first + else: + mode, data = (("120000", target.encode()) if kind == "link" + else ("100644", (path + "\n").encode())) + oid = blob_of(repository, data) + git("update-index", "--add", "--cacheinfo", + f"{mode},{oid},{path}", cwd=repository) + git("commit", "-qm", "Add paths", cwd=repository) + return add + + +def as_the_twin(assembly: Path, at: str, twin: str) -> None: + """What a clone on a disk that ignores case leaves at `at`, a folder + (a gitlink's empty one among them), beside its twin `twin`, which + sorts after it: `twin` alone, which both names reach, as r1237 and + r1237b measured. Made on a disk that keeps case: the folder gives way + to a symlink leading where `twin` leads, or to a hard link of `twin`'s + file. Where the clone's own disk ignores case, git did this itself, + and nothing is done.""" + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + return + shutil.rmtree(assembly / at) + if (assembly / twin).is_symlink(): + os.symlink(os.readlink(assembly / twin), assembly / at) + else: + os.link(assembly / twin, assembly / at) + + +def outside_the_assembly(base: Path) -> Path: + """`base/victim`, a folder outside the assembly holding `x` and + `keep`.""" + victim = base / "victim" + victim.mkdir() + (victim / "x").write_bytes(b"victim\n") + (victim / "keep").write_bytes(b"keep\n") + return victim + + +#: `(the paths a source commit gains, the kind of each that is no file +#: (`with_kinds`), each `(at, twin)` the clone left as `twin` alone +#: (`as_the_twin`), the paths the plan sends to a leg or drops, the `git +#: rm` and `git checkout-index` the mount runs)`. On 63c52718 the mount +#: ran `git rm -f` of the gitlink `D` (or of `M`), which emptied the folder +#: the symlink leads to and stopped at `fatal: could not remove 'D'` +#: (`'M/D'`), or stopped there where the gitlink was a file or below one. +#: The last two shapes pin what 63c52718 already ran: every path the index +#: holds under a removed path is asked, not only the twins (r1237b's +#: SHOULD 1), and a path under one is the path or below it after a `/` +#: (its NIT 1). +AT_A_GITLINK_OR_BEYOND_A_LINK = [ + pytest.param(("D", "d"), {"D": GITLINK, "d": LINK_OUT}, (("D", "d"),), + {"drop": ["D"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-gitlink-the-clone-left-the-link-dropped"), + pytest.param(("D", "d"), {"D": GITLINK, "d": LINK_OUT}, (("D", "d"),), + {"code": ["D"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-gitlink-the-clone-left-the-link-sent-to-a-leg"), + pytest.param(("M/D", "M/d", "M/z"), {"M/D": GITLINK, "M/d": LINK_OUT}, + (("M/D", "M/d"),), {"drop": ["M/"]}, + [([*CACHED_AS, "M/D"], None), ([*FORCED_AS, "M"], None)], + id="a-gitlink-the-clone-left-the-link-in-a-dropped-folder"), + pytest.param(("D", "d"), {"D": GITLINK}, (("D", "d"),), {"drop": ["D"]}, + [([*CACHED_AS, "D"], None), (WRITTEN_AS, b"d\0")], + id="a-gitlink-the-clone-left-a-file"), + pytest.param(("M/D", "m"), {"M/D": GITLINK}, (("M", "m"),), + {"drop": ["M/"]}, + [([*CACHED_AS, "M/D"], None), (WRITTEN_AS, b"m\0")], + id="a-gitlink-below-a-file-the-clone-left"), + pytest.param(("S/D/x", "S/z", "s/d"), {"s/d": LINK_OUT}, + (("S/D", "s/d"),), {"code": ["S/"]}, + [([*CACHED_AS, "S/D"], None), ([*FORCED_AS, "S"], None), + (WRITTEN_AS, b"s/d\0")], + id="a-file-beside-the-folder-the-clone-left-the-link"), + pytest.param(("D/x", "DD/y", "d", "dd"), + {"d": LINK_OUT, "dd": LINK_OUT}, (("D", "d"), ("DD", "dd")), + {"code": ["D/", "DD/"]}, + [([*CACHED_AS, "D"], None), ([*CACHED_AS, "DD"], None), + (WRITTEN_AS, b"d\0dd\0")], + id="two-folders-one-name-a-prefix-of-the-other"), +] + + +@pytest.mark.parametrize(("paths", "kinds", "made", "moved", "commands"), + AT_A_GITLINK_OR_BEYOND_A_LINK) +def test_nothing_is_removed_through_a_symlink_at_a_gitlink_or_beyond( + adopter, monkeypatch, tmp_path, paths, kinds, made, moved, commands): + """`git rm` of a gitlink's entry removes the folder at its path with + all it holds, and follows a symlink there: on 63c52718, where the clone + left the gitlink `D` the symlink `d`, the mount's `git rm -f -- D` + emptied the folder `d` leads to, outside the assembly, and stopped + after both legs were pushed (r1237b). Now a gitlink whose own path the + disk holds as no folder leaves the index alone, `git rm --cached`, as + each path beyond a symlink on the way does; the folder holds what it + held, and the root's index holds each path the plan keeps there with + its own blob, and none that goes.""" + victim = outside_the_assembly(tmp_path) + assembly, blobs = an_assembly(tmp_path, paths, with_kinds( + {path: (kind, target and target.format(victim=victim)) + for path, (kind, target) in kinds.items()})) + for at, twin in made: + as_the_twin(assembly, at, twin) + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, moved) + assert what_it_holds(victim) == {"x": b"victim\n", "keep": b"keep\n"} + assert ran == commands + removed = [path.rstrip("/") for leg in moved.values() for path in leg] + left = staged(assembly) + # A disk that keeps case holds what was made at `at` beside `twin`, and + # `git add -A` adds it there; on a disk that ignores case the two are + # one, `twin`'s own. + for at, twin in made: + assert left.pop(at, blobs[twin]) == blobs[twin] + assert left == {path: oid for path, oid in blobs.items() + if not any(at_or_under(path, top) for top in removed)} + + +def test_a_gitlink_the_disk_holds_a_folder_at_is_still_removed_with_f( + adopter, monkeypatch, tmp_path): + """The control: the gitlink `D` is dropped and `d`, a symlink to a + folder outside the assembly, stays in the root, on a disk that keeps + case, where `D` is the empty folder the clone wrote for the gitlink. + `D` is removed with `-f`, as on 63c52718: the folder is gone, the + folder `d` leads to holds what it held, and `d` is written back.""" + victim = outside_the_assembly(tmp_path) + assembly, blobs = an_assembly(tmp_path, ("D", "d"), with_kinds( + {"D": GITLINK, "d": ("link", str(victim))})) + if git("config", "--get", "core.ignorecase", cwd=assembly, + check=False).stdout.strip() == "true": + pytest.skip("on a disk that ignores case the clone leaves `D` the " + "symlink `d`: the rows above") + assert [child.name for child in (assembly / "D").iterdir()] == [] + ran = git_commands_of_the_mount(adopter, monkeypatch, tmp_path, + assembly, {"drop": ["D"]}) + assert ran == [([*FORCED_AS, "D"], None), (WRITTEN_AS, b"d\0")] + assert not os.path.lexists(assembly / "D") + assert what_it_holds(victim) == {"x": b"victim\n", "keep": b"keep\n"} + assert staged(assembly) == {path: oid for path, oid in blobs.items() + if path != "D"} + + +def test_the_folder_on_the_way_is_asked_as_bytes_under_a_c_locale(tmp_path): + """`Ü/x` goes to the code leg and `ü`, a symlink to a folder + outside the assembly, stays in the root, the clone having left + `Ü` that symlink; `_mount_the_legs` runs in a child under + `LC_ALL=C PYTHONUTF8=0`, whose file names are ASCII. `lstat` is handed + the bytes the tree holds, so `Ü` is the symlink and `Ü/x` + leaves the index alone. Handed the text, `lstat` raises + `UnicodeEncodeError`, a `ValueError` `_is_itself_a` reads as nothing + there, and `git rm -f` unlinks `x` in the folder the symlink leads to + (r1237b's R03).""" + env = c_locale_or_skip() + victim = outside_the_assembly(tmp_path) + folder, link = "Ü", "ü" + assembly, _ = an_assembly(tmp_path, (f"{folder}/x", link), with_kinds( + {link: ("link", str(victim))})) + as_the_twin(assembly, folder, link) + child = subprocess.run( + [sys.executable, "-c", MOUNT_IN_THE_C_LOCALE, + os.fsencode(ADOPT).hex(), os.fsencode(assembly).hex(), + os.fsencode(tmp_path / "work").hex(), + json.dumps({"spec": [], "code": [f"{folder}/"], "drop": []})], + capture_output=True, env=env, stdin=subprocess.DEVNULL, check=False) + assert child.returncode == 0, child.stderr.decode("utf-8", "replace") + answer = json.loads(child.stdout.decode("ascii")) + assert what_it_holds(victim) == {"x": b"victim\n", "keep": b"keep\n"} + assert "raised" not in answer, answer + held = z_rows(assembly, "ls-files", "-z") + assert f"{folder}/x" not in held + assert link in held + + +needs_filter_repo = pytest.mark.skipif( + shutil.which("git-filter-repo") is None, + reason="git filter-repo is not installed: `pip install git-filter-repo`") + + +def executed(base: Path, paths: tuple[str, ...], answers: dict): + """`execute` of a plan for `a_source` whose commit gains `paths`, each + with a blob of its own, `answers` given, with its legs under + `--local-remote-dir` and its work directory in `base`.""" + source = a_source(base) + with_paths(source, paths, own_blobs=True) + plan = planned(source, base, answers) + return run_script(ADOPT, "execute", "--plan", str(plan), "--yes", + "--local-remote-dir", str(base / "remotes"), + "--work-dir", str(base / "work")) + + +def assert_adopted(result) -> None: + assert result.returncode == 0, result.stderr + result.stdout + assert "adoption verified" in result.stdout, result.stdout + + +#: `(the paths, the plan's answers)`, run whole. On 4d40ce9f, on a disk +#: that ignores case, `execute` exited 1 with `adopt-lost` for `A` and +#: `D/x` (`across`) and `A` (`dropped`), and 2 with `REFUSED mounting the +#: legs` (`across-the-other-way`, `one-to-each-leg`), each after both legs +#: and, for `adopt-lost`, the split were pushed. +SPLIT_EXECUTED = [ + pytest.param(("A", "a", "D/x", "d/x"), + {"A": ROOT, "a": "code", "D/": ROOT, "d/": "code"}, + id="across"), + pytest.param(("A", "a", "D/x", "d/x"), + {"A": "code", "a": ROOT, "D/": "code", "d/": ROOT}, + id="across-the-other-way"), + pytest.param(("A", "a"), {"A": ROOT, "a": "drop"}, id="dropped"), + pytest.param(("A", "a"), {"A": "spec", "a": "code"}, + id="one-to-each-leg"), +] + + +@needs_filter_repo +@pytest.mark.parametrize(("paths", "answers"), SPLIT_EXECUTED) +def test_on_a_disk_that_ignores_case_execute_keeps_twins_it_splits( + tmp_path, paths, answers): + """`execute` whole, its work directory and its legs on the disk `git + init` is asked about; skipped where that disk keeps case, as every CI + disk does (CI is Linux only since #597), where the rows above make the + one file themselves.""" + probe = tmp_path / "probe" + git("init", "-q", str(probe), cwd=tmp_path) + if git("config", "--local", "--get", "core.ignorecase", cwd=probe, + check=False).stdout.strip() != "true": + pytest.skip("needs a disk that ignores case: `git init` wrote no " + "`core.ignorecase=true` in the work directory, so that " + "disk keeps case; the rows that drive the mount make " + "each set one file on every disk") + assert_adopted(executed(tmp_path, paths, answers)) + + +@needs_filter_repo +def test_execute_adopts_twins_it_sends_to_different_repositories(tmp_path): + """The control, on any disk: `A` and `D/x` kept in the root, `a` and + `d/x` sent to the code leg, `execute` whole. On a disk that keeps case + it was adopted on 4d40ce9f too, and it still is.""" + assert_adopted(executed(tmp_path, ("A", "a", "D/x", "d/x"), + {"A": ROOT, "a": "code", "D/": ROOT, + "d/": "code"})) + + # --------------------------------------------------------------------------- # L1-92: in the work directory # ---------------------------------------------------------------------------