diff --git a/adopt-project.py b/adopt-project.py index fb753e54..bd15286b 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -218,6 +218,12 @@ from shape_materialize import ( # noqa: E402 ATTRIBUTES, eol_block_paths, eol_line, with_eol_block, ) +# The copies the shape pins, in the pin's order, and which paths a +# `.gitattributes` line can name literally: asked of a `.gitattributes` the +# plan keeps below the root, over a pinned copy (#530). +from shape_materialize import ( # noqa: E402 + COPIED_FROM_SHAPE, COPIED_VERBATIM, EOL_PATH_RE, +) # What the project's own `validate-manifest.py` would say of the values written # into its manifest, and whether a repository's name can be a directory's: # asked of the plan's values by `plan`, `check` and `execute` (#261). @@ -5927,7 +5933,11 @@ def _entry_findings(plan: Plan, source: Source, project's YAML for its loader to read back as something else (#181), then the source's own submodules, then each symlink the plan keeps in the root that a shape file cannot - be written through (#178, #183), then each value `execute` writes into + be written through (#178, #183), then each `.gitattributes` the plan + keeps below the root that outranks the root's line-ending rule for a + pinned copy (#530), then the source's own root `.gitattributes`, which + the split writes into, where this machine's `git add` would not store + what it writes there (#530's D5(a)), then each value `execute` writes into a file that no file can hold (#255), then what the project's own `validate-manifest.py` would refuse of the values `execute` writes into its manifest (#261), then the split branch's name and the tracking @@ -6040,6 +6050,8 @@ def _entry_findings(plan: Plan, source: Source, else (MANIFEST_FIELDS[TRACKING_KEY],)) return (findings + misread + split + _root_symlink_findings(plan, source, tree) + + _nested_attributes_findings(plan, source, tree) + + _reencoded_attributes_findings(plan, source, tree) + unwritable_findings + value_findings + names + tracking_findings + moved + field_findings, unsafe_remediation @@ -7357,6 +7369,561 @@ def _refuse_symlinks_checked_out_as_files(plan: Plan, source: Source, "again.") +# --------------------------------------------------------------------------- +# A `.gitattributes` the plan keeps BELOW the root, over a pinned copy (#530) +# --------------------------------------------------------------------------- +# +# #471 gives every pinned copy at the root ONE line-ending rule: a `/ +# text eol=lf` line in the block the split appends to the source's own root +# `.gitattributes` (`shape_materialize.with_eol_block`), or, where the source +# keeps none, the shape's own `* text=auto eol=lf`. Git reads attributes for a +# path from the `.gitattributes` of its own directory and of each directory +# above it, and a deeper file outranks a shallower one, so a source's +# `.github/.gitattributes` or `scripts/.gitattributes` that the plan keeps in +# the root outranks that rule for each copy under it. Measured on main at +# 2247dbff with git 2.43: `*.yml eol=crlf` in a kept `.github/.gitattributes` +# -- or `* text=auto eol=crlf`, or a macro the root defines to say it -- +# checked `.github/workflows/validate.yml` out with CRLF in every clone, +# Linux's included, and `* !eol` did so in every clone that writes CRLF +# (`core.autocrlf=true`, Git for Windows' default, or `core.eol=crlf`). Each +# time `execute` said `adoption verified` and exited 0, and the root's first +# `make bootstrap` reported shape-copy-drift: `validate-pins.py` digests the +# bytes on disk (#51). +# +# A FINDING, NOT A WRITE. Brett Heap ruled at 2026-10-12T22:48Z that the +# root's `.gitattributes` is the ONE file of the source's that the split +# writes into, because no file beside it can reach the copies at the root. A +# nested file is not that case: the person can end it, in the source, with +# the lines the finding names, or keep it out of the root, and neither needs +# this tool to write a second file of theirs, to make a second exception to +# "by blob sha" in the verification, or to give `update-shape.py --add` a +# second block to extend. So `check` finds it, and `execute` refuses it with +# every other finding of `check`'s, before any leg exists (#168). +# +# GIT IS ASKED, NEVER IMITATED (`_AttributesProbe`): patterns, macros, +# precedence and what `crlf`, `text` and `eol` make of each other are git's. +# A throwaway repository holds, where the split's tree will hold them, only +# the root's `.gitattributes` as the split writes it, the shape's +# `shape/.gitattributes` where that goes beside the source's, and the kept +# nested files, and `git check-attr` is asked about each pinned copy where +# the split puts it: `_PlannedRoot`'s walks, which follow the root's symlinks +# as the materializer does. No system or global attributes file is read, nor +# a tree `GIT_ATTR_SOURCE` or `attr.tree` names, nor system or global config, +# since a clone on another machine has none of them (`_probe_env`). What a +# checkout then writes over the copy's LF blob is `checked_out_as`: the blob +# itself where the copy is not text or says `eol=lf`, CRLF everywhere where +# it says `eol=crlf`, and CRLF wherever git is set to write it where it is +# text and says no `eol`; git reads `text` first, and `crlf` only where +# `text` is unspecified, both measured with real checkouts. So +# `-text` and `binary` are no finding: a checkout writes a file that is not +# text as its blob, and the blob is LF (measured too). +# +# ONE FINDING PER CAUSE. A copy is outranked when the root's own files keep +# it LF and the kept nested files, added, do not. Each kept file that, +# added to the root's own files alone, makes a checkout write such a copy +# with CRLF is named once (and only where no file alone does, each that +# changes what git says of the copy), with every outranked copy it reaches +# and the line its own directory spells for each; a copy still outranked +# once those lines are in is named again by the next `check`, under the +# file that then decides it. A deeper kept file that restores LF leaves +# nothing to find, and one that alone keeps the copy LF is not named. A root +# `.gitattributes` that is a symlink, which git reads no rule from, is +# #471's follow-up, not this finding: its copies are asked against the block +# that follow-up asks a regular file to hold, so a kept nested file is still +# found. Asked as a disk that keeps case does and again as one that ignores +# it, as #262's findings are: there `core.ignorecase` folds git's patterns, +# and a kept `.GitHub/.gitattributes` is read for `.github/`. A finding only +# the second makes says so. + +#: `check`'s finding, and the code of `execute`'s refusal, for a kept +#: `.gitattributes` below the root that outranks the root's line-ending rule +#: for a pinned copy (#530). +NESTED_ATTRIBUTES = "plan-nested-attributes" + +#: The copies the shape pins, in the pin's order: what `_materialize` lists +#: in `Materialized.shape_files`, each at its own path or under `shape/`. +PINNED_COPIES = (*COPIED_VERBATIM, *(rel for _, rel in COPIED_FROM_SHAPE)) + +#: The attributes a checkout reads a file's line endings from. Git's +#: `convert.c` reads `text` first, and `crlf`, its old spelling, only where +#: `text` is unspecified. +LINE_ENDING_ATTRIBUTES = ("crlf", "text", "eol") + +#: What `crlf` or `text` asks of a checkout, as git's `git_path_check_crlf` +#: reads either one: not text, text checked out as it was checked in, and +#: the rest. Any other value is no answer, and the next is read. +_NOT_TEXT, _AS_CHECKED_IN = "binary", "input" +_TEXT_ACTIONS = {"set": "text", "unset": _NOT_TEXT, + _AS_CHECKED_IN: _AS_CHECKED_IN, "auto": "auto"} + +#: What a checkout writes over a pinned copy's LF blob (`checked_out_as`). +KEPT = "kept" +CRLF_EVERYWHERE = "everywhere" +CRLF_BY_CONFIG = "by config" + +#: What a question to `git check-attr` must not inherit: a repository, an +#: index or a work tree named for another one; a tree to read attributes +#: from instead of the work tree (`GIT_ATTR_SOURCE`); and a parent git's +#: `-c` settings, whose quoted form is not read here and none of which the +#: question needs (`GIT_CONFIG_PARAMETERS`). +_NOT_INHERITED = ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", + "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ATTR_SOURCE", + "GIT_CONFIG_PARAMETERS") +#: The setting that reads attributes from a tree instead of the work tree. +ATTR_TREE = "attr.tree" + + +def _inherited_env() -> dict: + """This process's environment, as a question to a throwaway repository + inherits it: with nothing of `_NOT_INHERITED`, and no `attr.tree` in + git's command scope (`GIT_CONFIG_COUNT`), whose other entries are kept + in their order. Git 2.43 reads `attr.tree` from any config, and one + naming the empty tree made `check-attr` read no rule at all, which + silenced the finding; `GIT_ATTR_SOURCE=HEAD` made git fail in the empty + repository.""" + count = os.environ.get("GIT_CONFIG_COUNT", "") + scoped = [(os.environ.get(f"GIT_CONFIG_KEY_{n}", ""), + os.environ.get(f"GIT_CONFIG_VALUE_{n}", "")) + for n in range(int(count) if count.isdigit() else 0)] + env = {name: value for name, value in os.environ.items() + if name.upper() not in _NOT_INHERITED + and not re.fullmatch(r"GIT_CONFIG_(COUNT|KEY_\d+|VALUE_\d+)", + name.upper())} + kept = [(key, value) for key, value in scoped + if key.lower() != ATTR_TREE] + for n, (key, value) in enumerate(kept): + env[f"GIT_CONFIG_KEY_{n}"], env[f"GIT_CONFIG_VALUE_{n}"] = key, value + if kept: + env["GIT_CONFIG_COUNT"] = str(len(kept)) + return env + + +def _probe_env(nothing: Path) -> dict: + """The environment `git check-attr` is asked in: `_inherited_env()`, + with no system config and the empty file `nothing` as the global + config. Each could steer the answer here and holds nothing on another + machine, as `attr.tree` from either did.""" + env = _inherited_env() + env.update(GIT_ATTR_NOSYSTEM="1", GIT_CONFIG_NOSYSTEM="1", + GIT_CONFIG_GLOBAL=str(nothing)) + return env + + +def checked_out_as(values: dict) -> str: + """What a checkout writes over an LF blob whose `crlf`, `text` and `eol` + are `values`, as `git check-attr` prints them: `KEPT`, the blob itself + on every platform; `CRLF_EVERYWHERE`; or `CRLF_BY_CONFIG`, CRLF wherever + `core.autocrlf` or `core.eol` asks for it, as Git for Windows' default + does. + + Git's `convert.c`, `convert_attrs`, read for a checkout: `text`, else + `crlf`, says whether the file is text, and one that is not is written as + its blob. Then `eol=lf` keeps LF and `eol=crlf` writes CRLF; with + neither, only `input` keeps LF without asking the configuration.""" + crlf, text, eol = (values.get(name, "") for name in LINE_ENDING_ATTRIBUTES) + action = _TEXT_ACTIONS.get(text) or _TEXT_ACTIONS.get(crlf) + if action == _NOT_TEXT or eol == "lf": + return KEPT + if eol == "crlf": + return CRLF_EVERYWHERE + return KEPT if action == _AS_CHECKED_IN else CRLF_BY_CONFIG + + +class _AttributesProbe: + """A throwaway repository to ask `git check-attr` in: each question's + `.gitattributes` files are written into a work tree of their own, which + holds nothing else. Made where `_ask_git_branch` makes its own, in the + system's temporary directory, and removed on leaving the `with`.""" + + def __init__(self) -> None: + self._scratch = Path(tempfile.mkdtemp( + prefix="openreposhape-attributes-")) + self._asked = 0 + #: An empty file, read in place of the person's own global config + #: and `core.attributesFile`. + self._nothing = self._scratch / "nothing" + self._env = _probe_env(self._nothing) + self._git_dir = self._scratch / "repository" / ".git" + + def __enter__(self) -> "_AttributesProbe": + try: + empty = self._scratch / "no-template" + empty.mkdir() + self._nothing.write_bytes(b"") + git_out(["init", "-q", f"--template={empty}", + str(self._git_dir.parent)], cwd=self._scratch, + env=self._env) + except BaseException: + self.__exit__() + raise + return self + + def __exit__(self, *_exc) -> None: + shutil.rmtree(self._scratch, ignore_errors=True) + + def ask(self, layout: dict, paths: list[str], + fold: bool) -> dict[str, dict[str, str]]: + """`{path: {attribute: value}}` for each of `paths`, with the + `.gitattributes` files `layout` holds (`{path: bytes}`) where git + reads them, and git ignoring case where `fold` says so.""" + self._asked += 1 + tree = self._scratch / f"tree-{self._asked}" + tree.mkdir() + for rel, data in layout.items(): + target = tree.joinpath(*rel.split("/")) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + raw = git_out([f"--git-dir={self._git_dir}", f"--work-tree={tree}", + "-c", f"core.ignorecase={str(fold).lower()}", + "-c", f"core.attributesFile={self._nothing}", + "check-attr", "-z", *LINE_ENDING_ATTRIBUTES, "--", + *paths], cwd=tree, binary=True, env=self._env) + fields = raw.split(b"\0") + said: dict[str, dict[str, str]] = {} + for at in range(0, len(fields) - 2, 3): + path, name, value = (field.decode("utf-8", "surrogateescape") + for field in fields[at:at + 3]) + said.setdefault(path, {})[name] = value + return said + + +def _kept_nested_attributes(plan: Plan, tree: list) -> list[tuple]: + """`(path, mode, blob, entry)` for each `.gitattributes` below the root + that the plan keeps in the root, in the tree's order, one so named only + once case is ignored among them, for the disk that ignores it. + + A SYMLINK IS ONE. Git follows none to read attributes, but a checkout + reads each `.gitattributes` from the index first, and there a symlink's + blob is its target, which git then reads as the file's rules: measured + with git 2.43, a kept `.github/.gitattributes` linking to `*.yml + eol=crlf` checked `.github/workflows/validate.yml` out with CRLF. A + target that is a path names no attribute, and changes nothing.""" + blobs = {path: oid for path, _, oid, _ in tree} + return [(path, mode, blobs[path], entry) + for path, mode, entry in _kept_in_root(plan.entries, tree) + if "/" in path and mode in (*FILE_MODES, SYMLINK_MODE) + and path.rsplit("/", 1)[1].casefold() == ATTRIBUTES] + + +def _pinned_landings(walks: list) -> list[tuple[str, str]]: + """`(the name the pin gives it, where the split puts it)` for each copy + the shape pins, in the pin's order, as `place` writes it: at its own + path, or under `shape/` where that is taken, followed through the + root's symlinks. A copy a walk cannot write, or one it writes outside + the assembly, is another finding's, and is left out.""" + by_path = {walk.path: walk for walk in walks} + landings = [] + for rel in PINNED_COPIES: + name, walk = rel, by_path.get(rel) + if walk is not None and walk.problem is None and walk.taken: + name = f"{COLLISION_DIR}/{rel}" + walk = by_path.get(name) + if walk is None or walk.problem is not None or walk.outside \ + or walk.taken: + continue + landings.append((name, walk.landed)) + return landings + + +def _root_attributes(source: Source, walks: list, disk: _KeptTree, + tree: list, names: list[str]) -> dict: + """The `.gitattributes` the split puts at the root, and under `shape/`, + as `{path: bytes}`. Where the root keeps none, the shape's own. Where it + keeps one, the shape's goes beside it and the split appends the block + for the copies `names` outside `shape/` (#471); a kept one that is not a + regular file gets no block, so it is asked as the block alone, the lines + #471's follow-up asks a regular file in its place to hold.""" + shapes = (ASSEMBLY_TEMPLATE / ATTRIBUTES).read_text( + encoding="utf-8").encode("utf-8") + walk = next((walk for walk in walks if walk.path == ATTRIBUTES), None) + if walk is None or not walk.taken: + return {ATTRIBUTES: shapes} + row = disk.at(ATTRIBUTES) + blobs = {path: oid for path, _, oid, _ in tree} + own = (_blob_bytes(source.path, blobs[row[0]]) + if row and row[1] in FILE_MODES else b"") + return {ATTRIBUTES: with_eol_block(own, eol_block_paths(names, + COLLISION_DIR)), + f"{COLLISION_DIR}/{ATTRIBUTES}": shapes} + + +def _placed_attributes(nested: list[tuple], landed: list[str], + fold: bool) -> dict: + """`{where git reads it: row}` for each row of `nested` in a directory + above where a pinned copy lands, spelled as that place spells it; on a + disk that ignores case, the first of two twins.""" + def key(path: str) -> str: + return path.casefold() if fold else path + + above: dict[str, str] = {} + for path in landed: + parts = path.split("/") + for end in range(1, len(parts)): + here = "/".join(parts[:end]) + above.setdefault(key(here), here) + placed: dict[str, tuple] = {} + for row in nested: + directory, _, name = row[0].rpartition("/") + spelled = above.get(key(directory)) + if spelled is not None and key(name) == ATTRIBUTES: + placed.setdefault(f"{spelled}/{ATTRIBUTES}", row) + return placed + + +def _copy_line(rel: str) -> str: + """The line a kept file's own directory spells for the copy at `rel` + below it, between backticks; for a name no line can spell literally, + which a symlink the root keeps can lead a copy to, what to write.""" + if EOL_PATH_RE.fullmatch(rel): + return f"`{eol_line(rel)}`" + return f"a `text eol=lf` line that names {_spelled(rel)}" + + +def _nested_attributes_finding(row: tuple, at: str, copies: list[str], + written: list[str], aside: str) -> str: + """`check`'s line for the kept file `row`, which git reads at `at`, and + the outranked `copies` it reaches, each written as `written` says.""" + path, mode, _, entry = row + directory = at[:-len(ATTRIBUTES) - 1] + many = len(copies) > 1 + under = "" if entry == path else f" under the entry {_spelled(entry)}" + where = (("on such a disk, whatever git is set to" if aside else + "on every platform, Linux and macOS included") + if CRLF_EVERYWHERE in written else + "wherever git writes them, as Git for Windows does by default " + "(`core.autocrlf=true`)") + lines = ", ".join(_copy_line(copy[len(directory) + 1:]) + for copy in copies) + end = (f"replace the symlink {_spelled(path)} in the source with a " + "regular file that ends" if mode == SYMLINK_MODE else + f"end {_spelled(path)} in the source") + return ( + f"FINDING {NESTED_ATTRIBUTES}: the plan keeps {_spelled(path)} in " + f"the assembly root{under}, and git reads it after the root's own " + f"`{ATTRIBUTES}` for each path under {_spelled(directory + '/')}, so " + "its rules outrank the root's line-ending rule for the pinned " + f"{'copies' if many else 'copy'} " + f"{_first_and_more(copies, _spelled)}{aside}: a clone would check " + f"{'them' if many else 'it'} out with CRLF line endings {where}, and " + "the root's first `make bootstrap` would report shape-copy-drift on " + f"{'each' if many else 'it'}, after `execute` had said `adoption " + f"verified`. Send {_spelled(entry)} to a leg or `drop` it, or {end} " + f"with {'these lines, one per copy:' if many else 'the line'} " + f"{lines}, commit, and re-run `plan`.") + + +def _outranking(asked: _AttributesProbe, fold: bool, landed: list[str], + root: dict, placed: dict) -> list[tuple[str, str]]: + """`(path, finding)` for each kept nested file that outranks the root's + rule for a copy landed at one of `landed`, asked of git on the disk + `fold` says. `root` is `_root_attributes`', and `placed` is + `{where git reads it: (row, its bytes)}`. + + A file is named for a copy when, added to the root's files alone, it + makes a checkout write that copy with CRLF. Where no file alone does, + and two or more do it only together, each that changes what git says + of the copy is named.""" + held = {at: data for at, (_, data) in placed.items()} + before = asked.ask(root, landed, fold) + after = asked.ask({**root, **held}, landed, fold) + outranked = [path for path in landed + if checked_out_as(before[path]) == KEPT + and checked_out_as(after[path]) != KEPT] + alone = {} + for at, data in held.items(): + under = [path for path in outranked + if path.startswith(at[:-len(ATTRIBUTES)])] + alone[at] = asked.ask({**root, at: data}, under, fold) if under else {} + breaks = {(at, path) for at, said in alone.items() for path in said + if checked_out_as(said[path]) != KEPT} + broken = {path for _, path in breaks} + found = [] + for at, (row, _) in placed.items(): + copies = [path for path in alone[at] if (at, path) in breaks + or (path not in broken + and alone[at][path] != before[path])] + if copies: + found.append((row[0], _nested_attributes_finding( + row, at, copies, [checked_out_as(after[path]) + for path in copies], + _CASE_ASIDE if fold else ""))) + return found + + +def _nested_attributes_findings(plan: Plan, source: Source, + tree: list) -> list[str]: + """A FINDING for each `.gitattributes` the plan keeps below the root + whose rules outrank the root's line-ending rule for a pinned copy (#530), + in the tree's order. A source that keeps none above a copy asks git + nothing, and adopts as it did.""" + nested = _kept_nested_attributes(plan, tree) + if not nested: + return [] + kept = _kept_in_root(plan.entries, tree) + probe = _SymlinkProbe(source.work_root) + questions = [] + for fold in (False, True): + walks = _PlannedRoot(plan, source, tree, probe=probe, + fold=fold).walks() + landings = _pinned_landings(walks) + landed = [where for _, where in landings] + placed = _placed_attributes(nested, landed, fold) + if not placed: + continue + root = _root_attributes(source, walks, _KeptTree(kept, fold), tree, + [name for name, _ in landings]) + # The shape's own `shape/.gitattributes` is written where the source + # keeps one only if nothing is there: `plan-root-blocks-shape-copy` + # names what is (#262), and it is not named again here. + placed = {at: (row, _blob_bytes(source.path, row[2])) + for at, row in placed.items() if at not in root} + if placed: + questions.append((fold, landed, root, placed)) + if not questions: + return [] + found: dict[str, str] = {} + with _AttributesProbe() as asked: + for fold, landed, root, placed in questions: + for path, finding in _outranking(asked, fold, landed, root, + placed): + found.setdefault(path, finding) + return [found[row[0]] for row in nested if row[0] in found] + + +# --------------------------------------------------------------------------- +# The root's own `.gitattributes`, as `git add` stores it (#530, D5(a)) +# --------------------------------------------------------------------------- +# +# THE ONE FILE OF THE SOURCE'S THE SPLIT WRITES INTO (#471) is staged again +# by the split's `git add -A`, and git stores it through its rules for it. A +# blob the source committed with CRLF under `* text eol=lf`, `* text +# eol=crlf` or `* eol=lf` was stored as LF, so the root held neither the +# source's blob nor its bytes followed by the block, and the verification +# ended `adopt-lost` after both legs existed (measured at 2423eaa2: lane 2's +# D5(a) line on #530; `* ident` over a `$Id: deadbeef $` did the same). +# `text=auto` leaves alone a file whose indexed blob already holds CRLF, so +# a CRLF blob under it adopted, as an LF blob did under every rule. +# +# GIT IS ASKED, AS THE SPLIT ASKS IT (`_stored_as_written`). A throwaway +# repository's index holds the source's blob at `.gitattributes`, as the +# assembly's checkout does, so the check-in rules that read the index answer +# as they do there; its work tree holds what the verification expects at the +# root (`_land_the_eol_block`), the source's bytes followed by the block +# (`_root_attributes`); and `git add` there stores those bytes or not. The +# materializer appends the block to the checkout's bytes, and git's check-in +# undoes what its checkout did to line endings, an `ident` or an encoding, +# so the blob's own bytes get the same answer. UNLIKE THE NESTED QUESTION, +# THIS MACHINE'S SETTINGS ARE READ: the split's `git add` runs here, with +# this machine's config and attributes (`core.autocrlf`, `core.safecrlf`, a +# `filter` driver, `core.attributesFile`), so only `_inherited_env` is taken +# from them: no other repository, and no tree to read attributes from +# (`GIT_ATTR_SOURCE`, `attr.tree`), of which the throwaway repository has +# none. What `git add` refuses there -- `core.safecrlf=true` over such a +# CRLF, or a `working-tree-encoding` the bytes are not in -- would stop the +# split committing after both legs exist, and is found too, in git's words. +# +# A FINDING, as the nested one is: the fix is the source's, a commit of the +# file as git stores it, and `execute` refuses it before any leg exists. A +# root that keeps no `.gitattributes` of the source's, or keeps a symlink +# there, which the split writes nothing into, asks git nothing. Another kept +# file whose blob `git add` would re-encode fails the same way when git reads +# it again; asking about each one needs a checkout of the whole source, and +# is not done here. + +#: `check`'s finding, and the code of `execute`'s refusal, for the source's +#: root `.gitattributes` that `git add` would not store as the split writes +#: it (#530, D5(a)). +REENCODED_ATTRIBUTES = "plan-attributes-reencoded" + + +def _stored_as_written(blob: bytes, mode: str, written: bytes) -> str | None: + """None when this machine's `git add` stores `written` at the root's + `.gitattributes`, over an index that holds `blob` there with `mode`, as + exactly those bytes; else what it did: git's last line where it + refused, or "" where it stored other bytes. Asked in a throwaway + repository in the system's temporary directory, removed afterwards.""" + scratch = Path(tempfile.mkdtemp(prefix="openreposhape-attributes-")) + env = _inherited_env() + # No fsmonitor daemon is started for a repository that is about to go. + git = ["-c", "core.fsmonitor=false"] + repository = scratch / "repository" + + def raw_oid(data: bytes, name: str, *write: str) -> str: + """The oid of `data` as they are, no rule applied; `-w` stores it.""" + (scratch / name).write_bytes(data) + return git_out([*git, "hash-object", *write, "--no-filters", "--", + str(scratch / name)], cwd=repository, env=env) + + try: + empty = scratch / "no-template" + empty.mkdir() + git_out(["init", "-q", f"--template={empty}", str(repository)], + cwd=scratch, env=env) + git_out([*git, "update-index", "--add", "--cacheinfo", + f"{mode},{raw_oid(blob, 'held', '-w')},{ATTRIBUTES}"], + cwd=repository, env=env) + (repository / ATTRIBUTES).write_bytes(written) + added = subprocess.run(["git", *git, "add", "--", ATTRIBUTES], + cwd=str(repository), env=env, + capture_output=True, check=False) + if added.returncode: + said = added.stderr.decode(errors="replace").strip() + return said.splitlines()[-1] if said else \ + f"exit {added.returncode}" + staged = git_out([*git, "ls-files", "-s", "--", ATTRIBUTES], + cwd=repository, env=env).split()[1] + return None if staged == raw_oid(written, "wanted") else "" + finally: + shutil.rmtree(scratch, ignore_errors=True) + + +def _reencoded_attributes_finding(said: str) -> str: + """`check`'s line for the root's `.gitattributes`, which this machine's + `git add` stores as other bytes (`said` empty) or refuses (`said`, in + git's words).""" + if said: + what = (f"refuses what the split writes there (git said " + f"{_spelled(said)}), so the split's own `git add` would fail, " + "after both legs exist.") + else: + what = ("would store other bytes there than the source's followed by " + "the block, as git does where its rules for the file make it " + "text and the blob holds CRLF line endings, or where an " + "`ident`, `working-tree-encoding` or `filter` rule reaches " + f"it. The verification would then report the source's " + f"`{ATTRIBUTES}` as `adopt-lost`, in no leg and not in the " + "root tree, after both legs exist.") + return (f"FINDING {REENCODED_ATTRIBUTES}: the plan keeps the source's " + f"`{ATTRIBUTES}` in the assembly root, where the split appends " + f"#471's block to it, and this machine's `git add` {what} Commit " + "the file in the source as git stores it (`git add --renormalize " + f"-- {ATTRIBUTES}`, then commit), or change the rule git applies " + "to it, and re-run `plan`.") + + +def _reencoded_attributes_findings(plan: Plan, source: Source, + tree: list) -> list[str]: + """A FINDING where the plan keeps the source's own `.gitattributes` in + the root, which the split appends the block to, and this machine's `git + add` would not store what the split writes there as those bytes (#530, + D5(a)). A root that keeps none, or keeps a symlink there, asks git + nothing.""" + disk = _KeptTree(_kept_in_root(plan.entries, tree), False) + row = disk.at(ATTRIBUTES) + if row is None or row[1] not in FILE_MODES: + return [] + walks = _PlannedRoot(plan, source, tree, + probe=_SymlinkProbe(source.work_root)).walks() + written = _root_attributes(source, walks, disk, tree, + [name for name, _ in _pinned_landings(walks)]) + if f"{COLLISION_DIR}/{ATTRIBUTES}" not in written: + return [] + blobs = {path: oid for path, _, oid, _ in tree} + said = _stored_as_written(_blob_bytes(source.path, blobs[row[0]]), + row[1], written[ATTRIBUTES]) + return [] if said is None else [_reencoded_attributes_finding(said)] + + def cmd_check(args) -> int: plan = Plan.load(Path(args.plan)) # FIRST, before anything is opened: the keys the rest is read from (#386). diff --git a/tests/test_adopt_nested_attributes.py b/tests/test_adopt_nested_attributes.py new file mode 100644 index 00000000..629b209c --- /dev/null +++ b/tests/test_adopt_nested_attributes.py @@ -0,0 +1,740 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A `.gitattributes` the plan keeps BELOW the root, over a pinned copy (#530). + +#471 (PR #511) gives every pinned copy at the assembly root one line-ending +rule: a `/ text eol=lf` line in the block the split appends to the +source's own root `.gitattributes`, or the shape's own `* text=auto eol=lf` +where the source keeps none. Git reads attributes for a path from the +`.gitattributes` of its own directory and of each directory above it, and a +deeper file outranks a shallower one. So a source's `.github/.gitattributes` +or `scripts/.gitattributes` that the plan keeps in the root outranks that +rule for every copy under it. + +Measured on main at 2247dbff with git 2.43: `*.yml eol=crlf` there, `* +text=auto eol=crlf`, or a macro the root defines to say the same, checked +`.github/workflows/validate.yml` out with CRLF in EVERY clone, Linux's +included; `* !eol` did so in every clone that writes CRLF. Each time +`execute` printed `adoption verified` and exited 0, and the root's first +`make bootstrap` reported shape-copy-drift. `* -text` and `* binary` changed +no byte: a checkout writes a file that is not text as its blob, and the blob +is LF. + +`check` now finds it as `plan-nested-attributes`, and `execute` refuses it +before any leg exists. The finding names the kept file, the copies it reaches +and the lines that keep them LF. The question is asked of git itself +(`git check-attr`), so the tests that need no `git filter-repo` run on every +CI job; the end-to-end ones adopt the remedied source, clone it the ways +Windows clones it, and read `git check-attr` on every pinned copy. + +The root's OWN `.gitattributes` is the one file of the source's the split +writes into (#471), and `git add` stores what it writes there through git's +rules for that file. A blob committed CRLF under `* text eol=lf` was stored as +LF, and `execute` ended `adopt-lost` after both legs existed (lane 2's D5(a) +line on this issue). `check` now asks this machine's `git add` and finds it as +`plan-attributes-reencoded`, and `execute` refuses it before any leg exists. +""" + +from __future__ import annotations + +import importlib.util +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import (ADOPT, REPO, clone_just_pushed, findings, git, + make_source_repo, planned, refused_before_any_leg, + run_script) + +sys.path.insert(0, str(REPO / "scripts")) +from repo_shape import parse_yaml # noqa: E402 +from shape_materialize import CASE_ASIDE # noqa: E402 + +#: The finding, and the code of `execute`'s refusal. +NESTED = "plan-nested-attributes" +ROOT_ATTRIBUTES = ".gitattributes" +GITHUB_ATTRIBUTES = ".github/.gitattributes" +WORKFLOWS_ATTRIBUTES = ".github/workflows/.gitattributes" +VALIDATE = ".github/workflows/validate.yml" +#: The line that keeps the workflow copy LF, written in `.github/`. +VALIDATE_LINE = "/workflows/validate.yml text eol=lf" +TEXT_AUTO = "* text=auto\n" +YML_CRLF = "*.yml eol=crlf\n" +ALL_CRLF = "* eol=crlf\n" +MINUS_TEXT = "* -text\n" +TEXT_SET = "* text\n" +#: The finding's own remedy for `.github/.gitattributes`, applied. +REMEDIED = f"{YML_CRLF}{VALIDATE_LINE}\n" +ROOT = "root" +UTF8 = "utf-8" +#: The tree git knows with no object stored: one with nothing in it. +EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +#: How the finding says where the copies go CRLF. +EVERYWHERE = "with CRLF line endings on every platform" +BY_CONFIG = "with CRLF line endings wherever git writes them" +#: The shape's scripts, in the pin's order, as `scripts/` holds them. +SCRIPTS = ("validate-pins.py", "validate-manifest.py", "bootstrap.py", + "repo_shape.py", "validate-repository-naming.py") +BRANCH = "adopt/three-repo-shape" +PIN = "contracts/shape-pin.yaml" +#: Where a root `.gitattributes` that is a symlink leads. +LINKED_RULES = "config/attributes" + +#: A source with nothing for the plan to ask about, and a workflow of its own +#: under `.github/workflows/`, which the policy sends to the code leg. +SOURCE = {"README.md": "# Thing\n", ".gitignore": "__pycache__/\n", + "specs/001-feature/spec.md": "# The feature\n", + "src/app/main.py": "print()\n", + ".github/workflows/ci.yml": "on: [push]\n"} +#: `plan` asks about `.github/.gitattributes`; these answers keep it. +KEEP_GITHUB = {GITHUB_ATTRIBUTES: ROOT} +KEEP_BOTH = {GITHUB_ATTRIBUTES: ROOT, ".github/workflows/": ROOT} + + +def github(rules: str, root: str | None = TEXT_AUTO) -> dict: + """A source's `.github/.gitattributes` holding `rules`, beside its own + root `.gitattributes` holding `root`, or none.""" + files = {GITHUB_ATTRIBUTES: rules} + if root is not None: + files[ROOT_ATTRIBUTES] = root + return files + + +def adoption_plan(base: Path, files: dict, answers: dict) -> Path: + """The plan for `SOURCE` and `files`, with `answers` given.""" + source = make_source_repo(base / "Thing", tree={**SOURCE, **files}, + edits=()) + return planned(source, base, answers) + + +def lines_of(names) -> list[str]: + return [f"`/{name} text eol=lf`" for name in names] + + +@pytest.fixture(scope="module") +def adopter(): + spec = importlib.util.spec_from_file_location("adopt_nested_attributes", + ADOPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- +# What a checkout does with a copy's LF blob, as git's `convert.c` decides it +# --------------------------------------------------------------------------- + +#: `crlf text eol => what a checkout writes`, each value as `git check-attr` +#: prints it: the blob, CRLF everywhere, or what `core.autocrlf` and +#: `core.eol` say. `crlf` is the old spelling of `text`, read only where +#: `text` is unspecified. Each row was checked against real checkouts with +#: git 2.43: as configured, with `core.autocrlf=true` and with +#: `core.eol=crlf` (#530's review found three rows backwards). +CHECKOUTS = [ + "unspecified set lf => kept", + "unspecified auto lf => kept", + "unspecified unspecified lf => kept", + "unspecified unset crlf => kept", + "unset set crlf => everywhere", + "unset unspecified crlf => kept", + "input unspecified unspecified => kept", + "unspecified input unset => kept", + "input set crlf => everywhere", + "unspecified auto crlf => everywhere", + "set unset crlf => kept", + "unspecified set unspecified => by config", + "unspecified auto unset => by config", + "unspecified unspecified unspecified => by config", + "set unset native => kept", + "unset set unspecified => by config", +] + + +@pytest.mark.parametrize("row", CHECKOUTS) +def test_what_a_checkout_writes_is_read_as_git_reads_it(adopter, row): + values, written = row.split(" => ") + crlf, text, eol = values.split() + assert adopter.checked_out_as( + {"crlf": crlf, "text": text, "eol": eol}) == written + + +# --------------------------------------------------------------------------- +# `check` finds it, and `execute` refuses it before any leg exists +# --------------------------------------------------------------------------- + +#: Each way a kept file outranks the root's rule: what the source adds, the +#: plan's answers, how the finding names the copies, the lines it gives, and +#: where it says the copies go CRLF. +OUTRANKED = [ + pytest.param(github(YML_CRLF), KEEP_GITHUB, f"`{VALIDATE}`", + [f"`{VALIDATE_LINE}`"], EVERYWHERE, id="eol-crlf"), + pytest.param(github("* text=auto eol=crlf\n"), KEEP_GITHUB, + f"`{VALIDATE}`", [f"`{VALIDATE_LINE}`"], EVERYWHERE, + id="text-auto-eol-crlf"), + pytest.param(github("*.yml winfile\n", + "[attr]winfile text eol=crlf\n" + TEXT_AUTO), + KEEP_GITHUB, f"`{VALIDATE}`", [f"`{VALIDATE_LINE}`"], + EVERYWHERE, id="a-macro-the-root-defines"), + pytest.param(github("* !eol\n"), KEEP_GITHUB, f"`{VALIDATE}`", + [f"`{VALIDATE_LINE}`"], BY_CONFIG, id="eol-unspecified"), + pytest.param(github(YML_CRLF, None), KEEP_GITHUB, f"`{VALIDATE}`", + [f"`{VALIDATE_LINE}`"], EVERYWHERE, + id="no-root-attributes"), + pytest.param(github("*.yml -crlf eol=crlf\n"), KEEP_GITHUB, + f"`{VALIDATE}`", [f"`{VALIDATE_LINE}`"], EVERYWHERE, + id="crlf-unset-beside-the-roots-text"), + pytest.param({ROOT_ATTRIBUTES: TEXT_AUTO, + "scripts/.gitattributes": "*.py eol=crlf\n", + "scripts/run.sh": "echo hi\n"}, {"scripts/": ROOT}, + "`scripts/validate-pins.py` and 4 more", lines_of(SCRIPTS), + EVERYWHERE, id="a-kept-directory"), +] + + +@pytest.mark.parametrize(("files", "answers", "copies", "lines", "where"), + OUTRANKED) +def test_check_names_the_kept_file_the_copies_and_the_lines( + tmp_path, files, answers, copies, lines, where): + found = findings(adoption_plan(tmp_path, files, answers)) + assert [code for code, _ in found] == [NESTED], found + detail = found[0][1] + kept = next(path for path in files if path.endswith(f"/{ROOT_ATTRIBUTES}")) + assert detail.startswith( + f"the plan keeps `{kept}` in the assembly root"), detail + assert f"the pinned cop{'ies' if len(lines) > 1 else 'y'} {copies}:" \ + in detail + assert where in detail + assert "shape-copy-drift" in detail + for line in lines: + assert line in detail, line + + +def test_execute_refuses_it_before_any_leg_exists(adopter, monkeypatch, + tmp_path): + plan = adoption_plan(tmp_path, github(YML_CRLF), KEEP_GITHUB) + refusal = refused_before_any_leg(adopter, monkeypatch, plan, tmp_path) + assert refusal.code == NESTED + + +def index_link(source: Path, link: str, target: str) -> None: + """Commit `link -> target` in `source`'s tree only, as Git for Windows' + `core.symlinks=false` leaves it: no symlink on the disk, so no privilege + is needed, and `plan` reads the tree.""" + blob = subprocess.run(["git", "hash-object", "-w", "--stdin"], + cwd=str(source), input=target.encode(UTF8), + capture_output=True, check=True) + git("update-index", "--add", "--cacheinfo", + f"120000,{blob.stdout.decode('ascii').strip()},{link}", cwd=source) + git("commit", "-qm", f"Link {link}", cwd=source) + + +def test_a_root_attributes_symlink_still_has_its_copies_outranked(tmp_path): + """The root's `.gitattributes` is a symlink, which the split appends + nothing to (#471's follow-up says so and lists the lines a regular file + needs). The kept `.github/.gitattributes` is asked against the rule that + follow-up asks for, so its own `eol=crlf`, which writes CRLF on every + platform, is still found.""" + source = make_source_repo( + tmp_path / "Thing", tree={**SOURCE, **github(YML_CRLF, None), + LINKED_RULES: TEXT_AUTO}, edits=()) + index_link(source, ROOT_ATTRIBUTES, LINKED_RULES) + found = findings(planned(source, tmp_path, + {**KEEP_GITHUB, "config/": ROOT})) + assert [code for code, _ in found] == [NESTED], found + assert f"`{VALIDATE_LINE}`" in found[0][1] + + +def test_a_kept_symlink_is_read_as_its_target_text(tmp_path): + """A checkout reads a `.gitattributes` from the index first, and there a + symlink's blob is its target: git 2.43 checked the copy out with CRLF + under a kept `.github/.gitattributes` linking to `*.yml eol=crlf`. The + remedy says to replace the link, which nothing can end.""" + source = make_source_repo(tmp_path / "Thing", tree=dict(SOURCE), + edits=()) + index_link(source, GITHUB_ATTRIBUTES, YML_CRLF.strip()) + found = findings(planned(source, tmp_path, KEEP_GITHUB)) + assert [code for code, _ in found] == [NESTED], found + assert (f"or replace the symlink `{GITHUB_ATTRIBUTES}` in the source with " + f"a regular file that ends with the line `{VALIDATE_LINE}`, " + "commit") in found[0][1] + + +@pytest.mark.parametrize(("root", "named"), [ + pytest.param(b"* text=auto eol=lf\n", 1, id="the-root-keeps-it-lf"), + pytest.param(TEXT_AUTO.encode(UTF8), 0, id="the-root-leaves-it-crlf")]) +def test_only_a_copy_the_roots_own_files_keep_lf_is_outranked(adopter, root, + named): + """One finding per cause: a copy the root's own files already leave to + the configuration -- a symlink the root keeps leads it where #471's + block, which names it by its pinned path, does not reach -- is not the + kept nested file's to answer for, whatever that file says.""" + row = (GITHUB_ATTRIBUTES, "100644", "0" * 40, GITHUB_ATTRIBUTES) + with adopter._AttributesProbe() as asked: + found = adopter._outranking( + asked, False, [VALIDATE], {ROOT_ATTRIBUTES: root}, + {GITHUB_ATTRIBUTES: (row, YML_CRLF.encode(UTF8))}) + assert len(found) == named, found + + +def test_a_kept_shape_attributes_file_is_the_copys_finding_alone(tmp_path): + """The source keeps its own `.gitattributes` and `shape/.gitattributes` + in the root, where the shape writes its copy of the first: + `plan-root-blocks-shape-copy` names that (#262), and the kept file is + not asked about the copies under `shape/` as well.""" + files = {ROOT_ATTRIBUTES: TEXT_AUTO, "shape/.gitattributes": YML_CRLF} + found = findings(adoption_plan(tmp_path, files, {"shape/": ROOT})) + assert [code for code, _ in found] == ["plan-root-blocks-shape-copy"], \ + found + + +#: Kept files that leave every copy as its pin records it: not text, text +#: with the root's `eol`, another path, the finding's own remedy, a file a +#: leg takes, and a rule a deeper kept file restores. +KEPT_LF = [ + pytest.param(github(MINUS_TEXT), KEEP_GITHUB, id="minus-text"), + pytest.param(github("* binary\n"), KEEP_GITHUB, id="binary"), + pytest.param(github(TEXT_SET), KEEP_GITHUB, id="text-alone"), + pytest.param(github("* -crlf\n* crlf=input\n"), KEEP_GITHUB, + id="the-old-crlf-spelling"), + pytest.param(github("*.png binary\n"), KEEP_GITHUB, id="another-path"), + pytest.param(github("*.yml crlf -text eol=crlf\n"), KEEP_GITHUB, + id="crlf-set-under-minus-text"), + pytest.param(github(REMEDIED), KEEP_GITHUB, id="the-remedy"), + pytest.param(github(YML_CRLF), {GITHUB_ATTRIBUTES: "code"}, + id="sent-to-a-leg"), + pytest.param({**github(ALL_CRLF), + WORKFLOWS_ATTRIBUTES: "*.yml eol=lf\n"}, KEEP_BOTH, + id="restored-below-it"), +] + + +@pytest.mark.parametrize(("files", "answers"), KEPT_LF) +def test_a_kept_file_that_leaves_each_copy_lf_is_no_finding(tmp_path, files, + answers): + assert findings(adoption_plan(tmp_path, files, answers)) == [] + + +def test_only_a_file_that_changes_a_copy_is_named(tmp_path): + """`.github/workflows/.gitattributes` says `text`, which the root's rule + says already: the copy's CRLF is `.github/.gitattributes`' alone.""" + files = {**github(ALL_CRLF), WORKFLOWS_ATTRIBUTES: TEXT_SET} + found = findings(adoption_plan(tmp_path, files, KEEP_BOTH)) + assert [code for code, _ in found] == [NESTED], found + assert found[0][1].startswith(f"the plan keeps `{GITHUB_ATTRIBUTES}`") + assert WORKFLOWS_ATTRIBUTES not in found[0][1] + + +def test_a_file_that_alone_keeps_the_copy_lf_is_not_named(tmp_path): + """`.github/.gitattributes` says `* text=auto`, which changes what git + says of the copy but, alone over the root's rule, leaves it LF: the + copy's CRLF is the deeper `* eol=crlf`'s, which alone is named, and + whose line alone keeps the copy LF (#530's review, F3).""" + files = {**github(TEXT_AUTO), WORKFLOWS_ATTRIBUTES: ALL_CRLF} + found = findings(adoption_plan(tmp_path, files, KEEP_BOTH)) + assert [detail.split("`")[1] for _, detail in found] == [ + WORKFLOWS_ATTRIBUTES], found + + +def test_a_cause_two_files_make_only_together_is_still_named(tmp_path): + """`* -text eol=crlf` alone leaves the copy as its blob, and `* text` + alone says what the root says; together they write it CRLF. No file + alone breaks the copy, so the one that changes what git says of it is + named, and its line keeps the copy LF.""" + files = {**github("* -text eol=crlf\n"), WORKFLOWS_ATTRIBUTES: TEXT_SET} + found = findings(adoption_plan(tmp_path, files, KEEP_BOTH)) + assert [detail.split("`")[1] for _, detail in found] == [ + GITHUB_ATTRIBUTES], found + + +def test_each_file_that_changes_a_copy_has_a_finding_with_its_own_lines( + tmp_path): + """Both kept files change the copy's `eol`, so each is named once, with + the line its own directory spells: the copy is LF only once both end + with one.""" + files = {**github(ALL_CRLF), WORKFLOWS_ATTRIBUTES: "*.yml !eol\n"} + found = findings(adoption_plan(tmp_path, files, KEEP_BOTH)) + assert [detail.split("`")[1] for _, detail in found] == [ + GITHUB_ATTRIBUTES, WORKFLOWS_ATTRIBUTES], found + assert f"`{VALIDATE_LINE}`" in found[0][1] + assert "`/validate.yml text eol=lf`" in found[1][1] + + +def test_a_pattern_that_matches_only_once_case_is_ignored_says_so(tmp_path): + """`*.YML` reaches `validate.yml` only where git ignores case, as it + does on a macOS or Windows disk (`core.ignorecase`).""" + found = findings(adoption_plan(tmp_path, github("*.YML eol=crlf\n"), + KEEP_GITHUB)) + assert [code for code, _ in found] == [NESTED], found + assert f"`{VALIDATE}`{CASE_ASIDE}:" in found[0][1] + assert "line endings on such a disk, whatever git is set to," \ + in found[0][1] + assert "Linux and macOS included" not in found[0][1] + + +def command_scope(key: str, value: str) -> dict: + """`key=value` in git's command scope (`GIT_CONFIG_COUNT`), appended to + what the environment holds: the Windows job sets `core.autocrlf` there, + and the last entry of a key is the one git reads.""" + held = int(os.environ.get("GIT_CONFIG_COUNT") or 0) + return {"GIT_CONFIG_COUNT": str(held + 1), + f"GIT_CONFIG_KEY_{held}": key, f"GIT_CONFIG_VALUE_{held}": value} + + +def checked(plan: Path, env: dict): + """`check` of `plan`, run with `env` laid over this process's own.""" + return run_script(ADOPT, "check", "--plan", str(plan), env=env) + + +def checked_with(tmp_path: Path, env: dict, rules: str = YML_CRLF): + """`check` of the plan keeping `.github/.gitattributes` holding + `rules`, run with `env` laid over this process's own.""" + return checked(adoption_plan(tmp_path, github(rules), KEEP_GITHUB), env) + + +def test_this_machines_own_attributes_are_not_read(tmp_path): + """A clone on another machine reads none of this one's attributes. A + macro this machine's global attributes file defines, `[attr]keepbytes + -text`, would make the kept `*.yml keepbytes eol=crlf` say `text: + unset`, which `check` would then call kept, so that file is not read: + `core.attributesFile` is set by this run's git's command scope + (`GIT_CONFIG_COUNT`, appended to what the environment holds), and the + finding still stands.""" + rules = tmp_path / "global-attributes" + rules.write_bytes(b"[attr]keepbytes -text\n") + checked = checked_with( + tmp_path, command_scope("core.attributesFile", str(rules)), + "*.yml keepbytes eol=crlf\n") + assert f"FINDING {NESTED}: " in checked.stderr, checked.stderr + + +def global_attr_tree(tmp_path: Path) -> dict: + """A global config naming the empty tree as `attr.tree`.""" + config = tmp_path / "global-config" + config.write_bytes(f"[attr]\n\ttree = {EMPTY_TREE}\n".encode(UTF8)) + return {"GIT_CONFIG_GLOBAL": str(config)} + + +@pytest.mark.parametrize("where", [ + pytest.param(lambda _: {"GIT_ATTR_SOURCE": "HEAD"}, + id="git-attr-source-head"), + pytest.param(lambda _: {"GIT_ATTR_SOURCE": EMPTY_TREE}, + id="git-attr-source-the-empty-tree"), + pytest.param(global_attr_tree, id="attr-tree-in-the-global-config"), + pytest.param(lambda _: command_scope("attr.tree", EMPTY_TREE), + id="attr-tree-in-the-command-scope")]) +def test_no_tree_this_machine_names_steers_the_question(tmp_path, where): + """`GIT_ATTR_SOURCE` and `attr.tree` make git read attributes from a + tree instead of the work tree the question is written into (#530's review, + checklist item 2). With git 2.43, `GIT_ATTR_SOURCE=HEAD` failed there, + an empty repository, so `check` printed `REFUSED git-failed`; and the + empty tree, named by either, made git read no rule at all, which + silenced the finding. None of them is read, and the finding stands.""" + checked = checked_with(tmp_path, where(tmp_path)) + assert "REFUSED" not in checked.stderr, checked.stderr + assert f"FINDING {NESTED}: " in checked.stderr, checked.stderr + + +@pytest.mark.parametrize(("rel", "said"), [ + pytest.param("workflows/validate.yml", f"`{VALIDATE_LINE}`", id="plain"), + pytest.param("ci files/validate.yml", + "a `text eol=lf` line that names `ci files/validate.yml`", + id="a-name-a-line-cannot-spell")]) +def test_a_copys_line_is_spelled_only_where_git_reads_it_literally( + adopter, rel, said): + """A symlink the root keeps can lead a copy to a name with a space, + which a `.gitattributes` line would read as a pattern and an attribute: + the finding names the copy instead of a line that says something else.""" + assert adopter._copy_line(rel) == said + + +class _NoProbe: + def __init__(self, *_args, **_kwargs) -> None: + raise AssertionError("git was asked about attributes") + + +@pytest.mark.parametrize(("files", "answers"), [ + pytest.param({}, {}, id="no-nested-file"), + pytest.param({"notes/.gitattributes": YML_CRLF, "notes/a.md": "# A\n"}, + {"notes/.gitattributes": ROOT}, id="above-no-copy"), +]) +def test_a_source_with_no_kept_file_above_a_copy_asks_git_nothing( + adopter, monkeypatch, tmp_path, files, answers): + """Every source with no nested `.gitattributes` above a pinned copy + adopts as it did on main: nothing is asked, and nothing is found.""" + plan = adopter.Plan.load(adoption_plan(tmp_path, files, answers)) + (tmp_path / "work").mkdir() + source = plan.open_source(None, tmp_path / "work") + monkeypatch.setattr(adopter, "_AttributesProbe", _NoProbe) + assert adopter._nested_attributes_findings(plan, source, + source.tree()) == [] + + +# --------------------------------------------------------------------------- +# The root's own `.gitattributes`, which `git add` would re-encode (#530, D5) +# --------------------------------------------------------------------------- + +#: The finding, and the code of `execute`'s refusal. +REENCODED = "plan-attributes-reencoded" +#: The start of every such finding. +KEPT_AND_APPENDED = ("the plan keeps the source's `.gitattributes` in the " + "assembly root, where the split appends #471's block to " + "it, and this machine's `git add` ") +STORES_OTHER_BYTES = "would store other bytes there" +CRLF_UNDER_TEXT = b"* text eol=lf\r\n" + + +def committed_raw(source: Path, name: str, data: bytes) -> None: + """Commit `data` at `name` in `source` byte for byte, whatever the + attributes say, as a file committed before its rules were, or by a + machine that wrote it CRLF, is held: `hash-object --no-filters`, then + `update-index`.""" + target = source / name + target.write_bytes(data) + blob = git("hash-object", "-w", "--no-filters", "--", name, cwd=source) + git("update-index", "--add", "--cacheinfo", + f"100644,{blob.stdout.strip()},{name}", cwd=source) + git("commit", "-qm", f"Commit {name} as it is", cwd=source) + + +def root_attributes_plan(base: Path, data: bytes) -> Path: + """The plan for `SOURCE` with a root `.gitattributes` committed as + `data`, which `plan` asks nothing about.""" + source = make_source_repo(base / "Thing", tree=dict(SOURCE), edits=()) + committed_raw(source, ROOT_ATTRIBUTES, data) + return planned(source, base, {}) + + +#: Root `.gitattributes` blobs whose own rules make `git add` store them as +#: other bytes once the split appends the block: CRLF under `text`, under +#: `eol` alone (which says `text`), and an `ident` collapsed on the way in. +#: Each of the first three ended `adopt-lost` after both legs at 2423eaa2. +REENCODED_ROOTS = [ + pytest.param(CRLF_UNDER_TEXT, id="crlf-under-text-eol-lf"), + pytest.param(b"* text eol=crlf\r\n", id="crlf-under-text-eol-crlf"), + pytest.param(b"* eol=lf\r\n", id="crlf-under-eol-alone"), + pytest.param(b"* text\r\n", id="crlf-under-text"), + pytest.param(b"# $Id: deadbeef $\n* ident\n", id="an-expanded-ident"), +] + + +@pytest.mark.parametrize("data", REENCODED_ROOTS) +def test_a_root_attributes_git_add_would_reencode_is_found(tmp_path, data): + """lane 2's D5(a) line, measured at 2423eaa2: `check` exited 0 and + `execute` 1, `FINDING adopt-lost: .gitattributes`, after both leg remotes + were made. `check` now finds it, says why, and names the way out.""" + found = findings(root_attributes_plan(tmp_path, data)) + assert [code for code, _ in found] == [REENCODED], found + detail = found[0][1] + assert detail.startswith(KEPT_AND_APPENDED + STORES_OTHER_BYTES), detail + assert ("The verification would then report the source's `.gitattributes` " + "as `adopt-lost`, in no leg and not in the root tree, after both " + "legs exist.") in detail + assert ("Commit the file in the source as git stores it (`git add " + "--renormalize -- .gitattributes`, then commit), or change the " + "rule git applies to it, and re-run `plan`.") in detail + + +def test_an_encoding_the_root_attributes_is_not_in_is_found(tmp_path): + """`* working-tree-encoding=UTF-16LE` reaches the root's own + `.gitattributes`, which is UTF-8: `git add` stores it decoded as + UTF-16, or refuses it, as the length of what the split writes falls. + Either is found, before any leg; at 2423eaa2 `execute` died after both.""" + found = findings(root_attributes_plan( + tmp_path, b"* working-tree-encoding=UTF-16LE\n")) + assert [code for code, _ in found] == [REENCODED], found + + +#: Root `.gitattributes` blobs `git add` stores as the split writes them: a +#: CRLF blob `text=auto` leaves alone because the index already holds it +#: with CRLF, an LF blob, a file its own rules leave as it is, and one no +#: rule names. The first two adopted at 2423eaa2, as main does. +STORED_AS_WRITTEN = [ + pytest.param(b"* text=auto\r\n", id="crlf-under-text-auto"), + pytest.param(b"* text=auto eol=lf\r\n", id="crlf-under-text-auto-eol-lf"), + pytest.param(b"* text eol=lf\n", id="lf-under-text-eol-lf"), + pytest.param(b"* text eol=crlf\n", id="lf-under-text-eol-crlf"), + pytest.param(CRLF_UNDER_TEXT + b".gitattributes -text\r\n", + id="crlf-its-own-rules-leave-alone"), + pytest.param(b"*.md text\r\n", id="crlf-no-rule-names-it"), +] + + +@pytest.mark.parametrize("data", STORED_AS_WRITTEN) +def test_a_root_attributes_git_add_stores_as_written_is_no_finding(tmp_path, + data): + assert findings(root_attributes_plan(tmp_path, data)) == [] + + +def test_execute_refuses_a_reencoded_root_attributes_before_any_leg( + adopter, monkeypatch, tmp_path): + plan = root_attributes_plan(tmp_path, CRLF_UNDER_TEXT) + refusal = refused_before_any_leg(adopter, monkeypatch, plan, tmp_path) + assert refusal.code == REENCODED + + +def checked_root_with(tmp_path: Path, env: dict, + data: bytes = CRLF_UNDER_TEXT): + """`check` of the plan keeping a root `.gitattributes` committed as + `data`, run with `env` laid over this process's own.""" + return checked(root_attributes_plan(tmp_path, data), env) + + +def reencoded(checked) -> str: + """The one `plan-attributes-reencoded` line `check` printed.""" + lines = [line for line in checked.stderr.splitlines() + if line.startswith(f"FINDING {REENCODED}: ")] + assert len(lines) == 1, checked.stderr + return lines[0] + + +def test_what_this_machines_git_add_refuses_is_found_in_gits_words(tmp_path): + """`core.safecrlf=true` makes this machine's `git add` refuse a CRLF it + would store as LF, so the split would die committing, not verifying.""" + line = reencoded(checked_root_with( + tmp_path, command_scope("core.safecrlf", "true"))) + assert (KEPT_AND_APPENDED + "refuses what the split writes there (git " + "said `fatal: CRLF would be replaced by LF in .gitattributes`), " + "so the split's own `git add` would fail, after both legs " + "exist.") in line, line + + +def test_this_machines_own_attributes_are_read_for_the_split(tmp_path): + """Unlike the nested question, which a clone elsewhere answers, the + split's `git add` runs here and reads this machine's attributes: a + global file saying `.gitattributes text eol=lf` makes it store a CRLF + blob whose own rules say nothing of it as LF.""" + rules = tmp_path / "global-attributes" + rules.write_bytes(b".gitattributes text eol=lf\n") + checked = checked_root_with( + tmp_path, command_scope("core.attributesFile", str(rules)), + b"*.md text\r\n") + assert STORES_OTHER_BYTES in reencoded(checked) + + +@pytest.mark.parametrize("where", [ + pytest.param(lambda: {"GIT_ATTR_SOURCE": "HEAD"}, + id="git-attr-source-head"), + pytest.param(lambda: {"GIT_ATTR_SOURCE": EMPTY_TREE}, + id="git-attr-source-the-empty-tree"), + pytest.param(lambda: command_scope("attr.tree", EMPTY_TREE), + id="attr-tree-in-the-command-scope")]) +def test_no_tree_this_machine_names_steers_the_split_question(tmp_path, + where): + """The question's throwaway repository has no commit for `HEAD` to name, + and the empty tree would make git read no rule at all: neither is + inherited, so git neither fails nor goes quiet.""" + checked = checked_root_with(tmp_path, where()) + assert "REFUSED" not in checked.stderr, checked.stderr + assert STORES_OTHER_BYTES in reencoded(checked) + + +def _never_asked(*_args, **_kwargs): + raise AssertionError("git was asked what `git add` stores") + + +def no_root_attributes(base: Path) -> Path: + return make_source_repo(base / "Thing", tree=dict(SOURCE), edits=()) + + +def a_root_attributes_symlink(base: Path) -> Path: + source = make_source_repo( + base / "Thing", tree={**SOURCE, LINKED_RULES: TEXT_AUTO}, + edits=()) + index_link(source, ROOT_ATTRIBUTES, LINKED_RULES) + return source + + +@pytest.mark.parametrize(("made", "answers"), [ + pytest.param(no_root_attributes, {}, id="no-root-attributes"), + pytest.param(a_root_attributes_symlink, {"config/": ROOT}, + id="a-root-attributes-symlink")]) +def test_a_root_attributes_the_split_does_not_write_into_asks_git_nothing( + adopter, monkeypatch, tmp_path, made, answers): + """The split writes into the source's root `.gitattributes` only where + the root keeps it as a regular file: where it keeps none the shape's own + is written, and through a symlink nothing is (#471's follow-up).""" + plan = adopter.Plan.load(planned(made(tmp_path), tmp_path, answers)) + (tmp_path / "work").mkdir() + source = plan.open_source(None, tmp_path / "work") + monkeypatch.setattr(adopter, "_stored_as_written", _never_asked) + assert adopter._reencoded_attributes_findings(plan, source, + source.tree()) == [] + + +# --------------------------------------------------------------------------- +# End to end: the remedy adopts, and every pinned copy keeps its bytes +# --------------------------------------------------------------------------- + +needs_filter_repo = pytest.mark.skipif( + shutil.which("git-filter-repo") is None, + reason="git filter-repo is not installed: `pip install git-filter-repo`") + + +def adopted_by_execute(plan: Path, base: Path) -> None: + """`execute --yes` of `plan`, its remotes and work under `base`; it + exits 0 and says `adoption verified`.""" + result = run_script(ADOPT, "execute", "--plan", str(plan), "--yes", + "--local-remote-dir", str(base / "remotes"), + "--work-dir", str(base / "work")) + assert result.returncode == 0, result.stderr + result.stdout + assert "adoption verified" in result.stdout, result.stdout + + +@pytest.fixture(scope="module", params=[ + pytest.param(github(REMEDIED), id="the-remedy"), + pytest.param(github(MINUS_TEXT), id="minus-text")]) +def adopted(request, tmp_path_factory) -> Path: + """One real adoption that keeps `.github/.gitattributes` in the root: + the source the finding's remedy was applied to, and one whose kept file + says `-text`.""" + base = tmp_path_factory.mktemp("nested-attributes") + adopted_by_execute(adoption_plan(base, request.param, KEEP_GITHUB), base) + return base / "Thing" + + +@needs_filter_repo +@pytest.mark.parametrize("config", [ + pytest.param([], id="as-configured"), + pytest.param(["-c", "core.autocrlf=true"], id="autocrlf"), + pytest.param(["-c", "core.eol=crlf"], id="eol-crlf")]) +def test_every_pinned_copy_keeps_its_bytes_in_the_adopted_clone( + adopted, tmp_path, config): + """`git check-attr` on every pinned copy of the clone says `eol: lf`, + the bytes on disk hold no CRLF, and the first bootstrap passes. The + source's own README, which no copy rule reaches, is the control: where + the clone was asked to write CRLF, it did.""" + clone = tmp_path / "Northwind" + cloned = clone_just_pushed(adopted, clone, BRANCH, config=config) + assert cloned.returncode == 0, cloned.stderr + pinned = [row["path"] for row in parse_yaml( + (clone / PIN).read_text(encoding=UTF8))["files"]] + said = git("check-attr", "-z", "eol", "--", *pinned, cwd=clone).stdout + fields = said.split("\0") + eol = dict(zip(fields[0::3], fields[2::3])) + assert {path: eol.get(path) for path in pinned} == dict.fromkeys( + pinned, "lf") + assert [path for path in pinned + if b"\r\n" in (clone / path).read_bytes()] == [] + if config: + assert b"\r\n" in (clone / "README.md").read_bytes(), \ + "this clone did not write CRLF, so it tests nothing" + booted = run_script(clone / "scripts" / "bootstrap.py", cwd=clone) + assert booted.returncode == 0, booted.stderr + booted.stdout + + +@needs_filter_repo +def test_a_renormalized_root_attributes_adopts(tmp_path): + """The finding's way out, taken: the CRLF root `.gitattributes` + committed again as git stores it, and the plan made again. `check` finds + nothing, and `execute` verifies the source's bytes followed by the + block.""" + source = make_source_repo(tmp_path / "Thing", tree=dict(SOURCE), edits=()) + committed_raw(source, ROOT_ATTRIBUTES, CRLF_UNDER_TEXT) + git("add", "--renormalize", "--", ROOT_ATTRIBUTES, cwd=source) + git("commit", "-qm", "Renormalize .gitattributes", cwd=source) + plan = planned(source, tmp_path, {}) + assert findings(plan) == [] + adopted_by_execute(plan, tmp_path)