diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index e31385cd..1a18259b 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -38,9 +38,17 @@ old commit, and one made as `main` could hide the commits the twin holds behind the new ref. No ref moves and none is made; the line says where the leg is, and gives `git -C switch ` as - the way out, or, for a branch that is not there, to keep the commits - on a branch named otherwise and check out the pin, whose sha it - gives (#896). A leg + the way out, or, for a branch that is not there, `git -C + branch -m `, which renames the twin and moves HEAD with + it, and the pin's sha (#896). A twin is also one in Unicode form: + `cafe` and U+0301 (NFD) is a twin of `caf` and U+00E9 (NFC), which + APFS reads as one file (#962). And a branch that is not there is not + made while any branch under `refs/heads/`, packed or loose, is a + twin of it, whether or not HEAD is on it and whatever else holds its + commits: a new file `refs/heads/main` shadows the twin's name on such + a disk, its entry stays in `packed-refs`, and a later `branch -D` of + the twin would unlink the file `main`. The line names the twin and + gives the same rename, to a name that is not there (#962). A leg with a merge, a cherry-pick, a revert, a rebase or an `am` in progress is not moved at all: a checkout ends the first three, carrying what they staged onto the branch, and leaves the last two to go on over @@ -112,6 +120,7 @@ import shlex import subprocess import sys +import unicodedata from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -688,13 +697,11 @@ def _say_left(head: str | None, current: str, named: str) -> None: f"{named} now") -def _say_keep(options: str = "the pin or the branch", - kept: str = "a branch") -> None: +def _say_keep(options: str = "the pin or the branch") -> None: """What to do about a checkout step (a) will not move, in the words - both refusals of it share, and a third's (`_say_twin`, which names the - branch the commits are kept on, `kept`). Nothing moves.""" + both refusals of it share. Nothing moves.""" print(" not moving the checkout off it. Keep any commit made " - f"there on {kept}, check out {options} in the leg, and run " + f"there on a branch, check out {options} in the leg, and run " "bootstrap again.") @@ -718,10 +725,19 @@ def _on_a_branch(current: str | None) -> bool: return current is not None and current.startswith("refs/heads/") +def _fold(name: str) -> str: + """`name` as a disk that folds case, or Unicode forms, reads it: in NFC, + then `str.casefold()`ed (#962). APFS reads the NFC and the NFD spelling + of a name as one file, and a surrogate escape (a name that is not UTF-8, + #704) passes through `unicodedata.normalize` as the text it is.""" + return unicodedata.normalize("NFC", name).casefold() + + def _folded_twin(current: str | None, branch: str) -> bool: """Whether HEAD's ref is a TWIN of the tracking branch: a branch under - `refs/heads/` whose name is the branch's under `str.casefold()` and not - byte for byte, `refs/heads/Main` for `main` (#896). + `refs/heads/` whose name is the branch's under `_fold` and not byte for + byte, `refs/heads/Main` for `main`, or `cafe` and U+0301 (NFD) for `caf` + and U+00E9 (NFC) (#896, #962). WHAT IS WRONG WITH ONE. A disk that folds case (macOS APFS and Windows NTFS by default) reads the loose files `refs/heads/Main` and @@ -739,19 +755,34 @@ def _folded_twin(current: str | None, branch: str) -> bool: twin. The names after `refs/heads/` are compared, and a name this system holds as surrogate escapes (a branch that is not UTF-8, #704) folds as the text it is, so no name raises here. NFC and NFD spellings - of one name are NOT folded: APFS reads those as one file too, and that - is not handled here. + of one name are twins: APFS reads those as one file too, so a leg on + the NFD spelling did #896's harm (#962). """ if current is None or not current.startswith("refs/heads/"): return False held = current[len("refs/heads/"):] - return held != branch and held.casefold() == branch.casefold() + return held != branch and _fold(held) == _fold(branch) + +def _twin_words(twin: str, branch: str) -> tuple[str, str, str]: + """How a twin differs from the branch, what reads the two as one, and + what a name must differ in to be no twin, as the lines of `_say_fold` + word them (#962). A twin that is one under `str.casefold()` alone is + #896's, in #896's words, byte for byte; any other differs in its + Unicode form too (NFC and NFD), which only macOS folds.""" + if twin.casefold() == branch.casefold(): + return ("only in case", "a disk that folds case (macOS, Windows)", + "case") + return ("only in case or Unicode form", + "a disk that folds case or Unicode forms (macOS folds both, " + "Windows folds case)", "case or Unicode form") -#: What step (a) says a HEAD on a twin of the tracking branch could do, by -#: what it would otherwise have done (#896): `move` is `fast_forward_to_pin`'s + +#: What step (a) says a twin of the tracking branch could do, by what it +#: would otherwise have done (#896): `move` is `fast_forward_to_pin`'s #: `branch -f`, `place` is `place_at_pin`'s checkout and its line that the -#: leg left a branch, and `make` is `create_at_pin`'s `checkout -b`. +#: leg left a branch, and `make` is `create_at_pin`'s `checkout -b`, said by +#: `_say_spelling` (#962). TWIN_RISKS = { "move": "moving {branch} could move the ref HEAD stands on and leave " "the index at the old commit", @@ -767,7 +798,9 @@ def _say_twin(label: str, path: str, branch: str, current: str, state: str, """A leg whose HEAD is on a twin of the tracking branch (`_folded_twin`, #896): nothing moves and no branch is made, and the lines say where the leg is, why it is left and the way out. `state` says what the branch is - to the pin, and `act` which of `TWIN_RISKS` is the harm. + to the pin, and `act` which of `TWIN_RISKS` is the harm: `move` or + `place`. The branch that is not there (`make`) is `_say_spelling`'s, + for HEAD's twin as for any other (#962). THE WORDS HOLD ON A DISK THAT KEEPS CASE TOO, where `Main` and `main` are two branches: they say only that a disk that folds case reads them @@ -780,30 +813,105 @@ def _say_twin(label: str, path: str, branch: str, current: str, state: str, disk that folds case it re-spells HEAD and moves no file, and the next run takes the on-branch `merge --ff-only` of `fast_forward_to_pin` or the no-op checkout of `place_at_pin`; where the two are branches, it - puts the leg on the real one, and `Main` keeps its commits. Where the - branch is NOT there (`make`), that command is no way out on a disk that - keeps case, so the commits `Main` holds are kept on a branch of another - name, and the pin is checked out: on a disk that folds case, - `checkout -b main` would write the file `Main` reads, and hide behind - it what `Main` holds in `packed-refs`. A loose `Main` there makes - `refs/heads/main` resolve, so this path never runs for it. + puts the leg on the real one, and `Main` keeps its commits. """ - named = spelled(branch) - twin = spelled(current[len("refs/heads/"):]) - risk = TWIN_RISKS[act].format(branch=named, twin=twin) - print(f" {label}: {state}, and the checkout is on {twin}, which differs " - f"from {named} only in case") - print(f" a disk that folds case (macOS, Windows) reads {twin} " - f"and {named} as one branch, so {risk}; the leg is left as it is") - if act == "make": - _say_keep("the pin", "a branch whose name differs from " - f"{named} by more than case") - return + twin = current[len("refs/heads/"):] + _say_fold(label, state, "the checkout is on", twin, branch, act) command = f"git -C {_paste_word(path)} switch {_paste_word(branch)}" print(f" run `{command}`, which moves no branch, and then run " "bootstrap again.") +def _say_fold(label: str, state: str, where: str, twin: str, branch: str, + act: str) -> None: + """The first two lines of a refusal over a twin, `where` saying how the + leg has it (`the checkout is on` it, or `the leg has the branch`): where + the leg is, and why it is left (`TWIN_RISKS`). Every name is spelled + (#391), and the second line says only what a disk that folds could do, + never that the two are one (#962).""" + named = spelled(branch) + held = spelled(twin) + how, reads, _ = _twin_words(twin, branch) + risk = TWIN_RISKS[act].format(branch=named, twin=held) + print(f" {label}: {state}, and {where} {held}, which differs from " + f"{named} {how}") + print(f" {reads} reads {held} and {named} as one branch, so " + f"{risk}; the leg is left as it is") + + +def _heads(sub: Path) -> list[str]: + """Every ref under `refs/heads/`, packed or loose, as `git for-each-ref` + lists them, or none where git says nothing (#962).""" + listed = _try_git(["for-each-ref", "--format=%(refname)", + "refs/heads/"], sub) + return (listed or "").splitlines() + + +def _folded_spelling(sub: Path, branch: str) -> str | None: + """The first branch under `refs/heads/`, packed or loose, that is a twin + of `branch` (`_folded_twin`), or `None` (#962). + + WHAT WAS WRONG. `create_at_pin` asked only about HEAD's own ref, so a + leg detached at the pin over a packed `Main`, or on an unborn `Main` + beside a packed `MAIN`, ran `checkout -b main `, which wrote the + loose file a disk that folds case reads for `Main` and `MAIN`. Reasoned + from git's files backend, and not measured on such a disk (rv984): the + packed entry `Main` is left where it was, `for-each-ref` goes on listing + it at its old sha while `rev-parse Main` reads the new file `main`, and + a later `git branch -D Main` unlinks that file. The commit is still in + a ref, but its name is shadowed, and the state is fragile. + + EVERY TWIN IS A REASON TO LEAVE, whether or not another ref holds its + commits (a tag, a remote-tracking ref, another branch, or the pin, which + the new branch holds): the ref made is the one bootstrap never makes, a + file a disk that folds case reads as a branch that is already there. + `_say_spelling` names it and gives the rename that ends the refusal.""" + for ref in _heads(sub): + if _folded_twin(ref, branch): + return ref[len("refs/heads/"):] + return None + + +def _kept_name(sub: Path, spelling: str) -> str: + """A name for the branch `spelling` is renamed to, so the line that says + to rename it can be pasted as it is: `spelling` and `-kept`, then + `-kept-2` and on, the first that no branch of the leg folds to + (`_fold`, so a name only in another case or form is taken too). It is + longer than `spelling`, so it differs from the tracking branch by more + than case or Unicode form, and a suffix of letters and dashes leaves a + valid ref name valid (#962).""" + held = {_fold(ref[len("refs/heads/"):]) for ref in _heads(sub)} + name, turn = f"{spelling}-kept", 1 + while _fold(name) in held: + turn += 1 + name = f"{spelling}-kept-{turn}" + return name + + +def _say_spelling(sub: Path, label: str, path: str, branch: str, + spelling: str, state: str, + where: str = "the leg has the branch") -> None: + """A leg with a twin of the missing tracking branch, `spelling`, in the + way of making it (`_folded_spelling`, or HEAD's own, `where` being `the + checkout is on`, #962): nothing is made, and the lines name the twin, + say why, and give the way out: rename it, `git -C branch -m + `, `_kept_name` being a name that is not there, and run + bootstrap again. The twin is then no twin, its commits stay on it, and + the next run makes the branch. Git moves HEAD with the rename of the + branch HEAD is on, an unborn one included, so a leg ON the twin takes + the way out as any other does, and the next run says `the leg was on + , which keeps its commits`. The twin may be a branch of its own, + on a disk that keeps case, and the words are true there too.""" + _say_fold(label, state, where, spelling, branch, "make") + more = _twin_words(spelling, branch)[2] + command = (f"git -C {_paste_word(path)} branch -m " + f"{_paste_word(spelling)} " + f"{_paste_word(_kept_name(sub, spelling))}") + print(f" rename {spelled(spelling)} to a name that differs from " + f"{spelled(branch)} by more than {more}, its commits staying on " + f"it: `{command}`, and run bootstrap again.") + + def _in_progress(sub: Path) -> tuple[tuple[str, str, str], str] | None: """The `IN_PROGRESS` row of the operation git has in progress in the leg, with where its marker is (`git dir` or `ref store`), or `None` @@ -1204,11 +1312,17 @@ def create_at_pin(sub: Path, label: str, path: str, branch: str, pin: str, HEAD on a TWIN of the branch, one that differs from it only in case (`Main` for `main`, `_folded_twin`, #896), and no branch is made: a disk that folds case reads the new file `refs/heads/main` as the file - `refs/heads/Main`, so the commits `Main` holds, packed as they are - after `git pack-refs`, are left behind it, in no ref anyone reads. - `switch ` is no way out of a leg with no such branch, so - `_say_twin` says to keep the commits on a branch of another name and to - check out the pin; `path` is the leg's. + `refs/heads/Main`, so the name `Main` is shadowed by it, and its + packed entry is left behind it. `switch ` is no way out of a + leg with no such branch, so `_say_spelling` says to rename the twin, + which moves HEAD with it (#962); `path` is the leg's. + + The same harm comes with HEAD ELSEWHERE (#962): detached at the pin over + a packed `Main`, or on an unborn `Main` beside a packed `MAIN`. Before + the checkout, `_folded_spelling` asks every branch under `refs/heads/`, + packed or loose, for a twin of `branch`, and `_say_spelling` leaves the + leg, naming it. A twin in NFD or NFC is one too (`_fold`), and EVERY + twin is a reason to leave, whatever else holds its commits. The checkout is judged by WHERE HEAD IS AFTERWARDS, as `_check_out` judges one (#533; rv704's S3): a post-checkout hook that fails after @@ -1220,9 +1334,10 @@ def create_at_pin(sub: Path, label: str, path: str, branch: str, pin: str, named = spelled(branch) head, current = _head_of(sub) if _folded_twin(current, branch): - _say_twin(label, path, branch, current, - f"branch {named} does not exist (the pin is " - f"{_short(pin)})", "make") + _say_spelling(sub, label, path, branch, + current[len("refs/heads/"):], + f"branch {named} does not exist (the pin is " + f"{_short(pin)})", "the checkout is on") return if head is not None and current is None and head != pin: # `checkout -b ` from a leg detached anywhere else @@ -1242,6 +1357,12 @@ def create_at_pin(sub: Path, label: str, path: str, branch: str, pin: str, "branch") _say_keep("the pin") return + spelling = _folded_spelling(sub, branch) + if spelling is not None: + _say_spelling(sub, label, path, branch, spelling, + f"branch {named} does not exist (the pin is " + f"{_short(pin)})") + return why = _git_says(["checkout", "-q", "--no-overwrite-ignore", "-b", branch, pin], sub) if _head_of(sub) != (pin, f"refs/heads/{branch}"): diff --git a/tests/test_bootstrap_casefold_twin.py b/tests/test_bootstrap_casefold_twin.py index 1631b28e..5c7b591e 100644 --- a/tests/test_bootstrap_casefold_twin.py +++ b/tests/test_bootstrap_casefold_twin.py @@ -31,8 +31,9 @@ moves and none is made. The lines say where the leg is, why it is left, and the way out: `git -C switch ` for T7 and P9, which on a disk that folds case re-spells HEAD and moves no file, and for the missing branch, -where that command has no branch to go to, keeping the commits on a branch of -another name and checking out the pin. +where that command has no branch to go to, renaming the twin, which moves HEAD +with it (#962: it first said to keep the commits on a branch of another name +and check out the pin, which leaves the twin where it is). THE WORDS HOLD ON A DISK THAT KEEPS CASE TOO, where `Main` and `main` are two branches: they say that a disk that folds case reads them as one, never that @@ -83,10 +84,11 @@ #: The way out of a leg with a branch to go to (T7, P9). SWITCH = (" run `git -C spec switch main`, which moves no branch, " "and then run bootstrap again.\n") -#: The way out of a leg with no `main` (#896's missing-branch path). -KEEP = (" not moving the checkout off it. Keep any commit made " - "there on a branch whose name differs from main by more than case, " - "check out the pin in the leg, and run bootstrap again.\n") +#: The way out of a leg with no `main` (#896's missing-branch path, #962's +#: rename of the twin, which git moves HEAD with). +RENAME = (" rename Main to a name that differs from main by more " + "than case, its commits staying on it: `git -C spec branch -m " + "Main Main-kept`, and run bootstrap again.\n") HEADING = "\n(a) legs on tracking branches, pins untouched\n" @@ -171,6 +173,13 @@ def printed_way_out(stdout: str) -> str: return line.split("`")[1] +def printed_rename(stdout: str) -> str: + """The command the `rename ...` line gives, between its backticks.""" + line = next(row for row in stdout.splitlines() + if row.startswith(" rename ")) + return line.split("`")[1] + + def status(leg: Path) -> str: return git("status", "--short", cwd=leg).stdout @@ -258,7 +267,9 @@ def test_the_missing_branch_path_leaves_an_unborn_twin_and_makes_nothing( """`Main` unborn (no commit, the dangling symlink `Main -> main` a disk that folds case reads as one), `main` not there. `checkout -b main` made the file `Main` reads, and said the leg had been on a branch with no - commit yet.""" + commit yet. The way out, run as printed through `/bin/sh`, renames the + unborn branch HEAD is on: HEAD goes with it, and the next run makes + `main` and says the leg was on `Main-kept`, a branch with no commit.""" leg = root / "spec" pin = without_main(leg) twin_on_disk(leg) @@ -268,21 +279,33 @@ def test_the_missing_branch_path_leaves_an_unborn_twin_and_makes_nothing( assert result.returncode == 0, result.stderr + result.stdout assert step_a(result.stdout) == twin_lines( f"branch main does not exist (the pin is {pin[:12]})", "make", - KEEP), result.stdout + RENAME), result.stdout assert main_of(leg) is None, "no `main` is made" assert not (gitdir_of(leg) / "refs" / "heads" / "main").exists() assert head_ref(leg) == TWIN_REF, "HEAD stays where it was" assert pin == rev(root, "HEAD:spec") + pasted(root, printed_rename(result.stdout)) + assert head_ref(leg) == "refs/heads/Main-kept", "HEAD goes with it" + again = skip_validators(root) + assert again.returncode == 0, again.stderr + again.stdout + assert step_a(again.stdout) == ( + f" [spec] spec: branch main created at {pin[:12]} — at the pin\n" + " the leg was on Main-kept, a branch with no commit yet; " + "it is on main now\n"), again.stdout + assert head_ref(leg) == MAIN + assert rev(leg) == pin + def test_the_missing_branch_path_leaves_a_packed_twin_with_commits( root, tmp_path): """The state that loses something on a disk that folds case: `Main` holds a commit of its own, packed (`git pack-refs --all`, which is what a `gc` leaves), and `main` is not there. A new loose `main` is the - file `Main` reads, which hides what `Main` holds. Left, with the way - out run as printed: the commit kept on a branch of another name, the - pin checked out, bootstrap again, which makes `main` at the pin.""" + file `Main` reads, which shadows the name `Main`. Left, with the way + out run as printed through `/bin/sh`: `Main` renamed, which moves HEAD + with it, and bootstrap again, which makes `main` at the pin and says + the leg was on `Main-kept`, which keeps its commits.""" needs_a_disk_that_keeps_case(tmp_path) leg = root / "spec" pin = rev(leg) @@ -295,25 +318,29 @@ def test_the_missing_branch_path_leaves_a_packed_twin_with_commits( assert result.returncode == 0, result.stderr + result.stdout assert step_a(result.stdout) == twin_lines( f"branch main does not exist (the pin is {pin[:12]})", "make", - KEEP), result.stdout + RENAME), result.stdout assert main_of(leg) is None, "no `main` is made" assert head_ref(leg) == TWIN_REF assert rev(leg) == mine, "HEAD stays on `Main`'s commit" assert status(leg) == "" - - git("branch", "-q", "kept-from-Main", cwd=leg) shown = re.search(r"\(the pin is ([0-9a-f]{12})\)", result.stdout) - assert shown is not None, "the line prints the pin" - git("checkout", "-q", "--detach", shown.group(1), cwd=leg) - assert rev(leg) == pin, "the printed sha is the pin" + assert shown is not None and shown.group(1) == pin[:12], ( + "the line prints the pin") + + pasted(root, printed_rename(result.stdout)) + assert head_ref(leg) == "refs/heads/Main-kept", "HEAD goes with it" + assert rev(leg) == mine again = skip_validators(root) assert again.returncode == 0, again.stderr + again.stdout assert step_a(again.stdout) == ( - f" [spec] spec: branch main created at {pin[:12]} — at the pin\n"), ( - again.stdout) + f" [spec] spec: branch main created at {pin[:12]} — at the pin\n" + " the leg was on Main-kept, which keeps its commits; it is " + "on main now\n"), again.stdout assert head_ref(leg) == MAIN - assert rev(leg, "refs/heads/kept-from-Main") == mine - assert rev(leg, TWIN_REF) == mine, "`Main` keeps its commit here" + assert rev(leg) == pin + assert rev(leg, "refs/heads/Main-kept") == mine + assert git("rev-parse", "--verify", "--quiet", TWIN_REF, cwd=leg, + check=False).stdout.strip() == "", "`Main` is renamed" def test_on_a_disk_that_keeps_case_the_words_are_true_too( @@ -510,7 +537,8 @@ def test_a_shell_reads_each_word_of_the_way_out_as_the_bytes_of_its_name( assert read == os.fsencode(path) + b"\0" + os.fsencode(branch) -def test_the_lines_spell_a_name_that_is_not_utf_8(bootstrap_module, capsys): +def test_the_lines_spell_a_name_that_is_not_utf_8( + bootstrap_module, capsys, tmp_path): """A twin named with a byte that is not UTF-8 holds a surrogate escape here (#704): printed raw it would raise `UnicodeEncodeError`, so each name in the lines is spelled (#391), the leg's path too.""" @@ -518,9 +546,9 @@ def test_the_lines_spell_a_name_that_is_not_utf_8(bootstrap_module, capsys): branch = f"a{SURROGATE}b" leg = f"a{chr(0x1B)}b" spelled = bootstrap_module.spelled - bootstrap_module._say_twin( - f"[spec] {spelled(leg)}", leg, branch, f"refs/heads/{twin}", - f"branch {spelled(branch)} does not exist", "make") + bootstrap_module._say_spelling( + tmp_path, f"[spec] {spelled(leg)}", leg, branch, twin, + f"branch {spelled(branch)} does not exist", "the checkout is on") out = capsys.readouterr().out assert out.isascii(), "every name is written as an escape" assert chr(0x1B) not in out diff --git a/tests/test_bootstrap_create_twins.py b/tests/test_bootstrap_create_twins.py new file mode 100644 index 00000000..dc99eef1 --- /dev/null +++ b/tests/test_bootstrap_create_twins.py @@ -0,0 +1,520 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`make bootstrap` does not make a tracking branch a disk that folds case, or +Unicode forms, reads as a branch that is there, and a twin is folded in NFC +too (#962). + +WHAT #948 (#896) LEFT. `_folded_twin` asked only about HEAD's own ref, and +`create_at_pin` checked no other spelling of the branch it made, so on a disk +that folds case (macOS, Windows) `checkout -b main ` wrote the loose file +that `Main` and `MAIN` are read from. What such a disk then does to a packed +`Main` is reasoned from git's files backend (rv984), and not measured on one: +the packed entry stays, `for-each-ref` goes on listing it at its old sha while +`rev-parse Main` reads the new file `main`, and a later `branch -D Main` +unlinks that file. The commit is still in a ref; its name is shadowed, and the +state is fragile. Three states did it: + +- a leg DETACHED AT THE PIN over a packed `Main`, and no `main`; +- a leg on an UNBORN `Main` beside a packed `MAIN`: #948 refuses the unborn + `Main`, whose way out is to check out the pin, and the next run then made + `main` over `MAIN`; +- the same with NFC and NFD, which APFS reads as one file as it reads `Main` + and `main`: a leg on the NFD spelling of `café`, `str.casefold()` alone + calling it no twin, was moved under its own HEAD as #896's was. + +THE RULE NOW. `_fold` is NFC, then `casefold()`, and `_folded_twin` asks it +of HEAD's ref. Before `checkout -b`, `create_at_pin` asks `git for-each-ref +refs/heads/`, which lists the packed ones too, for ANY twin, and leaves the +leg, naming it: whether or not a tag, a remote-tracking ref, another branch +or the pin holds the twin's commits, since the ref made is one a folding disk +reads as a branch that is already there. The way out RENAMES the twin, `git -C + branch -m -kept`, for a twin elsewhere and for the twin +HEAD is on alike: git moves HEAD with the rename, and the next run makes `main` +(an earlier way out, to keep the commits on another branch and check out the +pin, left the twin where it was, so the refusal came back). + +THE WORDS HOLD ON A DISK THAT KEEPS CASE TOO, where a real `Main` is a branch +of its own: every state here that makes a real name, packed or loose, runs the +way out there, and skips on a disk that folds case, where it cannot be made. +A test with a symlink skips as `test_bootstrap_casefold_twin.py`'s do. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import unicodedata +from pathlib import Path + +import pytest + +from conftest import git, run_script +from test_bootstrap_after_bump import ( # noqa: F401 + bootstrapped, head_ref, rev, root, skip_validators, +) +from test_bootstrap_branch_bytes import bootstrap_module # noqa: F401 +from test_bootstrap_casefold_twin import ( + behind, commit_on, gitdir_of, needs_a_disk_that_keeps_case, + printed_rename, printed_way_out, status, step_a, twin_on_disk, +) +from test_bootstrap_leg_moves import main_of +from test_bootstrap_non_utf8_branch import NEEDS_A_UTF8_PROCESS, head_bytes +from test_bootstrap_pasteable_lines import pasted + +NFC = "caf" + chr(0xE9) +NFD = unicodedata.normalize("NFD", NFC) +#: What a refusal says a twin differs from the branch in, what reads the two +#: as one, and what a name must differ in to be no twin (`_twin_words`). +CASE = ("only in case", "a disk that folds case (macOS, Windows)", "case") +FORM = ("only in case or Unicode form", + "a disk that folds case or Unicode forms (macOS folds both, Windows " + "folds case)", "case or Unicode form") +MAKE = ("making {b} could hide the commits a branch spelled like {b} holds " + "behind the new ref") +MOVE = ("moving {b} could move the ref HEAD stands on and leave the index " + "at the old commit") +ON = "the checkout is on" +HAS = "the leg has the branch" + + +def spelling_lines(state: str, spelling: str, branch: str = "main", + kind=CASE, where: str = HAS, kept: str = "") -> str: + """The three lines step (a) prints of a leg that has a twin of the + missing branch (`_say_spelling`), byte for byte, `where` saying how the + leg has it and `kept` being the name the rename gives it.""" + how, reads, more = kind + kept = kept or f"{spelling}-kept" + return (f" [spec] spec: {state}, and {where} {spelling}, " + f"which differs from {branch} {how}\n" + f" {reads} reads {spelling} and {branch} as one branch, " + f"so {MAKE.format(b=branch)}; the leg is left as it is\n" + f" rename {spelling} to a name that differs from " + f"{branch} by more than {more}, its commits staying on it: " + f"`git -C spec branch -m {shlex.quote(spelling)} " + f"{shlex.quote(kept)}`, and run bootstrap again.\n") + + +def missing(pin: str) -> str: + return f"branch main does not exist (the pin is {pin[:12]})" + + +def created(pin: str) -> str: + return f" [spec] spec: branch main created at {pin[:12]} — at the pin\n" + + +def sha_of(leg: Path, ref: str) -> str: + """What `ref` names in the leg, or the empty string where it is none.""" + return git("rev-parse", "--verify", "--quiet", ref, cwd=leg, + check=False).stdout.strip() + + +def detached_without_main(leg: Path) -> str: + """The leg detached at the pin, `main` gone. Returns the pin.""" + pin = rev(leg) + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + return pin + + +def keeps_forms(directory: Path) -> bool: + """Whether the disk tells the NFC from the NFD spelling of `café`.""" + (directory / NFC).write_bytes(b"") + return not (directory / NFD).exists() + + +@pytest.mark.parametrize("packed", [True, False], + ids=["packed Main", "loose Main"]) +def test_a_leg_detached_at_the_pin_over_a_real_spelling_is_left( + root, tmp_path, packed): + """(b) HEAD detached at the pin, `Main` holding a commit of its own, no + `main`. `checkout -b main ` exited 0 and wrote the file `Main` is + read from on a disk that folds case. Nothing is made; the lines name + `Main`, and the way out, run as printed through `/bin/sh`, renames it, so + the next run makes `main`, and the commit stays on `Main-kept`. Loose + and packed give the same lines (a loose `Main` cannot stand beside a + missing `main` on a disk that folds case: it IS `main` there).""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", "Main", cwd=leg) + mine = commit_on(leg, "MINE.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + if packed: + git("pack-refs", "--all", cwd=leg) + + first = skip_validators(root) + assert first.returncode == 0, first.stderr + first.stdout + assert step_a(first.stdout) == spelling_lines(missing(pin), "Main"), ( + first.stdout) + assert main_of(leg) is None, "no `main` is made" + assert head_ref(leg) is None, "HEAD stays detached at the pin" + assert rev(leg) == pin + assert rev(leg, "refs/heads/Main") == mine + assert status(leg) == "" + + pasted(root, printed_rename(first.stdout)) + assert rev(leg, "refs/heads/Main-kept") == mine + assert sha_of(leg, "refs/heads/Main") == "", "`Main` is renamed" + again = skip_validators(root) + assert again.returncode == 0, again.stderr + again.stdout + assert step_a(again.stdout) == created(pin), again.stdout + assert head_ref(leg) == "refs/heads/main" + assert rev(leg, "refs/heads/Main-kept") == mine + + +def test_a_leg_on_a_twin_takes_the_rename_and_head_goes_with_it( + root, tmp_path): + """#948's HEAD-twin line, for a branch that is not there, said to keep + the commits on another branch and check out the pin. The rename gives the + same ending to a leg ON the twin: git moves HEAD with the renamed + branch, the next run finds no twin, makes `main`, and says the leg was on + `Main-kept`, which keeps its commits.""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", "Main", cwd=leg) + mine = commit_on(leg, "MINE.md") + git("branch", "-q", "-D", "main", cwd=leg) + git("pack-refs", "--all", cwd=leg) + + first = skip_validators(root) + assert first.returncode == 0, first.stderr + first.stdout + assert step_a(first.stdout) == spelling_lines( + missing(pin), "Main", where=ON), first.stdout + assert main_of(leg) is None, "no `main` is made" + assert head_ref(leg) == "refs/heads/Main" + assert rev(leg) == mine + assert status(leg) == "" + + pasted(root, printed_rename(first.stdout)) + assert head_ref(leg) == "refs/heads/Main-kept", "HEAD goes with it" + assert rev(leg) == mine + again = skip_validators(root) + assert again.returncode == 0, again.stderr + again.stdout + assert step_a(again.stdout) == created(pin) + ( + " the leg was on Main-kept, which keeps its commits; it is " + "on main now\n"), again.stdout + assert head_ref(leg) == "refs/heads/main" + assert rev(leg) == pin + assert rev(leg, "refs/heads/Main-kept") == mine + + +def test_an_unborn_twin_beside_a_packed_third_spelling_is_renamed_in_turn( + root, tmp_path): + """(c) HEAD on an unborn `Main` while a packed `MAIN` holds a commit, and + no `main`. #948 refuses the unborn `Main`; its way out, checking out the + pin, ran `checkout -b main` over `MAIN`. Now each twin is named in turn + and renamed as printed: the unborn `Main` first (HEAD goes with it), then + `MAIN`, and the third run makes `main`.""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", "MAIN", cwd=leg) + mine = commit_on(leg, "MINE.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + git("pack-refs", "--all", cwd=leg) + git("symbolic-ref", "HEAD", "refs/heads/Main", cwd=leg) + + first = skip_validators(root) + assert first.returncode == 0, first.stderr + first.stdout + assert step_a(first.stdout) == spelling_lines( + missing(pin), "Main", where=ON), first.stdout + assert main_of(leg) is None + pasted(root, printed_rename(first.stdout)) + assert head_ref(leg) == "refs/heads/Main-kept", "HEAD goes with it" + + second = skip_validators(root) + assert second.returncode == 0, second.stderr + second.stdout + assert step_a(second.stdout) == spelling_lines(missing(pin), "MAIN"), ( + second.stdout) + assert main_of(leg) is None, "no `main` is made" + assert rev(leg, "refs/heads/MAIN") == mine + pasted(root, printed_rename(second.stdout)) + + third = skip_validators(root) + assert third.returncode == 0, third.stderr + third.stdout + assert step_a(third.stdout) == created(pin) + ( + " the leg was on Main-kept, a branch with no commit yet; " + "it is on main now\n"), third.stdout + assert rev(leg, "refs/heads/MAIN-kept") == mine + assert sha_of(leg, "refs/heads/MAIN") == "" + + +def test_two_twins_are_named_in_turn_and_the_second_name_is_taken_in_case( + root, tmp_path): + """Two twins, `MAIN` (first in the list) and `Main`, each with a commit: + the refusal names `MAIN`, and once it is renamed, `Main`. The second + rename is `Main-kept-2`, not `Main-kept`, which differs from the first + rename, `MAIN-kept`, only in case: on a disk that folds case it would be + the same file.""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", "MAIN", cwd=leg) + one = commit_on(leg, "ONE.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("checkout", "-q", "-b", "Main", cwd=leg) + two = commit_on(leg, "TWO.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + git("pack-refs", "--all", cwd=leg) + + first = skip_validators(root) + assert first.returncode == 0, first.stderr + first.stdout + assert step_a(first.stdout) == spelling_lines(missing(pin), "MAIN"), ( + first.stdout) + pasted(root, printed_rename(first.stdout)) + + second = skip_validators(root) + assert second.returncode == 0, second.stderr + second.stdout + assert step_a(second.stdout) == spelling_lines( + missing(pin), "Main", kept="Main-kept-2"), second.stdout + assert main_of(leg) is None + pasted(root, printed_rename(second.stdout)) + + third = skip_validators(root) + assert third.returncode == 0, third.stderr + third.stdout + assert step_a(third.stdout) == created(pin), third.stdout + assert rev(leg, "refs/heads/MAIN-kept") == one + assert rev(leg, "refs/heads/Main-kept-2") == two + + +@pytest.mark.parametrize("held", ["the pin alone", "another branch", + "a tag", "a remote-tracking ref", + "the stash"]) +def test_a_twin_is_a_reason_to_leave_however_its_commit_is_held( + root, tmp_path, held): + """The ref bootstrap would make is one a disk that folds case reads as a + branch that is already there, so a twin is a reason to leave whether or + not something else holds its commit: the pin (`Main` at the pin itself), + another branch, a tag, a remote-tracking ref or the stash (rv984 measured + the four that hold a commit the pin lacks, which an earlier rule + spared). The rename ends it, and the commit is where it was.""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", "Main", cwd=leg) + mine = pin if held == "the pin alone" else commit_on(leg, "MINE.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + if held == "another branch": + git("branch", "-q", "kept", mine, cwd=leg) + if held == "a tag": + git("tag", "kept-tag", mine, cwd=leg) + if held == "a remote-tracking ref": + git("update-ref", "refs/remotes/origin/Main", mine, cwd=leg) + if held == "the stash": + git("update-ref", "refs/stash", mine, cwd=leg) + git("pack-refs", "--all", cwd=leg) + + first = skip_validators(root) + assert first.returncode == 0, first.stderr + first.stdout + assert step_a(first.stdout) == spelling_lines(missing(pin), "Main"), ( + first.stdout) + assert main_of(leg) is None, "no `main` is made" + assert head_ref(leg) is None + assert rev(leg, "refs/heads/Main") == mine + + pasted(root, printed_rename(first.stdout)) + again = skip_validators(root) + assert again.returncode == 0, again.stderr + again.stdout + assert step_a(again.stdout) == created(pin), again.stdout + assert rev(leg, "refs/heads/Main-kept") == mine + + +def test_a_branch_that_is_no_twin_is_no_reason_to_leave(root, tmp_path): + """`mains` holds a commit and is no spelling of `main`: it is no one's + business, and `main` is made, as it was. This passes on the base by + design.""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", "mains", cwd=leg) + mine = commit_on(leg, "MINE.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + git("pack-refs", "--all", cwd=leg) + + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + assert step_a(result.stdout) == created(pin), result.stdout + assert head_ref(leg) == "refs/heads/main" + assert rev(leg, "refs/heads/mains") == mine + + +@NEEDS_A_UTF8_PROCESS +def test_a_leg_on_the_nfd_spelling_of_its_branch_is_left_and_the_way_out_works( + root, bootstrapped, tmp_path): + """T7 with the tracking branch `café` in NFC, behind the pin, and HEAD on + its NFD spelling (a symlink, as APFS reads the two as one file). The + byte-for-byte and the `casefold` compares called it no twin, `branch -f` + moved `café` under HEAD, and `D NEW-CONTRACT.md;M README.md` followed. + Now the leg is left, the lines say it differs in case or Unicode form, + and the printed `switch`, run as printed, puts HEAD on the NFC name byte + for byte, which the next run fast-forwards.""" + old, new = behind(root, bootstrapped["remotes"], tmp_path) + leg = root / "spec" + git("branch", "-q", "-m", "main", NFC, cwd=leg) + twin_on_disk(leg, NFD, NFC) + bootstrap = root / "scripts" / "bootstrap.py" + + first = run_script(bootstrap, "--skip-validators", "--branch", NFC, + cwd=root) + assert first.returncode == 0, first.stderr + first.stdout + how, reads, _ = FORM + assert step_a(first.stdout) == ( + f" [spec] spec: branch {NFC} tip {old[:12]} is behind the PIN " + f"{new[:12]}, and the checkout is on {NFD}, which differs from {NFC} " + f"{how}\n" + f" {reads} reads {NFD} and {NFC} as one branch, so " + f"{MOVE.format(b=NFC)}; the leg is left as it is\n" + f" run `git -C spec switch {shlex.quote(NFC)}`, which moves " + "no branch, and then run bootstrap again.\n"), first.stdout + assert head_bytes(leg) == f"refs/heads/{NFD}\n".encode("utf-8") + assert rev(leg, f"refs/heads/{NFC}") == old + assert status(leg) == "" + + pasted(root, printed_way_out(first.stdout)) + assert head_bytes(leg) == f"refs/heads/{NFC}\n".encode("utf-8") + + again = run_script(bootstrap, "--skip-validators", "--branch", NFC, + cwd=root) + assert again.returncode == 0, again.stderr + again.stdout + assert (f" [spec] spec: branch {NFC} fast-forwarded {old[:12]} -> " + f"{new[:12]}, the pin\n") in again.stdout + assert rev(leg) == new + assert status(leg) == "" + + +@NEEDS_A_UTF8_PROCESS +def test_a_missing_branch_beside_its_nfd_spelling_is_left_and_renamed( + root, tmp_path): + """The missing `café` in NFC beside a packed branch `café` in NFD that + holds a commit: on APFS, which reads the two as one file, `checkout -b` + would shadow it. The lines name the NFD branch and say it differs in case + or Unicode form; the rename, run as printed, gives the next run a branch + to make; skips on a disk that reads the two as one.""" + if not keeps_forms(tmp_path): + pytest.skip("this disk reads NFC and NFD as one file") + leg = root / "spec" + pin = rev(leg) + git("checkout", "-q", "-b", NFD, cwd=leg) + mine = commit_on(leg, "MINE.md") + git("checkout", "-q", "--detach", pin, cwd=leg) + git("branch", "-q", "-D", "main", cwd=leg) + git("pack-refs", "--all", cwd=leg) + bootstrap = root / "scripts" / "bootstrap.py" + + result = run_script(bootstrap, "--skip-validators", "--branch", NFC, + cwd=root) + assert result.returncode == 0, result.stderr + result.stdout + state = f"branch {NFC} does not exist (the pin is {pin[:12]})" + assert step_a(result.stdout) == spelling_lines( + state, NFD, NFC, FORM), result.stdout + assert os.path.exists(gitdir_of(leg) / "packed-refs") + assert sha_of(leg, f"refs/heads/{NFC}") == "", "none is made" + assert rev(leg, f"refs/heads/{NFD}") == mine + + pasted(root, printed_rename(result.stdout)) + assert rev(leg, f"refs/heads/{NFD}-kept") == mine + again = run_script(bootstrap, "--skip-validators", "--branch", NFC, + cwd=root) + assert again.returncode == 0, again.stderr + again.stdout + assert step_a(again.stdout) == ( + f" [spec] spec: branch {NFC} created at {pin[:12]} — at the pin\n" + ), again.stdout + + +@pytest.mark.parametrize("names, spelling, expected", [ + pytest.param([], "Main", "Main-kept", id="nothing taken"), + pytest.param(["main-kept"], "Main", "Main-kept-2", + id="the name taken in another case"), + pytest.param(["Main-kept", "Main-kept-2"], "Main", "Main-kept-3", + id="two taken"), + pytest.param(["mains"], "Main", "Main-kept", id="another name is no bar"), +]) +def test_the_rename_is_to_a_name_no_branch_folds_to( + bootstrap_module, root, tmp_path, names, spelling, expected): + """`_kept_name`: the spelling and `-kept`, then `-kept-2` and on, the + first that no branch of the leg folds to, so the printed line can be + pasted as it is and names no branch that is there.""" + needs_a_disk_that_keeps_case(tmp_path) + leg = root / "spec" + for name in names: + git("branch", "-q", name, cwd=leg) + assert bootstrap_module._kept_name(leg, spelling) == expected + + +@NEEDS_A_UTF8_PROCESS +def test_the_rename_is_taken_in_unicode_form_too( + bootstrap_module, root, tmp_path): + """The NFD spelling of `café-kept` is taken: the NFC spelling of the + rename is not free, though the two differ in bytes.""" + if not keeps_forms(tmp_path): + pytest.skip("this disk reads NFC and NFD as one file") + leg = root / "spec" + git("branch", "-q", f"{NFD}-kept", cwd=leg) + assert bootstrap_module._kept_name(leg, NFC) == f"{NFC}-kept-2" + + +@pytest.mark.skipif(os.name == "nt", reason="the words are `sh`'s") +@pytest.mark.parametrize("path, spelling", [ + pytest.param("spec", "Main", id="ordinary names"), + pytest.param("my spec", "Main", id="path with a space"), + pytest.param("spec", f"A{chr(0xDCFF)}b", id="spelling with the byte"), + pytest.param(f"-sp{chr(0xDCFF)}ec", f"A{chr(0xDCFF)}b", + id="both, the path leading with a dash"), +]) +def test_a_shell_reads_each_word_of_the_rename_as_the_bytes_of_its_name( + bootstrap_module, capsys, tmp_path, path, spelling): + """`_paste_word` for the path, the twin and the name it is renamed to: + the words `/bin/sh` reads from the printed command are the bytes of the + names, the name a byte that is not UTF-8 is spelled with included. `git` + is a function here that prints its arguments, NUL-separated.""" + if not os.path.exists("/bin/sh"): + pytest.skip("this system has no /bin/sh") + branch = spelling.lower() + bootstrap_module._say_spelling( + tmp_path, "[spec] x", path, branch, spelling, "branch is not there") + command = printed_rename(capsys.readouterr().out) + read = subprocess.run( + ["/bin/sh", "-c", 'git() { printf "%s\\0" "$@"; }\n' + command], + capture_output=True, check=True).stdout.split(b"\0")[:-1] + assert read == [b"-C", os.fsencode(path), b"branch", b"-m", + os.fsencode(spelling), os.fsencode(spelling) + b"-kept"] + + +@pytest.mark.parametrize("current, branch, twin", [ + pytest.param(f"refs/heads/{NFD}", NFC, True, id="NFD for NFC"), + pytest.param(f"refs/heads/{NFC}", NFD, True, id="NFC for NFD"), + pytest.param(f"refs/heads/{NFC}", NFC, False, id="NFC itself"), + pytest.param(f"refs/heads/{NFD}", NFD, False, id="NFD itself"), + pytest.param(f"refs/heads/{NFD.upper()}", NFC, True, + id="NFD in capitals for NFC"), + pytest.param("refs/heads/" + NFD, "caf" + chr(0xE8), False, + id="another letter"), + pytest.param("refs/heads/main" + chr(0xB2), "main2", False, + id="a superscript two is no two (NFKC would fold it)"), + pytest.param("refs/heads/" + chr(0x2460), "1", False, + id="a circled one is no one (NFKC would fold it)"), +]) +def test_the_helper_folds_unicode_forms_too( + bootstrap_module, current, branch, twin): + """`_folded_twin` folds in NFC before `casefold()`: the NFC and the NFD + spelling of a name are twins, in either order and in capitals, a name is + no twin of itself, and another letter is no twin. A compatibility form + (`main` and U+00B2, `1` and U+2460) is none: APFS does not fold those, + and NFKC would.""" + assert bootstrap_module._folded_twin(current, branch) is twin + + +def test_the_fold_raises_nothing_on_a_name_that_is_not_utf_8( + bootstrap_module): + """A name held as surrogate escapes (a branch that is not UTF-8, #704) + passes through the NFC step as the text it is.""" + name = f"a{chr(0xDCFF)}b" + assert bootstrap_module._fold(name) == name + assert bootstrap_module._folded_twin(f"refs/heads/A{chr(0xDCFF)}b", + name) is True