diff --git a/templates/assembly-root/scripts/bootstrap.py b/templates/assembly-root/scripts/bootstrap.py index a05dab46..7a03472f 100755 --- a/templates/assembly-root/scripts/bootstrap.py +++ b/templates/assembly-root/scripts/bootstrap.py @@ -67,13 +67,21 @@ too, the line gives git's words. Every name git prints back is read as bytes and decoded so that it reaches git again as the bytes it was (#710). The commands it prints for a person to paste (the - `merge --ff-only` that brings a moved-back branch forward, and the - `--continue` and `--abort` of an operation in progress) name the - leg's path and the upstream as ONE shell word each: quoted, and, - where the name holds a byte that is not UTF-8 or a character a - terminal must not be handed, a `printf` that prints its bytes - again, so a paste names what is on disk and not its escape (#932). - Advancing a pin + `merge --ff-only` that brings a moved-back branch forward, the + `--continue` and `--abort` of an operation in progress, and the + `git submodule update --init --` of a leg that is not there) name + the leg's path, and the upstream where there is one, as ONE shell + word each: quoted (a word that begins with `=` too, which zsh would + expand), and, where the name holds a byte that is not UTF-8, a + character a terminal must not be handed, or a character this stdout + cannot write, or writes as other bytes than git is handed, a + `printf` that prints those bytes again, so a paste names what is on + disk and not its escape (#932, #961). The `--init` of a path that + begins with `:`, which git reads as pathspec magic even after the + `--`, names `./:spec`. A name no `printf` can print, one that ends + in a line feed, holds a NUL or has a character git cannot be handed + (a lone surrogate, or one `os.fsencode` cannot write), is printed + spelled, as it was. Advancing a pin remains an explicit commit in this repository — see `README.md`, "The lockstep invariant". This is why `.gitmodules` carries no `branch=`: that would buy the same ergonomics by weakening the pin, and then "what @@ -529,15 +537,23 @@ def _initialized(root: Path, sub: Path, where: str, path: str) -> bool: one, or `_say_unreadable` where the `stat` of its `.git` is refused (#430). `Path.exists()` raised that refusal out of the run up to Python 3.13, a traceback, and from 3.14 answered False, so an initialized leg - in a mount this user cannot search was NOT INITIALIZED.""" + in a mount this user cannot search was NOT INITIALIZED. + + The command is for a person to paste, so the leg's path is ONE word of + it, as `_paste_word` words it, after a `--`: `my spec` was two + pathspecs, `=spec` was expanded by zsh, a byte that is not UTF-8 was + its escape text and `-spec` was an option. The path is a pathspec, so + one that begins with `:` is read as magic even after the `--` + (`:spec` matched nothing): `./` before it makes it the path (#961).""" try: there = _found(sub / ".git") except _Unreadable as unread: _say_unreadable(where, root, unread) return False if not there: + word = _paste_word(f"./{path}" if path.startswith(":") else path) print(f" {where}: NOT INITIALIZED. Run " - f"`git submodule update --init {spelled(path)}` — the " + f"`git submodule update --init -- {word}` — the " f"validators below will refuse until you do.") return there @@ -1247,24 +1263,50 @@ def _sequence_in_progress(gitdir: Path) -> tuple[str, str, str] | None: return SEQUENCER_TODO, "a cherry-pick", "cherry-pick" +def _handed_to_git(text: str) -> bytes: + """The bytes git is handed for `text` as a name (#961): `os.fsencode`'s + on POSIX, which is what `subprocess` passes. On Windows git is handed + UTF-16 and stores UTF-8, which is what Git Bash reads a paste as, so + they are UTF-8 there, and not what `os.fsencode` makes of the text + under `PYTHONLEGACYWINDOWSFSENCODING=1` (`mbcs`, a character the code + page lacks replaced). A name git cannot be handed raises + `UnicodeEncodeError`: a lone surrogate on Windows, which UTF-8 has no + bytes for, and on POSIX one the filesystem encoding cannot write.""" + if os.name == "nt": + return text.encode("utf-8") + return os.fsencode(text) + + def _prints_as_it_is(text: str) -> bool: - """Whether `print` writes `text` as it is (#932): `spelled` has no - escape to make in it, and this stdout holds every character of it, so - what the terminal shows is the text and a paste of it names the same.""" + """Whether `print` writes `text` as it is, and a paste of that gives git + the bytes it is handed (#932, #961): `spelled` has no escape to make in + it, this stdout holds every character of it and, where the text is not + ASCII, writes the bytes git is handed (`_handed_to_git`), so what the + terminal shows is the text and a paste of it names the same.""" + if spelled(text) != text: + return False + encoding = getattr(sys.stdout, "encoding", None) or "utf-8" try: - text.encode(getattr(sys.stdout, "encoding", None) or "utf-8") + written = text.encode(encoding) + return text.isascii() or written == _handed_to_git(text) except (LookupError, UnicodeEncodeError): return False - return spelled(text) == text def _paste_word(name: str) -> str: """`name`, a leg's path or a ref's name, as ONE word of a command line - printed for a person to paste into a POSIX shell (#932). + printed for a person to paste into a POSIX shell (#932, #961). WHERE THE TEXT PRINTS AS IT IS it is `spelled(shlex.quote(name))`, as every such word was: an ordinary name, and a printable one that is not - ASCII, are byte for byte what they were (`_prints_as_it_is`). + ASCII where this stdout writes the bytes git is handed, are byte for + byte what they were (`_prints_as_it_is`). + + ONE CHANGE: a word that begins with `=` is quoted, as + `repo_shape.shell_quoted` quotes it (#486). zsh expands a bare `=ls` to + `/usr/bin/ls` and stops a line at `=x` with `zsh: x not found`, where + dash and bash read both as they are; `=` anywhere else is itself to all + three. `shlex.quote` leaves a word bare only where it holds no `'`. WHAT WAS WRONG. `spelled` writes what a terminal must not be handed as an escape (#391), and a shell reads that text as the characters it is. @@ -1274,8 +1316,8 @@ def _paste_word(name: str) -> str: something we can merge`. A path with a tab, an ESC or a U+2028 in it, or one this stdout cannot write, was printed as its escape too. - ANY OTHER NAME is written as the bytes `os.fsencode` gives it, in a - command substitution that prints them again: + ANY OTHER NAME is written as the bytes git is handed (`_handed_to_git`), + in a command substitution that prints them again: `"$(printf -- 'origin/a\\377b')"`. Each byte outside `PRINTF_BARE` is a three-digit octal escape, which the `printf` of dash, bash and zsh reads, and a dash before 0.5.13 has no `$'...'` for. `--` keeps a name @@ -1284,15 +1326,22 @@ def _paste_word(name: str) -> str: ASCII, so it is the same on any stdout. It is `sh`'s dialect, as `shlex.quote` already was. - A NAME WITH NO SUCH FORM keeps `spelled`'s text, as it was: one whose - bytes do not read back as it (`os.fsdecode` of them is another name: a - lone surrogate with no byte, or a Windows code page that replaces), and - one that ENDS IN A LINE FEED, which `$(...)` strips, so the word would - name another path. A ref's name is neither: git refuses a control - character in one, and `_try_git` reads its bytes back. So it is a leg's - path, and one with no bytes is no directory a leg is moved in; a path - that ends in a line feed, which only a hand-made manifest holds, is the - one name left with no pasteable word. + A NAME WITH NO SUCH FORM keeps `spelled`'s text, as it was: + - one git cannot be handed, which `_handed_to_git` raises on: a lone + surrogate with no byte, or on POSIX a character the filesystem + encoding lacks; + - one whose bytes `$(...)` does not give back, so the word would name + another path: it strips a trailing line feed and drops a NUL. + + The bytes are not read back as the name (#961): git IS handed them, so + a name `os.fsdecode` would read as another has its word, as + `sp\\udcc3\\udca9c` has `"$(printf -- 'sp\\303\\251c')"`, the bytes of the + directory `spéc`. On Windows they are UTF-8 whatever the code page, + which Git Bash reads a paste as. A ref's name is neither: git refuses a + control character in one, and `_try_git` reads its bytes back. So it is + a leg's path, and one with no bytes is no directory a leg is moved in; + a path that ends in a line feed, which only a hand-made manifest holds, + is the one name left with no pasteable word. A COPY, as `inherited_child_environment` is: a root re-copies its pinned files one row at a time, so this file may meet a `repo_shape.py` older @@ -1300,13 +1349,15 @@ def _paste_word(name: str) -> str: `tests/test_bootstrap_pasteable_lines.py` runs each form it writes. """ quoted = shlex.quote(name) + if quoted.startswith("="): + quoted = f"'{quoted}'" if _prints_as_it_is(quoted): return quoted try: - raw = os.fsencode(name) + raw = _handed_to_git(name) except UnicodeEncodeError: return spelled(quoted) - if os.fsdecode(raw) != name or raw.endswith(b"\n"): + if raw.endswith(b"\n") or b"\0" in raw: return spelled(quoted) escaped = "".join(chr(byte) if byte in PRINTF_BARE else f"\\{byte:03o}" for byte in raw) diff --git a/tests/test_assembly_root_unreadable_paths.py b/tests/test_assembly_root_unreadable_paths.py index 16581ca8..02fb1722 100644 --- a/tests/test_assembly_root_unreadable_paths.py +++ b/tests/test_assembly_root_unreadable_paths.py @@ -74,6 +74,9 @@ from conftest import (NO_BYTECODE, REPO, clear_ambient_pin_sources, git, load_without_bytecode, run_script) +from test_bootstrap_non_utf8_branch import ( + NEEDS_A_UTF8_PROCESS, needs_the_byte, +) from test_bump_leg_unopenable import refused_stat from test_manifest_validator import as_a_descendant_by_chain, link_tree from test_validate_pins_named_and_spelled import ( @@ -152,11 +155,14 @@ def nothing_claimed(named: str, label: str = MAYBE, f"be read ({reason})", f" {advice}{AGAIN}"] -def not_initialized(shown: str) -> str: - """Step (a)'s line for a spec leg at `shown` that is not there.""" +def not_initialized(shown: str, word: str | None = None) -> str: + """Step (a)'s line for a spec leg at `shown` that is not there. The + command that makes it one holds the path as ONE shell word, `word` + (`shown` itself where the name needs no quoting), after a `--`, which + keeps a path that begins with `-` from being read as an option (#961).""" return (f" [spec] {shown}: NOT INITIALIZED. Run `git submodule update " - f"--init {shown}` {DASH} the validators below will refuse until " - "you do.") + f"--init -- {shown if word is None else word}` {DASH} the " + "validators below will refuse until you do.") def refusing(monkeypatch, name: str, failure: OSError | None, @@ -634,7 +640,45 @@ def test_a_leg_path_holding_a_nul_is_not_initialized(project): shown = leg_path(project, "spec", "sp\x00ec") result = bootstrap(project, "--skip-validators") assert result.stdout == skipping_validators( - project, [not_initialized(shown), placed(project, "code")], + project, [not_initialized(shown, f"'{shown}'"), + placed(project, "code")], + DEGRADE), result.stdout + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("typed, word", [ + pytest.param("my spec", "'my spec'", id="space"), + pytest.param("=spec", "'=spec'", id="leading-equals"), + pytest.param("-spec", "-spec", id="leading-dash"), + # Git reads a leading `:` as pathspec magic, even after the `--`. + pytest.param(":spec", "./:spec", id="leading-colon"), + # `./` goes before the path, inside the one word: `./':my spec'` is the + # same name to a shell, and is not the word. + pytest.param(":my spec", "'./:my spec'", id="leading-colon-and-space"), + pytest.param(":sp\udcffec", r'''"$(printf -- './:sp\377ec')"''', + id="leading-colon-and-byte-0xff"), + pytest.param("spec ", "'spec '", id="trailing-space"), + pytest.param(" spec", "' spec'", id="leading-space"), + pytest.param("sp\x1bec", r'''"$(printf -- 'sp\033ec')"''', id="escape"), + pytest.param("sp\u2028ec", r'''"$(printf -- 'sp\342\200\250ec')"''', + id="line-separator", marks=NEEDS_A_UTF8_PROCESS), + pytest.param("sp\udcffec", r'''"$(printf -- 'sp\377ec')"''', + id="byte-0xff"), +]) +def test_the_command_that_initializes_a_leg_holds_its_path_as_one_word( + project, tmp_path, typed, word): + """THE ISSUE (#961, rv932's SHOULD 3): the line printed + `--init my spec` (two pathspecs), `--init =spec` (zsh expands it), + `--init sp\\x1bec` and `--init sp\\udcffec` (the escape text, which + names no path) and `--init -spec` (an option). The path is one word, + `_paste_word`'s, after a `--`; `tests/test_bootstrap_pasteable_lines.py` + pastes each into a shell.""" + if "\udcff" in typed: + needs_the_byte(tmp_path) + shown = leg_path(project, "spec", typed) + result = bootstrap(project, "--skip-validators") + assert result.stdout == skipping_validators( + project, [not_initialized(shown, word), placed(project, "code")], DEGRADE), result.stdout assert result.returncode == 0, result.stderr diff --git a/tests/test_bootstrap_after_bump.py b/tests/test_bootstrap_after_bump.py index fa802e53..381566d9 100644 --- a/tests/test_bootstrap_after_bump.py +++ b/tests/test_bootstrap_after_bump.py @@ -534,14 +534,17 @@ def printed() -> str: def test_the_branch_and_its_upstream_are_spelled_in_the_move_back(root): - """The line that moves a branch back names the branch, its upstream and - a command with both in it; each is spelled (#391). The state is built - by hand: a commit past the pin, a remote-tracking ref at it, the branch - tracking that ref, and the leg detached at the pin. Named with the byte - 0xff since #715, as the test above is, and read from `tracking_branch:`; - the line is asked as the escape it must print, not as `spelled` makes - it. Its upstream is read as the system decodes it since #824 (#710), and - HEAD is read here as bytes, which git's answer is.""" + """The line that moves a branch back names the branch and its upstream, + and holds a command with both in it. Each is spelled where the line + names it (#391); the command is not spelled: `_paste_word` words each + for a shell (#932), and a byte that is not UTF-8 is a `printf` that + prints it again. The state is built by hand: a commit past the pin, a + remote-tracking ref at it, the branch tracking that ref, and the leg + detached at the pin. Named with the byte 0xff since #715, as the test + above is, and read from `tracking_branch:`; the line is asked as the + escape it must print, not as `spelled` makes it. Its upstream is read + as the system decodes it since #824 (#710), and HEAD is read here as + bytes, which git's answer is.""" needs_a_byte_name(root.parent) manifest = root / "project.yaml" text = manifest.read_bytes() diff --git a/tests/test_bootstrap_pasteable_lines.py b/tests/test_bootstrap_pasteable_lines.py index 9efa73b4..ccf8bdb6 100644 --- a/tests/test_bootstrap_pasteable_lines.py +++ b/tests/test_bootstrap_pasteable_lines.py @@ -4,24 +4,40 @@ `move_back_to_pin` prints `git -C merge --ff-only `, and `_say_in_progress` a leg's `git -C --continue` and `--abort`. -Both went through `spelled(shlex.quote(...))`. `spelled` writes a byte that +Both went through `spelled(shlex.quote(...))`, and `_initialized` (#961) put +`spelled(path)` in `git submodule update --init `, for a leg that is not +there, with no quote and no `--`. `spelled` writes a byte that is not UTF-8 as #391's `\\udcXX` text, and a control character, a line separator and a bidirectional control as the escape Python spells it, and a shell reads `'origin/a\\udcffb'` as the characters it is. RV824 measured it on #824: for an upstream `origin/a<0xff>b` the line named no ref, and git answered `not something we can merge`. -THE RULE NOW (`_paste_word`). A name that `spelled` has nothing to escape in, -and this stdout can write, is the word it was, byte for byte: `shlex.quote`. -Any other is `"$(printf -- '')"`, which `sh` turns back into -the bytes of the name. A name that has no such word (one that ends in a line -feed, which `$(...)` strips, and one `os.fsencode` cannot write) keeps what -`spelled` made of it. - -EACH TEST BELOW RUNS WHAT WAS PRINTED, AS PRINTED: the command in the line, -through `/bin/sh -c` from the root, and asks where the branch is, or the -operation, afterwards. Asked of the words themselves it would hold only what -the helper says of itself, which the first fix would have changed with it. +THE RULE NOW (`_paste_word`). A name that `spelled` has nothing to escape in +is the word it was, byte for byte (`shlex.quote`), where this stdout can +write it as the bytes git is handed: an ASCII name on any stdout, and one +that is not ASCII where the stdout's codec and the filesystem's give the +same bytes for it. A word that begins with `=` is quoted, as zsh expands a +bare one (#961). Any other is `"$(printf -- '')"`, which `sh` +turns back into the bytes `os.fsencode` gives the name. A name that has no +such word (one that ends in a line feed, which `$(...)` strips, one with a +NUL, which it drops, and one `os.fsencode` cannot write) keeps what +`spelled` made of it. The leg's path in `git submodule update --init -- ` +is such a word too, after a `--`, which keeps a path that begins with `-` +from being read as an option; a path that begins with `:` is `./:spec` there, +as git reads a leading `:` as pathspec magic even after the `--`. The bytes +git is handed are `os.fsencode`'s on POSIX and UTF-8 on Windows, where git is +handed UTF-16 and stores UTF-8, both in the comparison with the stdout and in +the `printf` word. + +THE TESTS THAT TAKE A `root` RUN WHAT WAS PRINTED, AS PRINTED: the command +in the line, through a shell's `-c` from the root, and ask where the branch +is, or the operation, afterwards. Asked of the words themselves it would +hold only what the helper says of itself, which the first fix would have +changed with it. The tests that take `bootstrap_module` ask `_paste_word` +for its words, in process, and where a shell's reading is the question run +the word through that shell; the stdout and the filesystem encoding they +vary are stubs, as `LegacyWindowsOs` is. The state is the one `tests/test_bootstrap_non_utf8_branch.py` builds (a `main` past the pin, with no commit of its own, tracking a remote-tracking @@ -43,6 +59,7 @@ import pytest from conftest import IDENTITY_EMAIL, IDENTITY_NAME, REPO, git, run_script +from repo_shape import shell_quoted from test_bootstrap_after_bump import ( # noqa: F401 MAIN, bootstrapped, head_ref, rev, root, skip_validators, ) @@ -90,11 +107,81 @@ r'''"$(printf -- '-sp\377ec')"''', id="leading-dash"), pytest.param("sp%s\udcffec", r"sp%s\udcffec", r'''"$(printf -- 'sp\045s\377ec')"''', id="percent"), + # Two escapes that are the bytes of a character (`spéc`, U+DCC3 U+DCA9): + # git is handed those bytes, and a `printf` prints them (#961). + pytest.param("sp\udcc3\udca9c", r"sp\udcc3\udca9c", + r'''"$(printf -- 'sp\303\251c')"''', id="UTF-8-as-bytes"), + # zsh expands a bare `=spec` to the path of a command (#961). + pytest.param("=spec", "=spec", "'=spec'", id="leading-equals"), + pytest.param("my spec", "my spec", "'my spec'", id="space"), + # A name is not stripped: its spaces are part of the path (rv973's R-H). + pytest.param("spec ", "spec ", "'spec '", id="trailing-space"), + pytest.param(" spec", " spec", "' spec'", id="leading-space"), + # `git -C -spec` takes the path; a `--init -spec` is an option (#961). + pytest.param("-spec", "-spec", "-spec", id="leading-dash-ASCII"), ] SHELLS = [pytest.param(shell, id=shell.rsplit("/", 1)[1]) for shell in ("/bin/sh", "/bin/dash", "/bin/bash", "/bin/zsh")] +class Stdout: + """The one thing `_prints_as_it_is` reads of `sys.stdout`: its codec.""" + + def __init__(self, encoding: str): + self.encoding = encoding + + +class Latin1Os: + """`os` where the filesystem encoding is latin-1: git is handed `é` as + the byte 0xe9, and each byte reads back as the character of that number. + Only what `bootstrap.py` reads of it differs; everything else is this + system's `os`, as `LegacyWindowsOs`'s is.""" + + @staticmethod + def fsencode(name: str) -> bytes: + return name.encode("latin-1", "surrogateescape") + + @staticmethod + def fsdecode(raw: bytes) -> str: + return raw.decode("latin-1", "surrogateescape") + + def __getattr__(self, attr: str): + return getattr(os, attr) + + +LATIN1_FS = Latin1Os() +#: The `printf` word of `origin/café` as git is handed it under UTF-8. +CAFE_AS_BYTES = r'''"$(printf -- 'origin/caf\303\251')"''' + + +class WindowsOs: + """`os` on Windows in its default mode: `nt`, and a UTF-8 `fsencode`, + which this system's is. Only `name` differs from this system's `os`.""" + + name = "nt" + + def __getattr__(self, attr: str): + return getattr(os, attr) + + +class LegacyMbcsOs(WindowsOs): + """`os` on Windows under `PYTHONLEGACYWINDOWSFSENCODING=1`: `fsencode` + is the code page's (`cp1252` here) with `replace`, so it is not the bytes + git is handed, which are UTF-8, as `LegacyWindowsOs` is for `fsdecode`.""" + + @staticmethod + def fsencode(name: str) -> bytes: + return name.encode("cp1252", "replace") + + @staticmethod + def fsdecode(raw: bytes) -> str: + return raw.decode("cp1252", "replace") + + +WINDOWS_FS = WindowsOs() +LEGACY_MBCS_FS = LegacyMbcsOs() + + def pasted(root: Path, command: str, shell: str = "/bin/sh") -> None: """`command`, as step (a) printed it, run from the root by `shell -c` as a person pastes it, with an editor that keeps the message git offers and @@ -113,6 +200,48 @@ def pasted(root: Path, command: str, shell: str = "/bin/sh") -> None: "replace")) +def emptied_leg_line(root: Path, tmp_path: Path, path: str + ) -> tuple[Path, str]: + """The spec leg mounted at `path` and emptied (`git submodule deinit`), + so that step (a) finds it not there: its directory, and the line step + (a) prints for it (#961).""" + if "\udcff" in path: + needs_the_byte(tmp_path) + leg = mounted_at(root, path) + # `./` before a path that begins with `:`, which is pathspec magic even + # after the `--`. Bytes: git's own line names the path, which may be the + # byte 0xff. + pathspec = f"./{path}" if path.startswith(":") else path + emptied = subprocess.run( + ["git", "submodule", "deinit", "-f", "--", pathspec], + cwd=str(root), capture_output=True, check=False) + assert emptied.returncode == 0, emptied.stderr.decode("utf-8", "replace") + assert not os.path.lexists(leg / ".git"), "the leg is emptied" + return leg, not_initialized_line(root) + + +def not_initialized_line(root: Path) -> str: + """The one line of step (a) that says a leg is NOT INITIALIZED.""" + result = skip_validators(root) + assert result.returncode == 0, result.stderr + result.stdout + lines = [line for line in step_a(result.stdout) + if "NOT INITIALIZED" in line] + assert len(lines) == 1, result.stdout + return lines[0] + + +def printed_by(shell: str, word: str) -> bytes: + """What `shell` prints for `printf %s `: the bytes it reads the + word as. Skips where this system has no such shell.""" + if not os.path.exists(shell): + pytest.skip(f"this system has no {shell}") + proc = subprocess.run([shell, "-c", f"printf %s {word}"], + capture_output=True, check=False) + assert proc.returncode == 0, word + "\n" + proc.stderr.decode( + "utf-8", "replace") + return proc.stdout + + def mounted_at(root: Path, path: str) -> Path: """The spec leg moved to `path`, in the tree and in the manifest, as a hand-written manifest holds it (`tests/test_bootstrap_after_bump.py`'s @@ -206,7 +335,9 @@ def test_the_move_back_command_is_pasteable_whatever_the_legs_path_is_named( and `project.yaml` can hold it), the command names that directory when run as printed; so with a leading `-`, which `printf` would read as an option without its `--`, and a `%`, which it would read as a conversion - without its escape.""" + without its escape. A path made of the escapes of a character's bytes + (`spc`, the directory `spéc`) is that directory too, and + one that begins with `=` is quoted, which zsh would expand (#961).""" pin, ahead, lines = moved_back_lines(root, tmp_path, path=path) command = f"git -C {word} merge --ff-only origin/main" @@ -218,6 +349,19 @@ def test_the_move_back_command_is_pasteable_whatever_the_legs_path_is_named( assert rev(leg, MAIN) == ahead +@pytest.mark.parametrize("shell", SHELLS) +def test_a_leg_path_that_begins_with_an_equals_sign_is_pasteable_in_a_shell( + root, tmp_path, shell): + """THE ISSUE (#961): `git -C =spec merge ...` is read by zsh as the path + of a command called `spec`, and stops at `zsh: spec not found`. Quoted, + the printed command brings `main` forward in each shell, zsh included + (dash and bash read the bare word too, which is why only zsh failed).""" + pin, ahead, lines = moved_back_lines(root, tmp_path, path="=spec") + + pasted(root, command_of(lines[0]), shell) + assert rev(root / "=spec", MAIN) == ahead + + # --- `_say_in_progress`'s two commands -------------------------------------- @pytest.mark.parametrize("which, done", [ @@ -249,6 +393,76 @@ def test_the_commands_that_end_an_operation_are_pasteable_whatever_the_path( check=False).returncode == 1, f"the merge is {done}" +# --- the command that initializes a leg that is not there ------------------- + +@pytest.mark.parametrize("path, label, word", [ + row for row in PATHS if not row.values[0].startswith("-")]) +def test_the_initializing_command_is_pasteable_whatever_the_legs_path_is( + root, tmp_path, path, label, word): + """THE ISSUE (#961, rv932's SHOULD 3): `_initialized` printed `git + submodule update --init {spelled(path)}`, bare, which was two pathspecs + for `my spec`, expanded by zsh for `=spec`, an option for `-spec` and the + escape text for a byte that is not UTF-8. The leg is emptied (`git + submodule deinit`) and step (a) says it is not there; the command it + prints, run as printed from the root, puts the leg back at its pin: + its `.git` is there again, and its HEAD is the commit the root's index + records (the move of the leg is staged, not committed). A path that + begins with `-` is the next test's, as git holds no submodule there.""" + leg, line = emptied_leg_line(root, tmp_path, path) + + assert line == (f" [spec] {label}: NOT INITIALIZED. Run `git submodule " + f"update --init -- {word}` \u2014 the validators below " + "will refuse until you do."), line + pasted(root, command_of(line)) + assert os.path.lexists(leg / ".git"), "the leg is initialized again" + pin = git("rev-parse", f":{path}", cwd=root).stdout.strip() + assert rev(leg) == pin + + +def test_the_initializing_command_reads_a_path_that_begins_with_a_dash( + root): + """Git ignores a `submodule..path` that begins with `-` (it warns + that it may be read as an option), so no command can initialize a leg + there. What `--` changes is what git says of it: that it found no + submodule at the PATH `-spec` (in words that differ by version), where + `--init -spec` is an option (#961).""" + leg = mounted_at(root, "-spec") + (leg / ".git").unlink() + line = not_initialized_line(root) + + assert command_of(line) == "git submodule update --init -- -spec", line + proc = subprocess.run(["/bin/sh", "-c", command_of(line)], cwd=str(root), + capture_output=True, check=False) + assert b"path '-spec'" in proc.stderr, proc.stderr + + +@pytest.mark.parametrize("shell", SHELLS) +def test_the_initializing_command_names_a_path_that_begins_with_a_colon( + root, tmp_path, shell): + """Git reads a word that begins with `:` as pathspec magic, even after + the `--`: `--init -- :spec` is `pathspec ':spec' did not match any + file(s) known to git`, and the leg stays empty. `./` before the path + makes it the path, and the printed command puts the leg back in each + shell (rv973's NIT 1, #961).""" + leg, line = emptied_leg_line(root, tmp_path, ":spec") + + assert command_of(line) == "git submodule update --init -- ./:spec", line + pasted(root, command_of(line), shell) + assert os.path.lexists(leg / ".git"), "the leg is initialized again" + + +@pytest.mark.parametrize("shell", SHELLS) +def test_the_initializing_command_of_an_equals_sign_path_pastes_in_a_shell( + root, tmp_path, shell): + """A leg at `=spec`: zsh expands the bare word of `--init =spec` to the + path of a command called `spec` and stops, so the word is quoted, and + the command puts the leg back in each shell (#961).""" + leg, line = emptied_leg_line(root, tmp_path, "=spec") + + pasted(root, command_of(line), shell) + assert os.path.lexists(leg / ".git"), "the leg is initialized again" + + # --- every ordinary line is what it was ------------------------------------- @pytest.mark.parametrize("name, word", [ @@ -257,7 +471,7 @@ def test_the_commands_that_end_an_operation_are_pasteable_whatever_the_path( ("my spec", "'my spec'"), ("it's", "'it'\"'\"'s'"), ("", "''"), - ("=x", "=x"), + ("a=b", "a=b"), pytest.param("origin/café", "'origin/café'", id="printable-UTF-8"), pytest.param("日本", "'日本'", id="CJK"), pytest.param("a\u00a0b", "'a\u00a0b'", id="no-break-space"), @@ -266,11 +480,42 @@ def test_a_name_with_nothing_to_escape_is_the_word_it_was( bootstrap_module, name, word): """Byte for byte `spelled(shlex.quote(name))`, which every such word was: no `printf`, no new quoting. A U+00A0 is no escape to `spelled`, - and prints as it is.""" + and prints as it is, and an `=` that does not begin the word is itself + to every shell.""" assert bootstrap_module._paste_word(name) == word assert word == bootstrap_module.spelled(shlex.quote(name)) +@pytest.mark.parametrize("name, word", [ + ("=x", "'=x'"), + ("=ls", "'=ls'"), + ("=", "'='"), + ("==x", "'==x'"), + ("=%", "'=%'"), + ("=a b", "'=a b'"), + ("origin/=x", "origin/=x"), +], ids=lambda value: None if value else "empty") +def test_a_word_that_begins_with_an_equals_sign_is_quoted( + bootstrap_module, name, word): + """`shell_quoted` quotes a word that begins with `=` (#486), and so does + `_paste_word` (#961), once: a word `shlex.quote` quotes already is not + quoted again, and an `=` inside a name is left as it was. The word is + `shell_quoted`'s, as the doctor and `family.py` word theirs.""" + assert bootstrap_module._paste_word(name) == word + assert word == shell_quoted(name, "posix") + + +@pytest.mark.parametrize("shell", SHELLS) +@pytest.mark.parametrize("name", ["=x", "=ls", "=", "==x", "=%", "=a b"]) +def test_a_word_that_begins_with_an_equals_sign_reads_back_in_every_shell( + bootstrap_module, shell, name): + """zsh expands a bare `=ls` to `/usr/bin/ls` and stops at `=x`, so what + is asked is the shell: the word prints the name, in dash, bash and zsh + alike (#961).""" + word = bootstrap_module._paste_word(name) + assert printed_by(shell, word) == name.encode("ascii") + + @pytest.mark.parametrize("upstream, word", [ pytest.param("main", "origin/main", id="ASCII"), pytest.param("it's", "'origin/it'\"'\"'s'", id="a-quote"), @@ -292,24 +537,36 @@ def test_the_move_back_line_of_a_printable_upstream_is_the_one_it_was( assert rev(root / "spec", MAIN) == ahead -# --- a stdout that cannot write the name ------------------------------------ +# --- a stdout that does not write the bytes git is handed ------------------- @NEEDS_A_UTF8_PROCESS -def test_a_stdout_that_cannot_write_the_name_gets_the_printf_word( - root, tmp_path): +@pytest.mark.parametrize("encoding, held", [ + pytest.param("ascii", r"origin/caf\xe9", id="cannot-write-it"), + pytest.param("latin-1", "origin/café", id="writes-other-bytes"), +]) +def test_a_stdout_of_another_codec_than_the_names_gets_the_printf_word( + root, tmp_path, encoding, held): """`PYTHONIOENCODING=ascii` writes `é` as `\\xe9`, which no shell - reads back as the name, so the command holds its bytes as `printf` - does; the prose names the upstream as the stdout writes it.""" + reads back as the name, and `latin-1` writes it as the byte 0xe9, where + git was handed 0xc3 0xa9 (#961): pasted, the quoted word named no ref. + So the command holds the bytes git is handed, as `printf` does; the + prose names the upstream as the stdout writes it, read back here in that + codec.""" leg = root / "spec" pin, ahead = past_the_pin(leg, "café", "main") - result = run_script(root / "scripts" / "bootstrap.py", "--skip-validators", - cwd=root, env={"PYTHONIOENCODING": "ascii"}) - assert result.returncode == 0, result.stderr + result.stdout + result = subprocess.run( + [sys.executable, str(root / "scripts" / "bootstrap.py"), + "--skip-validators"], cwd=str(root), stdin=subprocess.DEVNULL, + env={**os.environ, "PYTHONIOENCODING": encoding}, + capture_output=True, check=False) + printed = result.stdout.decode(encoding) + assert result.returncode == 0, printed + result.stderr.decode( + encoding, "replace") word = r'''"$(printf -- 'origin/caf\303\251')"''' command = f"git -C spec merge --ff-only {word}" - line = moved_back_line("spec", ahead, pin, r"origin/caf\xe9", command) - assert line in result.stdout.split("\n"), result.stdout + line = moved_back_line("spec", ahead, pin, held, command) + assert line in printed.split("\n"), printed pasted(root, command_of(line)) assert rev(leg, MAIN) == ahead @@ -328,6 +585,10 @@ def test_a_stdout_that_cannot_write_the_name_gets_the_printf_word( ("a\nb", r'''"$(printf -- 'a\012b')"'''), ("dé\udcff", r'''"$(printf -- 'd\303\251\377')"'''), ("=x\udcff", r'''"$(printf -- '=x\377')"'''), + # Escapes that are the bytes of a character, which `os.fsdecode` reads + # as that character: git is handed those bytes, so they have a word. + ("sp\udcc3\udca9c", r'''"$(printf -- 'sp\303\251c')"'''), + ("caf\udcc3\udca9\udcff", r'''"$(printf -- 'caf\303\251\377')"'''), # An escape is ALWAYS three digits: a digit that follows is the name's. ("a\x1b7\udcff", r'''"$(printf -- 'a\0337\377')"'''), ("\t1\udcff", r'''"$(printf -- '\0111\377')"'''), @@ -337,38 +598,176 @@ def test_the_printf_word_prints_the_bytes_of_the_name( """The word for each byte class, exactly, and what `/bin/sh` makes of it is the bytes `os.fsencode` gives the name: a byte outside the safe set is a three-digit octal escape, `%` and the backslash and the apostrophe - included, and a newline in the middle of a name is kept.""" + included, and a newline in the middle of a name is kept. So are the + escapes of a character's bytes (#961): `os.fsdecode` of them is that + character, which is no reason to keep the escape text.""" assert bootstrap_module._paste_word(name) == word - out = subprocess.run(["/bin/sh", "-c", f"printf %s {word}"], - capture_output=True, check=True).stdout - assert out == os.fsencode(name) + assert printed_by("/bin/sh", word) == os.fsencode(name) -@pytest.mark.parametrize("name", ["a\udcffb\n", "sp\n", "\ud83d"], - ids=["byte-then-LF", "ends-in-LF", "lone-surrogate"]) +@pytest.mark.parametrize("name", ["a\udcffb\n", "sp\n", "\ud83d", "a\x00b"], + ids=["byte-then-LF", "ends-in-LF", "lone-surrogate", + "NUL"]) def test_a_name_with_no_printf_word_keeps_the_text_it_had( bootstrap_module, name): - """`$(...)` strips a trailing line feed, so a word for such a name would - be another path, and a lone surrogate has no byte at all: both keep what - `spelled(shlex.quote(name))` made of them, as before, and are never the + """`$(...)` strips a trailing line feed and drops a NUL, so a word for + such a name would be another path (the NUL's was `ab` in dash and bash, + #961), and a lone surrogate has no byte at all: each keeps what + `spelled(shlex.quote(name))` made of it, as before, and is never the stripped word.""" assert bootstrap_module._paste_word(name) == bootstrap_module.spelled( shlex.quote(name)) -def test_a_name_the_codec_would_not_read_back_keeps_the_text_it_had( - bootstrap_module, monkeypatch): - """The word is the bytes `os.fsencode` gives the name, so it is the name - only where `os.fsdecode` gives the name back from them. Under Windows' - legacy code page (`LegacyWindowsOs`: `fsdecode` replaces what the page - lacks) it does not, and a word would name another path: the name keeps - what `spelled(shlex.quote(name))` made of it.""" - monkeypatch.setattr(bootstrap_module, "os", LegacyWindowsOs()) +@pytest.mark.parametrize("filesystem", [ + pytest.param(LegacyWindowsOs(), id="fsdecode-replaces"), + pytest.param(WINDOWS_FS, id="default-mode"), + pytest.param(LEGACY_MBCS_FS, id="legacy-mode"), +]) +def test_on_windows_a_lone_surrogate_has_no_bytes_and_keeps_the_text_it_had( + bootstrap_module, monkeypatch, filesystem): + """Windows hands git UTF-16 and takes UTF-8 back, and UTF-8 has no bytes + for a lone surrogate, so there is nothing for a `printf` to print: the + name keeps what `spelled(shlex.quote(name))` made of it. It was the + read-back of `os.fsdecode` that kept it (`LegacyWindowsOs`, #944's shim: + it replaces what the page lacks), and under the default mode `os.fsencode` + gave a surrogate three junk bytes, which the read-back let through + (rv973's NIT 2, #961).""" + monkeypatch.setattr(bootstrap_module, "os", filesystem) name = "a\udcffb" assert bootstrap_module._paste_word(name) == bootstrap_module.spelled( shlex.quote(name)) +@pytest.mark.parametrize("encoding, filesystem, word", [ + pytest.param("utf-8", None, "'origin/café'", id="both-UTF-8"), + pytest.param("latin-1", None, CAFE_AS_BYTES, id="stdout-latin-1"), + pytest.param("cp1252", None, CAFE_AS_BYTES, id="stdout-cp1252"), + pytest.param("utf-8", LATIN1_FS, r'''"$(printf -- 'origin/caf\351')"''', + id="filesystem-latin-1"), + pytest.param("latin-1", LATIN1_FS, "'origin/café'", id="both-latin-1"), + pytest.param("cp1252", LATIN1_FS, "'origin/café'", + id="two-codecs-one-byte-for-this-name"), +]) +def test_a_name_that_is_not_ascii_is_quoted_where_both_codecs_give_its_bytes( + bootstrap_module, monkeypatch, encoding, filesystem, word): + """A quoted `é` is what the terminal shows, and a paste of it is the + bytes the stdout's codec made of it: only where those are the bytes + `os.fsencode` gives the name, the ones git is handed, does the paste name + the same. Under a latin-1 stdout and a UTF-8 filesystem (a stdout the + locale does not set, `PYTHONIOENCODING`, a terminal emulator that is + not UTF-8) the word was `'origin/café'`, pasted as 0xe9 where the ref is + 0xc3 0xa9; under a latin-1 filesystem and a UTF-8 stdout it was the + reverse (#961). `cp1252` and `latin-1` are two codecs and one byte for + `é`, and that is all the name asks. The filesystem encoding is the + shim's `os.fsencode`, as `LegacyWindowsOs`'s is.""" + monkeypatch.setattr(sys, "stdout", Stdout(encoding)) + if filesystem is not None: + monkeypatch.setattr(bootstrap_module, "os", filesystem) + assert bootstrap_module._paste_word("origin/café") == word + + +@pytest.mark.parametrize("encoding", ["latin-1", "ascii", "utf-16"]) +def test_an_ascii_name_is_the_word_it_was_on_any_stdout( + bootstrap_module, monkeypatch, encoding): + """An ASCII name shows as itself on every stdout that can write it, so + the filesystem's codec is not asked of it (#961): `origin/main` stays + bare, under a UTF-16 stdout too, whose bytes are not git's.""" + monkeypatch.setattr(sys, "stdout", Stdout(encoding)) + monkeypatch.setattr(bootstrap_module, "os", LATIN1_FS) + assert bootstrap_module._paste_word("origin/main") == "origin/main" + + +@pytest.mark.parametrize("encoding, filesystem, word", [ + pytest.param("utf-8", LEGACY_MBCS_FS, "'origin/café'", + id="legacy-mode-UTF-8-stdout"), + pytest.param("cp1252", WINDOWS_FS, CAFE_AS_BYTES, + id="default-mode-cp1252-stdout"), + pytest.param("utf-8", WINDOWS_FS, "'origin/café'", + id="default-mode-UTF-8-stdout"), +]) +def test_on_windows_the_bytes_git_is_handed_are_utf_8( + bootstrap_module, monkeypatch, encoding, filesystem, word): + """git for Windows is handed UTF-16 and stores UTF-8, and Git Bash reads + a paste as UTF-8, so the stdout is compared with the UTF-8 of the name + and not with `os.fsencode`: under `PYTHONLEGACYWINDOWSFSENCODING=1` that + is `mbcs`, and a UTF-8 stdout got `origin/caf\\351`, a name Git Bash does + not hold, where the quoted word was right (rv973's NIT 1, #961). In + the default mode the two are the same bytes.""" + monkeypatch.setattr(sys, "stdout", Stdout(encoding)) + monkeypatch.setattr(bootstrap_module, "os", filesystem) + assert bootstrap_module._paste_word("origin/café") == word + + +@pytest.mark.parametrize("filesystem, name, word", [ + pytest.param(WINDOWS_FS, "origin/café", CAFE_AS_BYTES, id="default-mode"), + pytest.param(LEGACY_MBCS_FS, "origin/café", CAFE_AS_BYTES, + id="legacy-mode"), + pytest.param(LEGACY_MBCS_FS, "\u20ac5", + r'''"$(printf -- '\342\202\2545')"''', id="legacy-mode-euro"), + pytest.param(LEGACY_MBCS_FS, "日本", + r'''"$(printf -- '\346\227\245\346\234\254')"''', + id="legacy-mode-no-such-character-in-the-code-page"), +]) +def test_on_windows_the_printf_word_is_made_of_utf_8( + bootstrap_module, monkeypatch, filesystem, name, word): + """A stdout that cannot write the name (`ascii`) gets the `printf` word, + and Git Bash reads what it prints as UTF-8: so those are the bytes, and + not what `os.fsencode` makes of the name under + `PYTHONLEGACYWINDOWSFSENCODING=1`, which is `mbcs`: `\\351` for e-acute, + `\\2005` for the euro sign, and `?` for a character the code page + lacks (rv973's NIT 2, #961).""" + monkeypatch.setattr(sys, "stdout", Stdout("ascii")) + monkeypatch.setattr(bootstrap_module, "os", filesystem) + assert bootstrap_module._paste_word(name) == word + + +def test_the_euro_sign_is_quoted_on_a_utf_8_stdout_under_the_legacy_mode( + bootstrap_module, monkeypatch): + """The euro sign is one byte in `cp1252`, 0x80, and three in UTF-8: the + `printf` word of `os.fsencode`'s bytes was `\\2005` for `5`, which + is no name Git Bash holds (rv973's NIT 1, #961).""" + monkeypatch.setattr(sys, "stdout", Stdout("utf-8")) + monkeypatch.setattr(bootstrap_module, "os", LEGACY_MBCS_FS) + assert bootstrap_module._paste_word("\u20ac5") == "'\u20ac5'" + + +def test_a_name_the_filesystem_encoding_cannot_write_keeps_the_text_it_had( + bootstrap_module, monkeypatch): + """`日本` is printable and this stdout writes it, but a latin-1 filesystem + has no bytes for it, so git cannot be handed it: the word is the text it + was, and the comparison of the bytes, asked inside `_prints_as_it_is`'s + `try`, raises nothing (rv973's NIT 2: moved out of it, `os.fsencode` + raised `UnicodeEncodeError`).""" + monkeypatch.setattr(sys, "stdout", Stdout("utf-8")) + monkeypatch.setattr(bootstrap_module, "os", LATIN1_FS) + assert bootstrap_module._paste_word("日本") == "'日本'" + + +def test_a_stdout_with_no_such_codec_gets_the_printf_word( + bootstrap_module, monkeypatch): + """A stream a caller replaced may name any encoding, and `str.encode` + answers an unknown one with a `LookupError`, which is no + `UnicodeEncodeError`: no codec holds the name, so it gets the `printf` + word, which is ASCII, and nothing raises (#961).""" + monkeypatch.setattr(sys, "stdout", Stdout("no-such-codec")) + assert bootstrap_module._paste_word("origin/main") == ( + r'''"$(printf -- 'origin/main')"''') + + +def test_the_printf_word_is_the_bytes_the_filesystem_encoding_gives( + bootstrap_module, monkeypatch): + """Git is handed `os.fsencode`'s bytes, so the word prints those: under + a latin-1 filesystem `é` is the byte 0xe9, and the UTF-8 bytes of the + character (`name.encode("utf-8", "surrogateescape")`) would name another + path, `café`. Run through `/bin/sh`, the word prints what the shim's + `fsencode` gives (#961).""" + monkeypatch.setattr(bootstrap_module, "os", LATIN1_FS) + word = bootstrap_module._paste_word("café\t") + assert word == r'''"$(printf -- 'caf\351\011')"''' + assert printed_by("/bin/sh", word) == b"caf\xe9\t" + + def test_a_stdout_that_is_not_there_is_asked_as_utf8( bootstrap_module, monkeypatch): """`pythonw` has no stdout (`sys.stdout` is None): the word is the one diff --git a/tests/test_bootstrap_spelling.py b/tests/test_bootstrap_spelling.py index ef4ec397..a6f43cbc 100644 --- a/tests/test_bootstrap_spelling.py +++ b/tests/test_bootstrap_spelling.py @@ -374,23 +374,29 @@ def test_a_legs_role_is_spelled(project, tmp_path): assert "\x1b" not in out, out -@pytest.mark.parametrize("path, spelling", [ - (r'"sp\x1bec"', r"sp\x1bec"), - (r'"sp\ud83dec"', r"sp\ud83dec"), - (r'"sp\u2028ec"', r"sp\u2028ec"), +@pytest.mark.parametrize("path, spelling, word", [ + (r'"sp\x1bec"', r"sp\x1bec", r'''"$(printf -- 'sp\033ec')"'''), + (r'"sp\ud83dec"', r"sp\ud83dec", r"'sp\ud83dec'"), + (r'"sp\u2028ec"', r"sp\u2028ec", + r'''"$(printf -- 'sp\342\200\250ec')"'''), ]) def test_a_legs_path_is_spelled_where_it_is_not_initialized( - project, tmp_path, path, spelling): + project, tmp_path, path, spelling, word): """A path the manifest holds that is no directory: step (a) says so, and says what to run, before any validator has had its say. A lone surrogate - here raised from the `print`.""" + here raised from the `print`. The line names the leg spelled; the + command to paste holds the path as `_paste_word` words it (#961): the + bytes as a `printf`, and a lone surrogate, which has none, as its + escape text in quotes. The child runs in UTF-8 mode, so that its + filesystem encoding is the one the words are made of.""" edit(project, "project.yaml", ' path: "spec"', f" path: {path}") code, out, err = run(bootstrap_of(project), "--skip-validators", encoding="utf-8:strict", tmp_path=tmp_path, - cwd=project) - assert code == 0 and "Traceback" not in err, out + err + cwd=project, changes={"PYTHONUTF8": "1"}) + assert code == 0, out + err + assert "Traceback" not in err, out + err assert (f" [spec] {spelling}: NOT INITIALIZED. Run `git submodule update " - f"--init {spelling}` {DASH} the validators below will refuse " + f"--init -- {word}` {DASH} the validators below will refuse " "until you do.") in out.split("\n"), out assert not {"\x1b", "\u2028"} & set(out), out