tests: one skip helper for names and branches that are not UTF-8, Windows guards on make_file and the --branch a\udcffb test by emulation, and the UTF-8 promise and the leg finding's role pinned (#663)

tests/conftest.py

@@ -25,6 +25,7 @@
 
 from __future__ import annotations
 
+import errno
 import functools
 import importlib.util
 import os
@@ -1068,6 +1069,91 @@ def a_hook_that_unstages(hooks: Path, path: str = PINNED_COPY) -> Path:
     return hooks
 
 
+# --- a file name or a branch that is not UTF-8 (#663) ------------------------
+#
+# A test that needs a file name or a branch holding a byte that is not UTF-8
+# (Python keeps the byte as a lone surrogate, U+DC80 to U+DCFF, which
+# `os.fsencode` gives back as the byte on POSIX) asks these helpers whether
+# this system can carry it, so each system's answer is written in one place
+# and a change to it reaches every test that asks.
+# `tests/test_non_utf8_name_helpers.py` holds them to it, Windows included, by
+# emulating its `os.fsencode`: CI runs the suite on Linux only (#597).
+
+#: The surrogate the helpers ask about: the byte 0xFF, which no UTF-8 text has.
+NOT_UTF_8_PROBE = "\udcff"
+
+
+def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None:
+    """Skip where `text` holds a byte-carrying surrogate (U+DC80 to U+DCFF)
+    and this system's `os.fsencode` makes no single byte of one.
+
+    That is Windows: its `os.fsencode` is UTF-8 with `surrogatepass`, so it
+    writes the three bytes of a sequence that is not UTF-8 for EVERY
+    surrogate, and none of them is a byte of a name. A disk probe cannot be
+    trusted to ask it: CPython hands a str path to `CreateFileW` as UTF-16
+    and NTFS most likely stores a lone surrogate unchecked (read, not run),
+    so the file is made, and git for Windows then sees the name through its
+    own UTF-16 conversion, which is not the byte the test meant. A command
+    line is the same (`CreateProcessW`), which is why `bootstrap.py`'s
+    `_argument_fault` refuses U+DC80 to U+DCFF there. Asked through the `os`
+    module at call time, so a test emulates Windows with
+    `monkeypatch.setattr(os, "fsencode", <a function that gives three bytes>)`.
+    """
+    if (any("\udc80" <= char <= "\udcff" for char in text)
+            and len(os.fsencode(NOT_UTF_8_PROBE)) != 1):
+        pytest.skip("this system's `os.fsencode` makes no byte of a surrogate "
+                    "that stands for a byte that is not UTF-8")
+
+
+def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None:
+    """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not
+    UTF-8 cannot be made under `directory`. Two systems say so:
+
+    - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first,
+      before the disk is touched;
+    - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte
+      sequence"), so no clone can sit at such a path there.
+
+    The second is asked by making a directory `probe<U+DCFF>` under
+    `directory` and removing it; any other error is the test's own and is
+    raised.
+    """
+    probe = directory / f"probe{NOT_UTF_8_PROBE}"
+    skip_where_fsencode_gives_a_surrogate_no_byte(probe.name)
+    try:
+        probe.mkdir()
+    except OSError as exc:
+        if exc.errno != errno.EILSEQ:
+            raise
+        pytest.skip("needs a file name that is not UTF-8, which this "
+                    "filesystem refuses (EILSEQ)")
+    probe.rmdir()
+
+
+def windows_fsencode(text: str | bytes) -> bytes:
+    """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every
+    lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes
+    go through as they are, as `os.fsencode` passes them on every system. For
+    `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the
+    guard above in front of a test that Linux would let through."""
+    if isinstance(text, bytes):
+        return text
+    return text.encode("utf-8", "surrogatepass")
+
+
+def skip_message(call, *args) -> str | None:
+    """The reason `call(*args)` skips the running test with, or None where it
+    returns; any other exception goes on. For a test OF a skip helper: a
+    skip that escapes a test ends it SKIPPED, which no run fails on, so a
+    helper that skipped where it should not would pass `pytest.raises` and
+    every other assertion that expects no skip."""
+    try:
+        call(*args)
+    except pytest.skip.Exception as skipped:
+        return skipped.msg
+    return None
+
+
 # --- the `gh` guard (#296) ---------------------------------------------------
 #
 # NO TEST REACHES THE OPERATOR'S `gh`, and the doctor's `machine` row is still

tests/test_adopt_pins.py

@@ -34,7 +34,6 @@
 
 from __future__ import annotations
 
-import errno
 import importlib.util
 import json
 import os
@@ -50,7 +49,8 @@
 from conftest import (ADOPT, REPO, SCAFFOLD, SYNTHETIC_TREE, WINDOWS_SKIP,
                       as_human, cloned_just_pushed, findings, git,
                       make_source_repo, planned, refused_before_any_leg,
-                      resolve, rmtree, run_script)
+                      resolve, rmtree, run_script,
+                      skip_where_a_name_cannot_be_not_utf_8)
 
 sys.path.insert(0, str(REPO / "scripts"))
 import shape_materialize  # noqa: E402
@@ -890,22 +890,6 @@ def test_a_pin_source_check_would_refuse_is_refused_by_plan(
 UNREAD = "unreadclone"
 
 
-def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None:
-    """Skip, as `POSIX_NAMES` skips on Windows, on a filesystem that refuses
-    a file name that is not UTF-8: macOS answers EILSEQ ("Illegal byte
-    sequence"), so no clone can sit at such a path there. Asked of a
-    directory made under `directory`; any other error is the test's own."""
-    probe = directory / "probe\udcff"
-    try:
-        probe.mkdir()
-    except OSError as exc:
-        if exc.errno != errno.EILSEQ:
-            raise
-        pytest.skip("needs a file name that is not UTF-8, which this "
-                    "filesystem refuses (EILSEQ)")
-    probe.rmdir()
-
-
 @pytest.mark.parametrize("name, reason", [
     pytest.param(f"{UNREAD}‍x",
                  "it holds an invisible formatting character", id="zwj"),

tests/test_bootstrap_branch_bytes.py

@@ -45,7 +45,10 @@
 
 import pytest
 
-from conftest import REPO, git, load_without_bytecode
+from conftest import (REPO, git, load_without_bytecode, skip_message,
+                      skip_where_a_name_cannot_be_not_utf_8,
+                      skip_where_fsencode_gives_a_surrogate_no_byte,
+                      windows_fsencode)
 from test_bootstrap_spelling import (ASSEMBLY_TEMPLATE, DASH, bootstrap_of,
                                      edit, pin12, run)
 from test_stdout_encoding import child_environ, run_tool_bytes
@@ -349,12 +352,12 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made(
     `os.fsencode` carries it, and git is handed that byte, as on origin/main.
     `--branch` is the only way it reaches step (a) with the validators out of
     it (the manifest validator names it, #280's rule for a surrogate). Made on
-    a disk that takes the byte, skipped on one that does not."""
-    probe = tmp_path / "probe-\udcff"
-    try:
-        probe.write_bytes(b"")
-    except (OSError, ValueError):
-        pytest.skip("this disk takes no file name with the byte 0xff")
+    a system and a disk that take the byte, skipped on one that does not: the
+    shared helper asks `os.fsencode` first, which is what Windows answers
+    (NTFS takes the name, and `_argument_fault` refuses the branch there,
+    which `test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused`
+    holds), and then the disk, which macOS refuses (#663)."""
+    skip_where_a_name_cannot_be_not_utf_8(tmp_path)
     code, out, err = run(bootstrap_of(project), "--skip-validators",
                          "--branch", "a\udcffb", encoding=ENCODING,
                          tmp_path=tmp_path, cwd=project)
@@ -370,12 +373,40 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made(
         assert head == b"refs/heads/a\xffb\n", head
 
 
+def test_the_branch_test_skips_where_fsencode_makes_no_byte_of_the_surrogate(
+        project, tmp_path, monkeypatch):
+    """THE WINDOWS GUARD, EMULATED (#663, R455 F1): on NTFS the disk takes
+    `probe\\udcff`, the child is given `--branch a\\udcffb`, and
+    `_argument_fault` refuses U+DCFF where `os.fsencode` writes three bytes,
+    so the `branch a\\udcffb created` line the test expects is absent. With
+    Windows' `os.fsencode` the test is therefore SKIPPED, before it runs
+    bootstrap: it is called here as it is, under the emulation, and the legs
+    stand as they were."""
+    before = legs_state(project)
+    monkeypatch.setattr(os, "fsencode", windows_fsencode)
+    reason = skip_message(
+        test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made,
+        project, tmp_path)
+    assert reason is not None, "the test ran where `os.fsencode` gives no byte"
+    assert "os.fsencode" in reason, reason
+    assert legs_state(project) == before
+
+
 @pytest.fixture(scope="module")
 def bootstrap_module():
     """The template run as a module, no bytecode written beside it."""
     return load_without_bytecode(ASSEMBLY_TEMPLATE, "bootstrap_branch_bytes")
 
 
+@pytest.fixture(scope="module")
+def validator_module():
+    """`validate-manifest.py`, the same template, as a module: the file is a
+    pinned copy, so its functions are read and never changed here."""
+    return load_without_bytecode(
+        REPO / "templates" / "assembly-root" / "scripts"
+        / "validate-manifest.py", "validate_manifest_branch_bytes")
+
+
 @pytest.mark.parametrize("branch", [
     "main", "release/1.2", "feature/日本", "a‍b", "a\xe9b", "a b",
     "a\x1bb", "a..b", "", "\udc80", "a\udcffb"])
@@ -385,12 +416,28 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch):
     the byte-carrying surrogates U+DC80 to U+DCFF are the commands'. On a
     system whose `os.fsencode` does not carry U+DC80 and U+DCFF (Windows) the
     next test is the one that holds."""
-    if any("\udc80" <= char <= "\udcff" for char in branch) and len(
-            os.fsencode("\udcff")) != 1:
-        pytest.skip("this system's `os.fsencode` makes no byte of U+DCFF")
+    skip_where_fsencode_gives_a_surrogate_no_byte(branch)
     assert bootstrap_module._argument_fault(branch) is None
 
 
+@pytest.mark.parametrize("branch, skipped", [
+    pytest.param("a\udcffb", True, id="byte-carrying-surrogate"),
+    pytest.param("\udc80", True, id="the-first-one"),
+    pytest.param("release/1.2", False, id="plain"),
+    pytest.param("a\xe9b", False, id="text"),
+])
+def test_the_no_fault_test_skips_where_fsencode_makes_no_byte_of_the_branch(
+        bootstrap_module, monkeypatch, branch, skipped):
+    """THE WINDOWS GUARD, EMULATED (#663): under Windows' `os.fsencode` the
+    test above skips a branch holding a byte-carrying surrogate, which
+    `_argument_fault` refuses there, and runs every other (the next test
+    holds the refusal)."""
+    monkeypatch.setattr(os, "fsencode", windows_fsencode)
+    reason = skip_message(test_a_branch_the_system_carries_has_no_fault,
+                          bootstrap_module, branch)
+    assert (reason is not None) is skipped, reason
+
+
 @pytest.mark.parametrize("branch, fault", [
     ("a\x00b", "U+0000, a NUL"),
     ("\x00", "U+0000, a NUL"),
@@ -424,13 +471,28 @@ def test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused(
     assert fault.startswith("U+DC"), fault
 
 
-def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module):
+def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module,
+                                                    validator_module):
     """`validate-manifest.py` finds ANY surrogate (#280's rule, which a UTF-8
     file settles), where `bootstrap.py` asks only whether a command can be
     given the value: so every value bootstrap refuses, the validator names."""
-    validator = load_without_bytecode(
-        REPO / "templates" / "assembly-root" / "scripts"
-        / "validate-manifest.py", "validate_manifest_branch_bytes")
     for branch in ("a\x00b", "a\ud83db", "a\ude00", "\udc00"):
         assert bootstrap_module._argument_fault(branch) is not None
-        assert validator._branch_finding("c", "w", branch) != []
+        assert validator_module._branch_finding("c", "w", branch) != []
+
+
+@pytest.mark.parametrize("role", ["spec", "code"])
+def test_the_leg_finding_names_the_leg_by_its_role_and_not_its_path(
+        validator_module, role):
+    """THE NAME IS THE ROLE (#663, R455 F3, mutant R2: naming `path` passed
+    all 54 tests). The templates' roles equal their paths (`spec` is mounted
+    at `spec`), so nothing the scaffold writes can tell the two apart, and
+    the finding is asked of a leg whose path is not its role, as a spec leg
+    mounted at `contracts` is. A unit test on the validator, which this file
+    loads for its last test too: it is a pinned copy and is not edited."""
+    [finding] = validator_module._leg_branch_findings(
+        [{"role": role, "path": "contracts", "branch": "a\x00b"}])
+    assert finding.startswith(
+        f"FINDING manifest-leg-branch: leg '{role}': branch is 'a\\x00b', "
+        "which holds U+0000, a NUL"), finding
+    assert "contracts" not in finding, finding