From d6d1df2340652efe87c0cefbcad4eac8f804ce32 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 14:05:35 +0000 Subject: [PATCH 1/3] conftest.py and four test files: one helper asks whether a system can carry a name or branch that is not UTF-8, `os.fsencode` first and the disk's EILSEQ second, and two pins hold `bump-leg.py`'s UTF-8 decode and the leg finding's role, so a test no longer skips on the disk alone or passes under a decode as ASCII or a finding named by path (#663) #461's five test lines, lane 2's L2-52 and lane 4's R423 N1 and N2 and R455 F1 and F3. Test only: no tool and no template changes, and `validate-manifest.py`, a pinned copy, is read and not edited. - `tests/conftest.py` holds `skip_where_a_name_cannot_be_not_utf_8`, which `tests/test_adopt_pins.py` and `tests/test_pin_source_unreadable.py` each had a copy of. It asks `skip_where_fsencode_gives_a_surrogate_no_byte` first (Windows' `os.fsencode` makes three bytes of U+DCFF, so no name that stands for a byte is carried there, though NTFS would take the file), then the disk (macOS answers EILSEQ); any other error is the test's own. - `make_file` and the `--branch a\udcffb` test call it, so they skip on Windows too, where the disk takes the name and the child then refuses the branch. `test_a_branch_the_system_carries_has_no_fault` asks the same function instead of its own copy of the question. - `test_a_name_that_is_utf8_under_quote_path_false_is_still_refused` asserts that the line naming `caf.yml` holds no literal `\x`, in any locale, so `bump-leg.py` decoding git's bytes as ASCII fails it. - `_leg_branch_findings` is asked of a leg whose role is not its path, so naming the leg by `path` fails it. - `tests/test_non_utf8_name_helpers.py` holds the helpers to this, with Windows' `os.fsencode` and macOS's EILSEQ emulated: CI is Linux only, and no Windows or macOS run was made. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 83 +++++++++++++++++ tests/test_adopt_pins.py | 20 +--- tests/test_bootstrap_branch_bytes.py | 74 ++++++++++++--- tests/test_non_utf8_git_reads.py | 77 +++++++++++++--- tests/test_non_utf8_name_helpers.py | 133 +++++++++++++++++++++++++++ tests/test_pin_source_unreadable.py | 21 +---- 6 files changed, 346 insertions(+), 62 deletions(-) create mode 100644 tests/test_non_utf8_name_helpers.py diff --git a/tests/conftest.py b/tests/conftest.py index 33cbf476..0e9da01d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ from __future__ import annotations +import errno import functools import importlib.util import os @@ -864,6 +865,88 @@ def a_hook_that_unstages(hooks: Path, path: str = PINNED_COPY) -> Path: return hooks +# --- a file name or a branch that is not UTF-8 (#663) ------------------------ +# +# A test that needs a file name or a branch holding a byte that is not UTF-8 +# (Python keeps the byte as a lone surrogate, U+DC80 to U+DCFF, which +# `os.fsencode` gives back as the byte on POSIX) asks these helpers whether +# this system can carry it, so each system's answer is written in one place +# and a change to it reaches every test that asks. +# `tests/test_non_utf8_name_helpers.py` holds them to it, Windows included, by +# emulating its `os.fsencode`: CI runs the suite on Linux only (#597). + +#: The surrogate the helpers ask about: the byte 0xFF, which no UTF-8 text has. +NOT_UTF_8_PROBE = "\udcff" + + +def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None: + """Skip where `text` holds a byte-carrying surrogate (U+DC80 to U+DCFF) + and this system's `os.fsencode` makes no single byte of one. + + That is Windows: its `os.fsencode` is UTF-8 with `surrogatepass`, so it + writes the three bytes of a sequence that is not UTF-8 for EVERY + surrogate, and none of them is a byte of a name. A disk probe cannot be + trusted to ask it: CPython hands a str path to `CreateFileW` as UTF-16 + and NTFS most likely stores a lone surrogate unchecked (read, not run), + so the file is made, and git for Windows then sees the name through its + own UTF-16 conversion, which is not the byte the test meant. A command + line is the same (`CreateProcessW`), which is why `bootstrap.py`'s + `_argument_fault` refuses U+DC80 to U+DCFF there. Asked through the `os` + module at call time, so a test emulates Windows with + `monkeypatch.setattr(os, "fsencode", )`. + """ + if (any("\udc80" <= char <= "\udcff" for char in text) + and len(os.fsencode(NOT_UTF_8_PROBE)) != 1): + pytest.skip("this system's `os.fsencode` makes no byte of a surrogate " + "that stands for a byte that is not UTF-8") + + +def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: + """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not + UTF-8 cannot be made under `directory`. Two systems say so: + + - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first, + before the disk is touched; + - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte + sequence"), so no clone can sit at such a path there. + + The second is asked by making a directory `probe` under + `directory` and removing it; any other error is the test's own and is + raised. + """ + probe = directory / f"probe{NOT_UTF_8_PROBE}" + skip_where_fsencode_gives_a_surrogate_no_byte(probe.name) + try: + probe.mkdir() + except OSError as exc: + if exc.errno != errno.EILSEQ: + raise + pytest.skip("needs a file name that is not UTF-8, which this " + "filesystem refuses (EILSEQ)") + probe.rmdir() + + +def windows_fsencode(text: str) -> bytes: + """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every + lone surrogate is the three bytes of a sequence that is not UTF-8. For + `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the + guard above in front of a test that Linux would let through.""" + return text.encode("utf-8", "surrogatepass") + + +def skip_message(call, *args) -> str | None: + """The reason `call(*args)` skips the running test with, or None where it + returns; any other exception goes on. For a test OF a skip helper: a + skip that escapes a test ends it SKIPPED, which no run fails on, so a + helper that skipped where it should not would pass `pytest.raises` and + every other assertion that expects no skip.""" + try: + call(*args) + except pytest.skip.Exception as skipped: + return skipped.msg + return None + + # --- the `gh` guard (#296) --------------------------------------------------- # # NO TEST REACHES THE OPERATOR'S `gh`, and the doctor's `machine` row is still diff --git a/tests/test_adopt_pins.py b/tests/test_adopt_pins.py index 719af0f7..7b3bb313 100644 --- a/tests/test_adopt_pins.py +++ b/tests/test_adopt_pins.py @@ -34,7 +34,6 @@ from __future__ import annotations -import errno import importlib.util import json import os @@ -50,7 +49,8 @@ from conftest import (ADOPT, REPO, SCAFFOLD, SYNTHETIC_TREE, WINDOWS_SKIP, as_human, cloned_just_pushed, findings, git, make_source_repo, planned, refused_before_any_leg, - resolve, rmtree, run_script) + resolve, rmtree, run_script, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -855,22 +855,6 @@ def test_a_pin_source_check_would_refuse_is_refused_by_plan( UNREAD = "unreadclone" -def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: - """Skip, as `POSIX_NAMES` skips on Windows, on a filesystem that refuses - a file name that is not UTF-8: macOS answers EILSEQ ("Illegal byte - sequence"), so no clone can sit at such a path there. Asked of a - directory made under `directory`; any other error is the test's own.""" - probe = directory / "probe\udcff" - try: - probe.mkdir() - except OSError as exc: - if exc.errno != errno.EILSEQ: - raise - pytest.skip("needs a file name that is not UTF-8, which this " - "filesystem refuses (EILSEQ)") - probe.rmdir() - - @pytest.mark.parametrize("name, reason", [ pytest.param(f"{UNREAD}‍x", "it holds an invisible formatting character", id="zwj"), diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index bab82102..3f63761e 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -45,7 +45,10 @@ import pytest -from conftest import REPO, git, load_without_bytecode +from conftest import (REPO, git, load_without_bytecode, skip_message, + skip_where_a_name_cannot_be_not_utf_8, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) from test_bootstrap_spelling import (ASSEMBLY_TEMPLATE, DASH, bootstrap_of, edit, pin12, run) from test_stdout_encoding import child_environ, run_tool_bytes @@ -349,12 +352,12 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made( `os.fsencode` carries it, and git is handed that byte, as on origin/main. `--branch` is the only way it reaches step (a) with the validators out of it (the manifest validator names it, #280's rule for a surrogate). Made on - a disk that takes the byte, skipped on one that does not.""" - probe = tmp_path / "probe-\udcff" - try: - probe.write_bytes(b"") - except (OSError, ValueError): - pytest.skip("this disk takes no file name with the byte 0xff") + a system and a disk that take the byte, skipped on one that does not: the + shared helper asks `os.fsencode` first, which is what Windows answers + (NTFS takes the name, and `_argument_fault` refuses the branch there, + which `test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused` + holds), and then the disk, which macOS refuses (#663).""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) code, out, err = run(bootstrap_of(project), "--skip-validators", "--branch", "a\udcffb", encoding=ENCODING, tmp_path=tmp_path, cwd=project) @@ -370,12 +373,40 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made( assert head == b"refs/heads/a\xffb\n", head +def test_the_branch_test_skips_where_fsencode_makes_no_byte_of_the_surrogate( + project, tmp_path, monkeypatch): + """THE WINDOWS GUARD, EMULATED (#663, R455 F1): on NTFS the disk takes + `probe\\udcff`, the child is given `--branch a\\udcffb`, and + `_argument_fault` refuses U+DCFF where `os.fsencode` writes three bytes, + so the `branch a\\udcffb created` line the test expects is absent. With + Windows' `os.fsencode` the test is therefore SKIPPED, before it runs + bootstrap: it is called here as it is, under the emulation, and the legs + stand as they were.""" + before = legs_state(project) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message( + test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made, + project, tmp_path) + assert reason is not None, "the test ran where `os.fsencode` gives no byte" + assert "os.fsencode" in reason, reason + assert legs_state(project) == before + + @pytest.fixture(scope="module") def bootstrap_module(): """The template run as a module, no bytecode written beside it.""" return load_without_bytecode(ASSEMBLY_TEMPLATE, "bootstrap_branch_bytes") +@pytest.fixture(scope="module") +def validator_module(): + """`validate-manifest.py`, the same template, as a module: the file is a + pinned copy, so its functions are read and never changed here.""" + return load_without_bytecode( + REPO / "templates" / "assembly-root" / "scripts" + / "validate-manifest.py", "validate_manifest_branch_bytes") + + @pytest.mark.parametrize("branch", [ "main", "release/1.2", "feature/日本", "a‍b", "a\xe9b", "a b", "a\x1bb", "a..b", "", "\udc80", "a\udcffb"]) @@ -385,9 +416,7 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): the byte-carrying surrogates U+DC80 to U+DCFF are the commands'. On a system whose `os.fsencode` does not carry U+DC80 and U+DCFF (Windows) the next test is the one that holds.""" - if any("\udc80" <= char <= "\udcff" for char in branch) and len( - os.fsencode("\udcff")) != 1: - pytest.skip("this system's `os.fsencode` makes no byte of U+DCFF") + skip_where_fsencode_gives_a_surrogate_no_byte(branch) assert bootstrap_module._argument_fault(branch) is None @@ -424,13 +453,28 @@ def test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused( assert fault.startswith("U+DC"), fault -def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module): +def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module, + validator_module): """`validate-manifest.py` finds ANY surrogate (#280's rule, which a UTF-8 file settles), where `bootstrap.py` asks only whether a command can be given the value: so every value bootstrap refuses, the validator names.""" - validator = load_without_bytecode( - REPO / "templates" / "assembly-root" / "scripts" - / "validate-manifest.py", "validate_manifest_branch_bytes") for branch in ("a\x00b", "a\ud83db", "a\ude00", "\udc00"): assert bootstrap_module._argument_fault(branch) is not None - assert validator._branch_finding("c", "w", branch) != [] + assert validator_module._branch_finding("c", "w", branch) != [] + + +@pytest.mark.parametrize("role", ["spec", "code"]) +def test_the_leg_finding_names_the_leg_by_its_role_and_not_its_path( + validator_module, role): + """THE NAME IS THE ROLE (#663, R455 F3, mutant R2: naming `path` passed + all 54 tests). The templates' roles equal their paths (`spec` is mounted + at `spec`), so nothing the scaffold writes can tell the two apart, and + the finding is asked of a leg whose path is not its role, as a spec leg + mounted at `contracts` is. A unit test on the validator, which this file + loads for its last test too: it is a pinned copy and is not edited.""" + [finding] = validator_module._leg_branch_findings( + [{"role": role, "path": "contracts", "branch": "a\x00b"}]) + assert finding.startswith( + f"FINDING manifest-leg-branch: leg '{role}': branch is 'a\\x00b', " + "which holds U+0000, a NUL"), finding + assert "contracts" not in finding, finding diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index c79bada2..18fbb7e9 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -32,11 +32,13 @@ NO NETWORK AND NO GITHUB. The scaffold and `family.py init` here are run as child processes into LOCAL bare remotes (`--local-remote-dir`), which never calls `gh`. A name that is not UTF-8 is made by `os.fsdecode` of its bytes, so -a POSIX filesystem takes it as it is; where the disk refuses (macOS, Windows) -the test skips, and where the ENVIRONMENT cannot carry a byte (Windows') the -`user.name` rows skip. The checkout under test for `_shape_revision` and -`_shape_snapshot` is a COPY of the standard in a repository of its own -(`standard`), made dirty one test at a time, never this checkout. +a POSIX filesystem takes it as it is; where the system cannot carry the byte +the test skips (`make_file`: Windows' `os.fsencode` makes no byte of it, and +macOS's disk refuses it), and where the ENVIRONMENT cannot carry a byte +(Windows') the `user.name` rows skip. The checkout under test for +`_shape_revision` and `_shape_snapshot` is a COPY of the standard in a +repository of its own (`standard`), made dirty one test at a time, never this +checkout. """ from __future__ import annotations @@ -49,7 +51,10 @@ import pytest -from conftest import ORG, PROJECT, REPO, cloned_just_pushed, git, run_script +from conftest import (ORG, PROJECT, REPO, cloned_just_pushed, git, run_script, + skip_message, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) from test_bump_leg import branched, bump sys.path.insert(0, str(REPO / "scripts")) @@ -85,7 +90,11 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: """Write `data` at `path`, or skip: a name that is not UTF-8 needs a - filesystem that stores bytes (Linux does; APFS and NTFS refuse it).""" + system that carries the byte and a filesystem that stores it (Linux does; + APFS refuses it, and on Windows `os.fsencode` makes no byte of it, which + is asked FIRST: NTFS most likely takes the surrogate, and the file would + then not be the one the test means). A name that is UTF-8 is written.""" + skip_where_fsencode_gives_a_surrogate_no_byte(path.name) try: path.write_bytes(data) except (OSError, UnicodeError) as exc: @@ -93,6 +102,47 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: f"UTF-8: {exc}") +def test_make_file_skips_where_fsencode_makes_no_byte_and_writes_nothing( + tmp_path, monkeypatch): + """THE WINDOWS GUARD, EMULATED (#663, R423 N1): where `os.fsencode` gives + a surrogate three bytes, NTFS most likely takes `caf\\udce9.yml` anyway + and git for Windows would see another name, so `make_file` skips BEFORE it + writes, and nothing is on the disk. Linux takes the name, so a file + standing there would be the failure.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + target = tmp_path / NOT_UTF8 + reason = skip_message(make_file, target) + assert reason is not None, "make_file wrote the name instead of skipping" + assert "os.fsencode" in reason, reason + assert not target.exists(), "the file was written before the skip" + + +def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( + tmp_path, monkeypatch): + """The guard is for the byte and no wider: Windows holds `caf\\u00e9.yml` + as text, and the tests that use it must run there.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + target = tmp_path / UTF8 + assert skip_message(make_file, target, b"on: [pull_request]\n") is None + assert target.read_bytes() == b"on: [pull_request]\n" + + +def test_make_file_writes_the_name_that_is_not_utf8_where_the_system_has_it( + tmp_path): + """The control: where `os.fsencode` carries the byte and the disk takes it + (Linux), the file is written under the name that is not UTF-8, so the + guard skipped nothing.""" + if len(os.fsencode("\udce9")) != 1: + pytest.skip("this system's `os.fsencode` makes no byte of U+DCE9") + target = tmp_path / NOT_UTF8 + reason = skip_message(make_file, target, b"on: [push]\n") + if reason is not None and "filesystem" in reason: + pytest.skip(reason) + assert reason is None, reason + assert os.fsencode(target.name) == b"caf\xe9.yml" + assert target.read_bytes() == b"on: [push]\n" + + def git_bytes(*args: str, cwd: Path) -> bytes: """`git ` in `cwd`, stdout as the bytes git wrote. `conftest.git` decodes as text, which is the failure under test.""" @@ -205,16 +255,21 @@ def test_a_name_that_is_utf8_under_quote_path_false_is_still_refused( Its bytes decode, and it was refused on main; it still is, and the refusal still prints it as git did. (The name is matched on its ASCII ends: a child's stderr is decoded by the locale here, which is not UTF-8 - on every platform.)""" + on every platform. What the middle of the name is NOT is held instead, in + any locale: the tool decodes git's bytes as UTF-8, so `\\u00e9` is one + character, where a decode as ASCII would show its two bytes as `\\xc3\\xa9`, + and a locale's decode of what the child printed is mojibake at worst and + never a `\\x`: #423's mutant R1, #663.)""" dirty_root(project, UTF8, quote_path=False) before = found(project) result = bump(project, scaffolded, advanced, *extra) output = result.stderr + result.stdout assert result.returncode == 2, output assert "REFUSED bump-leg-root-dirty: " in result.stderr, output - assert any(line.startswith(" M .github/workflows/caf") - and line.endswith(".yml") - for line in result.stderr.splitlines()), output + [shown] = [line for line in result.stderr.splitlines() + if line.startswith(" M .github/workflows/caf")] + assert shown.endswith(".yml"), output + assert "\\x" not in shown, output assert "Traceback" not in output, output assert found(project) == before diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py new file mode 100644 index 00000000..8c387eee --- /dev/null +++ b/tests/test_non_utf8_name_helpers.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The two helpers a test asks whether a name that is not UTF-8 can be +carried here (#663, #461's L2-52). + +`skip_where_a_name_cannot_be_not_utf_8` was two helpers with one body +(`tests/test_adopt_pins.py` and `tests/test_pin_source_unreadable.py`), and +`make_file` and the `--branch a\\udcffb` test each asked a smaller question of +their own, or none: on Windows the disk takes the name, so no probe of the +disk skips there. Now `tests/conftest.py` holds both questions once, the +system's first (`os.fsencode`) and the disk's second (EILSEQ), and this file +holds them to it. + +CI runs the suite on Linux only (#597), so what Windows answers is EMULATED: +`os.fsencode` is replaced for one test by `windows_fsencode`, which gives a +surrogate three bytes, as Windows' does. No Windows run was made. macOS's +EILSEQ is emulated the same way, by a `mkdir` that answers it. +""" + +from __future__ import annotations + +import errno +import os +import sys +from pathlib import Path + +import pytest + +from conftest import (NOT_UTF_8_PROBE, skip_message, + skip_where_a_name_cannot_be_not_utf_8, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) + +LINUX_ONLY = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="Linux stores any byte in a file name; the other systems are " + "asked by the tests below that emulate them") + + +def mkdir_answers(code: int, said: str): + """A `Path.mkdir` that fails as the disk would with `code`.""" + def refuse(self, *args, **kwargs): + raise OSError(code, said, str(self)) + return refuse + + +# --- the system's question: what `os.fsencode` makes of the surrogate -------- + +@LINUX_ONLY +def test_a_system_that_carries_the_byte_is_not_skipped(): + """The control: Linux's `os.fsencode` gives U+DCFF back as the one byte + 0xFF, so no text is skipped.""" + assert os.fsencode(NOT_UTF_8_PROBE) == b"\xff" + assert skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + f"caf{NOT_UTF_8_PROBE}.yml") is None + + +def test_where_fsencode_makes_three_bytes_of_the_surrogate_the_name_is_skipped( + monkeypatch): + """Windows' `os.fsencode` (emulated): U+DCFF is the three bytes of a + sequence that is not UTF-8, none of them 0xFF, so a name that holds a + byte-carrying surrogate is skipped, and each one from U+DC80 to U+DCFF.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + assert len(os.fsencode(NOT_UTF_8_PROBE)) == 3 + for surrogate in ("\udc80", "\udce9", "\udcff"): + reason = skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + f"a{surrogate}b") + assert reason is not None, f"U+{ord(surrogate):04X} was not skipped" + assert "os.fsencode" in reason, reason + + +@pytest.mark.parametrize("text", [ + "plain", "café.yml", "feature/日本", "", "a\ud83db"], + ids=["ascii", "text", "cjk", "empty", "surrogate-without-a-byte"]) +def test_where_fsencode_makes_three_bytes_a_name_that_is_text_is_not_skipped( + monkeypatch, text): + """The guard is for the surrogates that stand for a byte (U+DC80 to + U+DCFF) and no wider: text is the same on every system, and a surrogate + that stands for no byte is not this helper's to judge (`_argument_fault` + refuses it, as `bootstrap.py`'s tests hold).""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + assert skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + text) is None + + +# --- the shared helper: the system, then the disk ---------------------------- + +@LINUX_ONLY +def test_a_disk_that_takes_the_name_is_not_skipped_and_the_probe_is_removed( + tmp_path): + """The control: no skip, and nothing is left in the directory.""" + assert skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path) is None + assert list(tmp_path.iterdir()) == [] + + +def test_where_fsencode_makes_three_bytes_the_disk_is_not_asked( + tmp_path, monkeypatch): + """THE SYSTEM IS ASKED FIRST (#663): on Windows the disk would take the + name, so a skip that waited for the disk would never come. The directory + does not exist, so an `mkdir` under it would be FileNotFoundError and not + a skip: the skip is the answer, and the disk was not touched.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path / "absent") + assert reason is not None, "the helper went on to ask the disk" + assert "os.fsencode" in reason, reason + + +def test_a_disk_that_answers_eilseq_is_skipped(tmp_path, monkeypatch): + """macOS (emulated): APFS answers EILSEQ, `Illegal byte sequence`, to a + file name that is not UTF-8, and no clone can sit at such a path.""" + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + monkeypatch.setattr(Path, "mkdir", + mkdir_answers(errno.EILSEQ, "Illegal byte sequence")) + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) + assert reason is not None, "EILSEQ was not a skip" + assert "EILSEQ" in reason, reason + + +@pytest.mark.parametrize("code", [errno.EACCES, errno.ENOSPC, errno.EEXIST], + ids=["EACCES", "ENOSPC", "EEXIST"]) +def test_any_other_error_from_the_disk_is_the_tests_own( + tmp_path, monkeypatch, code): + """Only EILSEQ is the filesystem's refusal of the name: a full disk, a + folder nobody can write in or a probe left by an earlier run is a + failure of the test, raised as it was, and never a skip that hides it.""" + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + monkeypatch.setattr(Path, "mkdir", mkdir_answers(code, os.strerror(code))) + with pytest.raises(OSError) as raised: + skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) + assert raised.value.errno == code diff --git a/tests/test_pin_source_unreadable.py b/tests/test_pin_source_unreadable.py index 177e8c94..c43a41ad 100644 --- a/tests/test_pin_source_unreadable.py +++ b/tests/test_pin_source_unreadable.py @@ -41,7 +41,8 @@ import pytest -from conftest import ADOPT, ORG, REPO, SCAFFOLD, git, make_source_repo, run_script +from conftest import (ADOPT, ORG, REPO, SCAFFOLD, git, make_source_repo, + run_script, skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) from repo_shape import load_yaml, parse_yaml, tree_digest # noqa: E402 @@ -592,22 +593,6 @@ def test_a_typed_dot_is_still_the_directory_the_tool_runs_in( ] -def skip_where_no_file_name_is_not_utf_8(directory: Path) -> None: - """Skip, as a Windows command line skips, where the filesystem refuses - a file name that is not UTF-8, so that no clone can sit at one: macOS - answers EILSEQ ("Illegal byte sequence"). Asked by making one under - `directory`; any other error is the test's to report.""" - probe = directory / "probe\udcff" - try: - probe.mkdir() - except OSError as exc: - if exc.errno != errno.EILSEQ: - raise - pytest.skip("needs a file name that is not UTF-8, which this " - "filesystem refuses (EILSEQ)") - probe.rmdir() - - @pytest.mark.parametrize("name", CLONE_NAMES) def test_the_pin_file_says_how_the_digest_was_read_and_not_where( name, tmp_path, adoptable, glass): @@ -618,7 +603,7 @@ def test_the_pin_file_says_how_the_digest_was_read_and_not_where( --all`, read as bytes).""" product, commit = glass if "\udcff" in name: - skip_where_no_file_name_is_not_utf_8(tmp_path) + skip_where_a_name_cannot_be_not_utf_8(tmp_path) clones = tmp_path / "clones" shutil.copytree(product, clones / name, symlinks=True) result = run("scaffold-run", tmp_path, adoptable, str(clones / name), From 0b1d1ddbeeca662500534f58c39e5b5a0c67a70d Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 14:15:43 +0000 Subject: [PATCH 2/3] test_bootstrap_branch_bytes.py: the no-fault test's Windows guard is held by an emulation, so dropping the shared helper from it fails a test on Linux too (#663) `test_a_branch_the_system_carries_has_no_fault` now asks `skip_where_fsencode_gives_a_surrogate_no_byte` where it had its own copy of the question. The new test calls it under Windows' `os.fsencode` (emulated): a branch holding a byte-carrying surrogate is skipped, plain text and text with an accent run. Removing the call fails the two skipped rows. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_bootstrap_branch_bytes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index 3f63761e..5a79ff8e 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -420,6 +420,24 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): assert bootstrap_module._argument_fault(branch) is None +@pytest.mark.parametrize("branch, skipped", [ + pytest.param("a\udcffb", True, id="byte-carrying-surrogate"), + pytest.param("\udc80", True, id="the-first-one"), + pytest.param("release/1.2", False, id="plain"), + pytest.param("a\xe9b", False, id="text"), +]) +def test_the_no_fault_test_skips_where_fsencode_makes_no_byte_of_the_branch( + bootstrap_module, monkeypatch, branch, skipped): + """THE WINDOWS GUARD, EMULATED (#663): under Windows' `os.fsencode` the + test above skips a branch holding a byte-carrying surrogate, which + `_argument_fault` refuses there, and runs every other (the next test + holds the refusal).""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(test_a_branch_the_system_carries_has_no_fault, + bootstrap_module, branch) + assert (reason is not None) is skipped, reason + + @pytest.mark.parametrize("branch, fault", [ ("a\x00b", "U+0000, a NUL"), ("\x00", "U+0000, a NUL"), From 6ba8c5d2d6fec3a779963713593e5377c76bbadf Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 18:40:04 +0000 Subject: [PATCH 3/3] conftest.py and two test files: `skip_message` is held by a test that fails where a skip escapes it, `windows_fsencode` passes bytes through as `os.fsencode` does, and the no-`\x` check skips where the file name encoding is not UTF-8, so a leaked skip, an emulation that breaks on bytes and a Latin-1 host no longer pass silently or fail on a tool that is right (#663) The review of #723 (0b1d1ddb) found no MUST and three things to take. SHOULD-1: `skip_message` had no test of its own. A skip that escapes it ends the test SKIPPED, and no run fails on a skip, so the mutant where one escapes gave `10 passed, 3 skipped` on the helpers file and seven emulation tests silently skipped. `tests/test_non_utf8_name_helpers.py` now holds it with three tests: it returns the reason a call skips with, it returns None where the call does not skip, and it lets any other exception go on. The first catches the escape and fails it with `pytest.fail`, because a bare assert on an escaped skip would end that test SKIPPED too. NIT-2: `windows_fsencode` took `str` only. `os.fsencode` passes bytes through on every system, so the emulation now does, and two tests hold it (the surrogate and text, and the bytes, compared with `os.fsencode` itself). NIT-1: on a Latin-1 file name encoding `make_file` writes `caf\xe9.yml` as the single byte 0xE9, the tool prints `caf\xe9.yml`, and the new `"\\x" not in shown` assertion would fail although the tool is right. It is now `assert_shown_without_hex_escapes`, which skips where `os.fsencode(UTF8) != UTF8.encode("utf-8")` and is called after the test's other assertions, so a skip loses none of them. Two tests hold it under an emulated Latin-1 `os.fsencode` (it skips) and Windows' (it asserts, and fails a line that holds a `\x`). Tests only; no tool and no template changes. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 7 +++-- tests/test_non_utf8_git_reads.py | 44 ++++++++++++++++++++++++++- tests/test_non_utf8_name_helpers.py | 47 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0e9da01d..a5313d72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -926,11 +926,14 @@ def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: probe.rmdir() -def windows_fsencode(text: str) -> bytes: +def windows_fsencode(text: str | bytes) -> bytes: """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every - lone surrogate is the three bytes of a sequence that is not UTF-8. For + lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes + go through as they are, as `os.fsencode` passes them on every system. For `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the guard above in front of a test that Linux would let through.""" + if isinstance(text, bytes): + return text return text.encode("utf-8", "surrogatepass") diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index 18fbb7e9..4d19a998 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -143,6 +143,48 @@ def test_make_file_writes_the_name_that_is_not_utf8_where_the_system_has_it( assert target.read_bytes() == b"on: [push]\n" +def assert_shown_without_hex_escapes(shown: str, output: str) -> None: + """Assert that the refusal's line for `caf\u00e9.yml` holds no literal + `\\x`, or skip where this system's file name encoding is not UTF-8. + + On a Latin-1 host `make_file` writes the one byte 0xE9 for it, and the + tool, which decodes git's bytes as UTF-8, rightly prints `caf\\xe9.yml`: + the assertion would fail though the tool is right (#663, NIT-1). So it + is asked after the system's encoding is, and a system whose encoding is + not UTF-8 skips. (Where `make_file` could not write the name at all, an + ASCII host, it skipped already.)""" + if os.fsencode(UTF8) != UTF8.encode("utf-8"): + pytest.skip("this system's file name encoding is not UTF-8, so a " + "UTF-8 name is not written as the bytes the tool decodes") + assert "\\x" not in shown, output + + +def test_the_hex_escape_check_skips_where_the_file_name_encoding_is_not_utf_8( + monkeypatch): + """THE LATIN-1 HOST, EMULATED (#663, NIT-1): `os.fsencode` gives the name + `caf\u00e9.yml` as the one byte 0xE9, the line holds a `\\x`, and the + check skips, where it would otherwise fail on a tool that is right.""" + monkeypatch.setattr(os, "fsencode", lambda name: name.encode("latin-1")) + reason = skip_message(assert_shown_without_hex_escapes, + r" M .github/workflows/caf\xe9.yml", "output") + assert reason is not None, "the check asserted instead of skipping" + assert "UTF-8" in reason, reason + + +def test_the_hex_escape_check_asserts_where_the_file_name_encoding_is_utf_8( + monkeypatch): + """The control: Windows' `os.fsencode` (emulated), like Linux's and + macOS's, is UTF-8 for text, so the check is not skipped, and it still + fails a line that holds a `\\x` and passes one that does not.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(assert_shown_without_hex_escapes, + " M .github/workflows/caf\u00e9.yml", "output") + assert reason is None, reason + with pytest.raises(AssertionError): + skip_message(assert_shown_without_hex_escapes, + r" M .github/workflows/caf\xc3\xa9.yml", "output") + + def git_bytes(*args: str, cwd: Path) -> bytes: """`git ` in `cwd`, stdout as the bytes git wrote. `conftest.git` decodes as text, which is the failure under test.""" @@ -269,9 +311,9 @@ def test_a_name_that_is_utf8_under_quote_path_false_is_still_refused( [shown] = [line for line in result.stderr.splitlines() if line.startswith(" M .github/workflows/caf")] assert shown.endswith(".yml"), output - assert "\\x" not in shown, output assert "Traceback" not in output, output assert found(project) == before + assert_shown_without_hex_escapes(shown, output) def test_the_refusal_names_every_dirty_file_whatever_its_bytes( diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 8c387eee..47ae0d13 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -14,6 +14,10 @@ `os.fsencode` is replaced for one test by `windows_fsencode`, which gives a surrogate three bytes, as Windows' does. No Windows run was made. macOS's EILSEQ is emulated the same way, by a `mkdir` that answers it. + +The two tools the tests below are written with are held here too: +`skip_message`, which every one of them reads a skip with, and +`windows_fsencode`, the emulation itself. """ from __future__ import annotations @@ -43,6 +47,49 @@ def refuse(self, *args, **kwargs): return refuse +# --- `skip_message`: how every test below reads a skip ----------------------- +# +# A skip that ESCAPES a helper ends its test SKIPPED, and no run fails on a +# skip: a `skip_message` that let one out would turn every assertion below +# that expects a skip, or expects none, into a silent skip. So the first test +# catches the escape and fails it. + +def test_skip_message_returns_the_reason_a_call_skips_with(): + try: + reason = skip_message(lambda: pytest.skip("x")) + except pytest.skip.Exception: + pytest.fail("the skip escaped skip_message, which would end the test " + "SKIPPED, and no run fails on that") + assert reason == "x" + + +def test_skip_message_returns_none_where_the_call_does_not_skip(): + assert skip_message(lambda: None) is None + + +def test_skip_message_lets_any_other_exception_go_on(): + def not_a_skip(): + raise ZeroDivisionError("not a skip") + + with pytest.raises(ZeroDivisionError): + skip_message(not_a_skip) + + +# --- `windows_fsencode`: the emulation is Windows' `os.fsencode` -------------- + +def test_windows_fsencode_makes_three_bytes_of_a_surrogate_and_utf_8_of_text(): + assert windows_fsencode("\udcff") == b"\xed\xb3\xbf" + assert windows_fsencode("caf\u00e9.yml") == "caf\u00e9.yml".encode("utf-8") + + +def test_windows_fsencode_passes_bytes_through_as_os_fsencode_does(): + """`os.fsencode` takes bytes on every system and gives them back; the + emulation that stands in for it must not fail on them.""" + data = b"caf\xe9.yml" + assert os.fsencode(data) == data + assert windows_fsencode(data) == data + + # --- the system's question: what `os.fsencode` makes of the surrogate -------- @LINUX_ONLY