From d6d1df2340652efe87c0cefbcad4eac8f804ce32 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 14:05:35 +0000 Subject: [PATCH 1/5] conftest.py and four test files: one helper asks whether a system can carry a name or branch that is not UTF-8, `os.fsencode` first and the disk's EILSEQ second, and two pins hold `bump-leg.py`'s UTF-8 decode and the leg finding's role, so a test no longer skips on the disk alone or passes under a decode as ASCII or a finding named by path (#663) #461's five test lines, lane 2's L2-52 and lane 4's R423 N1 and N2 and R455 F1 and F3. Test only: no tool and no template changes, and `validate-manifest.py`, a pinned copy, is read and not edited. - `tests/conftest.py` holds `skip_where_a_name_cannot_be_not_utf_8`, which `tests/test_adopt_pins.py` and `tests/test_pin_source_unreadable.py` each had a copy of. It asks `skip_where_fsencode_gives_a_surrogate_no_byte` first (Windows' `os.fsencode` makes three bytes of U+DCFF, so no name that stands for a byte is carried there, though NTFS would take the file), then the disk (macOS answers EILSEQ); any other error is the test's own. - `make_file` and the `--branch a\udcffb` test call it, so they skip on Windows too, where the disk takes the name and the child then refuses the branch. `test_a_branch_the_system_carries_has_no_fault` asks the same function instead of its own copy of the question. - `test_a_name_that_is_utf8_under_quote_path_false_is_still_refused` asserts that the line naming `caf.yml` holds no literal `\x`, in any locale, so `bump-leg.py` decoding git's bytes as ASCII fails it. - `_leg_branch_findings` is asked of a leg whose role is not its path, so naming the leg by `path` fails it. - `tests/test_non_utf8_name_helpers.py` holds the helpers to this, with Windows' `os.fsencode` and macOS's EILSEQ emulated: CI is Linux only, and no Windows or macOS run was made. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 83 +++++++++++++++++ tests/test_adopt_pins.py | 20 +--- tests/test_bootstrap_branch_bytes.py | 74 ++++++++++++--- tests/test_non_utf8_git_reads.py | 77 +++++++++++++--- tests/test_non_utf8_name_helpers.py | 133 +++++++++++++++++++++++++++ tests/test_pin_source_unreadable.py | 21 +---- 6 files changed, 346 insertions(+), 62 deletions(-) create mode 100644 tests/test_non_utf8_name_helpers.py diff --git a/tests/conftest.py b/tests/conftest.py index 33cbf476..0e9da01d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ from __future__ import annotations +import errno import functools import importlib.util import os @@ -864,6 +865,88 @@ def a_hook_that_unstages(hooks: Path, path: str = PINNED_COPY) -> Path: return hooks +# --- a file name or a branch that is not UTF-8 (#663) ------------------------ +# +# A test that needs a file name or a branch holding a byte that is not UTF-8 +# (Python keeps the byte as a lone surrogate, U+DC80 to U+DCFF, which +# `os.fsencode` gives back as the byte on POSIX) asks these helpers whether +# this system can carry it, so each system's answer is written in one place +# and a change to it reaches every test that asks. +# `tests/test_non_utf8_name_helpers.py` holds them to it, Windows included, by +# emulating its `os.fsencode`: CI runs the suite on Linux only (#597). + +#: The surrogate the helpers ask about: the byte 0xFF, which no UTF-8 text has. +NOT_UTF_8_PROBE = "\udcff" + + +def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None: + """Skip where `text` holds a byte-carrying surrogate (U+DC80 to U+DCFF) + and this system's `os.fsencode` makes no single byte of one. + + That is Windows: its `os.fsencode` is UTF-8 with `surrogatepass`, so it + writes the three bytes of a sequence that is not UTF-8 for EVERY + surrogate, and none of them is a byte of a name. A disk probe cannot be + trusted to ask it: CPython hands a str path to `CreateFileW` as UTF-16 + and NTFS most likely stores a lone surrogate unchecked (read, not run), + so the file is made, and git for Windows then sees the name through its + own UTF-16 conversion, which is not the byte the test meant. A command + line is the same (`CreateProcessW`), which is why `bootstrap.py`'s + `_argument_fault` refuses U+DC80 to U+DCFF there. Asked through the `os` + module at call time, so a test emulates Windows with + `monkeypatch.setattr(os, "fsencode", )`. + """ + if (any("\udc80" <= char <= "\udcff" for char in text) + and len(os.fsencode(NOT_UTF_8_PROBE)) != 1): + pytest.skip("this system's `os.fsencode` makes no byte of a surrogate " + "that stands for a byte that is not UTF-8") + + +def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: + """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not + UTF-8 cannot be made under `directory`. Two systems say so: + + - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first, + before the disk is touched; + - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte + sequence"), so no clone can sit at such a path there. + + The second is asked by making a directory `probe` under + `directory` and removing it; any other error is the test's own and is + raised. + """ + probe = directory / f"probe{NOT_UTF_8_PROBE}" + skip_where_fsencode_gives_a_surrogate_no_byte(probe.name) + try: + probe.mkdir() + except OSError as exc: + if exc.errno != errno.EILSEQ: + raise + pytest.skip("needs a file name that is not UTF-8, which this " + "filesystem refuses (EILSEQ)") + probe.rmdir() + + +def windows_fsencode(text: str) -> bytes: + """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every + lone surrogate is the three bytes of a sequence that is not UTF-8. For + `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the + guard above in front of a test that Linux would let through.""" + return text.encode("utf-8", "surrogatepass") + + +def skip_message(call, *args) -> str | None: + """The reason `call(*args)` skips the running test with, or None where it + returns; any other exception goes on. For a test OF a skip helper: a + skip that escapes a test ends it SKIPPED, which no run fails on, so a + helper that skipped where it should not would pass `pytest.raises` and + every other assertion that expects no skip.""" + try: + call(*args) + except pytest.skip.Exception as skipped: + return skipped.msg + return None + + # --- the `gh` guard (#296) --------------------------------------------------- # # NO TEST REACHES THE OPERATOR'S `gh`, and the doctor's `machine` row is still diff --git a/tests/test_adopt_pins.py b/tests/test_adopt_pins.py index 719af0f7..7b3bb313 100644 --- a/tests/test_adopt_pins.py +++ b/tests/test_adopt_pins.py @@ -34,7 +34,6 @@ from __future__ import annotations -import errno import importlib.util import json import os @@ -50,7 +49,8 @@ from conftest import (ADOPT, REPO, SCAFFOLD, SYNTHETIC_TREE, WINDOWS_SKIP, as_human, cloned_just_pushed, findings, git, make_source_repo, planned, refused_before_any_leg, - resolve, rmtree, run_script) + resolve, rmtree, run_script, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -855,22 +855,6 @@ def test_a_pin_source_check_would_refuse_is_refused_by_plan( UNREAD = "unreadclone" -def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: - """Skip, as `POSIX_NAMES` skips on Windows, on a filesystem that refuses - a file name that is not UTF-8: macOS answers EILSEQ ("Illegal byte - sequence"), so no clone can sit at such a path there. Asked of a - directory made under `directory`; any other error is the test's own.""" - probe = directory / "probe\udcff" - try: - probe.mkdir() - except OSError as exc: - if exc.errno != errno.EILSEQ: - raise - pytest.skip("needs a file name that is not UTF-8, which this " - "filesystem refuses (EILSEQ)") - probe.rmdir() - - @pytest.mark.parametrize("name, reason", [ pytest.param(f"{UNREAD}‍x", "it holds an invisible formatting character", id="zwj"), diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index bab82102..3f63761e 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -45,7 +45,10 @@ import pytest -from conftest import REPO, git, load_without_bytecode +from conftest import (REPO, git, load_without_bytecode, skip_message, + skip_where_a_name_cannot_be_not_utf_8, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) from test_bootstrap_spelling import (ASSEMBLY_TEMPLATE, DASH, bootstrap_of, edit, pin12, run) from test_stdout_encoding import child_environ, run_tool_bytes @@ -349,12 +352,12 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made( `os.fsencode` carries it, and git is handed that byte, as on origin/main. `--branch` is the only way it reaches step (a) with the validators out of it (the manifest validator names it, #280's rule for a surrogate). Made on - a disk that takes the byte, skipped on one that does not.""" - probe = tmp_path / "probe-\udcff" - try: - probe.write_bytes(b"") - except (OSError, ValueError): - pytest.skip("this disk takes no file name with the byte 0xff") + a system and a disk that take the byte, skipped on one that does not: the + shared helper asks `os.fsencode` first, which is what Windows answers + (NTFS takes the name, and `_argument_fault` refuses the branch there, + which `test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused` + holds), and then the disk, which macOS refuses (#663).""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) code, out, err = run(bootstrap_of(project), "--skip-validators", "--branch", "a\udcffb", encoding=ENCODING, tmp_path=tmp_path, cwd=project) @@ -370,12 +373,40 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made( assert head == b"refs/heads/a\xffb\n", head +def test_the_branch_test_skips_where_fsencode_makes_no_byte_of_the_surrogate( + project, tmp_path, monkeypatch): + """THE WINDOWS GUARD, EMULATED (#663, R455 F1): on NTFS the disk takes + `probe\\udcff`, the child is given `--branch a\\udcffb`, and + `_argument_fault` refuses U+DCFF where `os.fsencode` writes three bytes, + so the `branch a\\udcffb created` line the test expects is absent. With + Windows' `os.fsencode` the test is therefore SKIPPED, before it runs + bootstrap: it is called here as it is, under the emulation, and the legs + stand as they were.""" + before = legs_state(project) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message( + test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made, + project, tmp_path) + assert reason is not None, "the test ran where `os.fsencode` gives no byte" + assert "os.fsencode" in reason, reason + assert legs_state(project) == before + + @pytest.fixture(scope="module") def bootstrap_module(): """The template run as a module, no bytecode written beside it.""" return load_without_bytecode(ASSEMBLY_TEMPLATE, "bootstrap_branch_bytes") +@pytest.fixture(scope="module") +def validator_module(): + """`validate-manifest.py`, the same template, as a module: the file is a + pinned copy, so its functions are read and never changed here.""" + return load_without_bytecode( + REPO / "templates" / "assembly-root" / "scripts" + / "validate-manifest.py", "validate_manifest_branch_bytes") + + @pytest.mark.parametrize("branch", [ "main", "release/1.2", "feature/日本", "a‍b", "a\xe9b", "a b", "a\x1bb", "a..b", "", "\udc80", "a\udcffb"]) @@ -385,9 +416,7 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): the byte-carrying surrogates U+DC80 to U+DCFF are the commands'. On a system whose `os.fsencode` does not carry U+DC80 and U+DCFF (Windows) the next test is the one that holds.""" - if any("\udc80" <= char <= "\udcff" for char in branch) and len( - os.fsencode("\udcff")) != 1: - pytest.skip("this system's `os.fsencode` makes no byte of U+DCFF") + skip_where_fsencode_gives_a_surrogate_no_byte(branch) assert bootstrap_module._argument_fault(branch) is None @@ -424,13 +453,28 @@ def test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused( assert fault.startswith("U+DC"), fault -def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module): +def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module, + validator_module): """`validate-manifest.py` finds ANY surrogate (#280's rule, which a UTF-8 file settles), where `bootstrap.py` asks only whether a command can be given the value: so every value bootstrap refuses, the validator names.""" - validator = load_without_bytecode( - REPO / "templates" / "assembly-root" / "scripts" - / "validate-manifest.py", "validate_manifest_branch_bytes") for branch in ("a\x00b", "a\ud83db", "a\ude00", "\udc00"): assert bootstrap_module._argument_fault(branch) is not None - assert validator._branch_finding("c", "w", branch) != [] + assert validator_module._branch_finding("c", "w", branch) != [] + + +@pytest.mark.parametrize("role", ["spec", "code"]) +def test_the_leg_finding_names_the_leg_by_its_role_and_not_its_path( + validator_module, role): + """THE NAME IS THE ROLE (#663, R455 F3, mutant R2: naming `path` passed + all 54 tests). The templates' roles equal their paths (`spec` is mounted + at `spec`), so nothing the scaffold writes can tell the two apart, and + the finding is asked of a leg whose path is not its role, as a spec leg + mounted at `contracts` is. A unit test on the validator, which this file + loads for its last test too: it is a pinned copy and is not edited.""" + [finding] = validator_module._leg_branch_findings( + [{"role": role, "path": "contracts", "branch": "a\x00b"}]) + assert finding.startswith( + f"FINDING manifest-leg-branch: leg '{role}': branch is 'a\\x00b', " + "which holds U+0000, a NUL"), finding + assert "contracts" not in finding, finding diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index c79bada2..18fbb7e9 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -32,11 +32,13 @@ NO NETWORK AND NO GITHUB. The scaffold and `family.py init` here are run as child processes into LOCAL bare remotes (`--local-remote-dir`), which never calls `gh`. A name that is not UTF-8 is made by `os.fsdecode` of its bytes, so -a POSIX filesystem takes it as it is; where the disk refuses (macOS, Windows) -the test skips, and where the ENVIRONMENT cannot carry a byte (Windows') the -`user.name` rows skip. The checkout under test for `_shape_revision` and -`_shape_snapshot` is a COPY of the standard in a repository of its own -(`standard`), made dirty one test at a time, never this checkout. +a POSIX filesystem takes it as it is; where the system cannot carry the byte +the test skips (`make_file`: Windows' `os.fsencode` makes no byte of it, and +macOS's disk refuses it), and where the ENVIRONMENT cannot carry a byte +(Windows') the `user.name` rows skip. The checkout under test for +`_shape_revision` and `_shape_snapshot` is a COPY of the standard in a +repository of its own (`standard`), made dirty one test at a time, never this +checkout. """ from __future__ import annotations @@ -49,7 +51,10 @@ import pytest -from conftest import ORG, PROJECT, REPO, cloned_just_pushed, git, run_script +from conftest import (ORG, PROJECT, REPO, cloned_just_pushed, git, run_script, + skip_message, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) from test_bump_leg import branched, bump sys.path.insert(0, str(REPO / "scripts")) @@ -85,7 +90,11 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: """Write `data` at `path`, or skip: a name that is not UTF-8 needs a - filesystem that stores bytes (Linux does; APFS and NTFS refuse it).""" + system that carries the byte and a filesystem that stores it (Linux does; + APFS refuses it, and on Windows `os.fsencode` makes no byte of it, which + is asked FIRST: NTFS most likely takes the surrogate, and the file would + then not be the one the test means). A name that is UTF-8 is written.""" + skip_where_fsencode_gives_a_surrogate_no_byte(path.name) try: path.write_bytes(data) except (OSError, UnicodeError) as exc: @@ -93,6 +102,47 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: f"UTF-8: {exc}") +def test_make_file_skips_where_fsencode_makes_no_byte_and_writes_nothing( + tmp_path, monkeypatch): + """THE WINDOWS GUARD, EMULATED (#663, R423 N1): where `os.fsencode` gives + a surrogate three bytes, NTFS most likely takes `caf\\udce9.yml` anyway + and git for Windows would see another name, so `make_file` skips BEFORE it + writes, and nothing is on the disk. Linux takes the name, so a file + standing there would be the failure.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + target = tmp_path / NOT_UTF8 + reason = skip_message(make_file, target) + assert reason is not None, "make_file wrote the name instead of skipping" + assert "os.fsencode" in reason, reason + assert not target.exists(), "the file was written before the skip" + + +def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( + tmp_path, monkeypatch): + """The guard is for the byte and no wider: Windows holds `caf\\u00e9.yml` + as text, and the tests that use it must run there.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + target = tmp_path / UTF8 + assert skip_message(make_file, target, b"on: [pull_request]\n") is None + assert target.read_bytes() == b"on: [pull_request]\n" + + +def test_make_file_writes_the_name_that_is_not_utf8_where_the_system_has_it( + tmp_path): + """The control: where `os.fsencode` carries the byte and the disk takes it + (Linux), the file is written under the name that is not UTF-8, so the + guard skipped nothing.""" + if len(os.fsencode("\udce9")) != 1: + pytest.skip("this system's `os.fsencode` makes no byte of U+DCE9") + target = tmp_path / NOT_UTF8 + reason = skip_message(make_file, target, b"on: [push]\n") + if reason is not None and "filesystem" in reason: + pytest.skip(reason) + assert reason is None, reason + assert os.fsencode(target.name) == b"caf\xe9.yml" + assert target.read_bytes() == b"on: [push]\n" + + def git_bytes(*args: str, cwd: Path) -> bytes: """`git ` in `cwd`, stdout as the bytes git wrote. `conftest.git` decodes as text, which is the failure under test.""" @@ -205,16 +255,21 @@ def test_a_name_that_is_utf8_under_quote_path_false_is_still_refused( Its bytes decode, and it was refused on main; it still is, and the refusal still prints it as git did. (The name is matched on its ASCII ends: a child's stderr is decoded by the locale here, which is not UTF-8 - on every platform.)""" + on every platform. What the middle of the name is NOT is held instead, in + any locale: the tool decodes git's bytes as UTF-8, so `\\u00e9` is one + character, where a decode as ASCII would show its two bytes as `\\xc3\\xa9`, + and a locale's decode of what the child printed is mojibake at worst and + never a `\\x`: #423's mutant R1, #663.)""" dirty_root(project, UTF8, quote_path=False) before = found(project) result = bump(project, scaffolded, advanced, *extra) output = result.stderr + result.stdout assert result.returncode == 2, output assert "REFUSED bump-leg-root-dirty: " in result.stderr, output - assert any(line.startswith(" M .github/workflows/caf") - and line.endswith(".yml") - for line in result.stderr.splitlines()), output + [shown] = [line for line in result.stderr.splitlines() + if line.startswith(" M .github/workflows/caf")] + assert shown.endswith(".yml"), output + assert "\\x" not in shown, output assert "Traceback" not in output, output assert found(project) == before diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py new file mode 100644 index 00000000..8c387eee --- /dev/null +++ b/tests/test_non_utf8_name_helpers.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The two helpers a test asks whether a name that is not UTF-8 can be +carried here (#663, #461's L2-52). + +`skip_where_a_name_cannot_be_not_utf_8` was two helpers with one body +(`tests/test_adopt_pins.py` and `tests/test_pin_source_unreadable.py`), and +`make_file` and the `--branch a\\udcffb` test each asked a smaller question of +their own, or none: on Windows the disk takes the name, so no probe of the +disk skips there. Now `tests/conftest.py` holds both questions once, the +system's first (`os.fsencode`) and the disk's second (EILSEQ), and this file +holds them to it. + +CI runs the suite on Linux only (#597), so what Windows answers is EMULATED: +`os.fsencode` is replaced for one test by `windows_fsencode`, which gives a +surrogate three bytes, as Windows' does. No Windows run was made. macOS's +EILSEQ is emulated the same way, by a `mkdir` that answers it. +""" + +from __future__ import annotations + +import errno +import os +import sys +from pathlib import Path + +import pytest + +from conftest import (NOT_UTF_8_PROBE, skip_message, + skip_where_a_name_cannot_be_not_utf_8, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) + +LINUX_ONLY = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="Linux stores any byte in a file name; the other systems are " + "asked by the tests below that emulate them") + + +def mkdir_answers(code: int, said: str): + """A `Path.mkdir` that fails as the disk would with `code`.""" + def refuse(self, *args, **kwargs): + raise OSError(code, said, str(self)) + return refuse + + +# --- the system's question: what `os.fsencode` makes of the surrogate -------- + +@LINUX_ONLY +def test_a_system_that_carries_the_byte_is_not_skipped(): + """The control: Linux's `os.fsencode` gives U+DCFF back as the one byte + 0xFF, so no text is skipped.""" + assert os.fsencode(NOT_UTF_8_PROBE) == b"\xff" + assert skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + f"caf{NOT_UTF_8_PROBE}.yml") is None + + +def test_where_fsencode_makes_three_bytes_of_the_surrogate_the_name_is_skipped( + monkeypatch): + """Windows' `os.fsencode` (emulated): U+DCFF is the three bytes of a + sequence that is not UTF-8, none of them 0xFF, so a name that holds a + byte-carrying surrogate is skipped, and each one from U+DC80 to U+DCFF.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + assert len(os.fsencode(NOT_UTF_8_PROBE)) == 3 + for surrogate in ("\udc80", "\udce9", "\udcff"): + reason = skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + f"a{surrogate}b") + assert reason is not None, f"U+{ord(surrogate):04X} was not skipped" + assert "os.fsencode" in reason, reason + + +@pytest.mark.parametrize("text", [ + "plain", "café.yml", "feature/日本", "", "a\ud83db"], + ids=["ascii", "text", "cjk", "empty", "surrogate-without-a-byte"]) +def test_where_fsencode_makes_three_bytes_a_name_that_is_text_is_not_skipped( + monkeypatch, text): + """The guard is for the surrogates that stand for a byte (U+DC80 to + U+DCFF) and no wider: text is the same on every system, and a surrogate + that stands for no byte is not this helper's to judge (`_argument_fault` + refuses it, as `bootstrap.py`'s tests hold).""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + assert skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + text) is None + + +# --- the shared helper: the system, then the disk ---------------------------- + +@LINUX_ONLY +def test_a_disk_that_takes_the_name_is_not_skipped_and_the_probe_is_removed( + tmp_path): + """The control: no skip, and nothing is left in the directory.""" + assert skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path) is None + assert list(tmp_path.iterdir()) == [] + + +def test_where_fsencode_makes_three_bytes_the_disk_is_not_asked( + tmp_path, monkeypatch): + """THE SYSTEM IS ASKED FIRST (#663): on Windows the disk would take the + name, so a skip that waited for the disk would never come. The directory + does not exist, so an `mkdir` under it would be FileNotFoundError and not + a skip: the skip is the answer, and the disk was not touched.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path / "absent") + assert reason is not None, "the helper went on to ask the disk" + assert "os.fsencode" in reason, reason + + +def test_a_disk_that_answers_eilseq_is_skipped(tmp_path, monkeypatch): + """macOS (emulated): APFS answers EILSEQ, `Illegal byte sequence`, to a + file name that is not UTF-8, and no clone can sit at such a path.""" + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + monkeypatch.setattr(Path, "mkdir", + mkdir_answers(errno.EILSEQ, "Illegal byte sequence")) + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) + assert reason is not None, "EILSEQ was not a skip" + assert "EILSEQ" in reason, reason + + +@pytest.mark.parametrize("code", [errno.EACCES, errno.ENOSPC, errno.EEXIST], + ids=["EACCES", "ENOSPC", "EEXIST"]) +def test_any_other_error_from_the_disk_is_the_tests_own( + tmp_path, monkeypatch, code): + """Only EILSEQ is the filesystem's refusal of the name: a full disk, a + folder nobody can write in or a probe left by an earlier run is a + failure of the test, raised as it was, and never a skip that hides it.""" + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + monkeypatch.setattr(Path, "mkdir", mkdir_answers(code, os.strerror(code))) + with pytest.raises(OSError) as raised: + skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) + assert raised.value.errno == code diff --git a/tests/test_pin_source_unreadable.py b/tests/test_pin_source_unreadable.py index 177e8c94..c43a41ad 100644 --- a/tests/test_pin_source_unreadable.py +++ b/tests/test_pin_source_unreadable.py @@ -41,7 +41,8 @@ import pytest -from conftest import ADOPT, ORG, REPO, SCAFFOLD, git, make_source_repo, run_script +from conftest import (ADOPT, ORG, REPO, SCAFFOLD, git, make_source_repo, + run_script, skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) from repo_shape import load_yaml, parse_yaml, tree_digest # noqa: E402 @@ -592,22 +593,6 @@ def test_a_typed_dot_is_still_the_directory_the_tool_runs_in( ] -def skip_where_no_file_name_is_not_utf_8(directory: Path) -> None: - """Skip, as a Windows command line skips, where the filesystem refuses - a file name that is not UTF-8, so that no clone can sit at one: macOS - answers EILSEQ ("Illegal byte sequence"). Asked by making one under - `directory`; any other error is the test's to report.""" - probe = directory / "probe\udcff" - try: - probe.mkdir() - except OSError as exc: - if exc.errno != errno.EILSEQ: - raise - pytest.skip("needs a file name that is not UTF-8, which this " - "filesystem refuses (EILSEQ)") - probe.rmdir() - - @pytest.mark.parametrize("name", CLONE_NAMES) def test_the_pin_file_says_how_the_digest_was_read_and_not_where( name, tmp_path, adoptable, glass): @@ -618,7 +603,7 @@ def test_the_pin_file_says_how_the_digest_was_read_and_not_where( --all`, read as bytes).""" product, commit = glass if "\udcff" in name: - skip_where_no_file_name_is_not_utf_8(tmp_path) + skip_where_a_name_cannot_be_not_utf_8(tmp_path) clones = tmp_path / "clones" shutil.copytree(product, clones / name, symlinks=True) result = run("scaffold-run", tmp_path, adoptable, str(clones / name), From 0b1d1ddbeeca662500534f58c39e5b5a0c67a70d Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 14:15:43 +0000 Subject: [PATCH 2/5] test_bootstrap_branch_bytes.py: the no-fault test's Windows guard is held by an emulation, so dropping the shared helper from it fails a test on Linux too (#663) `test_a_branch_the_system_carries_has_no_fault` now asks `skip_where_fsencode_gives_a_surrogate_no_byte` where it had its own copy of the question. The new test calls it under Windows' `os.fsencode` (emulated): a branch holding a byte-carrying surrogate is skipped, plain text and text with an accent run. Removing the call fails the two skipped rows. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_bootstrap_branch_bytes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index 3f63761e..5a79ff8e 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -420,6 +420,24 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): assert bootstrap_module._argument_fault(branch) is None +@pytest.mark.parametrize("branch, skipped", [ + pytest.param("a\udcffb", True, id="byte-carrying-surrogate"), + pytest.param("\udc80", True, id="the-first-one"), + pytest.param("release/1.2", False, id="plain"), + pytest.param("a\xe9b", False, id="text"), +]) +def test_the_no_fault_test_skips_where_fsencode_makes_no_byte_of_the_branch( + bootstrap_module, monkeypatch, branch, skipped): + """THE WINDOWS GUARD, EMULATED (#663): under Windows' `os.fsencode` the + test above skips a branch holding a byte-carrying surrogate, which + `_argument_fault` refuses there, and runs every other (the next test + holds the refusal).""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(test_a_branch_the_system_carries_has_no_fault, + bootstrap_module, branch) + assert (reason is not None) is skipped, reason + + @pytest.mark.parametrize("branch, fault", [ ("a\x00b", "U+0000, a NUL"), ("\x00", "U+0000, a NUL"), From 6ba8c5d2d6fec3a779963713593e5377c76bbadf Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 18:40:04 +0000 Subject: [PATCH 3/5] conftest.py and two test files: `skip_message` is held by a test that fails where a skip escapes it, `windows_fsencode` passes bytes through as `os.fsencode` does, and the no-`\x` check skips where the file name encoding is not UTF-8, so a leaked skip, an emulation that breaks on bytes and a Latin-1 host no longer pass silently or fail on a tool that is right (#663) The review of #723 (0b1d1ddb) found no MUST and three things to take. SHOULD-1: `skip_message` had no test of its own. A skip that escapes it ends the test SKIPPED, and no run fails on a skip, so the mutant where one escapes gave `10 passed, 3 skipped` on the helpers file and seven emulation tests silently skipped. `tests/test_non_utf8_name_helpers.py` now holds it with three tests: it returns the reason a call skips with, it returns None where the call does not skip, and it lets any other exception go on. The first catches the escape and fails it with `pytest.fail`, because a bare assert on an escaped skip would end that test SKIPPED too. NIT-2: `windows_fsencode` took `str` only. `os.fsencode` passes bytes through on every system, so the emulation now does, and two tests hold it (the surrogate and text, and the bytes, compared with `os.fsencode` itself). NIT-1: on a Latin-1 file name encoding `make_file` writes `caf\xe9.yml` as the single byte 0xE9, the tool prints `caf\xe9.yml`, and the new `"\\x" not in shown` assertion would fail although the tool is right. It is now `assert_shown_without_hex_escapes`, which skips where `os.fsencode(UTF8) != UTF8.encode("utf-8")` and is called after the test's other assertions, so a skip loses none of them. Two tests hold it under an emulated Latin-1 `os.fsencode` (it skips) and Windows' (it asserts, and fails a line that holds a `\x`). Tests only; no tool and no template changes. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 7 +++-- tests/test_non_utf8_git_reads.py | 44 ++++++++++++++++++++++++++- tests/test_non_utf8_name_helpers.py | 47 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0e9da01d..a5313d72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -926,11 +926,14 @@ def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: probe.rmdir() -def windows_fsencode(text: str) -> bytes: +def windows_fsencode(text: str | bytes) -> bytes: """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every - lone surrogate is the three bytes of a sequence that is not UTF-8. For + lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes + go through as they are, as `os.fsencode` passes them on every system. For `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the guard above in front of a test that Linux would let through.""" + if isinstance(text, bytes): + return text return text.encode("utf-8", "surrogatepass") diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index 18fbb7e9..4d19a998 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -143,6 +143,48 @@ def test_make_file_writes_the_name_that_is_not_utf8_where_the_system_has_it( assert target.read_bytes() == b"on: [push]\n" +def assert_shown_without_hex_escapes(shown: str, output: str) -> None: + """Assert that the refusal's line for `caf\u00e9.yml` holds no literal + `\\x`, or skip where this system's file name encoding is not UTF-8. + + On a Latin-1 host `make_file` writes the one byte 0xE9 for it, and the + tool, which decodes git's bytes as UTF-8, rightly prints `caf\\xe9.yml`: + the assertion would fail though the tool is right (#663, NIT-1). So it + is asked after the system's encoding is, and a system whose encoding is + not UTF-8 skips. (Where `make_file` could not write the name at all, an + ASCII host, it skipped already.)""" + if os.fsencode(UTF8) != UTF8.encode("utf-8"): + pytest.skip("this system's file name encoding is not UTF-8, so a " + "UTF-8 name is not written as the bytes the tool decodes") + assert "\\x" not in shown, output + + +def test_the_hex_escape_check_skips_where_the_file_name_encoding_is_not_utf_8( + monkeypatch): + """THE LATIN-1 HOST, EMULATED (#663, NIT-1): `os.fsencode` gives the name + `caf\u00e9.yml` as the one byte 0xE9, the line holds a `\\x`, and the + check skips, where it would otherwise fail on a tool that is right.""" + monkeypatch.setattr(os, "fsencode", lambda name: name.encode("latin-1")) + reason = skip_message(assert_shown_without_hex_escapes, + r" M .github/workflows/caf\xe9.yml", "output") + assert reason is not None, "the check asserted instead of skipping" + assert "UTF-8" in reason, reason + + +def test_the_hex_escape_check_asserts_where_the_file_name_encoding_is_utf_8( + monkeypatch): + """The control: Windows' `os.fsencode` (emulated), like Linux's and + macOS's, is UTF-8 for text, so the check is not skipped, and it still + fails a line that holds a `\\x` and passes one that does not.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(assert_shown_without_hex_escapes, + " M .github/workflows/caf\u00e9.yml", "output") + assert reason is None, reason + with pytest.raises(AssertionError): + skip_message(assert_shown_without_hex_escapes, + r" M .github/workflows/caf\xc3\xa9.yml", "output") + + def git_bytes(*args: str, cwd: Path) -> bytes: """`git ` in `cwd`, stdout as the bytes git wrote. `conftest.git` decodes as text, which is the failure under test.""" @@ -269,9 +311,9 @@ def test_a_name_that_is_utf8_under_quote_path_false_is_still_refused( [shown] = [line for line in result.stderr.splitlines() if line.startswith(" M .github/workflows/caf")] assert shown.endswith(".yml"), output - assert "\\x" not in shown, output assert "Traceback" not in output, output assert found(project) == before + assert_shown_without_hex_escapes(shown, output) def test_the_refusal_names_every_dirty_file_whatever_its_bytes( diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 8c387eee..47ae0d13 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -14,6 +14,10 @@ `os.fsencode` is replaced for one test by `windows_fsencode`, which gives a surrogate three bytes, as Windows' does. No Windows run was made. macOS's EILSEQ is emulated the same way, by a `mkdir` that answers it. + +The two tools the tests below are written with are held here too: +`skip_message`, which every one of them reads a skip with, and +`windows_fsencode`, the emulation itself. """ from __future__ import annotations @@ -43,6 +47,49 @@ def refuse(self, *args, **kwargs): return refuse +# --- `skip_message`: how every test below reads a skip ----------------------- +# +# A skip that ESCAPES a helper ends its test SKIPPED, and no run fails on a +# skip: a `skip_message` that let one out would turn every assertion below +# that expects a skip, or expects none, into a silent skip. So the first test +# catches the escape and fails it. + +def test_skip_message_returns_the_reason_a_call_skips_with(): + try: + reason = skip_message(lambda: pytest.skip("x")) + except pytest.skip.Exception: + pytest.fail("the skip escaped skip_message, which would end the test " + "SKIPPED, and no run fails on that") + assert reason == "x" + + +def test_skip_message_returns_none_where_the_call_does_not_skip(): + assert skip_message(lambda: None) is None + + +def test_skip_message_lets_any_other_exception_go_on(): + def not_a_skip(): + raise ZeroDivisionError("not a skip") + + with pytest.raises(ZeroDivisionError): + skip_message(not_a_skip) + + +# --- `windows_fsencode`: the emulation is Windows' `os.fsencode` -------------- + +def test_windows_fsencode_makes_three_bytes_of_a_surrogate_and_utf_8_of_text(): + assert windows_fsencode("\udcff") == b"\xed\xb3\xbf" + assert windows_fsencode("caf\u00e9.yml") == "caf\u00e9.yml".encode("utf-8") + + +def test_windows_fsencode_passes_bytes_through_as_os_fsencode_does(): + """`os.fsencode` takes bytes on every system and gives them back; the + emulation that stands in for it must not fail on them.""" + data = b"caf\xe9.yml" + assert os.fsencode(data) == data + assert windows_fsencode(data) == data + + # --- the system's question: what `os.fsencode` makes of the surrogate -------- @LINUX_ONLY From 41849725ce9e309eb364233e6adcfa61e3f34882 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 10:40:26 +0000 Subject: [PATCH 4/5] test_stdout_encoding.py, test_adopt_git_reads.py, test_adopt_template_values.py, test_adopt_plan_escapes.py, test_clone_url_bytes.py and test_non_utf8_git_reads.py: the four tests and `directory_named` that made a directory named with a byte that is not UTF-8 ask `skip_where_a_name_cannot_be_not_utf_8` and skip on its two answers only, `directory_named` keeps its own Windows skip for a control character and no longer skips a bidi override there, and three tests of `test_non_utf8_git_reads.py` skip where an ASCII file name encoding cannot carry their text, so a full disk or a `tmp_path` nobody can write in fails the test instead of skipping it and a C locale shows no failure that is not a defect (#757) The four tests (in `test_stdout_encoding.py`, `test_adopt_git_reads.py`, `test_adopt_template_values.py` and `test_adopt_plan_escapes.py`) and `directory_named` each made the directory themselves and turned ANY `OSError` into `pytest.skip`, beside the helper #663 added to `tests/conftest.py`. That helper skips on the two answers that mean "this system cannot carry the name" (Windows' `os.fsencode` first, macOS's EILSEQ second) and raises the rest. They now call it, and each keeps its own second skip (git will not work under the name; a listing that shows a rewritten name); the directory they make after the probe is no longer guarded. The file twin in `test_adopt_plan_escapes.py` asks the same question of a file name and calls the helper too. `directory_named` asks the helper for a name that is not UTF-8 and keeps its own Windows skip for a control character, which Windows refuses in any name. The `bidi-override` row, which NTFS holds, now runs on Windows (reasoned; CI is Linux only, no Windows run was made). Under `PYTHONUTF8=0 LC_ALL=C PYTHONCOERCECLOCALE=0` the file name encoding is ASCII, and `tests/test_non_utf8_git_reads.py` failed six tests that hold no defect: `test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte` (the issue's line) and the five rows that put `Jos\u00E9`, a U+200C name or a U+202E name into a child's environment, where `subprocess` raises `UnicodeEncodeError` before the tool runs. The first now ends as its neighbour does, on a skip that names the filesystem; the others skip where `os.fsencode` cannot carry the value. `tests/test_non_utf8_name_callers.py` calls each caller as a function with the disk answering as macOS (EILSEQ on the probe: a skip), as a folder nobody can write in (EACCES on the probe, and on the test's own directory: a failure), and `directory_named` as Windows (a control character skips, a bidi override does not), and holds the two C-locale skips with an ASCII `os.fsencode` and a `write_bytes` that raises as it would. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_adopt_git_reads.py | 13 +- tests/test_adopt_plan_escapes.py | 21 +- tests/test_adopt_template_values.py | 13 +- tests/test_clone_url_bytes.py | 39 ++-- tests/test_non_utf8_git_reads.py | 23 ++- tests/test_non_utf8_name_callers.py | 285 ++++++++++++++++++++++++++++ tests/test_stdout_encoding.py | 9 +- 7 files changed, 358 insertions(+), 45 deletions(-) create mode 100644 tests/test_non_utf8_name_callers.py diff --git a/tests/test_adopt_git_reads.py b/tests/test_adopt_git_reads.py index a327e56f..d97353a4 100644 --- a/tests/test_adopt_git_reads.py +++ b/tests/test_adopt_git_reads.py @@ -31,7 +31,8 @@ import pytest -from conftest import REPO, git, make_source_repo +from conftest import (REPO, git, make_source_repo, + skip_where_a_name_cannot_be_not_utf_8) from test_adopt_template_values import (ADOPT_REFUSED, PLAN_FILE, PLAN_OK, PROJECT, REMOTES, VERIFIED, WORK, adopted, answer, @@ -427,12 +428,10 @@ def test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path): --source ` ended in this issue's traceback; it now adopts. Skipped where no directory can be named with the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" - try: - parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9"))) - parent.mkdir() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no directory here is named b'caf\\xe9': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), + b"caf\xe9"))) + parent.mkdir() try: source = make_source_repo(parent / "Thing") except AssertionError as exc: diff --git a/tests/test_adopt_plan_escapes.py b/tests/test_adopt_plan_escapes.py index cc1ef280..c6041a72 100644 --- a/tests/test_adopt_plan_escapes.py +++ b/tests/test_adopt_plan_escapes.py @@ -37,7 +37,8 @@ import pytest from conftest import (ADOPT, REPO, blank_unnamed_pin_sources, git, - make_source_repo, resolve) + make_source_repo, resolve, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import repo_shape # noqa: E402 @@ -398,12 +399,10 @@ def test_a_source_directory_that_is_not_utf8_is_shown_spelled(adopter, `UnicodeEncodeError` on a strict UTF-8 stdout after the plan was written. Both show it spelled now. Skipped where no directory can be named with the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" - try: - parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9"))) - parent.mkdir() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no directory here is named b'caf\\xe9': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), + b"caf\xe9"))) + parent.mkdir() try: source = make_source_repo(parent / "Thing") except AssertionError as exc: @@ -458,13 +457,7 @@ def test_a_leg_extracts_a_path_that_is_not_utf8(adopter, tmp_path): lifting that refusal reaches this, so the extraction is tested here, directly. Skipped where no file can be named with that byte (Windows, macOS's APFS): filter-repo works in a clone, which checks the name out.""" - try: - probe = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9.txt"))) - probe.write_bytes(b"") - probe.unlink() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no file here is named b'caf\\xe9.txt': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) source = odd_source(tmp_path, b"caf\xe9.txt") commit = git("rev-parse", "HEAD", cwd=source).stdout.strip() remote = tmp_path / "Northwind-code.git" diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index faf3a008..1351c43a 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -50,7 +50,8 @@ import pytest from conftest import (ADOPT, REPO, WINDOWS_SKIP, blank_unnamed_pin_sources, - make_source_repo, resolve) + make_source_repo, resolve, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -292,12 +293,10 @@ def test_execute_refuses_a_local_source_directory_that_is_not_utf8(tmp_path): path, which `_remote_repository` reads as UTF-8. Skipped where no directory can be named with the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" - try: - parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9"))) - parent.mkdir() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no directory here is named b'caf\\xe9': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), + b"caf\xe9"))) + parent.mkdir() refused_then_moved(tmp_path, parent, b"a byte that is not UTF-8") diff --git a/tests/test_clone_url_bytes.py b/tests/test_clone_url_bytes.py index 373b8de7..13c0bca5 100644 --- a/tests/test_clone_url_bytes.py +++ b/tests/test_clone_url_bytes.py @@ -44,7 +44,8 @@ import pytest -from conftest import ORG, SCAFFOLD, blank_unnamed_pin_sources +from conftest import (ORG, SCAFFOLD, blank_unnamed_pin_sources, + skip_where_a_name_cannot_be_not_utf_8) # Fixtures of the #282 file, on the harness this one shares with it: the # scaffold loaded as a module, and the guard that no process is started. from test_scaffold_free_text_values import ( # noqa: F401 - fixtures @@ -133,18 +134,34 @@ def family_refusal(url: str, reason: str) -> str: f"{FAMILY_REMEDIATION}\n") +def skip_where_this_system_cannot_name(tmp_path: Path, raw: bytes) -> None: + """Skip where `tmp_path/` cannot be made on this system. Two + questions, each asked only of the names it is about: + + - a name that is not UTF-8 goes to `skip_where_a_name_cannot_be_not_utf_8`, + which skips on the two answers that mean "this system cannot carry + it" (Windows names its files in UTF-16, and macOS answers EILSEQ) and + raises any other error of the disk; + - a control character, 0x00 to 0x1F, is refused by Windows in any name, + and a name that holds one is skipped there. + + Nothing else is skipped: U+202E, which NTFS holds, is a name Windows + makes as Linux does (#757; reasoned, no Windows run: CI is Linux only). + `tests/test_non_utf8_name_callers.py` holds the answers by emulation.""" + try: + raw.decode("utf-8") + except UnicodeDecodeError: + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + if os.name == "nt" and any(byte < 0x20 for byte in raw): + pytest.skip("Windows refuses a control character in a file name") + + def directory_named(tmp_path: Path, raw: bytes) -> Path: - """`tmp_path/` made from BYTES, empty; the test is skipped where the - disk cannot hold such a name (macOS refuses a name that is not UTF-8 and - Windows names its files in UTF-16, and refuses a control character).""" - if os.name == "nt": - pytest.skip("Windows names its files in UTF-16, which has no byte " - "that is not UTF-8, and refuses a control character") + """`tmp_path/` made from BYTES, empty; the test is skipped where this + system cannot hold such a name, and fails on any other error of the disk.""" + skip_where_this_system_cannot_name(tmp_path, raw) made = os.fsencode(tmp_path) + os.fsencode(os.sep) + raw - try: - os.mkdir(made) - except (OSError, ValueError, UnicodeError) as error: - pytest.skip(f"this disk cannot hold the name {raw!r}: {error}") + os.mkdir(made) path = Path(os.fsdecode(made)) assert path.is_dir() assert os.listdir(path) == [] diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index 4d19a998..ebe28bcd 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -123,7 +123,10 @@ def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( as text, and the tests that use it must run there.""" monkeypatch.setattr(os, "fsencode", windows_fsencode) target = tmp_path / UTF8 - assert skip_message(make_file, target, b"on: [pull_request]\n") is None + reason = skip_message(make_file, target, b"on: [pull_request]\n") + if reason is not None and "filesystem" in reason: + pytest.skip(reason) + assert reason is None, reason assert target.read_bytes() == b"on: [pull_request]\n" @@ -490,6 +493,22 @@ def test_a_clean_standard_prints_no_warning(standard, tmp_path, run, warning, "Windows converts it") LEGACY_NAME = "Jos\udce9" + +def skip_where_the_environment_cannot_carry(value: str) -> None: + """Skip where this system's file name encoding cannot encode `value`, as + under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII): `subprocess` + puts each environment value through `os.fsencode` before it starts the + child, so a `user.name` of `Jos\\u00e9` raises `UnicodeEncodeError` in the + test and not in the tool (#757). A name that is ASCII, or a lone + surrogate that `os.fsencode` gives back as its byte, is carried.""" + try: + os.fsencode(value) + except UnicodeEncodeError: + pytest.skip(f"this system's file name encoding " + f"({sys.getfilesystemencoding()}) cannot carry {value!r} " + "in an environment") + + REFUSED = "REFUSED family-unwritable-value: " #: How the refusal names the creator when no flag supplied it: `_manifest_flags` #: (#307), so that the two refusals agree on who is meant. @@ -623,6 +642,7 @@ def test_a_user_name_that_is_text_still_initialises(tmp_path, value, dry_run): exit 0, `created_by: "Jose"`. Text a file holds is not refused: an accent, a U+200C ZERO WIDTH NON-JOINER, a backslash and a quote, which the scaffold lets through as well.""" + skip_where_the_environment_cannot_carry(value) result = init(tmp_path, user_name(value), dry_run=dry_run) assert result.returncode == 0, result.stderr + result.stdout assert "REFUSED" not in result.stderr, result.stderr @@ -643,6 +663,7 @@ def test_every_other_name_no_file_can_hold_is_refused_the_same_way( asks it of the same setting: a line break, a bidi control and a control character in `git config user.name` are refused in its words, where the byte that is not UTF-8 is the case the issue lists.""" + skip_where_the_environment_cannot_carry(value) result = init(tmp_path, user_name(value), dry_run=True) assert result.returncode == 2, result.stderr + result.stdout assert result.stderr.startswith( diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py new file mode 100644 index 00000000..57f4dbc0 --- /dev/null +++ b/tests/test_non_utf8_name_callers.py @@ -0,0 +1,285 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A test that needs a directory named with a byte that is not UTF-8 asks the +one helper whether this system can carry it, and every other error of the disk +is the test's own (#757, #461). + +Five tests and `directory_named` each made the directory themselves and turned +ANY `OSError` into `pytest.skip`, beside `skip_where_a_name_cannot_be_not_utf_8` +(#663), which skips on the two answers that mean "this system cannot carry the +name" -- Windows' `os.fsencode` and macOS's EILSEQ -- and raises the rest. So a +disk that filled up, or a `tmp_path` this user cannot write in, read as SKIPPED +there and as a failure in the helper's own tests, and no run fails on a skip. + +Each caller is called here as a plain function, with the disk answering as it +would on the systems this suite does not run on (CI is Linux only, #597), and +asked what it did: + +- the PROBE is refused with EILSEQ (macOS): the test skips, on the helper's + reason; +- the PROBE is refused with EACCES: the test fails with `PermissionError`; +- the test's OWN directory is refused with EACCES, the probe not: the test + fails with `PermissionError`, where it skipped. + +Windows is emulated too, for the one caller whose own question is wider than +the helper's: `directory_named` skips there for a control character, which +Windows refuses in any file name, and for no other name that is text. No +Windows run was made. + +The last two groups hold the C-locale half of the same issue. Under +`LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` the file name encoding is ASCII, +so `make_file` cannot write `caf\\u00e9.yml` and `subprocess` cannot put a +`user.name` of `Jos\\u00e9` into a child's environment: the tests that do it +skip where this system cannot, and are held to it here with an ASCII +`os.fsencode` and a `write_bytes` that raises as it would. +""" + +from __future__ import annotations + +import errno +import os +import sys +from pathlib import Path + +import pytest + +import test_adopt_git_reads as git_reads +import test_adopt_plan_escapes as plan_escapes +import test_adopt_template_values as template_values +import test_clone_url_bytes as clone_url_bytes +import test_non_utf8_git_reads as non_utf8_git_reads +import test_stdout_encoding as stdout_encoding +from conftest import NOT_UTF_8_PROBE, skip_message, windows_fsencode +# The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a +# module, which two of its tests ask for. +from test_adopt_plan_escapes import adopter # noqa: F401 - fixture + +#: Where the probe of the helper is: a directory named `probe`. +PROBE = "probe" + +#: Every caller of the helper, as `(id, the call, whether it makes a directory +#: of its own after the probe)`; the call is given a `tmp_path` and the +#: `adopter` fixture. `directory_named` is called with the name the issue's +#: own row uses, `caf` and byte 0xE9. +CALLERS = [ + pytest.param( + lambda tmp_path, adopter: stdout_encoding + .test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes( + tmp_path, "utf-8:surrogateescape"), + True, id="stdout-encoding"), + pytest.param( + lambda tmp_path, adopter: git_reads + .test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path), + True, id="adopt-git-reads"), + pytest.param( + lambda tmp_path, adopter: template_values + .test_execute_refuses_a_local_source_directory_that_is_not_utf8( + tmp_path), + True, id="adopt-template-values"), + pytest.param( + lambda tmp_path, adopter: plan_escapes + .test_a_source_directory_that_is_not_utf8_is_shown_spelled( + adopter, tmp_path), + True, id="adopt-plan-escapes-directory"), + pytest.param( + lambda tmp_path, adopter: clone_url_bytes.directory_named( + tmp_path, b"caf\xe9"), + True, id="clone-url-bytes"), + pytest.param( + lambda tmp_path, adopter: plan_escapes + .test_a_leg_extracts_a_path_that_is_not_utf8(adopter, tmp_path), + False, id="adopt-plan-escapes-file"), +] +EVERY_CALLER = [pytest.param(row.values[0], id=row.id) for row in CALLERS] +THOSE_WITH_A_DIRECTORY_OF_THEIR_OWN = [ + pytest.param(row.values[0], id=row.id) for row in CALLERS if row.values[1]] + + +def refusing(code: int, *, probe: bool): + """An `os.mkdir` that fails with `code` for the probe (`probe=True`) or + for every directory but the probe, and makes the rest. `Path.mkdir` calls + it too.""" + real = os.mkdir + + def mkdir(path, *args, **kwargs): + name = os.path.basename(os.fsdecode(path)) + if name.startswith(PROBE) is probe: + raise OSError(code, os.strerror(code), os.fsdecode(path)) + return real(path, *args, **kwargs) + return mkdir + + +@pytest.fixture +def a_system_that_carries_the_byte(): + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + + +# --- the disk's refusals, on the systems CI does not run -------------------- + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", EVERY_CALLER) +def test_a_disk_that_refuses_the_name_with_eilseq_skips_the_caller( + tmp_path, monkeypatch, adopter, call): + """macOS (emulated): the probe is refused as `Illegal byte sequence`, so + the test is SKIPPED, on the helper's reason.""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + reason = skip_message(call, tmp_path, adopter) + assert reason is not None, "the caller went on past the probe's EILSEQ" + assert "EILSEQ" in reason, reason + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", EVERY_CALLER) +def test_a_disk_that_refuses_the_probe_with_eacces_fails_the_caller( + tmp_path, monkeypatch, adopter, call): + """A folder nobody can write in: the helper raises it, as its own tests + hold, so the test FAILS, and no run reads it as a skip.""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=True)) + with pytest.raises(PermissionError): + reason = skip_message(call, tmp_path, adopter) + pytest.fail("the caller ended " + + (f"SKIPPED: {reason}" if reason else "without an error")) + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", THOSE_WITH_A_DIRECTORY_OF_THEIR_OWN) +def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( + tmp_path, monkeypatch, adopter, call): + """The probe is made and removed, and then the directory the test means + to use is refused with EACCES: that is the test's own, raised, where it + was a skip that said "no directory here is named ...".""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=False)) + with pytest.raises(PermissionError): + reason = skip_message(call, tmp_path, adopter) + pytest.fail("the caller ended " + + (f"SKIPPED: {reason}" if reason else "without an error")) + + +# --- `directory_named`'s own question: Windows, emulated -------------------- + +def windows_names_it(monkeypatch) -> None: + """Windows, as far as `skip_where_this_system_cannot_name` can ask it: + `os.name` is `nt` and `os.fsencode` gives a surrogate three bytes.""" + monkeypatch.setattr(os, "name", "nt") + monkeypatch.setattr(os, "fsencode", windows_fsencode) + + +def test_where_windows_cannot_carry_the_byte_directory_named_skips_and_makes_nothing( + tmp_path, monkeypatch): + windows_names_it(monkeypatch) + reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, b"caf\xe9") + assert reason is not None, "a name that is not UTF-8 was not skipped" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("raw", [b"line\nbreak", b"a\x1bb"], + ids=["line-feed", "escape"]) +def test_where_windows_refuses_a_control_character_directory_named_skips( + tmp_path, monkeypatch, raw): + windows_names_it(monkeypatch) + reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, raw) + assert reason is not None, f"{raw!r} was not skipped on Windows" + assert "control character" in reason, reason + + +def test_where_windows_holds_a_bidi_override_directory_named_does_not_skip( + tmp_path, monkeypatch): + """The row #459 names (R433 O5): NTFS holds U+202E, so the `bidi-override` + row of the clone-URL tests runs on Windows as it does here. Reasoned, and + asked here only of the skip: no Windows run was made.""" + windows_names_it(monkeypatch) + raw = "\u202eevil".encode("utf-8") + assert skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, raw) is None + + +@pytest.mark.skipif(os.name == "nt", reason="Windows is the case above") +@pytest.mark.parametrize("raw", [b"line\nbreak", b"a\x1bb"], + ids=["line-feed", "escape"]) +def test_a_system_that_is_not_windows_makes_a_control_character_directory( + tmp_path, raw): + """The control: a POSIX disk takes the name, so no skip.""" + assert skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, raw) is None + assert os.path.basename(clone_url_bytes.directory_named( + tmp_path, raw)) == os.fsdecode(raw) + + +# --- under an ASCII file name encoding (LC_ALL=C PYTHONUTF8=0) -------------- + +def ascii_fsencode(value): + """`os.fsencode` where the file name encoding is ASCII, with + `surrogateescape`: text outside ASCII raises, and a lone surrogate that + stands for a byte is that byte.""" + value = os.fspath(value) + if isinstance(value, bytes): + return value + return value.encode("ascii", "surrogateescape") + + +def test_make_files_utf8_name_test_skips_where_the_encoding_is_ascii( + tmp_path, monkeypatch): + """`make_file` cannot write `caf\\u00e9.yml` where the file name encoding is + ASCII, and says so as a skip on `filesystem`. The test about the guard + reads that skip as it reads it in its neighbour, and is SKIPPED: it failed + on `assert skip_message(make_file, ...) is None`, on a system that holds + no defect. Emulated: `write_bytes` raises what the ASCII encoding does.""" + real = Path.write_bytes + + def write_bytes(self, data): + for index, char in enumerate(self.name): + if not char.isascii(): + raise UnicodeEncodeError("ascii", self.name, index, index + 1, + "ordinal not in range(128)") + return real(self, data) + monkeypatch.setattr(Path, "write_bytes", write_bytes) + reason = skip_message( + non_utf8_git_reads + .test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte, + tmp_path, monkeypatch) + assert reason is not None, "the test went on past an ASCII encoding" + assert "filesystem" in reason, reason + + +@pytest.mark.parametrize("value", ["Jos\u00e9", "Mehr\u200cdad", "\u202eevil"], + ids=["accent", "zwnj", "u202e"]) +def test_a_user_name_the_encoding_cannot_carry_skips_where_it_is_ascii( + monkeypatch, value): + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message( + non_utf8_git_reads.skip_where_the_environment_cannot_carry, value) + assert reason is not None, f"{value!r} was carried by an ASCII encoding" + assert "environment" in reason, reason + + +@pytest.mark.parametrize("value", ["Jose", "Jos\udce9", "Ada\nLovelace", + "a\x1bb"], + ids=["ascii", "byte", "line-feed", "escape"]) +def test_a_user_name_the_encoding_carries_is_not_skipped_even_where_it_is_ascii( + monkeypatch, value): + """ASCII text, a lone surrogate that is a byte, and the control characters + are the environment's own: the tests of them still run.""" + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + assert skip_message( + non_utf8_git_reads.skip_where_the_environment_cannot_carry, + value) is None + + +@pytest.mark.parametrize("test, args", [ + pytest.param("test_a_user_name_that_is_text_still_initialises", + ("Jos\u00e9", True), id="text-still-initialises"), + pytest.param("test_every_other_name_no_file_can_hold_is_refused_the_same_way", + ("\u202eevil", "it holds a bidirectional control"), + id="every-other-name"), +]) +def test_the_two_tests_that_pass_the_name_in_an_environment_skip_where_it_is_ascii( + tmp_path, monkeypatch, test, args): + """Both of them fail on the `UnicodeEncodeError` that `subprocess` raises + in the test (the tool is not reached), under an ASCII encoding.""" + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(getattr(non_utf8_git_reads, test), tmp_path, *args) + assert reason is not None, "the test ran, and raised or passed on ASCII" + assert "environment" in reason, reason diff --git a/tests/test_stdout_encoding.py b/tests/test_stdout_encoding.py index 621b43c7..d3e0f7e9 100644 --- a/tests/test_stdout_encoding.py +++ b/tests/test_stdout_encoding.py @@ -51,7 +51,8 @@ import pytest from conftest import (ADOPT, ORG, PROJECT, REPO, SCAFFOLD, - blank_unnamed_pin_sources, git, make_source_repo) + blank_unnamed_pin_sources, git, make_source_repo, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -206,11 +207,9 @@ def test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes(tmp_path, `日本.md` -- escaped on cp1252 -- where plain `backslashreplace` would print `\\udcff` instead. POSIX only: neither Windows nor macOS holds a file name that is not UTF-8.""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = os.path.join(os.fsencode(tmp_path), b"b\xffd") - try: - os.mkdir(parent) - except (OSError, UnicodeError) as exc: - pytest.skip(f"this filesystem holds no name that is not UTF-8: {exc}") + os.mkdir(parent) if b"b\xffd" not in os.listdir(os.fsencode(tmp_path)): pytest.skip("this filesystem rewrote a name that is not UTF-8") source = make_source_repo(tmp_path / "Thing", tree={ From 6b5e9074ab01fcc89381b98d97444d370c27634a Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 11:24:28 +0000 Subject: [PATCH 5/5] test_non_utf8_git_reads.py and test_non_utf8_name_callers.py: the `make_file` test asks the file name encoding first and keeps its strict assert, the rows that let the probe through ask the real disk first, and each `pytest.raises` block holds one call, so a write the disk refuses fails the test and no skip is made of it, a disk that refuses the probe skips the rows and does not fail them, and the new file has no S5778 smell (#757) A fix round for lane 1's review of #806 at 41849725 (three SHOULD FIX, one nit). 1. The `make_file` test skipped on `make_file`'s own "filesystem" reason, which `make_file` gives for ANY `OSError` of its write, so a write refused with EACCES or ENOSPC for the plain UTF-8 name `caf` and U+00E9 plus `.yml` ended the test SKIPPED, and the reason it gave was false. The test now calls `skip_where_the_environment_cannot_carry(UTF8)` first, before it swaps in the Windows `os.fsencode`, and keeps the strict `assert skip_message(make_file, ...) is None`: it skips for the file name encoding and for nothing else. The helper's reason loses " in an environment", which was wrong for a file name; the callers file reads "file name encoding". The ASCII row now patches `os.fsencode`, and two rows make `Path.write_bytes` raise EACCES and ENOSPC and expect the test to FAIL. 2. The own-directory rows let the probe through to the real disk, and their fixture asked only `os.fsencode`, so on macOS's APFS or a ZFS with `utf8only` the real probe answers EILSEQ, the helper skips, and the row failed. `a_disk_that_takes_the_probe` asks the real disk first and those rows alone use it; a row makes the disk answer EILSEQ and asks the fixture. 3. `fail_where_the_caller_ends` holds the `pytest.fail`, so each `pytest.raises` block holds one call (python:S5778, two places, and the new EACCES/ENOSPC rows). 4. The Windows rows for a byte and a control character now call `directory_named` itself, so its order (ask, then make) is held: a version that skipped after `os.mkdir` leaves a directory or fails on `os.fsencode` of a path. The bidi row asks `skip_where_this_system_cannot_name` and is named for it, because `directory_named` goes on to make a `Path`, and under an emulated `os.name` of `nt` that is a `WindowsPath`. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_non_utf8_git_reads.py | 26 +++--- tests/test_non_utf8_name_callers.py | 124 +++++++++++++++++++--------- 2 files changed, 100 insertions(+), 50 deletions(-) diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index ebe28bcd..ff81a3b3 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -120,13 +120,14 @@ def test_make_file_skips_where_fsencode_makes_no_byte_and_writes_nothing( def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( tmp_path, monkeypatch): """The guard is for the byte and no wider: Windows holds `caf\\u00e9.yml` - as text, and the tests that use it must run there.""" + as text, and the tests that use it must run there. Skipped, before the + Windows `os.fsencode` is put in, where this system's own file name + encoding cannot write `caf\\u00e9.yml` (ASCII); and for nothing else, so a + write the disk refuses fails the test (#757).""" + skip_where_the_environment_cannot_carry(UTF8) monkeypatch.setattr(os, "fsencode", windows_fsencode) target = tmp_path / UTF8 - reason = skip_message(make_file, target, b"on: [pull_request]\n") - if reason is not None and "filesystem" in reason: - pytest.skip(reason) - assert reason is None, reason + assert skip_message(make_file, target, b"on: [pull_request]\n") is None assert target.read_bytes() == b"on: [pull_request]\n" @@ -496,17 +497,18 @@ def test_a_clean_standard_prints_no_warning(standard, tmp_path, run, warning, def skip_where_the_environment_cannot_carry(value: str) -> None: """Skip where this system's file name encoding cannot encode `value`, as - under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII): `subprocess` - puts each environment value through `os.fsencode` before it starts the - child, so a `user.name` of `Jos\\u00e9` raises `UnicodeEncodeError` in the - test and not in the tool (#757). A name that is ASCII, or a lone - surrogate that `os.fsencode` gives back as its byte, is carried.""" + under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII). It is asked of + a file name (`make_file` cannot write `caf\\u00e9.yml`) and of an + environment value: `subprocess` puts each one through `os.fsencode` + before it starts the child, so a `user.name` of `Jos\\u00e9` raises + `UnicodeEncodeError` in the test and not in the tool (#757). Text that is + ASCII, or a lone surrogate that `os.fsencode` gives back as its byte, is + carried.""" try: os.fsencode(value) except UnicodeEncodeError: pytest.skip(f"this system's file name encoding " - f"({sys.getfilesystemencoding()}) cannot carry {value!r} " - "in an environment") + f"({sys.getfilesystemencoding()}) cannot carry {value!r}") REFUSED = "REFUSED family-unwritable-value: " diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 57f4dbc0..3b45ffe0 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -18,26 +18,30 @@ reason; - the PROBE is refused with EACCES: the test fails with `PermissionError`; - the test's OWN directory is refused with EACCES, the probe not: the test - fails with `PermissionError`, where it skipped. + fails with `PermissionError`, where it skipped. These rows let the probe + through to the real disk, so they ask it first and skip where it answers + EILSEQ (macOS's APFS, a ZFS with `utf8only`): they would FAIL there on a + system that holds no defect. Windows is emulated too, for the one caller whose own question is wider than the helper's: `directory_named` skips there for a control character, which Windows refuses in any file name, and for no other name that is text. No Windows run was made. -The last two groups hold the C-locale half of the same issue. Under +The last groups hold the C-locale half of the same issue. Under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` the file name encoding is ASCII, so `make_file` cannot write `caf\\u00e9.yml` and `subprocess` cannot put a -`user.name` of `Jos\\u00e9` into a child's environment: the tests that do it -skip where this system cannot, and are held to it here with an ASCII -`os.fsencode` and a `write_bytes` that raises as it would. +`user.name` of `Jos\\u00e9` into a child's environment: the three tests that +do it skip where this system's encoding cannot, held here with an ASCII +`os.fsencode`, and for that reason ALONE: a write the disk refuses (EACCES, +ENOSPC) still fails the `make_file` test, which a skip on `make_file`'s own +"filesystem" reason would have turned into a skip. """ from __future__ import annotations import errno import os -import sys from pathlib import Path import pytest @@ -48,7 +52,8 @@ import test_clone_url_bytes as clone_url_bytes import test_non_utf8_git_reads as non_utf8_git_reads import test_stdout_encoding as stdout_encoding -from conftest import NOT_UTF_8_PROBE, skip_message, windows_fsencode +from conftest import (NOT_UTF_8_PROBE, skip_message, + skip_where_a_name_cannot_be_not_utf_8, windows_fsencode) # The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a # module, which two of its tests ask for. from test_adopt_plan_escapes import adopter # noqa: F401 - fixture @@ -108,12 +113,29 @@ def mkdir(path, *args, **kwargs): return mkdir +def fail_where_the_caller_ends(call, *args) -> None: + """Fail where `call(*args)` ends without an error, SKIPPED or not: the + caller was to raise. A `pytest.raises` block may hold one call and no + more (python:S5778), so the failure is made here and not beside it.""" + reason = skip_message(call, *args) + pytest.fail("the caller ended " + + (f"SKIPPED: {reason}" if reason else "without an error")) + + @pytest.fixture def a_system_that_carries_the_byte(): if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: pytest.skip("on this system the first question is the one that skips") +@pytest.fixture +def a_disk_that_takes_the_probe(tmp_path, a_system_that_carries_the_byte): + """The rows that let the probe through to the real disk ask it first: + macOS's APFS, or a ZFS with `utf8only`, answers EILSEQ to it, and those + rows would FAIL there on a system that holds no defect.""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + + # --- the disk's refusals, on the systems CI does not run -------------------- @pytest.mark.usefixtures("a_system_that_carries_the_byte") @@ -136,12 +158,10 @@ def test_a_disk_that_refuses_the_probe_with_eacces_fails_the_caller( hold, so the test FAILS, and no run reads it as a skip.""" monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=True)) with pytest.raises(PermissionError): - reason = skip_message(call, tmp_path, adopter) - pytest.fail("the caller ended " - + (f"SKIPPED: {reason}" if reason else "without an error")) + fail_where_the_caller_ends(call, tmp_path, adopter) -@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") @pytest.mark.parametrize("call", THOSE_WITH_A_DIRECTORY_OF_THEIR_OWN) def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( tmp_path, monkeypatch, adopter, call): @@ -150,9 +170,20 @@ def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( was a skip that said "no directory here is named ...".""" monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=False)) with pytest.raises(PermissionError): - reason = skip_message(call, tmp_path, adopter) - pytest.fail("the caller ended " - + (f"SKIPPED: {reason}" if reason else "without an error")) + fail_where_the_caller_ends(call, tmp_path, adopter) + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +def test_the_fixture_that_guards_those_rows_asks_the_real_disk( + tmp_path, monkeypatch, request): + """`a_disk_that_takes_the_probe` skips where the disk answers EILSEQ to the + probe, as APFS does; asked here after the disk is made to, because a + fixture runs before the test body and Linux's disk would never say it.""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + reason = skip_message(request.getfixturevalue, + "a_disk_that_takes_the_probe") + assert reason is not None, "the fixture let a refused probe through" + assert "EILSEQ" in reason, reason # --- `directory_named`'s own question: Windows, emulated -------------------- @@ -166,9 +197,12 @@ def windows_names_it(monkeypatch) -> None: def test_where_windows_cannot_carry_the_byte_directory_named_skips_and_makes_nothing( tmp_path, monkeypatch): + """`directory_named` itself is called, so that it asks before it makes + anything: a version that skipped after its `os.mkdir` would leave a + directory here, or fail on `os.fsencode` of a path under the emulation.""" windows_names_it(monkeypatch) - reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, - tmp_path, b"caf\xe9") + reason = skip_message(clone_url_bytes.directory_named, tmp_path, + b"caf\xe9") assert reason is not None, "a name that is not UTF-8 was not skipped" assert "os.fsencode" in reason, reason assert list(tmp_path.iterdir()) == [] @@ -179,17 +213,20 @@ def test_where_windows_cannot_carry_the_byte_directory_named_skips_and_makes_not def test_where_windows_refuses_a_control_character_directory_named_skips( tmp_path, monkeypatch, raw): windows_names_it(monkeypatch) - reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, - tmp_path, raw) + reason = skip_message(clone_url_bytes.directory_named, tmp_path, raw) assert reason is not None, f"{raw!r} was not skipped on Windows" assert "control character" in reason, reason + assert list(tmp_path.iterdir()) == [] -def test_where_windows_holds_a_bidi_override_directory_named_does_not_skip( +def test_where_windows_holds_a_bidi_override_the_name_check_does_not_skip( tmp_path, monkeypatch): """The row #459 names (R433 O5): NTFS holds U+202E, so the `bidi-override` row of the clone-URL tests runs on Windows as it does here. Reasoned, and - asked here only of the skip: no Windows run was made.""" + asked here only of the skip: no Windows run was made. It asks + `skip_where_this_system_cannot_name` and not `directory_named`, which + goes on to make a `Path`, and under an emulated `os.name` of `nt` that is + a `WindowsPath`, which Linux cannot make.""" windows_names_it(monkeypatch) raw = "\u202eevil".encode("utf-8") assert skip_message(clone_url_bytes.skip_where_this_system_cannot_name, @@ -220,28 +257,39 @@ def ascii_fsencode(value): return value.encode("ascii", "surrogateescape") +MAKE_FILE_TEST = ( + non_utf8_git_reads + .test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte) + + def test_make_files_utf8_name_test_skips_where_the_encoding_is_ascii( tmp_path, monkeypatch): """`make_file` cannot write `caf\\u00e9.yml` where the file name encoding is - ASCII, and says so as a skip on `filesystem`. The test about the guard - reads that skip as it reads it in its neighbour, and is SKIPPED: it failed - on `assert skip_message(make_file, ...) is None`, on a system that holds - no defect. Emulated: `write_bytes` raises what the ASCII encoding does.""" - real = Path.write_bytes + ASCII, so the test about the guard failed on `assert skip_message(...) + is None`, on a system that holds no defect. It asks the encoding first, + and is SKIPPED on that reason.""" + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(MAKE_FILE_TEST, tmp_path, monkeypatch) + assert reason is not None, "the test went on past an ASCII encoding" + assert "file name encoding" in reason, reason + + +@pytest.mark.parametrize("code", [errno.EACCES, errno.ENOSPC], + ids=["EACCES", "ENOSPC"]) +def test_a_write_the_disk_refuses_still_fails_the_make_file_test( + tmp_path, monkeypatch, code): + """The name is plain UTF-8, so a refused write is the disk's and not the + encoding's: the test FAILS on its assert, as it does without this change. + A skip on `make_file`'s own "filesystem" reason would end it SKIPPED, and + no run fails on that.""" + non_utf8_git_reads.skip_where_the_environment_cannot_carry( + non_utf8_git_reads.UTF8) def write_bytes(self, data): - for index, char in enumerate(self.name): - if not char.isascii(): - raise UnicodeEncodeError("ascii", self.name, index, index + 1, - "ordinal not in range(128)") - return real(self, data) + raise OSError(code, os.strerror(code), str(self)) monkeypatch.setattr(Path, "write_bytes", write_bytes) - reason = skip_message( - non_utf8_git_reads - .test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte, - tmp_path, monkeypatch) - assert reason is not None, "the test went on past an ASCII encoding" - assert "filesystem" in reason, reason + with pytest.raises(AssertionError): + fail_where_the_caller_ends(MAKE_FILE_TEST, tmp_path, monkeypatch) @pytest.mark.parametrize("value", ["Jos\u00e9", "Mehr\u200cdad", "\u202eevil"], @@ -252,7 +300,7 @@ def test_a_user_name_the_encoding_cannot_carry_skips_where_it_is_ascii( reason = skip_message( non_utf8_git_reads.skip_where_the_environment_cannot_carry, value) assert reason is not None, f"{value!r} was carried by an ASCII encoding" - assert "environment" in reason, reason + assert "file name encoding" in reason, reason @pytest.mark.parametrize("value", ["Jose", "Jos\udce9", "Ada\nLovelace", @@ -282,4 +330,4 @@ def test_the_two_tests_that_pass_the_name_in_an_environment_skip_where_it_is_asc monkeypatch.setattr(os, "fsencode", ascii_fsencode) reason = skip_message(getattr(non_utf8_git_reads, test), tmp_path, *args) assert reason is not None, "the test ran, and raised or passed on ASCII" - assert "environment" in reason, reason + assert "file name encoding" in reason, reason