diff --git a/tests/conftest.py b/tests/conftest.py index 243b334f..06669dd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -732,8 +732,8 @@ def clear_ambient_pin_sources(monkeypatch: pytest.MonkeyPatch) -> None: def run_script(script: Path, *args: str, cwd: Path | None = None, env: dict | None = None, input: str | None = None, - stdin: int | None = None, - merge_stderr: bool = False) -> subprocess.CompletedProcess: + stdin: int | None = None, merge_stderr: bool = False, + encoding: str | None = None) -> subprocess.CompletedProcess: """Run one tool under test, with stdin closed by default. A test in this suite is often asserting what a tool does when nobody is @@ -754,6 +754,12 @@ def run_script(script: Path, *args: str, cwd: Path | None = None, does, and leaves `stderr` None. A tool's piped stdout is block-buffered and its stderr is not, so this is the only way a test sees the order a person reading that log sees. + + `encoding` is what the output is decoded in, and None is this process's + own locale, as `subprocess.run(text=True)` reads it. A caller that tells + the child `PYTHONIOENCODING=utf-8` names it too (#1196): a parent in a + locale of one byte to a character, Latin-1 say, would read the two bytes + of U+00E9 as two characters. """ asked = env or {} env = {**os.environ, **asked} @@ -794,8 +800,8 @@ def run_script(script: Path, *args: str, cwd: Path | None = None, stdout=subprocess.PIPE, stderr=(subprocess.STDOUT if merge_stderr else subprocess.PIPE), - text=True, check=False, env=env, input=input, - stdin=stdin) + text=True, encoding=encoding, check=False, env=env, + input=input, stdin=stdin) def scaffold_dry_run_on_github(monkeypatch, capsys, *args: str): @@ -1156,19 +1162,32 @@ def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None: def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not - UTF-8 cannot be made under `directory`. Two systems say so: + UTF-8 cannot be made under `directory`. Three answers say so: - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first, before the disk is touched; + - a locale of one byte to a character, Latin-1 say (#1196), asked second, + also before the disk is touched: there the byte 0xFF is the character + U+00FF, so `os.fsdecode` gives it as text and not as the surrogate + U+DCFF, and a name that holds it is not one that is not UTF-8. A tool + prints it as it is, where the test expects it spelled. The disk and + `os.fsencode` take the byte in every locale, so only `os.fsdecode` can + say. UTF-8 and ASCII give U+DCFF, and neither skips; - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte sequence"), so no clone can sit at such a path there. - The second is asked by making a directory `probe` under + The third is asked by making a directory `probe` under `directory` and removing it; any other error is the test's own and is raised. """ probe = directory / f"probe{NOT_UTF_8_PROBE}" skip_where_fsencode_gives_a_surrogate_no_byte(probe.name) + read = os.fsdecode(b"\xff") + if read != NOT_UTF_8_PROBE: + pytest.skip(f"this system's file name encoding " + f"({sys.getfilesystemencoding()}) reads the byte 0xFF as " + f"{ascii(read)}, not as {ascii(NOT_UTF_8_PROBE)}, so no " + f"file name here is one that is not UTF-8") try: probe.mkdir() except OSError as exc: diff --git a/tests/test_adopt_plan_paths_spelled.py b/tests/test_adopt_plan_paths_spelled.py index 3cbf5170..5ef0342f 100644 --- a/tests/test_adopt_plan_paths_spelled.py +++ b/tests/test_adopt_plan_paths_spelled.py @@ -29,6 +29,17 @@ `plan` says, and each is run here: `execute` keeps the clone it makes and hands `Source.clone` no list of them, where `plan` and `check` do. +A LOCALE OF ONE BYTE TO A CHARACTER, Latin-1 say, is not one these rows ran +under, and the tool is right in it (#1196, from #1185's review). Every child +is told `PYTHONIOENCODING=utf-8`, so every run reads what it wrote as UTF-8, +and the two bytes of U+00E9 are one character in the parent too. The row of a +`--work-dir` that is not UTF-8 skips where the locale reads the byte 0xFF as +U+00FF, as Latin-1 does: that is a printable name there, which `_shown` +prints as it was typed, and the row's premise, that Python reads it in as +U+DCFF, does not hold (`skip_where_a_name_cannot_be_not_utf_8` asks). A +character Latin-1 cannot spell, U+202E and the rest, is skipped as under +ASCII. + THE TEST SOURCE IS ASCII: every character that is not printable is written as an escape. """ @@ -43,7 +54,8 @@ import pytest -from conftest import (ADOPT, NOT_UTF_8_PROBE, make_source_repo, run_script, +from conftest import (ADOPT, IDENTITY_EMAIL, IDENTITY_NAME, NOT_UTF_8_PROBE, + blank_unnamed_pin_sources, make_source_repo, run_script, skip_where_a_name_cannot_be_not_utf_8, skip_where_the_environment_cannot_carry) @@ -95,7 +107,7 @@ def plan_of_a_local_source(tmp_path: Path, out: Path | str, is written there.""" source = make_source_repo(tmp_path / PROJECT, tree=TREE, edits=()) return run_script(ADOPT, *plan_arguments(str(source), out), cwd=cwd, - env={"PYTHONIOENCODING": "utf-8"}) + env={"PYTHONIOENCODING": "utf-8"}, encoding="utf-8") def a_gh_that_refuses(base: Path) -> dict: @@ -122,7 +134,8 @@ def plan_of_a_repository_on_github(tmp_path: Path, stand_in = a_gh_that_refuses(tmp_path) return run_script( ADOPT, *plan_arguments(REPOSITORY, tmp_path / "plan.yaml", - "--work-dir", str(work)), env=stand_in) + "--work-dir", str(work)), env=stand_in, + encoding="utf-8") def where_the_clone_goes(work: Path) -> str: @@ -289,15 +302,32 @@ def test_a_printable_non_ascii_work_dir_is_named_as_it_was_typed(tmp_path): ], result.stdout +def the_environment_of_a_child_read_as_bytes(tmp_path: Path) -> dict: + """The environment `run_script` gives a child, with the stand-in `gh` that + refuses and a `surrogateescape` stdout (#1196): no lane, no pin-source + override of the machine running the suite, and an identity git can commit + with. The run below ends at the clone refusal, before a pin is read or a + commit is made, so none of the three changes what it says; they are here + so that this child is started as every other one is, as + `test_stdout_encoding.py`'s `run_tool_bytes` starts its own.""" + environment = blank_unnamed_pin_sources( + {**os.environ, "LANES_LANE": "", **a_gh_that_refuses(tmp_path), + "PYTHONIOENCODING": "utf-8:surrogateescape"}, {}) + for name, value in (("GIT_AUTHOR_NAME", IDENTITY_NAME), + ("GIT_AUTHOR_EMAIL", IDENTITY_EMAIL), + ("GIT_COMMITTER_NAME", IDENTITY_NAME), + ("GIT_COMMITTER_EMAIL", IDENTITY_EMAIL)): + environment.setdefault(name, value) + return environment + + def plan_of_a_repository_on_github_as_bytes( tmp_path: Path, work: Path) -> tuple[int, bytes, bytes]: """`(exit, stdout, stderr)` of `plan --source testorg/Thing --work-dir ` with the stand-in `gh` that refuses, and a `surrogateescape` stdout: the bytes the child wrote. `run_script` decodes them in the parent's own locale, which a raw byte that is not UTF-8 fails.""" - environment = {**os.environ, "LANES_LANE": "", - **a_gh_that_refuses(tmp_path), - "PYTHONIOENCODING": "utf-8:surrogateescape"} + environment = the_environment_of_a_child_read_as_bytes(tmp_path) arguments = plan_arguments(REPOSITORY, tmp_path / "plan.yaml", "--work-dir", str(work)) proc = subprocess.run([sys.executable, str(ADOPT), *arguments], @@ -312,7 +342,8 @@ def test_a_work_dir_that_is_not_utf8_is_spelled_and_no_raw_byte_is_written( it in as U+DCFF, which a `surrogateescape` stdout writes back as the raw byte. The line is the one `_shown` writes, `\\udcff` between backticks, and no raw byte reaches stdout. The `--out` row of `test_stdout_encoding.py` - holds the same for `plan written to`.""" + holds the same for `plan written to`. Skipped where the locale reads the + byte as U+00FF, Latin-1 say, where it is the printable `wdk`.""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) work = tmp_path / f"wd{NOT_UTF_8_PROBE}k" code, stdout, stderr = plan_of_a_repository_on_github_as_bytes( @@ -365,7 +396,7 @@ def test_each_of_the_two_paths_is_spelled_in_one_run(tmp_path): work = tmp_path / f"wd{typed}k" result = run_script( ADOPT, *plan_arguments(REPOSITORY, out, "--work-dir", str(work)), - env=a_gh_that_clones(tmp_path, origin)) + env=a_gh_that_clones(tmp_path, origin), encoding="utf-8") assert result.returncode == 0, result.stdout + result.stderr assert out.is_file(), result.stdout base = os.path.realpath(tmp_path) @@ -390,7 +421,7 @@ def a_command_on_github(tmp_path: Path, command: str, assert planned.returncode == 0, planned.stdout + planned.stderr return run_script(ADOPT, command, "--plan", str(plan), "--source", REPOSITORY, "--work-dir", str(work), - env=a_gh_that_refuses(tmp_path)) + env=a_gh_that_refuses(tmp_path), encoding="utf-8") @pytest.mark.parametrize("command", ["check", "execute"]) diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index e4123a2e..acb5a63d 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -128,8 +128,12 @@ def fail_where_the_caller_ends(call, *args, said: str = "") -> None: @pytest.fixture def a_system_that_carries_the_byte(): - if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: - pytest.skip("on this system the first question is the one that skips") + """The rows that let the probe through to the disk ask the system's + question and the locale's first (#1196): the helper skips on either, on + its own reason, and not on the one these rows hold.""" + if (len(os.fsencode(NOT_UTF_8_PROBE)) != 1 + or os.fsdecode(b"\xff") != NOT_UTF_8_PROBE): + pytest.skip("on this system an earlier question is the one that skips") @pytest.fixture diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 89613f03..e7466f8b 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -15,6 +15,14 @@ surrogate three bytes, as Windows' does. No Windows run was made. macOS's EILSEQ is emulated the same way, by a `mkdir` that answers it. +The locale's question is asked between the two (#1196): in a locale of one +byte to a character, Latin-1 say, `os.fsdecode(b"\\xff")` is the character +U+00FF and not the surrogate U+DCFF, so a name that holds the byte is text +there, and a test that expects it spelled as a name that is not UTF-8 would +fail on a tool that is right. `os.fsdecode` is EMULATED too, for one test: no +Latin-1 locale is built here, as `tests/test_adopt_one_byte_locale.py` builds +one for a child. + The two tools the tests below are written with are held here too: `skip_message`, which every one of them reads a skip with, and `windows_fsencode`, the emulation itself. @@ -47,6 +55,28 @@ "asked by the tests below that emulate them") +def fsdecode_in(encoding: str): + """`os.fsdecode` as a locale whose file name encoding is `encoding` has + it: text is returned as it is, and bytes are read in `encoding`, with + `surrogateescape` for each one it cannot read (a UTF-8 locale gives the + byte 0xFF as U+DCFF, and a Latin-1 one as U+00FF).""" + def fsdecode(filename): + filename = os.fspath(filename) + if isinstance(filename, str): + return filename + return filename.decode(encoding, "surrogateescape") + return fsdecode + + +def skip_where_an_earlier_question_skips() -> None: + """The rows that let the probe through to the real disk ask the system's + question and the locale's first (#1196), and skip where either would: the + helper then skips on its own reason, and not on the one they hold.""" + if (len(os.fsencode(NOT_UTF_8_PROBE)) != 1 + or os.fsdecode(b"\xff") != NOT_UTF_8_PROBE): + pytest.skip("on this system an earlier question is the one that skips") + + def mkdir_answers(code: int, said: str): """A `Path.mkdir` that fails as the disk would with `code`.""" def refuse(self, *args, **kwargs): @@ -151,6 +181,7 @@ def test_where_fsencode_makes_three_bytes_a_name_that_is_text_is_not_skipped( def test_a_disk_that_takes_the_name_is_not_skipped_and_the_probe_is_removed( tmp_path): """The control: no skip, and nothing is left in the directory.""" + skip_where_an_earlier_question_skips() assert skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) is None assert list(tmp_path.iterdir()) == [] @@ -169,11 +200,40 @@ def test_where_fsencode_makes_three_bytes_the_disk_is_not_asked( assert "os.fsencode" in reason, reason +@LINUX_ONLY +@pytest.mark.parametrize("encoding", ["utf-8", "ascii"]) +def test_a_locale_that_reads_the_byte_as_the_surrogate_is_not_skipped( + tmp_path, monkeypatch, encoding): + """The control of the locale's question (#1196): UTF-8 and ASCII give the + byte 0xFF back as U+DCFF, which is `NOT_UTF_8_PROBE`, so no test that + asks is skipped, and the probe is removed.""" + monkeypatch.setattr(os, "fsdecode", fsdecode_in(encoding)) + assert os.fsdecode(b"\xff") == NOT_UTF_8_PROBE + assert skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path) is None + assert list(tmp_path.iterdir()) == [] + + +def test_a_locale_that_reads_the_byte_as_a_character_skips_before_the_disk( + tmp_path, monkeypatch): + """Latin-1 (emulated): the byte 0xFF is U+00FF, a printable character, + so no name holding it is one that is not UTF-8, and the name is skipped. + THE LOCALE IS ASKED BEFORE THE DISK (#1196), as the system is: the + directory does not exist, so an `mkdir` under it would be + FileNotFoundError and not a skip, and the skip is the answer.""" + monkeypatch.setattr(os, "fsdecode", fsdecode_in("iso8859-1")) + assert os.fsdecode(b"\xff") == "\u00ff" + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path / "absent") + assert reason is not None, "the helper went on to ask the disk" + assert "0xFF" in reason, reason + assert "'\\xff'" in reason, reason + + def test_a_disk_that_answers_eilseq_is_skipped(tmp_path, monkeypatch): """macOS (emulated): APFS answers EILSEQ, `Illegal byte sequence`, to a file name that is not UTF-8, and no clone can sit at such a path.""" - if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: - pytest.skip("on this system the first question is the one that skips") + skip_where_an_earlier_question_skips() monkeypatch.setattr(Path, "mkdir", mkdir_answers(errno.EILSEQ, "Illegal byte sequence")) reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) @@ -188,8 +248,7 @@ def test_any_other_error_from_the_disk_is_the_tests_own( """Only EILSEQ is the filesystem's refusal of the name: a full disk, a folder nobody can write in or a probe left by an earlier run is a failure of the test, raised as it was, and never a skip that hides it.""" - if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: - pytest.skip("on this system the first question is the one that skips") + skip_where_an_earlier_question_skips() monkeypatch.setattr(Path, "mkdir", mkdir_answers(code, os.strerror(code))) with pytest.raises(OSError) as raised: skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) diff --git a/tests/test_run_script_encoding.py b/tests/test_run_script_encoding.py new file mode 100644 index 00000000..28cce0ad --- /dev/null +++ b/tests/test_run_script_encoding.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`run_script` decodes a child's output in the encoding the caller names +(#1196, from #1185's review). + +A child told `PYTHONIOENCODING=utf-8` writes U+00E9 as the two bytes 0xC3 +0xA9. `run_script` read them in the PARENT's locale, and a parent in a locale +of one byte to a character, Latin-1 say, read two characters: the rows of +`tests/test_adopt_plan_paths_spelled.py` that name `pl\\u00e9an.yaml` failed +on a tool that was right. They name `encoding="utf-8"` now. Each row here +names an encoding that reads the two bytes one way and no other, so the +keyword is held in any locale the suite runs in, with no Latin-1 locale +built: the child is the same, and only the name differs. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from conftest import run_script + +#: A child that writes the two bytes of U+00E9 in UTF-8 to stdout and to +#: stderr, as one told `PYTHONIOENCODING=utf-8` does. +SAYS_E_ACUTE = ("import sys\n" + "sys.stdout.buffer.write(bytes([0xC3, 0xA9]))\n" + "sys.stderr.buffer.write(bytes([0xC3, 0xA9]))\n") + + +@pytest.mark.parametrize("encoding, read", [ + pytest.param("utf-8", "\u00e9", id="utf-8-reads-one-character"), + pytest.param("latin-1", "\u00c3\u00a9", + id="latin-1-reads-two"), +]) +def test_the_output_is_read_in_the_encoding_the_caller_names( + tmp_path, encoding, read): + """Stdout and stderr both, as the named encoding reads the two bytes: + one character in UTF-8 and two in Latin-1. A `run_script` that dropped + the keyword would read both in this process's own locale, which is one + answer for the two rows, so one of them would fail.""" + script = tmp_path / "say_e_acute.py" + script.write_text(SAYS_E_ACUTE, encoding="ascii") + result = run_script(script, encoding=encoding) + assert result.returncode == 0, result.stderr + assert result.stdout == read + assert result.stderr == read