tests/conftest.py and tests/test_adopt_plan_paths_spelled.py: under a one-byte locale such as Latin-1 the 0xff --work-dir row skips and the two U+00E9 rows read the child's UTF-8, so none of the three fails on a tool that is right, and the bytes helper starts its child as run_script does (#1196)

tests/conftest.py

@@ -732,8 +732,8 @@ def clear_ambient_pin_sources(monkeypatch: pytest.MonkeyPatch) -> None:
 
 def run_script(script: Path, *args: str, cwd: Path | None = None,
                env: dict | None = None, input: str | None = None,
-               stdin: int | None = None,
-               merge_stderr: bool = False) -> subprocess.CompletedProcess:
+               stdin: int | None = None, merge_stderr: bool = False,
+               encoding: str | None = None) -> subprocess.CompletedProcess:
     """Run one tool under test, with stdin closed by default.
 
     A test in this suite is often asserting what a tool does when nobody is
@@ -754,6 +754,12 @@ def run_script(script: Path, *args: str, cwd: Path | None = None,
     does, and leaves `stderr` None. A tool's piped stdout is block-buffered
     and its stderr is not, so this is the only way a test sees the order a
     person reading that log sees.
+
+    `encoding` is what the output is decoded in, and None is this process's
+    own locale, as `subprocess.run(text=True)` reads it. A caller that tells
+    the child `PYTHONIOENCODING=utf-8` names it too (#1196): a parent in a
+    locale of one byte to a character, Latin-1 say, would read the two bytes
+    of U+00E9 as two characters.
     """
     asked = env or {}
     env = {**os.environ, **asked}
@@ -794,8 +800,8 @@ def run_script(script: Path, *args: str, cwd: Path | None = None,
                           stdout=subprocess.PIPE,
                           stderr=(subprocess.STDOUT if merge_stderr
                                   else subprocess.PIPE),
-                          text=True, check=False, env=env, input=input,
-                          stdin=stdin)
+                          text=True, encoding=encoding, check=False, env=env,
+                          input=input, stdin=stdin)
 
 
 def scaffold_dry_run_on_github(monkeypatch, capsys, *args: str):
@@ -1156,19 +1162,32 @@ def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None:
 
 def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None:
     """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not
-    UTF-8 cannot be made under `directory`. Two systems say so:
+    UTF-8 cannot be made under `directory`. Three answers say so:
 
     - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first,
       before the disk is touched;
+    - a locale of one byte to a character, Latin-1 say (#1196), asked second,
+      also before the disk is touched: there the byte 0xFF is the character
+      U+00FF, so `os.fsdecode` gives it as text and not as the surrogate
+      U+DCFF, and a name that holds it is not one that is not UTF-8. A tool
+      prints it as it is, where the test expects it spelled. The disk and
+      `os.fsencode` take the byte in every locale, so only `os.fsdecode` can
+      say. UTF-8 and ASCII give U+DCFF, and neither skips;
     - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte
       sequence"), so no clone can sit at such a path there.
 
-    The second is asked by making a directory `probe<U+DCFF>` under
+    The third is asked by making a directory `probe<U+DCFF>` under
     `directory` and removing it; any other error is the test's own and is
     raised.
     """
     probe = directory / f"probe{NOT_UTF_8_PROBE}"
     skip_where_fsencode_gives_a_surrogate_no_byte(probe.name)
+    read = os.fsdecode(b"\xff")
+    if read != NOT_UTF_8_PROBE:
+        pytest.skip(f"this system's file name encoding "
+                    f"({sys.getfilesystemencoding()}) reads the byte 0xFF as "
+                    f"{ascii(read)}, not as {ascii(NOT_UTF_8_PROBE)}, so no "
+                    f"file name here is one that is not UTF-8")
     try:
         probe.mkdir()
     except OSError as exc:

tests/test_adopt_plan_paths_spelled.py

@@ -29,6 +29,17 @@
 `plan` says, and each is run here: `execute` keeps the clone it makes and
 hands `Source.clone` no list of them, where `plan` and `check` do.
 
+A LOCALE OF ONE BYTE TO A CHARACTER, Latin-1 say, is not one these rows ran
+under, and the tool is right in it (#1196, from #1185's review). Every child
+is told `PYTHONIOENCODING=utf-8`, so every run reads what it wrote as UTF-8,
+and the two bytes of U+00E9 are one character in the parent too. The row of a
+`--work-dir` that is not UTF-8 skips where the locale reads the byte 0xFF as
+U+00FF, as Latin-1 does: that is a printable name there, which `_shown`
+prints as it was typed, and the row's premise, that Python reads it in as
+U+DCFF, does not hold (`skip_where_a_name_cannot_be_not_utf_8` asks). A
+character Latin-1 cannot spell, U+202E and the rest, is skipped as under
+ASCII.
+
 THE TEST SOURCE IS ASCII: every character that is not printable is written as
 an escape.
 """
@@ -43,7 +54,8 @@
 
 import pytest
 
-from conftest import (ADOPT, NOT_UTF_8_PROBE, make_source_repo, run_script,
+from conftest import (ADOPT, IDENTITY_EMAIL, IDENTITY_NAME, NOT_UTF_8_PROBE,
+                      blank_unnamed_pin_sources, make_source_repo, run_script,
                       skip_where_a_name_cannot_be_not_utf_8,
                       skip_where_the_environment_cannot_carry)
 
@@ -95,7 +107,7 @@ def plan_of_a_local_source(tmp_path: Path, out: Path | str,
     is written there."""
     source = make_source_repo(tmp_path / PROJECT, tree=TREE, edits=())
     return run_script(ADOPT, *plan_arguments(str(source), out), cwd=cwd,
-                      env={"PYTHONIOENCODING": "utf-8"})
+                      env={"PYTHONIOENCODING": "utf-8"}, encoding="utf-8")
 
 
 def a_gh_that_refuses(base: Path) -> dict:
@@ -122,7 +134,8 @@ def plan_of_a_repository_on_github(tmp_path: Path,
     stand_in = a_gh_that_refuses(tmp_path)
     return run_script(
         ADOPT, *plan_arguments(REPOSITORY, tmp_path / "plan.yaml",
-                               "--work-dir", str(work)), env=stand_in)
+                               "--work-dir", str(work)), env=stand_in,
+        encoding="utf-8")
 
 
 def where_the_clone_goes(work: Path) -> str:
@@ -289,15 +302,32 @@ def test_a_printable_non_ascii_work_dir_is_named_as_it_was_typed(tmp_path):
     ], result.stdout
 
 
+def the_environment_of_a_child_read_as_bytes(tmp_path: Path) -> dict:
+    """The environment `run_script` gives a child, with the stand-in `gh` that
+    refuses and a `surrogateescape` stdout (#1196): no lane, no pin-source
+    override of the machine running the suite, and an identity git can commit
+    with. The run below ends at the clone refusal, before a pin is read or a
+    commit is made, so none of the three changes what it says; they are here
+    so that this child is started as every other one is, as
+    `test_stdout_encoding.py`'s `run_tool_bytes` starts its own."""
+    environment = blank_unnamed_pin_sources(
+        {**os.environ, "LANES_LANE": "", **a_gh_that_refuses(tmp_path),
+         "PYTHONIOENCODING": "utf-8:surrogateescape"}, {})
+    for name, value in (("GIT_AUTHOR_NAME", IDENTITY_NAME),
+                        ("GIT_AUTHOR_EMAIL", IDENTITY_EMAIL),
+                        ("GIT_COMMITTER_NAME", IDENTITY_NAME),
+                        ("GIT_COMMITTER_EMAIL", IDENTITY_EMAIL)):
+        environment.setdefault(name, value)
+    return environment
+
+
 def plan_of_a_repository_on_github_as_bytes(
         tmp_path: Path, work: Path) -> tuple[int, bytes, bytes]:
     """`(exit, stdout, stderr)` of `plan --source testorg/Thing --work-dir
     <work>` with the stand-in `gh` that refuses, and a `surrogateescape`
     stdout: the bytes the child wrote. `run_script` decodes them in the
     parent's own locale, which a raw byte that is not UTF-8 fails."""
-    environment = {**os.environ, "LANES_LANE": "",
-                   **a_gh_that_refuses(tmp_path),
-                   "PYTHONIOENCODING": "utf-8:surrogateescape"}
+    environment = the_environment_of_a_child_read_as_bytes(tmp_path)
     arguments = plan_arguments(REPOSITORY, tmp_path / "plan.yaml",
                                "--work-dir", str(work))
     proc = subprocess.run([sys.executable, str(ADOPT), *arguments],
@@ -312,7 +342,8 @@ def test_a_work_dir_that_is_not_utf8_is_spelled_and_no_raw_byte_is_written(
     it in as U+DCFF, which a `surrogateescape` stdout writes back as the raw
     byte. The line is the one `_shown` writes, `\\udcff` between backticks, and
     no raw byte reaches stdout. The `--out` row of `test_stdout_encoding.py`
-    holds the same for `plan written to`."""
+    holds the same for `plan written to`. Skipped where the locale reads the
+    byte as U+00FF, Latin-1 say, where it is the printable `wd<U+00FF>k`."""
     skip_where_a_name_cannot_be_not_utf_8(tmp_path)
     work = tmp_path / f"wd{NOT_UTF_8_PROBE}k"
     code, stdout, stderr = plan_of_a_repository_on_github_as_bytes(
@@ -365,7 +396,7 @@ def test_each_of_the_two_paths_is_spelled_in_one_run(tmp_path):
     work = tmp_path / f"wd{typed}k"
     result = run_script(
         ADOPT, *plan_arguments(REPOSITORY, out, "--work-dir", str(work)),
-        env=a_gh_that_clones(tmp_path, origin))
+        env=a_gh_that_clones(tmp_path, origin), encoding="utf-8")
     assert result.returncode == 0, result.stdout + result.stderr
     assert out.is_file(), result.stdout
     base = os.path.realpath(tmp_path)
@@ -390,7 +421,7 @@ def a_command_on_github(tmp_path: Path, command: str,
     assert planned.returncode == 0, planned.stdout + planned.stderr
     return run_script(ADOPT, command, "--plan", str(plan), "--source",
                       REPOSITORY, "--work-dir", str(work),
-                      env=a_gh_that_refuses(tmp_path))
+                      env=a_gh_that_refuses(tmp_path), encoding="utf-8")
 
 
 @pytest.mark.parametrize("command", ["check", "execute"])

tests/test_non_utf8_name_callers.py

@@ -128,8 +128,12 @@ def fail_where_the_caller_ends(call, *args, said: str = "") -> None:
 
 @pytest.fixture
 def a_system_that_carries_the_byte():
-    if len(os.fsencode(NOT_UTF_8_PROBE)) != 1:
-        pytest.skip("on this system the first question is the one that skips")
+    """The rows that let the probe through to the disk ask the system's
+    question and the locale's first (#1196): the helper skips on either, on
+    its own reason, and not on the one these rows hold."""
+    if (len(os.fsencode(NOT_UTF_8_PROBE)) != 1
+            or os.fsdecode(b"\xff") != NOT_UTF_8_PROBE):
+        pytest.skip("on this system an earlier question is the one that skips")
 
 
 @pytest.fixture

tests/test_non_utf8_name_helpers.py

@@ -15,6 +15,14 @@
 surrogate three bytes, as Windows' does. No Windows run was made. macOS's
 EILSEQ is emulated the same way, by a `mkdir` that answers it.
 
+The locale's question is asked between the two (#1196): in a locale of one
+byte to a character, Latin-1 say, `os.fsdecode(b"\\xff")` is the character
+U+00FF and not the surrogate U+DCFF, so a name that holds the byte is text
+there, and a test that expects it spelled as a name that is not UTF-8 would
+fail on a tool that is right. `os.fsdecode` is EMULATED too, for one test: no
+Latin-1 locale is built here, as `tests/test_adopt_one_byte_locale.py` builds
+one for a child.
+
 The two tools the tests below are written with are held here too:
 `skip_message`, which every one of them reads a skip with, and
 `windows_fsencode`, the emulation itself.
@@ -47,6 +55,28 @@
            "asked by the tests below that emulate them")
 
 
+def fsdecode_in(encoding: str):
+    """`os.fsdecode` as a locale whose file name encoding is `encoding` has
+    it: text is returned as it is, and bytes are read in `encoding`, with
+    `surrogateescape` for each one it cannot read (a UTF-8 locale gives the
+    byte 0xFF as U+DCFF, and a Latin-1 one as U+00FF)."""
+    def fsdecode(filename):
+        filename = os.fspath(filename)
+        if isinstance(filename, str):
+            return filename
+        return filename.decode(encoding, "surrogateescape")
+    return fsdecode
+
+
+def skip_where_an_earlier_question_skips() -> None:
+    """The rows that let the probe through to the real disk ask the system's
+    question and the locale's first (#1196), and skip where either would: the
+    helper then skips on its own reason, and not on the one they hold."""
+    if (len(os.fsencode(NOT_UTF_8_PROBE)) != 1
+            or os.fsdecode(b"\xff") != NOT_UTF_8_PROBE):
+        pytest.skip("on this system an earlier question is the one that skips")
+
+
 def mkdir_answers(code: int, said: str):
     """A `Path.mkdir` that fails as the disk would with `code`."""
     def refuse(self, *args, **kwargs):
@@ -151,6 +181,7 @@ def test_where_fsencode_makes_three_bytes_a_name_that_is_text_is_not_skipped(
 def test_a_disk_that_takes_the_name_is_not_skipped_and_the_probe_is_removed(
         tmp_path):
     """The control: no skip, and nothing is left in the directory."""
+    skip_where_an_earlier_question_skips()
     assert skip_message(skip_where_a_name_cannot_be_not_utf_8,
                         tmp_path) is None
     assert list(tmp_path.iterdir()) == []
@@ -169,11 +200,40 @@ def test_where_fsencode_makes_three_bytes_the_disk_is_not_asked(
     assert "os.fsencode" in reason, reason
 
 
+@LINUX_ONLY
+@pytest.mark.parametrize("encoding", ["utf-8", "ascii"])
+def test_a_locale_that_reads_the_byte_as_the_surrogate_is_not_skipped(
+        tmp_path, monkeypatch, encoding):
+    """The control of the locale's question (#1196): UTF-8 and ASCII give the
+    byte 0xFF back as U+DCFF, which is `NOT_UTF_8_PROBE`, so no test that
+    asks is skipped, and the probe is removed."""
+    monkeypatch.setattr(os, "fsdecode", fsdecode_in(encoding))
+    assert os.fsdecode(b"\xff") == NOT_UTF_8_PROBE
+    assert skip_message(skip_where_a_name_cannot_be_not_utf_8,
+                        tmp_path) is None
+    assert list(tmp_path.iterdir()) == []
+
+
+def test_a_locale_that_reads_the_byte_as_a_character_skips_before_the_disk(
+        tmp_path, monkeypatch):
+    """Latin-1 (emulated): the byte 0xFF is U+00FF, a printable character,
+    so no name holding it is one that is not UTF-8, and the name is skipped.
+    THE LOCALE IS ASKED BEFORE THE DISK (#1196), as the system is: the
+    directory does not exist, so an `mkdir` under it would be
+    FileNotFoundError and not a skip, and the skip is the answer."""
+    monkeypatch.setattr(os, "fsdecode", fsdecode_in("iso8859-1"))
+    assert os.fsdecode(b"\xff") == "\u00ff"
+    reason = skip_message(skip_where_a_name_cannot_be_not_utf_8,
+                          tmp_path / "absent")
+    assert reason is not None, "the helper went on to ask the disk"
+    assert "0xFF" in reason, reason
+    assert "'\\xff'" in reason, reason
+
+
 def test_a_disk_that_answers_eilseq_is_skipped(tmp_path, monkeypatch):
     """macOS (emulated): APFS answers EILSEQ, `Illegal byte sequence`, to a
     file name that is not UTF-8, and no clone can sit at such a path."""
-    if len(os.fsencode(NOT_UTF_8_PROBE)) != 1:
-        pytest.skip("on this system the first question is the one that skips")
+    skip_where_an_earlier_question_skips()
     monkeypatch.setattr(Path, "mkdir",
                         mkdir_answers(errno.EILSEQ, "Illegal byte sequence"))
     reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path)
@@ -188,8 +248,7 @@ def test_any_other_error_from_the_disk_is_the_tests_own(
     """Only EILSEQ is the filesystem's refusal of the name: a full disk, a
     folder nobody can write in or a probe left by an earlier run is a
     failure of the test, raised as it was, and never a skip that hides it."""
-    if len(os.fsencode(NOT_UTF_8_PROBE)) != 1:
-        pytest.skip("on this system the first question is the one that skips")
+    skip_where_an_earlier_question_skips()
     monkeypatch.setattr(Path, "mkdir", mkdir_answers(code, os.strerror(code)))
     with pytest.raises(OSError) as raised:
         skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path)

tests/test_run_script_encoding.py

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: Apache-2.0
+"""`run_script` decodes a child's output in the encoding the caller names
+(#1196, from #1185's review).
+
+A child told `PYTHONIOENCODING=utf-8` writes U+00E9 as the two bytes 0xC3
+0xA9. `run_script` read them in the PARENT's locale, and a parent in a locale
+of one byte to a character, Latin-1 say, read two characters: the rows of
+`tests/test_adopt_plan_paths_spelled.py` that name `pl\\u00e9an.yaml` failed
+on a tool that was right. They name `encoding="utf-8"` now. Each row here
+names an encoding that reads the two bytes one way and no other, so the
+keyword is held in any locale the suite runs in, with no Latin-1 locale
+built: the child is the same, and only the name differs.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from conftest import run_script
+
+#: A child that writes the two bytes of U+00E9 in UTF-8 to stdout and to
+#: stderr, as one told `PYTHONIOENCODING=utf-8` does.
+SAYS_E_ACUTE = ("import sys\n"
+                "sys.stdout.buffer.write(bytes([0xC3, 0xA9]))\n"
+                "sys.stderr.buffer.write(bytes([0xC3, 0xA9]))\n")
+
+
+@pytest.mark.parametrize("encoding, read", [
+    pytest.param("utf-8", "\u00e9", id="utf-8-reads-one-character"),
+    pytest.param("latin-1", "\u00c3\u00a9",
+                 id="latin-1-reads-two"),
+])
+def test_the_output_is_read_in_the_encoding_the_caller_names(
+        tmp_path, encoding, read):
+    """Stdout and stderr both, as the named encoding reads the two bytes:
+    one character in UTF-8 and two in Latin-1. A `run_script` that dropped
+    the keyword would read both in this process's own locale, which is one
+    answer for the two rows, so one of them would fail."""
+    script = tmp_path / "say_e_acute.py"
+    script.write_text(SAYS_E_ACUTE, encoding="ascii")
+    result = run_script(script, encoding=encoding)
+    assert result.returncode == 0, result.stderr
+    assert result.stdout == read
+    assert result.stderr == read