From 8a7828332b3599a8312e60d4e404898d2b1cdf80 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Wed, 21 Oct 2026 16:28:51 +0000 Subject: [PATCH 1/4] adopt-project.py: `plan written to` and `cloning (read only) into` name the caller's `--out` and `--work-dir` with `_shown`, as the other lines of the report name a path, so a bidi control, a line feed or a tab in either no longer reaches the terminal as it was typed (#1115) `cmd_plan` printed `plan written to {out}` and `Source.clone` printed `cloning {repository} (read only) into {path}` with the typed path as it was. A path holding U+202E or U+2066 turned the line round, a line feed split it and a tab padded it, where the `source` line, the table and the follow-ups spell such a path with `_shown` (#199, #900). #1115 named the `--out` line; RV1120's review of #1120 (its N4) found the `--work-dir` line, which is said for an `org/repo` source by `plan`, `check` and `execute` alike. A path whose every character is printable prints exactly as before. The `--out` line also spells a byte that is not UTF-8 now, `\udcff` between backticks where it was written back as 0xff, as `_shown` spells it in every other line; `test_stdout_encoding.py` pinned the raw byte for that one line, and its row now holds the spelled one. tests/test_adopt_plan_paths_spelled.py runs `plan` as a child with U+202E, U+2066, a line feed, a tab and all four in `--out` and in `--work-dir`, and with a plain path that must print unchanged. The `--work-dir` rows use a `gh` that refuses: the line is said before `gh` runs. Co-Authored-By: Claude Sonnet 5.5 --- adopt-project.py | 13 +- tests/test_adopt_plan_paths_spelled.py | 260 +++++++++++++++++++++++++ tests/test_non_utf8_name_callers.py | 2 +- tests/test_stdout_encoding.py | 21 +- 4 files changed, 283 insertions(+), 13 deletions(-) create mode 100644 tests/test_adopt_plan_paths_spelled.py diff --git a/adopt-project.py b/adopt-project.py index ed9c8b1d..01d37475 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -2759,7 +2759,12 @@ def clone(cls, repository: str, work_root: Path, if made is not None: made.append(claim) try: - print(f"cloning {repository} (read only) into {path}") + # The path is the caller's typed `--work-dir`, made absolute, so + # it is `_shown` as `plan`'s other lines name a path (#1115, + # #1120's N4): printed as it was, a bidi control in it reached + # the terminal and a line feed split the line. + print(f"cloning {repository} (read only) into " + f"{_shown(str(path))}") run(["gh", "repo", "clone", repository, str(path), "--", "--quiet"]) except BaseException as exc: @@ -6334,8 +6339,10 @@ def cmd_plan(args) -> int: args.plan_written = True # And it is SAID from here, before the report reads more of the source: # the report's last reads (`symlink_targets`) can still refuse, exit 2, - # and the plan is on disk (#799's L1-79). - print(f"\nplan written to {out}") + # and the plan is on disk (#799's L1-79). The path is the caller's typed + # `--out`, so it is `_shown` as every other path this report names is + # (#1115): a bidi control in it reached the terminal as it was typed. + print(f"\nplan written to {_shown(str(out))}") _print_plan_report(args, source, tree, entries, follow_ups, seeded, pins) return 0 diff --git a/tests/test_adopt_plan_paths_spelled.py b/tests/test_adopt_plan_paths_spelled.py new file mode 100644 index 00000000..7b15d2aa --- /dev/null +++ b/tests/test_adopt_plan_paths_spelled.py @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`plan` names the two paths the caller typed, `--out` and `--work-dir`, as +every other path in its report is named (#1115, and #1120's N4). + +`plan written to {out}` printed the typed `--out` as it was. A path holding +U+202E (a bidi override), U+2066 (a bidi isolate), a line feed or a tab +reached the terminal unspelled: the two bidi controls reverse or isolate what +is printed after them, a line feed splits the line in two, and a tab pads it. +`cloning {repository} (read only) into {path}` did the same with `--work-dir`, +for an `org/repo` source, and `Source.clone` prints that line for `plan`, +`check` and `execute` alike. The `source` line, the table and the follow-ups +already name such a path with `_shown` (#199, #900): as it is when every +character in it is printable, and otherwise between two backticks, each +character as the plan's own `path:` writes it. + +NO NETWORK AND NO GITHUB. `plan` is run as a child, as a person runs it. The +`--out` rows read a local source. The `--work-dir` rows name `testorg/Thing`, +which a `gh` stand-in first on PATH refuses to clone: the line is printed +before `gh` runs, so the refusal changes nothing it says. The stand-in is a +`#!` script, and no Windows file name holds a line feed or a tab, so the +module is POSIX only. + +THE TEST SOURCE IS ASCII: every character that is not printable is written as +an escape. +""" + +from __future__ import annotations + +import os +import stat +import subprocess +from pathlib import Path + +import pytest + +from conftest import (ADOPT, make_source_repo, run_script, + skip_where_the_environment_cannot_carry) + +pytestmark = pytest.mark.skipif( + os.name == "nt", + reason="the `gh` stand-in is a `#!` script, and a Windows file name " + "holds no line feed or tab") + +ORG = "testorg" +PROJECT = "Thing" +REPOSITORY = f"{ORG}/{PROJECT}" +SOURCE_READONLY = "source-readonly" +TREE = {"README.md": "# Thing\n", "src/main.py": "x = 1\n"} +#: `plan` refuses a clone of `gh` that fails, as this. +UNCLONABLE = "REFUSED source-unclonable: " + +#: Each character a terminal does not show as itself, and how `_shown` writes +#: it: the escape `repo_shape.double_quoted` writes for it in the plan. +CHARACTERS = [ + pytest.param("\u202e", "\\u202e", id="bidi-override-U+202E"), + pytest.param("\u2066", "\\u2066", id="bidi-isolate-U+2066"), + pytest.param("\n", "\\n", id="line-feed"), + pytest.param("\t", "\\t", id="tab"), + pytest.param("\u202e\u2066\n\t", "\\u202e\\u2066\\n\\t", id="all-four"), +] + + +def lines_starting(stdout: str, start: str) -> list[str]: + """The lines of `stdout` that begin with `start`, as a terminal breaks + them: at each line feed, so a name that holds one is two lines.""" + return [line for line in stdout.split("\n") if line.startswith(start)] + + +def plan_arguments(source: str, out: Path | str, *extra: str) -> list[str]: + """`plan --source ` for the project `testorg/Thing`, writing + `out`.""" + return ["plan", "--source", source, "--project", PROJECT, "--org", ORG, + "--elected-by", "Test Human", "--elected-on", "2026-09-02", + "--out", str(out), *extra] + + +def plan_of_a_local_source(tmp_path: Path, + out: Path | str) -> subprocess.CompletedProcess: + """`plan` over a repository on disk, which it reads in place, writing + `out` as it is typed.""" + source = make_source_repo(tmp_path / PROJECT, tree=TREE, edits=()) + return run_script(ADOPT, *plan_arguments(str(source), out), + env={"PYTHONIOENCODING": "utf-8"}) + + +def a_gh_that_refuses(base: Path) -> dict: + """The environment that puts a `gh` first on PATH that refuses every + call, and writes its arguments, one to a line, to `base/gh.log`: the + clone `plan` asks for is never made.""" + binary = base / "bin" + binary.mkdir() + script = binary / "gh" + script.write_bytes(b"#!/bin/sh\n" + b"printf '%s\\n' \"$@\" >> \"$GH_STUB_LOG\"\n" + b"echo 'stub gh: refusing' >&2\n" + b"exit 1\n") + script.chmod(script.stat().st_mode | stat.S_IEXEC) + return {"PATH": f"{binary}{os.pathsep}{os.environ.get('PATH', '')}", + "GH_STUB_LOG": str(base / "gh.log"), + "PYTHONIOENCODING": "utf-8"} + + +def plan_of_a_repository_on_github(tmp_path: Path, + work: Path) -> subprocess.CompletedProcess: + """`plan --source testorg/Thing --work-dir `: the work directory is + made, the `cloning` line is said, and the stand-in `gh` refuses.""" + stand_in = a_gh_that_refuses(tmp_path) + return run_script( + ADOPT, *plan_arguments(REPOSITORY, tmp_path / "plan.yaml", + "--work-dir", str(work)), env=stand_in) + + +def where_the_clone_goes(work: Path) -> str: + """The path `Source.clone` clones into: the work directory as + `os.path.realpath` reads it, then `source-readonly/`.""" + return f"{os.path.realpath(work)}/{SOURCE_READONLY}/{PROJECT}" + + +# --- `--out`: `plan written to` ---------------------------------------------- + +@pytest.mark.parametrize("typed, spelled", CHARACTERS) +def test_plan_written_to_spells_an_out_path_a_terminal_would_not_show( + tmp_path, typed, spelled): + """On main the line was `plan written to `: the bidi + controls reached the terminal, and a line feed or a tab broke the line. + Now it is the one line `_shown` writes, between backticks.""" + skip_where_the_environment_cannot_carry(typed) + out = tmp_path / f"pl{typed}an.yaml" + result = plan_of_a_local_source(tmp_path, out) + assert result.returncode == 0, result.stdout + result.stderr + assert out.is_file(), "the plan was not written where --out named" + expected = f"plan written to `{os.fspath(tmp_path)}/pl{spelled}an.yaml`" + assert lines_starting(result.stdout, "plan written to") == [expected], \ + result.stdout + + +@pytest.mark.parametrize("typed", ["\u202e", "\u2066"]) +def test_the_out_path_reaches_no_bidi_control_unspelled(tmp_path, typed): + """The same rows from the other side: whatever else the report prints, + none of it holds the control, which would turn the line around.""" + skip_where_the_environment_cannot_carry(typed) + result = plan_of_a_local_source(tmp_path, tmp_path / f"pl{typed}an.yaml") + assert result.returncode == 0, result.stdout + result.stderr + assert typed not in result.stdout, result.stdout + + +def test_a_plain_out_path_is_named_exactly_as_it_was(tmp_path): + """The control, which passes on main as well: a path whose every + character is printable is not respelled and not wrapped in backticks.""" + out = tmp_path / "adoption-plan.yaml" + result = plan_of_a_local_source(tmp_path, out) + assert result.returncode == 0, result.stdout + result.stderr + assert lines_starting(result.stdout, "plan written to") == [ + f"plan written to {os.fspath(out)}"], result.stdout + + +def test_an_out_path_typed_with_a_dot_segment_is_named_as_it_always_was( + tmp_path): + """Also the control: `plan` names the path as `Path` reads it, so + `/./adoption-plan.yaml` is named `/adoption-plan.yaml`. What + `_shown` spells is that path, and not the text that was typed.""" + out = tmp_path / "adoption-plan.yaml" + result = plan_of_a_local_source( + tmp_path, f"{os.fspath(tmp_path)}{os.sep}.{os.sep}{out.name}") + assert result.returncode == 0, result.stdout + result.stderr + assert lines_starting(result.stdout, "plan written to") == [ + f"plan written to {os.fspath(out)}"], result.stdout + + +# --- `--work-dir`: `cloning (read only) into` ------------------- + +@pytest.mark.parametrize("typed, spelled", CHARACTERS) +def test_cloning_spells_a_work_dir_a_terminal_would_not_show(tmp_path, typed, + spelled): + """On main the line was `cloning testorg/Thing (read only) into `, for an `org/repo` source only: a local source is read in + place and clones nothing. Now it is the one line `_shown` writes.""" + skip_where_the_environment_cannot_carry(typed) + work = tmp_path / f"wd{typed}k" + result = plan_of_a_repository_on_github(tmp_path, work) + assert result.returncode == 2, result.stdout + result.stderr + assert UNCLONABLE in result.stderr, result.stdout + result.stderr + base = os.path.realpath(tmp_path) + expected = (f"cloning {REPOSITORY} (read only) into " + f"`{base}/wd{spelled}k/{SOURCE_READONLY}/{PROJECT}`") + assert lines_starting(result.stdout, "cloning ") == [expected], \ + result.stdout + + +@pytest.mark.parametrize("typed", ["\u202e", "\u2066"]) +def test_the_work_dir_reaches_no_bidi_control_unspelled(tmp_path, typed): + """The same rows from the other side, as for `--out`.""" + skip_where_the_environment_cannot_carry(typed) + result = plan_of_a_repository_on_github(tmp_path, + tmp_path / f"wd{typed}k") + assert result.returncode == 2, result.stdout + result.stderr + assert typed not in result.stdout, result.stdout + + +def test_a_plain_work_dir_is_named_exactly_as_it_was(tmp_path): + """The control, which passes on main as well, and which holds that the + stand-in `gh` ran: it was asked to clone the one repository.""" + work = tmp_path / "work" + result = plan_of_a_repository_on_github(tmp_path, work) + assert result.returncode == 2, result.stdout + result.stderr + assert lines_starting(result.stdout, "cloning ") == [ + f"cloning {REPOSITORY} (read only) into {where_the_clone_goes(work)}" + ], result.stdout + log = (tmp_path / "gh.log").read_text(encoding="utf-8").splitlines() + assert log[:3] == ["repo", "clone", REPOSITORY], log + + +# --- both at once ------------------------------------------------------------ + +def a_gh_that_clones(base: Path, origin: Path) -> dict: + """The environment that puts a `gh` first on PATH that makes the one + clone `plan` asks for: `gh repo clone -- + --quiet` is a `git clone` of `origin`, its own `origin` then set to the + GitHub url, as `gh` leaves it. Any other call is refused.""" + binary = base / "bin" + binary.mkdir() + script = binary / "gh" + script.write_bytes( + b"#!/bin/sh\n" + b"if [ \"$1 $2\" = \"repo clone\" ]; then\n" + b" git clone -q --no-local \"$STUB_ORIGIN\" \"$4\" || exit 1\n" + b" exec git -C \"$4\" remote set-url origin " + b"\"https://github.com/$3.git\"\n" + b"fi\n" + b"echo 'stub gh: refusing' >&2\n" + b"exit 1\n") + script.chmod(script.stat().st_mode | stat.S_IEXEC) + return {"PATH": f"{binary}{os.pathsep}{os.environ.get('PATH', '')}", + "STUB_ORIGIN": str(origin), "PYTHONIOENCODING": "utf-8"} + + +def test_each_of_the_two_paths_is_spelled_in_one_run(tmp_path): + """`--out` and `--work-dir` typed with the same control in one run that + clones and writes its plan: both lines are said, and both are spelled, so + a spelling given to one flag and not the other fails here whichever it + is.""" + typed = "\u202e" + skip_where_the_environment_cannot_carry(typed) + origin = make_source_repo(tmp_path / "github" / PROJECT, tree=TREE, + edits=()) + out = tmp_path / f"pl{typed}an.yaml" + work = tmp_path / f"wd{typed}k" + result = run_script( + ADOPT, *plan_arguments(REPOSITORY, out, "--work-dir", str(work)), + env=a_gh_that_clones(tmp_path, origin)) + assert result.returncode == 0, result.stdout + result.stderr + assert out.is_file(), result.stdout + base = os.path.realpath(tmp_path) + assert lines_starting(result.stdout, "cloning ") == [ + f"cloning {REPOSITORY} (read only) into " + f"`{base}/wd\\u202ek/{SOURCE_READONLY}/{PROJECT}`"], result.stdout + assert lines_starting(result.stdout, "plan written to") == [ + f"plan written to `{os.fspath(tmp_path)}/pl\\u202ean.yaml`" + ], result.stdout + assert typed not in result.stdout, result.stdout diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index dd6c7386..05e84702 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -70,7 +70,7 @@ CALLERS = [ pytest.param( lambda tmp_path, adopter: stdout_encoding - .test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes( + .test_a_name_that_is_not_utf8_is_spelled_and_the_report_still_comes_out( tmp_path, "utf-8:surrogateescape"), True, id="stdout-encoding"), pytest.param( diff --git a/tests/test_stdout_encoding.py b/tests/test_stdout_encoding.py index 03283488..fd8e4a09 100644 --- a/tests/test_stdout_encoding.py +++ b/tests/test_stdout_encoding.py @@ -199,14 +199,17 @@ def test_plan_reports_every_name_whatever_its_stdout_can_encode(tmp_path, "`os.fsencode` and `os.fsdecode` are UTF-8, which " "has no byte 0xff, so the byte path below raises " "before any directory is made") -def test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes(tmp_path, - encoding): +def test_a_name_that_is_not_utf8_is_spelled_and_the_report_still_comes_out( + tmp_path, encoding): """`plan --out` into a directory whose name holds byte 0xff, which is not - UTF-8: Python reads it in as U+DCFF, and a `surrogateescape` stdout - writes it back as 0xff. It still does, in the same report that shows - `日本.md` -- escaped on cp1252 -- where plain `backslashreplace` would - print `\\udcff` instead. POSIX only: neither Windows nor macOS holds a - file name that is not UTF-8.""" + UTF-8: Python reads it in as U+DCFF, which a `surrogateescape` stdout + writes back as 0xff. `plan written to` no longer prints that path as it + is: it spells the byte as `_shown` spells every name in the report, + `\\udcff` between backticks (#1115), so no raw byte reaches the + terminal. The report still comes out whole, with `日本.md` in the table + -- escaped on cp1252 -- as it did where the byte was written back. + POSIX only: neither Windows nor macOS holds a file name that is not + UTF-8.""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = os.path.join(os.fsencode(tmp_path), b"b\xffd") os.mkdir(parent) @@ -225,8 +228,8 @@ def test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes(tmp_path, written = [line for line in stdout.splitlines() if line.startswith(b"plan written to ")] assert len(written) == 1, stdout - assert written[0].endswith(b"/b\xffd/plan.yaml"), written - assert b"\\udcff" not in stdout, stdout + assert written[0].endswith(b"/b\\udcffd/plan.yaml`"), written + assert b"\xff" not in stdout, stdout codec = encoding.split(":")[0] assert shown(ANSWERED, encoding).encode(codec) in stdout, stdout From 5dc90602fbc19291234f6c02ce4a84555a61af65 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Thu, 22 Oct 2026 21:46:31 +0000 Subject: [PATCH 2/4] tests/test_adopt_plan_paths_spelled.py: seven mutants of the `plan written to` and `cloning ... into` lines that no row killed now fail one, so a printable non-ASCII path prints as typed, ESC, CR and a non-UTF-8 `--work-dir` are spelled, a relative `--out` is not made absolute, an `--out` that ends in a space or is wrapped in backticks is spelled, and `execute` spells the clone line as `plan` does (#1167) Follows #1154 (for #1115), from RV1115's review of it (#1154 issuecomment-6344547992): its mutants R1, R2, R3, R4, R6, R7 and R12 left the 18 rows of the file green, and the code of #1154 is right on each case. No change to adopt-project.py. 13 rows are added, and the first 18 are unchanged: - ESC and CR join `CHARACTERS`, which feeds the `--out` rows and the `--work-dir` rows alike (R6, R7; 4 rows); - a `--work-dir` that is not UTF-8, under a `surrogateescape` stdout, is spelled `\udcff` between backticks and no raw byte is written (R7; 1 row); - a printable non-ASCII `--out` and `--work-dir` (U+00E9) print as typed, with no backticks (R2, R3; 2 rows); - a relative `--out`, `plan.yaml` and `./plan.yaml`, prints as `plan.yaml` (R4; 2 rows); - an `--out` that ends in a space, and a relative one that begins and ends with a backtick, are printable and still spelled, as `_shown` has it since #900 (R1; 2 rows); - `check` and `execute` over a plan from a local source, handed `--source testorg/Thing` and a `--work-dir` holding U+202E, refuse as `source-unclonable` after the spelled `cloning` line (R12; 2 rows). Only the `execute` row kills R12: `plan` and `check` hand `Source.clone` a list of the clones they made, `execute` none. Each new row passes on 8a782733 and fails on its mutant, and each mutant leaves the first 18 rows green. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_adopt_plan_paths_spelled.py | 164 ++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 5 deletions(-) diff --git a/tests/test_adopt_plan_paths_spelled.py b/tests/test_adopt_plan_paths_spelled.py index 7b15d2aa..f98c7088 100644 --- a/tests/test_adopt_plan_paths_spelled.py +++ b/tests/test_adopt_plan_paths_spelled.py @@ -20,6 +20,15 @@ `#!` script, and no Windows file name holds a line feed or a tab, so the module is POSIX only. +`_shown`'s rule is held from both ends (#1167, from #1154's review). A +printable path is named as it was typed, a non-ASCII one too (U+00E9), and a +relative `--out` as `Path` reads it, which drops a leading `./`. A printable +`--out` that ends in a space, or that begins and ends with a backtick, is +spelled all the same (#900). A `--work-dir` that is not UTF-8 is spelled, and +no raw byte reaches stdout. `check` and `execute` say the `cloning` line +`plan` says, and each is run here: `execute` keeps the clone it makes and +hands `Source.clone` no list of them, where `plan` and `check` do. + THE TEST SOURCE IS ASCII: every character that is not printable is written as an escape. """ @@ -29,11 +38,13 @@ import os import stat import subprocess +import sys from pathlib import Path import pytest -from conftest import (ADOPT, make_source_repo, run_script, +from conftest import (ADOPT, NOT_UTF_8_PROBE, make_source_repo, run_script, + skip_where_a_name_cannot_be_not_utf_8, skip_where_the_environment_cannot_carry) pytestmark = pytest.mark.skipif( @@ -57,6 +68,8 @@ pytest.param("\n", "\\n", id="line-feed"), pytest.param("\t", "\\t", id="tab"), pytest.param("\u202e\u2066\n\t", "\\u202e\\u2066\\n\\t", id="all-four"), + pytest.param("\x1b", "\\x1b", id="escape"), + pytest.param("\r", "\\r", id="carriage-return"), ] @@ -74,12 +87,14 @@ def plan_arguments(source: str, out: Path | str, *extra: str) -> list[str]: "--out", str(out), *extra] -def plan_of_a_local_source(tmp_path: Path, - out: Path | str) -> subprocess.CompletedProcess: +def plan_of_a_local_source(tmp_path: Path, out: Path | str, + cwd: Path | None = None + ) -> subprocess.CompletedProcess: """`plan` over a repository on disk, which it reads in place, writing - `out` as it is typed.""" + `out` as it is typed, run in `cwd` where that is given: a relative `out` + is written there.""" source = make_source_repo(tmp_path / PROJECT, tree=TREE, edits=()) - return run_script(ADOPT, *plan_arguments(str(source), out), + return run_script(ADOPT, *plan_arguments(str(source), out), cwd=cwd, env={"PYTHONIOENCODING": "utf-8"}) @@ -167,6 +182,56 @@ def test_an_out_path_typed_with_a_dot_segment_is_named_as_it_always_was( f"plan written to {os.fspath(out)}"], result.stdout +def test_a_printable_non_ascii_out_path_is_named_as_it_was_typed(tmp_path): + """A character that is not ASCII is not one a terminal does not show: + `plan.yaml` is printable, so `_shown` returns it as it is, with + no backticks and no escape. A spelling of every non-ASCII path passes the + rows above, and fails here.""" + typed = "\u00e9" + skip_where_the_environment_cannot_carry(typed) + out = tmp_path / f"pl{typed}an.yaml" + result = plan_of_a_local_source(tmp_path, out) + assert result.returncode == 0, result.stdout + result.stderr + assert out.is_file(), "the plan was not written where --out named" + assert lines_starting(result.stdout, "plan written to") == [ + f"plan written to {os.fspath(out)}"], result.stdout + + +@pytest.mark.parametrize("typed", ["plan.yaml", "./plan.yaml"]) +def test_a_relative_out_path_is_named_as_it_was_typed(tmp_path, typed): + """`plan` names the path as `Path` reads it: `plan.yaml` is `plan.yaml` + and `./plan.yaml` is too, and neither is made absolute. Run in `tmp_path`, + so the plan is written there. The dot-segment row above types an absolute + path, which an `os.path.abspath` would not change.""" + result = plan_of_a_local_source(tmp_path, typed, cwd=tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "plan.yaml").is_file(), \ + "the plan was not written where --out named" + assert lines_starting(result.stdout, "plan written to") == [ + "plan written to plan.yaml"], result.stdout + + +@pytest.mark.parametrize("typed, spelled", [ + pytest.param("plan.yaml ", "`plan.yaml `", id="trailing-space"), + pytest.param("`plan.yaml`", "`\\x60plan.yaml\\x60`", + id="begins-and-ends-with-a-backtick"), +]) +def test_a_printable_out_path_that_reads_as_another_is_spelled( + tmp_path, typed, spelled): + """`_shown`'s rule from #900, which `isprintable()` alone is not: a path + that ends in a space reads as the one without it, and one that begins and + ends with a backtick reads as the spelling of the path between them. Both + are printable, and both are spelled. Each is a relative path, run in + `tmp_path`. A `--work-dir` cannot reach either case: the path the clone + line prints ends in `source-readonly/` and begins with a slash.""" + result = plan_of_a_local_source(tmp_path, typed, cwd=tmp_path) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / typed).is_file(), \ + "the plan was not written where --out named" + assert lines_starting(result.stdout, "plan written to") == [ + f"plan written to {spelled}"], result.stdout + + # --- `--work-dir`: `cloning (read only) into` ------------------- @pytest.mark.parametrize("typed, spelled", CHARACTERS) @@ -210,6 +275,58 @@ def test_a_plain_work_dir_is_named_exactly_as_it_was(tmp_path): assert log[:3] == ["repo", "clone", REPOSITORY], log +def test_a_printable_non_ascii_work_dir_is_named_as_it_was_typed(tmp_path): + """The same control as for `--out`: `wdk` is printable, so the + line names it as it is, with no backticks and no escape.""" + typed = "\u00e9" + skip_where_the_environment_cannot_carry(typed) + work = tmp_path / f"wd{typed}k" + result = plan_of_a_repository_on_github(tmp_path, work) + assert result.returncode == 2, result.stdout + result.stderr + assert UNCLONABLE in result.stderr, result.stdout + result.stderr + assert lines_starting(result.stdout, "cloning ") == [ + f"cloning {REPOSITORY} (read only) into {where_the_clone_goes(work)}" + ], result.stdout + + +def plan_of_a_repository_on_github_as_bytes( + tmp_path: Path, work: Path) -> tuple[int, bytes, bytes]: + """`(exit, stdout, stderr)` of `plan --source testorg/Thing --work-dir + ` with the stand-in `gh` that refuses, and a `surrogateescape` + stdout: the bytes the child wrote. `run_script` decodes them in the + parent's own locale, which a raw byte that is not UTF-8 fails.""" + environment = {**os.environ, "LANES_LANE": "", + **a_gh_that_refuses(tmp_path), + "PYTHONIOENCODING": "utf-8:surrogateescape"} + arguments = plan_arguments(REPOSITORY, tmp_path / "plan.yaml", + "--work-dir", str(work)) + proc = subprocess.run([sys.executable, str(ADOPT), *arguments], + capture_output=True, env=environment, + stdin=subprocess.DEVNULL, check=False) + return proc.returncode, proc.stdout, proc.stderr + + +def test_a_work_dir_that_is_not_utf8_is_spelled_and_no_raw_byte_is_written( + tmp_path): + """`--work-dir` holding the byte 0xff, which is not UTF-8: Python reads + it in as U+DCFF, which a `surrogateescape` stdout writes back as the raw + byte. The line is the one `_shown` writes, `\\udcff` between backticks, and + no raw byte reaches stdout. The `--out` row of `test_stdout_encoding.py` + holds the same for `plan written to`.""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + work = tmp_path / f"wd{NOT_UTF_8_PROBE}k" + code, stdout, stderr = plan_of_a_repository_on_github_as_bytes( + tmp_path, work) + assert code == 2 and b"Traceback" not in stderr, stdout + stderr + assert UNCLONABLE.encode() in stderr, stdout + stderr + clone = f"{os.path.realpath(tmp_path)}/wd\\udcffk/{SOURCE_READONLY}" + assert [line for line in stdout.split(b"\n") + if line.startswith(b"cloning ")] == [ + f"cloning {REPOSITORY} (read only) into `{clone}/{PROJECT}`".encode() + ], stdout + assert b"\xff" not in stdout, stdout + + # --- both at once ------------------------------------------------------------ def a_gh_that_clones(base: Path, origin: Path) -> dict: @@ -258,3 +375,40 @@ def test_each_of_the_two_paths_is_spelled_in_one_run(tmp_path): f"plan written to `{os.fspath(tmp_path)}/pl\\u202ean.yaml`" ], result.stdout assert typed not in result.stdout, result.stdout + + +# --- `check` and `execute` say the same line --------------------------------- + +def a_command_on_github(tmp_path: Path, command: str, + work: Path) -> subprocess.CompletedProcess: + """`check` or `execute` over the plan `plan` wrote from a local source, + handed `--source testorg/Thing --work-dir `: the work directory is + made, the `cloning` line is said, and the stand-in `gh` refuses.""" + plan = tmp_path / "plan.yaml" + planned = plan_of_a_local_source(tmp_path, plan) + assert planned.returncode == 0, planned.stdout + planned.stderr + return run_script(ADOPT, command, "--plan", str(plan), "--source", + REPOSITORY, "--work-dir", str(work), + env=a_gh_that_refuses(tmp_path)) + + +@pytest.mark.parametrize("command", ["check", "execute"]) +def test_check_and_execute_spell_the_work_dir_as_plan_does(tmp_path, command): + """`Source.clone` prints the `cloning` line for `plan`, `check` and + `execute` alike. `plan` and `check` hand it a list of the clones they + made, and `execute` none, as it keeps its clone: a spelling that holds + only where the caller keeps that list leaves `execute` printing the + U+202E as it was typed, which the `execute` row catches and `check`'s + does not.""" + typed, spelled = "\u202e", "\\u202e" + skip_where_the_environment_cannot_carry(typed) + result = a_command_on_github(tmp_path, command, + tmp_path / f"wd{typed}k") + assert result.returncode == 2, result.stdout + result.stderr + assert "REFUSED source-unclonable" in result.stderr, \ + result.stdout + result.stderr + base = os.path.realpath(tmp_path) + assert lines_starting(result.stdout, "cloning ") == [ + f"cloning {REPOSITORY} (read only) into " + f"`{base}/wd{spelled}k/{SOURCE_READONLY}/{PROJECT}`"], result.stdout + assert typed not in result.stdout, result.stdout From d193617c77c034c765e2cb60216b59a3494fac39 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 23 Oct 2026 07:41:03 +0000 Subject: [PATCH 3/4] tests/test_adopt_plan_paths_spelled.py: the row's exit code and its missing traceback are two asserts, so a failure names the one that broke (#1167) The 0xff `--work-dir` row asserted `code == 2 and b"Traceback" not in stderr` as one composite assert, which SonarCloud flags as python:S9073 on #1185 (RV1185's S1). It is now `assert code == 2` and `assert b"Traceback" not in stderr`, each with the same message, the child's stdout and stderr. The row asserts what it asserted before. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_adopt_plan_paths_spelled.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_adopt_plan_paths_spelled.py b/tests/test_adopt_plan_paths_spelled.py index f98c7088..3cbf5170 100644 --- a/tests/test_adopt_plan_paths_spelled.py +++ b/tests/test_adopt_plan_paths_spelled.py @@ -317,7 +317,8 @@ def test_a_work_dir_that_is_not_utf8_is_spelled_and_no_raw_byte_is_written( work = tmp_path / f"wd{NOT_UTF_8_PROBE}k" code, stdout, stderr = plan_of_a_repository_on_github_as_bytes( tmp_path, work) - assert code == 2 and b"Traceback" not in stderr, stdout + stderr + assert code == 2, stdout + stderr + assert b"Traceback" not in stderr, stdout + stderr assert UNCLONABLE.encode() in stderr, stdout + stderr clone = f"{os.path.realpath(tmp_path)}/wd\\udcffk/{SOURCE_READONLY}" assert [line for line in stdout.split(b"\n") From 13a2c58cb79d640ed6392edef77cf5d27ed9d2b3 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 23 Oct 2026 12:59:52 +0000 Subject: [PATCH 4/4] tests/conftest.py and tests/test_adopt_plan_paths_spelled.py: under a one-byte locale such as Latin-1 the 0xff `--work-dir` row skips and the two U+00E9 rows read the child's UTF-8, so none of the three fails on a tool that is right, and the bytes helper starts its child as `run_script` does (#1196) Follows #1185 (for #1167), from RV1185's review of it (#1185 issuecomment-6356497748): its N1 and N2. No change to adopt-project.py, and the tool is right in every case below. Under a Latin-1 locale built as #1153's test_adopt_one_byte_locale.py builds one, the new file and the two non-UTF-8 `--out` rows of test_stdout_encoding.py gave 5 failed, 15 passed, 13 skipped; they give 0 failed, 17 passed, 16 skipped now. N1, the three rows that failed under a one-byte locale: - `skip_where_a_name_cannot_be_not_utf_8` (tests/conftest.py) also skips where `os.fsdecode(b"\xff")` is not U+DCFF, before the disk is touched. Latin-1 reads the byte as U+00FF, a printable character, so `_shown` prints `wdk` as typed, and the 0xff row's premise, that Python reads it in as U+DCFF, does not hold. The disk and `os.fsencode` take the byte in every locale, so neither could say. UTF-8 and ASCII give U+DCFF and skip no more than before: the 1086 tests of the 22 files that ask the helper, and of test_repo_hygiene.py, have under UTF-8 the outcome they had. The two non-UTF-8 `--out` rows of test_stdout_encoding.py ask it too, and skip under Latin-1 where they failed. - `run_script` takes `encoding`, None unless named, and the four runs of tests/test_adopt_plan_paths_spelled.py name "utf-8": each child is told `PYTHONIOENCODING=utf-8`, and a parent in Latin-1 read the two bytes of U+00E9 as two characters (`..._non_ascii_out_path_...` and `..._non_ascii_work_dir_...`). Both rows pass there now. - The rows of test_non_utf8_name_callers.py (its fixture, which test_non_utf8_git_failures.py and test_non_utf8_oserror_sites.py use) and test_non_utf8_name_helpers.py that let the probe through to the disk skip where the helper skips on the new question, as they skip where it skips on the first: 28 of them failed under Latin-1 without it. N2: `plan_of_a_repository_on_github_as_bytes` builds its child's environment with `blank_unnamed_pin_sources` and the git identity defaults, as `run_script` and test_stdout_encoding.py's `run_tool_bytes` do. It had neither; the run ends at the clone refusal, so nothing it says changes, and no row sees it. Five rows are added and no other row changes: three in test_non_utf8_name_helpers.py (the new question asked of an emulated Latin-1 `os.fsdecode`, before the disk, and the UTF-8 and ASCII controls) and two in the new tests/test_run_script_encoding.py (the keyword reads stdout and stderr in the encoding named). The 31 rows of the spelled file are unchanged: 31 pass under UTF-8, and under ASCII 16 pass and 15 skip. Lane: openreposhape-4 (openRepoShape-4) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 31 +++++++++--- tests/test_adopt_plan_paths_spelled.py | 49 +++++++++++++++---- tests/test_non_utf8_name_callers.py | 8 ++- tests/test_non_utf8_name_helpers.py | 67 ++++++++++++++++++++++++-- tests/test_run_script_encoding.py | 46 ++++++++++++++++++ 5 files changed, 180 insertions(+), 21 deletions(-) create mode 100644 tests/test_run_script_encoding.py diff --git a/tests/conftest.py b/tests/conftest.py index d2bf4e33..61ef4826 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -732,8 +732,8 @@ def clear_ambient_pin_sources(monkeypatch: pytest.MonkeyPatch) -> None: def run_script(script: Path, *args: str, cwd: Path | None = None, env: dict | None = None, input: str | None = None, - stdin: int | None = None, - merge_stderr: bool = False) -> subprocess.CompletedProcess: + stdin: int | None = None, merge_stderr: bool = False, + encoding: str | None = None) -> subprocess.CompletedProcess: """Run one tool under test, with stdin closed by default. A test in this suite is often asserting what a tool does when nobody is @@ -754,6 +754,12 @@ def run_script(script: Path, *args: str, cwd: Path | None = None, does, and leaves `stderr` None. A tool's piped stdout is block-buffered and its stderr is not, so this is the only way a test sees the order a person reading that log sees. + + `encoding` is what the output is decoded in, and None is this process's + own locale, as `subprocess.run(text=True)` reads it. A caller that tells + the child `PYTHONIOENCODING=utf-8` names it too (#1196): a parent in a + locale of one byte to a character, Latin-1 say, would read the two bytes + of U+00E9 as two characters. """ asked = env or {} env = {**os.environ, **asked} @@ -794,8 +800,8 @@ def run_script(script: Path, *args: str, cwd: Path | None = None, stdout=subprocess.PIPE, stderr=(subprocess.STDOUT if merge_stderr else subprocess.PIPE), - text=True, check=False, env=env, input=input, - stdin=stdin) + text=True, encoding=encoding, check=False, env=env, + input=input, stdin=stdin) def scaffold_dry_run_on_github(monkeypatch, capsys, *args: str): @@ -1156,19 +1162,32 @@ def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None: def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not - UTF-8 cannot be made under `directory`. Two systems say so: + UTF-8 cannot be made under `directory`. Three answers say so: - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first, before the disk is touched; + - a locale of one byte to a character, Latin-1 say (#1196), asked second, + also before the disk is touched: there the byte 0xFF is the character + U+00FF, so `os.fsdecode` gives it as text and not as the surrogate + U+DCFF, and a name that holds it is not one that is not UTF-8. A tool + prints it as it is, where the test expects it spelled. The disk and + `os.fsencode` take the byte in every locale, so only `os.fsdecode` can + say. UTF-8 and ASCII give U+DCFF, and neither skips; - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte sequence"), so no clone can sit at such a path there. - The second is asked by making a directory `probe` under + The third is asked by making a directory `probe` under `directory` and removing it; any other error is the test's own and is raised. """ probe = directory / f"probe{NOT_UTF_8_PROBE}" skip_where_fsencode_gives_a_surrogate_no_byte(probe.name) + read = os.fsdecode(b"\xff") + if read != NOT_UTF_8_PROBE: + pytest.skip(f"this system's file name encoding " + f"({sys.getfilesystemencoding()}) reads the byte 0xFF as " + f"{ascii(read)}, not as {ascii(NOT_UTF_8_PROBE)}, so no " + f"file name here is one that is not UTF-8") try: probe.mkdir() except OSError as exc: diff --git a/tests/test_adopt_plan_paths_spelled.py b/tests/test_adopt_plan_paths_spelled.py index 3cbf5170..5ef0342f 100644 --- a/tests/test_adopt_plan_paths_spelled.py +++ b/tests/test_adopt_plan_paths_spelled.py @@ -29,6 +29,17 @@ `plan` says, and each is run here: `execute` keeps the clone it makes and hands `Source.clone` no list of them, where `plan` and `check` do. +A LOCALE OF ONE BYTE TO A CHARACTER, Latin-1 say, is not one these rows ran +under, and the tool is right in it (#1196, from #1185's review). Every child +is told `PYTHONIOENCODING=utf-8`, so every run reads what it wrote as UTF-8, +and the two bytes of U+00E9 are one character in the parent too. The row of a +`--work-dir` that is not UTF-8 skips where the locale reads the byte 0xFF as +U+00FF, as Latin-1 does: that is a printable name there, which `_shown` +prints as it was typed, and the row's premise, that Python reads it in as +U+DCFF, does not hold (`skip_where_a_name_cannot_be_not_utf_8` asks). A +character Latin-1 cannot spell, U+202E and the rest, is skipped as under +ASCII. + THE TEST SOURCE IS ASCII: every character that is not printable is written as an escape. """ @@ -43,7 +54,8 @@ import pytest -from conftest import (ADOPT, NOT_UTF_8_PROBE, make_source_repo, run_script, +from conftest import (ADOPT, IDENTITY_EMAIL, IDENTITY_NAME, NOT_UTF_8_PROBE, + blank_unnamed_pin_sources, make_source_repo, run_script, skip_where_a_name_cannot_be_not_utf_8, skip_where_the_environment_cannot_carry) @@ -95,7 +107,7 @@ def plan_of_a_local_source(tmp_path: Path, out: Path | str, is written there.""" source = make_source_repo(tmp_path / PROJECT, tree=TREE, edits=()) return run_script(ADOPT, *plan_arguments(str(source), out), cwd=cwd, - env={"PYTHONIOENCODING": "utf-8"}) + env={"PYTHONIOENCODING": "utf-8"}, encoding="utf-8") def a_gh_that_refuses(base: Path) -> dict: @@ -122,7 +134,8 @@ def plan_of_a_repository_on_github(tmp_path: Path, stand_in = a_gh_that_refuses(tmp_path) return run_script( ADOPT, *plan_arguments(REPOSITORY, tmp_path / "plan.yaml", - "--work-dir", str(work)), env=stand_in) + "--work-dir", str(work)), env=stand_in, + encoding="utf-8") def where_the_clone_goes(work: Path) -> str: @@ -289,15 +302,32 @@ def test_a_printable_non_ascii_work_dir_is_named_as_it_was_typed(tmp_path): ], result.stdout +def the_environment_of_a_child_read_as_bytes(tmp_path: Path) -> dict: + """The environment `run_script` gives a child, with the stand-in `gh` that + refuses and a `surrogateescape` stdout (#1196): no lane, no pin-source + override of the machine running the suite, and an identity git can commit + with. The run below ends at the clone refusal, before a pin is read or a + commit is made, so none of the three changes what it says; they are here + so that this child is started as every other one is, as + `test_stdout_encoding.py`'s `run_tool_bytes` starts its own.""" + environment = blank_unnamed_pin_sources( + {**os.environ, "LANES_LANE": "", **a_gh_that_refuses(tmp_path), + "PYTHONIOENCODING": "utf-8:surrogateescape"}, {}) + for name, value in (("GIT_AUTHOR_NAME", IDENTITY_NAME), + ("GIT_AUTHOR_EMAIL", IDENTITY_EMAIL), + ("GIT_COMMITTER_NAME", IDENTITY_NAME), + ("GIT_COMMITTER_EMAIL", IDENTITY_EMAIL)): + environment.setdefault(name, value) + return environment + + def plan_of_a_repository_on_github_as_bytes( tmp_path: Path, work: Path) -> tuple[int, bytes, bytes]: """`(exit, stdout, stderr)` of `plan --source testorg/Thing --work-dir ` with the stand-in `gh` that refuses, and a `surrogateescape` stdout: the bytes the child wrote. `run_script` decodes them in the parent's own locale, which a raw byte that is not UTF-8 fails.""" - environment = {**os.environ, "LANES_LANE": "", - **a_gh_that_refuses(tmp_path), - "PYTHONIOENCODING": "utf-8:surrogateescape"} + environment = the_environment_of_a_child_read_as_bytes(tmp_path) arguments = plan_arguments(REPOSITORY, tmp_path / "plan.yaml", "--work-dir", str(work)) proc = subprocess.run([sys.executable, str(ADOPT), *arguments], @@ -312,7 +342,8 @@ def test_a_work_dir_that_is_not_utf8_is_spelled_and_no_raw_byte_is_written( it in as U+DCFF, which a `surrogateescape` stdout writes back as the raw byte. The line is the one `_shown` writes, `\\udcff` between backticks, and no raw byte reaches stdout. The `--out` row of `test_stdout_encoding.py` - holds the same for `plan written to`.""" + holds the same for `plan written to`. Skipped where the locale reads the + byte as U+00FF, Latin-1 say, where it is the printable `wdk`.""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) work = tmp_path / f"wd{NOT_UTF_8_PROBE}k" code, stdout, stderr = plan_of_a_repository_on_github_as_bytes( @@ -365,7 +396,7 @@ def test_each_of_the_two_paths_is_spelled_in_one_run(tmp_path): work = tmp_path / f"wd{typed}k" result = run_script( ADOPT, *plan_arguments(REPOSITORY, out, "--work-dir", str(work)), - env=a_gh_that_clones(tmp_path, origin)) + env=a_gh_that_clones(tmp_path, origin), encoding="utf-8") assert result.returncode == 0, result.stdout + result.stderr assert out.is_file(), result.stdout base = os.path.realpath(tmp_path) @@ -390,7 +421,7 @@ def a_command_on_github(tmp_path: Path, command: str, assert planned.returncode == 0, planned.stdout + planned.stderr return run_script(ADOPT, command, "--plan", str(plan), "--source", REPOSITORY, "--work-dir", str(work), - env=a_gh_that_refuses(tmp_path)) + env=a_gh_that_refuses(tmp_path), encoding="utf-8") @pytest.mark.parametrize("command", ["check", "execute"]) diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 05e84702..def6b622 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -128,8 +128,12 @@ def fail_where_the_caller_ends(call, *args, said: str = "") -> None: @pytest.fixture def a_system_that_carries_the_byte(): - if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: - pytest.skip("on this system the first question is the one that skips") + """The rows that let the probe through to the disk ask the system's + question and the locale's first (#1196): the helper skips on either, on + its own reason, and not on the one these rows hold.""" + if (len(os.fsencode(NOT_UTF_8_PROBE)) != 1 + or os.fsdecode(b"\xff") != NOT_UTF_8_PROBE): + pytest.skip("on this system an earlier question is the one that skips") @pytest.fixture diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 8594c6b6..a10979f8 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -15,6 +15,14 @@ surrogate three bytes, as Windows' does. No Windows run was made. macOS's EILSEQ is emulated the same way, by a `mkdir` that answers it. +The locale's question is asked between the two (#1196): in a locale of one +byte to a character, Latin-1 say, `os.fsdecode(b"\\xff")` is the character +U+00FF and not the surrogate U+DCFF, so a name that holds the byte is text +there, and a test that expects it spelled as a name that is not UTF-8 would +fail on a tool that is right. `os.fsdecode` is EMULATED too, for one test: no +Latin-1 locale is built here, as `tests/test_adopt_one_byte_locale.py` builds +one for a child. + The two tools the tests below are written with are held here too: `skip_message`, which every one of them reads a skip with, and `windows_fsencode`, the emulation itself. @@ -40,6 +48,28 @@ "asked by the tests below that emulate them") +def fsdecode_in(encoding: str): + """`os.fsdecode` as a locale whose file name encoding is `encoding` has + it: text is returned as it is, and bytes are read in `encoding`, with + `surrogateescape` for each one it cannot read (a UTF-8 locale gives the + byte 0xFF as U+DCFF, and a Latin-1 one as U+00FF).""" + def fsdecode(filename): + filename = os.fspath(filename) + if isinstance(filename, str): + return filename + return filename.decode(encoding, "surrogateescape") + return fsdecode + + +def skip_where_an_earlier_question_skips() -> None: + """The rows that let the probe through to the real disk ask the system's + question and the locale's first (#1196), and skip where either would: the + helper then skips on its own reason, and not on the one they hold.""" + if (len(os.fsencode(NOT_UTF_8_PROBE)) != 1 + or os.fsdecode(b"\xff") != NOT_UTF_8_PROBE): + pytest.skip("on this system an earlier question is the one that skips") + + def mkdir_answers(code: int, said: str): """A `Path.mkdir` that fails as the disk would with `code`.""" def refuse(self, *args, **kwargs): @@ -144,6 +174,7 @@ def test_where_fsencode_makes_three_bytes_a_name_that_is_text_is_not_skipped( def test_a_disk_that_takes_the_name_is_not_skipped_and_the_probe_is_removed( tmp_path): """The control: no skip, and nothing is left in the directory.""" + skip_where_an_earlier_question_skips() assert skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) is None assert list(tmp_path.iterdir()) == [] @@ -162,11 +193,40 @@ def test_where_fsencode_makes_three_bytes_the_disk_is_not_asked( assert "os.fsencode" in reason, reason +@LINUX_ONLY +@pytest.mark.parametrize("encoding", ["utf-8", "ascii"]) +def test_a_locale_that_reads_the_byte_as_the_surrogate_is_not_skipped( + tmp_path, monkeypatch, encoding): + """The control of the locale's question (#1196): UTF-8 and ASCII give the + byte 0xFF back as U+DCFF, which is `NOT_UTF_8_PROBE`, so no test that + asks is skipped, and the probe is removed.""" + monkeypatch.setattr(os, "fsdecode", fsdecode_in(encoding)) + assert os.fsdecode(b"\xff") == NOT_UTF_8_PROBE + assert skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path) is None + assert list(tmp_path.iterdir()) == [] + + +def test_a_locale_that_reads_the_byte_as_a_character_skips_before_the_disk( + tmp_path, monkeypatch): + """Latin-1 (emulated): the byte 0xFF is U+00FF, a printable character, + so no name holding it is one that is not UTF-8, and the name is skipped. + THE LOCALE IS ASKED BEFORE THE DISK (#1196), as the system is: the + directory does not exist, so an `mkdir` under it would be + FileNotFoundError and not a skip, and the skip is the answer.""" + monkeypatch.setattr(os, "fsdecode", fsdecode_in("iso8859-1")) + assert os.fsdecode(b"\xff") == "\u00ff" + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path / "absent") + assert reason is not None, "the helper went on to ask the disk" + assert "0xFF" in reason, reason + assert "'\\xff'" in reason, reason + + def test_a_disk_that_answers_eilseq_is_skipped(tmp_path, monkeypatch): """macOS (emulated): APFS answers EILSEQ, `Illegal byte sequence`, to a file name that is not UTF-8, and no clone can sit at such a path.""" - if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: - pytest.skip("on this system the first question is the one that skips") + skip_where_an_earlier_question_skips() monkeypatch.setattr(Path, "mkdir", mkdir_answers(errno.EILSEQ, "Illegal byte sequence")) reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) @@ -181,8 +241,7 @@ def test_any_other_error_from_the_disk_is_the_tests_own( """Only EILSEQ is the filesystem's refusal of the name: a full disk, a folder nobody can write in or a probe left by an earlier run is a failure of the test, raised as it was, and never a skip that hides it.""" - if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: - pytest.skip("on this system the first question is the one that skips") + skip_where_an_earlier_question_skips() monkeypatch.setattr(Path, "mkdir", mkdir_answers(code, os.strerror(code))) with pytest.raises(OSError) as raised: skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) diff --git a/tests/test_run_script_encoding.py b/tests/test_run_script_encoding.py new file mode 100644 index 00000000..28cce0ad --- /dev/null +++ b/tests/test_run_script_encoding.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`run_script` decodes a child's output in the encoding the caller names +(#1196, from #1185's review). + +A child told `PYTHONIOENCODING=utf-8` writes U+00E9 as the two bytes 0xC3 +0xA9. `run_script` read them in the PARENT's locale, and a parent in a locale +of one byte to a character, Latin-1 say, read two characters: the rows of +`tests/test_adopt_plan_paths_spelled.py` that name `pl\\u00e9an.yaml` failed +on a tool that was right. They name `encoding="utf-8"` now. Each row here +names an encoding that reads the two bytes one way and no other, so the +keyword is held in any locale the suite runs in, with no Latin-1 locale +built: the child is the same, and only the name differs. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from conftest import run_script + +#: A child that writes the two bytes of U+00E9 in UTF-8 to stdout and to +#: stderr, as one told `PYTHONIOENCODING=utf-8` does. +SAYS_E_ACUTE = ("import sys\n" + "sys.stdout.buffer.write(bytes([0xC3, 0xA9]))\n" + "sys.stderr.buffer.write(bytes([0xC3, 0xA9]))\n") + + +@pytest.mark.parametrize("encoding, read", [ + pytest.param("utf-8", "\u00e9", id="utf-8-reads-one-character"), + pytest.param("latin-1", "\u00c3\u00a9", + id="latin-1-reads-two"), +]) +def test_the_output_is_read_in_the_encoding_the_caller_names( + tmp_path, encoding, read): + """Stdout and stderr both, as the named encoding reads the two bytes: + one character in UTF-8 and two in Latin-1. A `run_script` that dropped + the keyword would read both in this process's own locale, which is one + answer for the two rows, so one of them would fail.""" + script = tmp_path / "say_e_acute.py" + script.write_text(SAYS_E_ACUTE, encoding="ascii") + result = run_script(script, encoding=encoding) + assert result.returncode == 0, result.stderr + assert result.stdout == read + assert result.stderr == read