diff --git a/tests/conftest.py b/tests/conftest.py index 9f8dedf6..c964dcd2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1235,12 +1235,14 @@ def on_a_branch_that_is_not_utf_8(repo: Path) -> dict: return held_by_git(repo) -def windows_fsencode(text: str | bytes) -> bytes: +def windows_fsencode(text: str | bytes | os.PathLike) -> bytes: """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every - lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes - go through as they are, as `os.fsencode` passes them on every system. For + lone surrogate is the three bytes of a sequence that is not UTF-8. A path + is taken as the real `os.fsencode` takes it (`os.fspath` first), and + bytes go through as they are, as it passes them on every system. For `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the guard above in front of a test that Linux would let through.""" + text = os.fspath(text) if isinstance(text, bytes): return text return text.encode("utf-8", "surrogatepass") @@ -1259,6 +1261,25 @@ def skip_message(call, *args) -> str | None: return None +def skip_where_the_environment_cannot_carry(value: str) -> None: + """Skip where this system's file name encoding cannot encode `value`, as + under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII). It is asked of + a file name (`make_file` cannot write `caf\\u00e9.yml`, and `mkdir` cannot + make `\\u202ecaf`), of an environment value and of a command-line + argument: `subprocess` puts each one through `os.fsencode` before it + starts the child, so a `user.name` of `Jos\\u00e9` or a branch + `main\\u00a0` raises `UnicodeEncodeError` in the test and not in the tool + (#757, #853, #854). Text that is ASCII, or a lone surrogate that + `os.fsencode` gives back as its byte, is carried. Asked through the `os` + module at call time, so a test emulates ASCII with + `monkeypatch.setattr(os, "fsencode", ...)`.""" + try: + os.fsencode(value) + except UnicodeEncodeError: + pytest.skip(f"this system's file name encoding " + f"({sys.getfilesystemencoding()}) cannot spell {value!r}") + + # --- the `gh` guard (#296) --------------------------------------------------- # # NO TEST REACHES THE OPERATOR'S `gh`, and the doctor's `machine` row is still diff --git a/tests/test_adopt_git_reads.py b/tests/test_adopt_git_reads.py index a8059442..fc86e7f6 100644 --- a/tests/test_adopt_git_reads.py +++ b/tests/test_adopt_git_reads.py @@ -44,7 +44,8 @@ import pytest from conftest import (REPO, appended_to_command_scope, git, make_source_repo, - skip_where_a_name_cannot_be_not_utf_8) + skip_where_a_name_cannot_be_not_utf_8, + skip_where_the_environment_cannot_carry) from test_adopt_template_values import (ADOPT_REFUSED, PLAN_FILE, PLAN_OK, PROJECT, REMOTES, SPLIT_BRANCH, VERIFIED, WORK, adopted, answer, @@ -462,15 +463,12 @@ def test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path): at a path that is text has the old path as its origin, so `execute --source ` ended in this issue's traceback; it now adopts. Skipped where no directory can be named with the byte 0xE9 (Windows, - macOS's APFS) or git will not work in one.""" + macOS's APFS); a failure of git under it fails the test (#853).""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), b"caf\xe9"))) parent.mkdir() - try: - source = make_source_repo(parent / "Thing") - except AssertionError as exc: - pytest.skip(f"git will not work under {parent.name!r} here: {exc}") + source = make_source_repo(parent / "Thing") clone = tmp_path / "clone" git("clone", "-q", "--", str(source), str(clone), cwd=tmp_path) runs = {"plan": planned(tmp_path, source)} @@ -522,6 +520,25 @@ def test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path): LOST_REFUSAL = b"REFUSED plan-tracking-branch-missing: " +def skip_where_no_ref_can_be_named(source: Path, branch: str) -> None: + """Skip where this system cannot give `source` a ref named `branch`, + asked before any git call makes one (#853). A name that is not UTF-8 + asks `skip_where_a_name_cannot_be_not_utf_8` of the directory `source` + is in, on the disk its ref files go to: Windows' `os.fsencode` first, + then macOS's EILSEQ. A name that is text asks only whether this + system's file name encoding can spell it, as a command line must + (`subprocess` puts each argument through `os.fsencode`; ASCII under + `LC_ALL=C` cannot), since every disk git runs on holds text. What git + does after it is the test's own: a git call that fails, on a full disk + or a directory it cannot write in, fails the test.""" + try: + branch.encode("utf-8") + except UnicodeEncodeError: + skip_where_a_name_cannot_be_not_utf_8(source.parent) + else: + skip_where_the_environment_cannot_carry(branch) + + def with_default_branch(source: Path, where: str, branch: str = BRANCH) -> Path: """`source`, on `main`, with `branch` (`BRANCH` unless given) as its @@ -531,23 +548,21 @@ def with_default_branch(source: Path, where: str, source: a plan that read the wrong branch would adopt the wrong commit, and a `--tracking-branch main` names a branch it has. - Skipped where no branch can be named so (the byte 0xE9 on Windows or - macOS's APFS), or git here does not give the name back as it was - written.""" + Skipped where this system cannot name the branch + (`skip_where_no_ref_can_be_named`: the byte 0xE9 on Windows or macOS's + APFS, or text an ASCII file name encoding cannot spell), asked before + git is, and where git here does not give the name back as it was + written. A git call between the two that fails fails the test (#853).""" ref = "refs/remotes/origin/HEAD" if where == ORIGIN_HEAD else "HEAD" written = branch.encode("utf-8", "surrogateescape") - try: - git("switch", "-q", "-c", branch, cwd=source) - git("commit", "-q", "--allow-empty", "-m", "One ahead", cwd=source) - if where == ORIGIN_HEAD: - git("switch", "-q", "main", cwd=source) - for name in (branch, "main"): - git("update-ref", f"refs/remotes/origin/{name}", name, - cwd=source) - git("symbolic-ref", ref, f"refs/remotes/origin/{branch}", - cwd=source) - except (AssertionError, UnicodeError, OSError) as exc: - pytest.skip(f"no branch here is named {written!r}: {exc!r}") + skip_where_no_ref_can_be_named(source, branch) + git("switch", "-q", "-c", branch, cwd=source) + git("commit", "-q", "--allow-empty", "-m", "One ahead", cwd=source) + if where == ORIGIN_HEAD: + git("switch", "-q", "main", cwd=source) + for name in (branch, "main"): + git("update-ref", f"refs/remotes/origin/{name}", name, cwd=source) + git("symbolic-ref", ref, f"refs/remotes/origin/{branch}", cwd=source) read = subprocess.run(["git", "symbolic-ref", ref], cwd=str(source), capture_output=True, check=False) if not read.stdout.rstrip(b"\r\n").endswith(b"/" + written): @@ -559,15 +574,14 @@ def moved_to_a_branch_that_is_not_utf8(base: Path) -> Path: """A source on `main`, planned and answered, whose `main` is then renamed `BRANCH`: a plan that tracks a branch the source no longer has, and a source whose HEAD is on one that is not UTF-8. Skipped as - `with_default_branch` is.""" + `with_default_branch` is, before anything is made; a failure of git + fails the test (#853).""" + skip_where_no_ref_can_be_named(base / "Thing", BRANCH) source = make_source_repo(base / "Thing") made = planned(base, source) assert made.returncode == 0, said(made) answer(base) - try: - git("branch", "-m", "main", BRANCH, cwd=source) - except (AssertionError, UnicodeError, OSError) as exc: - pytest.skip(f"no branch here is named {NOT_UTF8_BRANCH!r}: {exc!r}") + git("branch", "-m", "main", BRANCH, cwd=source) return source @@ -720,15 +734,14 @@ def test_plan_reads_a_default_branch_ending_in_white_space_whole(tmp_path, def remembered_on(source: Path, branch: str) -> Path: """`source` with `origin/HEAD` naming `origin/`, which it has fetched, and no branch `branch` of its own, as `git clone -b main` of a - repository whose default branch is `branch` leaves it. Skipped as - `with_default_branch` is, where git refuses or alters the name.""" - try: - git("update-ref", f"refs/remotes/origin/{branch}", "refs/heads/main", - cwd=source) - git("symbolic-ref", "refs/remotes/origin/HEAD", - f"refs/remotes/origin/{branch}", cwd=source) - except (AssertionError, UnicodeError, OSError) as exc: - pytest.skip(f"no ref here is named {branch!r}: {exc!r}") + repository whose default branch is `branch` leaves it. Skipped where + this system cannot name the ref, asked before git is, as + `with_default_branch` is; a failure of git fails the test (#853).""" + skip_where_no_ref_can_be_named(source, branch) + git("update-ref", f"refs/remotes/origin/{branch}", "refs/heads/main", + cwd=source) + git("symbolic-ref", "refs/remotes/origin/HEAD", + f"refs/remotes/origin/{branch}", cwd=source) return source diff --git a/tests/test_adopt_plan_escapes.py b/tests/test_adopt_plan_escapes.py index c6041a72..d94446ec 100644 --- a/tests/test_adopt_plan_escapes.py +++ b/tests/test_adopt_plan_escapes.py @@ -398,15 +398,13 @@ def test_a_source_directory_that_is_not_utf8_is_shown_spelled(adopter, so did `check`'s `WHAT WILL HAPPEN, against @`, and each raised `UnicodeEncodeError` on a strict UTF-8 stdout after the plan was written. Both show it spelled now. Skipped where no directory can be named with - the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" + the byte 0xE9 (Windows, macOS's APFS); a failure of git under it fails + the test (#853).""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), b"caf\xe9"))) parent.mkdir() - try: - source = make_source_repo(parent / "Thing") - except AssertionError as exc: - pytest.skip(f"git will not work under b'caf\\xe9' here: {exc}") + source = make_source_repo(parent / "Thing") shown = adopter._spelled(str(source.resolve())) assert "\\udce9" in shown plan = tmp_path / PLAN_FILE diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index 991687f5..c2944c07 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -51,7 +51,8 @@ from conftest import (ADOPT, REPO, WINDOWS_SKIP, blank_unnamed_pin_sources, make_source_repo, resolve, - skip_where_a_name_cannot_be_not_utf_8) + skip_where_a_name_cannot_be_not_utf_8, + skip_where_the_environment_cannot_carry) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -197,7 +198,11 @@ def test_plan_refuses_a_flag_no_file_can_hold(tmp_path, flag, value): `plan ok`, and `execute` raised in `write_lf` after both legs were pushed (a surrogate), or adopted with the line break, the tab or the bidi control in the manifest and the README. Now `plan` refuses, and - writes no plan.""" + writes no plan. Skipped where this system's file name encoding cannot + spell `value` (U+2028, U+202E or U+2066 in ASCII under `LC_ALL=C`), + asked first: `subprocess` cannot put it on `plan`'s command line there, + and raised `UnicodeEncodeError` in the test, not in the tool.""" + skip_where_the_environment_cannot_carry(value) source = make_source_repo(tmp_path / "Thing") runs = plan_check_execute(tmp_path, source, flag, value) assert_no_traceback_and_no_leg(tmp_path, runs) @@ -246,8 +251,7 @@ def test_plan_names_the_elector_it_took_from_git_config(tmp_path): b"again") -def refused_then_moved(base: Path, parent: Path, reason: bytes, - skip_where_git_fails: bool = True) -> None: +def refused_then_moved(base: Path, parent: Path, reason: bytes) -> None: """A source in the directory `parent`: `plan` and `check` pass it, as #247 made them, and `execute` refuses its path as the assembly root's clone URL, giving `reason`, before any leg exists. Then the remediation @@ -255,17 +259,12 @@ def refused_then_moved(base: Path, parent: Path, reason: bytes, `plan --source `, `check` and `execute` run again -- ends `adoption verified`. - A source repository that git cannot make under `parent` skips the test - where `skip_where_git_fails` is true, for a name that is not UTF-8, which - git may not work under. A name that is valid text asks for it false, and - the failure is raised: the cause is the disk or git, never the name - (#834).""" - try: - source = make_source_repo(parent / "Thing") - except AssertionError as exc: - if not skip_where_git_fails: - raise - pytest.skip(f"git will not work under {parent.name!r} here: {exc}") + The source repository is made under `parent` with no guard: the caller + has asked whether this system can carry the name, and git works under + any directory a system carries, so a failure of git there (a disk that + filled after the `mkdir`) is the test's own and is raised, for a name + that is not UTF-8 (#853) as for one that is valid text (#834).""" + source = make_source_repo(parent / "Thing") runs = plan_check_execute(base, source) assert_no_traceback_and_no_leg(base, runs) assert runs["plan"].returncode == 0, said(runs["plan"]) @@ -300,8 +299,8 @@ def test_execute_refuses_a_local_source_directory_that_is_not_utf8(tmp_path): once said to clone the source to a UTF-8 path and pass that with --source, which ended in a traceback: the clone's `origin` is the old path, which `_remote_repository` reads as UTF-8. Skipped where no - directory can be named with the byte 0xE9 (Windows, macOS's APFS) or - git will not work in one.""" + directory can be named with the byte 0xE9 (Windows, macOS's APFS); a + failure of git under it fails the test (#853).""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), b"caf\xe9"))) @@ -315,15 +314,18 @@ def test_execute_refuses_a_local_source_directory_holding_a_bidi_control( """A source under a directory whose name begins with U+202E RIGHT-TO-LEFT OVERRIDE: on main `execute` adopted it, and the shape's README showed the rest of its `git clone` line reversed. Now it is refused as the - byte is, and moving the source adopts. Nothing here is guarded: U+202E - is valid text, which a POSIX disk, NTFS and APFS hold and git works - under (reasoned, not run, on the last two), so an error of the disk when - the directory is made, and a failure of git when the source is, are the - test's own and are raised, not skipped (#834).""" + byte is, and moving the source adopts. Skipped where this system's file + name encoding cannot spell U+202E (ASCII under `LC_ALL=C`), which is + asked before the `mkdir`, where it raised `UnicodeEncodeError` (#854). + Nothing else is guarded: U+202E is valid text, which a POSIX disk, NTFS + and APFS hold and git works under (reasoned, not run, on the last two), + so an error of the disk when the directory is made, and a failure of git + when the source is, are the test's own and are raised, not skipped + (#834).""" parent = tmp_path / "\u202ecaf" + skip_where_the_environment_cannot_carry(parent.name) parent.mkdir() - refused_then_moved(tmp_path, parent, b"a bidirectional control", - skip_where_git_fails=False) + refused_then_moved(tmp_path, parent, b"a bidirectional control") # --- `check` and `execute`: a plan value edited by hand ---------------------- @@ -647,7 +649,13 @@ def test_a_valid_non_ascii_elector_still_adopts(tmp_path, elected_by): The manifest is asked through the reader, not for a line: since #250 a YAML template's value is written as `double_quoted` spells it, so the - line holds U+200C as an escape there, and as it stands before.""" + line holds U+200C as an escape there, and as it stands before. + + Skipped where this system's file name encoding cannot spell the name + (ASCII under `LC_ALL=C`), asked first: `subprocess` cannot put it on + `plan`'s command line there, and raised `UnicodeEncodeError` in the + test, not in the tool.""" + skip_where_the_environment_cannot_carry(elected_by) source = make_source_repo(tmp_path / "Thing") runs = plan_check_execute(tmp_path, source, "--elected-by", elected_by) assert_no_traceback(runs) @@ -666,8 +674,13 @@ def test_a_valid_non_ascii_elector_still_adopts(tmp_path, elected_by): def test_a_utf8_source_directory_still_adopts_with_local_remotes(tmp_path): """A source under a directory `café`: its path is the clone URL, which a file holds, so it adopts, and the shape's README (beside the source's - own) names it.""" - source = make_source_repo(tmp_path / "café" / "Thing") + own) names it. Skipped where this system's file name encoding cannot + spell the directory's name (ASCII under `LC_ALL=C`), asked before the + directory is made, where `mkdir` raised `UnicodeEncodeError` in the + test.""" + parent = tmp_path / "caf\u00e9" + skip_where_the_environment_cannot_carry(parent.name) + source = make_source_repo(parent / "Thing") runs = plan_check_execute(tmp_path, source) assert_no_traceback(runs) assert runs["check"].returncode == 0, said(runs["check"]) diff --git a/tests/test_non_utf8_git_failures.py b/tests/test_non_utf8_git_failures.py new file mode 100644 index 00000000..6f657c21 --- /dev/null +++ b/tests/test_non_utf8_git_failures.py @@ -0,0 +1,482 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A test that needs a directory, or a branch, named with a byte that is not +UTF-8 asks the system once whether it can carry the name, and a failure of the +git call that follows fails the test (#853, #461). + +Six places turned a failure of git into `pytest.skip`. The issue's scan of +`tests/*.py` found five; the sixth, `remembered_on`, came with #830 (#797) +after it: + +- `refused_then_moved` (`tests/test_adopt_template_values.py`), + `test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts` + (`tests/test_adopt_git_reads.py`) and + `test_a_source_directory_that_is_not_utf8_is_shown_spelled` + (`tests/test_adopt_plan_escapes.py`) skipped "git will not work under ..." + on an `AssertionError` of `make_source_repo` under the directory `caf` and + byte 0xE9; +- `with_default_branch`, `moved_to_a_branch_that_is_not_utf8` and + `remembered_on` (`tests/test_adopt_git_reads.py`) skipped "no branch here + is named ..." on an `AssertionError`, a `UnicodeError` or an `OSError` of + the git calls that make the branch, and the first and the last are called + with a name that is text too (`main` and U+00A0, `ma`, U+2028 and `in`). + +A git process that fails raises `AssertionError` (`conftest.git`, and +`make_source_repo`'s own commits), so a disk that filled after the `mkdir`, or +a directory git could no longer write in, read as SKIPPED, and no run fails +on a skip. The answers that mean "this system cannot carry the name" are +asked before git now, and every git call after them is unguarded: for a +directory, `skip_where_a_name_cannot_be_not_utf_8`, which the three tests +already asked (#757); for a branch, `skip_where_no_ref_can_be_named`, which +asks that helper for a name that is not UTF-8 and the file name encoding for +a name that is text. The skip after `git symbolic-ref` stays: whether git +gives the name back as it was written is the system's answer, and no +exception is behind it. + +#854 is the C-locale half: under `PYTHONUTF8=0 LC_ALL=C PYTHONCOERCECLOCALE=0` +the file name encoding is ASCII, and the U+202E test's `mkdir` raised +`UnicodeEncodeError`, a `ValueError` its old `except OSError` never caught, +so the test ended in an error. It asks the encoding before the `mkdir` now. +The six tests of `tests/test_adopt_template_values.py` that ask it before +`make_source_repo` (a flag, an elector or a directory that ASCII cannot +spell) are held for CI, which runs UTF-8, by rows that emulate ASCII. + +Each place is called here as a plain function, with git made to fail as a +full disk makes `git commit` fail, or with a system emulated that CI does not +run (CI is Linux only, #597; no Windows or macOS run was made): + +- git fails: the test FAILS with git's error, for a name that is not UTF-8 + and for one that is text; +- Windows' `os.fsencode` (a surrogate is three bytes): the three tests of the + byte directory are skipped on the helper's reason before their `mkdir` and + before git, and make nothing (macOS's EILSEQ on the probe is held for them + by #757's rows in `tests/test_non_utf8_name_callers.py`); +- macOS's EILSEQ on the probe, or Windows' `os.fsencode`: a branch that is + not UTF-8 is skipped on the helper's reason before git is called and + before anything is made; a branch that is text asks neither and goes on + to git; +- an ASCII file name encoding: a branch that is text, and the U+202E test, are + skipped on the encoding's reason, before git and before the `mkdir`; so + are the six tests of `tests/test_adopt_template_values.py` that ask the + encoding first, before `make_source_repo` (which would fail here) and + before anything is made under `tmp_path`; +- every `tests/*.py`, read as #853's scan reads it and wider: no `except` + that catches an `AssertionError` (one that names it, `Exception` or + `BaseException`, or a bare one) holds a `pytest.skip`. +""" + +from __future__ import annotations + +import ast +import errno +import os +from textwrap import dedent + +import pytest + +import test_adopt_git_reads as git_reads +import test_adopt_plan_escapes as plan_escapes +import test_adopt_template_values as template_values +from conftest import (REPO, skip_message, + skip_where_the_environment_cannot_carry, + windows_fsencode) +# The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a +# module, which its directory test asks for. +from test_adopt_plan_escapes import adopter # noqa: F401 - fixture +# The fixtures and the emulations of #757's callers file, and the failure of +# git and the way of asking for an error of #834's sites file. +from test_non_utf8_name_callers import ( # noqa: F401 - fixtures + a_disk_that_takes_the_probe, a_system_that_carries_the_byte, + ascii_fsencode, refusing) +from test_non_utf8_oserror_sites import (BIDI_TEST, GIT_FAILED, + a_source_git_cannot_make, + the_error_of) + +# --- a source repository under a directory that is not UTF-8 ---------------- + +#: The three tests that make a source repository under the directory `caf` and +#: byte 0xE9, each with the module whose `make_source_repo` it calls, called +#: with a `tmp_path` and the `adopter` fixture. +UNDER_A_BYTE_DIRECTORY = [ + pytest.param( + template_values, + lambda tmp_path, adopter: template_values + .test_execute_refuses_a_local_source_directory_that_is_not_utf8( + tmp_path), + id="adopt-template-values"), + pytest.param( + git_reads, + lambda tmp_path, adopter: git_reads + .test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path), + id="adopt-git-reads"), + pytest.param( + plan_escapes, + lambda tmp_path, adopter: plan_escapes + .test_a_source_directory_that_is_not_utf8_is_shown_spelled( + adopter, tmp_path), + id="adopt-plan-escapes"), +] + + +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") +@pytest.mark.parametrize("module, call", UNDER_A_BYTE_DIRECTORY) +def test_a_failure_of_git_under_the_byte_directory_fails_the_test( + tmp_path, monkeypatch, adopter, module, call): + """The directory is made, and then git cannot make the source repository + in it, as on a disk that filled after the `mkdir`: the test FAILS with + git's error, where it was SKIPPED as "git will not work under ...". The + rows ask the real disk for the probe first, and skip where it answers + EILSEQ, as the test does before git is reached.""" + monkeypatch.setattr(module, "make_source_repo", a_source_git_cannot_make) + error = the_error_of(AssertionError, call, tmp_path, adopter) + assert str(error) == GIT_FAILED + + +@pytest.mark.parametrize("module, call", UNDER_A_BYTE_DIRECTORY) +def test_where_fsencode_makes_no_byte_the_byte_directory_test_skips_first( + tmp_path, monkeypatch, adopter, module, call): + """Windows (emulated): `os.fsencode` gives the surrogate three bytes, so + the test skips on the helper's reason before it makes its directory or + asks git (which would fail here), and nothing is made under `tmp_path`. + A test that asked after its `mkdir` would leave the directory here, and + on Windows raise at `os.fsdecode` of the name instead of skipping.""" + monkeypatch.setattr(module, "make_source_repo", a_source_git_cannot_make) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(call, tmp_path, adopter) + assert reason is not None, "the test went on where fsencode makes none" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +# --- a branch or a ref that is not UTF-8, or is text ------------------------ + +#: The three helpers of `tests/test_adopt_git_reads.py` that make a branch or a +#: ref with git, given a name that is not UTF-8 (`BRANCH`, `caf` and byte +#: 0xE9), each called with a `tmp_path`. The source is `tmp_path / "Thing"`; +#: only `moved_to_a_branch_that_is_not_utf8` makes it itself. +NOT_UTF_8_NAMES = [ + pytest.param( + lambda base: git_reads.with_default_branch(base / "Thing", + git_reads.ORIGIN_HEAD), + id="with-default-branch-origin-head"), + pytest.param( + lambda base: git_reads.with_default_branch(base / "Thing", "head"), + id="with-default-branch-head"), + pytest.param(git_reads.moved_to_a_branch_that_is_not_utf8, + id="moved-to-a-branch-that-is-not-utf8"), + pytest.param( + lambda base: git_reads.remembered_on(base / "Thing", git_reads.BRANCH), + id="remembered-on"), +] +#: The two of them that are also given a name that is text, with that name. +TEXT_NAMES = [ + pytest.param( + lambda base: git_reads.with_default_branch( + base / "Thing", git_reads.ORIGIN_HEAD, git_reads.NBSP_BRANCH), + git_reads.NBSP_BRANCH, id="with-default-branch-nbsp"), + pytest.param( + lambda base: git_reads.remembered_on(base / "Thing", + git_reads.LINE_SEPARATED), + git_reads.LINE_SEPARATED, id="remembered-on-u2028"), +] + + +def git_fails(monkeypatch) -> None: + """From here on every git call the module makes itself fails as + `conftest.git` fails where `git commit` cannot write an object. Only the + module's own name is replaced: `make_source_repo` and the tool's runs + still reach the real git.""" + monkeypatch.setattr(git_reads, "git", a_source_git_cannot_make) + + +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") +@pytest.mark.parametrize("call", NOT_UTF_8_NAMES) +def test_a_failure_of_git_making_a_branch_that_is_not_utf8_fails_the_test( + tmp_path, monkeypatch, call): + """The system carries the name, so the helper goes on to git, and git's + failure reaches the test as a failure, where it was SKIPPED as "no + branch here is named ...". The rows ask the real disk for the probe + first, as the helper does.""" + git_fails(monkeypatch) + error = the_error_of(AssertionError, call, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.parametrize("call, name", TEXT_NAMES) +def test_a_failure_of_git_making_a_branch_that_is_text_fails_the_test( + tmp_path, monkeypatch, call, name): + """No system refuses a name that is text, so every skip there hid a + failure. Skipped where this system's file name encoding cannot spell the + name (ASCII under `LC_ALL=C`): the helper skips there first.""" + skip_where_the_environment_cannot_carry(name) + git_fails(monkeypatch) + error = the_error_of(AssertionError, call, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", NOT_UTF_8_NAMES) +def test_a_disk_that_refuses_the_probe_with_eilseq_skips_before_git( + tmp_path, monkeypatch, call): + """macOS (emulated): the probe is `Illegal byte sequence`, so the helper + skips, on its reason, before any git call (each would fail here) and + before anything is made under `tmp_path`.""" + git_fails(monkeypatch) + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + reason = skip_message(call, tmp_path) + assert reason is not None, "the helper went on past the probe's EILSEQ" + assert "EILSEQ" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("call", NOT_UTF_8_NAMES) +def test_where_fsencode_makes_no_byte_of_the_surrogate_the_branch_skips_first( + tmp_path, monkeypatch, call): + """Windows (emulated): `os.fsencode` gives the surrogate three bytes, so + no argument can carry the byte. The helper skips on that reason before + the disk and git are asked, and nothing is made under `tmp_path`.""" + git_fails(monkeypatch) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, "the helper went on where fsencode makes none" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("call, name", TEXT_NAMES) +def test_a_branch_that_is_text_asks_nothing_of_a_byte( + tmp_path, monkeypatch, call, name): + """Neither macOS's EILSEQ to a name that is not UTF-8 nor Windows' + `os.fsencode` of a surrogate is the answer for a name that is text, so + with both in place the helper goes on to git, and git's failure is the + test's. Skipped where the file name encoding cannot spell the name, as + the test above is.""" + skip_where_the_environment_cannot_carry(name) + git_fails(monkeypatch) + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + error = the_error_of(AssertionError, call, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.parametrize("call, name", TEXT_NAMES) +def test_where_the_file_name_encoding_is_ascii_a_branch_that_is_text_skips( + tmp_path, monkeypatch, call, name): + """`LC_ALL=C` (emulated): `subprocess` puts each argument through + `os.fsencode`, which cannot spell U+00A0 or U+2028 in ASCII, so the + helper skips on the encoding's reason before git is called, where the + git call raised `UnicodeEncodeError` in the test.""" + git_fails(monkeypatch) + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, f"{name!r} went on to git under ASCII" + assert "file name encoding" in reason, reason + + +# --- the U+202E directory under an ASCII file name encoding (#854) ---------- + +def test_where_the_file_name_encoding_cannot_spell_u202e_the_bidi_test_skips( + tmp_path, monkeypatch): + """`LC_ALL=C` (emulated as the real one answers): `os.fsencode` is ASCII, + and `os.mkdir` raises `UnicodeEncodeError` for a name ASCII cannot + encode. The test asks the encoding before its `mkdir` and skips on the + encoding's reason, worded as `make_file`'s is, and makes nothing; it + ended in that `UnicodeEncodeError`.""" + real_mkdir = os.mkdir + + def mkdir(path, *args, **kwargs): + ascii_fsencode(path) + return real_mkdir(path, *args, **kwargs) + + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + monkeypatch.setattr(os, "mkdir", mkdir) + reason = skip_message(BIDI_TEST, tmp_path) + assert reason is not None, "the test went on past an ASCII encoding" + assert "this system's file name encoding" in reason, reason + assert "cannot spell" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +#: The six tests of `tests/test_adopt_template_values.py` that skip where the +#: file name encoding cannot spell their value (#854), each called with a +#: `tmp_path`: the flag test with the three values ASCII cannot spell (U+2028, +#: U+202E and U+2066), the elector test with `Jos\u00e9` and a name joined +#: with U+200C, and the test of a source under a directory `caf\u00e9`. +WHERE_ASCII_CANNOT_SPELL = [ + pytest.param( + lambda base: template_values.test_plan_refuses_a_flag_no_file_can_hold( + base, "--elected-by", "Jos\u2028e"), + id="flag-u2028"), + pytest.param( + lambda base: template_values.test_plan_refuses_a_flag_no_file_can_hold( + base, "--elected-by", "\u202eevil"), + id="flag-u202e"), + pytest.param( + lambda base: template_values.test_plan_refuses_a_flag_no_file_can_hold( + base, "--reference", "ref \u2066x\u2069 y"), + id="flag-u2066"), + pytest.param( + lambda base: template_values + .test_a_valid_non_ascii_elector_still_adopts(base, "Jos\u00e9"), + id="elector-accented"), + pytest.param( + lambda base: template_values + .test_a_valid_non_ascii_elector_still_adopts(base, "Ali\u200cReza"), + id="elector-zwnj"), + pytest.param( + template_values + .test_a_utf8_source_directory_still_adopts_with_local_remotes, + id="cafe-directory"), +] + + +@pytest.mark.parametrize("call", WHERE_ASCII_CANNOT_SPELL) +def test_where_the_file_name_encoding_is_ascii_the_template_values_tests_skip( + tmp_path, monkeypatch, call): + """`LC_ALL=C` (emulated): `os.fsencode` is ASCII, so each test skips on + the encoding's reason before `make_source_repo` (which fails here, as git + fails on a full disk) and before any directory is made, and makes nothing + under `tmp_path`. CI runs UTF-8, where a test that asked after either + would still pass; these rows fail it there, as the real `LC_ALL=C` run + does.""" + monkeypatch.setattr(template_values, "make_source_repo", + a_source_git_cannot_make) + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, "the test went on under an ASCII encoding" + assert "file name encoding" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +# --- no test skips on an `AssertionError` ----------------------------------- + +def names_in(node: ast.expr | None) -> set[str]: + """The names an `except` clause catches: one, or each of a tuple.""" + if isinstance(node, ast.Tuple): + return {name for element in node.elts for name in names_in(element)} + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, ast.Attribute): + return {node.attr} + return set() + + +def is_pytest_skip(node: ast.AST) -> bool: + return (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "skip" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "pytest") + + +#: What an `except` that catches an `AssertionError` names: the error itself +#: or a class above it. A bare `except` catches it too. +CATCH_AN_ASSERTION_ERROR = {"AssertionError", "Exception", "BaseException"} + + +def catches_an_assertion_error(handler: ast.ExceptHandler) -> bool: + """A bare `except`, or one that names `AssertionError`, `Exception` or + `BaseException`, alone or in a tuple.""" + return (handler.type is None + or bool(names_in(handler.type) & CATCH_AN_ASSERTION_ERROR)) + + +def skips_that_catch_an_assertion_error(source: str) -> list[int]: + """The line of each `except` in `source` that catches an + `AssertionError` and holds a call of `pytest.skip` anywhere in its body: + #853's scan, of an `except` that names `AssertionError`, and with it one + that catches the error without naming it (`Exception`, `BaseException` + or a bare `except`), which would hide a failed git call as well.""" + return [node.lineno for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.ExceptHandler) + and catches_an_assertion_error(node) + and any(is_pytest_skip(inner) for inner in ast.walk(node))] + + +def test_no_test_skips_on_an_assertion_error(): + """`conftest.git` and `make_source_repo` raise `AssertionError` for a git + process that fails, whatever made it fail, so an `except` that catches it + and skips turns a full disk into SKIPPED, whether it names the error, + `Exception` or `BaseException`, or nothing. Where a system cannot carry a + name, a test asks it first and skips on that answer; the git call after + it is unguarded. Every `tests/*.py`, this file and `conftest.py` + included.""" + found = {} + for path in sorted((REPO / "tests").glob("*.py")): + lines = skips_that_catch_an_assertion_error( + path.read_text(encoding="utf-8")) + if lines: + found[path.name] = lines + assert found == {}, found + + +@pytest.mark.parametrize("source, lines", [ + pytest.param(""" + try: + make() + except AssertionError as exc: + pytest.skip(f"git will not work here: {exc}") + """, [4], id="alone"), + pytest.param(""" + try: + make() + except (AssertionError, UnicodeError, OSError) as exc: + pytest.skip(f"no branch here is named: {exc!r}") + """, [4], id="in-a-tuple"), + pytest.param(""" + try: + make() + except AssertionError: + if not asked: + raise + pytest.skip("git will not work here") + """, [4], id="behind-a-raise"), + pytest.param(""" + try: + make() + except Exception as exc: + pytest.skip(f"no branch here is named: {exc!r}") + """, [4], id="exception"), + pytest.param(""" + try: + make() + except (OSError, BaseException): + pytest.skip("anything at all") + """, [4], id="base-exception-in-a-tuple"), + pytest.param(""" + try: + make() + except: + pytest.skip("anything at all") + """, [4], id="bare"), + pytest.param(""" + try: + make() + except AssertionError: + raise + """, [], id="no-skip"), + pytest.param(""" + try: + make() + except Exception: + raise + """, [], id="exception-no-skip"), + pytest.param(""" + try: + make() + except OSError: + pytest.skip("another error") + """, [], id="another-error"), + pytest.param(""" + if not read.endswith(written): + pytest.skip("git here does not give the name back") + """, [], id="a-skip-on-an-answer"), +]) +def test_the_scan_finds_a_skip_on_an_assertion_error_and_nothing_else( + source, lines): + """A skip behind an `if` in the `except` is found too, as #845's + `skip_where_git_fails` had it, and so is one in an `except` that catches + the error without naming it: `Exception`, `BaseException`, or a bare + `except`. An `except` that does not skip, one of another error, and the + skip after `git symbolic-ref`, which answers a question and catches + nothing, are not.""" + assert skips_that_catch_an_assertion_error(dedent(source)) == lines diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index a53d6547..29728bbc 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -55,6 +55,7 @@ from conftest import (ORG, PROJECT, REPO, appended_to_command_scope, cloned_just_pushed, git, run_script, skip_message, skip_where_fsencode_gives_a_surrogate_no_byte, + skip_where_the_environment_cannot_carry, windows_fsencode) from test_bump_leg import branched, bump @@ -513,22 +514,6 @@ def test_a_clean_standard_prints_no_warning(standard, tmp_path, run, warning, LEGACY_NAME = "Jos\udce9" -def skip_where_the_environment_cannot_carry(value: str) -> None: - """Skip where this system's file name encoding cannot encode `value`, as - under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII). It is asked of - a file name (`make_file` cannot write `caf\\u00e9.yml`) and of an - environment value: `subprocess` puts each one through `os.fsencode` - before it starts the child, so a `user.name` of `Jos\\u00e9` raises - `UnicodeEncodeError` in the test and not in the tool (#757). Text that is - ASCII, or a lone surrogate that `os.fsencode` gives back as its byte, is - carried.""" - try: - os.fsencode(value) - except UnicodeEncodeError: - pytest.skip(f"this system's file name encoding " - f"({sys.getfilesystemencoding()}) cannot carry {value!r}") - - REFUSED = "REFUSED family-unwritable-value: " #: How the refusal names the creator when no flag supplied it: `_manifest_flags` #: (#307), so that the two refusals agree on who is meant. diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 76db45df..dd6c7386 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -53,7 +53,9 @@ import test_non_utf8_git_reads as non_utf8_git_reads import test_stdout_encoding as stdout_encoding from conftest import (NOT_UTF_8_PROBE, skip_message, - skip_where_a_name_cannot_be_not_utf_8, windows_fsencode) + skip_where_a_name_cannot_be_not_utf_8, + skip_where_the_environment_cannot_carry, + windows_fsencode) # The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a # module, which two of its tests ask for. from test_adopt_plan_escapes import adopter # noqa: F401 - fixture @@ -285,8 +287,7 @@ def test_a_write_the_disk_refuses_still_fails_the_make_file_test( skip on `make_file`'s own "filesystem" reason would end it SKIPPED, and no run fails on that; before #834 it did skip, and the test failed on its assert that the call is not a skip.""" - non_utf8_git_reads.skip_where_the_environment_cannot_carry( - non_utf8_git_reads.UTF8) + skip_where_the_environment_cannot_carry(non_utf8_git_reads.UTF8) def write_bytes(self, data): raise OSError(code, os.strerror(code), str(self)) @@ -301,8 +302,7 @@ def write_bytes(self, data): def test_a_user_name_the_encoding_cannot_carry_skips_where_it_is_ascii( monkeypatch, value): monkeypatch.setattr(os, "fsencode", ascii_fsencode) - reason = skip_message( - non_utf8_git_reads.skip_where_the_environment_cannot_carry, value) + reason = skip_message(skip_where_the_environment_cannot_carry, value) assert reason is not None, f"{value!r} was carried by an ASCII encoding" assert "file name encoding" in reason, reason @@ -315,9 +315,8 @@ def test_a_user_name_the_encoding_carries_is_not_skipped_even_where_it_is_ascii( """ASCII text, a lone surrogate that is a byte, and the control characters are the environment's own: the tests of them still run.""" monkeypatch.setattr(os, "fsencode", ascii_fsencode) - assert skip_message( - non_utf8_git_reads.skip_where_the_environment_cannot_carry, - value) is None + assert skip_message(skip_where_the_environment_cannot_carry, + value) is None @pytest.mark.parametrize("test, args", [ diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 47ae0d13..8594c6b6 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -90,6 +90,15 @@ def test_windows_fsencode_passes_bytes_through_as_os_fsencode_does(): assert windows_fsencode(data) == data +def test_windows_fsencode_takes_a_path_as_os_fsencode_does(): + """`os.fsencode` takes a path-like object (`os.fspath` first), and a test + asks it for one (`os.fsencode(tmp_path)`), so the emulation must not fail + on a `Path` with an `AttributeError`, which would end such a test before + the question it is written to hold. A surrogate in a path is the three + bytes of one in text.""" + assert windows_fsencode(Path("caf\udcff")) == b"caf\xed\xb3\xbf" + + # --- the system's question: what `os.fsencode` makes of the surrogate -------- @LINUX_ONLY diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py index 181ed792..428828dd 100644 --- a/tests/test_non_utf8_oserror_sites.py +++ b/tests/test_non_utf8_oserror_sites.py @@ -26,7 +26,7 @@ and raises the rest. `make_file` skips on EILSEQ and on a `UnicodeError` (an ASCII file name encoding, #757), each on a reason of its own, and raises the rest. The U+202E test makes its directory, and its source repository, with no -guard. +guard but the file name encoding's question, asked before its `mkdir` (#854). Each is called here as a plain function, with the disk answering as it would on a system this suite does not run on (CI is Linux only, #597), and asked what @@ -41,9 +41,11 @@ EDQUOT, fails here; - Windows, by an `os.fsencode` that gives a surrogate three bytes: the three tests that ask the helper skip before the disk is touched; -- git failing under the U+202E directory fails the test, and under the - directory named with a byte that is not UTF-8 still skips it (#757 kept that - second skip, and it is a known limit: a full disk skips there). +- git failing under the U+202E directory fails the test. Under the directory + named with a byte that is not UTF-8 it fails the test too since #853, which + dropped the second skip #757 kept there: + `tests/test_non_utf8_git_failures.py` holds that, for each test that makes + a source repository under such a directory. """ from __future__ import annotations @@ -59,7 +61,8 @@ import test_cli_reference as cli_reference import test_doctor_validator_verdicts as doctor_verdicts import test_non_utf8_git_reads as non_utf8_git_reads -from conftest import skip_message, windows_fsencode +from conftest import (skip_message, skip_where_the_environment_cannot_carry, + windows_fsencode) # The fixtures and the two functions of #757's callers file, which ask the same # questions of the same helper. from test_non_utf8_name_callers import ( # noqa: F401 - fixtures @@ -283,8 +286,6 @@ def test_make_file_raises_any_other_error_of_the_disk( BIDI = chr(0x202E) BIDI_TEST = (template_values .test_execute_refuses_a_local_source_directory_holding_a_bidi_control) -BYTE_TEST = (template_values - .test_execute_refuses_a_local_source_directory_that_is_not_utf8) def test_a_disk_that_refuses_the_bidi_directory_fails_the_test( @@ -292,7 +293,10 @@ def test_a_disk_that_refuses_the_bidi_directory_fails_the_test( """U+202E is valid text: no probe is asked, and the directory is made with no guard, so what the disk answers to it is raised, whichever errno it is. It was a skip that said "no directory here is named ..." for a full disk - or a read-only one.""" + or a read-only one. Skipped where this system's file name encoding cannot + spell U+202E (ASCII under `LC_ALL=C`): the test skips there before it + asks the disk (#854).""" + skip_where_the_environment_cannot_carry(BIDI) disk = a_disk_that_refuses(monkeypatch, errno.EACCES) raised_for_every_errno(disk, BIDI_TEST, tmp_path) @@ -303,28 +307,15 @@ def test_a_failure_of_git_under_the_bidi_directory_fails_the_test( where git cannot (a disk that filled after the `mkdir`), the failure is raised, as the name is valid text and not the cause. It was a skip that said "git will not work under ...". Skipped where this system's file name - encoding cannot spell U+202E (ASCII under `LC_ALL=C`): the directory is - really made here, and `mkdir` raises `UnicodeEncodeError` first.""" - non_utf8_git_reads.skip_where_the_environment_cannot_carry(BIDI) + encoding cannot spell U+202E (ASCII under `LC_ALL=C`): the test skips + there before it makes the directory (#854).""" + skip_where_the_environment_cannot_carry(BIDI) monkeypatch.setattr(template_values, "make_source_repo", a_source_git_cannot_make) error = the_error_of(AssertionError, BIDI_TEST, tmp_path) assert str(error) == GIT_FAILED -@pytest.mark.usefixtures("a_disk_that_takes_the_probe") -def test_a_failure_of_git_under_the_byte_directory_still_skips_the_test( - tmp_path, monkeypatch): - """The control, and the known limit: #757 kept the second skip for a name - that is not UTF-8, which git may not work under, so a disk that filled - after the `mkdir` still reads as SKIPPED there.""" - monkeypatch.setattr(template_values, "make_source_repo", - a_source_git_cannot_make) - reason = skip_message(BYTE_TEST, tmp_path) - assert reason is not None, "the test went on past git's failure" - assert "git will not work under" in reason, reason - - # --- the sweep itself ------------------------------------------------------- def test_the_sweep_names_the_first_errno_that_fails_and_stops_there():