diff --git a/adopt-project.py b/adopt-project.py index 07dd1cd2..5e12d8d0 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -189,7 +189,7 @@ VISIBILITY_CHOICES, NamingPolicy, Refusal, YamlError, accepts_role, checked_value, double_quoted, git_out, load_yaml, parse_yaml, redacted, remote_is_a_path, remote_key, remote_local_path, - tree_digest, + shown_path, spelled as spelled_as_git_out, tree_digest, ) from shape_materialize import ( # noqa: E402 PLACEHOLDER_RE, SHAPE_REPOSITORY, SYMLINK_HOPS, @@ -1028,14 +1028,110 @@ def _path_argument(path: str) -> str | bytes: `bytes` argument back into text there first, with `os.fsdecode` (`subprocess.list2cmdline`), which raises on a byte that is not UTF-8. - ONLY FOR A PATH READ FROM GIT. A path read from the disk -- `Path`, - `os.readlink`, `Path.resolve` -- is text `os.fsdecode` made, and the - `os.fsencode` that `subprocess` applies to it gives back the same bytes - in any locale, so `_stage_the_split` hands git such paths as they are. + ONLY FOR A PATH READ FROM GIT, or text spelled as one (`_run_exact`). A + path read from the disk -- `Path`, `os.readlink`, `Path.resolve` -- is + text `os.fsdecode` made, and the `os.fsencode` that `subprocess` applies + to it gives back the same bytes in any locale, so `_stage_the_split` + hands git such paths as they are. """ return path if os.name == "nt" else path.encode("utf-8", "surrogateescape") +def _git_spelling(path: str) -> str: + """`path`, text `os.fsdecode` made of a path on the disk, spelled as + `_git_path` spells a path git printed: the same bytes, so that + `_path_argument` hands git the bytes the disk holds (#653). + + Where a path from the disk meets a url git printed -- the base + `_resolved` resolves a relative submodule url against, and the working + tree git's relative answer is joined to -- both must reach git spelled + one way. Under a UTF-8 locale the two spellings are one, and under + `LC_ALL=C PYTHONUTF8=0` `os.fsdecode` keeps each byte outside ASCII as + the surrogate `_path_argument` writes back as that byte; but under a + locale of one byte per character, Latin-1 say, it reads the two bytes + of `\\u00e9` as two characters, which `_path_argument` would write as + four. Text on Windows, which `_path_argument` leaves as it is. + """ + return path if os.name == "nt" else _git_path(os.fsencode(path)) + + +def _the_locale_hands_it_alike(argument: str) -> bool: + """Whether `subprocess`, encoding `argument` with the locale's codec, + hands the program the bytes `_path_argument` makes of it: under every + UTF-8 locale, and for text that is all ASCII under any.""" + try: + return os.fsencode(argument) == _path_argument(argument) + except UnicodeEncodeError: + return False + + +def _run_exact(command: list[str], cwd: Path | None = None) -> str: + """`run(command, cwd)`, with each argument handed to `git` or `gh` as + the UTF-8 bytes it spells, under any locale (#653). + + `run` hands `subprocess` text, which it encodes with the locale's codec + when Python is not in UTF-8 mode: under `LC_ALL=C PYTHONUTF8=0` that is + ASCII, so the em dash in `gh repo create --description` raised + `UnicodeEncodeError` before any leg existed, for every source, and so + did a url `_resolved` handed `git config` with a character outside + ASCII. Every argument here is text git and gh read as UTF-8: a path or + url git printed, a path `_git_spelling` spelled, or text this tool + wrote. NEVER a path `os.fsdecode` made, which `run` hands git exactly. + + `run` ITSELF WHEREVER THE LOCALE HANDS EVERY ARGUMENT ALIKE, which is + every run under a UTF-8 locale: nothing there changes, a stand-in for + `run` included. Elsewhere `subprocess` is handed `_path_argument` of + each argument, and the rest is `run`'s: only `git` and `gh` + (`check_program`, asked of the command as the person reads it), what + the program printed back as text, stripped, and a refusal raised as + `CommandFailed` naming that command. + """ + if os.name == "nt" or all(map(_the_locale_hands_it_alike, command)): + # As each caller called `run` before: no `cwd` where it gave none. + return run(command) if cwd is None else run(command, cwd=cwd) + check_program(command) + proc = subprocess.run([_path_argument(argument) for argument in command], + cwd=str(cwd) if cwd else None, capture_output=True, + check=False) + if proc.returncode != 0: + raise CommandFailed(command, cwd, proc.returncode, + (proc.stderr + proc.stdout).decode("utf-8", + "replace")) + return proc.stdout.decode("utf-8", "replace").strip() + + +def _git_out_exact(args: list[str], cwd: Path) -> str: + """`git_out(args, cwd)`, with each argument handed to git as the UTF-8 + bytes it spells, under any locale (#653): what `_run_exact` is to `run`. + + `git_out` is `scripts/repo_shape.py`'s, a shape copy every project + pins, and it hands git text: `Source.open`'s `git rev-parse` of a + default branch named `caf\\u00e9` raised `UnicodeEncodeError` under + `LC_ALL=C PYTHONUTF8=0`, and `plan`, `check` and `execute` died before + anything was said of the branch. So `git_out` itself wherever the locale + hands every argument alike, and elsewhere git is run here, and a git + that refuses is refused as `git_out` refuses it: `git-failed`, naming + the command as the person reads it. + + THE REFUSAL IS `git_out`'S, WORD FOR WORD, AND MUST STAY IN STEP WITH + IT: the folder named with `shown_path` and what git printed `spelled` + as one line (#610). Raw, a folder named with a line break began a line + the tool never printed, under the C locale alone. A change to + `git_out`'s words is a change here too, and + `tests/test_adopt_c_locale_bytes.py` compares the two. + """ + if os.name == "nt" or all(map(_the_locale_hands_it_alike, args)): + return git_out(args, cwd=cwd) + proc = subprocess.run(["git", *map(_path_argument, args)], cwd=str(cwd), + capture_output=True, check=False) + if proc.returncode != 0: + raise Refusal("git-failed", "`git {}` in {} exited {}: {}".format( + " ".join(args), shown_path(cwd), proc.returncode, + spelled_as_git_out( + proc.stderr.decode("utf-8", "replace").strip()))) + return proc.stdout.decode("utf-8").strip() + + def _one_line(text: str) -> str: """What git said, on one line: `check` prints one FINDING per line.""" return " ".join(text.split()) @@ -1520,7 +1616,8 @@ def open(cls, spec: str, work_root: Path, "name, or clone it yourself and pass the path.") from exc cloned = True branch = _default_branch(path) - commit = git_out(["rev-parse", branch], cwd=path).lower() + # The name as the bytes git keeps it, under any locale (#653). + commit = _git_out_exact(["rev-parse", branch], path).lower() return cls(path, repository, branch, commit, work_root, cloned) def tree(self) -> list[tuple[str, str, str, int]]: @@ -2157,9 +2254,13 @@ def cross_leg_references(source: "Source", named, "`git grep` reads each line of a pattern as a pattern " "of its own, so it cannot search for this name")) continue + # The name and the code paths as the bytes the tree holds (#653): + # as text, `Übersicht/` raised `UnicodeEncodeError` under + # `LC_ALL=C PYTHONUTF8=0`, and `plan` wrote no plan. proc = subprocess.run( ["git", "--literal-pathspecs", "grep", "--no-color", "-z", "-l", - "-I", "-F", "-e", spec_dir, source.commit, "--", *code_paths], + "-I", "-F", "-e", _path_argument(spec_dir), source.commit, "--", + *map(_path_argument, code_paths)], cwd=str(source.path), capture_output=True, check=False) if proc.returncode not in (0, 1): why = _one_line(proc.stderr.decode("utf-8", "replace"))[:300] @@ -5207,7 +5308,8 @@ def same_as(self, other: "_Resolution") -> bool: symlinks -- and the `..` of a relative answer joined to the source's working tree (`_default_remote`) -- may make two spellings one directory: the order `same_repository_here` asks them in, so the - disk can add an answer and never take one away. + disk can add an answer and never take one away. Each is asked as + the bytes it spells (`_path_argument`, #653). """ if self.url is None or other.url is None: return False @@ -5215,7 +5317,8 @@ def same_as(self, other: "_Resolution") -> bool: return True paths = [remote_local_path(url) for url in (self.url, other.url)] return None not in paths \ - and os.path.realpath(paths[0]) == os.path.realpath(paths[1]) + and os.path.realpath(_path_argument(paths[0])) \ + == os.path.realpath(_path_argument(paths[1])) def __str__(self) -> str: # `redacted`: a source's remote may carry a token, and so does every @@ -5267,24 +5370,32 @@ def _default_remote(source: Source) -> tuple[str, str, Path | None]: `working_tree` is the source's own for such a remote -- a path, and not an absolute one (`remote_local_path` names absolute paths only), which is git's own test for the answers it leaves relative -- and None for - every other base.""" + every other base. + + The source's directory, the base or the working tree, is spelled as git + spells the url beside it (`_git_spelling`), so `_resolved` and + `same_as` hand both the bytes they name in any locale (#653).""" name = _config_value(source.path, f"branch.{source.branch}.remote") \ or "origin" url = _config_value(source.path, f"remote.{name}.url") if url is None: - return (source.path.as_posix(), + return (_git_spelling(source.path.as_posix()), f"its own directory, as it has no remote {_spelled(name)}", None) if remote_is_a_path(url) and remote_local_path(url) is None: return (url, f"its remote {_spelled(name)}, relative to its working " - "tree", source.path) + "tree", Path(_git_spelling(str(source.path)))) return url, f"its remote {_spelled(name)}", None def _config_value(repository: Path, key: str) -> str | None: """`key` in `repository`'s git config, as git reads it -- the last - value, when there are several; None when unset.""" - got = subprocess.run(["git", GIT_CONFIG, "-z", "--get", key], + value, when there are several; None when unset. The key as the bytes + it spells (`_path_argument`): a remote's name in it is one git printed, + and as text `remote.\\u00dcrsprung.url` raised `UnicodeEncodeError` + under `LC_ALL=C PYTHONUTF8=0` (#653).""" + got = subprocess.run(["git", GIT_CONFIG, "-z", "--get", + _path_argument(key)], cwd=repository, capture_output=True, check=False) return None if got.returncode else _git_path(got.stdout.rstrip(b"\x00")) @@ -5295,15 +5406,19 @@ def _resolved(base: str, urls: list[str], whose `origin` is `base`, in a scratch repository made for the purpose and removed after it; and, given a `working_tree`, git's relative answer joined to it, which is what the clone opens (`_default_remote`). A url - git cannot resolve is what git said: that is where the clone fails.""" + git cannot resolve is what git said: that is where the clone fails. + + `base` and each url reach git as the bytes they spell (`_run_exact`): + as text, `../\\u00dcnterlagen` raised `UnicodeEncodeError` under + `LC_ALL=C PYTHONUTF8=0`, and `check` and `execute` died (#653).""" scratch = Path(tempfile.mkdtemp(prefix="openreposhape-url-")) try: run(["git", "init", "-q", str(scratch)]) - run(["git", GIT_CONFIG, "remote.origin.url", base], cwd=scratch) + _run_exact(["git", GIT_CONFIG, "remote.origin.url", base], scratch) for index, url in enumerate(urls): for key, value in (("path", f"u{index}"), ("url", url)): - run(["git", GIT_CONFIG, "-f", GITMODULES, - f"submodule.u{index}.{key}", value], cwd=scratch) + _run_exact(["git", GIT_CONFIG, "-f", GITMODULES, + f"submodule.u{index}.{key}", value], scratch) # A gitlink names a commit nobody fetches here; any object id of the # repository's own format will do, and this one is at hand. oid = run(["git", "hash-object", GITMODULES], cwd=scratch) @@ -5361,8 +5476,9 @@ def _relative_url_findings(plan: Plan, source: Source, tree: list, leg, kept = _kept_relative_urls(plan.entries, tree, registered) if not kept: return [], [], {} - remote = _remote_urls(checked_value("org", plan.get("org")), plan.names(), - local_remote_dir, source)[leg] + remote = _git_spelling(_remote_urls(checked_value("org", plan.get("org")), + plan.names(), local_remote_dir, + source)[leg]) base, which, working_tree = _default_remote(source) urls = [url for _, _, url in kept] found, alike, from_the_leg = [], [], {} @@ -5480,14 +5596,15 @@ def _answers(urls: list[str]) -> list[tuple[str, str | None]]: its questions, so that no repository's own config around this process rewrites a url as the assembly's clone would not; and under ON_THIS_DISK_ONLY, so that nothing a config could do sends git to a - network.""" + network. Each url as the bytes it spells (`_path_argument`, #653).""" scratch = Path(tempfile.mkdtemp(prefix="openreposhape-ask-")) env = {**os.environ, **ON_THIS_DISK_ONLY} try: run(["git", "init", "-q", str(scratch)]) out = [] for url in urls: - got = subprocess.run(["git", "ls-remote", "--get-url", "--", url], + got = subprocess.run(["git", "ls-remote", "--get-url", "--", + _path_argument(url)], cwd=scratch, env=env, capture_output=True, check=False) expanded = ("" if got.returncode else @@ -5495,7 +5612,8 @@ def _answers(urls: list[str]) -> list[tuple[str, str | None]]: if not _on_this_disk(expanded): out.append((expanded, None)) continue - listed = subprocess.run(["git", "ls-remote", "--", url], + listed = subprocess.run(["git", "ls-remote", "--", + _path_argument(url)], cwd=scratch, env=env, capture_output=True, check=False) out.append((expanded, None if not listed.returncode @@ -9266,12 +9384,16 @@ def _root_visibility(repository: str) -> str | None: def _create_leg_remote(plan: Plan, names: dict, repositories: dict, urls: dict, tracking: str, local: bool, role: str) -> None: - """One leg's repository: `gh repo create`, or a bare one on disk.""" + """One leg's repository: `gh repo create`, or a bare one on disk. + + The description's em dash reaches gh as UTF-8 under any locale + (`_run_exact`): as text it raised `UnicodeEncodeError` under `LC_ALL=C + PYTHONUTF8=0`, before any leg existed, for every source (#653).""" if not local: - run(["gh", "repo", "create", repositories[role], - f"--{plan.get('visibility', 'private')}", "--description", - f"{names['assembly']} — {role} leg, extracted from " - f"{repositories['assembly']} with history"]) + _run_exact(["gh", "repo", "create", repositories[role], + f"--{plan.get('visibility', 'private')}", "--description", + f"{names['assembly']} — {role} leg, extracted from " + f"{repositories['assembly']} with history"]) print(f" gh {repositories[role]}") return bare = Path(urls[role]) @@ -9956,8 +10078,10 @@ def _open_the_pull_request(repositories: dict, names: dict, tracking: str, out of `cmd_execute` for #138. """ try: - url = run(_pull_request_command(repositories, names, tracking, branch, - message)) + # The body is the split's message, which names the paths it moved, + # as UTF-8 under any locale (`_run_exact`, #653). + url = _run_exact(_pull_request_command(repositories, names, tracking, + branch, message)) print(f" pull request {url}") except LATE_FAILURES as exc: _flush_stdout() # the pushed split is announced on stdout (#188) diff --git a/tests/test_adopt_c_locale_bytes.py b/tests/test_adopt_c_locale_bytes.py new file mode 100644 index 00000000..74c5838c --- /dev/null +++ b/tests/test_adopt_c_locale_bytes.py @@ -0,0 +1,502 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Under `LC_ALL=C PYTHONUTF8=0`, `plan`, `check` and `execute` hand git and +gh the bytes a name or a url spells, as #527 made `execute` hand `git rm` +the bytes the tree holds (#653). + +On POSIX `subprocess` encodes a `str` argument with the filesystem encoding, +which is the locale's codec when Python is not in UTF-8 mode, and under the +C locale that codec is ASCII. On main at a5245427 three places still handed +git or gh text, and each ended in a `UnicodeEncodeError` traceback, exit 1, +before any leg existed: + +- `gh repo create --description`, whose em dash is the tool's own, so every + `execute` against GitHub stopped there, an all-ASCII source included; and + the `gh pr create --body` after it, the split's message, names the paths + the split moved (L1-33); +- `plan`'s `git grep` for a spec directory or a code entry outside ASCII, + so `plan` wrote no plan (L1-34); +- `_resolved`'s `git config` of a relative submodule url outside ASCII, or + of a source remote outside ASCII, so `check` and `execute` died (L1-35). + +Ruled 2026-10-12, for #527: pass git the exact bytes the tree holds, under +any locale; and 2026-10-16 (#457's triage, D4): this locale is in scope. + +THE GITHUB RUNS ARE `tests/test_adopt_github_exits.py`'s, with nothing on a +network: its `gh` stand-in makes each repository a bare one, and a throwaway +global git config sends `https://github.com/` there. One line is added to +the stand-in: every call writes its arguments, NUL-separated, to a file of +its own, so the test reads the bytes gh was handed, not a spelling of them. + +What `check` and `execute` SAY is compared with what they say in UTF-8 mode +(`LC_ALL=C PYTHONUTF8=1`), once the C locale's escapes (`\\xdc` for `Ü`) +are undone: the same report, about the same repositories. A C locale is a +POSIX idea, so every test skips on Windows and where the filesystem +encoding stays UTF-8 under it (`c_locale_or_skip`, #527's own). +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +import test_adopt_github_exits as on_github +from conftest import (ADOPT, WINDOWS_SKIP, git, make_source_repo, run_script, + write_plan) +from test_adopt_git_rm_bytes import (C_LOCALE, NORTHWIND, SPLIT_BRANCH, + UTF8, c_locale_or_skip, + needs_filter_repo, tree_bytes) +from test_adopt_relative_submodule_url import (HTTPS_ORIGIN, RELATIVE, + RELATIVE_ORIGIN, REMOTES, + THING_REMOTE, VERIFIED, found, + its_org, planned, source_with) + +#: The same interpreter in UTF-8 mode under the same locale: what the C +#: locale's runs are compared with. +UTF8_MODE = {"LC_ALL": "C", "PYTHONUTF8": "1"} +ORG = "testorg" +#: A directory and a url outside ASCII: the source's or a remote's +#: directory, and a submodule url. +UMLAUT = "Ünter" +URL = "../Ünterlagen" +#: Where git resolves `../dependency` beside the source, or URL beside the +#: repositories `execute --local-remote-dir` makes. +SIBLING = "../dependency" +DEPENDENCY_DIR = "Ünterlagen" +#: A remote's name outside ASCII, which git takes as it takes `origin`. +REMOTE_NAME = "Ürsprung" +#: The line the stand-in `gh` gets, after its first one, to keep each call's +#: arguments as bytes: `printf` writes each argument as it was given. +RECORD_THE_ARGUMENTS = "printf '%s\\0' \"$@\" > \"$FAKE_GH_ROOT/argv.$$\"\n" +#: `gh repo create`'s description of a leg: the tool's own text, em dash +#: and all, which reaches gh as UTF-8. +DESCRIPTION = ("{project} \N{EM DASH} {role} leg, extracted from " + "{org}/{project} with history") + + +def unescaped(line: str) -> str: + """`line` as the C locale printed it, each character its stream could + not encode written back from the `\\xdc`, `\\u00dc` or `\\U...` escape + `backslashreplace` gave it.""" + return re.sub(r"\\x([0-9a-f]{2})|\\u([0-9a-f]{4})|\\U([0-9a-f]{8})", + lambda match: chr(int("".join(filter(None, match.groups())), + 16)), line) + + +def said(result: subprocess.CompletedProcess) -> list[str]: + """Every REFUSED, FINDING and NOTE line a run printed, on either + stream; a refusal of `execute` indents each FINDING it repeats.""" + lines = (result.stdout + result.stderr).splitlines() + return [line.strip() for line in lines + if line.lstrip().startswith(("REFUSED ", "FINDING ", "NOTE "))] + + +def assert_said_alike(c_locale, utf8_mode) -> None: + """The C locale's run exits as the UTF-8 run does and prints the same + findings and notes, once its escapes are undone, of which there is at + least one; and it printed no traceback, which `said` would not show.""" + assert "Traceback" not in c_locale.stderr, c_locale.stderr + assert c_locale.returncode == utf8_mode.returncode, ( + c_locale.stderr + c_locale.stdout) + assert said(utf8_mode), utf8_mode.stderr + utf8_mode.stdout + assert [unescaped(line) for line in said(c_locale)] == said(utf8_mode) + + +def in_the_locale(monkeypatch, locale: dict) -> None: + """Every child this test starts runs in `locale`, whatever the locale + of this process: the environment `c_locale_or_skip` gives, with + `locale` over it.""" + env = c_locale_or_skip() + for key in [key for key in os.environ if key not in env]: + monkeypatch.delenv(key) + for key, value in locale.items(): + monkeypatch.setenv(key, value) + + +# --- `execute` against GitHub (L1-33) --------------------------------------- + + +def called(github: Path) -> list[list[bytes]]: + """Each `gh` call's arguments, as the bytes gh was handed.""" + return [path.read_bytes().split(b"\0")[:-1] + for path in sorted(github.glob("argv.*"))] + + +def on_github_in_the_c_locale(monkeypatch, tmp_path: Path, tree: dict): + """`test_adopt_github_exits.github_run` of `tree`, `plan` and `execute` + both under the C locale, with the stand-in `gh` keeping each call's + arguments; `(result, github)` as it returns them.""" + in_the_locale(monkeypatch, C_LOCALE) + monkeypatch.setattr(on_github, "GH_SHIM", on_github.GH_SHIM.replace( + "\n", "\n" + RECORD_THE_ARGUMENTS, 1)) + return on_github.github_run(tmp_path, "", tree=tree) + + +@needs_filter_repo +@WINDOWS_SKIP +def test_execute_on_github_hands_gh_repo_create_its_description_as_utf_8( + monkeypatch, tmp_path): + """L1-33. On a5245427 `execute` exited 1 with `UnicodeEncodeError` at + the first `gh repo create`, for this all-ASCII source, before any + repository existed. Now both legs are created and the adoption is + verified, and gh was handed each description as its UTF-8 bytes, + unchanged.""" + result, github = on_github_in_the_c_locale(monkeypatch, tmp_path, + on_github.SOURCE_TREE) + assert result.returncode == 0, result.stderr + result.stdout + assert VERIFIED in result.stdout + created = sorted(argv for argv in called(github) + if argv[:2] == [b"repo", b"create"]) + assert created == [ + [b"repo", b"create", f"{ORG}/{NORTHWIND}-{role}".encode(), + b"--private", b"--description", + DESCRIPTION.format(project=NORTHWIND, role=role, org=ORG).encode()] + for role in ("code", "spec")] + + +@needs_filter_repo +@WINDOWS_SKIP +def test_execute_on_github_hands_gh_pr_create_the_split_message_as_utf_8( + monkeypatch, tmp_path): + """L1-33's second half. `Überblick.md` moves to the spec leg, so the + split's message names it, and that message is the pull request's body. + Handed to gh as text, it raised `UnicodeEncodeError` after both legs and + the split were pushed, and `execute` exited 2 with the pull request to + open by hand. Now the pull request is opened, and its body is the split + commit's message byte for byte, the name as the tree holds it.""" + moved = "Überblick.md" + result, github = on_github_in_the_c_locale( + monkeypatch, tmp_path, {**on_github.SOURCE_TREE, moved: "# Ü\n"}) + assert result.returncode == 0, result.stderr + result.stdout + assert VERIFIED in result.stdout + [opened] = [argv for argv in called(github) + if argv[:2] == [b"pr", b"create"]] + body = opened[opened.index(b"--body") + 1] + commit = subprocess.run(["git", "cat-file", "commit", SPLIT_BRANCH], + cwd=str(github / ORG / f"{NORTHWIND}.git"), + capture_output=True, check=True).stdout + assert body == commit.split(b"\n\n", 1)[1] + assert b"\n " + tree_bytes(moved) + b"\n" in body + + +# --- `plan`'s `git grep` (L1-34) -------------------------------------------- + + +#: A spec directory outside ASCII, and the two code paths outside ASCII that +#: name it: a file in a code directory, and a code file. +SPEC_DIR = "Ärger" +IN_A_CODE_DIR = "Werkzeuge-Ü/run.py" +A_CODE_FILE = "Übung.py" +#: Their source, where `src/app.py` names nothing. +NAMED_BY_CODE = { + "README.md": "# Northwind\n", + "src/app.py": "VALUE = 1\n", + f"{SPEC_DIR}/a.md": "a\n", + f"{SPEC_DIR}/b.md": "b\n", + IN_A_CODE_DIR: f'open("{SPEC_DIR}/a.md")\n', + "Werkzeuge-Ü/util.py": "X = 1\n", + A_CODE_FILE: f'PATH = "{SPEC_DIR}/b.md"\n', +} + + +def test_plan_greps_for_a_spec_directory_outside_ascii(tmp_path): + """L1-34. On a5245427 `plan` died with `UnicodeEncodeError` at the + `git grep` for `Ärger/` in the code paths, and wrote no plan. Now the + plan it writes is the one UTF-8 mode writes, byte for byte, and its + follow-ups name each code path that names the spec directory: git found + both.""" + env = c_locale_or_skip() + source = make_source_repo(tmp_path / NORTHWIND, NAMED_BY_CODE, edits=()) + + def written(locale: dict) -> bytes: + plan = tmp_path / f"plan-{locale['PYTHONUTF8']}" / "adoption-plan.yaml" + plan.parent.mkdir() + result = run_script(ADOPT, "plan", "--source", str(source), + "--project", NORTHWIND, "--org", ORG, + "--elected-by", "Test Human", "--elected-on", + "2026-09-02", "--out", str(plan), + env={**env, **locale}) + assert result.returncode == 0, result.stderr + result.stdout + return plan.read_bytes() + + text = written(C_LOCALE) + assert text == written(UTF8_MODE) + across = [line for line in text.decode().splitlines() + if f"names `{SPEC_DIR}/`" in line] + assert sorted(path for line in across + for path in (IN_A_CODE_DIR, A_CODE_FILE) + if path in line) == sorted((IN_A_CODE_DIR, A_CODE_FILE)) + + +# --- `check` and `execute`, a relative submodule url (L1-35) ---------------- + + +def checked(plan: Path, locale: dict) -> subprocess.CompletedProcess: + return run_script(ADOPT, "check", "--plan", str(plan), + env={**c_locale_or_skip(), **locale}) + + +def url_outside_ascii(base: Path) -> tuple[Path, Path, str]: + """The issue's: `../Ünterlagen`, from a source on GitHub.""" + return source_with(base, URL, HTTPS_ORIGIN), base, ORG + + +def origin_outside_ascii(base: Path) -> tuple[Path, Path, str]: + """A source whose remote is a directory outside ASCII: git's url, which + `_resolved` hands `git config` as the base.""" + origin = str(base / UMLAUT / ORG / THING_REMOTE) + return source_with(base, SIBLING, origin), base, its_org(origin) + + +def remote_name_outside_ascii(base: Path) -> tuple[Path, Path, str]: + """A source whose branch tracks a remote NAMED outside ASCII, on this + disk: `_default_remote` asks git for that remote's url by its name.""" + origin = str(base / ORG / THING_REMOTE) + source = source_with(base, SIBLING, origin) + git("remote", "rename", "origin", REMOTE_NAME, cwd=source) + git("config", "branch.main.remote", REMOTE_NAME, cwd=source) + return source, base, its_org(origin) + + +def own_directory_outside_ascii(base: Path) -> tuple[Path, Path, str]: + """A source with no remote, in a directory outside ASCII: git resolves + against that directory, as the disk spells it.""" + inside = base / UMLAUT + return source_with(inside, SIBLING), inside, ORG + + +def working_tree_outside_ascii(base: Path) -> tuple[Path, Path, str]: + """A source whose remote is relative to its working tree, which is in a + directory outside ASCII: git's answer is joined to that working tree.""" + inside = base / UMLAUT + return (source_with(inside, SIBLING, RELATIVE_ORIGIN), inside, + its_org(RELATIVE_ORIGIN)) + + +@pytest.mark.parametrize("layout", [ + url_outside_ascii, origin_outside_ascii, remote_name_outside_ascii, + own_directory_outside_ascii, working_tree_outside_ascii, +], ids=["url", "origin", "remote-name", "own-directory", "working-tree"]) +def test_check_resolves_a_relative_url_as_utf_8_mode_does(monkeypatch, + tmp_path, layout): + """L1-35. On a5245427 `check` died with `UnicodeEncodeError` where + `_resolved` handed `git config` the url (`url`) or the source's remote + (`origin`) as text, and where `_default_remote` asked `git config` for + the url of a remote by its name (`remote-name`). Now `plan` and `check` + under the C locale end as they do in UTF-8 mode, and say the same of + the same repositories: `plan ok` and the NOTEs for `url`, the finding + for the others. A directory read from the disk (`own-directory`, + `working-tree`) is named as UTF-8 mode names it, `Ü` where it said + `\\udcc3\\udc9c` (`_git_spelling`). Each locale writes its own plan, + because one locale cannot read the source directory another wrote: a + plan written in UTF-8 mode records `Ü`, which a path under the C + locale cannot hold.""" + c_locale_or_skip() + source, base, org = layout(tmp_path) + runs = [] + for locale in (C_LOCALE, UTF8_MODE): + with monkeypatch.context() as here: + in_the_locale(here, locale) + plan = planned(source, base, org, + name=f"plan-{locale['PYTHONUTF8']}") + runs.append(checked(plan, locale)) + assert_said_alike(*runs) + + +@needs_filter_repo +def test_execute_resolves_and_asks_a_relative_url_outside_ascii(tmp_path): + """L1-35, `execute --local-remote-dir`, in #190's layout: the source's + remote is `../remotes/Thing.git`, and `../Ünterlagen` names + `remotes/Ünterlagen` from the source and from the code leg's remote + beside it. On a5245427 `execute` died in `_resolved`; past it, in + `same_as`, which asks the disk whether two spellings are one directory, + and in #418's `git ls-remote` of the url. Now `check` finds what UTF-8 + mode finds (GitHub's leg resolves it elsewhere), `execute` with the + legs elsewhere, in a directory outside ASCII, refuses with UTF-8 mode's + words, and with the legs beside the dependency it adopts and verifies.""" + c_locale_or_skip() + remotes = tmp_path / REMOTES + source = source_with(tmp_path, URL, RELATIVE_ORIGIN, + remotes / DEPENDENCY_DIR) + plan = planned(source, tmp_path, its_org(RELATIVE_ORIGIN)) + result = checked(plan, C_LOCALE) + assert [code for code, _ in found(result.stderr)] == [RELATIVE] + + def executed(legs: Path, locale: dict, work: str): + return run_script(ADOPT, "execute", "--plan", str(plan), "--yes", + "--local-remote-dir", str(legs), "--work-dir", + str(tmp_path / work), + env={**c_locale_or_skip(), **locale}) + + elsewhere = tmp_path / UMLAUT + refused = executed(elsewhere, C_LOCALE, "work-c") + assert refused.returncode != 0, refused.stdout + assert_said_alike(refused, executed(elsewhere, UTF8_MODE, "work-utf-8")) + adopted = executed(remotes, C_LOCALE, "work") + assert adopted.returncode == 0, adopted.stderr + adopted.stdout + assert VERIFIED in adopted.stdout + + +# --- `Source.open`'s `git rev-parse` of the default branch ------------------ + + +#: A default branch whose name is UTF-8 and not ASCII (w425's probe). +BRANCH = "café" + + +def on_its_own_branch(base: Path) -> Path: + """A source on BRANCH, its only branch.""" + return make_source_repo(base / NORTHWIND, on_github.SOURCE_TREE, + branch=BRANCH, edits=()) + + +def its_branch_gone(base: Path) -> Path: + """A clone of a repository on BRANCH, whose `origin/HEAD` still names + BRANCH after the clone's own BRANCH was deleted: `git rev-parse` of the + name `_default_branch` reads finds nothing, and git refuses it.""" + upstream = make_source_repo(base / ORG / NORTHWIND, on_github.SOURCE_TREE, + branch=BRANCH, edits=()) + source = base / NORTHWIND + git("clone", "-q", str(upstream), str(source), cwd=base) + git("checkout", "-q", "-b", "other", cwd=source) + git("branch", "-q", "-D", BRANCH, cwd=source) + return source + + +@pytest.mark.parametrize("make_source, tracking", [ + (on_its_own_branch, ()), + (on_its_own_branch, ("--tracking-branch", "main")), + (its_branch_gone, ("--tracking-branch", "main")), +], ids=["tracked", "main", "gone"]) +def test_a_default_branch_outside_ascii_reads_as_in_utf_8_mode( + monkeypatch, tmp_path, make_source, tracking): + """On 9b8e1768, as on main, `plan` died with `UnicodeEncodeError` at + `Source.open`'s `git rev-parse café`, whatever it was told. Now under + the C locale it ends as in UTF-8 mode. Taking `café` as the tracking + branch (`tracked`), it is refused `unsafe-value` before any plan is + written. Told `--tracking-branch main` (`main`), it writes the plan + UTF-8 mode writes, byte for byte, and `check` finds `main` no branch of + the source, naming `café` as UTF-8 mode names it. Where git refuses + `café` (`gone`), the refusal is `git-failed`, naming the command as + UTF-8 mode's does.""" + c_locale_or_skip() + source = make_source(tmp_path) + plans, runs = [], [] + for locale in (C_LOCALE, UTF8_MODE): + plan = tmp_path / f"plan-{locale['PYTHONUTF8']}" / "adoption-plan.yaml" + plan.parent.mkdir() + with monkeypatch.context() as here: + in_the_locale(here, locale) + written = write_plan(source, plan, extra=tracking) + plans.append(plan.read_bytes() if written.returncode == 0 else None) + runs.append(checked(plan, locale) if written.returncode == 0 + else written) + assert plans[0] == plans[1] + assert_said_alike(*runs) + + +#: The child: `_git_out_exact(["rev-parse", ], )` under the +#: C locale. The script and the repository travel as the hex of their bytes, +#: and the name as JSON, which spells a surrogate as an escape, so every +#: argument is ASCII. It prints one line of ASCII JSON: the commit, or the +#: refusal's code and detail. +REV_PARSE_IN_THE_C_LOCALE = r""" +import importlib.util, json, os, sys +from pathlib import Path +spec = importlib.util.spec_from_file_location( + "adopt_rev_parse", os.fsdecode(bytes.fromhex(sys.argv[1]))) +adopter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(adopter) +repository = Path(os.fsdecode(bytes.fromhex(sys.argv[2]))) +try: + answer = {"commit": adopter._git_out_exact( + ["rev-parse", json.loads(sys.argv[3])], repository)} +except adopter.Refusal as exc: + answer = {"refused": exc.code, "detail": exc.detail} +print(json.dumps(answer)) +""" + +#: A name that is UTF-8, and one holding the byte 0xE9 where UTF-8 has +#: none, as #425's `_default_branch` keeps it: the surrogate U+DC00 plus +#: the byte. +NAMES = {"utf-8": BRANCH, "not-utf-8": "caf\udce9"} + + +@pytest.mark.parametrize("present", [True, False], + ids=["present", "missing"]) +@pytest.mark.parametrize("name", NAMES.values(), ids=NAMES.keys()) +def test_rev_parse_hands_git_the_name_s_bytes_and_refuses_it_by_name( + tmp_path, name, present): + """`Source.open`'s `git rev-parse` (#425's review). Under the C locale + the name reaches git as its bytes, UTF-8 with `surrogateescape`, so a + branch git keeps as `caf\\xe9` is the one asked, and the commit comes + back. Where git refuses it, the refusal is `git_out`'s own, + `git-failed`, naming the command as text: no bytes reach its words, + which `git_out` joins as text and would end in a `TypeError`.""" + env = c_locale_or_skip() + repository = make_source_repo(tmp_path / NORTHWIND, on_github.SOURCE_TREE, + edits=()) + if present: + subprocess.run(["git", "branch", tree_bytes(name)], + cwd=str(repository), check=True) + child = subprocess.run( + [sys.executable, "-c", REV_PARSE_IN_THE_C_LOCALE, + os.fsencode(ADOPT).hex(), os.fsencode(repository).hex(), + json.dumps(name)], + capture_output=True, env=env, stdin=subprocess.DEVNULL, check=False) + assert child.returncode == 0, child.stderr.decode(UTF8, "replace") + answer = json.loads(child.stdout.decode("ascii")) + if present: + head = subprocess.run(["git", "rev-parse", "HEAD"], + cwd=str(repository), capture_output=True, + check=True).stdout.decode("ascii").strip() + assert answer == {"commit": head} + return + assert answer.get("refused") == "git-failed", answer + assert answer["detail"].startswith(f"`git rev-parse {name}` in "), answer + + +@pytest.fixture(scope="module") +def adopter(): + """`adopt-project.py` as a module, in this process.""" + module_spec = importlib.util.spec_from_file_location( + "adopt_c_locale_bytes", ADOPT) + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + return module + + +#: A folder name `shown_path` spells, U+2028 not being printable, and one +#: NTFS holds. +UNPRINTABLE_DIR = "line
separator" + + +def test_git_out_exact_refuses_in_git_out_s_own_words(adopter, monkeypatch, + tmp_path): + """`_git_out_exact`'s refusal is a copy of the pinned `git_out`'s, and + #634 (#610) changed `git_out`'s after the copy was made: it names the + folder with `shown_path` and spells what git printed as one line. The + copy kept the folder and git's lines raw, so under the C locale a + folder named with a line break began a line the tool never printed. + Asked the same failing command, in a folder `shown_path` spells, of a + git whose message is three lines, the two now refuse alike, on one + line. `_git_out_exact` is made to take its own path, as it does under + the C locale for a name outside ASCII.""" + repository = make_source_repo(tmp_path / UNPRINTABLE_DIR / NORTHWIND, + on_github.SOURCE_TREE, edits=()) + args = ["rev-parse", "no-such-branch"] + with pytest.raises(adopter.Refusal) as by_git_out: + adopter.git_out(args, cwd=repository) + monkeypatch.setattr(adopter, "_the_locale_hands_it_alike", + lambda _argument: False) + with pytest.raises(adopter.Refusal) as by_hand: + adopter._git_out_exact(args, repository) + assert by_hand.value.code == by_git_out.value.code + assert by_hand.value.detail == by_git_out.value.detail + assert "\n" not in by_hand.value.detail