diff --git a/AGENTS.md b/AGENTS.md
index ee82ff3b..9e125f7a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -267,8 +267,8 @@ its last line names that directory, the human's to delete once those have run.
never changes the verdict: it says whether THIS workstation can run the fixes
the other rows named. Relay a missing prerequisite; `openRepoShape
--preflight` offers only the `gh` set-up, and the human types that yes.
-7. `--json` is the same report as one object, alone on stdout, each row
- carrying its `id`, for when you have to act on a row rather than quote it.
+7. `--json` is the same report as one object, alone on stdout, each row with
+ its `id`, to act on a row. A `next` with `$'…'` needs bash or zsh, not `sh`.
## Updating a project's shape
diff --git a/README.md b/README.md
index b0d3a957..5f4496ab 100644
--- a/README.md
+++ b/README.md
@@ -159,6 +159,8 @@ typed `yes`, which its preflight cannot take there. Printed commands are quoted
for a POSIX shell, so paste them into Git Bash. Only `shape-doctor.py`'s lines,
`setup-project.py`'s family lines and `family.py`'s `NEXT`, `switch -c`, `add`
and `git rm -r --cached` lines are quoted for PowerShell, on Windows.
+Off Windows, a path `shape-doctor.py` cannot print as itself is written `$'…'`:
+bash, zsh and ksh read that; dash (`/bin/sh` on Debian and Ubuntu) does not.
diff --git a/docs/handbook.html b/docs/handbook.html
index 8df78127..827c88bd 100644
--- a/docs/handbook.html
+++ b/docs/handbook.html
@@ -472,7 +472,7 @@
-
+
@@ -612,7 +612,7 @@ The one thing to read before electing it
-
+
Setup
Start here
@@ -738,7 +738,9 @@ The tool offers the rest
shape-doctor.py’s lines, setup-project.py’s family
lines and family.py’s NEXT, switch -c,
add and git rm -r --cached lines are quoted for PowerShell, on
- Windows.
+ Windows. Off Windows, a path shape-doctor.py cannot print as itself is
+ written $'…': bash, zsh and ksh read that; dash (/bin/sh
+ on Debian and Ubuntu) does not.
@@ -1087,7 +1089,7 @@ What you have afterwards — the same on every platform
-
+
Setup
If it fails part-way
@@ -1124,7 +1126,7 @@ If it fails part-way
-
+
@@ -1187,7 +1189,7 @@ Reading private legs in CI
-
+
@@ -1443,7 +1445,7 @@ The InkRouter example
-
+
@@ -1742,7 +1744,7 @@ Is this repository compliant? openRepoShape --doctor
-
+
diff --git a/scripts/shape_advisory.py b/scripts/shape_advisory.py
index 64a489e9..0d19f450 100644
--- a/scripts/shape_advisory.py
+++ b/scripts/shape_advisory.py
@@ -169,13 +169,15 @@
def printable(text: str) -> str:
"""`text` as ASCII, so printing it can never be what fails a run.
- THE SAME FLATTENING `shape-doctor.py`'s `ascii_text` applies to every row,
- for the same reason, here for the two tools that print the advisory
- themselves: a record's `kind:` is a person's text and may carry anything,
- and a console in an ANSI code page -- or a `PYTHONIOENCODING=ascii`
- pipe -- raises `UnicodeEncodeError` on the first character it cannot
- write. The typography this repository writes is spelled out; anything
- else becomes `?` rather than an exception.
+ THE FLATTENING `shape-doctor.py`'s `ascii_text` applied to every row
+ until #785, for the same reason, here for the two tools that print the
+ advisory themselves: a record's `kind:` is a person's text and may carry
+ anything, and a console in an ANSI code page -- or a
+ `PYTHONIOENCODING=ascii` pipe -- raises `UnicodeEncodeError` on the first
+ character it cannot write. The typography this repository writes is
+ spelled out; anything else becomes `?` rather than an exception. (The
+ doctor's now keeps a character its stdout can show, because a row names
+ paths a reader pastes; a `kind:` is never pasted.)
"""
for source, target in (("—", "--"), ("–", "-"),
("‘", "'"), ("’", "'"),
diff --git a/scripts/shape_materialize.py b/scripts/shape_materialize.py
index a26d2390..1171227d 100644
--- a/scripts/shape_materialize.py
+++ b/scripts/shape_materialize.py
@@ -162,11 +162,11 @@ def escape_what_stdout_cannot_encode(stream=None) -> None:
file it is, and two names that differ only outside the code page do not
both print as `??.md`. (A backslash is not escaped, so a name that holds
the six characters `\\u65e5` prints as one holding `日` does.) ONCE, AT
- THE START OF `main()`, and not at each `print`: `shape-doctor.py`'s
- `ascii_text` and `shape_advisory.printable` flatten their own text to
- ASCII on every console, which a name cannot afford -- it would change
- what a UTF-8 terminal shows -- and a guard at each site is a guard the
- next `print` goes without.
+ THE START OF `main()`, and not at each `print`: `shape_advisory.printable`
+ flattens its own text to ASCII on every console, as `shape-doctor.py`'s
+ `ascii_text` did until #785, which a name cannot afford -- it would
+ change what a UTF-8 terminal shows -- and a guard at each site is a
+ guard the next `print` goes without.
GUARDED, because failing to set a handler is no reason to refuse the
run: a stream a caller replaced may have no `reconfigure`, a closed or
diff --git a/shape-doctor.py b/shape-doctor.py
index 1e41af46..b3527bab 100755
--- a/shape-doctor.py
+++ b/shape-doctor.py
@@ -333,9 +333,14 @@
red validator: the conflict it leaves is what turns the other rows red, and
none of their fixes can be committed until it is finished.
-STANDARD LIBRARY ONLY, like everything else shipped here. Printed text is
-ASCII: a verdict a cp1252 console cannot render is a verdict nobody reads, and
-this tool is run on every platform the standard supports.
+STANDARD LIBRARY ONLY, like everything else shipped here. The report's own
+words are ASCII: a verdict a cp1252 console cannot render is a verdict nobody
+reads, and this tool is run on every platform the standard supports. A NAME
+is not the report's own words (#785): a path printed in a command is spelled
+so that the reader's shell turns it back into that path, as itself where the
+output can show it and escaped where it cannot (`quote_arg`), and a sentence
+shows it as itself where the output can and with `?` where it cannot
+(`ascii_text`).
"""
from __future__ import annotations
@@ -420,7 +425,8 @@ def not_in_the_standard(detail: str) -> int:
#: reason, in place of a copy of `plan`'s regex (`origin_org`, #521).
#: `spelled` is how `siblings.py` prints the values that row quotes (#673).
from repo_shape import ( # noqa: E402
- COMMIT_RE, NamingPolicy, PYTHON, Refusal, accepts_role, git_out,
+ COMMIT_RE, NamingPolicy, POWERSHELL_SINGLE_QUOTES, PYTHON,
+ Refusal, accepts_role, git_out,
load_yaml, recorded_gitlink, redacted, remote_local_path,
resolved_remote, same_repository, spelled, url_repository,
)
@@ -662,20 +668,93 @@ def not_in_the_standard(detail: str) -> int:
)
-def ascii_text(text: str) -> str:
- """`text` with the typography this repository is written in flattened.
+#: The typography this repository's prose is written in, and the ASCII
+#: `ascii_text` spells each one with. `quote_arg` never prints one of them as
+#: itself in a value (#785): `Row.__init__` passes every command and every
+#: `why` through `ascii_text`, which would make the value another path.
+FLATTENED = {"—": "--", "–": "-", "‘": "'", "’": "'", "“": '"', "”": '"',
+ "…": "...", "\N{NO-BREAK SPACE}": " "}
+
+
+#: The encoding the report is written for where that is not stdout's: UTF-8
+#: for `--json`, which `main` sets (#785, review S2 on PR #792). `json.dumps`
+#: writes ASCII whatever stdout's encoding is, and every character as an
+#: escape its reader decodes, so a JSON reader gets each character a UTF-8
+#: name holds as itself -- under `LC_ALL=C` and a `PYTHONIOENCODING=cp1252`
+#: pipe as under a UTF-8 terminal. None for the table, which IS stdout.
+REPORT_ENCODING: str | None = None
+
+
+def printed_encoding(encoding: str | None = None) -> str:
+ """`encoding`, or the one the report is written for: `REPORT_ENCODING`
+ where `main` set one, else `sys.stdout`'s, which `pythonw` and a stream
+ a caller replaced may not have (#785)."""
+ return (encoding or REPORT_ENCODING
+ or getattr(sys.stdout, "encoding", None) or "utf-8")
+
+
+def prints_as_itself(char: str, encoding: str) -> bool:
+ """Whether a report written in `encoding` shows `char` as itself (#785):
+ a printable character that `ascii_text` does not flatten, and that the
+ encoding writes and reads back unchanged.
+
+ PRINTABLE, because a control, a line separator or a bidirectional mark
+ is not seen as itself on any terminal: it moves the line it is printed
+ on, and a reader cannot copy what they cannot see. And not U+FFFD, the
+ mark a lossy read leaves where a byte did not decode, which a row has
+ printed as `?` since #385: it stands for no character a path holds."""
+ if (char in FLATTENED or char == "\N{REPLACEMENT CHARACTER}"
+ or not char.isprintable()):
+ return False
+ try:
+ return char.encode(encoding).decode(encoding) == char
+ except (UnicodeError, LookupError):
+ return False
+
+
+def ascii_text(text: str, encoding: str | None = None) -> str:
+ """`text` with the typography this repository is written in flattened,
+ and each character the report cannot show as itself replaced by `?`.
Validator output is quoted into rows, and this repository's prose is full
of em dashes, curly quotes and ellipses. A Windows console in the machine's
- ANSI code page raises on the first of them, in the middle of a report whose
+ ANSI code page raised on the first of them, in the middle of a report whose
whole job is to be read -- so the few characters that actually occur are
- spelled out and anything else is replaced rather than raised on.
- """
- for source, target in (("—", "--"), ("–", "-"), ("‘", "'"),
- ("’", "'"), ("“", '"'), ("”", '"'),
- ("…", "..."), (" ", " ")):
+ spelled out in ASCII on every console.
+
+ ANY OTHER CHARACTER IS REPLACED ONLY WHERE IT CANNOT BE SHOWN (#785). It
+ was replaced on every console, so `é` was `?` under a UTF-8 locale, whose
+ terminal shows it, and so was every character of a path `quote_arg` had
+ quoted for a command: a keep line named `caf?`, which is not there. Now a
+ character is kept where `prints_as_itself` says `encoding` (the report's
+ own when None, `printed_encoding`) shows it, and is `?` elsewhere, as
+ every one was: an ASCII console still reads an ASCII report, and a UTF-8
+ one reads the name. A value `quote_arg` spelled holds no character this
+ changes, so this pass leaves it whole.
+ """
+ for source, target in FLATTENED.items():
text = text.replace(source, target)
- return text.encode("ascii", "replace").decode("ascii")
+ encoding = printed_encoding(encoding)
+ return "".join(char if char.isascii() or prints_as_itself(char, encoding)
+ else "?" for char in text)
+
+
+def path_bytes(char: str) -> bytes:
+ """The bytes one character of a path stands for on a POSIX disk (#785).
+
+ A byte of a name that is not in the encoding it was read with is carried
+ by `surrogateescape` as U+DC80..U+DCFF, how `os.fsdecode` reads a path
+ and how this file reads one git prints, and stands for that byte. Any
+ other character is in the filesystem encoding, which is how Python names
+ a path on disk, or in UTF-8 where that encoding has no such character,
+ which is how git wrote a name it printed (`LC_ALL=C` with Python's
+ UTF-8 mode off makes the filesystem encoding ASCII)."""
+ if 0xDC80 <= ord(char) <= 0xDCFF:
+ return bytes([ord(char) - 0xDC00])
+ try:
+ return char.encode(sys.getfilesystemencoding())
+ except UnicodeEncodeError:
+ return char.encode("utf-8", "surrogatepass")
#: A value `sh` needs no quoting for: letters, digits, and the punctuation a
@@ -703,7 +782,7 @@ def ascii_text(text: str) -> str:
def quote_arg(value, platform: str | None = None, *,
- always: bool = False) -> str:
+ always: bool = False, encoding: str | None = None) -> str:
# A RAW docstring, because the Windows examples below are Windows paths:
# `C:\Users\...` is a `\U` escape to the interpreter, and a docstring that
# had to spell a reader's path with doubled backslashes would be a
@@ -760,20 +839,41 @@ def quote_arg(value, platform: str | None = None, *,
become an argument to `git`, where every command is a list run with
`shell=False` and the threat is argument injection, not a shell.
- AND IT NORMALIZES BEFORE IT QUOTES, NOT AFTER. `Row.__init__` runs
- `ascii_text` again, over the WHOLE assembled `next_command`, once every
- value in it already sits quoted -- so a curly apostrophe that reached
- this function unconverted came out the far side of THAT pass as a bare,
- undoubled one, breaking the very quoting it sat inside (Copilot, PR
- #102): `'O'Neill'` is an unterminated quotation to `sh`, and an
- un-doubled apostrophe is exactly as broken to PowerShell's own verbatim
- string. Normalizing HERE first leaves that later pass nothing to touch:
- `ascii_text` maps this repository's own typography to ASCII, so calling
- it twice on an already-ASCII string is a no-op. What the reader then sees
- is the ASCII FLATTENING of their path rather than its own bytes -- which
- is this file's PURE ASCII rule, stated at the top and true of every other
- string in the report, and a command that names a path plainly and is
- refused by name beats one no shell will parse.
+ A VALUE THAT IS NOT ASCII IS SPELLED AS THE PATH IT IS (#785). It was
+ flattened to ASCII first (Copilot, PR #102), because `Row.__init__`
+ runs `ascii_text` again over the whole assembled command, and a curly
+ apostrophe that pass found came out of it bare, breaking the quoting it
+ sat in. But the flattening named another path: every character that is
+ not ASCII became `?`, so a keep line under `Projets/Équipe` named
+ `Projets/?quipe`, which is not there, and the pasted line failed. Now
+ each character of such a value is one of two things:
+
+ - ITSELF, inside the quoting above, where `prints_as_itself` says the
+ report shows it: printable, not one of the typography `ascii_text`
+ flattens, and written by `encoding` (the report's own when None,
+ `printed_encoding`) -- for `sh`, as the very bytes the path holds
+ (`path_bytes`), so that what a terminal shows is what it pastes back;
+ - ESCAPED otherwise, and then the whole value is spelled that way. For
+ `sh` that is `$'...'`, every byte of such a character a three-digit
+ octal escape (`$'caf\303\251'`, `$'caf\351'` for a name that is not
+ UTF-8), which bash, zsh, ksh and a POSIX.1-2024 `sh` turn back into
+ those bytes; dash 0.5.12 (Ubuntu's `/bin/sh`, whose login shell is
+ bash) reads the `$` as itself. For PowerShell, which has no escape
+ inside a verbatim string, it is an expression in parentheses,
+ `('C:\Users\' + [char]0x00C9 + 'quipe')`, which is one argument to a
+ native command and a value to assign alike (`powershell_expression`).
+ On Windows that is rare: Python writes a console as UTF-8, which shows
+ every printable character, and only a stdout redirected to a file or a
+ pipe is in the ANSI code page.
+
+ So `ascii_text`'s later pass finds nothing in a value to change. The
+ report's encoding is stdout's for the table, and UTF-8 for `--json`,
+ whatever stdout's is (`REPORT_ENCODING`, review S2 on PR #792): a JSON
+ reader decodes each `\u` escape `json.dumps` writes into the character
+ it stands for, so a row's `next` holds a UTF-8 name as itself, and only
+ a byte that is not UTF-8, or a character `prints_as_itself` refuses, is
+ escaped there. The table escapes what its own encoding cannot write. A
+ value made of ASCII is quoted exactly as before, in both.
AND THE ALPHABET CHECK MUST COVER THE VALUE WHOLE. `$` is content to
match just before a trailing newline, so `.match` alone waved a value
@@ -806,10 +906,11 @@ def quote_arg(value, platform: str | None = None, *,
answer. PowerShell's dialect is NOT changed: it expands no `=word`, so
`UNQUOTED_NT` keeps its bare `=`, as in any other place in a word.
"""
- # ASCII first, so the alphabet check and the quoting below it both see
- # what `Row.__init__` will (Copilot, PR #102); see the docstring above.
- text = ascii_text(str(value))
+ text = str(value)
windows = (os.name if platform is None else platform) == "nt"
+ if not text.isascii():
+ # Never bare: neither alphabet holds a character that is not ASCII.
+ return quoted_beyond_ascii(text, windows, printed_encoding(encoding))
# `.fullmatch`, not `.match`: a pattern ending in `$` still matches just
# before a trailing newline, so `.match` alone would call "Atlas\n" bare
# (Copilot, PR #102). And a leading `=` is never bare for `sh` (#532).
@@ -827,6 +928,94 @@ def quote_arg(value, platform: str | None = None, *,
return "'" + text.replace("'", "'\\''") + "'"
+def quoted_beyond_ascii(text: str, windows: bool, encoding: str) -> str:
+ """`quote_arg` for a value holding a character that is not ASCII (#785):
+ quoted as any other value where every such character `prints_as_itself`
+ -- for `sh`, as the bytes the path holds -- and escaped whole where one
+ does not. See `quote_arg`'s docstring.
+
+ ON POSIX A NAME IS READ AS UTF-8 FIRST (review S2 on PR #792). Under
+ `LC_ALL=C` with Python's UTF-8 mode off the filesystem encoding is
+ ASCII, so `os.fsdecode` carries each byte of `é` as a `surrogateescape`
+ byte of its own, and a report in UTF-8 -- `--json` -- could not show
+ it. Those bytes are read again as UTF-8, which joins each run that
+ spells a character into that character and leaves every other byte as
+ it was. A value that holds a surrogate `surrogateescape` never makes
+ (outside U+DC80..U+DCFF) cannot be written back to bytes like that, and
+ is left as it is: such a character is not printable, so it is escaped.
+
+ AND THE JOINED NAME IS KEPT ONLY WHERE IT NAMES THE SAME BYTES (review
+ M1 on PR #792, round 2). `path_bytes` writes a joined character in the
+ filesystem encoding, which is ASCII or UTF-8 under `C` and gives the
+ bytes back; but under a legacy multibyte one -- EUC-JP, Big5, EUC-KR --
+ a directory named `日` in UTF-8 (`\\xe6\\x97\\xa5`) is read in as three
+ escaped bytes, and the joined `日` is written `\\xc6\\xfc` in EUC-JP:
+ another directory. So the join is checked byte for byte against the
+ value as it was read, and the escaped bytes stay where it differs."""
+ if not windows:
+ try:
+ joined = text.encode("utf-8", "surrogateescape").decode(
+ "utf-8", "surrogateescape")
+ except UnicodeEncodeError:
+ joined = text
+ if (b"".join(map(path_bytes, joined))
+ == b"".join(map(path_bytes, text))):
+ text = joined
+
+ def shown(char: str) -> bool:
+ if char.isascii():
+ return True
+ if not prints_as_itself(char, encoding):
+ return False
+ return windows or char.encode(encoding) == path_bytes(char)
+ if all(shown(char) for char in text):
+ if windows:
+ return powershell_verbatim(text)
+ return "'" + text.replace("'", "'\\''") + "'"
+ if windows:
+ return powershell_expression(text, shown)
+ return "$'" + "".join(
+ char if char.isascii() and char.isprintable() and char not in "\\'"
+ else "".join(f"\\{byte:03o}" for byte in path_bytes(char))
+ for char in text) + "'"
+
+
+def powershell_verbatim(text: str) -> str:
+ """`text` in PowerShell's verbatim string, each of the single quotes
+ PowerShell reads as one doubled, as `repo_shape.shell_quoted` doubles
+ them: the curly ones end the string too (#486)."""
+ return "'" + "".join(char * 2 if char in POWERSHELL_SINGLE_QUOTES
+ else char for char in text) + "'"
+
+
+def powershell_expression(text: str, shown) -> str:
+ """`text` as a PowerShell expression, `('...' + [char]0x00C9 + '...')`,
+ where a character of it is not `shown` (#785).
+
+ Each such character is a `[char]` of each of its UTF-16 code units, and
+ the characters between them are verbatim strings, joined by `+`. The
+ FIRST operand is always a string -- `''` where the value starts with an
+ escaped character -- because `+` on two `[char]`s adds them as numbers,
+ and a string on the left makes every `+` after it a concatenation. A
+ printable ASCII character is always shown; a control is escaped too, as
+ it is in `sh`'s `$'...'`."""
+ parts, run = [], ""
+ for char in text:
+ if char.isprintable() and shown(char):
+ run += char
+ continue
+ if run or not parts:
+ parts.append(powershell_verbatim(run))
+ run = ""
+ units = char.encode("utf-16-le", "surrogatepass")
+ for at in range(0, len(units), 2):
+ unit = int.from_bytes(units[at:at + 2], "little")
+ parts.append(f"[char]0x{unit:04X}")
+ if run:
+ parts.append(powershell_verbatim(run))
+ return "(" + " + ".join(parts) + ")"
+
+
# ---------------------------------------------------------------------------
# A row, a check, and the registry they live in
# ---------------------------------------------------------------------------
@@ -853,9 +1042,10 @@ def __init__(self, check_id: str, label: str, status: str, reason: str,
self.label = label
self.status = status
self.reason = ascii_text(reason)
- # `quote_arg` ascii-normalizes every value it quotes BEFORE it
- # quotes it (Copilot, PR #102), so this pass has nothing left to do
- # to a `next_command` already built from it -- it only does its
+ # `quote_arg` spells every value it quotes in characters this pass
+ # keeps -- each one itself only where `ascii_text` keeps it, and the
+ # value escaped otherwise (#785) -- so this pass has nothing to do
+ # to a value in a `next_command` or a `why`. It only does its
# ordinary job, flattening this repository's own typography, on the
# literal words a caller wrote around those values.
self.next_command = ascii_text(next_command) if next_command else None
@@ -10080,6 +10270,9 @@ def main(argv: list[str] | None = None) -> int:
# a name stdout cannot encode rather than raising on it (#214).
escape_what_stdout_cannot_encode()
args = build_parser().parse_args(argv)
+ # Before any row is built: `quote_arg` and `ascii_text` read it (#785).
+ global REPORT_ENCODING
+ REPORT_ENCODING = "utf-8" if args.as_json else None
if [marker for marker in SHAPE_MARKERS
if not (SHAPE_ROOT / marker).exists()]:
return not_in_the_standard("")
diff --git a/tests/readme-cap.d/785.txt b/tests/readme-cap.d/785.txt
new file mode 100644
index 00000000..6433e6ef
--- /dev/null
+++ b/tests/readme-cap.d/785.txt
@@ -0,0 +1,11 @@
++2
+2026-10-17
+Two lines for one sentence under "Windows, natively" (#785, review S1 on #792).
+Since #785 `shape-doctor.py` writes a path it cannot print as itself, in a
+POSIX line, as `$'…'` with an octal escape for each byte, so the line names
+that path rather than one with `?` in it. bash, zsh and ksh read that form;
+dash, which is `/bin/sh` on Debian and Ubuntu, reads the `$` as itself and
+hands the command another path. The section says printed commands are quoted
+for a POSIX shell, and `sh -c` is dash on those systems, so it has to say
+which shells read the form. The two lines name the form, the three shells
+that read it and the one that does not, and nothing else.
diff --git a/tests/test_doctor_non_ascii_paths.py b/tests/test_doctor_non_ascii_paths.py
new file mode 100644
index 00000000..6ba006b6
--- /dev/null
+++ b/tests/test_doctor_non_ascii_paths.py
@@ -0,0 +1,435 @@
+# SPDX-License-Identifier: Apache-2.0
+"""`shape-doctor.py` prints a path that is not ASCII as THAT path, in every
+command it hands a reader (#785).
+
+THE DEFECT. `ascii_text` ended in `encode("ascii", "replace")`, and both
+`quote_arg` and `Row` ran it, so each character of a path that is not ASCII
+printed as `?`, on every console, in the table and in `--json` alike. A
+holder under `Projets/Équipe` was handed `--root '.../Projets/?quipe/...'`
+and the keep line `git -C '.../?quipe/InkRouter/members/IRRS' branch NAME`;
+no such directory is there, so the pasted line failed -- under a UTF-8
+locale, whose terminal shows `é`, as much as under `LC_ALL=C`. A name that
+is not UTF-8 at all (`\\xc9quipe`, a legacy code page) fared the same.
+
+WHAT IT DOES NOW. `quote_arg` spells such a value as itself where the
+report's encoding writes the bytes the path holds, and escapes it whole
+where it does not: `$'...'` with octal escapes for `sh`, a `[char]`
+expression for PowerShell. `ascii_text` replaces only what the report cannot
+show. Each line is asserted here as a reader uses it: the table's bytes and
+the `--json` value, pasted into bash -- `sh` on Ubuntu is dash, which has no
+`$'...'` -- under a UTF-8 locale, under `LC_ALL=C`, and under `LC_ALL=C`
+with Python's UTF-8 mode off, where its filesystem encoding is ASCII, for a
+directory named in UTF-8 and one named in Latin-1. The NEXT is also read
+with stdout in cp1252 and in ASCII, an encoding that is not the filesystem's
+(review S3 on PR #792), and run in dash where the report shows the name as
+itself, which is what README says dash can run.
+
+The spellings are asserted on every OS. The runs are `WINDOWS_SKIP`: a
+Windows command line is text and cannot carry the byte 0xC9 alone. No
+PowerShell is run here; its spelling is read back by `powershell_value`.
+
+THE FIXTURES ARE `test_shape_doctor`'s, imported rather than rebuilt.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+from pathlib import Path
+
+import pytest
+
+from conftest import REPO, WINDOWS_SKIP, blank_unnamed_pin_sources
+from test_doctor_gitmodules_not_utf8 import listed_locale
+from test_doctor_legs_members_next import (
+ paste_env, commit_in, detached_member, emptied_with_its_repository_kept,
+)
+from test_family_printed_quoting import powershell_words
+from test_shape_doctor import ( # noqa: F401
+ DOCTOR, FAMILY_MEMBER, FAMILY_NAME, doctor_module, holder, standard,
+)
+
+#: A byte of a name that is not UTF-8, as `surrogateescape` reads it in.
+LATIN_1_E_ACUTE = chr(0xDCC9)
+#: The apostrophe this repository's own prose is written with, which
+#: `ascii_text` flattens, so a value holding it is always escaped.
+CURLY = chr(0x2019)
+
+#: `id: (value, encoding, spelling, bytes)` for `sh`: what `quote_arg`
+#: prints for `value` to a report in `encoding`, and the bytes a shell makes
+#: of that spelling.
+POSIX_CASES = {
+ "utf-8-shown": ("/srv/Équipe", "utf-8", "'/srv/Équipe'",
+ "/srv/Équipe".encode("utf-8")),
+ "ascii-escaped": ("/srv/Équipe", "ascii", r"$'/srv/\303\211quipe'",
+ "/srv/Équipe".encode("utf-8")),
+ # cp1252 holds `É`, as the byte 0xC9: not the bytes this path holds.
+ "cp1252-escaped": ("/srv/Équipe", "cp1252", r"$'/srv/\303\211quipe'",
+ "/srv/Équipe".encode("utf-8")),
+ "latin-1-byte": (f"/srv/{LATIN_1_E_ACUTE}quipe", "utf-8",
+ r"$'/srv/\311quipe'", b"/srv/\xc9quipe"),
+ "curly-apostrophe": (f"/srv/Dana{CURLY}s work", "utf-8",
+ r"$'/srv/Dana\342\200\231s work'",
+ f"/srv/Dana{CURLY}s work".encode("utf-8")),
+ "apostrophe-and-backslash": ("/srv/l'Équipe\\x", "ascii",
+ r"$'/srv/l\047\303\211quipe\134x'",
+ "/srv/l'Équipe\\x".encode("utf-8")),
+ "outside-cp1252": ("/srv/日本", "cp1252",
+ r"$'/srv/\346\227\245\346\234\254'",
+ "/srv/日本".encode("utf-8")),
+ # `é` as `os.fsdecode` reads it with an ASCII filesystem encoding: two
+ # `surrogateescape` bytes, read again as the UTF-8 they spell.
+ "utf-8-read-byte-by-byte": (f"/srv/caf{chr(0xDCC3)}{chr(0xDCA9)}",
+ "utf-8", "'/srv/café'",
+ "/srv/café".encode("utf-8")),
+ # A surrogate `surrogateescape` never makes: escaped, never raised on.
+ "stray-surrogate": (f"/srv/a{chr(0xD800)}", "utf-8",
+ r"$'/srv/a\355\240\200'", b"/srv/a\xed\xa0\x80"),
+}
+
+#: The same cases, as `pytest` parameters: the first three, or the last.
+SPELLED = [pytest.param(*case[:3], id=name)
+ for name, case in POSIX_CASES.items()]
+SPELLED_AND_RUN = [pytest.param(*case[1:], id=name)
+ for name, case in POSIX_CASES.items()]
+
+#: `(value, encoding, spelling)` for PowerShell, which compares no bytes:
+#: a Windows path is text.
+WINDOWS_CASES = [
+ pytest.param("C:\\Équipe", "utf-8", "'C:\\Équipe'", id="utf-8-shown"),
+ pytest.param("C:\\Équipe", "cp1252", "'C:\\Équipe'", id="cp1252-shown"),
+ pytest.param("C:\\Équipe", "ascii",
+ "('C:\\' + [char]0x00C9 + 'quipe')", id="ascii-escaped"),
+ pytest.param("Équipe", "ascii", "('' + [char]0x00C9 + 'quipe')",
+ id="first-operand-a-string"),
+ pytest.param("C:\\日本", "cp1252",
+ "('C:\\' + [char]0x65E5 + [char]0x672C)",
+ id="outside-cp1252"),
+ pytest.param(f"C:\\Dana{CURLY}s", "utf-8",
+ "('C:\\Dana' + [char]0x2019 + 's')", id="curly-apostrophe"),
+ pytest.param("C:\\" + chr(0x1F600), "cp1252",
+ "('C:\\' + [char]0xD83D + [char]0xDE00)",
+ id="two-code-units"),
+ pytest.param("C:\\l'É" + chr(0x201A), "utf-8",
+ "'C:\\l''É" + chr(0x201A) * 2 + "'",
+ id="quotes-doubled"),
+]
+
+
+@pytest.fixture(scope="module")
+def module():
+ return doctor_module(REPO)
+
+
+@pytest.mark.parametrize(("value", "encoding", "spelling"), SPELLED)
+def test_785_a_value_is_spelled_for_sh(module, value, encoding, spelling):
+ assert module.quote_arg(value, "posix", encoding=encoding) == spelling
+
+
+#: A legacy multibyte filesystem encoding, and a character it holds that a
+#: directory may be named in UTF-8 instead: the joined character would be
+#: written in that encoding, as other bytes (review M1 on PR #792).
+LEGACY = [pytest.param("euc_jp", "日", id="euc-jp"),
+ pytest.param("big5", "日", id="big5"),
+ pytest.param("euc_kr", "€", id="euc-kr")]
+
+
+@pytest.mark.parametrize("report", ["filesystem", "utf-8"])
+@pytest.mark.parametrize(("filesystem", "char"), LEGACY)
+def test_785_a_utf_8_name_under_a_legacy_filesystem_keeps_its_bytes(
+ module, monkeypatch, report, filesystem, char):
+ """The UTF-8 bytes of `char`, read in as escaped bytes under that
+ filesystem encoding, stay those bytes in the table (written in that
+ encoding) and in `--json` (in UTF-8): never `char` itself, whose
+ `path_bytes` there are another directory's."""
+ monkeypatch.setattr(sys, "getfilesystemencoding", lambda: filesystem)
+ raw = char.encode("utf-8")
+ value = "/srv/" + raw.decode(filesystem, "surrogateescape")
+ escaped = "".join(f"\\{byte:03o}" for byte in raw)
+ encoding = filesystem if report == "filesystem" else report
+ assert module.quote_arg(value, "posix", encoding=encoding) == (
+ f"$'/srv/{escaped}'")
+
+
+@pytest.mark.parametrize(("value", "encoding", "spelling"), WINDOWS_CASES)
+def test_785_a_value_is_spelled_for_powershell(module, value, encoding,
+ spelling):
+ assert module.quote_arg(value, "nt", encoding=encoding) == spelling
+
+
+@pytest.mark.parametrize(("value", "encoding", "spelling"), SPELLED)
+def test_785_the_rows_own_pass_leaves_a_spelling_whole(module, value,
+ encoding, spelling):
+ """`Row.__init__` runs `ascii_text` over every command and `why`, after
+ the values in it are quoted: it must find nothing in one to change."""
+ spelled = module.quote_arg(value, "posix", encoding=encoding)
+ assert module.ascii_text(spelled, encoding) == spelling
+
+
+def powershell_value(spelling: str) -> str:
+ """The value a PowerShell spelling of `quote_arg`'s stands for: one
+ verbatim string, or an expression of verbatim strings and `[char]`
+ code units joined by `+`. A reader of those two forms, not of
+ PowerShell."""
+ if not spelling.startswith("("):
+ return powershell_words(spelling)[0]
+ units = bytearray()
+ for part in re.findall(r"'(?:[^']|'')*'|\[char\]0x[0-9A-F]{4}",
+ spelling[1:-1]):
+ if part.startswith("'"):
+ units += powershell_words(part)[0].encode("utf-16-le")
+ else:
+ units += int(part[8:], 16).to_bytes(2, "little")
+ return units.decode("utf-16-le")
+
+
+@pytest.mark.parametrize(("value", "encoding", "spelling"), WINDOWS_CASES)
+def test_785_a_powershell_spelling_stands_for_the_value(module, value,
+ encoding, spelling):
+ assert powershell_value(module.quote_arg(value, "nt",
+ encoding=encoding)) == value
+
+
+def test_785_a_sentence_keeps_what_the_report_can_show(module):
+ assert module.ascii_text("members/café: ok", "utf-8") == (
+ "members/café: ok")
+
+
+@pytest.mark.parametrize(("text", "encoding", "shown"), [
+ pytest.param("members/café", "ascii", "members/caf?", id="ascii"),
+ pytest.param(f"caf{LATIN_1_E_ACUTE}", "utf-8", "caf?", id="not-utf-8"),
+ pytest.param(f"a{chr(0x202E)}b", "utf-8", "a?b", id="bidi-control"),
+ # The mark a lossy read leaves, printed as `?` since #385.
+ pytest.param(f"caf{chr(0xFFFD)}", "utf-8", "caf?", id="lossy-read"),
+ pytest.param(f"a {chr(0x2014)} {CURLY}b{CURLY}", "utf-8", "a -- 'b'",
+ id="typography"),
+])
+def test_785_a_sentence_replaces_what_the_report_cannot_show(module, text,
+ encoding, shown):
+ assert module.ascii_text(text, encoding) == shown
+
+
+@WINDOWS_SKIP
+@pytest.mark.parametrize("shell", ["bash", "zsh"])
+@pytest.mark.parametrize(("encoding", "spelling", "expected"),
+ SPELLED_AND_RUN)
+def test_785_a_shell_turns_the_spelling_back_into_the_paths_bytes(
+ shell, encoding, spelling, expected):
+ """The spelling as a report in `encoding` printed it, as bytes, so the
+ one that shows `É` hands the shell the bytes a terminal would; and
+ `expected`, the bytes of the path it was spelled from."""
+ if shutil.which(shell) is None:
+ pytest.skip(f"no {shell} here")
+ line = f"printf '%s\\0' {spelling}".encode(encoding)
+ printed = subprocess.run([shell, "-c", line],
+ capture_output=True, check=True,
+ stdin=subprocess.DEVNULL)
+ assert printed.stdout == expected + b"\0"
+
+
+# --- the doctor's own lines, pasted -----------------------------------------
+
+#: The directory a holder sits in, named in UTF-8 and in Latin-1.
+PARENTS = [pytest.param("Équipe".encode("utf-8"), id="utf-8-name"),
+ pytest.param(b"\xc9quipe", id="latin-1-name")]
+
+#: The locales the doctor and the pasted line run under: a UTF-8 one, `C`
+#: (where Python turns its UTF-8 mode on), and `C` with it off.
+LOCALES = [pytest.param("UTF-8", {}, id="utf-8-locale"),
+ pytest.param("C", {}, id="C"),
+ pytest.param("C", {"PYTHONUTF8": "0"}, id="C-ascii-fs")]
+
+#: Those, and a UTF-8 locale whose stdout is in cp1252 or in ASCII: Linux's
+#: stand-in for a Windows stdout redirected in the ANSI code page, an
+#: encoding that is not the filesystem's (review S3 on PR #792).
+STDOUTS = [*LOCALES,
+ pytest.param("UTF-8", {"PYTHONIOENCODING": "cp1252"},
+ id="cp1252-stdout"),
+ pytest.param("UTF-8", {"PYTHONIOENCODING": "ascii"},
+ id="ascii-stdout")]
+
+#: Where a line is read from: the table's bytes, or the `--json` value,
+#: which a reader's UTF-8 terminal hands the shell as UTF-8.
+SOURCES = ["table", "json"]
+
+#: A keep line naming a real directory: `git -C branch NAME`, or
+#: `git <--git-dir=...> [<--work-tree=...>] branch NAME `.
+KEEP = re.compile(rb"git (?:-C )?\S+(?: \S+)? branch NAME(?: [0-9a-f]{40})?")
+
+
+def environ(codeset: str, changes: dict) -> dict:
+ """`paste_env()` under `codeset`, with no locale or Python encoding the
+ person running the suite set, no lane and no pin-source override."""
+ env = {key: value for key, value in paste_env().items()
+ if not key.startswith(("LC_", "LANG", "PYTHONIO", "PYTHONUTF8"))}
+ env.update(LANES_LANE="", LC_ALL=listed_locale(codeset), **changes)
+ return blank_unnamed_pin_sources(env, {})
+
+
+def holder_under(holder, tmp_path: Path, parent: bytes) -> Path:
+ """A copy of the holder at `//InkRouter`, the directory
+ named in `parent`'s own bytes, its member detached."""
+ where = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), parent)))
+ where.mkdir()
+ root = where / FAMILY_NAME
+ shutil.copytree(holder["root"], root, symlinks=True)
+ detached_member(root)
+ return root
+
+
+def members_line(standard, root: Path, env: dict, source: str,
+ field: str) -> bytes:
+ """The members row's NEXT or WHY (`field`), as the bytes a reader
+ pastes: the table's line, or the `--json` value in UTF-8."""
+ if source == "json":
+ rows = json.loads(run_doctor(standard, root, env, "--json"))["rows"]
+ row = next(row for row in rows if row["id"] == "members")
+ return row[field.lower()].encode("utf-8")
+ label = None
+ for line in run_doctor(standard, root, env).splitlines():
+ if re.match(rb" \S", line):
+ label = line.split()[1]
+ elif label == b"members" and line.split()[:1] == [field.encode()]:
+ return line.split(field.encode(), 1)[1].strip()
+ raise AssertionError(f"no {field} on the members row")
+
+
+def run_doctor(standard, root: Path, env: dict, *args: str) -> bytes:
+ result = subprocess.run(
+ [sys.executable, os.fsencode(standard / DOCTOR), "--root",
+ os.fsencode(root), *args], capture_output=True, env=env,
+ stdin=subprocess.DEVNULL, check=False)
+ assert b"Traceback" not in result.stderr, result.stderr
+ return result.stdout
+
+
+def bash(line: bytes, env: dict, cwd: Path,
+ shell: str = "bash") -> subprocess.CompletedProcess:
+ """`line` pasted into bash, or `shell`, from a directory that is not
+ the root."""
+ cwd.mkdir(exist_ok=True)
+ return subprocess.run([shell, "-c", line], cwd=str(cwd), env=env,
+ capture_output=True, check=False,
+ stdin=subprocess.DEVNULL)
+
+
+def next_root(standard, holder, tmp_path: Path, parent: bytes, env: dict,
+ source: str, shell: str = "bash") -> tuple[bytes, bytes]:
+ """`(the --root word as `shell` reads the members row's NEXT, the root's
+ own bytes)`, for a holder under `parent` whose member is detached with
+ a commit on it: `printf` in place of the interpreter prints each word."""
+ root = holder_under(holder, tmp_path, parent)
+ commit_in(root / "members" / FAMILY_MEMBER)
+ line = members_line(standard, root, env, source, "NEXT")
+ words = bash(b"printf '%s\\0' " + line.split(b" ", 1)[1], env,
+ tmp_path / "away", shell).stdout.split(b"\0")
+ return words[words.index(b"--root") + 1], os.fsencode(root.resolve())
+
+
+@WINDOWS_SKIP
+@pytest.mark.parametrize("source", SOURCES)
+@pytest.mark.parametrize("state", ["detached", "emptied"])
+@pytest.mark.parametrize("parent", PARENTS)
+@pytest.mark.parametrize(("codeset", "changes"), LOCALES)
+def test_785_a_keep_line_naming_the_path_keeps_the_commit_as_printed(
+ standard, holder, tmp_path, source, state, parent, codeset,
+ changes):
+ """A commit on the member's detached HEAD, in its mount (`git -C`,
+ #413 b) or in the repository git kept when the mount was emptied
+ (`--git-dir`, #413 a): the keep line the row's WHY names keeps it on a
+ branch, run as printed."""
+ root = holder_under(holder, tmp_path, parent)
+ mount = f"members/{FAMILY_MEMBER}"
+ commit = (commit_in(root / mount) if state == "detached"
+ else emptied_with_its_repository_kept(root, mount))
+ env = environ(codeset, changes)
+ keeps = [line for line in KEEP.findall(
+ members_line(standard, root, env, source, "WHY"))
+ if b"LEG" not in line and b"REPO" not in line]
+ assert len(keeps) == 1, keeps
+ kept = bash(keeps[0].replace(b"NAME", b"kept"), env, tmp_path / "away")
+ assert kept.returncode == 0, kept.stderr
+ branch = subprocess.run(
+ ["git", "--git-dir", os.fsencode(root / ".git" / "modules" / mount),
+ "rev-parse", "--verify", "-q", "refs/heads/kept"],
+ capture_output=True, check=False)
+ assert branch.stdout.decode().strip() == commit
+
+
+@WINDOWS_SKIP
+@pytest.mark.parametrize("source", SOURCES)
+@pytest.mark.parametrize("parent", PARENTS)
+@pytest.mark.parametrize(("codeset", "changes"), STDOUTS)
+def test_785_the_members_next_names_the_real_root(standard, holder, tmp_path,
+ source, parent, codeset,
+ changes):
+ """The NEXT's `--root`, as bash reads it, is the root's own bytes, and
+ with stdout in cp1252 or ASCII too: a table that printed `É` in
+ cp1252 would hand the shell 0xC9, which is not the byte this path
+ holds."""
+ word, root = next_root(standard, holder, tmp_path, parent,
+ environ(codeset, changes), source)
+ assert word == root
+
+
+#: The cells dash can run, which are the ones that show a UTF-8 name as
+#: itself: the table under a UTF-8 locale and under `C`, where Python's UTF-8
+#: mode is on, and `--json` under every stdout (review N1 on PR #792, both
+#: rounds).
+DASH_CELLS = [*(pytest.param("table", *cell.values, id=f"table-{cell.id}")
+ for cell in LOCALES[:2]),
+ *(pytest.param("json", *cell.values, id=f"json-{cell.id}")
+ for cell in STDOUTS)]
+
+
+@WINDOWS_SKIP
+@pytest.mark.parametrize(("source", "codeset", "changes"), DASH_CELLS)
+def test_785_dash_runs_a_next_that_shows_the_name_as_itself(
+ standard, holder, tmp_path, source, codeset, changes):
+ """dash has no `$'...'`, as README says, so the lines it can run are
+ the ones that show a name as itself: a UTF-8 name, in the table where
+ stdout can show it, and in `--json` whatever stdout is."""
+ if shutil.which("dash") is None:
+ pytest.skip("no dash here")
+ word, root = next_root(standard, holder, tmp_path,
+ "Équipe".encode("utf-8"),
+ environ(codeset, changes), source, "dash")
+ assert word == root
+
+
+@WINDOWS_SKIP
+@pytest.mark.parametrize(("codeset", "changes"), STDOUTS)
+def test_785_json_holds_a_utf_8_name_as_itself_whatever_stdout_holds(
+ standard, holder, tmp_path, codeset, changes):
+ """`--json` is written for a reader who decodes it, so its `next`
+ holds a UTF-8 name as itself whatever stdout's encoding is, and under
+ `LC_ALL=C` with Python's UTF-8 mode off, where the name is read in as
+ bytes (review S2 on PR #792)."""
+ root = holder_under(holder, tmp_path, "Équipe".encode("utf-8"))
+ commit_in(root / "members" / FAMILY_MEMBER)
+ line = members_line(standard, root, environ(codeset, changes), "json",
+ "NEXT")
+ assert b"--root '" + os.fsencode(root.resolve()) + b"'" in line
+
+
+@WINDOWS_SKIP
+@pytest.mark.parametrize("parent", PARENTS)
+@pytest.mark.parametrize(("codeset", "changes"), LOCALES)
+def test_785_the_members_next_runs_as_printed_and_cures_the_row(
+ standard, holder, tmp_path, parent, codeset, changes):
+ """The holder's own bootstrap, pasted from the table, brings the
+ detached member back to its pin, and the row is asked again."""
+ root = holder_under(holder, tmp_path, parent)
+ commit_in(root / "members" / FAMILY_MEMBER)
+ env = environ(codeset, changes)
+ result = bash(members_line(standard, root, env, "table", "NEXT"), env,
+ tmp_path / "away")
+ assert result.returncode == 0, result.stdout + result.stderr
+ rows = json.loads(run_doctor(standard, root, env, "--json"))["rows"]
+ assert next(row for row in rows if row["id"] == "members")[
+ "status"] == "ok"
diff --git a/tests/test_shape_doctor.py b/tests/test_shape_doctor.py
index e48685aa..10722cfd 100644
--- a/tests/test_shape_doctor.py
+++ b/tests/test_shape_doctor.py
@@ -10089,22 +10089,26 @@ def test_the_interpreter_named_is_the_asked_for_platforms(platform, expected):
CURLY_APOSTROPHE = "O\u2019Neill"
-def test_a_curly_apostrophe_is_normalized_before_it_is_quoted():
- """`quote_arg` must ascii-normalize FIRST, or `Row.__init__` breaks it.
+@pytest.mark.parametrize(("platform", "spelled"), [
+ ("posix", r"$'O\342\200\231Neill'"),
+ ("nt", "('O' + [char]0x2019 + 'Neill')"),
+])
+def test_a_curly_apostrophe_is_escaped_where_the_row_cannot_reach_it(
+ platform, spelled):
+ """`quote_arg` must leave `Row.__init__` nothing to change in a value.
`Row.__init__` runs `ascii_text` again, after quoting, over the whole
assembled command -- so a curly apostrophe that reached this function
unconverted came back out of THAT pass as a bare, undoubled one, on the
far side of the quoting it sat inside. `'O'Neill'` is an unterminated
quotation to `sh`, and PowerShell's own verbatim string is exactly as
- broken by an apostrophe that never got doubled. Normalizing here first
- means the value quoted is already plain ASCII, so that later pass has
- nothing left to do.
+ broken by an apostrophe that never got doubled (Copilot, PR #102). It was
+ flattened to `'` here first, which named another path; it is ESCAPED
+ now, in ASCII the later pass leaves alone, and stands for this one
+ (#785, `tests/test_doctor_non_ascii_paths.py` runs it).
"""
module = doctor_module(REPO)
- posix = module.quote_arg(CURLY_APOSTROPHE, "posix")
- assert shlex.split(posix) == ["O'Neill"], posix
- assert module.quote_arg(CURLY_APOSTROPHE, "nt") == "'O''Neill'"
+ assert module.quote_arg(CURLY_APOSTROPHE, platform) == spelled
def test_a_trailing_newline_is_quoted_not_waved_through():
diff --git a/tests/test_stdout_encoding.py b/tests/test_stdout_encoding.py
index 621b43c7..252d1d51 100644
--- a/tests/test_stdout_encoding.py
+++ b/tests/test_stdout_encoding.py
@@ -312,7 +312,9 @@ def _adopt_check(tmp_path: Path, request) -> tuple:
def _doctor(tmp_path: Path, request) -> tuple:
- """The doctor's rows are ASCII already; its `root` line was not."""
+ """The doctor's `root` line names the root as it is. (Its rows spell a
+ name too since #785: as itself where stdout can show it, escaped in a
+ command and `?` in a sentence where it cannot.)"""
root = tmp_path / NAME
root.mkdir()
(root / "README.md").write_text("# here\n", encoding="utf-8")