adopt-project.py: _spelled writes a raw byte that is not UTF-8 as \udcNN and a character as \xNN, as the plan's own path: does, so the byte 0x85 and the character U+0085 no longer both read \x85, and 34 pairs of different names no longer read as one in a question or a finding (#329)

adopt-project.py

@@ -1208,12 +1208,19 @@ def _spelled(path: str) -> str:
     file can hold -- written raw, it raised mid-write and left `--out` empty
     (#166's NEW-2) -- and a name may hold a newline, which would end the
     line. So a character that is not printable is spelled with a backslash,
-    `\\xe9` for the byte 0xE9 and `\\n` for a newline, and so are a
-    backslash and a backtick, so that the spelling reads one way only and
-    the backticks still delimit it. The plan's own `path:` spells that byte
-    `\\udce9`, not `\\xe9` (`repo_shape.double_quoted`), because in YAML
-    `\\xe9` is the character U+00E9, which is another name: a question or a
-    report that says `caf\\xe9` is the entry `"caf\\udce9"` in the plan.
+    and so are a backslash and a backtick, so that the spelling reads one way
+    only and the backticks still delimit it.
+
+    THE SPELLING IS THE PLAN'S OWN (`_spelled_character`): the byte 0xE9 is
+    `\\udce9` and the character U+0085 is `\\x85`, as `path:` writes them
+    (`repo_shape.double_quoted`), so that a question or a report that says
+    `caf\\udce9` is the entry `"caf\\udce9"` in the plan. Each escape names
+    a code point, so a byte and a character never share one. They did: a
+    byte was `\\xe9` here, which in YAML is the character U+00E9, and the 32
+    C1 controls, U+00A0 and U+00AD were spelled as the bytes of the same
+    value, 34 pairs of different names that read alike (#329). A spelling is
+    only printed, in a finding, a refusal or a question the plan carries;
+    none is read back or handed to git.
     """
     return "`" + "".join(map(_spelled_character, path)) + "`"
 
@@ -1233,13 +1240,17 @@ def _shown(path: str) -> str:
 
 
 def _spelled_character(char: str) -> str:
+    """One character of `_spelled`: itself when it is printable, a backslash
+    before a backslash or a backtick, and otherwise the escape
+    `repo_shape.double_quoted` writes for it in the plan, so the plan's one
+    table and not a second one: `\\n` for a line feed, `\\x85` for U+0085,
+    `\\u2028` for U+2028, and `\\udc85` for the byte 0x85, which
+    `Source.tree` keeps as the surrogate U+DC00 plus the byte."""
     if char in "\\`":
         return "\\" + char
     if char.isprintable():
         return char
-    if "\udc80" <= char <= "\udcff":
-        return f"\\x{ord(char) - 0xDC00:02x}"
-    return ascii(char)[1:-1]
+    return double_quoted(char)[1:-1]
 
 
 def cross_leg_references(source: "Source", entries: list["Entry"],
@@ -3349,7 +3360,7 @@ def _refused_by_policy(path: str) -> str | None:
       * a byte that is not UTF-8, which `Source.tree` keeps as one of
         U+DC80 to U+DCFF (`surrogateescape`). Since #199 (PR #247)
         `write_paths_listing` writes it as that byte, and with this refusal
-        disabled a source holding `caf\\xe9.md` was extracted and verified,
+        disabled a source holding `caf\\udce9.md` was extracted and verified,
         the byte in the leg, measured. It is held until adopting such a name
         end to end is measured on macOS and Windows: a name that holds one
         prints as an escape, never as itself, and the split commit's message
@@ -4279,8 +4290,8 @@ def _index_entries(assembly: Path,
     as #169 refuses case-only twins of the two leg paths. `-z`, for a path
     git would otherwise quote. A name that is not UTF-8 keeps each byte as a
     surrogate, as `Source.tree` keeps it, so the refusal spells the byte 0xFF
-    `\\xff`, as every other finding does (#212). Decoded with `replace`,
-    `a<0xFF>` and `a<0xFE>` both read `a<U+FFFD>`.
+    `\\udcff`, as every other finding does (#212, #329). Decoded with
+    `replace`, `a<0xFF>` and `a<0xFE>` both read `a<U+FFFD>`.
     """
     raw = git_out(["--icase-pathspecs", "ls-files", "-s", "-z", "--",
                    *pathspecs], cwd=assembly, binary=True)

tests/test_adopt_entry_paths.py

@@ -177,9 +177,9 @@ def planned(tmp_path: Path, tree: dict,
     ("line\u2028sep.md", "line separator `\\u2028`"),
     # A byte that is not UTF-8, as `Source.tree` keeps it: since #199 the
     # path list carries it byte-exact.
-    ("caf\udce9.md", "`\\xe9`, which is not UTF-8"),
-    ("docs/\udcff/", "`\\xff`, which is not UTF-8"),
-    ("\udc80.md", "`\\x80`, which is not UTF-8"),
+    ("caf\udce9.md", "`\\udce9`, which is not UTF-8"),
+    ("docs/\udcff/", "`\\udcff`, which is not UTF-8"),
+    ("\udc80.md", "`\\udc80`, which is not UTF-8"),
 ]
 
 
@@ -653,8 +653,8 @@ def with_a_name_that_is_not_utf8(adopter, tmp_path: Path):
 #: The finding `check` must print first for `NOT_UTF8`, spelled as
 #: `_spelled` spells a byte that is not UTF-8.
 NOT_UTF8_FINDING = (
-    f"FINDING {UNSAFE}: `docs/caf\\xe9.md` has leg: spec, and `execute` "
-    "will not extract it into a leg: it holds `\\xe9`, which is not UTF-8 "
+    f"FINDING {UNSAFE}: `docs/caf\\udce9.md` has leg: spec, and `execute` "
+    "will not extract it into a leg: it holds `\\udce9`, which is not UTF-8 "
     "and which `execute` refuses as policy, though git and `git filter-repo` "
     "would take it literally: a name that holds one prints as an escape and "
     "never as itself, the split commit's message would list it with a `?` "

tests/test_adopt_finding_spelling.py

@@ -313,10 +313,11 @@ def test_adopt_mount_occupied_spells_the_kept_path_under_the_mount(
 def test_adopt_mount_occupied_spells_a_byte_that_is_not_utf8(adopter,
                                                              tmp_path):
     """`_index_entries` reads the index as `Source.tree` reads a tree: a byte
-    that is not UTF-8 is kept, so the refusal spells 0xFE `\\xfe`, as
-    `plan-uncovered` would. Decoded with `replace`, `a<0xFE>` and `a<0xFF>`
-    both read `a<U+FFFD>`. The entries go in with `git update-index
-    --index-info`, fed bytes, so no filesystem has to hold them."""
+    that is not UTF-8 is kept, so the refusal spells 0xFE `\\udcfe`, as
+    `plan-uncovered` would, and as the plan's own `path:` does (#329).
+    Decoded with `replace`, `a<0xFE>` and `a<0xFF>` both read `a<U+FFFD>`.
+    The entries go in with `git update-index --index-info`, fed bytes, so no
+    filesystem has to hold them."""
     repo = tmp_path / "assembly"
     repo.mkdir()
     git_out(repo, "init", "-q")
@@ -326,13 +327,13 @@ def test_adopt_mount_occupied_spells_a_byte_that_is_not_utf8(adopter,
         for name in (b"spec/a\xfe", b"spec/a\xff")))
     assert [adopter._entry_named(entry) for entry in
             adopter._index_entries(repo, ["spec"])] == [
-        "`spec/a\\xfe` (a file)", "`spec/a\\xff` (a file)"]
+        "`spec/a\\udcfe` (a file)", "`spec/a\\udcff` (a file)"]
     with pytest.raises(adopter.Refusal) as caught:
         adopter._refuse_an_unmountable_path(repo, "spec")
     assert caught.value.code == "adopt-mount-occupied"
     assert caught.value.detail == (
         "the assembly's index already holds paths under spec, the leg mount "
-        "path: `spec/a\\xfe` (a file) and 1 more")
+        "path: `spec/a\\udcfe` (a file) and 1 more")
 
 
 #: The root tree's two leg mounts, as `_verify` passes them to

tests/test_adopt_gitmodules_values.py

@@ -259,7 +259,7 @@ def test_the_finding_names_the_key_git_dies_on(adopter, tmp_path, gitmodules,
     (with_line(r'update = "bo\ngus"'),
      f"invalid value for `submodule.b.update`: `bo\\ngus`, {NOT_A_STRATEGY}"),
     (with_line("shallow = caf").encode()[:-1] + b"\xe9\n",
-     "bad boolean config value `caf\\xe9` for `submodule.b.shallow`"),
+     "bad boolean config value `caf\\udce9` for `submodule.b.shallow`"),
     (b'[submodule "b\x1bc"]\n\tpath = vendor/b\n\turl = ../b\n'
      b"\tfetchRecurseSubmodules = `x`\n",
      "bad `submodule.b\\x1bc.fetchrecursesubmodules` argument: `\\`x\\``, "

tests/test_adopt_literal_names.py

@@ -264,8 +264,8 @@ def test_a_colour_setting_in_the_source_is_not_in_a_follow_up(tmp_path):
 def test_a_name_with_a_line_break_or_a_byte_outside_utf8_is_one_spelled_line(
         tmp_path):
     """git's `-z` gives a name as its bytes, and the follow-up spells it as a
-    question spells a path: `\\n` for the line break, `\\xe9` for the byte,
-    on one line of UTF-8. On main each came back in git's C-quoted
+    question spells a path: `\\n` for the line break, `\\udce9` for the byte
+    (#329), on one line of UTF-8. On main each came back in git's C-quoted
     spelling, quotes and octal included. The two files are put in the TREE
     (`update-index -z --index-info`) and never on the disk, which macOS
     refuses for a name that is not UTF-8."""
@@ -285,7 +285,7 @@ def test_a_name_with_a_line_break_or_a_byte_outside_utf8_is_one_spelled_line(
 
     plan = planned(source, tmp_path)
 
-    assert named(plan) == [follow_up("src/caf\\xe9.py", "notes/"),
+    assert named(plan) == [follow_up("src/caf\\udce9.py", "notes/"),
                            follow_up("src/line\\nbreak.py", "notes/")]
 
 

tests/test_adopt_plan_escapes.py

@@ -409,7 +409,7 @@ def test_a_source_directory_that_is_not_utf8_is_shown_spelled(adopter,
     except AssertionError as exc:
         pytest.skip(f"git will not work under b'caf\\xe9' here: {exc}")
     shown = adopter._spelled(str(source.resolve()))
-    assert "\\xe9" in shown
+    assert "\\udce9" in shown
     plan = tmp_path / PLAN_FILE
     strict = {"PYTHONIOENCODING": "utf-8:strict"}