From d4f5283b01526246ad3f8fe9104a193e54c47a44 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Tue, 10 Nov 2026 12:37:06 +0000 Subject: [PATCH 1/3] tests: test_family.py reads and rewrites the UTF-8 files its rows assert on with `encoding="utf-8"`, and a pin source named `caf` plus U+00E9 skips where the file name encoding is ASCII, so 16 rows stop failing under an ASCII locale (#1285) The 16 rows of `tests/test_family.py` and `tests/test_pinned_copy_text.py` that failed under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` on main `3add78e1c` (the ids of #1285) no longer fail there. `tests/test_family.py`: 18 bare `read_text()` calls of UTF-8 files (the family root's `Makefile`, `README.md`, `AGENTS-shape.md` and `.gitattributes`, `family.yaml`, the copied and the upstream `validate-family.py`, a sibling's `project.yaml`) now read with `encoding="utf-8"`, as #1278's `test_init_writes_the_holders_agent_files` does. The 13 `write_text()` calls that write back what such a read returned take `encoding="utf-8"` too: under ASCII, and under Latin-1 for U+2014, the text read as UTF-8 cannot be written in the locale's encoding. `tests/test_pinned_copy_text.py`: `test_a_printable_pin_source_that_is_not_a_repository_is_named_as_typed` asks `skip_where_the_environment_cannot_carry(name)` just before `source.mkdir()`, as #1278's rows do, so `[caf\xe9]` skips under ASCII and `[plain]` still runs. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_family.py | 71 ++++++++++++++++++++-------------- tests/test_pinned_copy_text.py | 5 ++- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/tests/test_family.py b/tests/test_family.py index 16e2a4a4..10dfd8c3 100644 --- a/tests/test_family.py +++ b/tests/test_family.py @@ -317,11 +317,13 @@ def test_init_copies_the_siblings_utility_and_the_makefile_target(family, monkey if os.name != "nt": assert copied.stat().st_mode & 0o111, ( "`FAMILY_EXECUTABLE` names it, so the materializer chmods it") - makefile = (root / "Makefile").read_text() + makefile = (root / "Makefile").read_text(encoding="utf-8") assert "siblings:" in makefile assert "$(PYTHON) scripts/siblings.py" in makefile - assert "make siblings" in (root / "README.md").read_text() - assert "make siblings" in (root / "AGENTS-shape.md").read_text() + assert "make siblings" in (root / "README.md").read_text( + encoding="utf-8") + assert "make siblings" in (root / "AGENTS-shape.md").read_text( + encoding="utf-8") def test_init_writes_the_holders_agent_files(family, monkeypatch): @@ -381,7 +383,7 @@ def test_the_holder_says_what_its_bytes_are(family, monkeypatch): assert copied.read_bytes() == template.read_bytes(), "verbatim, like the rest" assert b"\r" not in copied.read_bytes(), ( "the file that says LF is itself written LF, on every platform") - assert "* text=auto eol=lf" in copied.read_text() + assert "* text=auto eol=lf" in copied.read_text(encoding="utf-8") rows = {row["path"]: row["sha256"].lower() for row in load_yaml(root / "contracts" / "shape-pin.yaml")["files"]} assert rows[".gitattributes"] == file_sha256(copied) @@ -722,7 +724,7 @@ def test_validate_refuses_a_missing_gitlink(holder): def test_validate_finds_a_pin_that_disagrees_with_the_gitlink(holder): """THE LOCKSTEP RULE. Moving the pin alone is exactly the failure the xFactory aggregation shipped seven times in a row.""" - text = (holder / "family.yaml").read_text() + text = (holder / "family.yaml").read_text(encoding="utf-8") row = {r["project"]: r for r in manifest(holder)["members"]}["IRRS"] # A REAL commit that is not the gitlink: the member's own parent. Pinning # 40 zeros would prove only that an unresolvable commit is refused, which @@ -730,7 +732,7 @@ def test_validate_finds_a_pin_that_disagrees_with_the_gitlink(holder): parent = git("rev-parse", "HEAD~1", cwd=holder / "members" / "IRRS").stdout.strip() (holder / "family.yaml").write_text( - text.replace(row["pin"]["commit"], parent)) + text.replace(row["pin"]["commit"], parent), encoding="utf-8") result = validate(holder) assert result.returncode == 1 assert "member-gitlink-mismatch" in result.stderr @@ -755,19 +757,20 @@ def test_validate_accepts_a_member_bump_that_is_staged_but_not_committed(holder) after = git("rev-parse", "HEAD", cwd=member).stdout.strip() git("add", "--", "members/IRRS", cwd=holder) # staged, NOT committed manifest_path = holder / "family.yaml" - text = manifest_path.read_text() + text = manifest_path.read_text(encoding="utf-8") assert text.count(before) == 1, "fixture drift: IRRS's pin is not unique" - manifest_path.write_text(text.replace(before, after, 1)) + manifest_path.write_text(text.replace(before, after, 1), + encoding="utf-8") result = validate(holder) assert result.returncode == 0, result.stderr + result.stdout assert "gitlink == pin" in result.stdout def test_validate_finds_a_digest_that_does_not_recompute(holder): - text = (holder / "family.yaml").read_text() + text = (holder / "family.yaml").read_text(encoding="utf-8") row = {r["project"]: r for r in manifest(holder)["members"]}["IRSS"] (holder / "family.yaml").write_text( - text.replace(row["pin"]["tree_sha256"], "b" * 64)) + text.replace(row["pin"]["tree_sha256"], "b" * 64), encoding="utf-8") result = validate(holder) assert result.returncode == 1 assert "member-digest-mismatch" in result.stderr @@ -776,9 +779,10 @@ def test_validate_finds_a_digest_that_does_not_recompute(holder): def test_validate_finds_an_id_that_is_not_the_project_mounted_there(holder): """A repository at the right commit is not by itself the project the row claims: `project.yaml` inside the member is the source.""" - text = (holder / "family.yaml").read_text() + text = (holder / "family.yaml").read_text(encoding="utf-8") (holder / "family.yaml").write_text( - text.replace(" id: irrs\n", " id: something-else\n")) + text.replace(" id: irrs\n", " id: something-else\n"), + encoding="utf-8") result = validate(holder) assert result.returncode == 1 assert "member-id-mismatch" in result.stderr @@ -871,7 +875,8 @@ def test_the_doctor_reports_a_project_pinned_twice(family, holder): def test_validate_finds_an_edited_shape_copy(holder): copy = holder / "scripts" / "validate-family.py" - copy.write_text(copy.read_text() + "\n# edited in place\n") + copy.write_text(copy.read_text(encoding="utf-8") + + "\n# edited in place\n", encoding="utf-8") result = validate(holder) assert result.returncode == 1 assert "shape-copy-drift" in result.stderr @@ -1232,8 +1237,10 @@ def test_bump_to_the_commit_already_pinned_leaves_an_uncommitted_edit_alone( `git status` says the same before and after.""" pinned = git("rev-parse", "HEAD:members/IRRS", cwd=holder).stdout.strip() manifest_path = holder / "family.yaml" - manifest_path.write_text(manifest_path.read_text().replace( - " id: irrs\n", " id: something-else\n")) + manifest_path.write_text( + manifest_path.read_text(encoding="utf-8").replace( + " id: irrs\n", " id: something-else\n"), + encoding="utf-8") edited = manifest_path.read_bytes() status = git("status", "--porcelain", cwd=holder).stdout assert status.strip() == "M family.yaml", status @@ -1302,9 +1309,10 @@ def test_bump_finishes_a_bump_an_earlier_run_staged_and_did_not_commit(holder): before = git("rev-parse", "HEAD:members/IRRS", cwd=holder).stdout.strip() after = commit_in_member(member) manifest_path = holder / "family.yaml" - text = manifest_path.read_text() + text = manifest_path.read_text(encoding="utf-8") assert text.count(before) == 1, "fixture drift: IRRS's pin is not unique" - manifest_path.write_text(text.replace(before, after, 1)) + manifest_path.write_text(text.replace(before, after, 1), + encoding="utf-8") git("add", "--", "members/IRRS", "family.yaml", cwd=holder) head = git("rev-parse", "HEAD", cwd=holder).stdout.strip() args = ["bump", "--family-root", str(holder), "--member", "IRRS", "--to", @@ -1365,8 +1373,9 @@ def test_bump_to_the_pinned_commit_repairs_a_digest_that_does_not_recompute( pinned = git("rev-parse", "HEAD:members/IRRS", cwd=holder).stdout.strip() row = {r["project"]: r for r in manifest(holder)["members"]}["IRRS"] manifest_path = holder / "family.yaml" - manifest_path.write_text(manifest_path.read_text().replace( - row["pin"]["tree_sha256"], "b" * 64)) + manifest_path.write_text( + manifest_path.read_text(encoding="utf-8").replace( + row["pin"]["tree_sha256"], "b" * 64), encoding="utf-8") git("add", "--", "family.yaml", cwd=holder) git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "Hand-edit a digest", cwd=holder) @@ -1691,8 +1700,9 @@ def test_update_shape_reads_a_family_root_and_mirrors_into_family_yaml( changed = "templates/family-root/scripts/validate-family.py" source = upstream / changed - source.write_text(source.read_text() - + "\n# An upstream fix that must reach every family.\n") + source.write_text(source.read_text(encoding="utf-8") + + "\n# An upstream fix that must reach every family.\n", + encoding="utf-8") git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "Fix the family validator", "--", changed, cwd=upstream) target_commit = git("rev-parse", "HEAD", cwd=upstream).stdout.strip() @@ -1714,7 +1724,7 @@ def test_update_shape_reads_a_family_root_and_mirrors_into_family_yaml( assert load_yaml(holder / "contracts" / "shape-pin.yaml")["commit"] \ == target_commit assert "must reach every family" in \ - (holder / "scripts" / "validate-family.py").read_text() + (holder / "scripts" / "validate-family.py").read_text(encoding="utf-8") assert validate(holder).returncode == 0 @@ -1731,8 +1741,9 @@ def test_update_shape_moves_the_holder_readmes_shape_line_with_the_pin( upstream = upstream_clone(tmp_path / "openRepoShape") changed = "templates/family-root/scripts/validate-family.py" source = upstream / changed - source.write_text(source.read_text() - + "\n# An upstream fix that must reach every family.\n") + source.write_text(source.read_text(encoding="utf-8") + + "\n# An upstream fix that must reach every family.\n", + encoding="utf-8") git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "Fix the family validator", "--", changed, cwd=upstream) target_commit = git("rev-parse", "HEAD", cwd=upstream).stdout.strip() @@ -1765,8 +1776,9 @@ def test_update_shape_with_no_shape_block_names_the_family_validator( upstream = upstream_clone(tmp_path / "openRepoShape") changed = "templates/family-root/scripts/validate-family.py" source = upstream / changed - source.write_text(source.read_text() - + "\n# forces a target commit past the family's pin.\n") + source.write_text(source.read_text(encoding="utf-8") + + "\n# forces a target commit past the family's pin.\n", + encoding="utf-8") git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "Force a target commit past the family's pin", "--", changed, cwd=upstream) @@ -1827,7 +1839,8 @@ def test_no_members_checks_the_envelope_and_the_copies_alone(holder): def test_no_members_still_finds_an_edited_copy(holder): copy = holder / "Makefile" - copy.write_text(copy.read_text() + "\n# edited in place\n") + copy.write_text(copy.read_text(encoding="utf-8") + + "\n# edited in place\n", encoding="utf-8") result = validate(holder, "--no-members") assert result.returncode == 1 assert "shape-copy-drift" in result.stderr @@ -2352,8 +2365,8 @@ def test_siblings_skips_a_clone_whose_project_id_is_not_the_rows(estate): sibling = estate["folder"] / "IRSS" manifest_path = sibling / "project.yaml" manifest_path.write_text( - manifest_path.read_text().replace('id: "irss"', 'id: "something-else"', - 1)) + manifest_path.read_text(encoding="utf-8").replace( + 'id: "irss"', 'id: "something-else"', 1), encoding="utf-8") result = siblings(estate["holder"]) assert result.returncode == 1, result.stdout + result.stderr assert "WRONG PROJECT" in result.stdout diff --git a/tests/test_pinned_copy_text.py b/tests/test_pinned_copy_text.py index d99414c8..37601690 100644 --- a/tests/test_pinned_copy_text.py +++ b/tests/test_pinned_copy_text.py @@ -45,7 +45,7 @@ from conftest import (PRINTED_AS_ITSELF, REPO, SCAFFOLD, clear_ambient_pin_sources, git, load_without_bytecode, - run_script, + run_script, skip_where_the_environment_cannot_carry, skip_where_the_locale_does_not_spell_it_as_utf_8) from test_link_source_chain_defaults import ( # noqa: F401 VALIDATOR, manifest, naming, pinned, scaffold, tree, validate_pins, @@ -670,6 +670,9 @@ def test_a_printable_pin_source_that_is_not_a_repository_is_named_as_typed( """The control: a printable folder, UTF-8 included, is named byte for byte, with no quotes.""" source = tmp_path / name + # `mkdir` cannot make "caf\u00e9" where the file name encoding is ASCII + # (#1285); `plain` still runs there. + skip_where_the_environment_cannot_carry(name) source.mkdir() commit = git("rev-parse", "HEAD", cwd=pinned["product"]).stdout.strip() result = validate_pins(pinned, "--pin-source", str(source)) From 84e006269ff49ae69b7a20be401a0bba11d9e69d Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Tue, 10 Nov 2026 16:24:21 +0000 Subject: [PATCH 2/3] tests: test_family.py rows 1, 3 and 11 assert a character or the bytes of a file they read, so a read of it as Latin-1 fails the row, as #1285's third box asks (#1285) Row 1 (test_init_copies_the_siblings_utility_and_the_makefile_target) asserts a phrase of the Makefile that holds its U+2014. Row 3 (test_validate_finds_a_pin_that_disagrees_with_the_gitlink) asserts that family.yaml is not ASCII and that its bytes after the rewrite equal the bytes before with only the pin's commit replaced. Row 11 (test_update_shape_reads_a_family_root_and_mirrors_into_family_yaml) asserts that the upstream validate-family.py is not ASCII and that its bytes after the rewrite equal the old bytes plus the appended line as UTF-8. A read as Latin-1 survived in every locale in these three rows before; no tool file and no other row changes. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_family.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_family.py b/tests/test_family.py index 10dfd8c3..d31efd53 100644 --- a/tests/test_family.py +++ b/tests/test_family.py @@ -320,6 +320,10 @@ def test_init_copies_the_siblings_utility_and_the_makefile_target(family, monkey makefile = (root / "Makefile").read_text(encoding="utf-8") assert "siblings:" in makefile assert "$(PYTHON) scripts/siblings.py" in makefile + assert "WORKING CLONES BESIDE THIS HOLDER — not in" in makefile, ( + "the Makefile tells a person that the sibling clones `siblings.py` " + "works on are BESIDE the holder, not in `members/`; its dash " + "is one character only when the file is read as UTF-8") assert "make siblings" in (root / "README.md").read_text( encoding="utf-8") assert "make siblings" in (root / "AGENTS-shape.md").read_text( @@ -724,6 +728,7 @@ def test_validate_refuses_a_missing_gitlink(holder): def test_validate_finds_a_pin_that_disagrees_with_the_gitlink(holder): """THE LOCKSTEP RULE. Moving the pin alone is exactly the failure the xFactory aggregation shipped seven times in a row.""" + before = (holder / "family.yaml").read_bytes() text = (holder / "family.yaml").read_text(encoding="utf-8") row = {r["project"]: r for r in manifest(holder)["members"]}["IRRS"] # A REAL commit that is not the gitlink: the member's own parent. Pinning @@ -733,6 +738,13 @@ def test_validate_finds_a_pin_that_disagrees_with_the_gitlink(holder): cwd=holder / "members" / "IRRS").stdout.strip() (holder / "family.yaml").write_text( text.replace(row["pin"]["commit"], parent), encoding="utf-8") + assert not before.isascii(), ( + "fixture drift: family.yaml holds no character outside ASCII, so the " + "next assert cannot tell a read as UTF-8 from a read as anything else") + assert (holder / "family.yaml").read_bytes() == before.replace( + row["pin"]["commit"].encode("ascii"), parent.encode("ascii")), ( + "the rewrite changed more than the pin's commit: family.yaml was read " + "or written as something other than UTF-8") result = validate(holder) assert result.returncode == 1 assert "member-gitlink-mismatch" in result.stderr @@ -1700,9 +1712,18 @@ def test_update_shape_reads_a_family_root_and_mirrors_into_family_yaml( changed = "templates/family-root/scripts/validate-family.py" source = upstream / changed - source.write_text(source.read_text(encoding="utf-8") - + "\n# An upstream fix that must reach every family.\n", + appended = "\n# An upstream fix that must reach every family.\n" + old_bytes = source.read_bytes() + source.write_text(source.read_text(encoding="utf-8") + appended, encoding="utf-8") + assert not old_bytes.isascii(), ( + "fixture drift: the upstream validate-family.py holds no character " + "outside ASCII, so the next assert cannot tell a read as UTF-8 from a " + "read as anything else") + assert source.read_bytes() == old_bytes + appended.encode("utf-8"), ( + "the rewrite changed more than the appended line: the upstream " + "validate-family.py was read or written as something other than " + "UTF-8") git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "Fix the family validator", "--", changed, cwd=upstream) target_commit = git("rev-parse", "HEAD", cwd=upstream).stdout.strip() From bd84004f3eea693729ece80f378d1e8bdea40555 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Tue, 10 Nov 2026 16:41:13 +0000 Subject: [PATCH 3/3] tests: test_family.py row 11 holds its bytes compare in a name, so a failure of it does not make pytest diff the 268 KB upstream validate-family.py for minutes (#1285) The bytes assert added for the row compared two 268 KB values in the assert itself. A mutant that read the file as Latin-1 failed it, and pytest then spent more than four minutes of CPU diffing the two values line by line before it reported the failure (stopped by hand; CI makes pytest diff at any verbosity). The compare is now a bool the assert names, with the failure message unchanged. Row 3's family.yaml is 4.6 KB and keeps its direct compare. Lane: openreposhape-1 (openRepoShape-1) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_family.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_family.py b/tests/test_family.py index d31efd53..4a7820b0 100644 --- a/tests/test_family.py +++ b/tests/test_family.py @@ -1720,7 +1720,11 @@ def test_update_shape_reads_a_family_root_and_mirrors_into_family_yaml( "fixture drift: the upstream validate-family.py holds no character " "outside ASCII, so the next assert cannot tell a read as UTF-8 from a " "read as anything else") - assert source.read_bytes() == old_bytes + appended.encode("utf-8"), ( + # The compare is held in a name: an `==` in the assert would make pytest + # diff the file's quarter of a megabyte line by line, for minutes, on a + # failure. + rewritten = source.read_bytes() == old_bytes + appended.encode("utf-8") + assert rewritten, ( "the rewrite changed more than the appended line: the upstream " "validate-family.py was read or written as something other than " "UTF-8")