From 21fbea4782bdbf6f6690827956e2a0a93863c722 Mon Sep 17 00:00:00 2001 From: Brett Heap <1513478+brettheap@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:33:23 +0000 Subject: [PATCH] repo_shape.py and validate-manifest.py: a plain flow scalar holding `[`, `]`, `{` or `}` is refused, and a display name holding a lone surrogate is a finding, so `pins: [a, b]]` no longer reads as the pin `b]` and `name: "a\ud83db"` no longer stops `make validate` on a traceback while `bump-leg.py` commits it as `a?b` (#301, #280) #301. In YAML a plain scalar inside a flow collection ends at any of `[ ] { } ,`. `_split_flow` counted brackets to find its commas and never asked whether the count went below zero or ended above it, so a stray bracket was kept as text: `k: [a]]` read `['a]']`, `k: [x [a, b]]` read `['x [a, b]']`, and since #284 `k: {a"]:}` read the key `a"]`. PyYAML refuses each. `_refuse_flow_indicator` now refuses a plain item or value (`_flow_value`) and a plain key (`_split_key_flow`) of a flow collection that holds one, as a `YamlError` (`REFUSED yaml-unparsable`). A quoted scalar's brackets, and a bracket in a block plain scalar, read as before. #280. The reader keeps the escape `\udXXXX` on purpose: `double_quoted` writes a non-UTF-8 path byte that way into an adoption plan (#199), and PyYAML's Python reader reads it the same. A display name is text a person chose, and no UTF-8 stream holds a surrogate: `validate-manifest.py`'s `manifest ok` line raised `UnicodeEncodeError`, so `make validate` and `shape-doctor.py` met a traceback, while `bump-leg.py` and `update-shape.py apply` passed the project and the bump's commit subject read `... in a?b`. `_name_finding` now reports a name holding any of U+D800..U+DFFF as `manifest-name`, and all four launchers stop at that finding. Evidence: all ten #301 rows and every #280 row reproduce on c7c02fa and are refused on this commit; on the seeded PyYAML-oracle fuzz (seed 2490250, 785,646 documents) no document c7c02fa read the way PyYAML reads it changes, 67,437 documents PyYAML refuses are refused instead of read, and 1,058 that c7c02fa misread (PyYAML's `#` leniency and single-pair flow mappings) are refused instead of misread; every tracked YAML file loads the same. Tests: tests/test_yaml_flow_brackets.py (new), the surrogate tests in tests/test_manifest_validator.py and tests/test_validator_child_encoding.py (a real bump, `update-shape.py apply`, the doctor and `make validate`), and the NIT-A count in tests/test_yaml_quote_boundaries.py's docstring. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Opus 5.5 --- scripts/repo_shape.py | 44 ++++- .../scripts/validate-manifest.py | 27 ++- tests/test_manifest_validator.py | 71 ++++++++ tests/test_validator_child_encoding.py | 108 +++++++++++ tests/test_yaml_flow_brackets.py | 171 ++++++++++++++++++ tests/test_yaml_quote_boundaries.py | 7 +- 6 files changed, 422 insertions(+), 6 deletions(-) create mode 100644 tests/test_yaml_flow_brackets.py diff --git a/scripts/repo_shape.py b/scripts/repo_shape.py index 38b45290..8e500e0f 100644 --- a/scripts/repo_shape.py +++ b/scripts/repo_shape.py @@ -461,6 +461,31 @@ def _scalar(raw: str) -> Any: return text +def _refuse_flow_indicator(text: str) -> None: + """Refuse a PLAIN scalar of a flow collection that holds `[`, `]`, `{` or + `}` (#301); a quoted one is `_scalar`'s to judge, and its brackets are + text. + + In YAML a plain scalar inside a flow collection ENDS at any of the four, + as it ends at the `,` `_split_flow` splits at, so a scalar holding one is + never the text it looks like, and PyYAML refuses the document. `k: [a]]` + closes the sequence after `a` and leaves a `]` nothing opened, and + `k: [x [a, b]]` opens a sequence while an item is still being read. + `_split_flow` counts brackets to find its commas and never asked whether + the count went below zero or ended above it, so the rest was read as + text: `['a]']`, `['x [a, b]']`, and the key of `k: {a"]:}` as `a"]` + since #284 stopped reading that `"` as a quote. Refused, a stray + bracket typed into `pins: [a, b]]` is a `yaml-unparsable` naming it, not + the pin `b]`. + """ + if text.startswith(('"', "'")): + return + held = next((char for char in text if char in "[]{}"), None) + if held is not None: + raise YamlError(f"a plain scalar in a flow collection holds `{held}`, " + f"which ends it in YAML: {text!r}") + + def _flow_value(text: str) -> Any: """A flow entry is itself a flow collection when it OPENS with `[` or `{`. @@ -468,8 +493,14 @@ def _flow_value(text: str) -> Any: and `'' in "[{"` is True for any string, so the test used to run `_flow` on it and refuse `{a: }` as `not a flow collection: ''`, a text nobody wrote. `_scalar('')` is the null a block `a:` reads. + + ONE THAT OPENS NOTHING HOLDS NO BRACKET OR BRACE (#301): a plain item or + value holding one is refused by `_refuse_flow_indicator`. """ - return _flow(text) if text.startswith(("[", "{")) else _scalar(text) + if text.startswith(("[", "{")): + return _flow(text) + _refuse_flow_indicator(text) + return _scalar(text) def _flow(text: str) -> Any: @@ -506,10 +537,19 @@ def _split_key_flow(text: str) -> tuple[str, str] | None: names its line, and a flow collection has none to name, so this names the entry. A QUOTED key, `"? a"`, does not open with the `?`, and `?a` is not the indicator, which needs a space after it: both read as they did. + + A PLAIN KEY HOLDS NO `]` OR `}` EITHER (#301). `_split_key` refuses a + `[` or `{` later in a key, and in a flow a closer ends the key as surely, + so `{a"]:}` read as the key `a"]` and `{]:}` as `]`, where PyYAML + refuses both. `_refuse_flow_indicator` refuses them; a quoted key is + still text. """ if text.startswith("? "): raise YamlError(f"explicit keys are not supported: {text!r}") - return _split_key(text) + kv = _split_key(text) + if kv is not None and not text.startswith(('"', "'")): + _refuse_flow_indicator(kv[0]) + return kv def _split_flow(body: str) -> list[str]: diff --git a/templates/assembly-root/scripts/validate-manifest.py b/templates/assembly-root/scripts/validate-manifest.py index 12ff6a57..6312dbf7 100755 --- a/templates/assembly-root/scripts/validate-manifest.py +++ b/templates/assembly-root/scripts/validate-manifest.py @@ -378,9 +378,32 @@ def _name_finding(manifest: dict) -> list[str]: """`name`. Split from `_findings` for #132. + + A NAME HOLDING A LONE SURROGATE IS A FINDING (#280). The double-quoted + escape `"a\\ud83db"` reads as one, as PyYAML's own Python reader reads it + (libyaml refuses the file). The reader keeps that escape on purpose: it + is how an adoption plan spells a path byte that is not UTF-8 (#199). A + display name is text a person chose, and no UTF-8 file, commit message + or console can hold a surrogate, so this file's `manifest ok` line + raised `UnicodeEncodeError` on it; `make validate` and `shape-doctor.py` + reported that traceback, while `bump-leg.py` and `update-shape.py apply` + passed the name, and the bump wrote it into its commit as `a?b`. Every + launcher now meets this finding instead. A pair of `\\u` escapes is two + surrogates here, as it is to PyYAML, so the finding asks for the + character itself or its one `\\U` escape. """ - if not isinstance(manifest.get("name"), str) or not manifest.get("name").strip(): - return [_finding("manifest-name", f"name is {manifest.get('name')!r}")] + name = manifest.get("name") + if not isinstance(name, str) or not name.strip(): + return [_finding("manifest-name", f"name is {name!r}")] + surrogate = next((char for char in name if "\ud800" <= char <= "\udfff"), + None) + if surrogate is not None: + return [_finding( + "manifest-name", + f"name is {name!r}, which holds U+{ord(surrogate):04X}, a lone " + "surrogate: no UTF-8 file, commit message or console can hold " + "it. Write each character as itself, or as one `\\U` escape " + "(`\\U0001F600`, not the pair `\\uD83D\\uDE00`).")] return [] diff --git a/tests/test_manifest_validator.py b/tests/test_manifest_validator.py index b79bc4f8..9aa2a803 100644 --- a/tests/test_manifest_validator.py +++ b/tests/test_manifest_validator.py @@ -3,6 +3,10 @@ from __future__ import annotations +import os +import subprocess +import sys + import pytest from conftest import run_script @@ -326,6 +330,73 @@ def test_a_manifest_that_is_not_utf8_refuses_rather_than_raising(project): assert "manifest ok" not in result.stdout +# --- a display name no UTF-8 text can hold (#280) --------------------------- + +def rename(project, escape: str) -> None: + """Put `name: ""` in place of the scaffolded `name: "Atlas"`, as + BYTES: `escape` is the YAML spelling, backslash and all, and the file + stays ASCII and keeps whatever line endings it has.""" + manifest = project / "project.yaml" + data = manifest.read_bytes() + assert data.count(b'\nname: "Atlas"') == 1, "fixture drift: the name line" + manifest.write_bytes(data.replace(b'\nname: "Atlas"', + f'\nname: "{escape}"'.encode("ascii"))) + + +def validate_bytes(project, **env: str): + """`(exit code, stdout, stderr)` as BYTES, so no codec of this process + reads them, with `env` set over this process's environment.""" + proc = subprocess.run( + [sys.executable, str(project / "scripts" / "validate-manifest.py")], + cwd=str(project), capture_output=True, stdin=subprocess.DEVNULL, + env={**os.environ, **env}, check=False) + return proc.returncode, proc.stdout, proc.stderr + + +@pytest.mark.parametrize("escape, code_point", [ + pytest.param("a\\ud83db", "U+D83D", id="the-issue-row-high-surrogate"), + pytest.param("caf\\udce9", "U+DCE9", id="the-byte-surrogateescape-keeps"), + pytest.param("a\\ud800b", "U+D800", id="the-first-surrogate"), + pytest.param("a\\udfffb", "U+DFFF", id="the-last-surrogate"), + pytest.param("\\ud83d\\ude00", "U+D83D", id="a-pair-is-two-surrogates"), +]) +@pytest.mark.parametrize("ioencoding", [None, "utf-8"]) +def test_a_name_holding_a_lone_surrogate_is_a_finding(project, escape, + code_point, ioencoding): + """On c7c02fa the reader took the escape, the validator accepted the + name, and its `manifest ok` line raised `UnicodeEncodeError`: exit 1 and + a traceback, under a pipe and under `PYTHONIOENCODING=utf-8` (the + doctor's) alike. It is a finding now, on stderr, with the name escaped + by `repr`, before that line is reached.""" + rename(project, escape) + env = {} if ioencoding is None else {"PYTHONIOENCODING": ioencoding} + code, stdout, stderr = validate_bytes(project, **env) + shown = (stdout + stderr).decode("utf-8", "backslashreplace") + assert code == 1, shown + assert b"Traceback" not in stderr and b"UnicodeEncodeError" not in stderr, \ + shown + assert b"manifest ok" not in stdout, shown + finding = (f"FINDING manifest-name: name is '{escape}', which holds " + f"{code_point}, a lone surrogate").encode("ascii") + assert finding in stderr, shown + assert b"1 finding(s)" in stderr, "the name is the only finding" + + +@pytest.mark.parametrize("escape", [ + pytest.param("\\U0001F600", id="an-astral-character-as-one-escape"), + pytest.param("\\u00e9t\\u00e9", id="a-bmp-character-as-an-escape"), + pytest.param("\\ue000", id="private-use-just-past-the-surrogates"), + pytest.param("\\ud7ff", id="the-last-character-before-them"), +]) +def test_a_name_spelled_without_a_surrogate_is_still_ok(project, escape): + """What the finding must not take: every other code point, spelled as an + escape, including the two either side of U+D800..U+DFFF.""" + rename(project, escape) + code, stdout, stderr = validate_bytes(project, PYTHONIOENCODING="utf-8") + assert code == 0, stderr.decode("utf-8", "backslashreplace") + assert stdout.startswith(b"manifest ok: "), stdout + + # --- the ruling: a chain of pins may reach the referent (2026-09-05) -------- # # "elect the shape for both, follow the pin chain, no family yet" — Brett Heap, diff --git a/tests/test_validator_child_encoding.py b/tests/test_validator_child_encoding.py index bdf6a6ce..2cc826c6 100644 --- a/tests/test_validator_child_encoding.py +++ b/tests/test_validator_child_encoding.py @@ -30,6 +30,10 @@ - the environment helpers themselves, each one handed to a real child that prints the name, so a handler a child does not know fails here; and a guard that no tool runs the interpreter without one. + +One name no stdout can hold at all, a lone surrogate (#280), is refused by the +validator as a finding, and the two tools, the doctor and `make validate` are +each run on it. """ from __future__ import annotations @@ -345,6 +349,110 @@ def test_an_apply_under_a_c_locale_lands(upstream_and_project, shape_root, assert b"pins ok" in stdout, stdout +# --- a display name no UTF-8 text can hold (#280) --------------------------- +# +# The escape `"a\ud83db"` reads as a string holding the lone surrogate U+D83D, +# as PyYAML's own Python reader reads it. No UTF-8 stream can hold one, so the +# validator's `manifest ok` line raised: `make validate` and the doctor met a +# traceback, while the two callers above, which hand their child +# `backslashreplace` (#254), passed the project. The validator now reports the +# name as a finding, and all four launchers stop at it. + +#: The display name as the YAML spells it, and the finding the project's own +#: validator reports for it. +SURROGATE_ESCAPE = "a\\ud83db" +SURROGATE_FINDING = (b"FINDING manifest-name: name is 'a\\ud83db', which " + b"holds U+D83D, a lone surrogate") + + +def rename_to_a_lone_surrogate(project: Path) -> str: + """Rename the display name to `SURROGATE_ESCAPE` and commit it on its own + branch; returns the sha it left HEAD at. Edited as bytes.""" + manifest = project / "project.yaml" + body = manifest.read_bytes() + line = f'\nname: "{PROJECT}"'.encode("ascii") + assert body.count(line) == 1, body + manifest.write_bytes(body.replace( + line, f'\nname: "{SURROGATE_ESCAPE}"'.encode("ascii"))) + git("checkout", "-q", "-b", WORKING_BRANCH, cwd=project) + git("-c", "user.name=t", "-c", "user.email=t@t.invalid", "commit", "-q", + "-am", "A display name holding a lone surrogate", cwd=project) + return git("rev-parse", "HEAD", cwd=project).stdout.strip() + + +def assert_stopped_at_the_finding(code: int, stdout: bytes, stderr: bytes, + exit_code: int, refusal: bytes = b"") -> None: + """The launcher exits `exit_code` with the validator's finding in its + output, `refusal` named where it has one, and no traceback anywhere.""" + both = stdout + stderr + shown = both.decode("utf-8", "backslashreplace") + assert code == exit_code, shown + assert SURROGATE_FINDING in both, shown + assert refusal in both, shown + assert b"Traceback" not in both and b"UnicodeEncodeError" not in both, \ + shown + assert b"manifest ok" not in both, shown + + +def test_a_real_bump_of_a_project_whose_name_holds_a_surrogate_is_refused( + project, scaffolded, advanced, tmp_path): + """On c7c02fa this bump exited 0: its child printed `manifest ok: + a\\ud83db`, and the commit it made read `Bump spec leg to in a?b`, + the name lost to the `replace` its message is written with. It is the + validator's finding now, and every byte is rolled back.""" + before = rename_to_a_lone_surrogate(project) + pinned = git("rev-parse", "HEAD:spec", cwd=project).stdout.strip() + code, stdout, stderr = run_in( + BUMP, *bump_arguments(project, scaffolded, advanced), + encoding="utf-8:strict", tmp_path=tmp_path) + assert_stopped_at_the_finding(code, stdout, stderr, 2, + b"bump-leg-validators-red") + assert git("rev-parse", "HEAD", cwd=project).stdout.strip() == before + assert git("rev-parse", "HEAD:spec", cwd=project).stdout.strip() == pinned + assert git("status", "--porcelain", cwd=project).stdout == "" + + +def test_an_apply_on_a_project_whose_name_holds_a_surrogate_is_refused( + upstream_and_project, shape_root, tmp_path): + """On c7c02fa `apply` exited 0 and printed `manifest ok: a\\ud83db`. The + validator it runs is the one B wrote, which carries the finding.""" + before = rename_to_a_lone_surrogate(shape_root) + copy = (shape_root / CHANGED).read_bytes() + code, stdout, stderr = run_in( + UPDATE, *update_arguments(shape_root, upstream_and_project), + encoding="utf-8:strict", tmp_path=tmp_path) + assert_stopped_at_the_finding(code, stdout, stderr, 2, + b"update-validators-red") + assert git("rev-parse", "HEAD", cwd=shape_root).stdout.strip() == before + assert (shape_root / CHANGED).read_bytes() == copy, "rolled back" + assert git("status", "--porcelain", cwd=shape_root).stdout == "" + + +def test_the_doctor_quotes_the_finding_for_a_name_holding_a_surrogate( + project, tmp_path): + """On c7c02fa the `manifest` row quoted the traceback's last line, + `UnicodeEncodeError: 'utf-8' codec can't encode character '\\ud83d'`.""" + rename_to_a_lone_surrogate(project) + code, stdout, stderr = run_in( + REPO / "shape-doctor.py", "--root", str(project), + encoding="utf-8:strict", tmp_path=tmp_path) + assert_stopped_at_the_finding(code, stdout, stderr, 1) + row = [line for line in stdout.splitlines() if SURROGATE_FINDING in line] + assert len(row) == 1 and row[0].split()[:2] == [b"FINDING", b"manifest"], \ + stdout.decode("utf-8", "backslashreplace") + assert b"INVALID (manifest)" in stdout + + +@pytest.mark.skipif(shutil.which("make") is None, reason="needs `make` on PATH") +def test_make_validate_stops_at_the_finding_for_a_surrogate(project, tmp_path): + """On c7c02fa `make validate` stopped on the validator's traceback.""" + rename_to_a_lone_surrogate(project) + proc = subprocess.run(["make", "validate"], cwd=str(project), + capture_output=True, stdin=subprocess.DEVNULL, + env=child_environ(tmp_path, None), check=False) + assert_stopped_at_the_finding(proc.returncode, proc.stdout, proc.stderr, 2) + + # --- the environment each tool builds --------------------------------------- #: What a child prints: `manifest ok: 日本`, spelled with `chr` so the program diff --git a/tests/test_yaml_flow_brackets.py b/tests/test_yaml_flow_brackets.py new file mode 100644 index 00000000..2305d0a9 --- /dev/null +++ b/tests/test_yaml_flow_brackets.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A plain scalar in a flow collection holds no `[`, `]`, `{` or `}` (#301). + +In YAML a plain scalar inside a flow collection ENDS at any of `[ ] { } ,`. +`_split_flow` splits a flow body at its top-level commas by counting brackets, +and never asked whether the count went below zero or ended above it, so a body +that closes a bracket it never opened, or opens one inside an item, was read +with the stray bracket kept as text: `pins: [a, b]]` read as the pin `b]`, +`k: [x [a, b]]` as the one item `x [a, b]`, and since #284, which reads a +quote only where a scalar begins, `k: {a"]:}` as the key `a"]`. PyYAML +refuses each. `_refuse_flow_indicator` now refuses a plain item, key or value +of a flow collection that holds one, as a `YamlError`, which `load_yaml` turns +into `REFUSED yaml-unparsable`. A QUOTED scalar's brackets are text, and a +bracket in a BLOCK plain scalar (`k: a[b]`) is a character of it, as before. + +Every refused document below is one PyYAML 6.0.3 refuses, except three: +- `k: {a:}#}: b}` and `k: {}#":}`, which PyYAML reads as `{'k': {'a': None}}` + and `{'k': {}}` only because it takes a `#` with no space before it for a + comment, which YAML forbids; the subset refuses them as it refuses + `k: "a"#c`; +- `k: [a: [b]]`, a single-pair mapping in a flow sequence, which PyYAML reads + as `[{'a': ['b']}]` and this reader read as the string `'a: [b]'`; a + refusal replaces that misread. +Every expected read is PyYAML's. PyYAML is not imported here; nothing in this +suite depends on it. + +NO NETWORK AND NO GITHUB: strings, one temporary file and an import. +""" + +from __future__ import annotations + +import sys + +import pytest + +from conftest import REPO + +sys.path.insert(0, str(REPO / "scripts")) +from repo_shape import (Refusal, YamlError, double_quoted, # noqa: E402 + load_yaml, parse_yaml) + +#: The phrase every refusal here is made with. +REFUSAL = "a plain scalar in a flow collection holds" + +#: #301's own ten documents: the stray closer in an item, in a key and in a +#: key `#284` stopped hiding behind a quote, the opener in an item, and the +#: two PyYAML reads through its `#` leniency. The text is raw. +ISSUE_ROWS = [ + pytest.param("pins: [a, b]]", id="closer-after-the-last-item"), + pytest.param('k: {a"]:}', id="closer-in-a-key-after-a-quote"), + pytest.param("k: {a]x:}", id="closer-in-a-key"), + pytest.param("k: {]:}", id="closer-as-the-key"), + pytest.param("k: [a]]", id="closer-after-the-only-item"), + pytest.param("k: [a}]", id="brace-closer-in-an-item"), + pytest.param("k: [x [a, b]]", id="opener-in-an-item"), + pytest.param("k: [a[]", id="opener-never-closed"), + pytest.param("k: {a:}#}: b}", id="pyyaml-reads-a-hash-comment"), + pytest.param('k: {}#":}', id="pyyaml-reads-an-empty-mapping"), +] + + +@pytest.mark.parametrize("text", ISSUE_ROWS) +def test_the_issue_rows_are_refused(text): + with pytest.raises(YamlError, match=REFUSAL): + parse_yaml(text + "\n") + + +#: The same defect wherever a plain flow scalar stands: in a value, nested +#: one or two levels down, in a block sequence's flow item, after another +#: entry, an item that is only a bracket, and two collections where one was +#: meant. The text is raw. +EVERYWHERE_ELSE = [ + pytest.param("k: {a: b}}", id="closer-in-a-value"), + pytest.param("k: {a: b, c: d]}", id="closer-in-a-later-value"), + pytest.param("k: {a: [b]]}", id="closer-in-a-nested-sequence"), + pytest.param("k: [[a]]]", id="closer-after-a-nested-sequence"), + pytest.param("k: [{a: b}}]", id="closer-after-a-nested-mapping"), + pytest.param("k: [a, ]]", id="an-item-that-is-a-closer"), + pytest.param("k: [a], [b]", id="two-sequences-on-one-line"), + pytest.param("k: {a: b}, {c: d}", id="two-mappings-on-one-line"), + pytest.param("- [a]]", id="block-sequence-flow-item"), + pytest.param("k:\n - {a: b}}", id="nested-block-sequence-flow-item"), + pytest.param("k: {t-'': x{}", id="a-value-opening-a-brace"), + pytest.param("k: {a: x{y}}", id="a-value-holding-braces"), + pytest.param("k: [see [1]]", id="an-item-holding-a-bracket-pair"), + pytest.param("k: {a b]: c}", id="closer-in-a-key-with-a-space"), + pytest.param("k: {x: 1, a}: 2}", id="closer-in-a-later-key"), + # PyYAML READS this one, as a single-pair mapping, `[{'a': ['b']}]`; the + # subset reads no single-pair mapping and read it as the string `a: [b]`. + pytest.param("k: [a: [b]]", id="single-pair-mapping-once-misread"), +] + + +@pytest.mark.parametrize("text", EVERYWHERE_ELSE) +def test_a_bracket_in_a_plain_flow_scalar_is_refused_wherever_it_stands(text): + with pytest.raises(YamlError, match=REFUSAL): + parse_yaml(text + "\n") + + +@pytest.mark.parametrize("text, held, scalar", [ + pytest.param("pins: [a, b]]", "]", "b]", id="item"), + pytest.param('k: {a"]:}', "]", 'a"]', id="key"), + pytest.param("k: {a: b}}", "}", "b}", id="value"), + pytest.param("k: [x [a, b]]", "[", "x [a, b]", id="opener"), +]) +def test_the_refusal_names_the_bracket_and_the_scalar(text, held, scalar): + with pytest.raises(YamlError) as caught: + parse_yaml(text + "\n") + assert str(caught.value) == (f"{REFUSAL} `{held}`, which ends it in YAML: " + f"{scalar!r}") + + +def test_a_stray_bracket_in_a_file_is_a_refusal_naming_the_file(tmp_path): + """The issue's case, read the way every caller reads one: a stray `]` + typed into a pin list. Bytes, so the file is LF on every platform.""" + path = tmp_path / "project.yaml" + path.write_bytes(b"id: atlas\nneutral_product_pins: [openGlass, openDox]]\n") + with pytest.raises(Refusal) as caught: + load_yaml(path) + assert caught.value.code == "yaml-unparsable" + assert str(path) in caught.value.detail + assert f"{REFUSAL} `]`, which ends it in YAML: 'openDox]'" in \ + caught.value.detail + + +#: What must NOT change: a bracket inside a quoted flow scalar, nested and +#: empty flow collections, a bracket in a BLOCK plain scalar, a bracket in a +#: comment after a flow, and the other characters a plain flow item may hold. +STILL_READ = [ + pytest.param('k: ["a]", \'b[\', {"c}": "d{"}]', + {"k": ["a]", "b[", {"c}": "d{"}]}, id="quoted-brackets"), + pytest.param("k: {'x]y': \"[z]\"}", {"k": {"x]y": "[z]"}}, + id="quoted-key-and-value"), + pytest.param("k: {a: [b, c], d: {e: f}}", + {"k": {"a": ["b", "c"], "d": {"e": "f"}}}, id="nested-flows"), + pytest.param("k: [[a, [b]], {c: [d, {e: f}]}]", + {"k": [["a", ["b"]], {"c": ["d", {"e": "f"}]}]}, + id="deeply-nested-flows"), + pytest.param("k: [[], {}, [ ], { }]", {"k": [[], {}, [], {}]}, + id="empty-flows"), + pytest.param("k: [ ]", {"k": []}, id="an-empty-sequence-with-a-space"), + pytest.param("k: a]", {"k": "a]"}, id="block-value-closer"), + pytest.param("k: a[b", {"k": "a[b"}, id="block-value-opener"), + pytest.param("k: a, b]", {"k": "a, b]"}, id="block-value-comma-closer"), + pytest.param("k: x {y} z", {"k": "x {y} z"}, id="block-value-braces"), + pytest.param("- see [1]", ["see [1]"], id="block-item-bracket-pair"), + pytest.param("k: [a] # ]", {"k": ["a"]}, id="closer-in-a-comment"), + pytest.param("k: {a: b} # }", {"k": {"a": "b"}}, + id="brace-in-a-comment"), + pytest.param("k: [a#b, c:d, e-f]", {"k": ["a#b", "c:d", "e-f"]}, + id="other-characters-in-plain-items"), + pytest.param("k: {a: b, c: }", {"k": {"a": "b", "c": None}}, + id="an-empty-value"), +] + + +@pytest.mark.parametrize("text, expected", STILL_READ) +def test_what_is_not_a_bracket_in_a_plain_flow_scalar_still_reads(text, + expected): + assert parse_yaml(text + "\n") == expected + + +@pytest.mark.parametrize("value", ["]", "[", "}", "{", "a]b", "x [a, b]", + "{a: b}", 'a"]', "]: x"]) +def test_the_writer_spells_a_bracket_so_a_flow_reads_it_back(value): + """`double_quoted` is the subset's writer. A value holding a bracket is + always written QUOTED, so it reads back as an item, a key and a value of + a flow collection, where the refusal never looks.""" + spelled = double_quoted(value) + assert parse_yaml(f"k: [{spelled}, b]\n") == {"k": [value, "b"]} + assert parse_yaml(f"k: {{{spelled}: {spelled}}}\n") == {"k": {value: value}} diff --git a/tests/test_yaml_quote_boundaries.py b/tests/test_yaml_quote_boundaries.py index 74ca0d1f..fa847169 100644 --- a/tests/test_yaml_quote_boundaries.py +++ b/tests/test_yaml_quote_boundaries.py @@ -28,8 +28,11 @@ (`- [a, b]`, `- {a: b}`). Every expected read below is PyYAML 6.0.3's, and every refused document is -one PyYAML refuses, except `k: "a"#c`: PyYAML reads `a`, but YAML requires a -space before a comment, and the subset refuses rather than guess. PyYAML is +one PyYAML refuses, except eight. `k: "a"#c`: PyYAML reads `a`, but YAML +requires a space before a comment, and the subset refuses rather than guess. +And seven of the eight `BRACKET_IN_A_KEY` rows, every one but the flow entry +`k: {a[b]: c}`: PyYAML reads each block key, and the subset refuses them by +design, as the paragraph above says (#301 corrected this count). PyYAML is not imported here; nothing in this suite depends on it. NO NETWORK AND NO GITHUB: strings, one temporary file and an import.