From 5f46cd302e1ff644d949c3e1e4683c8ddce62130 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Mon, 19 Oct 2026 12:01:56 +0000 Subject: [PATCH] scaffold-project.py, adopt-project.py and shape_materialize.py: one function both tools call spells a `neutral_product_pins:` entry, so a `--pin` of `0x1F`, `1_000`, `1e3`, `3dGlass` or `-x` is double-quoted from the scaffold as from adopt (#924) The scaffold wrote a pin's product bare wherever the project's reader read it back, and adopt wrote it as `y` writes the plan's `product:`, so one `--pin` wrote `- 0x1F` in a scaffolded manifest and `- "0x1F"` in an adopted one, and PyYAML read the scaffold's `0x1F` as 31 and `1_000` as 1000. `shape_materialize.neutral_pin_entry` is adopt's rule, moved: bare only for a plain scalar (`PLAIN_SCALAR_RE`, which adopt's `y` now uses as its `_PLAIN_RE`) that the reader reads back after `key: `, as a `- ` item and as the list's entry; double-quoted otherwise. Both tools import it as their `_neutral_pin_entry`. `openGlass`, `007` and `yes` are written as both tools wrote them. tests/test_neutral_pin_entry_one_spelling.py renders the issue's list through both tools and compares the lines; the scaffold's spelling test moves `0x1F`, `1_000`, `1e3`, `3dGlass`, `-x` and `--` to the quoted group. Co-Authored-By: Claude Opus 5.5 --- adopt-project.py | 55 +++--- scaffold-project.py | 37 ++-- scripts/shape_materialize.py | 59 +++++++ tests/test_neutral_pin_entry_one_spelling.py | 169 +++++++++++++++++++ tests/test_scaffold_pin_and_reuse.py | 27 ++- 5 files changed, 282 insertions(+), 65 deletions(-) create mode 100644 tests/test_neutral_pin_entry_one_spelling.py diff --git a/adopt-project.py b/adopt-project.py index b3d7554c..52c2bdd0 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -368,6 +368,12 @@ from shape_materialize import pin_source_not_top_level # noqa: E402 # One reading of an `origin`, shared with `shape-doctor.py` (#521). from repo_shape import origin_is_a_path, url_repository # noqa: E402 +# The one spelling of a `neutral_product_pins:` entry, which +# `scaffold-project.py` calls too, and the plain scalar it and `y` write a +# text bare by (#924). +from shape_materialize import ( # noqa: E402 + PLAIN_SCALAR_RE, neutral_pin_entry as _neutral_pin_entry, +) #: The keys `plan` records for each pin, in the order it writes them, and the #: value of `contracts/neutral-product-pin.yaml` each one fills. @@ -1021,9 +1027,11 @@ def _pin_digest_findings(plan: "Plan") -> tuple[list[str], list[str]]: # `tests/test_adopt_plan.py` round-trips a plan through both, which is what # keeps the writer and the reader honest about each other. -#: Ends at `\Z`, not `$`: `$` also matches before a FINAL newline, and wrote a -#: file named `Icon\n` bare, over two lines of the plan (#199). -_PLAIN_RE = re.compile(r"^[A-Za-z_.][A-Za-z0-9_./@:+-]*\Z") +#: `shape_materialize`'s plain scalar since #924, the rule `neutral_pin_entry` +#: writes a pin's product bare by, so the manifest's `neutral_product_pins:` +#: entry is bare only where `y` writes the plan's `product:` bare. It ends +#: at `\Z`, not `$` (#199). +_PLAIN_RE = PLAIN_SCALAR_RE def loader_reading(line: str): @@ -13371,18 +13379,20 @@ def _neutral_product_pins(pins) -> str: `1.5` for numbers, and its `validate-pins.py` refused the entry as `neutral-pin-entry-malformed`, so the project's first CI run was red. - EACH ENTRY IS SPELLED AS `y` SPELLS THE PLAN'S `product:`. Of the names - `--pin` takes, `y` writes bare only one that begins with a letter, `_` - or `.` and that the reader reads back as the same text, after `key: ` - and as a `- ` item alike (#181), so `openGlass` is the line it always - was, `007` is `"007"`, and `true` and `null` are `"true"` and `"null"`. - The reader is asked once more of the entry as this list holds it, as - `family.py` asks it of a member's id (#259), rather than a list of its - words, and an entry it would not read back as the same text is - double-quoted all the same. Such a name reaches here from `plan` as - well as from a plan edited by hand: `plan` records it quoted, and - `check` passes it. The list is a block placeholder, so `render` writes - it as it stands, and nothing else quotes it. + EACH ENTRY IS SPELLED AS `y` SPELLS THE PLAN'S `product:`, by + `shape_materialize.neutral_pin_entry`, the one function + `scaffold-project.py` calls too (#924). Of the names `--pin` takes, it + writes bare only one that begins with a letter, `_` or `.` (`y`'s + `_PLAIN_RE`, the same rule) and that the reader reads back as the same + text, after `key: ` and as a `- ` item alike (#181), so `openGlass` is + the line it always was, `007` is `"007"`, and `true` and `null` are + `"true"` and `"null"`. The reader is asked once more of the entry as + this list holds it, as `family.py` asks it of a member's id (#259), + rather than a list of its words, and an entry it would not read back as + the same text is double-quoted all the same. Such a name reaches here + from `plan` as well as from a plan edited by hand: `plan` records it + quoted, and `check` passes it. The list is a block placeholder, so + `render` writes it as it stands, and nothing else quotes it. """ if not pins: return "[]" @@ -13390,21 +13400,6 @@ def _neutral_product_pins(pins) -> str: for product in sorted(pins)) -def _neutral_pin_entry(product: str) -> str: - """One `neutral_product_pins:` entry: `y`'s spelling of `product` where - the project's reader reads it back as `product`, and `product` - double-quoted where it does not.""" - spelled = y(product) - try: - if parse_yaml(f"neutral_product_pins:\n - {spelled}\n") \ - == {"neutral_product_pins": [product]}: - return spelled - except YamlError: - # A spelling the reader refuses is not read back as anything. - pass - return double_quoted(product) - - def _template_values(plan: Plan, names, repositories, urls, spec_path, code_path, tracking, leg_commits, leg_digests, shape_commit, pins, naming) -> dict[str, str]: diff --git a/scaffold-project.py b/scaffold-project.py index b3af3609..6b6c56ae 100755 --- a/scaffold-project.py +++ b/scaffold-project.py @@ -141,9 +141,12 @@ from shape_materialize import ( # noqa: E402 assembly_root_paths, neutral_pin_path, shape_path_collisions, ) -# The project's own reader, and the writer of the double-quoted scalars it -# reads back, asked of each `neutral_product_pins:` entry (#516). -from repo_shape import YamlError, double_quoted, parse_yaml # noqa: E402 +# The one spelling of a `neutral_product_pins:` entry (#516), which +# `adopt-project.py execute` calls too, so the same `--pin` writes the same +# line whichever tool made the root (#924). +from shape_materialize import ( # noqa: E402 + neutral_pin_entry as _neutral_pin_entry, +) # What both runs ask before the first write, as `family.py init` asks it # (#609): each folder this run makes, each working tree it writes, and gh's # not-found answer as the one failure of `gh repo view` that means absent. @@ -1909,6 +1912,11 @@ def _neutral_product_pins(products) -> str: first CI run was red. `adopt-project.py execute` wrote the same line (#514). The list is a block placeholder, so `render` writes it as it stands, and nothing else quotes it. + + EACH ENTRY IS `shape_materialize.neutral_pin_entry`'s, the one function + `adopt-project.py execute` calls too (#924). The scaffold spelled its + own, bare wherever the reader read it back, so `0x1F`, `1_000`, `1e3`, + `3dGlass` and `-x` were bare here and double-quoted in an adopted root. """ if not products: return "[]" @@ -1916,29 +1924,6 @@ def _neutral_product_pins(products) -> str: for product in products) -def _neutral_pin_entry(product: str) -> str: - """One `neutral_product_pins:` entry: `product` bare where the project's - reader reads that entry back as `product`, and `double_quoted` (#284) - where it does not. - - THE READER IS ASKED, as `family.py` asks it of a member's id (#259), - rather than a list of the words it takes for something else, and it is - asked of the LIST ENTRY: ` - - a` is a list of its own, where `id: - a` - is the text (#715; `id: -` is refused since #393). A name it reads back - bare, `openGlass` and every other plain name, is the line it always was; - `007`, `2026`, `1.5`, `true` and `null` are double-quoted, and a - double-quoted name always reads back. - """ - try: - if parse_yaml(f"neutral_product_pins:\n - {product}\n") \ - == {"neutral_product_pins": [product]}: - return product - except YamlError: - # A spelling the reader refuses is not read back as anything. - pass - return double_quoted(product) - - def _scaffold(args) -> int: """The whole act, as the phases above, in the order they must happen. diff --git a/scripts/shape_materialize.py b/scripts/shape_materialize.py index b95a7da2..284a53ab 100644 --- a/scripts/shape_materialize.py +++ b/scripts/shape_materialize.py @@ -511,6 +511,65 @@ def neutral_pin_path(product: str) -> str: return f"contracts/{product.lower()}-pin.yaml" +#: THE PLAIN SCALAR: a text that begins with a letter, `_` or `.`, and holds +#: only letters, digits and `_./@:+-`. `adopt-project.py`'s `y` writes a +#: plan's text bare only where it is one and the reader reads it back +#: (#181), and `neutral_pin_entry` writes a pin's product bare only where +#: `y` would, so the rule is here, once, for both (#924). It ends at `\Z`, +#: not `$`: `$` also matches before a FINAL newline, and wrote a file named +#: `Icon\n` bare, over two lines of the plan (#199). +PLAIN_SCALAR_RE = re.compile(r"^[A-Za-z_.][A-Za-z0-9_./@:+-]*\Z") + + +def neutral_pin_entry(product: str) -> str: + """One `neutral_product_pins:` entry, as `scaffold-project.py` and + `adopt-project.py execute` both write it: `product` bare where it is a + plain scalar (`PLAIN_SCALAR_RE`) that the project's reader reads back as + `product`, and `double_quoted` (#284) where it is not. + + ONE SPELLING, WHICHEVER TOOL MADE THE ROOT (#924). The scaffold wrote a + product bare wherever the reader read it back, and adopt wrote it as `y` + writes the plan's `product:`, so `--pin 0x1F@` was `- 0x1F` in a + scaffolded manifest and `- "0x1F"` in an adopted one, and `1_000`, + `1e3`, `3dGlass` and `-x` the same. The project's reader reads both back + as typed, but a standard YAML reader (PyYAML's `safe_load`) took the + scaffold's `0x1F` for 31 and its `1_000` for 1000. Both tools call this + one function now, and it is adopt's rule: a name that begins with a + digit or `-` is double-quoted, as `plan` quotes it in `product:`, and + `openGlass`, `007` and `yes` are written as both tools wrote them. A + bare `yes`, `on`, `.5` or `.inf` stays bare: the project's reader reads + it back as typed, though PyYAML does not (#457's L2-14, #912). + + THE READER IS ASKED, as `family.py` asks it of a member's id (#259), + rather than a list of the words it takes for something else: in the two + places `y` asks it, after `key: ` and as a `- ` item (#181), and once + more of the LIST ENTRY, as the manifest's list holds it: + ` - - a` is a list of its own, where `id: - a` is the text (#715; + `id: -` is refused since #393). A spelling the reader refuses is read + back as nothing, so it is double-quoted too. A name it reads back bare, + `openGlass` and every other plain name, is the line it always was; + `007`, `2026`, `1.5`, `true` and `null` are double-quoted, and a + double-quoted name always reads back. + """ + if PLAIN_SCALAR_RE.match(product) and all( + _reads_back(written, read) for written, read in ( + (f"key: {product}", {"key": product}), + (f"- {product}", [product]), + (f"neutral_product_pins:\n - {product}", + {"neutral_product_pins": [product]}))): + return product + return double_quoted(product) + + +def _reads_back(written: str, read) -> bool: + """Does the project's reader read `written`, a line or two of YAML, as + `read`? A spelling it refuses is not read back as anything.""" + try: + return parse_yaml(written + "\n") == read + except YamlError: + return False + + # --------------------------------------------------------------------------- # The FAMILY root's own lists (2026-09-04) # --------------------------------------------------------------------------- diff --git a/tests/test_neutral_pin_entry_one_spelling.py b/tests/test_neutral_pin_entry_one_spelling.py new file mode 100644 index 00000000..f1fefef8 --- /dev/null +++ b/tests/test_neutral_pin_entry_one_spelling.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: Apache-2.0 +"""One spelling of a `neutral_product_pins:` entry, whichever tool made the +root (#924). + +`scaffold-project.py` wrote a pin's product bare wherever the project's own +reader read it back, and `adopt-project.py execute` wrote it as `y` writes +the plan's `product:`, double-quoted where the reader does not read that +spelling back. So one `--pin` wrote two lines. Measured at v1.1.1's +`5017830`: + + product scaffold adopt + 0x1F 0x1F "0x1F" + 1_000 1_000 "1_000" + 1e3 1e3 "1e3" + 3dGlass 3dGlass "3dGlass" + -x -x "-x" + +`openGlass`, `007` and `yes` came out the same from both. The project's +reader read every one back and no gate failed, but PyYAML's `safe_load` +read the scaffold's `0x1F` as 31 and its `1_000` as 1000. Both tools now +call `shape_materialize.neutral_pin_entry`, which is adopt's rule. + +Here one list is rendered through both tools' `_neutral_product_pins` into +the manifest template's own line, and the lines are compared with each +other and with adopt's spelling. No network, no git and no GitHub: the two +tools are loaded as modules, and nothing is run. +""" + +from __future__ import annotations + +import importlib.util +import itertools +import sys +from pathlib import Path + +import pytest + +from conftest import ADOPT, REPO, SCAFFOLD + +sys.path.insert(0, str(REPO / "scripts")) +from repo_shape import YamlError, double_quoted, parse_yaml # noqa: E402 +from shape_materialize import neutral_pin_entry, render # noqa: E402 + +try: + import yaml as pyyaml # the oracle only: nothing shipped imports it +except ImportError: # pragma: no cover - depends on the machine + pyyaml = None + +NEEDS_PYYAML = pytest.mark.skipif( + pyyaml is None, + reason="PyYAML is not installed here, so the comparison with a stock " + "reader is skipped; the project's own reader is asked by the " + "tests beside this one") + +#: The issue's names, each with the line both tools now write for it: +#: adopt's spelling, double-quoted where `y` quotes the plan's `product:`. +SPELLED = { + "0x1F": '"0x1F"', + "1_000": '"1_000"', + "1e3": '"1e3"', + "3dGlass": '"3dGlass"', + "-x": '"-x"', + "openGlass": "openGlass", + "007": '"007"', + "yes": "yes", +} +#: Text `y`'s rule is asked of: every text of one or two characters from +#: the characters YAML gives a meaning to, beside letters and digits. +YAML_ALPHABET = "aZ09_.-:@/+ #'\"&*!?[{%|>," +TEXTS = tuple("".join(chars) for size in (1, 2) + for chars in itertools.product(YAML_ALPHABET, repeat=size)) + + +def load(path: Path, name: str): + """`path` loaded as a module called `name`, so its functions are asked + here and nothing it does at its entry point runs.""" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def scaffolder(): + """`scaffold-project.py`, to ask how it lists a manifest's pins.""" + return load(SCAFFOLD, "scaffold_one_pin_spelling") + + +@pytest.fixture(scope="module") +def adopter(): + """`adopt-project.py`, to ask how it lists a manifest's pins.""" + return load(ADOPT, "adopt_one_pin_spelling") + + +def rendered(entries: str) -> str: + """`entries`, a `NEUTRAL_PRODUCT_PINS` value, in the manifest template's + own `neutral_product_pins:` line, rendered by `render` as both tools + render it.""" + template = REPO / "templates" / "assembly-root" / "project.yaml" + line, = (line for line in template.read_text(encoding="utf-8") + .splitlines() if "{{NEUTRAL_PRODUCT_PINS}}" in line) + return render(f"{line}\n", {"NEUTRAL_PRODUCT_PINS": entries}, + "project.yaml") + + +def test_one_list_renders_the_same_lines_through_both_tools(scaffolder, + adopter): + """The issue's names as one list, through the scaffold (in the order + given, here adopt's sorted order) and through adopt: the same lines, + each adopt's spelling, and the project's reader reads each back as the + name typed.""" + names = sorted(SPELLED) + scaffolded = rendered(scaffolder._neutral_product_pins(names)) + adopted = rendered(adopter._neutral_product_pins(set(names))) + assert scaffolded == adopted + assert scaffolded.splitlines()[1:] == [f" - {SPELLED[name]}" + for name in names] + assert parse_yaml(scaffolded)["neutral_product_pins"] == names + + +def test_both_tools_call_the_one_function(scaffolder, adopter): + """Each tool's `_neutral_pin_entry` is `shape_materialize`'s + `neutral_pin_entry`, not a copy of it, so the two cannot drift apart + again.""" + shared = (REPO / "scripts" / "shape_materialize.py").resolve() + for tool in (scaffolder, adopter): + entry = tool._neutral_pin_entry + assert (entry.__module__, entry.__qualname__) == ( + "shape_materialize", "neutral_pin_entry"), tool.__name__ + assert Path(entry.__code__.co_filename).resolve() == shared + + +def adopts_old_entry(adopter, text: str) -> str: + """How `adopt-project.py`'s own `_neutral_pin_entry` spelled `text` until + #924: `y`'s spelling where the entry reads back as `text`, and `text` + double-quoted where it does not.""" + spelled = adopter.y(text) + try: + if parse_yaml(f"neutral_product_pins:\n - {spelled}\n") \ + == {"neutral_product_pins": [text]}: + return spelled + except YamlError: + pass + return double_quoted(text) + + +def test_an_entry_is_bare_exactly_where_y_writes_the_plan_bare(adopter): + """The shared function is adopt's rule, moved: for every short text + YAML gives a meaning to, and the issue's names, it writes what adopt's + own function wrote, which is `y`'s spelling of the plan's `product:` + wherever the entry reads back. A change to `y`'s rule that left the + entry's behind, or the other way round, fails here.""" + differ = {text: (neutral_pin_entry(text), adopts_old_entry(adopter, text)) + for text in (*TEXTS, *SPELLED) + if neutral_pin_entry(text) != adopts_old_entry(adopter, text)} + assert differ == {} + + +@NEEDS_PYYAML +def test_a_stock_reader_reads_a_name_that_opens_with_a_digit_as_text( + scaffolder): + """The point of #924: PyYAML reads the scaffold's `0x1F` and `1_000` as + the text typed, as it reads adopt's. A bare `yes` it still reads as + True, from both tools alike: the project's own reader reads it back, + and whether to quote it is #457's L2-14, on #912.""" + names = [name for name in sorted(SPELLED) if not name[0].isalpha()] + entries = scaffolder._neutral_product_pins(names) + assert pyyaml.safe_load(rendered(entries))["neutral_product_pins"] \ + == names diff --git a/tests/test_scaffold_pin_and_reuse.py b/tests/test_scaffold_pin_and_reuse.py index b4bcbb10..cba0d6eb 100644 --- a/tests/test_scaffold_pin_and_reuse.py +++ b/tests/test_scaffold_pin_and_reuse.py @@ -761,12 +761,18 @@ def scaffolder(): "Null", "NULL"), "list": ("-",), } -#: Names it reads back bare as the same text, so each is written bare, as -#: main wrote it: the plain names, and those that only look like a number -#: or a word to a reader other than the project's. +#: Names that begin with a letter, `_` or `.` and that it reads back bare as +#: the same text, so each is written bare, as main wrote it: the plain +#: names, and those that only look like a number or a word to a reader +#: other than the project's. READ_BARE = ("openGlass", "Openglass", "open-glass", "open_glass", - "openGlass2", "_x", "x.y", ".5", "3dGlass", "-x", "--", "0x1F", - "1e3", "1_000", ".inf", "yes", "off") + "openGlass2", "_x", "x.y", ".5", ".inf", "yes", "off") +#: Names it reads back bare as the same text, but that begin with a digit +#: or `-`, so `adopt-project.py`'s `y` quotes them in a plan's `product:`. +#: The scaffold wrote them bare until #924, and a standard YAML reader took +#: its `0x1F` for 31 and its `1_000` for 1000; both tools now spell an entry +#: by `shape_materialize.neutral_pin_entry`, which double-quotes them. +QUOTED_BY_Y = ("3dGlass", "-x", "--", "0x1F", "1e3", "1_000") #: Text the reader refuses outright bare. `--pin` admits none of it, and #: each is double-quoted all the same, which the reader reads back. REFUSED_BARE = ("[", "{", "&a", "*a", "!t", "'a", '"a') @@ -788,11 +794,14 @@ def listed(entries: str) -> object: *(pytest.param(name, id=f"{kind}-{name}") for kind, names in MISREAD_BARE.items() for name in names), *(pytest.param(name, id=f"bare-{name}") for name in READ_BARE), + *(pytest.param(name, id=f"quoted-by-y-{name}") for name in QUOTED_BY_Y), ]) def test_each_entry_reads_back_as_the_product(scaffolder, product): - """Every product reads back as its text. A name the reader reads back - bare is written bare, so a root's `openGlass` line does not move by a - byte; every other is double-quoted, which the reader always reads back.""" + """Every product reads back as its text. A name that begins with a + letter, `_` or `.` and that the reader reads back bare is written bare, + so a root's `openGlass` line does not move by a byte; every other is + double-quoted, as `adopt-project.py` writes it (#924), which the reader + always reads back.""" entries = scaffolder._neutral_product_pins([product]) assert listed(entries) == [product] spelled = product if product in READ_BARE else double_quoted(product) @@ -803,7 +812,7 @@ def test_every_entry_together_reads_back_in_the_order_given(scaffolder): """All of them in one list, as many `--pin`s write it: one line each, in the order the pins were given, as the scaffold has always listed them.""" every = [*(name for names in MISREAD_BARE.values() for name in names), - *READ_BARE] + *READ_BARE, *QUOTED_BY_Y] entries = scaffolder._neutral_product_pins(every) assert entries.count("\n") == len(every) assert listed(entries) == every