diff --git a/scaffold-project.py b/scaffold-project.py index 7a489bee..a903aef9 100755 --- a/scaffold-project.py +++ b/scaffold-project.py @@ -73,6 +73,9 @@ COMMIT_RE, NEUTRAL_PRODUCT_OWNER, PROJECT_ID_RE, TREE_DIGEST_DEFINITION, VISIBILITY_CHOICES, NamingPolicy, Refusal, accepts_role, checked_value, git_out, link_pins_from_trees, repo_basename, tree_digest, + # Which `--pin` products share a `SHAPE_PIN_SOURCE_` variable: the + # one answer, which the project's own `validate-pins.py` reports on (#275). + PIN_SOURCE_ENV_PREFIX, pin_source_env_collisions, ) from shape_materialize import ( # noqa: E402 RULESET_HINT, SHAPE_REPOSITORY, CommandFailed, @@ -513,6 +516,19 @@ def _exists_remedy(role: str, empty: bool, reuse_flag: bool) -> str: "each product once, by the name of the repository it pins." ) +#: What a refused pair of `--pin`s whose products share a source variable is +#: told to do (#275), after what `PIN_COLLISION_REMEDIATION` says when a file +#: collides too. +PIN_VARIABLE_REMEDIATION = ( + f"`{PIN_SOURCE_ENV_PREFIX}` names the local checkout of a " + "product, and is its name upper-cased with every character outside `A-Z`, " + "`0-9` and `_` turned into `_`, so names that differ only in `-`, `.` or " + "`_` share one. The project's own `validate-pins.py` reports such a pair " + "as `neutral-pin-source-collides`, and a variable two pins share is read " + "for neither. Pass one product of each such group, or products whose " + "names differ by more." +) + def _refuse_colliding_pins(pins, owner: str) -> None: """No `--pin` is written where the shape writes a file of its own, and no @@ -553,6 +569,13 @@ def _refuse_colliding_pins(pins, owner: str) -> None: contracts` was told it contains that file "and 4 more" where there were three (#197's re-review, N1). Refused first, no pin path it lists is the shape's or listed twice. + + TWO PINS THAT SHARE A SOURCE VARIABLE ARE REFUSED HERE TOO (#275), in the + same refusal and after the file collisions, by `_colliding_pin_variables`: + `--pin open-glass@ --pin open_glass@` are two files and one + `SHAPE_PIN_SOURCE_OPEN_GLASS`. The refusal is the one the project's own + `validate-pins.py` would make as a finding, said before three repositories + exist. """ # Folded on both sides, as `path_overlap` folds the leg paths: a future # mixed-case shape file under `contracts/` is still one path with a @@ -560,10 +583,12 @@ def _refuse_colliding_pins(pins, owner: str) -> None: shape_files = {shape_file.casefold() for shape_file in assembly_root_paths()} written: dict[str, list[tuple[str, str]]] = {} + typed_pins: list[tuple[str, str]] = [] for raw in pins: product = parse_pin(raw, owner)[0] path = neutral_pin_path(product) written.setdefault(path, []).append((raw, product)) + typed_pins.append((raw, product)) collisions = [] for path, typed in written.items(): in_shape = path.casefold() in shape_files @@ -583,9 +608,52 @@ def _refuse_colliding_pins(pins, owner: str) -> None: collisions.append( f"{said} at one file, `{path}`, which holds one pin: pass " "each product once.") - if collisions: - raise Refusal("scaffold-pin-collides", " ".join(collisions), - PIN_COLLISION_REMEDIATION) + shared = _colliding_pin_variables(typed_pins) + if collisions or shared: + raise Refusal("scaffold-pin-collides", " ".join(collisions + shared), + _pin_collision_remediation(bool(collisions), + bool(shared))) + + +def _colliding_pin_variables(typed_pins) -> list[str]: + """One sentence for each `SHAPE_PIN_SOURCE_` variable that two or + more of the products `--pin` names share, each argument as it was typed, + in the order each variable's first `--pin` was (#275). + + `open-glass`, `open_glass` and `open.glass` are three pin files and three + legal `--pin` names, and one variable: whichever checkout it names answers + for all of them, and the project's own `validate-pins.py` reported the + pin it was not a checkout of as `neutral-pin-source-unreadable`. The + question is `repo_shape.pin_source_env_collisions`'s, so it is the one + `validate-pins.py` answers, and it takes products as their pin files do: + the same product typed twice is #241's `one file`, said above, and is + never a collision here. + """ + typed = {} + for raw, product in typed_pins: + typed.setdefault(product.casefold(), []).append(raw) + out = [] + for variable, products in pin_source_env_collisions( + product for _, product in typed_pins).items(): + arguments = [f"--pin {raw!r}" for product in products + for raw in typed[product.casefold()]] + out.append( + " and ".join(arguments) + " name the products " + + " and ".join(f"`{product}`" for product in products) + + f", and one variable, `{variable}`, names the local checkout of " + "each: it cannot say which it holds.") + return out + + +def _pin_collision_remediation(file_collision: bool, + variable_collision: bool) -> str: + """The remediation for the collisions a refusal names: #241's alone for a + file, the variable's alone, or both said once each.""" + if not variable_collision: + return PIN_COLLISION_REMEDIATION + if not file_collision: + return f"Remediation: nothing was created. {PIN_VARIABLE_REMEDIATION}" + return f"{PIN_COLLISION_REMEDIATION} {PIN_VARIABLE_REMEDIATION}" def _leg_collision_remediation(neutral_pins=()) -> str: diff --git a/scripts/repo_shape.py b/scripts/repo_shape.py index 55fa5228..82346bec 100644 --- a/scripts/repo_shape.py +++ b/scripts/repo_shape.py @@ -1175,6 +1175,16 @@ def recorded_gitlink(repo: Path, path: str) -> str | None: #: ONE variable answers both questions asked of such a checkout: the digest #: `validate-pins.py` recomputes from it, and the chain-link declaration #: `resolve_referent` verifies against it. +#: +#: THE NAME IS NOT INJECTIVE (#275). `.`, `-` and `_` are all legal in a +#: product's name and all become `_`, so `open-glass`, `open_glass` and +#: `open.glass` share `SHAPE_PIN_SOURCE_OPEN_GLASS` while naming three pin +#: files. Every existing name is kept, because an operator may have set it and +#: the docs name it. What is done instead (`pin_source_env_collisions`): a +#: variable two declared products share answers for NEITHER, wherever it is +#: read; `scaffold-project.py --pin` refuses to create such a pair; and +#: `validate-pins.py` NOTES a pair a project already declares, and never fails +#: it, because the project did nothing wrong that an update could undo. PIN_SOURCE_ENV_PREFIX = "SHAPE_PIN_SOURCE_" @@ -1182,6 +1192,68 @@ def pin_source_env_name(product: str) -> str: return PIN_SOURCE_ENV_PREFIX + re.sub(r"[^A-Z0-9_]", "_", product.upper()) +def pin_product_key(product: str) -> str: + """What makes two declarations ONE product: the name its pin file carries, + `contracts/-pin.yaml` (`neutral_pin_path` in + `shape_materialize.py` writes it; `validate-pins.py` reads it, and spells + it again because it ships into projects). + + `lower()`, and not `casefold()`. The two agree over ASCII and part + company outside it: `casefold()` folds `ß` to `ss`, `ſ` to `s`, `fi` to + `fi` and `ς` to `σ`, which `lower()` leaves alone, so `openß` and `openss` + are two pin files that a casefold took for one product (#275's re-review, + N2). Whether two names are the same pin is whether they are the same + file, so that is what is asked here, and + `tests/test_pin_source_env_collisions.py` holds it to `neutral_pin_path`. + """ + return product.lower() + + +def pin_source_env_collisions(products) -> dict[str, list[str]]: + """The variables two or more DISTINCT products share, each with them: + `{variable: [product, ...]}` (#275). + + PURE, and the one place the question is answered, so `scaffold-project.py`, + `validate-pins.py` and the chain readers below cannot disagree about it. + `products` is any iterable of product NAMES (`openGlass`, never + `opensoft/openGlass@`): an entry that is not a non-empty string is + not a name and is left out, as `validate-pins.py` reports it as + malformed on its own. + + DISTINCT MEANS DISTINCT PIN FILES (`pin_product_key`): `openGlass` and + `OpenGlass` are one product and name one variable rightly, and whether the + same product is declared twice is a different question, which + `scaffold-project.py` answers as `scaffold-pin-collides` (#241). A product + listed twice is therefore never a collision here, and two that differ in + more than the case `lower()` folds are, when `pin_source_env_name` maps + them to one name. That includes names no ASCII `--pin` can spell, such as + `openß` and `openss`, which share `SHAPE_PIN_SOURCE_OPENSS` and have two + pin files. + + Variables are in the order their first product was given, and each + group's products are in that order too, each in the spelling it was first + given in. A variable only one product names is not in the result. + """ + by_variable: dict[str, dict[str, str]] = {} + for product in products or (): + if not isinstance(product, str) or not product: + continue + group = by_variable.setdefault(pin_source_env_name(product), {}) + group.setdefault(pin_product_key(product), product) + return {variable: list(group.values()) + for variable, group in by_variable.items() if len(group) > 1} + + +def pin_source_env_shared(products) -> set[str]: + """The `pin_product_key` of each of `products` whose variable another of + them shares (#275): the products a variable must not answer for, because + it does not say which. A caller asks `pin_product_key(name) in` the + result.""" + return {pin_product_key(product) + for group in pin_source_env_collisions(products).values() + for product in group} + + def form_id(family_id: str, role_id: str | None) -> str: """`("project-leg", "assembly")` -> `"project-leg/assembly"`.""" return f"{family_id}/{role_id}" if role_id else family_id @@ -1335,15 +1407,20 @@ def _chain_link_findings(chain, link_pins) -> tuple: available = _link_declarations(link_pins) warnings: list[str] = [] unverified: list[str] = [] + # A link whose variable another link of the chain shares is never offered + # that variable: it would be read for both (#275). + shared = pin_source_env_shared(chain) for holder, held in zip(chain, chain[1:]): declared = available.get(holder.casefold()) if declared is None: unverified.append(holder) + alone = pin_product_key(holder) not in shared + offered = (f"a sibling checkout, {pin_source_env_name(holder)}, or" + if alone else "a sibling checkout or") warnings.append( f"{CHAIN_UNVERIFIED}: {holder} declaring a pin on {held} " f"is a fact in {holder}'s own tree, which is not " - f"reachable here (a sibling checkout, " - f"{pin_source_env_name(holder)}, or --link-source " + f"reachable here ({offered} --link-source " f"{holder}= would let it be read). The recorded " "chain still classifies.") continue @@ -2511,20 +2588,26 @@ def declared_neutral_pins(manifest_path, def resolve_link_source(product: str, root: Path | None = None, keyed_sources: dict | None = None, - default_source=None) -> tuple[Path | None, str | None]: + default_source=None, + env_shared: bool = False + ) -> tuple[Path | None, str | None]: """Where `product`'s own manifest can be read, if anywhere at all. 1. `--link-source =` (case-insensitive on the name) 2. `SHAPE_PIN_SOURCE_` in the environment 3. a checkout sitting BESIDE this project (`../openXdox`) 4. a bare `--link-source ` with no `product=` prefix + + `env_shared` says another product asked about alongside this one names + the same variable, so rule 2 is skipped: a variable that does not say + which product it holds is not read for either (#275). """ candidates: list[tuple[Path, str]] = [] keyed = (keyed_sources or {}).get(product.casefold()) if keyed is not None: candidates.append((Path(keyed), f"--link-source {product}=...")) env_name = pin_source_env_name(product) - env_value = os.environ.get(env_name) + env_value = None if env_shared else os.environ.get(env_name) if env_value: candidates.append((Path(env_value), f"${env_name}")) if root is not None: @@ -2546,12 +2629,22 @@ def link_pins_from_trees(links, root: Path | None = None, A link whose tree is not on the disk is simply ABSENT from the mapping, which is what `NamingPolicy.resolve_referent` reads as declared-unverified. + + A link whose `SHAPE_PIN_SOURCE_` another of `links` names too is + not read through it (#275): one tree would answer for two links, and a + chain verified against the wrong tree is worse than one left + declared-unverified. Only a hand-written chain can hold two such links: + `scaffold-project.py` takes every link but the first from the naming + policy's `open` form, which has no `-`, `.` or `_` to fold. """ found: dict[str, set] = {} + links = list(links) + shared = pin_source_env_shared(repo_basename(str(link)) for link in links) for link in links: name = repo_basename(str(link)) path, _how = resolve_link_source(name, root, keyed_sources, - default_source) + default_source, + pin_product_key(name) in shared) if path is None: continue pins = declared_neutral_pins(path, key) diff --git a/templates/assembly-root/scripts/validate-pins.py b/templates/assembly-root/scripts/validate-pins.py index b32b0755..5f83eb75 100755 --- a/templates/assembly-root/scripts/validate-pins.py +++ b/templates/assembly-root/scripts/validate-pins.py @@ -68,8 +68,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) from repo_shape import ( # noqa: E402 COMMIT_RE, SHA256_RE, TREE_DIGEST_DEFINITION, Refusal, file_sha256, - find_repo_root, git_out, load_yaml, pin_source_env_name, recorded_gitlink, - repo_basename, tree_digest, tree_digest_from_gh, + find_repo_root, git_out, load_yaml, pin_product_key, + pin_source_env_collisions, pin_source_env_name, pin_source_env_shared, + recorded_gitlink, repo_basename, tree_digest, tree_digest_from_gh, ) # `PIN_SOURCE_ENV_PREFIX` / `pin_source_env_name` live in `repo_shape` because @@ -109,13 +110,18 @@ def skip(self, text: str) -> None: nonzero exit. A network absence is not evidence of anything.""" self.notes.append(f" skip {text}") - def unused(self, text: str) -> None: - """A `--pin-source` no declared pin read (#295) — printed beside the - skips and, like them, never a FINDING and never the reason for a - nonzero exit: the person named a source that did not answer, and - nothing about any pin is wrong for that.""" + def advise(self, text: str) -> None: + """A NOTE: something a person should know that is no fault of the + project's — printed beside the skips and, like them, never a FINDING + and never the reason for a nonzero exit.""" self.notes.append(f" note {text}") + def unused(self, text: str) -> None: + """A `--pin-source` no declared pin read (#295): the person named a + source that did not answer, and nothing about any pin is wrong for + that. A note, as every one of those is.""" + self.advise(text) + class _Unreadable(NamedTuple): """One workflow file that could not be read (#359): the line that names it, @@ -590,6 +596,7 @@ def read_as_path(self, typed: str | None) -> str: def resolve_neutral_pin_source(root: Path, product: str, repository: str, sources: PinSources, + env_shared: bool = False, ) -> tuple[Path | None, str | None, str | None]: """Where to read `product`'s tree OFFLINE, if anywhere at all. @@ -606,6 +613,12 @@ def resolve_neutral_pin_source(root: Path, product: str, repository: str, to whichever pin does not already have a more specific answer — the common case is exactly one declared neutral-product pin + Rule 2 is skipped when `env_shared` is true, which is when another + declared pin names the same variable (#275): a variable that cannot say + which product it holds answers for neither, and `neutral-pin-source- + collides` is noted for the pair. The pin still verifies through rule 1, + 3 or 4. + Returns `(path, how, typed)`, `typed` being the `--pin-source` value as given when one of those answered, so the caller can mark it used (#295). `(None, None, None)` when nothing above answers; the caller's next step @@ -616,7 +629,7 @@ def resolve_neutral_pin_source(root: Path, product: str, repository: str, typed, path = keyed return path, f"--pin-source {product}=...", typed env_name = pin_source_env_name(product) - env_value = os.environ.get(env_name) + env_value = None if env_shared else os.environ.get(env_name) if env_value: return Path(env_value), f"${env_name}", None sibling = root.parent / repo_basename(repository) @@ -759,7 +772,9 @@ def _also_a_directory(typed: str | None, sources: PinSources) -> str: def _recompute_neutral_pin_digest(root: Path, product: str, repository: str, commit: str, rel: str, sources: PinSources, - report: Report) -> tuple[str, str] | None: + report: Report, + shared: frozenset[str] = frozenset() + ) -> tuple[str, str] | None: """Recompute the referent's tree digest: offline where a source can be found, else from `gh api`, else a named SKIP that never fails the run. @@ -768,8 +783,9 @@ def _recompute_neutral_pin_digest(root: Path, product: str, repository: str, already reported in that case, exactly as the original loop's `continue` left it. """ + env_shared = pin_product_key(product) in shared source, how, typed = resolve_neutral_pin_source(root, product, repository, - sources) + sources, env_shared) if source is not None: if typed is not None: sources.used.add(typed) @@ -791,18 +807,26 @@ def _recompute_neutral_pin_digest(root: Path, product: str, repository: str, try: return tree_digest_from_gh(repository, commit), "gh-api-tree-recursive" except Refusal as exc: + # A variable another pin shares is not offered, and neither is the + # bare `--pin-source`, which answers for every pin that has no more + # specific answer: the keyed form is the one that says which pin it + # is for (#275). + offered = (f"A --pin-source {product}= or" if env_shared else + f"A --pin-source, {pin_source_env_name(product)}, or") report.skip( f"{product}: pin commit {commit[:12]} digest NOT " f"rechecked — no local checkout ({exc.code}: " - f"{exc.detail}). A --pin-source, {pin_source_env_name(product)}, " - f"or a checkout at ../{repo_basename(repository)} beside " + f"{exc.detail}). {offered} a checkout at " + f"../{repo_basename(repository)} beside " "this assembly root would verify it offline.", ) return None def _check_one_neutral_product_pin(root: Path, product, report: Report, - sources: PinSources) -> None: + sources: PinSources, + shared: frozenset[str] = frozenset() + ) -> None: """One `neutral_product_pins:` entry, start to finish. Split from `_check_neutral_product_pins` for #132 so the loop in that @@ -819,7 +843,8 @@ def _check_one_neutral_product_pin(root: Path, product, report: Report, return commit, recorded = fields recomputed = _recompute_neutral_pin_digest(root, product, repository, - commit, rel, sources, report) + commit, rel, sources, report, + shared) if recomputed is None: return actual, how = recomputed @@ -837,6 +862,36 @@ def _check_one_neutral_product_pin(root: Path, product, report: Report, f"recomputes ({how})") +def _report_pin_source_collisions(products, report: Report) -> None: + """One `neutral-pin-source-collides` NOTE for each variable two or more + declared products share (#275), naming every one of them and the + variable. It is a note and never a finding: such a project did nothing + the standard forbade when it was made (`scaffold-project.py` has refused + it only since #275), every pin in it still verifies through the keyed + `--pin-source =` or a checkout beside this assembly root, + and a finding would make `update-shape.py apply` refuse every shape + update (`update-validators-red`) until a person dropped a true + declaration of descent. Which declaration stays is that person's + decision, so the note says so and nothing here asks for one to go. + `pin_source_env_collisions` is `repo_shape`'s, the one answer + `scaffold-project.py` refuses on too: products are distinct by their pin + files, so a product listed twice is not a collision here. The names are + quoted with `repr`, as `neutral-pin-entry-malformed` quotes an entry: they + are the manifest's own text and may hold a newline, and a note is one + line.""" + for variable, group in pin_source_env_collisions(products).items(): + names = " and ".join(repr(product) for product in group) + report.advise( + "neutral-pin-source-collides: project.yaml neutral_product_pins " + f"declares {names}, and one variable, {variable}, names the local " + "checkout of each: it cannot say which it holds, so it is read " + "for none of them. Each pin still verifies from the keyed " + "`--pin-source =` or from a checkout beside this " + "assembly root; a bare `--pin-source` answers for every pin that " + "has no more specific source. Which declaration stays is the " + "human's decision, and nothing here fails on it.") + + def _check_neutral_product_pins(root: Path, manifest: dict, report: Report, pin_source_values: list[str]) -> None: """Re-check every `neutral_product_pins:` entry `project.yaml` declares. @@ -849,6 +904,18 @@ def _check_neutral_product_pins(root: Path, manifest: dict, report: Report, available is a SKIP, never a finding and never a refusal: the absence of a way to answer is not evidence that the pin is wrong. + TWO PRODUCTS THAT SHARE ONE VARIABLE ARE A NOTE (#275), and neither + reads it. `SHAPE_PIN_SOURCE_` is the product's name upper-cased + with every character outside `[A-Z0-9_]` turned into `_`, so `open-glass`, + `open_glass` and `open.glass` are three pin files and one variable. + Whichever checkout it named answered for all of them, and the pin it was + not a checkout of was reported as `neutral-pin-source-unreadable`: drift + in the project, said about a variable. Now no pin reads it, each pin + verifies from its own source, and `neutral-pin-source-collides` says so + and goes no further: it is not a finding, because a project that already + declares such a pair could not otherwise take a shape update, and which + declaration to drop is a human's to decide. + The `--pin-source` values are READ here rather than in `main`, because whether `a=b/openGlass` is keyed to a product or a bare path depends on the pins this manifest declares (#295); each one no pin used is noted @@ -856,8 +923,10 @@ def _check_neutral_product_pins(root: Path, manifest: dict, report: Report, """ products = manifest.get("neutral_product_pins") or [] sources = PinSources(pin_source_values, products) + _report_pin_source_collisions(products, report) + shared = frozenset(pin_source_env_shared(products)) for product in products: - _check_one_neutral_product_pin(root, product, report, sources) + _check_one_neutral_product_pin(root, product, report, sources, shared) _note_unused_pin_sources(sources, report) diff --git a/tests/test_pin_source_env_collisions.py b/tests/test_pin_source_env_collisions.py new file mode 100644 index 00000000..52207870 --- /dev/null +++ b/tests/test_pin_source_env_collisions.py @@ -0,0 +1,987 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Two neutral products must not share one `SHAPE_PIN_SOURCE_` (#275). + +`pin_source_env_name` turns every character outside `[A-Z0-9_]` into `_`, so +`open-glass`, `open_glass` and `open.glass` are three legal product names, with +three pin files (`contracts/-pin.yaml`), and ONE variable, +`SHAPE_PIN_SOURCE_OPEN_GLASS`. A project that pinned two of them got whichever +checkout the variable named for BOTH, and a `neutral-pin-source-unreadable` +FINDING (exit 1, a red `validate.yml`) for the one it was not a checkout of; +with no source given the same variable was offered as the way to verify either. + +Every existing variable name is kept: an operator may have set one and the docs +name them. What is done instead, in each place that declares or reads pins: + +- `scaffold-project.py --pin` REFUSES a set of products whose variables + collide, beside #241's file collisions, dry run and real run alike, before + any process starts: no new project is made with a variable it cannot use; +- the project's own `validate-pins.py` NOTES a pair a project already declares + (`neutral-pin-source-collides`) and never fails it, after which neither + product reads the variable and each is verified from its own source. It was + a finding in the first round, and a project declaring such a pair could not + take `update-shape.py apply` (`update-validators-red`) until a person dropped + a true declaration; the second round made it a note, and + `test_a_project_declaring_a_colliding_pair_takes_a_shape_update` holds that; +- the chain readers in `repo_shape.py` never read a variable two links of one + chain share and never offer it. + +Two declarations are one product when they are one PIN FILE (`pin_product_key`, +`lower()`), and not when `casefold()` says so: `openß` and `openss` share +`SHAPE_PIN_SOURCE_OPENSS` and have two files. + +`adopt-project.py --pin` is not here: its half arrives with #273's change, +which routes adopt through the one function this file pins. + +The tests that call `main()` refuse every process start, which shows the +refusal comes before `git` and `gh`. The validator runs against a copy of the +session's scaffolded project, whose own `validate-pins.py` and `repo_shape.py` +are the copies under test, with a `gh` that exits 97 and records its arguments +where the run would otherwise have reached the forge. +""" + +from __future__ import annotations + +import itertools +import json +import os +import shutil +import stat +import subprocess +import sys +import textwrap +import types +from pathlib import Path + +import pytest + +from conftest import ( + REPO, SCAFFOLD, WINDOWS_SKIP, clear_ambient_pin_sources, git, run_script, +) +from test_scaffold_pin_and_reuse import product_repo +from test_update_shape import ( + PROJECT, commit_upstream_changes, scaffold_upstream_and_project, verdicts, +) +# Fixtures and constants of #241's file, shared rather than re-made: the +# stand-in `openGlass` with two commits, the module-level `scaffold`, the +# refusal of every process start, and the remote and `gh` world. +from test_scaffold_pin_collisions import ( # noqa: F401 + BASE_ARGS, LEG_REFUSED, ONE_FILE, REFUSED, REMEDIATION, SHAPE_WRITES, + glass, listing, nothing_was_created, no_process, pin_args, pins_of, + scaffold, world, +) + +sys.path.insert(0, str(REPO / "scripts")) +import repo_shape # noqa: E402 +from repo_shape import NamingPolicy, Refusal, tree_digest # noqa: E402 +from shape_materialize import neutral_pin_path # noqa: E402 + +UTF8 = "utf-8" +POLICY_PATH = REPO / "contracts" / "repository-naming.yaml" +VALIDATE_PINS = (REPO / "templates" / "assembly-root" / "scripts" + / "validate-pins.py") + +#: The two refusals' remediation for the variable, word for word. Alone it +#: stands after `Remediation: nothing was created.`, and beside a file +#: collision it follows #241's remediation after a space. +VARIABLE_REMEDIATION = ( + "`SHAPE_PIN_SOURCE_` names the local checkout of a product, and " + "is its name upper-cased with every character outside `A-Z`, `0-9` and " + "`_` turned into `_`, so names that differ only in `-`, `.` or `_` share " + "one. The project's own `validate-pins.py` reports such a pair as " + "`neutral-pin-source-collides`, and a variable two pins share is read " + "for neither. Pass one product of each such group, or products whose " + "names differ by more.") +ALONE = f"Remediation: nothing was created. {VARIABLE_REMEDIATION}" +BESIDE_A_FILE = f"{REMEDIATION} {VARIABLE_REMEDIATION}" + + +def shared(variable: str) -> str: + """What the refusal says after it has named the products.""" + return (f", and one variable, `{variable}`, names the local checkout of " + "each: it cannot say which it holds.") + + +# --- the one function, and the names it must not change ----------------------- + +@pytest.mark.parametrize("product, variable", [ + pytest.param("openGlass", "SHAPE_PIN_SOURCE_OPENGLASS", id="camel"), + pytest.param("open-glass", "SHAPE_PIN_SOURCE_OPEN_GLASS", id="hyphen"), + pytest.param("open_glass", "SHAPE_PIN_SOURCE_OPEN_GLASS", id="underscore"), + pytest.param("open.glass", "SHAPE_PIN_SOURCE_OPEN_GLASS", id="dot"), + pytest.param("OpenGlass", "SHAPE_PIN_SOURCE_OPENGLASS", id="capital"), + pytest.param("openXdox", "SHAPE_PIN_SOURCE_OPENXDOX", id="openXdox"), +]) +def test_every_existing_variable_name_is_kept(product, variable): + """The refusal is the fix, so no name an operator may have set moves.""" + assert repo_shape.pin_source_env_name(product) == variable + + +def test_the_variable_two_products_share_is_found_with_all_of_them(): + found = repo_shape.pin_source_env_collisions( + ["open-glass", "openGlass", "open_glass", "open.glass"]) + assert found == { + "SHAPE_PIN_SOURCE_OPEN_GLASS": ["open-glass", "open_glass", + "open.glass"]} + + +def test_each_variable_and_each_product_keeps_the_order_first_given(): + found = repo_shape.pin_source_env_collisions( + ["x.y", "a-b", "lone", "a_b", "x_y", "x-y"]) + assert list(found) == ["SHAPE_PIN_SOURCE_X_Y", "SHAPE_PIN_SOURCE_A_B"] + assert found == {"SHAPE_PIN_SOURCE_X_Y": ["x.y", "x_y", "x-y"], + "SHAPE_PIN_SOURCE_A_B": ["a-b", "a_b"]} + + +@pytest.mark.parametrize("products", [ + pytest.param([], id="none"), + pytest.param(["openGlass"], id="one"), + pytest.param(["openGlass", "openPane"], id="two-that-differ"), + pytest.param(["open-glass", "openGlass"], id="hyphen-and-none"), + pytest.param(["openGlass", "OpenGlass", "OPENGLASS"], id="case-only"), + pytest.param(["open-glass", "open-glass", "Open-Glass"], id="repeats"), + pytest.param(["open-glass", None, "", 7, ["open_glass"]], + id="not-names-are-left-out"), +]) +def test_products_that_do_not_share_a_variable_are_no_collision(products): + """The same product declared twice, in any case, is ONE product: whether + that is wrong is #241's `scaffold-pin-collides` and not this function's. + An entry that is not a name is not one for this either.""" + assert repo_shape.pin_source_env_collisions(products) == {} + assert repo_shape.pin_source_env_shared(products) == set() + + +def test_what_is_shared_is_every_product_a_variable_cannot_tell_apart(): + assert repo_shape.pin_source_env_shared( + ["open-glass", "openGlass", "Open_Glass", "openPane"] + ) == {"open-glass", "open_glass"} + + +def test_it_reads_a_generator_once(): + """`link_pins_from_trees` and the scaffold hand it generators.""" + products = (p for p in ("a-b", "a_b")) + assert repo_shape.pin_source_env_collisions(products) == { + "SHAPE_PIN_SOURCE_A_B": ["a-b", "a_b"]} + + +# --- two declarations are one product when they are one PIN FILE (#275, N2) --- + +#: Pairs of names that share a variable and have TWO pin files, though +#: `casefold()` makes one of each. Hand-written `project.yaml` cases only: +#: scaffold's `--pin` and adopt's `pins:` take ASCII, as a GitHub repository +#: name is. +FOLDED_APART = [ + pytest.param("open\u00df", "openss", "SHAPE_PIN_SOURCE_OPENSS", + id="sharp-s"), + pytest.param("open\u017f", "opens", "SHAPE_PIN_SOURCE_OPENS", + id="long-s"), + pytest.param("open\ufb01", "openfi", "SHAPE_PIN_SOURCE_OPENFI", + id="fi-ligature"), + pytest.param("open\u03c2", "open\u03c3", "SHAPE_PIN_SOURCE_OPEN_", + id="final-sigma"), +] + + +@pytest.mark.parametrize("first, second, variable", FOLDED_APART) +def test_names_one_variable_cannot_tell_apart_are_found_beyond_ascii( + first, second, variable): + """The four pairs the first round missed. Each shares a variable and has + two pin files, so one checkout answered for both and the pin it was not a + checkout of was reported against the project; `casefold()`, which was the + rule, made each pair one product.""" + assert first.casefold() == second.casefold(), "the reason they were missed" + assert neutral_pin_path(first) != neutral_pin_path(second) + assert repo_shape.pin_source_env_name(first) == variable + assert repo_shape.pin_source_env_name(second) == variable + assert repo_shape.pin_source_env_collisions([first, second]) == { + variable: [first, second]} + assert repo_shape.pin_source_env_collisions([second, first]) == { + variable: [second, first]} + assert repo_shape.pin_source_env_shared([first, second]) == {first, second} + + +#: A pool of names covering every way two of them can relate: the same name in +#: another case, punctuation that folds to `_`, a non-ASCII letter whose +#: upper-case is ASCII (`\u0131` is `I`, `\u017f` is `S`), a character that +#: upper-cases to two (`\u00df`, `\ufb01`), one that upper-cases to a letter +#: that is no ASCII (`\u03c2`, `\u03c3`) and one that is only `_` once folded. +NAME_POOL = [ + "openGlass", "OpenGlass", "OPENGLASS", "open-glass", "Open-Glass", + "open_glass", "open.glass", "open-x", "open_x", "open.x", "openPane", + "open\u0131", "openi", "openI", "open\u00e9", "open-", "open_", + "open\uff47", "open\u00df", "openss", "openSS", "open\u017f", "opens", + "openS", "open\ufb01", "openfi", "openFI", "open\u03c2", "open\u03c3", + "open\u03a3", "open\u0130", +] + + +def test_every_pair_sharing_a_variable_with_two_pin_files_is_found(): + """Over every pair of the pool: found exactly when the variable is shared + AND the pin files are two. The oracle is `neutral_pin_path`, which names + the pin file, and `pin_source_env_name`, which names the variable; no + casefold, no case rule of this test's own.""" + for first, second in itertools.combinations(NAME_POOL, 2): + shares = (repo_shape.pin_source_env_name(first) + == repo_shape.pin_source_env_name(second)) + two_files = neutral_pin_path(first) != neutral_pin_path(second) + found = repo_shape.pin_source_env_collisions([first, second]) + assert bool(found) is (shares and two_files), (first, second, found) + if found: + (variable, group), = found.items() + assert group == [first, second] + assert variable == repo_shape.pin_source_env_name(first) + assert repo_shape.pin_source_env_shared([first, second]) == { + repo_shape.pin_product_key(first), + repo_shape.pin_product_key(second)} + + +@pytest.mark.parametrize("name", NAME_POOL) +def test_no_single_product_is_ever_a_collision(name): + """Nor is one product declared twice, in any spelling of its pin file.""" + assert repo_shape.pin_source_env_collisions([name]) == {} + assert repo_shape.pin_source_env_collisions([name, name]) == {} + assert repo_shape.pin_source_env_shared([name]) == set() + assert repo_shape.pin_source_env_collisions([name, name.lower()]) == {} + + +@pytest.mark.parametrize("name", NAME_POOL) +def test_the_key_is_the_name_the_pin_file_carries(name): + """`pin_product_key` is not a second rule: it is the stem of the file + `shape_materialize.neutral_pin_path` writes, and `validate-pins.py` reads + the same file under `contracts/-pin.yaml`.""" + assert neutral_pin_path(name) == ( + f"contracts/{repo_shape.pin_product_key(name)}-pin.yaml") + + +# --- scaffold-project.py --pin: refused, dry and real, before any process ---- + +#: `(the --pin values, what the refusal says, its remediation)`, `{A}` and +#: `{B}` being two commits of the `--pin-source` clone. +REFUSED_CASES = [ + pytest.param( + ["open-glass@{A}", "open_glass@{B}"], + "--pin 'open-glass@{A}' and --pin 'open_glass@{B}' name the products " + "`open-glass` and `open_glass`" + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + ALONE, id="hyphen-underscore"), + pytest.param( + ["open.glass@{A}", "open-glass@{B}"], + "--pin 'open.glass@{A}' and --pin 'open-glass@{B}' name the products " + "`open.glass` and `open-glass`" + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + ALONE, id="dot-hyphen"), + pytest.param( + ["open-glass@{A}", "openGlass@{A}", "open.glass@{B}", "open_glass@{A}"], + "--pin 'open-glass@{A}' and --pin 'open.glass@{B}' and --pin " + "'open_glass@{A}' name the products `open-glass` and `open.glass` and " + "`open_glass`" + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + ALONE, id="three-with-a-bystander"), + pytest.param( + ["opensoft/open-glass@{A}", "open_glass@{B}"], + "--pin 'opensoft/open-glass@{A}' and --pin 'open_glass@{B}' name the " + "products `open-glass` and `open_glass`" + + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + ALONE, id="owner-qualified-as-typed"), + pytest.param( + ["Open-Glass@{A}", "open_glass@{B}"], + "--pin 'Open-Glass@{A}' and --pin 'open_glass@{B}' name the products " + "`Open-Glass` and `open_glass`" + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + ALONE, id="capitals"), + pytest.param( + ["a-b@{A}", "x.y@{A}", "a_b@{B}", "x_y@{B}"], + "--pin 'a-b@{A}' and --pin 'a_b@{B}' name the products `a-b` and " + "`a_b`" + shared("SHAPE_PIN_SOURCE_A_B") + + " --pin 'x.y@{A}' and --pin 'x_y@{B}' name the products `x.y` and " + "`x_y`" + shared("SHAPE_PIN_SOURCE_X_Y"), + ALONE, id="two-groups-in-the-order-typed"), + pytest.param( + ["spec@{A}", "open-glass@{A}", "open_glass@{B}"], + "--pin 'spec@{A}' writes the pin of the product `spec` at " + "`contracts/spec-pin.yaml`, " + SHAPE_WRITES + + " --pin 'open-glass@{A}' and --pin 'open_glass@{B}' name the " + "products `open-glass` and `open_glass`" + + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + BESIDE_A_FILE, id="and-a-file-the-shape-writes"), + pytest.param( + ["open-glass@{A}", "Open-Glass@{B}", "open_glass@{A}"], + "--pin 'open-glass@{A}' and --pin 'Open-Glass@{B}' write the pins of " + "the products `open-glass` and `Open-Glass` at one file, " + "`contracts/open-glass-pin.yaml`, " + ONE_FILE + + " --pin 'open-glass@{A}' and --pin 'Open-Glass@{B}' and --pin " + "'open_glass@{A}' name the products `open-glass` and `open_glass`" + + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), + BESIDE_A_FILE, id="and-one-product-typed-twice"), +] + + +def refusal(scaffold, capsys, said: str, remediation: str, *args: str) -> str: + """`main()` with `args`: the pin refusal it printed, checked whole.""" + code = scaffold.main([*BASE_ARGS, *args]) + out, err = capsys.readouterr() + assert code == 2, out + err + assert out == "", "the refusal comes before the plan is printed" + assert err == f"{REFUSED}: {said}\n{remediation}\n", err + return err + + +@pytest.mark.parametrize("dry", [pytest.param(True, id="dry-run"), + pytest.param(False, id="real-run")]) +@pytest.mark.parametrize("values, said, remediation", REFUSED_CASES) +def test_pins_sharing_a_variable_are_refused_before_any_process( + scaffold, glass, no_process, capsys, tmp_path, values, said, + remediation, dry): + """The sentence names each argument as typed, the products and the + variable. A real run that got past it would read each pin's digest from + the `--pin-source` with `git`, which `no_process` refuses; and the remote + and work directories it was given were never made.""" + remotes, work = tmp_path / "remotes", tmp_path / "work" + _, first, second = glass + refusal(scaffold, capsys, said.format(A=first, B=second), remediation, + *pin_args(glass, values), "--local-remote-dir", str(remotes), + "--work-dir", str(work), *(("--dry-run",) if dry else ())) + assert not remotes.exists() and not work.exists() + + +@pytest.mark.parametrize("values, said, remediation", REFUSED_CASES) +def test_a_dry_run_refuses_exactly_as_the_real_run(glass, world, values, said, + remediation): + """On main through c7c02fa the dry run said yes and the real run made + three repositories holding two pins one variable named. Now both say the + same no, word for word, and nothing exists afterwards.""" + args = (*BASE_ARGS, *pin_args(glass, values), + "--local-remote-dir", str(world["remotes"]), + "--work-dir", str(world["work"])) + dry = run_script(SCAFFOLD, *args, "--dry-run", env=world["env"]) + real = run_script(SCAFFOLD, *args, env=world["env"]) + assert (dry.returncode, real.returncode) == (2, 2), ( + dry.stdout + dry.stderr + real.stdout + real.stderr) + assert dry.stderr == real.stderr + assert dry.stdout == real.stdout == "" + _, first, second = glass + assert real.stderr == (f"{REFUSED}: {said.format(A=first, B=second)}\n" + f"{remediation}\n") + nothing_was_created(world) + + +def test_the_variable_refusal_comes_before_the_leg_path_refusal( + scaffold, glass, no_process, capsys, monkeypatch): + """Where #241's file refusal runs, in `_refuse_colliding_pins`, and so + ahead of `_refuse_colliding_leg_paths`: a leg path over one of the two + pins' files is the pin refusal, and the leg-path question is never asked. + The control is the same leg path with one pin: the leg-path refusal.""" + asked = [] + original = scaffold._refuse_colliding_leg_paths + + def spy(*args, **kwargs): + asked.append(args) + return original(*args, **kwargs) + + monkeypatch.setattr(scaffold, "_refuse_colliding_leg_paths", spy) + legs = ("--code-path", "contracts/open-glass-pin.yaml") + _, first, second = glass + err = refusal( + scaffold, capsys, + f"--pin 'open-glass@{first}' and --pin 'open_glass@{second}' name " + "the products `open-glass` and `open_glass`" + + shared("SHAPE_PIN_SOURCE_OPEN_GLASS"), ALONE, *legs, + *pin_args(glass, ["open-glass@{A}", "open_glass@{B}"]), "--dry-run") + assert LEG_REFUSED not in err and asked == [] + code = scaffold.main([*BASE_ARGS, *legs, "--dry-run", + *pin_args(glass, ["open-glass@{A}"])]) + assert code == 2 + assert capsys.readouterr().err.startswith(f"{LEG_REFUSED}: ") + assert len(asked) == 1 + + +@WINDOWS_SKIP # the stub `gh` is a `#!` script; Windows executes none +def test_no_gh_command_runs_before_the_variable_refusal(world): + """Pointed at GitHub, with no `--pin-source`, so `gh api` would read each + pin's digest, the refusal comes first and `gh` is never run. The control + is the same run with one of the two pins: it reaches `gh`.""" + commit = "1" * 40 + common = (*BASE_ARGS, "--pin", f"open-glass@{commit}", + "--work-dir", str(world["work"])) + refused = run_script(SCAFFOLD, *common, "--pin", f"open_glass@{commit}", + env=world["env"]) + assert refused.returncode == 2, refused.stdout + refused.stderr + assert refused.stderr.startswith(f"{REFUSED}: --pin 'open-glass@{commit}' " + f"and --pin 'open_glass@{commit}' ") + nothing_was_created(world) + + control = run_script(SCAFFOLD, *common, env=world["env"]) + assert control.returncode == 2, control.stdout + control.stderr + assert "pin-unreadable" in control.stderr + assert world["log"].read_text(encoding=UTF8).startswith( + "api repos/opensoft/open-glass") + + +@pytest.mark.parametrize("values, files", [ + pytest.param(["open_glass@{A}"], ["contracts/open_glass-pin.yaml"], + id="one-punctuated-pin"), + pytest.param(["open-glass@{A}", "openGlass@{B}"], + ["contracts/open-glass-pin.yaml", + "contracts/openglass-pin.yaml"], id="variables-differ"), + pytest.param(["open-glass@{A}", "open-glass-2@{B}", "open.pane@{A}"], + ["contracts/open-glass-pin.yaml", + "contracts/open-glass-2-pin.yaml", + "contracts/open.pane-pin.yaml"], id="three-that-differ"), +]) +def test_pins_whose_variables_differ_still_scaffold(glass, world, values, + files): + """Nothing the refusal does touches a set whose variables differ: the + plan names each pin's file, and the pushed root holds each pin at its own + file with its own commit.""" + args = (*BASE_ARGS, *pin_args(glass, values), + "--local-remote-dir", str(world["remotes"]), + "--work-dir", str(world["work"])) + dry = run_script(SCAFFOLD, *args, "--dry-run", env=world["env"]) + assert dry.returncode == 0, dry.stdout + dry.stderr + planned = [line for line in dry.stdout.splitlines() + if line.startswith("pin ")] + assert [line.split(" -> ")[1].split(" (")[0] for line in planned] == files + result = run_script(SCAFFOLD, *args, env=world["env"]) + assert result.returncode == 0, result.stdout + result.stderr + assert REFUSED not in result.stderr + root = world["remotes"] / "Northwind.git" + for path, value in zip(files, pins_of(glass, values)): + pin = git("show", f"main:{path}", cwd=root).stdout + assert f'commit: "{value.rsplit("@", 1)[1]}"' in pin, pin + assert not world["log"].exists() + + +# --- validate-pins.py: a note naming each product and the variable ------------- + +PIN_FILE = textwrap.dedent("""\ + schema_version: 1 + kind: pinned_contract_manifest + pin_role: neutral-product + product: {name} + source_repository: opensoft/{name} + materialization: referenced + commit: "{commit}" + revision_kind: commit + digest_algorithm: sha256 + digest_definition: sorted-ls-tree-r-v1 + digests: + tree_sha256: "{digest}" + digest_source: stand-in + declared_in: project.yaml + verify_pin: scripts/validate-manifest.py + """) + + +def write(path: Path, text: str) -> None: + """LF on every platform, as the shape's own writers do.""" + with open(path, "w", encoding=UTF8, newline="\n") as handle: + handle.write(text) + + +def declare_pins(project: Path, base: Path, names) -> dict[str, Path]: + """Declare a pin on each of `names` in the scaffolded `project`, by hand: + a stand-in repository per name under `base`, a pin file written from it + and a `neutral_product_pins:` entry, as a hand-edited manifest, or one an + older scaffold wrote, holds them. `{name: the stand-in}`; `base` is no + sibling of `project`, so rule 3 of the lookup finds none of them.""" + stand_ins = {} + for name in names: + path, commit = product_repo(base, name) + stand_ins[name] = path + write(project / "contracts" / f"{name.lower()}-pin.yaml", + PIN_FILE.format(name=name, commit=commit, + digest=tree_digest(path, commit))) + manifest = project / "project.yaml" + text = manifest.read_text(encoding=UTF8) + assert "neutral_product_pins: []" in text, "fixture drift" + write(manifest, text.replace( + "neutral_product_pins: []", + "neutral_product_pins:\n" + "\n".join(f" - {name}" for name in names), + 1)) + return stand_ins + + +def validate(project: Path, *args: str, env=None): + return run_script(project / "scripts" / "validate-pins.py", *args, + cwd=project, env=env) + + +@pytest.fixture +def gh_shim(tmp_path): + """A `gh` that records its arguments and fails, first on PATH: the run + that reaches the forge says so in `log`, and a project whose pins were + all answered locally leaves it absent. Windows executes no `#!` script, + so the tests that need it are skipped there.""" + bin_dir, log = tmp_path / "shim", tmp_path / "gh.log" + bin_dir.mkdir() + gh = bin_dir / "gh" + write(gh, textwrap.dedent("""\ + #!/usr/bin/env python3 + import os, sys + with open(os.environ["GH_STUB_LOG"], "a", encoding="utf-8") as log: + log.write(" ".join(sys.argv[1:]) + "\\n") + print("gh shim: refused", file=sys.stderr) + sys.exit(97) + """)) + gh.chmod(gh.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return {"log": log, "env": { + "PATH": f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}", + "GH_STUB_LOG": str(log)}} + + +#: The note's line, and what it says after naming the products and the +#: variable, word for word. It is a NOTE (#275, fix round 1, S1): the lines the +#: validator prints that fail nothing are `ok`, `skip` and `note`. +NOTE = " note neutral-pin-source-collides: " + + +def collides(names: str, variable: str) -> str: + """The whole note, after its code, for the products `names` quotes.""" + return (f"project.yaml neutral_product_pins declares {names}, and one " + f"variable, {variable}, names the local checkout of each: it " + "cannot say which it holds, so it is read for none of them. Each " + "pin still verifies from the keyed `--pin-source " + "=` or from a checkout beside this assembly " + "root; a bare `--pin-source` answers for every pin that has no " + "more specific source. Which declaration stays is the human's " + "decision, and nothing here fails on it.") + + +def collision_notes(result) -> list[str]: + return [line for line in result.stdout.splitlines() + if line.startswith(NOTE)] + + +def test_two_pins_sharing_a_variable_are_one_note_naming_both( + project, tmp_path): + """Every source named by product answers, so what is said about the + declaration is a note and nothing else: one line, both products, the + variable, and a run that passes.""" + stand_ins = declare_pins(project, tmp_path / "products", + ["open-glass", "open_glass"]) + result = validate(project, "--pin-source", + f"open-glass={stand_ins['open-glass']}", + "--pin-source", f"open_glass={stand_ins['open_glass']}") + assert result.returncode == 0, result.stdout + result.stderr + assert "FINDING" not in result.stderr and "finding(s)" not in result.stderr + assert collision_notes(result) == [ + NOTE + collides("'open-glass' and 'open_glass'", + "SHAPE_PIN_SOURCE_OPEN_GLASS")] + assert result.stdout.rstrip().endswith("pins ok") + for name in ("open-glass", "open_glass"): + assert f" ok {name}: pin commit " in result.stdout + assert f"digest recomputes (--pin-source {name}=...)" in result.stdout + + +def test_three_products_one_variable_are_one_note_naming_all_three( + project, tmp_path): + names = ["open-glass", "open.glass", "open_glass"] + stand_ins = declare_pins(project, tmp_path / "products", + [*names, "openGlass"]) + result = validate(project, *(arg for name in names + for arg in ("--pin-source", + f"{name}={stand_ins[name]}")), + "--pin-source", f"openGlass={stand_ins['openGlass']}") + assert result.returncode == 0, result.stdout + result.stderr + assert "FINDING" not in result.stderr + assert collision_notes(result) == [ + NOTE + collides("'open-glass' and 'open.glass' and 'open_glass'", + "SHAPE_PIN_SOURCE_OPEN_GLASS")] + assert "openGlass'" not in collision_notes(result)[0] + assert result.stdout.count("digest recomputes (--pin-source") == 4 + + +def test_a_quiet_run_prints_no_note_and_still_passes(project, tmp_path): + """`--quiet` prints the notes of no kind, as it always did: nothing else + is said about a pair that nothing fails on.""" + stand_ins = declare_pins(project, tmp_path / "products", + ["open-glass", "open_glass"]) + result = validate(project, "--quiet", "--pin-source", + f"open-glass={stand_ins['open-glass']}", + "--pin-source", f"open_glass={stand_ins['open_glass']}") + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "pins ok" and result.stderr == "" + + +@WINDOWS_SKIP # the stub `gh` is a `#!` script; Windows executes none +@pytest.mark.parametrize("named", ["open-glass", "open_glass"]) +def test_a_shared_variable_answers_for_neither_pin(project, tmp_path, gh_shim, + named): + """THE ISSUE'S PROBE. The variable names one product's checkout and, on + main through c7c02fa, answered for both: the other was reported + `neutral-pin-source-unreadable`, a finding about the project made by a + checkout of another. Now it answers for neither, so both fall through to + `gh` (stubbed, and asked about each); the declaration is a note, so the + run passes with the two skips; and the hint offers the keyed + `--pin-source =`, the one source that says which pin it is + for, and never the variable or the bare form.""" + stand_ins = declare_pins(project, tmp_path / "products", + ["open-glass", "open_glass"]) + env = {**gh_shim["env"], + "SHAPE_PIN_SOURCE_OPEN_GLASS": str(stand_ins[named])} + result = validate(project, env=env) + assert result.returncode == 0, result.stdout + result.stderr + assert "neutral-pin-source-unreadable" not in result.stderr + assert "FINDING" not in result.stderr + assert len(collision_notes(result)) == 1 + calls = gh_shim["log"].read_text(encoding=UTF8).splitlines() + assert [call.split("/git/trees")[0] for call in calls] == [ + "api repos/opensoft/open-glass", "api repos/opensoft/open_glass"] + skips = [line for line in result.stdout.splitlines() + if line.startswith(" skip ")] + assert len(skips) == 2 + for name, line in zip(("open-glass", "open_glass"), skips): + assert "SHAPE_PIN_SOURCE" not in line + assert f"A --pin-source {name}= or a checkout at ../{name} " \ + "beside this assembly root would verify it offline." in line + + +@pytest.mark.parametrize("named", ["open-glass", "open_glass"]) +def test_a_shared_variable_answers_for_neither_pin_with_no_gh(project, named): + """THE SAME GUARD, WITH NO `gh` AT ALL, so it runs on Windows too (the + test above needs a `#!` stub). Each pin has a checkout BESIDE the project, + `../open-glass` and `../open_glass`, and the variable names ONE of them. + On ab623dd the variable answered for both pins, so the other was + `neutral-pin-source-unreadable` (a commit its checkout does not hold), exit + 1. Now neither reads it: each pin is recomputed from its own sibling, the + variable is named by no line, and only the note is said. No pin reaches + `gh`, which the session's guard would fail this test for.""" + stand_ins = declare_pins(project, project.parent, + ["open-glass", "open_glass"]) + result = validate(project, env={ + "SHAPE_PIN_SOURCE_OPEN_GLASS": str(stand_ins[named])}) + assert result.returncode == 0, result.stdout + result.stderr + assert "neutral-pin-source-unreadable" not in result.stderr + assert "FINDING" not in result.stderr + assert len(collision_notes(result)) == 1 + for name in ("open-glass", "open_glass"): + line = next(line for line in result.stdout.splitlines() + if line.startswith(f" ok {name}: pin commit ")) + assert "digest recomputes (sibling checkout (" in line, line + assert "($SHAPE_PIN_SOURCE" not in result.stdout + + +@WINDOWS_SKIP # the stub `gh` is a `#!` script; Windows executes none +def test_a_pin_no_other_shares_a_variable_with_is_offered_it(project, tmp_path, + gh_shim): + """The control: one punctuated pin keeps its variable in the hint, as it + always did, and the run is green with no source (a skip is no finding).""" + declare_pins(project, tmp_path / "products", ["open-glass"]) + result = validate(project, env=gh_shim["env"]) + assert result.returncode == 0, result.stdout + result.stderr + assert "FINDING" not in result.stderr + skip = next(line for line in result.stdout.splitlines() + if line.startswith(" skip open-glass")) + assert ("A --pin-source, SHAPE_PIN_SOURCE_OPEN_GLASS, or a checkout at " + "../open-glass beside this assembly root") in skip + + +def test_pins_whose_variables_differ_each_read_their_own(project, tmp_path): + """The names are kept: `open-glass` is still `SHAPE_PIN_SOURCE_OPEN_GLASS` + and `openGlass` is still `SHAPE_PIN_SOURCE_OPENGLASS`, two variables for + two pins, and each answers for its own product.""" + stand_ins = declare_pins(project, tmp_path / "products", + ["open-glass", "openGlass"]) + result = validate(project, env={ + "SHAPE_PIN_SOURCE_OPEN_GLASS": str(stand_ins["open-glass"]), + "SHAPE_PIN_SOURCE_OPENGLASS": str(stand_ins["openGlass"])}) + assert result.returncode == 0, result.stdout + result.stderr + assert "FINDING" not in result.stderr + assert ("open-glass: pin commit" in result.stdout + and "digest recomputes ($SHAPE_PIN_SOURCE_OPEN_GLASS)" + in result.stdout) + assert "digest recomputes ($SHAPE_PIN_SOURCE_OPENGLASS)" in result.stdout + assert "pins ok" in result.stdout + + +def test_a_product_declared_twice_is_not_a_shared_variable(project, tmp_path): + """`open-glass` and `Open-Glass` are one pin file and so one product, and + one variable rightly: no collision, and the variable answers.""" + stand_ins = declare_pins(project, tmp_path / "products", ["open-glass"]) + manifest = project / "project.yaml" + write(manifest, manifest.read_text(encoding=UTF8).replace( + " - open-glass", " - open-glass\n - Open-Glass", 1)) + result = validate(project, env={ + "SHAPE_PIN_SOURCE_OPEN_GLASS": str(stand_ins["open-glass"])}) + assert result.returncode == 0, result.stdout + result.stderr + assert "neutral-pin-source-collides" not in result.stderr + assert result.stdout.count("digest recomputes ($SHAPE_PIN_SOURCE_OPEN_GLASS)" + ) == 2 + + +# --- the skip hint, with no `gh` and no filesystem: runs on every OS --------- + +@pytest.fixture(scope="module") +def validate_pins(): + """The template's `validate-pins.py` as a module, so its functions are + called in-process. A hyphenated name is not importable, and + `importlib.util.spec_from_file_location` (which `tests/test_update_shape.py` + uses for `update-shape.py`) would write `__pycache__` BESIDE the template, + inside `templates/assembly-root`, where `tests/test_scaffold_leg_path_ + collisions.py` reads every file under that directory as text and dies on + the bytecode. So the source is compiled and run into a module here, and + nothing is written. `repo_shape` resolves to the module cached above, so + `Refusal` is the one the file raises.""" + module = types.ModuleType("validate_pins_under_test") + module.__file__ = str(VALIDATE_PINS) + exec(compile(VALIDATE_PINS.read_bytes(), str(VALIDATE_PINS), "exec", + dont_inherit=True), module.__dict__) + return module + + +def skip_hint(validate_pins, monkeypatch, tmp_path, products, product): + """The `skip` line `product` gets when nothing answers for it: no source, + no sibling, and a `gh` that is not there (`tree_digest_from_gh` refuses as + it does when `gh` is not on PATH, which is no process at all).""" + clear_ambient_pin_sources(monkeypatch) + + def no_gh(repository, commit): + raise Refusal("gh-not-found", "the `gh` CLI is not on PATH") + + monkeypatch.setattr(validate_pins, "tree_digest_from_gh", no_gh) + report = validate_pins.Report() + answered = validate_pins._recompute_neutral_pin_digest( + tmp_path / PROJECT, product, f"opensoft/{product}", "1" * 40, + f"contracts/{product}-pin.yaml", + validate_pins.PinSources([], products), report, + frozenset(repo_shape.pin_source_env_shared(products))) + assert answered is None + assert len(report.notes) == 1 and report.findings == [] + assert report.notes[0].startswith(f" skip {product}: pin commit ") + return report.notes[0] + + +def test_a_product_two_pins_share_is_offered_the_keyed_pin_source( + validate_pins, monkeypatch, tmp_path): + """A shared product's hint offers `--pin-source =` (N3): the + bare `--pin-source` answers for every pin with no more specific source, + so with two pins it gives the other the finding #275 was about, and the + variable answers for neither. The keyed form is the one source that says + which pin it is for.""" + for product in ("open-glass", "open_glass"): + line = skip_hint(validate_pins, monkeypatch, tmp_path, + ["open-glass", "open_glass"], product) + assert line.endswith( + f"A --pin-source {product}= or a checkout at ../{product} " + "beside this assembly root would verify it offline.") + assert "SHAPE_PIN_SOURCE" not in line + assert "A --pin-source or" not in line + + +@pytest.mark.parametrize("products", [ + pytest.param(["open-glass"], id="alone"), + pytest.param(["open-glass", "openGlass"], id="variables-differ"), + pytest.param(["open-glass", "Open-Glass"], id="one-product-twice"), +]) +def test_a_product_with_a_variable_to_itself_is_offered_it_as_before( + validate_pins, monkeypatch, tmp_path, products): + """The control: the hint's text for every product no other shares a + variable with is unchanged, the variable and the bare `--pin-source`.""" + line = skip_hint(validate_pins, monkeypatch, tmp_path, products, + "open-glass") + assert line.endswith( + "A --pin-source, SHAPE_PIN_SOURCE_OPEN_GLASS, or a checkout at " + "../open-glass beside this assembly root would verify it offline.") + + +@pytest.mark.parametrize("first, second, variable", FOLDED_APART) +def test_the_note_is_made_for_names_casefold_took_for_one_product( + validate_pins, tmp_path, first, second, variable): + """A hand-written `project.yaml` naming `openß` and `openss` (and the + three other pairs) declares two pins one variable names: the manifest is + READ, as the validator reads it, and `_report_pin_source_collisions` is + asked of what it declares. Every name is written as an escape in this + file, and the manifest holds it as UTF-8, so no part of the run depends + on the console's code page.""" + manifest = tmp_path / "project.yaml" + write(manifest, f"schema_version: 1\nneutral_product_pins:\n" + f" - {first}\n - {second}\n") + products = repo_shape.load_yaml(manifest)["neutral_product_pins"] + assert products == [first, second] + report = validate_pins.Report() + validate_pins._report_pin_source_collisions(products, report) + assert report.findings == [] + assert report.notes == [ + NOTE + collides(f"{first!r} and {second!r}", variable)] + + +# --- a project that already declares such a pair takes a shape update --------- + +#: The two copied files #275's change ships into every project, as upstream +#: commit B changes them: a line appended to each, which is all an update that +#: carries `validate-pins.py` and `repo_shape.py` has to be for `apply` to +#: rewrite both and then run the project's validators. +VALIDATE_PINS_SOURCE = "templates/assembly-root/scripts/validate-pins.py" +SHIPPED = {VALIDATE_PINS_SOURCE: "validate-pins", + "scripts/repo_shape.py": "repo_shape"} +SHIPPED_COPIES = ("scripts/validate-pins.py", "scripts/repo_shape.py") + + +@pytest.fixture(scope="module") +def behind(tmp_path_factory) -> dict: + """A clone of this repository at A, and B, which changes the two files a + project's pin validator is made of.""" + _base, upstream, commit_a, clone = scaffold_upstream_and_project( + tmp_path_factory, "pin-source-collides-update") + commit_b = commit_upstream_changes( + upstream, "Ship the pin validator's change", + {source: f"\n# {label}, as an update ships it.\n" + for source, label in SHIPPED.items()}) + return {"upstream": upstream, "clone": clone, "a": commit_a, "b": commit_b} + + +def declared_project(behind, base: Path, names) -> Path: + """A copy of the project as it was scaffolded at A, then declaring a pin + on each of `names` as an older scaffold's project does (main through + ab623dd scaffolded `--pin open-glass@A --pin open_glass@B`, exit 0, and + this is what that left), committed. A stand-in repository for each name + sits BESIDE the project, so each pin is recomputed offline and no `gh` is + asked.""" + root = base / PROJECT + shutil.copytree(behind["clone"], root, symlinks=True) + declare_pins(root, base, names) + git("add", "-A", cwd=root) + git("commit", "-q", "-m", "Declare the pins", cwd=root) + return root + + +def update(behind, root: Path, verb: str, *args: str): + return run_script(behind["upstream"] / "update-shape.py", verb, "--root", + str(root), "--upstream", str(behind["upstream"]), + "--at", behind["b"], *args) + + +def doctor_of(behind, root: Path) -> dict: + result = run_script(behind["upstream"] / "shape-doctor.py", "--root", + str(root), "--json") + payload = json.loads(result.stdout) + return {"exit": result.returncode, "verdict": payload["verdict"], + "rows": {row["id"]: row for row in payload["rows"]}} + + +@pytest.mark.parametrize("names", [ + pytest.param(["open-glass", "open_glass"], id="hyphen-underscore"), + pytest.param(["open-glass", "open.glass", "open_glass"], id="three"), +]) +def test_a_project_declaring_a_colliding_pair_takes_a_shape_update( + behind, tmp_path, names): + """THE FIRST ROUND'S FINDING LOCKED SUCH A PROJECT OUT. Main through + ab623dd scaffolds `--pin open-glass@A --pin open_glass@B`, and the first + round made `neutral-pin-source-collides` a finding, so the NEW + `validate-pins.py` that `update-shape.py apply` runs over the rewritten + root exited 1 and the whole apply ended `REFUSED update-validators-red`, + rolled back, for this update and every later one, until a person dropped a + true declaration of descent. Now it is a note: the update applies, the + project's validator passes and says the note, each pin still verifies from + its own checkout, and the doctor's `pins` row stays `ok` before and after + and moves no verdict a project declaring no pair does not have.""" + root = declared_project(behind, tmp_path, names) + control = declared_project(behind, tmp_path / "control", ["openGlass"]) + + before, before_control = doctor_of(behind, root), doctor_of(behind, + control) + assert before["rows"]["pins"]["status"] == "ok", before["rows"]["pins"] + assert before["rows"]["shape-currency"]["status"] == "FINDING" + assert (before["verdict"], before["exit"]) == ( + before_control["verdict"], before_control["exit"]), (before, + before_control) + + checked = update(behind, root, "check") + assert checked.returncode == 1, checked.stdout + checked.stderr + behind_by = verdicts(checked.stdout) + for copy in SHIPPED_COPIES: + assert behind_by[copy] == "upstream-changed", checked.stdout + + applied = update(behind, root, "apply", "--yes") + assert applied.returncode == 0, applied.stdout + applied.stderr + assert "update-validators-red" not in applied.stdout + applied.stderr + # The blob at B, never the upstream's working-tree file: a helper that + # appended to it with `write_text` wrote CRLF on Windows. + shipped = subprocess.run( + ["git", "show", f"{behind['b']}:{VALIDATE_PINS_SOURCE}"], + cwd=str(behind["upstream"]), capture_output=True, check=True).stdout + assert (root / "scripts" / "validate-pins.py").read_bytes() == shipped, ( + "the update was applied") + + result = validate(root) + assert result.returncode == 0, result.stdout + result.stderr + assert "FINDING" not in result.stderr + (note,) = collision_notes(result) + assert "SHAPE_PIN_SOURCE_OPEN_GLASS" in note + assert all(repr(name) in note for name in names) + for name in names: + assert f" ok {name}: pin commit " in result.stdout + assert result.stdout.count("digest recomputes (sibling checkout (") == len( + names) + + controlled = update(behind, control, "apply", "--yes") + assert controlled.returncode == 0, controlled.stdout + controlled.stderr + after, after_control = doctor_of(behind, root), doctor_of(behind, control) + assert after["rows"]["pins"]["status"] == "ok", after["rows"]["pins"] + assert after["rows"]["shape-currency"]["status"] == "ok" + assert (after["verdict"], after["exit"]) == ( + after_control["verdict"], after_control["exit"]), (after, + after_control) + assert after["exit"] == 0, after + + +# --- the chain readers: a variable two links share is read for neither -------- + +def tree(path: Path, *pins: str) -> Path: + """A stand-in for a link's checkout: a manifest and its declaration.""" + path.mkdir(parents=True, exist_ok=True) + write(path / "project.yaml", + "schema_version: 1\nkind: project-manifest\nneutral_product_pins:\n" + + "".join(f" - {pin}\n" for pin in pins)) + return path + + +def test_a_variable_two_links_share_is_read_for_neither(tmp_path, monkeypatch): + """`open-x` and `open_x` are two links of one hand-written chain and one + variable. On main through c7c02fa the tree it named answered for both, so + `open_x` was read as declaring what `open-x`'s tree declares.""" + clear_ambient_pin_sources(monkeypatch) + only = tree(tmp_path / "open-x", "openDox") + monkeypatch.setenv("SHAPE_PIN_SOURCE_OPEN_X", str(only)) + assert repo_shape.link_pins_from_trees( + ("open-x", "open_x", "openDox"), root=None) == {} + + +def test_a_link_no_other_shares_a_variable_with_still_reads_it(tmp_path, + monkeypatch): + """The control: the same variable answers for `open-x` in a chain with no + other link it names, which is what every chain read before.""" + clear_ambient_pin_sources(monkeypatch) + only = tree(tmp_path / "open-x", "openDox") + monkeypatch.setenv("SHAPE_PIN_SOURCE_OPEN_X", str(only)) + assert repo_shape.link_pins_from_trees( + ("open-x", "openY", "openDox"), root=None) == {"open-x": {"openDox"}} + + +def test_links_sharing_a_variable_are_still_read_by_name(tmp_path, + monkeypatch): + """`--link-source =` is not the variable, and answers each.""" + clear_ambient_pin_sources(monkeypatch) + first = tree(tmp_path / "one", "open_x") + second = tree(tmp_path / "two", "openDox") + assert repo_shape.link_pins_from_trees( + ("open-x", "open_x", "openDox"), root=None, + keyed_sources={"open-x": first, "open_x": second} + ) == {"open-x": {"open_x"}, "open_x": {"openDox"}} + + +def test_a_shared_variable_is_not_offered_as_the_way_to_read_a_link(): + """The `declared-unverified` hint names the variable for a link that has + it to itself, as before, and names `--link-source` alone for two links + that share one.""" + policy = NamingPolicy.load(POLICY_PATH) + apart = policy.resolve_referent( + "MedxGlass", {"openXglass"}, ["openXglass", "openGlass"], {}) + assert list(apart.warnings) == [ + "declared-unverified: openXglass declaring a pin on openGlass is a " + "fact in openXglass's own tree, which is not reachable here (a " + "sibling checkout, SHAPE_PIN_SOURCE_OPENXGLASS, or --link-source " + "openXglass= would let it be read). The recorded chain still " + "classifies."] + together = policy.resolve_referent( + "MedxGlass", {"open-x"}, ["open-x", "open_x", "openGlass"], {}) + assert [w.split(" (")[1].split(" would")[0] + for w in together.warnings] == [ + "a sibling checkout or --link-source open-x=", + "a sibling checkout or --link-source open_x="] + assert "SHAPE_PIN_SOURCE" not in " ".join(together.warnings)