repo_shape.py and validate-repository-naming.py: a chain link, a declared pin and a --link-source key are one product by pin_product_key, the pin file's own fold, so a tree declaring openss no longer verifies a chain that says openß, a chain naming both is two links and not a cycle, and --link-source openss= no longer answers for openß (#444)

scaffold-project.py

@@ -854,10 +854,11 @@ def _refuse_malformed_chain_links(policy, chain: tuple) -> None:
 
     The FIRST and LAST links are left to the checks that already anchor them:
     the first must be a `--pin` product and the last the referent the name
-    claims, both compared casefolded because a pin names WHICH repository, not
-    how it was typed. Each must casefold to a name this run already holds
-    (`[A-Za-z0-9_.-]`, or `open<Product>`), so neither can carry a `#`, a
-    quote, a bracket, a comma or a space.
+    claims, both compared by `pin_product_key`, the pin file's own fold
+    (#444), because a pin names WHICH product, not how it was typed. Each
+    must fold to a name this run already holds (`[A-Za-z0-9_.-]`, or
+    `open<Product>`), so neither can carry a `#`, a quote, a bracket, a comma
+    or a space.
 
     Asked before `_declaration`, so before `gh api` reads a pin's tree and
     before `--dry-run` returns: a dry run refuses exactly as the real run

scripts/repo_shape.py

@@ -1374,14 +1374,28 @@ def __repr__(self) -> str:  # pragma: no cover - diagnostics only
 
 
 def _pinned_basenames(declared_pins) -> set[str]:
-    """The repositories a manifest's `neutral_product_pins:` name, casefolded.
+    """The products a manifest's `neutral_product_pins:` name, each keyed by
+    `pin_product_key`.
 
-    Casefolded because every comparison made against them — a referent
+    Folded because every comparison made against them — a referent
     template's output, a chain's first link — is written in whatever case its
-    own file chose, and a pin is a fact about WHICH repository, not about how
+    own file chose, and a pin is a fact about WHICH product, not about how
     it was typed.
-    """
-    return {repo_basename(str(pin)).casefold()
+
+    ONE IDENTITY FOR A PRODUCT (#444). Every reader of a chain — this, the
+    referent and link compares below, `resolve_link_source`'s keyed lookup,
+    `link_pins_from_trees`' table and the naming validator's `--link-source`
+    table — keys a name by `pin_product_key`, the pin file's own fold, which
+    `validate-pins.py` reads by. They used `casefold()`, so `openß` and
+    `openss`, two pin files, were one link: a holder whose tree declares
+    `openss` verified a chain that says `openß`, a project pinning `openss`
+    verified a chain beginning at `openß`, `--link-source openss=` answered
+    for `openß`, and a chain naming both was refused as a cycle. Over ASCII
+    the two folds make the same groups, and `lower()` only ever splits one
+    that `casefold()` makes, so no name GitHub, `--pin` or the naming policy
+    can spell answers differently.
+    """
+    return {pin_product_key(repo_basename(str(pin)))
             for pin in (declared_pins or ()) if pin}
 
 
@@ -1397,8 +1411,9 @@ def _chain_links(referent_chain) -> list[str]:
 
 
 def _referent_in(referents, wanted) -> str | None:
-    """The first of `referents` whose casefolded name is one of `wanted`."""
-    return next((r for r in referents if r.casefold() in wanted), None)
+    """The first of `referents` whose `pin_product_key` is one of `wanted`
+    (#444: the pin file's fold, as `_pinned_basenames` says)."""
+    return next((r for r in referents if pin_product_key(r) in wanted), None)
 
 
 def _link_declarations(link_pins) -> dict[str, set[str]]:
@@ -1408,9 +1423,9 @@ def _link_declarations(link_pins) -> dict[str, set[str]]:
     broken-link message quotes what the other tree actually says. A link whose
     tree could not be read is ABSENT rather than empty: the two mean opposite
     things — unverified, and declaring nothing at all — and the caller reads
-    the difference.
+    the difference. Each link is keyed by `pin_product_key` (#444).
     """
-    return {str(link).casefold():
+    return {pin_product_key(str(link)):
             {repo_basename(str(pin)) for pin in (pins_of or ()) if pin}
             for link, pins_of in (link_pins or {}).items()
             if pins_of is not None}
@@ -1429,6 +1444,10 @@ def _chain_link_findings(chain, link_pins) -> tuple:
     A `LinkPins` carries, beside the links that answered, why a tree that WAS
     reached could not: a warning for such a link says so, where a plain
     mapping's says the tree is not reachable (#338).
+
+    A holder's tree and the next link are matched by `pin_product_key`
+    (#444): a tree declaring `openss` does not declare `openß`, whose pin
+    file is another.
     """
     available = _link_declarations(link_pins)
     unreadable = getattr(link_pins, "unreadable", None) or {}
@@ -1438,8 +1457,8 @@ def _chain_link_findings(chain, link_pins) -> tuple:
     # that variable: it would be read for both (#275).
     shared = pin_source_env_shared(chain)
     for holder, held in zip(chain, chain[1:]):
-        declared = available.get(holder.casefold())
-        why = unreadable.get(holder.casefold())
+        declared = available.get(pin_product_key(holder))
+        why = unreadable.get(pin_product_key(holder))
         if declared is None and why is not None:
             # A source REACHED this tree and could not read a declaration out
             # of it, so "not reachable here" would be untrue (#338).
@@ -1462,7 +1481,8 @@ def _chain_link_findings(chain, link_pins) -> tuple:
                 f"{holder}=<path> would let it be read). The recorded "
                 "chain still classifies.")
             continue
-        if held.casefold() not in {pin.casefold() for pin in declared}:
+        if pin_product_key(held) not in {pin_product_key(pin)
+                                         for pin in declared}:
             return (f"{holder} declares "
                     + (", ".join(sorted(declared)) if declared else "no "
                        "neutral-product pin at all")
@@ -1769,7 +1789,7 @@ def broken(detail: str) -> ReferentResolution:
         problem = self._chain_shape_problem(name, chain, pins, referents)
         if problem is not None:
             return broken(problem)
-        final = _referent_in(referents, {chain[-1].casefold()})
+        final = _referent_in(referents, {pin_product_key(chain[-1])})
         contradiction, warnings, unverified = _chain_link_findings(chain,
                                                                    link_pins)
         if contradiction is not None:
@@ -1792,19 +1812,23 @@ def _chain_shape_problem(self, name: str, chain: list,
         have. Each answer becomes the `detail` of a `broken` resolution, which
         is where the reader is told WHICH link broke rather than merely that
         one did.
+
+        A link is one product by `pin_product_key` (#444), so `openß` and
+        `openss` in one chain are two links and not a repeat, and a chain
+        beginning at `openß` does not begin at a pin on `openss`.
         """
         max_length = int(self.chain_rule().get("max_length") or CHAIN_MAX_LENGTH)
         if len(chain) > max_length:
             return (f"it names {len(chain)} links and the policy admits "
                     f"at most {max_length}")
-        seen = [link.casefold() for link in chain]
+        seen = [pin_product_key(link) for link in chain]
         if len(set(seen)) != len(seen):
             return "it repeats a link, so it is a cycle rather than a chain"
-        if chain[0].casefold() not in pins:
+        if pin_product_key(chain[0]) not in pins:
             return (f"it begins at {chain[0]}, which is not in this "
                     "project's `neutral_product_pins:` — the first "
                     "entry is the pin this project actually holds")
-        if _referent_in(referents, {chain[-1].casefold()}) is None:
+        if _referent_in(referents, {pin_product_key(chain[-1])}) is None:
             return (f"it ends at {chain[-1]}, but {name} would need "
                     + " or ".join(referents))
         return None
@@ -2787,7 +2811,9 @@ def resolve_link_source(product: str, root: Path | None = None,
                         ) -> tuple[Path | None, str | None]:
     """Where `product`'s own manifest can be read, if anywhere at all.
 
-      1. `--link-source <product>=<path>` (case-insensitive on the name)
+      1. `--link-source <product>=<path>`, `keyed_sources` keyed by
+         `pin_product_key` (#444): case-insensitive on the name, as its pin
+         file is, so `openss=` answers for `openss` and never for `openß`
       2. `SHAPE_PIN_SOURCE_<PRODUCT>` in the environment
       3. a checkout sitting BESIDE this project (`../openXdox`)
       4. a bare `--link-source <path>` with no `product=` prefix, which is
@@ -2810,7 +2836,7 @@ def resolve_link_source(product: str, root: Path | None = None,
     which product it holds is not read for either (#275).
     """
     candidates: list[tuple[str, Path, str]] = []
-    keyed = (keyed_sources or {}).get(product.casefold())
+    keyed = (keyed_sources or {}).get(pin_product_key(product))
     if keyed is not None:
         candidates.append(("keyed", Path(keyed),
                            f"--link-source {product}=..."))
@@ -2836,8 +2862,9 @@ def resolve_link_source(product: str, root: Path | None = None,
 class LinkPins(dict):
     """`link_pins_from_trees`'s answer for a caller that asks it to say why:
     the same `{link: the pins ITS manifest declares}`, equal to a plain one,
-    with `unreadable` beside it, `{link casefolded: why}` for each link whose
-    source REACHED a manifest that could not be read as a declaration (#338).
+    with `unreadable` beside it, `{pin_product_key(link): why}` for each link
+    whose source REACHED a manifest that could not be read as a declaration
+    (#338, #444).
 
     A link in `unreadable` is absent from the mapping, as one that is not on
     the disk is. What differs is the warning `_chain_link_findings` writes for
@@ -2891,6 +2918,11 @@ def link_pins_from_trees(links, root: Path | None = None,
     openYdox, not openInk", and a chain that holds was broken. A later link
     is read through rules 1 to 3 of `resolve_link_source`, or is
     declared-unverified.
+
+    THE TABLE IS KEYED BY `pin_product_key` (#444), the key the variable
+    guard above already asks by. It was `casefold()`, so `openß` and
+    `openss`, each with its own tree, shared one entry and the later tree
+    answered for both.
     """
     found: dict[str, set] = {}
     unreadable: dict[str, str] = {}
@@ -2911,9 +2943,9 @@ def link_pins_from_trees(links, root: Path | None = None,
         if pins is None:
             rule, how, _path, _answered = asked[-1]
             asked[-1] = (rule, how, path, why)
-            unreadable[name.casefold()] = why
+            unreadable[pin_product_key(name)] = why
             continue
-        found[name.casefold()] = pins
+        found[pin_product_key(name)] = pins
     return LinkPins(found, unreadable) if with_reasons else found
 
 

scripts/validate-repository-naming.py

@@ -102,7 +102,7 @@
 from repo_shape import (  # noqa: E402
     CHAIN_RECORD_FIELD, LINK_SOURCE_ABSENT, LINK_SOURCE_NO_MANIFEST,
     UNAMBIGUOUS_FORMS, Classification, NamingPolicy, Refusal,
-    link_pins_from_trees, load_yaml, repo_basename,
+    link_pins_from_trees, load_yaml, pin_product_key, repo_basename,
 )
 
 DEFAULT_POLICY = Path(__file__).resolve().parents[1] / "contracts" / "repository-naming.yaml"
@@ -137,15 +137,25 @@ class LinkSources:
     that a source keyed to a link `a` that no chain names, which no lookup
     ever asked for — the link stayed declared-unverified, exit 0, under a
     WARNING suggesting the very `--link-source` the person had passed. So the
-    text before the first `=` is a LINK only when it casefolds to a link of a
+    text before the first `=` is a LINK only when it names a link of a
     referent chain this run reads (`--referent-chain`'s, or a leg's
-    `naming.referent_chain:` under `--project`), the casefold the lookup
-    uses, and any other value is a bare PATH, whole. A link's name wins even
-    where a directory is spelled like the whole value: the reading is decided
-    by the chains and never by the disk, and `./` asks for the directory.
+    `naming.referent_chain:` under `--project`) by `pin_product_key`, the key
+    the lookup uses, and any other value is a bare PATH, whole. A link's name
+    wins even where a directory is spelled like the whole value: the reading
+    is decided by the chains and never by the disk, and `./` asks for the
+    directory.
     That is `validate-pins.py`'s reading of `--pin-source` (#295), with a
     chain link where that file has a declared pin, so the two copied tools
-    read `[PRODUCT=]PATH` one way.
+    read `[PRODUCT=]PATH` one way OVER ASCII ONLY (#444). Outside it they part
+    company: `PinSources` (`templates/assembly-root/scripts/validate-pins.py`
+    `:831`, `:845`, `:848-849` and `:912`) still keys `--pin-source PRODUCT=`
+    by `casefold()`, so `--pin-source openss=` answers for a declared `openß`
+    where `--link-source openss=` no longer answers for the link `openß`. It
+    is left, and the issue leaves it as its own change, because a pin's digest
+    stops a wrong source from passing: `validate-pins.py` checks it, a wrong
+    source cannot match it, and so that reading can turn a wrong source into
+    a finding and never into a pass. The follow-up is on #459's checklist
+    (the shipped validators and the helpers they share).
 
     WHAT EACH VALUE CAME TO. Every lookup records what it asked
     (`link_pins_from_trees(..., tried=)`), so once every name is classified a
@@ -170,6 +180,13 @@ class LinkSources:
     `Path("")`, which is `.`: run from a project root it read the project's
     own manifest as openXdox's, and a chain that holds was broken. That and
     an empty bare value are in `no_path`, and `no_path_refusal` says so.
+
+    EVERY LINK IS KEYED BY `pin_product_key` (#444), the pin file's own fold,
+    as `repo_shape.py`'s chain readers key it. With `casefold()` here,
+    `openß` and `openss` were one link: `--link-source openss=` answered for
+    `openß`, two values for the pair replaced each other, and two chains
+    beginning at them counted one first link, so a bare PATH was read as both
+    products' trees.
     """
 
     def __init__(self, values: list[str], links, firsts=()) -> None:
@@ -178,18 +195,19 @@ def __init__(self, values: list[str], links, firsts=()) -> None:
         names: dict[str, str] = {}
         for link in links:
             name = repo_basename(str(link))
-            if name and name.casefold() not in names:
-                names[name.casefold()] = name
+            if name and pin_product_key(name) not in names:
+                names[pin_product_key(name)] = name
                 self.links.append(name)
-        #: `{casefolded: first spelling}` of the first link of every chain a
-        #: name here reads (#441). `link_pins` adds a chain's first link as it
-        #: reads it, which matters only for names read from stdin: they come
-        #: when no other name does, all with `--referent-chain`.
+        #: `{pin_product_key: first spelling}` of the first link of every
+        #: chain a name here reads (#441). `link_pins` adds a chain's first
+        #: link as it reads it, which matters only for names read from stdin:
+        #: they come when no other name does, all with `--referent-chain`.
         self.firsts: dict[str, str] = {}
         for link in firsts:
             name = repo_basename(str(link))
-            self.firsts.setdefault(name.casefold(), name)
-        #: `{link casefolded: (typed, path)}`, the last value given wins.
+            self.firsts.setdefault(pin_product_key(name), name)
+        #: `{pin_product_key(link): (typed, path)}`, the last value given
+        #: wins.
         self.keyed: dict[str, tuple[str, Path]] = {}
         #: `(typed, path)` of the last bare value, if any.
         self.bare: tuple[str, Path] | None = None
@@ -200,11 +218,11 @@ def __init__(self, values: list[str], links, firsts=()) -> None:
         self.replaced: dict[str, str] = {}
         for typed in values:
             key, sep, path = typed.partition("=")
-            link = names.get(key.casefold()) if sep else None
+            link = names.get(pin_product_key(key)) if sep else None
             self.reading[typed] = link
             if link is not None:
-                earlier = self.keyed.get(link.casefold())
-                self.keyed[link.casefold()] = (typed, Path(path))
+                earlier = self.keyed.get(pin_product_key(link))
+                self.keyed[pin_product_key(link)] = (typed, Path(path))
             else:
                 earlier = self.bare
                 self.bare = (typed, Path(typed))
@@ -216,7 +234,7 @@ def __init__(self, values: list[str], links, firsts=()) -> None:
         self.no_path: list[str] = [
             typed for typed, link in self.reading.items()
             if not (typed.partition("=")[2] if link is not None else typed)]
-        #: The links some name's lookup asked about, casefolded.
+        #: The links some name's lookup asked about, by `pin_product_key`.
         self.asked: set[str] = set()
         #: The values a link read: each the answer for one link (#441).
         self.used: set[str] = set()
@@ -242,7 +260,7 @@ def link_pins(self, policy: NamingPolicy, target: Target) -> dict:
             return {}
         if len(target.chain) > 1:
             first = repo_basename(str(target.chain[0]))
-            self.firsts.setdefault(first.casefold(), first)
+            self.firsts.setdefault(pin_product_key(first), first)
         tried: dict = {}
         found = link_pins_from_trees(
             target.chain, target.root,
@@ -253,12 +271,12 @@ def link_pins(self, policy: NamingPolicy, target: Target) -> dict:
         # (`_chain_link_findings`): a source asked only for the chain's end
         # decided nothing, and saying it answered, or failed to, would be a
         # report about a question nobody asked.
-        holders = {repo_basename(str(link)).casefold()
+        holders = {pin_product_key(repo_basename(str(link)))
                    for link in target.chain[:-1]}
         for link, asked in tried.items():
-            if link.casefold() not in holders:
+            if pin_product_key(link) not in holders:
                 continue
-            self.asked.add(link.casefold())
+            self.asked.add(pin_product_key(link))
             for rule, how, path, why in asked:
                 self._record(link, rule, how, path, why)
         return found
@@ -280,7 +298,7 @@ def _record(self, link: str, rule: str, how: str, path: Path,
         still could not answer, since one that is absent, or holds no
         manifest, is the ordinary case."""
         if rule in ("keyed", "bare"):
-            typed = (self.keyed[link.casefold()] if rule == "keyed"
+            typed = (self.keyed[pin_product_key(link)] if rule == "keyed"
                      else self.bare)[0]
             if why is None:
                 self.used.add(typed)