From d71cbb6190cb698411a3b987440c1f008ebe2e22 Mon Sep 17 00:00:00 2001 From: Sarah Gehrke <99770056+sagehrke@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:08:49 -0600 Subject: [PATCH 01/10] Update Community-Meetings.md Adding October 15th presenter. --- docs/get-involved/Community-Meetings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/get-involved/Community-Meetings.md b/docs/get-involved/Community-Meetings.md index b278ec082f..d932d91645 100644 --- a/docs/get-involved/Community-Meetings.md +++ b/docs/get-involved/Community-Meetings.md @@ -27,7 +27,7 @@ Join the LinkML community for regular sessions featuring presentations on LinkML | Date | Presenter 1 | Topic 1 | Presenter 2 | Topic 2 | | :---: | :---: | :---: | :----: | :---: | -| October 15, 2026|Piotr Sowiński|[LinkML-Scala](https://github.com/NeverBlink-OSS/linkml-scala)| | Open slot!
Volunteers welcome| +| October 15, 2026|Piotr Sowiński|[LinkML-Scala](https://github.com/NeverBlink-OSS/linkml-scala)| Maia Kapur | Autonomous science development at EMSL & model viewer in the BASALT GitHub docs | | [September 17, 2026](https://docs.google.com/presentation/d/1b_XGol4vnEKsAp0zwCKludfVP_IOFnYoxwW3gBRbbLU/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Matt Gehring| How sniff.world uses LinkML: from consuming Biolink to publishing its own schemas| Nolan Nicols |[Provenance as a Primitive: Building Knowledge Graphs with LOKF](https://docs.google.com/presentation/d/155JuRM4nBrLsFOFkMv3ZQfgQleqwieP27c0vWAo9SZM/edit?slide=id.p1#slide=id.p1)| | [August 20, 2026](https://docs.google.com/presentation/d/1_DgJTclLsi9XEcnmgAfftVD5HZ1Aa0TF6abGB_YCGx0/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Patrick Golden| Working across boundaries: LinkML in the [Zebrafish Phenotype Atlas Project](https://zappfish.org/)|Alex Anderson| Semantic Converters for Power Systems Data Interoperability and Agentic AI| | [July 16, 2026](https://docs.google.com/presentation/d/1A05qfTbmI8RXyvoBSplpjSU9e4OP6BlIKJvQ1-eWhGw/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Sierra Moxon | LinkML Microschemas |Stephan Heunis |[The ORINOCO project at Research Center Jülich - building self-hostable research information infrastructure](https://pool.psychoinformatics.de/ui/?sh%3ANodeShape=xyzri%3AXYZProject&pid=xyzrins%3Aprojects%2Forinoco) | From e34784bb85e043ccd63ffe0bdf6703da150cea61 Mon Sep 17 00:00:00 2001 From: Sarah Gehrke <99770056+sagehrke@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:13:45 -0600 Subject: [PATCH 02/10] Update Community-Meetings.md fix white space. my nemesis. --- docs/get-involved/Community-Meetings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/get-involved/Community-Meetings.md b/docs/get-involved/Community-Meetings.md index d932d91645..a7e6b8d4ed 100644 --- a/docs/get-involved/Community-Meetings.md +++ b/docs/get-involved/Community-Meetings.md @@ -27,7 +27,7 @@ Join the LinkML community for regular sessions featuring presentations on LinkML | Date | Presenter 1 | Topic 1 | Presenter 2 | Topic 2 | | :---: | :---: | :---: | :----: | :---: | -| October 15, 2026|Piotr Sowiński|[LinkML-Scala](https://github.com/NeverBlink-OSS/linkml-scala)| Maia Kapur | Autonomous science development at EMSL & model viewer in the BASALT GitHub docs | +| October 15, 2026|Piotr Sowiński|[LinkML-Scala](https://github.com/NeverBlink-OSS/linkml-scala)| Maia Kapur| Autonomous science development at EMSL & model viewer in the BASALT GitHub docs| | [September 17, 2026](https://docs.google.com/presentation/d/1b_XGol4vnEKsAp0zwCKludfVP_IOFnYoxwW3gBRbbLU/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Matt Gehring| How sniff.world uses LinkML: from consuming Biolink to publishing its own schemas| Nolan Nicols |[Provenance as a Primitive: Building Knowledge Graphs with LOKF](https://docs.google.com/presentation/d/155JuRM4nBrLsFOFkMv3ZQfgQleqwieP27c0vWAo9SZM/edit?slide=id.p1#slide=id.p1)| | [August 20, 2026](https://docs.google.com/presentation/d/1_DgJTclLsi9XEcnmgAfftVD5HZ1Aa0TF6abGB_YCGx0/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Patrick Golden| Working across boundaries: LinkML in the [Zebrafish Phenotype Atlas Project](https://zappfish.org/)|Alex Anderson| Semantic Converters for Power Systems Data Interoperability and Agentic AI| | [July 16, 2026](https://docs.google.com/presentation/d/1A05qfTbmI8RXyvoBSplpjSU9e4OP6BlIKJvQ1-eWhGw/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Sierra Moxon | LinkML Microschemas |Stephan Heunis |[The ORINOCO project at Research Center Jülich - building self-hostable research information infrastructure](https://pool.psychoinformatics.de/ui/?sh%3ANodeShape=xyzri%3AXYZProject&pid=xyzrins%3Aprojects%2Forinoco) | From a51b69515c540e6ae64ff026a23e3bbb8b0a79e1 Mon Sep 17 00:00:00 2001 From: "yusheng.wang" Date: Fri, 9 Oct 2026 11:38:01 +0800 Subject: [PATCH 03/10] docs: add FAQ entry for generator feature support Point readers to the generator dashboard and issue 2333 instead of a hand-curated matrix of generators and validators. --- docs/faq/tools.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/faq/tools.md b/docs/faq/tools.md index e6cee18d82..25f4acad37 100644 --- a/docs/faq/tools.md +++ b/docs/faq/tools.md @@ -69,6 +69,25 @@ There are a few strategies: You can use any of the generator tools distributed as part of linkml to check for errors in your schema. +## Is there a master list of which generators or validators support which LinkML features? + +There is no separately curated, complete matrix of every generator and validator against every metamodel slot. + +The closest source of truth is the [Generator Feature Dashboard](../generators/dashboard). It is generated from the [compliance test suite](https://github.com/linkml/linkml/tree/main/tests/linkml/test_compliance) and shows which metamodel features each tested generator implements, partially implements, ignores, or has not yet been tested for. + +That dashboard is the outcome of [issue 2333](https://github.com/linkml/linkml/issues/2333) (Generator Support Matrix). It is not exhaustive: + +* Only generators that participate in the compliance suite appear as columns. +* Cells can be unknown or partial; a result reflects the current tests, not a guarantee of every edge case. +* There is no equivalent complete matrix for validators such as JSON Schema, Pydantic, SHACL, or ShEx. Those validators typically consume generated artifacts, so generator coverage on the dashboard is the relevant proxy. + +For more detail, see: + +* [Generators](../generators/index) and the individual generator pages (JSON Schema, Pydantic, SHACL, ShEx, and others) +* [Validating data](../data/validating-data) for validation strategies and plugins + +If a feature you need is missing or marked unknown, please open an issue or add a compliance test rather than treating the dashboard as a finished catalog. + ## Is there a linter for LinkML? Yes! See the documentation for [the schema linter](https://linkml.io/linkml/schemas/linter.html). From 7ba2509b567a085c7762f5d05bf4da72605228b3 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Tue, 8 Sep 2026 18:09:43 +0200 Subject: [PATCH 04/10] feat(rdf): add opt-in diff-stable blank-node labels RDFC-1.0 canonicalization already makes RDF output deterministic: isomorphic graphs always serialize identically. It does not make output diffable. Blank nodes are numbered `c14nN` in a single global order, so inserting one class can renumber every blank node after it and rewrite most of the file. A one-line semantic change lands as a whole-file diff, which makes generated OWL/SHACL hard to review and noisy to keep under version control. Add a `diff_stable` argument to `canonicalize_rdf_graph()` and a `--diff-stable/--no-diff-stable` flag to the four RDF generators. When enabled, blank-node labels are derived from each node's own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels only the blank nodes it actually touches. Measured churn on a real schema (add one class, count changed lines): generator default --diff-stable owlgen 2091 17 shexgen 796 50 shaclgen 291 13 rdfgen 115 25 Output stays deterministic and isomorphic either way; only the choice of label changes. Off by default, because enabling it relabels existing output. The refinement itself lives in `diffable-rdf`, whose only dependencies (rdflib, pyoxigraph) are already linkml-runtime dependencies at higher versions, so this adds no new transitive dependencies. Signed-off-by: jdsika --- .../linkml/src/linkml/generators/owlgen.py | 28 +++- .../linkml/src/linkml/generators/rdfgen.py | 28 +++- .../linkml/src/linkml/generators/shaclgen.py | 28 +++- .../linkml/src/linkml/generators/shexgen.py | 28 +++- packages/linkml_runtime/pyproject.toml | 1 + .../linkml_runtime/utils/rdf_canonicalize.py | 23 ++- .../test_utils/test_rdf_canonicalize.py | 135 ++++++++++++++++++ 7 files changed, 266 insertions(+), 5 deletions(-) diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 7ba15df672..09db0357be 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,22 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +369,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1860,16 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..052fcff12e 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,22 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +105,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +153,16 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..5d42fea272 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,22 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +212,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +945,16 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..7c43b13bc0 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,22 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Label blank nodes so that unrelated edits leave them untouched. + + Output is already deterministic: RDFC-1.0 guarantees that isomorphic + graphs serialize identically. It does not guarantee that *similar* + graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a + global order, so adding one class can renumber every blank node after + it and rewrite most of the file. + + When ``True``, blank-node labels are instead derived from each node's + own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels + only the blank nodes it actually touches. The output stays + deterministic and isomorphic either way; only the choice of label + changes. Off by default because enabling it relabels existing output. + """ + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +193,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +274,16 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Derive blank-node labels from each node's own neighbourhood so that " + "unrelated edits leave them unchanged. Output is deterministic either " + "way; this makes successive versions of a file diff cleanly." + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 5f69503d48..1324790b58 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.2.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..e0fadae34a 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -42,6 +42,7 @@ import pyoxigraph as ox import rdflib +from diffable_rdf import wl_relabel_quads from rdflib.compare import to_canonical_graph @@ -287,6 +288,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,6 +302,12 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Derive blank-node labels from each node's own + neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that + editing one part of a schema does not renumber unrelated blank nodes. + Output is deterministic and isomorphic either way; only the choice of + label changes. Off by default because enabling it relabels existing + output. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -339,13 +347,26 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + # 3b. Optionally re-label blank nodes with locality-sensitive hashes. + # RDFC-1.0 guarantees that identical graphs get identical labels, but it + # does not guarantee that *similar* graphs get similar labels: the labels + # are assigned by a global ordering, so inserting one blank node can + # renumber every label after it and turn a one-line semantic change into a + # whole-file diff. Weisfeiler-Lehman labels are derived only from each + # node's local neighbourhood, so unrelated regions keep their labels. + # Output stays deterministic and isomorphic either way; this only changes + # which label each blank node receives. + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..0ca79c3c90 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -507,3 +507,138 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + import difflib + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_is_opt_in(): + """The default must keep producing exactly the output it produced before.""" + graph = _make_graph_with_bnodes() + assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) + + +def test_diff_stable_preserves_semantics(): + """Relabelling blank nodes must not change what the graph means.""" + graph = _make_graph_with_bnodes() + + plain = rdflib.Graph() + plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") + stable = rdflib.Graph() + stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") + + assert rdflib.compare.isomorphic(plain, stable) + + +def test_diff_stable_is_deterministic(): + """Diff stability must not cost determinism, which is the stronger property.""" + graph = _make_graph_with_bnodes() + outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} + assert len(outputs) == 1 + + +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): + """Inserting one subject must not relabel the blank nodes of the others. + + RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject + sorting before the others shifts every subsequent label and rewrites + most of the file. This is the entire reason the option exists, so the + assertion is on the *ratio*, not on an absolute line count that would + be brittle across rdflib versions. + """ + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +def _generator_cases(): + """The four generators that serialize RDF, with the args that make them do so.""" + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.rdfgen import RDFGenerator + from linkml.generators.shaclgen import ShaclGenerator + from linkml.generators.shexgen import ShExGenerator + + return [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); + # the `network` marker serves those from local stubs. See tests/conftest.py. + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + # ShExGenerator only emits RDF in this format; its default is ShExC text, + # where blank-node labelling does not apply. + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ] + + +@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) +def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): + """Every RDF generator must actually apply the option, not merely accept it. + + Asserting only that the attribute exists would pass even if a generator + forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this + checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, + while diff-stable labels are neighbourhood hashes, so a generator that + honours the flag emits no ``c14nN`` label at all. + """ + assert generator.diff_stable is False, f"{generator.__name__} must default to off" + + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") + + plain = generator(str(schema), **kwargs).serialize() + stable = generator(str(schema), diff_stable=True, **kwargs).serialize() + + # Guards the test itself: if the fixture stopped producing blank nodes the + # assertion below would hold vacuously. + assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" + assert not re.search(r"c14n\d+", stable), ( + f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " + "the flag is probably not threaded into canonicalize_rdf_graph()" + ) From 4cf96efa95a6c911c93946270d3a932ec71586c9 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 10:52:51 +0200 Subject: [PATCH 05/10] fix(rdf): bump diffable-rdf to 0.3.0 and stop diff_stable silently no-opping Bump the floor to diffable-rdf 0.3.0 and add the missing uv.lock entry: the dependency was declared in pyproject.toml but never locked, so "uv lock --check" and the "uv sync --frozen" anti-malware gate would both have failed CI. 0.3.0 also fixes two defects in the Weisfeiler-Lehman labelling this feature relies on. Disconnected blank-node components now converge independently, so an edit in one region no longer relabels an unrelated one. And the suffix used to tell structurally indistinguishable nodes apart was assigned in c14nN *text* order, so c14n10 sorted between c14n1 and c14n2 -- adding a tenth tied blank node relabelled eight of the nine already there, the exact opposite of what this labelling is for. Separately, diff_stable=True was silently ignored whenever pyoxigraph refused the graph and canonicalize_rdf_graph degraded to rdflib. Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, and that path exists precisely because there are none, so the argument could not be honoured -- but the caller was never told. "shaclgen --include-annotations --diff-stable" reaches it, via the literal predicate an annotation tag without a ':' produces, and returned output byte-identical to --no-diff-stable. It now warns, with a regression test asserting the warning and the byte-identical output that makes silence misleading. Signed-off-by: jdsika --- packages/linkml_runtime/pyproject.toml | 2 +- .../linkml_runtime/utils/rdf_canonicalize.py | 20 ++++++++++++++- .../test_utils/test_rdf_canonicalize.py | 25 +++++++++++++++++++ uv.lock | 15 +++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 1324790b58..9c555f4ace 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.2.0", + "diffable-rdf>=0.3.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index e0fadae34a..8e28488d55 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -307,7 +307,8 @@ def canonicalize_rdf_graph( editing one part of a schema does not renumber unrelated blank nodes. Output is deterministic and isomorphic either way; only the choice of label changes. Off by default because enabling it relabels existing - output. + output. Has no effect on the rdflib fallback path (non-standard RDF), + which warns rather than silently ignoring the request. :return: Deterministic string serialization of the graph. """ ox_format = _FORMAT_MAP.get(output_format.lower()) @@ -338,6 +339,23 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, + # and this path exists precisely because pyoxigraph refused the + # graph, so there are none to refine. The fallback is deterministic + # but not diff-stable: say so rather than returning output that + # silently ignores the argument. ``shaclgen --include-annotations`` + # reaches this path, because an annotation tag without a ``:`` + # becomes a literal predicate. + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 0ca79c3c90..1bc07583af 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -642,3 +642,28 @@ def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " "the flag is probably not threaded into canonicalize_rdf_graph()" ) + + +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): + """A request the fallback cannot honour must be reported, not ignored. + + ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib + fallback exists precisely because pyoxigraph refused the graph. Returning + the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without + saying so lets a caller believe the output is diff-stable when it is not. + """ + graph = _make_graph_with_bnodes() + # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and + # canonicalize_rdf_graph degrades to rdflib -- the same path that + # ``shaclgen --include-annotations`` takes via its literal predicates. + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + + # The warning is the contract: the bytes really are identical, which is + # exactly why staying silent would be misleading. + assert stable == plain diff --git a/uv.lock b/uv.lock index 3b8ef0ceb1..98077774d3 100644 --- a/uv.lock +++ b/uv.lock @@ -1075,6 +1075,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2609,6 +2622,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2641,6 +2655,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.3.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From b7e3b7443bf302eefb5b3b22b1f334325e675723 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 13:45:48 +0200 Subject: [PATCH 06/10] build(deps): require diffable-rdf 0.4.0 0.4.0 carries graph.base through the library's rdflib fallback, verifying that every absolute IRI of the source survives a re-read rather than dropping the directive outright, and adds a diff_stable parameter to canonicalize_rdf_graph. The lock entry is written by hand because the workspace sets exclude-newer = "7 days", which filters any release younger than that from resolution; 0.3.0 was pinned the same way for the same reason, and both become resolvable normally on 2026-09-18. uv lock --check and uv sync --all-groups both accept the entry. https://github.com/ASCS-eV/diffable-rdf/releases/tag/v0.4.0 Signed-off-by: jdsika --- packages/linkml_runtime/pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 9c555f4ace..7509ef60a6 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.3.0", + "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/uv.lock b/uv.lock index 98077774d3..4e5d9464b2 100644 --- a/uv.lock +++ b/uv.lock @@ -1077,15 +1077,15 @@ wheels = [ [[package]] name = "diffable-rdf" -version = "0.3.0" +version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyoxigraph" }, { name = "rdflib" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fe/f3/45771c536aea850f2884206349e35910bbabcaebeec890c5c0cc08b040f9/diffable_rdf-0.3.0.tar.gz", hash = "sha256:6fcf84d9323cd35fc75f423bd73836469cfad198ebb7e6ab86a8743724cd000d", size = 829395, upload-time = "2026-09-11T08:00:26.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/22/ef80dba96cd558f59d2ab225602bde318a925d437866379303e1f41a4e96/diffable_rdf-0.3.0-py3-none-any.whl", hash = "sha256:8756137423f9f28beacb5cd99ba8d66317328a0c936b1dbb4527831c7d9c6784", size = 43578, upload-time = "2026-09-11T08:00:24.542Z" }, + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, ] [[package]] @@ -2655,7 +2655,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, - { name = "diffable-rdf", specifier = ">=0.3.0" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, From 10b3b2d5b516e33432a1a0300e3504940627185a Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Mon, 14 Sep 2026 11:19:02 +0200 Subject: [PATCH 07/10] test(rdf): mark diff-stable tests for diffable-rdf canary CI Registers a diffable_rdf pytest marker and applies it to the six diff_stable tests (including the parametrized per-generator test), so an external CI job can run 'pytest -m diffable_rdf' against a local diffable-rdf checkout without pulling in the rest of the suite. Follow-up to a review comment on this PR requesting canary tests from linkml running in diffable-rdf's own CI, modeled on the numpydantic project's tests-linkml.yml workflow. Signed-off-by: jdsika --- pyproject.toml | 1 + tests/linkml_runtime/test_utils/test_rdf_canonicalize.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 559cf265ac..fd62dc2986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ markers = [ "yarrrml: End-to-end tests for the YARRRML generator", "typedbgen: Tests for the TypeDB generator", "arrays: Array specifications!", + "diffable_rdf: tests that exercise the diffable-rdf integration (--diff-stable); run as a canary against linkml in diffable-rdf's own CI", ] # https://docs.astral.sh/ruff/settings/ diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 1bc07583af..9fc94d38d4 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -529,12 +529,14 @@ def _changed_line_count(before: str, after: str) -> int: return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) +@pytest.mark.diffable_rdf def test_diff_stable_is_opt_in(): """The default must keep producing exactly the output it produced before.""" graph = _make_graph_with_bnodes() assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) +@pytest.mark.diffable_rdf def test_diff_stable_preserves_semantics(): """Relabelling blank nodes must not change what the graph means.""" graph = _make_graph_with_bnodes() @@ -547,6 +549,7 @@ def test_diff_stable_preserves_semantics(): assert rdflib.compare.isomorphic(plain, stable) +@pytest.mark.diffable_rdf def test_diff_stable_is_deterministic(): """Diff stability must not cost determinism, which is the stronger property.""" graph = _make_graph_with_bnodes() @@ -554,6 +557,7 @@ def test_diff_stable_is_deterministic(): assert len(outputs) == 1 +@pytest.mark.diffable_rdf def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): """Inserting one subject must not relabel the blank nodes of the others. @@ -617,6 +621,7 @@ def _generator_cases(): ] +@pytest.mark.diffable_rdf @pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): """Every RDF generator must actually apply the option, not merely accept it. @@ -644,6 +649,7 @@ def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): ) +@pytest.mark.diffable_rdf def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): """A request the fallback cannot honour must be reported, not ignored. From 73f1f5e0fdc78bc8f4e2d0b033c505ce2364fb9f Mon Sep 17 00:00:00 2001 From: jdsika Date: Wed, 16 Sep 2026 07:19:42 +0200 Subject: [PATCH 08/10] Make diff-stable serialization optional and address review feedback Expose runtime and LinkML extras with a lazy import and an installation hint. Move usage guidance into the collaboration guide and shorten generator help. Assert designed RDF edits, source preservation, generator output and CLI flags. Check ordinary and extra wheel installs in the existing package build job. Signed-off-by: jdsika --- .github/tests/test_diff_stable_install.py | 147 ++++++++++ .github/workflows/main.yaml | 26 ++ docs/howtos/collaborative-development.md | 30 ++ packages/linkml/pyproject.toml | 4 + .../linkml/src/linkml/generators/owlgen.py | 20 +- .../linkml/src/linkml/generators/rdfgen.py | 20 +- .../linkml/src/linkml/generators/shaclgen.py | 20 +- .../linkml/src/linkml/generators/shexgen.py | 20 +- packages/linkml_runtime/pyproject.toml | 2 +- .../linkml_runtime/utils/rdf_canonicalize.py | 37 +-- tests/linkml/test_scripts/test_diff_stable.py | 99 +++++++ .../test_utils/test_rdf_canonicalize.py | 271 ++++++++++++------ uv.lock | 19 +- 13 files changed, 525 insertions(+), 190 deletions(-) create mode 100644 .github/tests/test_diff_stable_install.py create mode 100644 tests/linkml/test_scripts/test_diff_stable.py diff --git a/.github/tests/test_diff_stable_install.py b/.github/tests/test_diff_stable_install.py new file mode 100644 index 0000000000..ef2927ccf4 --- /dev/null +++ b/.github/tests/test_diff_stable_install.py @@ -0,0 +1,147 @@ +"""Run each selection in a fresh environment containing the built wheels.""" + +import importlib +import re +import sys +import sysconfig +from importlib.metadata import metadata, version +from importlib.util import find_spec +from pathlib import Path + +import pytest +from packaging.requirements import Requirement + +_SCHEMA = """id: https://example.org/install +name: install +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/install/ +default_prefix: ex +imports: [linkml:types] +classes: + Thing: + attributes: + label: + range: string +""" + + +def _assert_installed(package: str) -> None: + """Require imports from this environment's installed package directory.""" + module = importlib.import_module(package) + assert Path(module.__file__).resolve().is_relative_to(Path(sysconfig.get_path("purelib")).resolve()) + + +def _assert_runtime_requirements() -> None: + """Check ordinary compatibility and the extra's forwarding in wheel metadata.""" + requirements = [Requirement(value) for value in metadata("linkml").get_all("Requires-Dist")] + (ordinary,) = [r for r in requirements if r.name == "linkml-runtime" and r.marker is None] + (forwarded,) = [r for r in requirements if r.name == "linkml-runtime" and r.marker is not None] + assert ordinary.extras == set() + assert forwarded.extras == {"diff-stable"} + assert forwarded.marker.evaluate({"extra": "diff-stable"}) + assert not forwarded.marker.evaluate({"extra": ""}) + assert ordinary.specifier == forwarded.specifier + assert ordinary.specifier.contains(version("linkml-runtime"), prereleases=True) + + +def _assert_diff_stable_labels(output: str) -> None: + """Require actual blank-node terms with the enabled label syntax.""" + import pyoxigraph as ox + + quads = ox.parse(output, format=ox.RdfFormat.TURTLE) + nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} + assert nodes + assert all(re.fullmatch(r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", node.value) for node in nodes) + + +def test_runtime_without_extra() -> None: + """Ordinary runtime installs work and explain how to enable the option.""" + assert find_spec("diffable_rdf") is None + _assert_installed("linkml_runtime") + from rdflib import Graph + from rdflib.compare import isomorphic + + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + + graph = Graph().parse(data=' [ "v" ] .') + for kwargs in ({}, {"diff_stable": False}): + output = canonicalize_rdf_graph(graph, **kwargs) + assert isomorphic(graph, Graph().parse(data=output, format="turtle")) + assert "diffable_rdf" not in sys.modules + with pytest.raises(ImportError, match=r"linkml-runtime\[diff-stable\]"): + canonicalize_rdf_graph(graph, diff_stable=True) + + +def test_linkml_without_extra(tmp_path: Path) -> None: + """Generator imports, help and default output do not require the extra.""" + assert find_spec("diffable_rdf") is None + _assert_installed("linkml_runtime") + _assert_installed("linkml") + _assert_runtime_requirements() + from click.testing import CliRunner + from rdflib import Graph + from rdflib.namespace import RDF, SH + + from linkml.generators.shaclgen import cli + + for name in ("owlgen", "rdfgen", "shaclgen", "shexgen"): + command = importlib.import_module(f"linkml.generators.{name}").cli + result = CliRunner().invoke(command, ["--help"]) + assert result.exit_code == 0, result.output + schema = tmp_path / "schema.yaml" + schema.write_text(_SCHEMA, encoding="utf-8") + result = CliRunner().invoke(cli, [str(schema)]) + assert result.exit_code == 0, result.output + graph = Graph().parse(data=result.stdout, format="turtle") + assert list(graph.subjects(RDF.type, SH.NodeShape)) + assert "diffable_rdf" not in sys.modules + result = CliRunner().invoke(cli, [str(schema), "--diff-stable"]) + assert result.exit_code != 0 + assert isinstance(result.exception, ImportError) + assert "linkml[diff-stable]" in str(result.exception) + + +def test_runtime_with_extra() -> None: + """The runtime extra supplies the relabeler and preserves the source graph.""" + assert find_spec("diffable_rdf") is not None + _assert_installed("linkml_runtime") + from rdflib import Graph + from rdflib.compare import isomorphic + + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + + graph = Graph().parse(data=' [ "v" ] .') + canonicalize_rdf_graph(graph) + assert "diffable_rdf" not in sys.modules + output = canonicalize_rdf_graph(graph, output_format="nt", diff_stable=True) + parsed = Graph().parse(data=output, format="nt") + assert len(parsed) == len(graph) + assert isomorphic(graph, parsed) + _assert_diff_stable_labels(output) + _assert_installed("diffable_rdf") + + +def test_linkml_with_extra(tmp_path: Path) -> None: + """The LinkML extra installs the relabeler through the runtime extra.""" + assert find_spec("diffable_rdf") is not None + _assert_installed("linkml_runtime") + _assert_installed("linkml") + _assert_runtime_requirements() + from click.testing import CliRunner + from rdflib import Graph, Namespace + from rdflib.namespace import RDF, SH + + from linkml.generators.shaclgen import cli + + schema = tmp_path / "schema.yaml" + schema.write_text(_SCHEMA, encoding="utf-8") + result = CliRunner().invoke(cli, [str(schema), "--diff-stable"]) + assert result.exit_code == 0, result.output + graph = Graph().parse(data=result.stdout, format="turtle") + ex = Namespace("https://example.org/install/") + assert (ex.Thing, RDF.type, SH.NodeShape) in graph + assert (ex.Thing, SH.targetClass, ex.Thing) in graph + assert {graph.value(node, SH.path) for node in graph.objects(ex.Thing, SH.property)} == {ex.label} + _assert_diff_stable_labels(result.stdout) + _assert_installed("diffable_rdf") diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a8c122e1bb..dcc4c52ca7 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -237,3 +237,29 @@ jobs: version: ${{ env.UV_VERSION }} - name: Build source and wheel archives run: uv build --all-packages + - name: Test optional diff-stable installations + shell: bash + run: | + shopt -s nullglob + runtime_wheels=(dist/linkml_runtime-*.whl) + linkml_wheels=(dist/linkml-*.whl) + test "${#runtime_wheels[@]}" -eq 1 + test "${#linkml_wheels[@]}" -eq 1 + for scenario in runtime_without_extra linkml_without_extra runtime_with_extra linkml_with_extra; do + scenario_env="$RUNNER_TEMP/$scenario" + uv venv --python 3.13 "$scenario_env" + packages=("${runtime_wheels[0]}" pytest) + case "$scenario" in + runtime_with_extra) packages=("${runtime_wheels[0]}[diff-stable]" pytest) ;; + linkml_without_extra) packages+=("${linkml_wheels[0]}") ;; + linkml_with_extra) packages+=("${linkml_wheels[0]}[diff-stable]") ;; + esac + uv pip install --python "$scenario_env/bin/python" "${packages[@]}" + uv pip check --python "$scenario_env/bin/python" + ( + cd "$RUNNER_TEMP" + uv run --no-project "$scenario_env/bin/python" -I -m pytest \ + --noconftest --import-mode=importlib \ + "$GITHUB_WORKSPACE/.github/tests/test_diff_stable_install.py" -k "$scenario" -v + ) + done diff --git a/docs/howtos/collaborative-development.md b/docs/howtos/collaborative-development.md index 94515df2d1..221337e843 100644 --- a/docs/howtos/collaborative-development.md +++ b/docs/howtos/collaborative-development.md @@ -40,6 +40,36 @@ Some projects may opt to manage their data alongside the schema in the same repo for "registry" style projects, but in other cases the data may be best managed separately in a dedicated store or a repository like Zenodo or Figshare. +(rdf-in-version-control)= +### Keep generated RDF diffs small + +When you track generated RDF in Git, adding a class or property can change many +blank-node labels. The default RDFC-1.0 numbering is repeatable for the same graph, +but an insertion can renumber existing nodes and obscure the actual schema edit. + +The optional `--diff-stable` flag uses +[diffable-rdf](https://github.com/ASCS-eV/diffable-rdf) to reduce this label churn. +In a uv project containing your schema, install LinkML with the extra and generate +SHACL as follows (using a release that includes this option): + +```bash +uv add 'linkml[diff-stable]' +uv run gen-shacl --diff-stable schema.yaml > schema.shacl.ttl +``` + +The flag also works with `gen-owl`, `gen-rdf`, and `gen-shex --format rdf`. +Python callers of `canonicalize_rdf_graph(..., diff_stable=True)` can install +`linkml-runtime[diff-stable]`. The feature is disabled by default; enabling it +changes existing labels once, so keep that regeneration separate from schema edits. + +Labels depend on blank-node neighborhoods. Edits within connected structures or +among symmetric nodes can still affect other labels; minimum diffs are not +guaranteed. Keep tool versions consistent when comparing generated files. +The relabeled output is not the standard RDFC canonical representation. +ShExC, ShEx JSON and instance-data conversion are outside this option's scope. +Non-standard RDF takes the existing fallback path, which warns that it cannot +apply diff-stable labels. + ### Permissively License Your Code and Data (adapted from [O3 guidelines](https://osf.io/vuzt3/)) diff --git a/packages/linkml/pyproject.toml b/packages/linkml/pyproject.toml index 3d8ee1cdf2..9ff6fd1404 100644 --- a/packages/linkml/pyproject.toml +++ b/packages/linkml/pyproject.toml @@ -68,6 +68,9 @@ dependencies = [ # Specifier syntax: https://peps.python.org/pep-0631/ "platformdirs>=4.12.2", ] +[project.optional-dependencies] +diff-stable = ["linkml-runtime[diff-stable]>=1.10.0,<2.0.0"] + [dependency-groups] lint = [ "black >= 24.0.0", @@ -79,6 +82,7 @@ shacl = [ "pyshacl>=0.40.1", ] tests = [ + "linkml-runtime[diff-stable]", { include-group = "lint" }, { include-group = "typing" }, { include-group = "shacl" }, diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 09db0357be..3f16c4965c 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -123,20 +123,7 @@ class OwlSchemaGenerator(Generator): # ObjectVars diff_stable: bool = False - """Label blank nodes so that unrelated edits leave them untouched. - - Output is already deterministic: RDFC-1.0 guarantees that isomorphic - graphs serialize identically. It does not guarantee that *similar* - graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a - global order, so adding one class can renumber every blank node after - it and rewrite most of the file. - - When ``True``, blank-node labels are instead derived from each node's - own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels - only the blank nodes it actually touches. The output stays - deterministic and isomorphic either way; only the choice of label - changes. Off by default because enabling it relabels existing output. - """ + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -1865,9 +1852,8 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: default=False, show_default=True, help=( - "Derive blank-node labels from each node's own neighbourhood so that " - "unrelated edits leave them unchanged. Output is deterministic either " - "way; this makes successive versions of a file diff cleanly." + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" ), ) @click.version_option(__version__, "-V", "--version") diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 052fcff12e..5b8f8dc1c7 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -79,20 +79,7 @@ class RDFGenerator(Generator): # ObjectVars diff_stable: bool = False - """Label blank nodes so that unrelated edits leave them untouched. - - Output is already deterministic: RDFC-1.0 guarantees that isomorphic - graphs serialize identically. It does not guarantee that *similar* - graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a - global order, so adding one class can renumber every blank node after - it and rewrite most of the file. - - When ``True``, blank-node labels are instead derived from each node's - own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels - only the blank nodes it actually touches. The output stays - deterministic and isomorphic either way; only the choice of label - changes. Off by default because enabling it relabels existing output. - """ + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" emit_metadata: bool = False context: list[str] = None @@ -158,9 +145,8 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str default=False, show_default=True, help=( - "Derive blank-node labels from each node's own neighbourhood so that " - "unrelated edits leave them unchanged. Output is deterministic either " - "way; this makes successive versions of a file diff cleanly." + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" ), ) @click.version_option(__version__, "-V", "--version") diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 5d42fea272..1804b1204e 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -143,20 +143,7 @@ class ShaclGenerator(Generator): """ diff_stable: bool = False - """Label blank nodes so that unrelated edits leave them untouched. - - Output is already deterministic: RDFC-1.0 guarantees that isomorphic - graphs serialize identically. It does not guarantee that *similar* - graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a - global order, so adding one class can renumber every blank node after - it and rewrite most of the file. - - When ``True``, blank-node labels are instead derived from each node's - own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels - only the blank nodes it actually touches. The output stays - deterministic and isomorphic either way; only the choice of label - changes. Off by default because enabling it relabels existing output. - """ + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -950,9 +937,8 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: default=False, show_default=True, help=( - "Derive blank-node labels from each node's own neighbourhood so that " - "unrelated edits leave them unchanged. Output is deterministic either " - "way; this makes successive versions of a file diff cleanly." + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" ), ) @click.version_option(__version__, "-V", "--version") diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 7c43b13bc0..e74e45225e 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -41,20 +41,7 @@ class ShExGenerator(Generator): # ObjectVars diff_stable: bool = False - """Label blank nodes so that unrelated edits leave them untouched. - - Output is already deterministic: RDFC-1.0 guarantees that isomorphic - graphs serialize identically. It does not guarantee that *similar* - graphs serialize *similarly* — blank nodes are numbered ``c14nN`` in a - global order, so adding one class can renumber every blank node after - it and rewrite most of the file. - - When ``True``, blank-node labels are instead derived from each node's - own neighbourhood via Weisfeiler-Lehman refinement, so an edit relabels - only the blank nodes it actually touches. The output stays - deterministic and isomorphic either way; only the choice of label - changes. Off by default because enabling it relabels existing output. - """ + """Reduce blank-node label churn in RDF output. See :ref:`rdf-in-version-control`.""" shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) @@ -279,9 +266,8 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: default=False, show_default=True, help=( - "Derive blank-node labels from each node's own neighbourhood so that " - "unrelated edits leave them unchanged. Output is deterministic either " - "way; this makes successive versions of a file diff cleanly." + "Reduce blank-node label churn across edits (--format rdf). See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" ), ) @click.version_option(__version__, "-V", "--version") diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 7509ef60a6..d4c0a2f167 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,7 +48,6 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", - "diffable-rdf>=0.4.0", "pydantic>=2.13.5,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] @@ -63,6 +62,7 @@ dev = [ linkml-normalize = "linkml_runtime.processing.referencevalidator:cli" [project.optional-dependencies] +diff-stable = ["diffable-rdf>=0.4.0"] dev = [ "coverage", "requests-cache>=1.3.3", diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 8e28488d55..d7a894be17 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -39,10 +39,10 @@ import io import re import warnings +from importlib.util import find_spec import pyoxigraph as ox import rdflib -from diffable_rdf import wl_relabel_quads from rdflib.compare import to_canonical_graph @@ -302,15 +302,20 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). - :param diff_stable: Derive blank-node labels from each node's own - neighbourhood instead of RDFC-1.0's global ``c14nN`` numbering, so that - editing one part of a schema does not renumber unrelated blank nodes. - Output is deterministic and isomorphic either way; only the choice of - label changes. Off by default because enabling it relabels existing - output. Has no effect on the rdflib fallback path (non-standard RDF), - which warns rather than silently ignoring the request. + :param diff_stable: Use diffable-rdf labels to reduce blank-node churn across + edits. Requires the ``diff-stable`` extra; disabled by default. The + non-standard RDF fallback warns when it cannot apply these labels. :return: Deterministic string serialization of the graph. + :raises ImportError: If diff-stable labels are requested without diffable-rdf. """ + if diff_stable: + if find_spec("diffable_rdf") is None: + raise ImportError( + "diff_stable=True requires diffable-rdf. Install " + "'linkml-runtime[diff-stable]' or 'linkml[diff-stable]'." + ) + from diffable_rdf import wl_relabel_quads + ox_format = _FORMAT_MAP.get(output_format.lower()) if ox_format is None: warnings.warn( @@ -340,13 +345,6 @@ def canonicalize_rdf_graph( stacklevel=2, ) if diff_stable: - # Weisfeiler-Lehman refinement consumes canonical pyoxigraph quads, - # and this path exists precisely because pyoxigraph refused the - # graph, so there are none to refine. The fallback is deterministic - # but not diff-stable: say so rather than returning output that - # silently ignores the argument. ``shaclgen --include-annotations`` - # reaches this path, because an annotation tag without a ``:`` - # becomes a literal predicate. warnings.warn( "diff_stable=True was requested but this graph took the rdflib fallback, " "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " @@ -367,15 +365,6 @@ def canonicalize_rdf_graph( quads = list(dataset) - # 3b. Optionally re-label blank nodes with locality-sensitive hashes. - # RDFC-1.0 guarantees that identical graphs get identical labels, but it - # does not guarantee that *similar* graphs get similar labels: the labels - # are assigned by a global ordering, so inserting one blank node can - # renumber every label after it and turn a one-line semantic change into a - # whole-file diff. Weisfeiler-Lehman labels are derived only from each - # node's local neighbourhood, so unrelated regions keep their labels. - # Output stays deterministic and isomorphic either way; this only changes - # which label each blank node receives. if diff_stable: quads = wl_relabel_quads(quads) diff --git a/tests/linkml/test_scripts/test_diff_stable.py b/tests/linkml/test_scripts/test_diff_stable.py new file mode 100644 index 0000000000..cd072aed38 --- /dev/null +++ b/tests/linkml/test_scripts/test_diff_stable.py @@ -0,0 +1,99 @@ +"""CLI integration checks for optional diff-stable RDF labels.""" + +import re +from pathlib import Path + +import click +import pyoxigraph as ox +import pytest +from click.testing import CliRunner +from rdflib import Graph, Namespace, URIRef +from rdflib.namespace import OWL, RDF, SH + +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.owlgen import cli as owl_cli +from linkml.generators.rdfgen import RDFGenerator +from linkml.generators.rdfgen import cli as rdf_cli +from linkml.generators.shaclgen import ShaclGenerator +from linkml.generators.shaclgen import cli as shacl_cli +from linkml.generators.shexgen import ShExGenerator +from linkml.generators.shexgen import cli as shex_cli +from linkml.utils.generator import Generator + + +@pytest.mark.parametrize( + ("command", "args", "rdf_type"), + [ + pytest.param(owl_cli, [], OWL.Class, id="owlgen"), + pytest.param( + rdf_cli, [], URIRef("https://w3id.org/linkml/ClassDefinition"), id="rdfgen", marks=pytest.mark.network + ), + pytest.param(shacl_cli, [], SH.NodeShape, id="shaclgen"), + pytest.param( + shex_cli, + ["--format", "rdf"], + URIRef("http://www.w3.org/ns/shex#Shape"), + id="shexgen", + marks=pytest.mark.network, + ), + ], +) +@pytest.mark.parametrize( + ("flag", "label_pattern"), + [ + pytest.param("--no-diff-stable", r"c14n[0-9]+", id="disabled"), + pytest.param("--diff-stable", r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", id="enabled"), + ], +) +def test_diff_stable_cli( + tmp_path: Path, command: click.Command, args: list[str], rdf_type: URIRef, flag: str, label_pattern: str +) -> None: + """Every RDF CLI produces the requested labels and the expected class.""" + schema = tmp_path / "schema.yaml" + schema.write_text( + """id: https://example.org/cli +name: cli +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/cli/ +default_prefix: ex +imports: [linkml:types] +classes: + Thing: + attributes: + label: + range: string +""", + encoding="utf-8", + ) + result = CliRunner().invoke(command, [str(schema), flag, *args]) + assert result.exit_code == 0, result.output + quads = list(ox.parse(result.stdout, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(label_pattern, node.value) for node in blank_nodes) + graph = Graph().parse(data=result.stdout, format="turtle") + assert (Namespace("https://example.org/cli/").Thing, RDF.type, rdf_type) in graph + + +@pytest.mark.parametrize( + ("generator", "command"), + [ + pytest.param(OwlSchemaGenerator, owl_cli, id="owlgen"), + pytest.param(RDFGenerator, rdf_cli, id="rdfgen"), + pytest.param(ShaclGenerator, shacl_cli, id="shaclgen"), + pytest.param(ShExGenerator, shex_cli, id="shexgen"), + ], +) +def test_diff_stable_defaults_and_help(generator: type[Generator], command: click.Command) -> None: + """Defaults are off and terminal help points to the shared guide.""" + assert generator.diff_stable is False + (option,) = [param for param in command.params if param.name == "diff_stable"] + assert option.default is False + result = CliRunner().invoke(command, ["--help"]) + assert result.exit_code == 0 + assert "--diff-stable / --no-diff-stable" in result.output + assert "[default: no-diff-stable]" in result.output + assert "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" in "".join( + result.output.split() + ) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 9fc94d38d4..c3c935a188 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,17 +1,25 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" +import difflib +import inspect import os import re import subprocess import sys import textwrap +from pathlib import Path import pyoxigraph as ox import pytest import rdflib -from rdflib import BNode, Graph, Literal, URIRef -from rdflib.namespace import RDF - +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import OWL, RDF, RDFS, SH, XSD + +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.rdfgen import RDFGenerator +from linkml.generators.shaclgen import ShaclGenerator +from linkml.generators.shexgen import ShExGenerator +from linkml.utils.generator import Generator from linkml_runtime.utils import rdf_canonicalize as rdf_canon_mod from linkml_runtime.utils.rdf_canonicalize import ( RDFCanonicalizationWarning, @@ -227,7 +235,8 @@ def test_curie_in_literal_with_hash_not_treated_as_comment(): assert "ex:bar\\. more" in str(obj) -def test_sort_is_load_bearing(): +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +def test_sort_is_load_bearing(diff_stable: bool) -> None: """Output is byte-identical across subprocesses with different PYTHONHASHSEED values. RDFC-1.0 stabilizes blank-node labels, but pyoxigraph's ``Dataset`` iteration @@ -240,6 +249,8 @@ def test_sort_is_load_bearing(): """ program = textwrap.dedent( """ + import sys + from rdflib import BNode, Graph, Literal, URIRef from rdflib.namespace import RDF from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph @@ -255,7 +266,7 @@ def test_sort_is_load_bearing(): g.add((URIRef("http://example.com/x"), URIRef("http://example.com/has"), bn)) g.add((bn, URIRef("http://example.com/q"), Literal(f"bn_{i}"))) - print(canonicalize_rdf_graph(g, output_format="turtle"), end="") + print(canonicalize_rdf_graph(g, output_format="turtle", diff_stable=sys.argv[1] == "True"), end="") """ ) @@ -264,7 +275,7 @@ def run(seed: str) -> str: # wholesale drops SystemRoot on Windows, which breaks winsock init. env = {**os.environ, "PYTHONHASHSEED": seed} result = subprocess.run( - [sys.executable, "-c", program], + [sys.executable, "-c", program, str(diff_stable)], check=True, capture_output=True, text=True, @@ -523,61 +534,120 @@ def _shapes_graph(count: int, extra: bool = False) -> Graph: def _changed_line_count(before: str, after: str) -> int: - import difflib + """Count added and removed content lines in a unified diff.""" diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) -@pytest.mark.diffable_rdf -def test_diff_stable_is_opt_in(): - """The default must keep producing exactly the output it produced before.""" - graph = _make_graph_with_bnodes() - assert canonicalize_rdf_graph(graph) == canonicalize_rdf_graph(graph, diff_stable=False) +def test_diff_stable_is_opt_in() -> None: + """Diff-stable labels require an explicit request.""" + assert inspect.signature(canonicalize_rdf_graph).parameters["diff_stable"].default is False -@pytest.mark.diffable_rdf -def test_diff_stable_preserves_semantics(): - """Relabelling blank nodes must not change what the graph means.""" +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +@pytest.mark.parametrize("output_format", ["nt", "turtle"]) +def test_diff_stable_preserves_semantics(diff_stable: bool, output_format: str) -> None: + """Each serialization preserves the source graph under blank-node renaming.""" graph = _make_graph_with_bnodes() - - plain = rdflib.Graph() - plain.parse(data=canonicalize_rdf_graph(graph), format="turtle") - stable = rdflib.Graph() - stable.parse(data=canonicalize_rdf_graph(graph, diff_stable=True), format="turtle") - - assert rdflib.compare.isomorphic(plain, stable) + output = canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) + parsed = Graph().parse(data=output, format=output_format) + assert len(parsed) == len(graph) + assert rdflib.compare.isomorphic(graph, parsed) @pytest.mark.diffable_rdf -def test_diff_stable_is_deterministic(): - """Diff stability must not cost determinism, which is the stronger property.""" - graph = _make_graph_with_bnodes() - outputs = {canonicalize_rdf_graph(graph, diff_stable=True) for _ in range(5)} - assert len(outputs) == 1 +def test_diff_stable_is_deterministic() -> None: + """A cycle's tied signatures remain independent of input blank-node IDs.""" + graphs = [] + predicate = URIRef("http://example.com/next") + for labels, reverse in [("abcd", False), ("dbac", True)]: + nodes = [BNode(label) for label in labels] + triples = [(node, predicate, nodes[(i + 1) % len(nodes)]) for i, node in enumerate(nodes)] + graph = Graph() + for triple in reversed(triples) if reverse else triples: + graph.add(triple) + graphs.append(graph) + + assert rdflib.compare.isomorphic(*graphs) + outputs = [canonicalize_rdf_graph(graph, output_format="nt", diff_stable=True) for graph in graphs] + assert outputs[0] == outputs[1] + for graph, output in zip(graphs, outputs): + assert rdflib.compare.isomorphic(graph, Graph().parse(data=output, format="nt")) + + +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +def test_diff_stable_insertion_labels(diff_stable: bool) -> None: + """An independent insertion renumbers RDFC labels but preserves the old WL label.""" + before = Graph().parse( + data="""@prefix ex: . + ex:Shape00 ex:property _:old . + _:old ex:path ex:p00 . + """, + format="turtle", + ) + after = Graph() + for triple in before: + after.add(triple) + inserted = BNode() + ex = Namespace("http://example.com/") + after.add((ex.ShapeAAAinserted, ex.property, inserted)) + after.add((inserted, ex.path, ex.pAAAinserted)) + + outputs = [canonicalize_rdf_graph(graph, output_format="nt", diff_stable=diff_stable) for graph in (before, after)] + parsed = [set(ox.parse(output, format=ox.RdfFormat.N_TRIPLES)) for output in outputs] + old_subject = ox.NamedNode(str(ex.Shape00)) + new_subject = ox.NamedNode(str(ex.ShapeAAAinserted)) + property_iri = ox.NamedNode(str(ex.property)) + path_iri = ox.NamedNode(str(ex.path)) + (old_before,) = {q.object for q in parsed[0] if q.subject == old_subject and q.predicate == property_iri} + (old_after,) = {q.object for q in parsed[1] if q.subject == old_subject and q.predicate == property_iri} + (new_after,) = {q.object for q in parsed[1] if q.subject == new_subject and q.predicate == property_iri} + assert all(isinstance(node, ox.BlankNode) for node in (old_before, old_after, new_after)) + assert old_after != new_after + inserted_quads = { + ox.Quad(new_subject, property_iri, new_after), + ox.Quad(new_after, path_iri, ox.NamedNode(str(ex.pAAAinserted))), + } + assert parsed[0] == { + ox.Quad(old_subject, property_iri, old_before), + ox.Quad(old_before, path_iri, ox.NamedNode(str(ex.p00))), + } + assert ( + parsed[1] + == { + ox.Quad(old_subject, property_iri, old_after), + ox.Quad(old_after, path_iri, ox.NamedNode(str(ex.p00))), + } + | inserted_quads + ) + + if diff_stable: + assert all(re.fullmatch(_STABLE_LABEL, node.value) for node in (old_before, old_after, new_after)) + assert old_before == old_after + assert parsed[1] - parsed[0] == inserted_quads + assert parsed[0] - parsed[1] == set() + else: + assert (old_before.value, old_after.value, new_after.value) == ("c14n0", "c14n1", "c14n0") + for graph, output in zip((before, after), outputs): + assert rdflib.compare.isomorphic(graph, Graph().parse(data=output, format="nt")) @pytest.mark.diffable_rdf -def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): - """Inserting one subject must not relabel the blank nodes of the others. - - RDFC-1.0 numbers blank nodes ``c14nN`` in a global order, so a subject - sorting before the others shifts every subsequent label and rewrites - most of the file. This is the entire reason the option exists, so the - assertion is on the *ratio*, not on an absolute line count that would - be brittle across rdflib versions. - """ +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches() -> None: + """Diff-stable labels reduce Turtle line churn for independent property nodes.""" before, after = _shapes_graph(20), _shapes_graph(20, extra=True) - baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) stable = _changed_line_count( canonicalize_rdf_graph(before, diff_stable=True), canonicalize_rdf_graph(after, diff_stable=True), ) - assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" +_STABLE_LABEL = r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?" + + _DIFF_STABLE_SCHEMA = """\ id: https://example.org/diffstable name: diffstable @@ -602,74 +672,87 @@ def test_diff_stable_confines_an_insertion_to_the_lines_it_touches(): """ -def _generator_cases(): - """The four generators that serialize RDF, with the args that make them do so.""" - from linkml.generators.owlgen import OwlSchemaGenerator - from linkml.generators.rdfgen import RDFGenerator - from linkml.generators.shaclgen import ShaclGenerator - from linkml.generators.shexgen import ShExGenerator +@pytest.fixture +def diff_stable_schema(tmp_path: Path) -> Path: + """Write the small schema for real generator calls.""" + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8") + return schema - return [ + +@pytest.mark.diffable_rdf +@pytest.mark.parametrize( + ("generator", "kwargs"), + [ pytest.param(OwlSchemaGenerator, {}, id="owlgen"), - # rdfgen and shexgen resolve JSON-LD contexts (linkml types, shex.jsonld); - # the `network` marker serves those from local stubs. See tests/conftest.py. pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), pytest.param(ShaclGenerator, {}, id="shaclgen"), - # ShExGenerator only emits RDF in this format; its default is ShExC text, - # where blank-node labelling does not apply. pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), - ] - - -@pytest.mark.diffable_rdf -@pytest.mark.parametrize(("generator", "kwargs"), _generator_cases()) -def test_diff_stable_reaches_every_rdf_generator(tmp_path, generator, kwargs): - """Every RDF generator must actually apply the option, not merely accept it. - - Asserting only that the attribute exists would pass even if a generator - forgot to pass it down to :func:`canonicalize_rdf_graph`. Instead this - checks the observable consequence: RDFC-1.0 names blank nodes ``c14nN``, - while diff-stable labels are neighbourhood hashes, so a generator that - honours the flag emits no ``c14nN`` label at all. - """ - assert generator.diff_stable is False, f"{generator.__name__} must default to off" - - schema = tmp_path / "schema.yaml" - schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8", newline="\n") - - plain = generator(str(schema), **kwargs).serialize() - stable = generator(str(schema), diff_stable=True, **kwargs).serialize() - - # Guards the test itself: if the fixture stopped producing blank nodes the - # assertion below would hold vacuously. - assert re.search(r"c14n\d+", plain), f"{generator.__name__} output has no blank nodes to relabel" - assert not re.search(r"c14n\d+", stable), ( - f"{generator.__name__} still emits RDFC-1.0 blank-node labels with diff_stable=True; " - "the flag is probably not threaded into canonicalize_rdf_graph()" - ) + ], +) +def test_diff_stable_reaches_every_rdf_generator( + diff_stable_schema: Path, generator: type[Generator], kwargs: dict[str, str] +) -> None: + """Every RDF generator emits the designed schema with diff-stable labels.""" + output = generator(str(diff_stable_schema), diff_stable=True, **kwargs).serialize() + quads = list(ox.parse(output, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for quad in quads for term in (quad.subject, quad.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(_STABLE_LABEL, node.value) for node in blank_nodes) + graph = Graph().parse(data=output, format="turtle") + ex = Namespace("https://example.org/diffstable/") + if generator is OwlSchemaGenerator: + assert (ex.Person, RDF.type, OWL.Class) in graph + assert (ex.Organization, RDF.type, OWL.Class) in graph + (restriction,) = { + node + for node in graph.objects(ex.Person, RDFS.subClassOf) + if (node, OWL.onProperty, ex.knows) in graph and (node, OWL.allValuesFrom, ex.Person) in graph + } + assert isinstance(restriction, BNode) + assert (restriction, RDF.type, OWL.Restriction) in graph + assert (restriction, OWL.allValuesFrom, ex.Person) in graph + elif generator is ShaclGenerator: + for subject, paths in [(ex.Person, {ex.name, ex.knows}), (ex.Organization, {ex.name})]: + assert (subject, RDF.type, SH.NodeShape) in graph + assert (subject, SH.targetClass, subject) in graph + properties = set(graph.objects(subject, SH.property)) + assert {graph.value(prop, SH.path) for prop in properties} == paths + (ignored,) = graph.objects(subject, SH.ignoredProperties) + assert set(graph.predicate_objects(ignored)) == {(RDF.first, RDF.type), (RDF.rest, RDF.nil)} + (knows,) = {node for node in graph.objects(ex.Person, SH.property) if (node, SH.path, ex.knows) in graph} + assert (knows, SH["class"], ex.Person) in graph + elif generator is RDFGenerator: + linkml = Namespace("https://w3id.org/linkml/") + schema = ex.diffstable + assert (schema, RDF.type, linkml.SchemaDefinition) in graph + assert set(graph.objects(schema, linkml.classes)) == {ex.Person, ex.Organization} + (prefix,) = {node for node in graph.objects(schema, SH.declare) if (node, SH.prefix, Literal("ex")) in graph} + assert (prefix, SH.namespace, Literal(str(ex), datatype=XSD.anyURI)) in graph + else: + assert generator is ShExGenerator + shex = Namespace("http://www.w3.org/ns/shex#") + assert (ex.Person, RDF.type, shex.Shape) in graph + (expression,) = graph.objects(ex.Person, shex.expression) + assert (expression, RDF.type, shex.EachOf) in graph + (expressions,) = graph.objects(expression, shex.expressions) + assert ex.Person_tes in set(graph.items(expressions)) + assert (ex.Person_tes, RDF.type, shex.EachOf) in graph + (slots,) = graph.objects(ex.Person_tes, shex.expressions) + (knows,) = {node for node in graph.items(slots) if (node, shex.predicate, ex.knows) in graph} + assert (knows, RDF.type, shex.TripleConstraint) in graph + assert (knows, shex.valueExpr, ex.Person) in graph + assert (knows, shex.min, Literal(0)) in graph + assert (knows, shex.max, Literal(-1)) in graph @pytest.mark.diffable_rdf -def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback(): - """A request the fallback cannot honour must be reported, not ignored. - - ``wl_relabel_quads`` consumes canonical pyoxigraph quads, and the rdflib - fallback exists precisely because pyoxigraph refused the graph. Returning - the same bytes for ``diff_stable=True`` and ``diff_stable=False`` without - saying so lets a caller believe the output is diff-stable when it is not. - """ +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback() -> None: + """A relative IRI requires the fallback and a warning that labels cannot apply.""" graph = _make_graph_with_bnodes() - # A relative IRI is non-standard RDF, so pyoxigraph rejects the graph and - # canonicalize_rdf_graph degrades to rdflib -- the same path that - # ``shaclgen --include-annotations`` takes via its literal predicates. graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) - with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): stable = canonicalize_rdf_graph(graph, diff_stable=True) - with pytest.warns(RDFCanonicalizationWarning): plain = canonicalize_rdf_graph(graph, diff_stable=False) - - # The warning is the contract: the bytes really are identical, which is - # exactly why staying silent would be misleading. assert stable == plain diff --git a/uv.lock b/uv.lock index 4e5d9464b2..7c22c3d061 100644 --- a/uv.lock +++ b/uv.lock @@ -2408,6 +2408,11 @@ dependencies = [ { name = "watchdog" }, ] +[package.optional-dependencies] +diff-stable = [ + { name = "linkml-runtime", extra = ["diff-stable"] }, +] + [package.dev-dependencies] bigquery = [ { name = "sqlalchemy-bigquery" }, @@ -2422,6 +2427,7 @@ dev = [ { name = "ipython-genutils" }, { name = "jsonpatch" }, { name = "jupyter" }, + { name = "linkml-runtime", extra = ["diff-stable"] }, { name = "mock" }, { name = "myst-nb" }, { name = "nbconvert" }, @@ -2474,6 +2480,7 @@ shacl = [ tests = [ { name = "black" }, { name = "duckdb" }, + { name = "linkml-runtime", extra = ["diff-stable"] }, { name = "numpydantic" }, { name = "pyshacl" }, { name = "sqlalchemy-bigquery" }, @@ -2513,6 +2520,7 @@ requires-dist = [ { name = "jsonasobj2", specifier = ">=1.0.3,<2.0.0" }, { name = "jsonschema", extras = ["format"], specifier = ">=4.26.0" }, { name = "linkml-runtime", editable = "packages/linkml_runtime" }, + { name = "linkml-runtime", extras = ["diff-stable"], marker = "extra == 'diff-stable'", editable = "packages/linkml_runtime" }, { name = "openapi-spec-validator", specifier = ">=0.8.4" }, { name = "openpyxl" }, { name = "parse" }, @@ -2533,6 +2541,7 @@ requires-dist = [ { name = "typing-extensions", marker = "python_full_version < '3.12'", specifier = ">=4.6.0" }, { name = "watchdog", specifier = ">=0.9.0" }, ] +provides-extras = ["diff-stable"] [package.metadata.requires-dev] bigquery = [{ name = "sqlalchemy-bigquery", specifier = ">=1.9.0" }] @@ -2546,6 +2555,7 @@ dev = [ { name = "ipython-genutils" }, { name = "jsonpatch", specifier = ">=1.33" }, { name = "jupyter" }, + { name = "linkml-runtime", extras = ["diff-stable"], editable = "packages/linkml_runtime" }, { name = "mock", specifier = ">=5.1.0" }, { name = "myst-nb", marker = "python_full_version >= '3.10'", specifier = ">=1.4.0" }, { name = "nbconvert" }, @@ -2593,6 +2603,7 @@ shacl = [{ name = "pyshacl", specifier = ">=0.40.1" }] tests = [ { name = "black", specifier = ">=24.0.0" }, { name = "duckdb", specifier = ">=1.5.5" }, + { name = "linkml-runtime", extras = ["diff-stable"], editable = "packages/linkml_runtime" }, { name = "numpydantic", specifier = ">=1.10.0" }, { name = "pyshacl", specifier = ">=0.40.1" }, { name = "sqlalchemy-bigquery", specifier = ">=1.17.2" }, @@ -2622,7 +2633,6 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, - { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2642,6 +2652,9 @@ dev = [ { name = "coverage" }, { name = "requests-cache" }, ] +diff-stable = [ + { name = "diffable-rdf" }, +] [package.dev-dependencies] dev = [ @@ -2655,7 +2668,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, - { name = "diffable-rdf", specifier = ">=0.4.0" }, + { name = "diffable-rdf", marker = "extra == 'diff-stable'", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, @@ -2670,7 +2683,7 @@ requires-dist = [ { name = "requests" }, { name = "requests-cache", marker = "extra == 'dev'", specifier = ">=1.3.3" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "diff-stable"] [package.metadata.requires-dev] dev = [ From 2989c7469dd33c7da1eab0de5c44e9e332b22969 Mon Sep 17 00:00:00 2001 From: jdsika Date: Fri, 2 Oct 2026 12:03:48 +0200 Subject: [PATCH 09/10] test(rdf): trim the diff-stable tests and reuse the existing fixture Addresses the second review round. Remove the installed-wheel checks and the CI step that drove them. They were a one-off apparatus for something the test suite already has a mechanism for, and part of it only asserted that uv builds wheel metadata correctly from pyproject.toml rather than testing anything about linkml. Test the missing extra with a new mock_missing_import fixture built on the existing MockImportErrorFinder, so the next optional dependency needs no fixture of its own. Guard the import the way bigquerygen already does: find_spec raises under that fixture, so the install hint never surfaced. Drop the default-value test, restore the pre-existing sort test, and cut the CLI matrix to the single case that shows the flag reaching the serializer. Record the convention in AGENTS.md, point MockImportErrorFinder at the general fixture, and link the guide from the diff_stable parameter. Signed-off-by: jdsika --- .github/tests/test_diff_stable_install.py | 147 ------------------ .github/workflows/main.yaml | 26 ---- AGENTS.md | 2 + .../linkml_runtime/utils/rdf_canonicalize.py | 9 +- tests/conftest.py | 37 ++++- tests/linkml/test_scripts/test_diff_stable.py | 90 +++-------- .../test_utils/test_rdf_canonicalize.py | 19 ++- 7 files changed, 70 insertions(+), 260 deletions(-) delete mode 100644 .github/tests/test_diff_stable_install.py diff --git a/.github/tests/test_diff_stable_install.py b/.github/tests/test_diff_stable_install.py deleted file mode 100644 index ef2927ccf4..0000000000 --- a/.github/tests/test_diff_stable_install.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Run each selection in a fresh environment containing the built wheels.""" - -import importlib -import re -import sys -import sysconfig -from importlib.metadata import metadata, version -from importlib.util import find_spec -from pathlib import Path - -import pytest -from packaging.requirements import Requirement - -_SCHEMA = """id: https://example.org/install -name: install -prefixes: - linkml: https://w3id.org/linkml/ - ex: https://example.org/install/ -default_prefix: ex -imports: [linkml:types] -classes: - Thing: - attributes: - label: - range: string -""" - - -def _assert_installed(package: str) -> None: - """Require imports from this environment's installed package directory.""" - module = importlib.import_module(package) - assert Path(module.__file__).resolve().is_relative_to(Path(sysconfig.get_path("purelib")).resolve()) - - -def _assert_runtime_requirements() -> None: - """Check ordinary compatibility and the extra's forwarding in wheel metadata.""" - requirements = [Requirement(value) for value in metadata("linkml").get_all("Requires-Dist")] - (ordinary,) = [r for r in requirements if r.name == "linkml-runtime" and r.marker is None] - (forwarded,) = [r for r in requirements if r.name == "linkml-runtime" and r.marker is not None] - assert ordinary.extras == set() - assert forwarded.extras == {"diff-stable"} - assert forwarded.marker.evaluate({"extra": "diff-stable"}) - assert not forwarded.marker.evaluate({"extra": ""}) - assert ordinary.specifier == forwarded.specifier - assert ordinary.specifier.contains(version("linkml-runtime"), prereleases=True) - - -def _assert_diff_stable_labels(output: str) -> None: - """Require actual blank-node terms with the enabled label syntax.""" - import pyoxigraph as ox - - quads = ox.parse(output, format=ox.RdfFormat.TURTLE) - nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} - assert nodes - assert all(re.fullmatch(r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", node.value) for node in nodes) - - -def test_runtime_without_extra() -> None: - """Ordinary runtime installs work and explain how to enable the option.""" - assert find_spec("diffable_rdf") is None - _assert_installed("linkml_runtime") - from rdflib import Graph - from rdflib.compare import isomorphic - - from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph - - graph = Graph().parse(data=' [ "v" ] .') - for kwargs in ({}, {"diff_stable": False}): - output = canonicalize_rdf_graph(graph, **kwargs) - assert isomorphic(graph, Graph().parse(data=output, format="turtle")) - assert "diffable_rdf" not in sys.modules - with pytest.raises(ImportError, match=r"linkml-runtime\[diff-stable\]"): - canonicalize_rdf_graph(graph, diff_stable=True) - - -def test_linkml_without_extra(tmp_path: Path) -> None: - """Generator imports, help and default output do not require the extra.""" - assert find_spec("diffable_rdf") is None - _assert_installed("linkml_runtime") - _assert_installed("linkml") - _assert_runtime_requirements() - from click.testing import CliRunner - from rdflib import Graph - from rdflib.namespace import RDF, SH - - from linkml.generators.shaclgen import cli - - for name in ("owlgen", "rdfgen", "shaclgen", "shexgen"): - command = importlib.import_module(f"linkml.generators.{name}").cli - result = CliRunner().invoke(command, ["--help"]) - assert result.exit_code == 0, result.output - schema = tmp_path / "schema.yaml" - schema.write_text(_SCHEMA, encoding="utf-8") - result = CliRunner().invoke(cli, [str(schema)]) - assert result.exit_code == 0, result.output - graph = Graph().parse(data=result.stdout, format="turtle") - assert list(graph.subjects(RDF.type, SH.NodeShape)) - assert "diffable_rdf" not in sys.modules - result = CliRunner().invoke(cli, [str(schema), "--diff-stable"]) - assert result.exit_code != 0 - assert isinstance(result.exception, ImportError) - assert "linkml[diff-stable]" in str(result.exception) - - -def test_runtime_with_extra() -> None: - """The runtime extra supplies the relabeler and preserves the source graph.""" - assert find_spec("diffable_rdf") is not None - _assert_installed("linkml_runtime") - from rdflib import Graph - from rdflib.compare import isomorphic - - from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph - - graph = Graph().parse(data=' [ "v" ] .') - canonicalize_rdf_graph(graph) - assert "diffable_rdf" not in sys.modules - output = canonicalize_rdf_graph(graph, output_format="nt", diff_stable=True) - parsed = Graph().parse(data=output, format="nt") - assert len(parsed) == len(graph) - assert isomorphic(graph, parsed) - _assert_diff_stable_labels(output) - _assert_installed("diffable_rdf") - - -def test_linkml_with_extra(tmp_path: Path) -> None: - """The LinkML extra installs the relabeler through the runtime extra.""" - assert find_spec("diffable_rdf") is not None - _assert_installed("linkml_runtime") - _assert_installed("linkml") - _assert_runtime_requirements() - from click.testing import CliRunner - from rdflib import Graph, Namespace - from rdflib.namespace import RDF, SH - - from linkml.generators.shaclgen import cli - - schema = tmp_path / "schema.yaml" - schema.write_text(_SCHEMA, encoding="utf-8") - result = CliRunner().invoke(cli, [str(schema), "--diff-stable"]) - assert result.exit_code == 0, result.output - graph = Graph().parse(data=result.stdout, format="turtle") - ex = Namespace("https://example.org/install/") - assert (ex.Thing, RDF.type, SH.NodeShape) in graph - assert (ex.Thing, SH.targetClass, ex.Thing) in graph - assert {graph.value(node, SH.path) for node in graph.objects(ex.Thing, SH.property)} == {ex.label} - _assert_diff_stable_labels(result.stdout) - _assert_installed("diffable_rdf") diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dcc4c52ca7..a8c122e1bb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -237,29 +237,3 @@ jobs: version: ${{ env.UV_VERSION }} - name: Build source and wheel archives run: uv build --all-packages - - name: Test optional diff-stable installations - shell: bash - run: | - shopt -s nullglob - runtime_wheels=(dist/linkml_runtime-*.whl) - linkml_wheels=(dist/linkml-*.whl) - test "${#runtime_wheels[@]}" -eq 1 - test "${#linkml_wheels[@]}" -eq 1 - for scenario in runtime_without_extra linkml_without_extra runtime_with_extra linkml_with_extra; do - scenario_env="$RUNNER_TEMP/$scenario" - uv venv --python 3.13 "$scenario_env" - packages=("${runtime_wheels[0]}" pytest) - case "$scenario" in - runtime_with_extra) packages=("${runtime_wheels[0]}[diff-stable]" pytest) ;; - linkml_without_extra) packages+=("${linkml_wheels[0]}") ;; - linkml_with_extra) packages+=("${linkml_wheels[0]}[diff-stable]") ;; - esac - uv pip install --python "$scenario_env/bin/python" "${packages[@]}" - uv pip check --python "$scenario_env/bin/python" - ( - cd "$RUNNER_TEMP" - uv run --no-project "$scenario_env/bin/python" -I -m pytest \ - --noconftest --import-mode=importlib \ - "$GITHUB_WORKSPACE/.github/tests/test_diff_stable_install.py" -k "$scenario" -v - ) - done diff --git a/AGENTS.md b/AGENTS.md index 772f7c59ab..c9a178e6c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,8 @@ All commands use `uv run` prefix (e.g., `uv run pytest`). * Do not "fix" issues by changing or weakening test conditions. Try harder, or ask questions if a test fails. * Avoid try/except blocks, these can mask bugs * Failing fast is a good principle +* Optional dependencies are the documented exception to the two rules above. Guard the import, not the spec: `try: import x` / `except ImportError as exc: raise ImportError("x is required. Install with: pip install 'linkml[extra]'") from exc` - see `generators/bigquerygen.py`. `importlib.util.find_spec` raises under the test fixture below, which hides your message. +* Testing that an optional dependency is absent is not a mock test: use the `mock_missing_import` fixture in `tests/conftest.py`. Do not build isolated environments in CI for this. * Follow the DRY principle * Avoid repeating chunks of code, but also avoid premature over-abstraction * Declarative principles are favored diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index d7a894be17..bc06a969b8 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -39,7 +39,6 @@ import io import re import warnings -from importlib.util import find_spec import pyoxigraph as ox import rdflib @@ -305,16 +304,18 @@ def canonicalize_rdf_graph( :param diff_stable: Use diffable-rdf labels to reduce blank-node churn across edits. Requires the ``diff-stable`` extra; disabled by default. The non-standard RDF fallback warns when it cannot apply these labels. + See https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control :return: Deterministic string serialization of the graph. :raises ImportError: If diff-stable labels are requested without diffable-rdf. """ if diff_stable: - if find_spec("diffable_rdf") is None: + try: + from diffable_rdf import wl_relabel_quads + except ImportError as exc: raise ImportError( "diff_stable=True requires diffable-rdf. Install " "'linkml-runtime[diff-stable]' or 'linkml[diff-stable]'." - ) - from diffable_rdf import wl_relabel_quads + ) from exc ox_format = _FORMAT_MAP.get(output_format.lower()) if ox_format is None: diff --git a/tests/conftest.py b/tests/conftest.py index bcf958b9e5..ba13d6f764 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -422,7 +422,8 @@ class MockImportErrorFinder(MetaPathFinder): """ Fake like we don't have a module when we really do. - see the ``mock_black_import`` fixture for example usage. + see the ``mock_missing_import`` fixture to hide any module, or ``mock_black_import`` + for a single-purpose example. .. note:: @@ -442,6 +443,40 @@ def find_spec(self, fullname, path, target): return None +@pytest.fixture(scope="function") +def mock_missing_import(): + """Pretend an optional dependency is not installed, for any module. + + Yields a callable taking a module name, so a test for one optional + dependency does not need a fixture of its own:: + + def test_requires_the_extra(mock_missing_import): + mock_missing_import("diffable_rdf") + with pytest.raises(ImportError, match="linkml-runtime[diff-stable]"): + ... + + See :class:`MockImportErrorFinder` for the caveat about reimporting + modules that already imported the one being hidden. + """ + removed = {} + finders = [] + + def hide(module: str) -> None: + for name, loaded in list(sys.modules.items()): + if name.startswith(module): + removed[name] = loaded + del sys.modules[name] + finder = MockImportErrorFinder(module) + finders.append(finder) + sys.meta_path.insert(0, finder) + + yield hide + + sys.modules.update(removed) + for finder in finders: + sys.meta_path.remove(finder) + + @pytest.fixture(scope="function") def mock_black_import(): """ diff --git a/tests/linkml/test_scripts/test_diff_stable.py b/tests/linkml/test_scripts/test_diff_stable.py index cd072aed38..d1c292276b 100644 --- a/tests/linkml/test_scripts/test_diff_stable.py +++ b/tests/linkml/test_scripts/test_diff_stable.py @@ -1,57 +1,15 @@ -"""CLI integration checks for optional diff-stable RDF labels.""" +"""CLI checks for the optional diff-stable RDF labels.""" import re from pathlib import Path -import click import pyoxigraph as ox import pytest from click.testing import CliRunner -from rdflib import Graph, Namespace, URIRef -from rdflib.namespace import OWL, RDF, SH -from linkml.generators.owlgen import OwlSchemaGenerator -from linkml.generators.owlgen import cli as owl_cli -from linkml.generators.rdfgen import RDFGenerator -from linkml.generators.rdfgen import cli as rdf_cli -from linkml.generators.shaclgen import ShaclGenerator from linkml.generators.shaclgen import cli as shacl_cli -from linkml.generators.shexgen import ShExGenerator -from linkml.generators.shexgen import cli as shex_cli -from linkml.utils.generator import Generator - -@pytest.mark.parametrize( - ("command", "args", "rdf_type"), - [ - pytest.param(owl_cli, [], OWL.Class, id="owlgen"), - pytest.param( - rdf_cli, [], URIRef("https://w3id.org/linkml/ClassDefinition"), id="rdfgen", marks=pytest.mark.network - ), - pytest.param(shacl_cli, [], SH.NodeShape, id="shaclgen"), - pytest.param( - shex_cli, - ["--format", "rdf"], - URIRef("http://www.w3.org/ns/shex#Shape"), - id="shexgen", - marks=pytest.mark.network, - ), - ], -) -@pytest.mark.parametrize( - ("flag", "label_pattern"), - [ - pytest.param("--no-diff-stable", r"c14n[0-9]+", id="disabled"), - pytest.param("--diff-stable", r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", id="enabled"), - ], -) -def test_diff_stable_cli( - tmp_path: Path, command: click.Command, args: list[str], rdf_type: URIRef, flag: str, label_pattern: str -) -> None: - """Every RDF CLI produces the requested labels and the expected class.""" - schema = tmp_path / "schema.yaml" - schema.write_text( - """id: https://example.org/cli +_SCHEMA = """id: https://example.org/cli name: cli prefixes: linkml: https://w3id.org/linkml/ @@ -63,37 +21,25 @@ def test_diff_stable_cli( attributes: label: range: string -""", - encoding="utf-8", - ) - result = CliRunner().invoke(command, [str(schema), flag, *args]) - assert result.exit_code == 0, result.output - quads = list(ox.parse(result.stdout, format=ox.RdfFormat.TURTLE)) - blank_nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} - assert blank_nodes - assert all(re.fullmatch(label_pattern, node.value) for node in blank_nodes) - graph = Graph().parse(data=result.stdout, format="turtle") - assert (Namespace("https://example.org/cli/").Thing, RDF.type, rdf_type) in graph +""" @pytest.mark.parametrize( - ("generator", "command"), + ("flag", "label_pattern"), [ - pytest.param(OwlSchemaGenerator, owl_cli, id="owlgen"), - pytest.param(RDFGenerator, rdf_cli, id="rdfgen"), - pytest.param(ShaclGenerator, shacl_cli, id="shaclgen"), - pytest.param(ShExGenerator, shex_cli, id="shexgen"), + pytest.param("--no-diff-stable", r"c14n[0-9]+", id="disabled"), + pytest.param("--diff-stable", r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", id="enabled", marks=pytest.mark.diffable_rdf), ], ) -def test_diff_stable_defaults_and_help(generator: type[Generator], command: click.Command) -> None: - """Defaults are off and terminal help points to the shared guide.""" - assert generator.diff_stable is False - (option,) = [param for param in command.params if param.name == "diff_stable"] - assert option.default is False - result = CliRunner().invoke(command, ["--help"]) - assert result.exit_code == 0 - assert "--diff-stable / --no-diff-stable" in result.output - assert "[default: no-diff-stable]" in result.output - assert "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" in "".join( - result.output.split() - ) +def test_the_flag_reaches_the_serializer(tmp_path: Path, flag: str, label_pattern: str) -> None: + """The CLI flag selects the blank-node labelling it names.""" + schema = tmp_path / "schema.yaml" + schema.write_text(_SCHEMA, encoding="utf-8") + + result = CliRunner().invoke(shacl_cli, [str(schema), flag]) + + assert result.exit_code == 0, result.output + quads = list(ox.parse(result.stdout, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(label_pattern, node.value) for node in blank_nodes) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index c3c935a188..8da48fa8b3 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,7 +1,6 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" import difflib -import inspect import os import re import subprocess @@ -235,8 +234,7 @@ def test_curie_in_literal_with_hash_not_treated_as_comment(): assert "ex:bar\\. more" in str(obj) -@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) -def test_sort_is_load_bearing(diff_stable: bool) -> None: +def test_sort_is_load_bearing(): """Output is byte-identical across subprocesses with different PYTHONHASHSEED values. RDFC-1.0 stabilizes blank-node labels, but pyoxigraph's ``Dataset`` iteration @@ -249,8 +247,6 @@ def test_sort_is_load_bearing(diff_stable: bool) -> None: """ program = textwrap.dedent( """ - import sys - from rdflib import BNode, Graph, Literal, URIRef from rdflib.namespace import RDF from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph @@ -266,7 +262,7 @@ def test_sort_is_load_bearing(diff_stable: bool) -> None: g.add((URIRef("http://example.com/x"), URIRef("http://example.com/has"), bn)) g.add((bn, URIRef("http://example.com/q"), Literal(f"bn_{i}"))) - print(canonicalize_rdf_graph(g, output_format="turtle", diff_stable=sys.argv[1] == "True"), end="") + print(canonicalize_rdf_graph(g, output_format="turtle"), end="") """ ) @@ -275,7 +271,7 @@ def run(seed: str) -> str: # wholesale drops SystemRoot on Windows, which breaks winsock init. env = {**os.environ, "PYTHONHASHSEED": seed} result = subprocess.run( - [sys.executable, "-c", program, str(diff_stable)], + [sys.executable, "-c", program], check=True, capture_output=True, text=True, @@ -540,9 +536,12 @@ def _changed_line_count(before: str, after: str) -> int: return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) -def test_diff_stable_is_opt_in() -> None: - """Diff-stable labels require an explicit request.""" - assert inspect.signature(canonicalize_rdf_graph).parameters["diff_stable"].default is False +def test_diff_stable_without_the_extra_names_the_extra(mock_missing_import) -> None: + """Requesting the opt-in path without diffable-rdf says how to install it.""" + mock_missing_import("diffable_rdf") + + with pytest.raises(ImportError, match=r"linkml-runtime\[diff-stable\]"): + canonicalize_rdf_graph(_make_graph_with_bnodes(), diff_stable=True) @pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) From 0c12c87fc20831b44fd68472adad20949043f2a0 Mon Sep 17 00:00:00 2001 From: Nico Matentzoglu Date: Wed, 14 Oct 2026 17:33:13 +0300 Subject: [PATCH 10/10] docs(schemaview): fix typos in get_subset and get_slot docstrings (#4039) Closes #4015 --- .../linkml_runtime/src/linkml_runtime/utils/schemaview.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py b/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py index 0291617950..31f3e80416 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py @@ -880,7 +880,7 @@ def get_slot( :param slot_name: name of the slot to be retrieved :param imports: include import closure :param attributes: include attributes - :param strict: raise ValueError is not found + :param strict: raise ValueError if not found :return: slot definition """ slot = self.all_slots(imports=imports, attributes=False).get(slot_name, None) @@ -904,8 +904,9 @@ def get_subset( ) -> SubsetDefinition | None: """Retrieve a subset from the schema. - :param subset_name: name of the subsey to be retrieved + :param subset_name: name of the subset to be retrieved :param imports: include import closure + :param strict: raise ValueError if not found :return: subset definition """ s = self.all_subsets(imports).get(subset_name, None)