diff --git a/AGENTS.md b/AGENTS.md index 772f7c59ab..c9a178e6c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,8 @@ All commands use `uv run` prefix (e.g., `uv run pytest`). * Do not "fix" issues by changing or weakening test conditions. Try harder, or ask questions if a test fails. * Avoid try/except blocks, these can mask bugs * Failing fast is a good principle +* Optional dependencies are the documented exception to the two rules above. Guard the import, not the spec: `try: import x` / `except ImportError as exc: raise ImportError("x is required. Install with: pip install 'linkml[extra]'") from exc` - see `generators/bigquerygen.py`. `importlib.util.find_spec` raises under the test fixture below, which hides your message. +* Testing that an optional dependency is absent is not a mock test: use the `mock_missing_import` fixture in `tests/conftest.py`. Do not build isolated environments in CI for this. * Follow the DRY principle * Avoid repeating chunks of code, but also avoid premature over-abstraction * Declarative principles are favored diff --git a/docs/faq/tools.md b/docs/faq/tools.md index e6cee18d82..25f4acad37 100644 --- a/docs/faq/tools.md +++ b/docs/faq/tools.md @@ -69,6 +69,25 @@ There are a few strategies: You can use any of the generator tools distributed as part of linkml to check for errors in your schema. +## Is there a master list of which generators or validators support which LinkML features? + +There is no separately curated, complete matrix of every generator and validator against every metamodel slot. + +The closest source of truth is the [Generator Feature Dashboard](../generators/dashboard). It is generated from the [compliance test suite](https://github.com/linkml/linkml/tree/main/tests/linkml/test_compliance) and shows which metamodel features each tested generator implements, partially implements, ignores, or has not yet been tested for. + +That dashboard is the outcome of [issue 2333](https://github.com/linkml/linkml/issues/2333) (Generator Support Matrix). It is not exhaustive: + +* Only generators that participate in the compliance suite appear as columns. +* Cells can be unknown or partial; a result reflects the current tests, not a guarantee of every edge case. +* There is no equivalent complete matrix for validators such as JSON Schema, Pydantic, SHACL, or ShEx. Those validators typically consume generated artifacts, so generator coverage on the dashboard is the relevant proxy. + +For more detail, see: + +* [Generators](../generators/index) and the individual generator pages (JSON Schema, Pydantic, SHACL, ShEx, and others) +* [Validating data](../data/validating-data) for validation strategies and plugins + +If a feature you need is missing or marked unknown, please open an issue or add a compliance test rather than treating the dashboard as a finished catalog. + ## Is there a linter for LinkML? Yes! See the documentation for [the schema linter](https://linkml.io/linkml/schemas/linter.html). diff --git a/docs/get-involved/Community-Meetings.md b/docs/get-involved/Community-Meetings.md index b278ec082f..a7e6b8d4ed 100644 --- a/docs/get-involved/Community-Meetings.md +++ b/docs/get-involved/Community-Meetings.md @@ -27,7 +27,7 @@ Join the LinkML community for regular sessions featuring presentations on LinkML | Date | Presenter 1 | Topic 1 | Presenter 2 | Topic 2 | | :---: | :---: | :---: | :----: | :---: | -| October 15, 2026|Piotr Sowiński|[LinkML-Scala](https://github.com/NeverBlink-OSS/linkml-scala)| | Open slot!
Volunteers welcome| +| October 15, 2026|Piotr Sowiński|[LinkML-Scala](https://github.com/NeverBlink-OSS/linkml-scala)| Maia Kapur| Autonomous science development at EMSL & model viewer in the BASALT GitHub docs| | [September 17, 2026](https://docs.google.com/presentation/d/1b_XGol4vnEKsAp0zwCKludfVP_IOFnYoxwW3gBRbbLU/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Matt Gehring| How sniff.world uses LinkML: from consuming Biolink to publishing its own schemas| Nolan Nicols |[Provenance as a Primitive: Building Knowledge Graphs with LOKF](https://docs.google.com/presentation/d/155JuRM4nBrLsFOFkMv3ZQfgQleqwieP27c0vWAo9SZM/edit?slide=id.p1#slide=id.p1)| | [August 20, 2026](https://docs.google.com/presentation/d/1_DgJTclLsi9XEcnmgAfftVD5HZ1Aa0TF6abGB_YCGx0/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Patrick Golden| Working across boundaries: LinkML in the [Zebrafish Phenotype Atlas Project](https://zappfish.org/)|Alex Anderson| Semantic Converters for Power Systems Data Interoperability and Agentic AI| | [July 16, 2026](https://docs.google.com/presentation/d/1A05qfTbmI8RXyvoBSplpjSU9e4OP6BlIKJvQ1-eWhGw/edit?slide=id.g36e69bd970c_1_50#slide=id.g36e69bd970c_1_50)| Sierra Moxon | LinkML Microschemas |Stephan Heunis |[The ORINOCO project at Research Center Jülich - building self-hostable research information infrastructure](https://pool.psychoinformatics.de/ui/?sh%3ANodeShape=xyzri%3AXYZProject&pid=xyzrins%3Aprojects%2Forinoco) | diff --git a/docs/howtos/collaborative-development.md b/docs/howtos/collaborative-development.md index 94515df2d1..221337e843 100644 --- a/docs/howtos/collaborative-development.md +++ b/docs/howtos/collaborative-development.md @@ -40,6 +40,36 @@ Some projects may opt to manage their data alongside the schema in the same repo for "registry" style projects, but in other cases the data may be best managed separately in a dedicated store or a repository like Zenodo or Figshare. +(rdf-in-version-control)= +### Keep generated RDF diffs small + +When you track generated RDF in Git, adding a class or property can change many +blank-node labels. The default RDFC-1.0 numbering is repeatable for the same graph, +but an insertion can renumber existing nodes and obscure the actual schema edit. + +The optional `--diff-stable` flag uses +[diffable-rdf](https://github.com/ASCS-eV/diffable-rdf) to reduce this label churn. +In a uv project containing your schema, install LinkML with the extra and generate +SHACL as follows (using a release that includes this option): + +```bash +uv add 'linkml[diff-stable]' +uv run gen-shacl --diff-stable schema.yaml > schema.shacl.ttl +``` + +The flag also works with `gen-owl`, `gen-rdf`, and `gen-shex --format rdf`. +Python callers of `canonicalize_rdf_graph(..., diff_stable=True)` can install +`linkml-runtime[diff-stable]`. The feature is disabled by default; enabling it +changes existing labels once, so keep that regeneration separate from schema edits. + +Labels depend on blank-node neighborhoods. Edits within connected structures or +among symmetric nodes can still affect other labels; minimum diffs are not +guaranteed. Keep tool versions consistent when comparing generated files. +The relabeled output is not the standard RDFC canonical representation. +ShExC, ShEx JSON and instance-data conversion are outside this option's scope. +Non-standard RDF takes the existing fallback path, which warns that it cannot +apply diff-stable labels. + ### Permissively License Your Code and Data (adapted from [O3 guidelines](https://osf.io/vuzt3/)) diff --git a/packages/linkml/pyproject.toml b/packages/linkml/pyproject.toml index 3d8ee1cdf2..9ff6fd1404 100644 --- a/packages/linkml/pyproject.toml +++ b/packages/linkml/pyproject.toml @@ -68,6 +68,9 @@ dependencies = [ # Specifier syntax: https://peps.python.org/pep-0631/ "platformdirs>=4.12.2", ] +[project.optional-dependencies] +diff-stable = ["linkml-runtime[diff-stable]>=1.10.0,<2.0.0"] + [dependency-groups] lint = [ "black >= 24.0.0", @@ -79,6 +82,7 @@ shacl = [ "pyshacl>=0.40.1", ] tests = [ + "linkml-runtime[diff-stable]", { include-group = "lint" }, { include-group = "typing" }, { include-group = "shacl" }, diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 7ba15df672..3f16c4965c 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -122,6 +122,9 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + diff_stable: bool = False + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" + metadata_profile: MetadataProfile | None = None """Deprecated - use metadata_profiles.""" @@ -353,7 +356,7 @@ def serialize(self, **kwargs: Any) -> str: """ self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(self.graph, output_format=fmt) + return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: """ @@ -1844,6 +1847,15 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "specified language tag. Element-level in_language overrides this." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/rdfgen.py b/packages/linkml/src/linkml/generators/rdfgen.py index 2da1701787..5b8f8dc1c7 100644 --- a/packages/linkml/src/linkml/generators/rdfgen.py +++ b/packages/linkml/src/linkml/generators/rdfgen.py @@ -78,6 +78,9 @@ class RDFGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" + emit_metadata: bool = False context: list[str] = None original_schema: SchemaDefinition = None @@ -89,7 +92,7 @@ def __post_init__(self): def _data(self, g: Graph) -> str: fmt = "turtle" if self.format == "ttl" else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def end_schema(self, output: str | None = None, context: str = None, **_) -> str: gen = JSONLDGenerator( @@ -137,6 +140,15 @@ def end_schema(self, output: str | None = None, context: str = None, **_) -> str multiple=True, help="JSONLD context file (default: vendored meta.context.jsonld)", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **kwargs): """Generate an RDF representation of a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4731b9f0b8..1804b1204e 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -142,6 +142,9 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + diff_stable: bool = False + """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" + emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. @@ -196,7 +199,7 @@ def generate_header(self) -> str: def serialize(self, **args) -> str: g = self.as_graph() fmt = "turtle" if self.format in ["owl", "ttl"] else self.format - return canonicalize_rdf_graph(g, output_format=fmt) + return canonicalize_rdf_graph(g, output_format=fmt, diff_stable=self.diff_stable) def as_graph(self) -> Graph: sv = self.schemaview @@ -929,6 +932,15 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits. See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index 40a93ffbc9..e74e45225e 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -40,6 +40,9 @@ class ShExGenerator(Generator): uses_schemaloader = True # ObjectVars + diff_stable: bool = False + """Reduce blank-node label churn in RDF output. See :ref:`rdf-in-version-control`.""" + shex: Schema = field(default_factory=lambda: Schema()) # ShEx Schema being generated shapes: list = field(default_factory=lambda: []) shape: Shape | None = None # Current shape being defined @@ -177,7 +180,7 @@ def end_schema(self, output: str | None = None, **_) -> str: g = Graph() g.parse(data=shex, format="json-ld", version="1.1") g.bind("owl", OWL) - shex = canonicalize_rdf_graph(g, output_format="turtle") + shex = canonicalize_rdf_graph(g, output_format="turtle", diff_stable=self.diff_stable) elif self.format == "shex": g = Graph() self.namespaces.load_graph(g) @@ -258,6 +261,15 @@ def _get_subproperty_values(self, slot: SlotDefinition) -> list: help="If --expand-subproperty-of (default), slots with subproperty_of will generate NodeConstraint " "values containing all slot descendants. Use --no-expand-subproperty-of to disable this behavior.", ) +@click.option( + "--diff-stable/--no-diff-stable", + default=False, + show_default=True, + help=( + "Reduce blank-node label churn across edits (--format rdf). See " + "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" + ), +) @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate a ShEx Schema for a LinkML model""" diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index 5f69503d48..d4c0a2f167 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -62,6 +62,7 @@ dev = [ linkml-normalize = "linkml_runtime.processing.referencevalidator:cli" [project.optional-dependencies] +diff-stable = ["diffable-rdf>=0.4.0"] dev = [ "coverage", "requests-cache>=1.3.3", diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index 31c5580348..bc06a969b8 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -287,6 +287,7 @@ def _is_safe_prefix_iri(iri: str) -> bool: def canonicalize_rdf_graph( graph: rdflib.Graph, output_format: str = "turtle", + diff_stable: bool = False, ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. @@ -300,8 +301,22 @@ def canonicalize_rdf_graph( :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). + :param diff_stable: Use diffable-rdf labels to reduce blank-node churn across + edits. Requires the ``diff-stable`` extra; disabled by default. The + non-standard RDF fallback warns when it cannot apply these labels. + See https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control :return: Deterministic string serialization of the graph. + :raises ImportError: If diff-stable labels are requested without diffable-rdf. """ + if diff_stable: + try: + from diffable_rdf import wl_relabel_quads + except ImportError as exc: + raise ImportError( + "diff_stable=True requires diffable-rdf. Install " + "'linkml-runtime[diff-stable]' or 'linkml[diff-stable]'." + ) from exc + ox_format = _FORMAT_MAP.get(output_format.lower()) if ox_format is None: warnings.warn( @@ -330,6 +345,16 @@ def canonicalize_rdf_graph( RDFCanonicalizationWarning, stacklevel=2, ) + if diff_stable: + warnings.warn( + "diff_stable=True was requested but this graph took the rdflib fallback, " + "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " + "deterministic but NOT diff-stable: an unrelated edit may still renumber " + "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " + "predicates) to get diff-stable labels.", + RDFCanonicalizationWarning, + stacklevel=2, + ) return _deterministic_fallback_serialize(graph, output_format) dataset = ox.Dataset() @@ -339,13 +364,17 @@ def canonicalize_rdf_graph( # 3. Canonicalize blank node labels with RDFC-1.0. dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) + quads = list(dataset) + + if diff_stable: + quads = wl_relabel_quads(quads) + # 4. Sort triples for deterministic ordering. # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset # iteration order is not sorted and varies across processes (verified # empirically against pyoxigraph 0.5.8). The explicit string-key sort # is load-bearing for byte-identical output across runs; see # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - quads = list(dataset) sorted_triples = sorted( (ox.Triple(q.subject, q.predicate, q.object) for q in quads), key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py b/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py index 0291617950..31f3e80416 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/schemaview.py @@ -880,7 +880,7 @@ def get_slot( :param slot_name: name of the slot to be retrieved :param imports: include import closure :param attributes: include attributes - :param strict: raise ValueError is not found + :param strict: raise ValueError if not found :return: slot definition """ slot = self.all_slots(imports=imports, attributes=False).get(slot_name, None) @@ -904,8 +904,9 @@ def get_subset( ) -> SubsetDefinition | None: """Retrieve a subset from the schema. - :param subset_name: name of the subsey to be retrieved + :param subset_name: name of the subset to be retrieved :param imports: include import closure + :param strict: raise ValueError if not found :return: subset definition """ s = self.all_subsets(imports).get(subset_name, None) diff --git a/pyproject.toml b/pyproject.toml index 559cf265ac..fd62dc2986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ markers = [ "yarrrml: End-to-end tests for the YARRRML generator", "typedbgen: Tests for the TypeDB generator", "arrays: Array specifications!", + "diffable_rdf: tests that exercise the diffable-rdf integration (--diff-stable); run as a canary against linkml in diffable-rdf's own CI", ] # https://docs.astral.sh/ruff/settings/ diff --git a/tests/conftest.py b/tests/conftest.py index bcf958b9e5..ba13d6f764 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -422,7 +422,8 @@ class MockImportErrorFinder(MetaPathFinder): """ Fake like we don't have a module when we really do. - see the ``mock_black_import`` fixture for example usage. + see the ``mock_missing_import`` fixture to hide any module, or ``mock_black_import`` + for a single-purpose example. .. note:: @@ -442,6 +443,40 @@ def find_spec(self, fullname, path, target): return None +@pytest.fixture(scope="function") +def mock_missing_import(): + """Pretend an optional dependency is not installed, for any module. + + Yields a callable taking a module name, so a test for one optional + dependency does not need a fixture of its own:: + + def test_requires_the_extra(mock_missing_import): + mock_missing_import("diffable_rdf") + with pytest.raises(ImportError, match="linkml-runtime[diff-stable]"): + ... + + See :class:`MockImportErrorFinder` for the caveat about reimporting + modules that already imported the one being hidden. + """ + removed = {} + finders = [] + + def hide(module: str) -> None: + for name, loaded in list(sys.modules.items()): + if name.startswith(module): + removed[name] = loaded + del sys.modules[name] + finder = MockImportErrorFinder(module) + finders.append(finder) + sys.meta_path.insert(0, finder) + + yield hide + + sys.modules.update(removed) + for finder in finders: + sys.meta_path.remove(finder) + + @pytest.fixture(scope="function") def mock_black_import(): """ diff --git a/tests/linkml/test_scripts/test_diff_stable.py b/tests/linkml/test_scripts/test_diff_stable.py new file mode 100644 index 0000000000..d1c292276b --- /dev/null +++ b/tests/linkml/test_scripts/test_diff_stable.py @@ -0,0 +1,45 @@ +"""CLI checks for the optional diff-stable RDF labels.""" + +import re +from pathlib import Path + +import pyoxigraph as ox +import pytest +from click.testing import CliRunner + +from linkml.generators.shaclgen import cli as shacl_cli + +_SCHEMA = """id: https://example.org/cli +name: cli +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/cli/ +default_prefix: ex +imports: [linkml:types] +classes: + Thing: + attributes: + label: + range: string +""" + + +@pytest.mark.parametrize( + ("flag", "label_pattern"), + [ + pytest.param("--no-diff-stable", r"c14n[0-9]+", id="disabled"), + pytest.param("--diff-stable", r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?", id="enabled", marks=pytest.mark.diffable_rdf), + ], +) +def test_the_flag_reaches_the_serializer(tmp_path: Path, flag: str, label_pattern: str) -> None: + """The CLI flag selects the blank-node labelling it names.""" + schema = tmp_path / "schema.yaml" + schema.write_text(_SCHEMA, encoding="utf-8") + + result = CliRunner().invoke(shacl_cli, [str(schema), flag]) + + assert result.exit_code == 0, result.output + quads = list(ox.parse(result.stdout, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for q in quads for term in (q.subject, q.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(label_pattern, node.value) for node in blank_nodes) diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 63dbf095a1..8da48fa8b3 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,17 +1,24 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" +import difflib import os import re import subprocess import sys import textwrap +from pathlib import Path import pyoxigraph as ox import pytest import rdflib -from rdflib import BNode, Graph, Literal, URIRef -from rdflib.namespace import RDF - +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import OWL, RDF, RDFS, SH, XSD + +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.rdfgen import RDFGenerator +from linkml.generators.shaclgen import ShaclGenerator +from linkml.generators.shexgen import ShExGenerator +from linkml.utils.generator import Generator from linkml_runtime.utils import rdf_canonicalize as rdf_canon_mod from linkml_runtime.utils.rdf_canonicalize import ( RDFCanonicalizationWarning, @@ -507,3 +514,244 @@ def run(seed: str) -> str: assert out_a == out_b, ( "Fallback output differs across PYTHONHASHSEED values; blank-node canonicalization may be missing" ) + + +def _shapes_graph(count: int, extra: bool = False) -> Graph: + """A graph shaped like generator output: one blank node per named subject.""" + g = Graph() + g.bind("ex", "http://example.com/") + names = [f"{i:02d}" for i in range(count)] + (["AAAinserted"] if extra else []) + for name in names: + subject = URIRef(f"http://example.com/Shape{name}") + prop = BNode() + g.add((subject, URIRef("http://example.com/property"), prop)) + g.add((prop, URIRef("http://example.com/path"), URIRef(f"http://example.com/p{name}"))) + return g + + +def _changed_line_count(before: str, after: str) -> int: + """Count added and removed content lines in a unified diff.""" + + diff = difflib.unified_diff(before.splitlines(), after.splitlines(), n=0, lineterm="") + return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) + + +def test_diff_stable_without_the_extra_names_the_extra(mock_missing_import) -> None: + """Requesting the opt-in path without diffable-rdf says how to install it.""" + mock_missing_import("diffable_rdf") + + with pytest.raises(ImportError, match=r"linkml-runtime\[diff-stable\]"): + canonicalize_rdf_graph(_make_graph_with_bnodes(), diff_stable=True) + + +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +@pytest.mark.parametrize("output_format", ["nt", "turtle"]) +def test_diff_stable_preserves_semantics(diff_stable: bool, output_format: str) -> None: + """Each serialization preserves the source graph under blank-node renaming.""" + graph = _make_graph_with_bnodes() + output = canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) + parsed = Graph().parse(data=output, format=output_format) + assert len(parsed) == len(graph) + assert rdflib.compare.isomorphic(graph, parsed) + + +@pytest.mark.diffable_rdf +def test_diff_stable_is_deterministic() -> None: + """A cycle's tied signatures remain independent of input blank-node IDs.""" + graphs = [] + predicate = URIRef("http://example.com/next") + for labels, reverse in [("abcd", False), ("dbac", True)]: + nodes = [BNode(label) for label in labels] + triples = [(node, predicate, nodes[(i + 1) % len(nodes)]) for i, node in enumerate(nodes)] + graph = Graph() + for triple in reversed(triples) if reverse else triples: + graph.add(triple) + graphs.append(graph) + + assert rdflib.compare.isomorphic(*graphs) + outputs = [canonicalize_rdf_graph(graph, output_format="nt", diff_stable=True) for graph in graphs] + assert outputs[0] == outputs[1] + for graph, output in zip(graphs, outputs): + assert rdflib.compare.isomorphic(graph, Graph().parse(data=output, format="nt")) + + +@pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) +def test_diff_stable_insertion_labels(diff_stable: bool) -> None: + """An independent insertion renumbers RDFC labels but preserves the old WL label.""" + before = Graph().parse( + data="""@prefix ex: . + ex:Shape00 ex:property _:old . + _:old ex:path ex:p00 . + """, + format="turtle", + ) + after = Graph() + for triple in before: + after.add(triple) + inserted = BNode() + ex = Namespace("http://example.com/") + after.add((ex.ShapeAAAinserted, ex.property, inserted)) + after.add((inserted, ex.path, ex.pAAAinserted)) + + outputs = [canonicalize_rdf_graph(graph, output_format="nt", diff_stable=diff_stable) for graph in (before, after)] + parsed = [set(ox.parse(output, format=ox.RdfFormat.N_TRIPLES)) for output in outputs] + old_subject = ox.NamedNode(str(ex.Shape00)) + new_subject = ox.NamedNode(str(ex.ShapeAAAinserted)) + property_iri = ox.NamedNode(str(ex.property)) + path_iri = ox.NamedNode(str(ex.path)) + (old_before,) = {q.object for q in parsed[0] if q.subject == old_subject and q.predicate == property_iri} + (old_after,) = {q.object for q in parsed[1] if q.subject == old_subject and q.predicate == property_iri} + (new_after,) = {q.object for q in parsed[1] if q.subject == new_subject and q.predicate == property_iri} + assert all(isinstance(node, ox.BlankNode) for node in (old_before, old_after, new_after)) + assert old_after != new_after + inserted_quads = { + ox.Quad(new_subject, property_iri, new_after), + ox.Quad(new_after, path_iri, ox.NamedNode(str(ex.pAAAinserted))), + } + assert parsed[0] == { + ox.Quad(old_subject, property_iri, old_before), + ox.Quad(old_before, path_iri, ox.NamedNode(str(ex.p00))), + } + assert ( + parsed[1] + == { + ox.Quad(old_subject, property_iri, old_after), + ox.Quad(old_after, path_iri, ox.NamedNode(str(ex.p00))), + } + | inserted_quads + ) + + if diff_stable: + assert all(re.fullmatch(_STABLE_LABEL, node.value) for node in (old_before, old_after, new_after)) + assert old_before == old_after + assert parsed[1] - parsed[0] == inserted_quads + assert parsed[0] - parsed[1] == set() + else: + assert (old_before.value, old_after.value, new_after.value) == ("c14n0", "c14n1", "c14n0") + for graph, output in zip((before, after), outputs): + assert rdflib.compare.isomorphic(graph, Graph().parse(data=output, format="nt")) + + +@pytest.mark.diffable_rdf +def test_diff_stable_confines_an_insertion_to_the_lines_it_touches() -> None: + """Diff-stable labels reduce Turtle line churn for independent property nodes.""" + before, after = _shapes_graph(20), _shapes_graph(20, extra=True) + baseline = _changed_line_count(canonicalize_rdf_graph(before), canonicalize_rdf_graph(after)) + stable = _changed_line_count( + canonicalize_rdf_graph(before, diff_stable=True), + canonicalize_rdf_graph(after, diff_stable=True), + ) + assert stable < baseline / 4, f"expected diff-stable output to churn far less; got {stable} vs baseline {baseline}" + + +_STABLE_LABEL = r"b[0-9a-f]{12}(?:_[1-9][0-9]*)?" + + +_DIFF_STABLE_SCHEMA = """\ +id: https://example.org/diffstable +name: diffstable +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/diffstable/ +default_prefix: ex +default_range: string +imports: + - linkml:types +classes: + Person: + slots: [name, knows] + Organization: + slots: [name] +slots: + name: + range: string + knows: + range: Person + multivalued: true +""" + + +@pytest.fixture +def diff_stable_schema(tmp_path: Path) -> Path: + """Write the small schema for real generator calls.""" + schema = tmp_path / "schema.yaml" + schema.write_text(_DIFF_STABLE_SCHEMA, encoding="utf-8") + return schema + + +@pytest.mark.diffable_rdf +@pytest.mark.parametrize( + ("generator", "kwargs"), + [ + pytest.param(OwlSchemaGenerator, {}, id="owlgen"), + pytest.param(RDFGenerator, {}, id="rdfgen", marks=pytest.mark.network), + pytest.param(ShaclGenerator, {}, id="shaclgen"), + pytest.param(ShExGenerator, {"format": "rdf"}, id="shexgen", marks=pytest.mark.network), + ], +) +def test_diff_stable_reaches_every_rdf_generator( + diff_stable_schema: Path, generator: type[Generator], kwargs: dict[str, str] +) -> None: + """Every RDF generator emits the designed schema with diff-stable labels.""" + output = generator(str(diff_stable_schema), diff_stable=True, **kwargs).serialize() + quads = list(ox.parse(output, format=ox.RdfFormat.TURTLE)) + blank_nodes = {term for quad in quads for term in (quad.subject, quad.object) if isinstance(term, ox.BlankNode)} + assert blank_nodes + assert all(re.fullmatch(_STABLE_LABEL, node.value) for node in blank_nodes) + graph = Graph().parse(data=output, format="turtle") + ex = Namespace("https://example.org/diffstable/") + if generator is OwlSchemaGenerator: + assert (ex.Person, RDF.type, OWL.Class) in graph + assert (ex.Organization, RDF.type, OWL.Class) in graph + (restriction,) = { + node + for node in graph.objects(ex.Person, RDFS.subClassOf) + if (node, OWL.onProperty, ex.knows) in graph and (node, OWL.allValuesFrom, ex.Person) in graph + } + assert isinstance(restriction, BNode) + assert (restriction, RDF.type, OWL.Restriction) in graph + assert (restriction, OWL.allValuesFrom, ex.Person) in graph + elif generator is ShaclGenerator: + for subject, paths in [(ex.Person, {ex.name, ex.knows}), (ex.Organization, {ex.name})]: + assert (subject, RDF.type, SH.NodeShape) in graph + assert (subject, SH.targetClass, subject) in graph + properties = set(graph.objects(subject, SH.property)) + assert {graph.value(prop, SH.path) for prop in properties} == paths + (ignored,) = graph.objects(subject, SH.ignoredProperties) + assert set(graph.predicate_objects(ignored)) == {(RDF.first, RDF.type), (RDF.rest, RDF.nil)} + (knows,) = {node for node in graph.objects(ex.Person, SH.property) if (node, SH.path, ex.knows) in graph} + assert (knows, SH["class"], ex.Person) in graph + elif generator is RDFGenerator: + linkml = Namespace("https://w3id.org/linkml/") + schema = ex.diffstable + assert (schema, RDF.type, linkml.SchemaDefinition) in graph + assert set(graph.objects(schema, linkml.classes)) == {ex.Person, ex.Organization} + (prefix,) = {node for node in graph.objects(schema, SH.declare) if (node, SH.prefix, Literal("ex")) in graph} + assert (prefix, SH.namespace, Literal(str(ex), datatype=XSD.anyURI)) in graph + else: + assert generator is ShExGenerator + shex = Namespace("http://www.w3.org/ns/shex#") + assert (ex.Person, RDF.type, shex.Shape) in graph + (expression,) = graph.objects(ex.Person, shex.expression) + assert (expression, RDF.type, shex.EachOf) in graph + (expressions,) = graph.objects(expression, shex.expressions) + assert ex.Person_tes in set(graph.items(expressions)) + assert (ex.Person_tes, RDF.type, shex.EachOf) in graph + (slots,) = graph.objects(ex.Person_tes, shex.expressions) + (knows,) = {node for node in graph.items(slots) if (node, shex.predicate, ex.knows) in graph} + assert (knows, RDF.type, shex.TripleConstraint) in graph + assert (knows, shex.valueExpr, ex.Person) in graph + assert (knows, shex.min, Literal(0)) in graph + assert (knows, shex.max, Literal(-1)) in graph + + +@pytest.mark.diffable_rdf +def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback() -> None: + """A relative IRI requires the fallback and a warning that labels cannot apply.""" + graph = _make_graph_with_bnodes() + graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) + with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + stable = canonicalize_rdf_graph(graph, diff_stable=True) + with pytest.warns(RDFCanonicalizationWarning): + plain = canonicalize_rdf_graph(graph, diff_stable=False) + assert stable == plain diff --git a/uv.lock b/uv.lock index 3b8ef0ceb1..7c22c3d061 100644 --- a/uv.lock +++ b/uv.lock @@ -1075,6 +1075,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "diffable-rdf" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyoxigraph" }, + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/86/df/1d71c0c984eac1cfd25531aabf84528941b01083c875da49d44882d1b5af/diffable_rdf-0.4.0.tar.gz", hash = "sha256:a84afaa10332e6a039b6ddcabf76bced3049e4da467c88ee786d0fa35218111e", size = 838040, upload-time = "2026-09-11T11:43:12.844Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/49/b62ee864fe6769b759f04e3709fd79711b1cdf49223b1bb0e985e507ead3/diffable_rdf-0.4.0-py3-none-any.whl", hash = "sha256:c59c57429465f786fdbc8d38ddbb687da16748c5752f0c2ac4a77b5527aa9aed", size = 46126, upload-time = "2026-09-11T11:43:11.536Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -2395,6 +2408,11 @@ dependencies = [ { name = "watchdog" }, ] +[package.optional-dependencies] +diff-stable = [ + { name = "linkml-runtime", extra = ["diff-stable"] }, +] + [package.dev-dependencies] bigquery = [ { name = "sqlalchemy-bigquery" }, @@ -2409,6 +2427,7 @@ dev = [ { name = "ipython-genutils" }, { name = "jsonpatch" }, { name = "jupyter" }, + { name = "linkml-runtime", extra = ["diff-stable"] }, { name = "mock" }, { name = "myst-nb" }, { name = "nbconvert" }, @@ -2461,6 +2480,7 @@ shacl = [ tests = [ { name = "black" }, { name = "duckdb" }, + { name = "linkml-runtime", extra = ["diff-stable"] }, { name = "numpydantic" }, { name = "pyshacl" }, { name = "sqlalchemy-bigquery" }, @@ -2500,6 +2520,7 @@ requires-dist = [ { name = "jsonasobj2", specifier = ">=1.0.3,<2.0.0" }, { name = "jsonschema", extras = ["format"], specifier = ">=4.26.0" }, { name = "linkml-runtime", editable = "packages/linkml_runtime" }, + { name = "linkml-runtime", extras = ["diff-stable"], marker = "extra == 'diff-stable'", editable = "packages/linkml_runtime" }, { name = "openapi-spec-validator", specifier = ">=0.8.4" }, { name = "openpyxl" }, { name = "parse" }, @@ -2520,6 +2541,7 @@ requires-dist = [ { name = "typing-extensions", marker = "python_full_version < '3.12'", specifier = ">=4.6.0" }, { name = "watchdog", specifier = ">=0.9.0" }, ] +provides-extras = ["diff-stable"] [package.metadata.requires-dev] bigquery = [{ name = "sqlalchemy-bigquery", specifier = ">=1.9.0" }] @@ -2533,6 +2555,7 @@ dev = [ { name = "ipython-genutils" }, { name = "jsonpatch", specifier = ">=1.33" }, { name = "jupyter" }, + { name = "linkml-runtime", extras = ["diff-stable"], editable = "packages/linkml_runtime" }, { name = "mock", specifier = ">=5.1.0" }, { name = "myst-nb", marker = "python_full_version >= '3.10'", specifier = ">=1.4.0" }, { name = "nbconvert" }, @@ -2580,6 +2603,7 @@ shacl = [{ name = "pyshacl", specifier = ">=0.40.1" }] tests = [ { name = "black", specifier = ">=24.0.0" }, { name = "duckdb", specifier = ">=1.5.5" }, + { name = "linkml-runtime", extras = ["diff-stable"], editable = "packages/linkml_runtime" }, { name = "numpydantic", specifier = ">=1.10.0" }, { name = "pyshacl", specifier = ">=0.40.1" }, { name = "sqlalchemy-bigquery", specifier = ">=1.17.2" }, @@ -2628,6 +2652,9 @@ dev = [ { name = "coverage" }, { name = "requests-cache" }, ] +diff-stable = [ + { name = "diffable-rdf" }, +] [package.dev-dependencies] dev = [ @@ -2641,6 +2668,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", marker = "extra == 'diff-stable'", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, { name = "json-flattener", specifier = ">=0.1.9" }, @@ -2655,7 +2683,7 @@ requires-dist = [ { name = "requests" }, { name = "requests-cache", marker = "extra == 'dev'", specifier = ">=1.3.3" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "diff-stable"] [package.metadata.requires-dev] dev = [