From 5d0f36ada57780dc06d1eaabc799a8b6af91af05 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 2 Oct 2026 13:23:41 +0200 Subject: [PATCH 01/17] feat(jsonschemagen): emit propertyNames from inlined-dict key slot constraints For an inlined-as-dict slot whose range class has an identifier/key slot, render the key slot's string-applicable constraints onto JSON Schema propertyNames (draft-06+) instead of dropping them. In the inlined-dict form the mapping key is the identifier value, so the key slot's constraints constrain the keys. JSON object keys are always strings, so only pattern, enum (equals_string_in) and a string const (equals_string) are emitted; numeric minimum/maximum, numeric const (equals_number) and allOf are excluded -- a numeric const would otherwise reject every key. structured_pattern is honored when materialize_patterns is enabled, consistent with value patterns. Backward compatible: emitted only when a string-applicable key constraint applies. Signed-off-by: Carlo van Driesten Includes the documentation for the emitted propertyNames constraint. Signed-off-by: jdsika --- docs/generators/json-schema.rst | 66 +++++++ .../src/linkml/generators/jsonschemagen.py | 39 +++++ .../test_generators/test_jsonschemagen.py | 162 ++++++++++++++++++ 3 files changed, 267 insertions(+) diff --git a/docs/generators/json-schema.rst b/docs/generators/json-schema.rst index 87c85552cb..be72b938bd 100644 --- a/docs/generators/json-schema.rst +++ b/docs/generators/json-schema.rst @@ -420,6 +420,72 @@ will generate: LinkML also supports `Structured patterns `_, these are compiled down to patterns during JSON Schema generation. +Dictionary key constraints (propertyNames) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A multivalued, inlined slot whose range class has an identifier slot is +compiled to a JSON object keyed by that identifier (see *Inlining* above). +When the identifier slot carries string-applicable constraints, they are +emitted as a `propertyNames `_ +schema on the container object, so the *keys* of the dictionary are validated, +not just the values: + +.. code-block:: yaml + + slots: + tags: + range: Tag + multivalued: true + inlined: true + uid: + identifier: true + pattern: "^(0|[1-9][0-9]*)$" + +generates on the container: + +.. code-block:: json + + "tags": { + "additionalProperties": {"$ref": "#/$defs/Tag"}, + "propertyNames": {"pattern": "^(0|[1-9][0-9]*)$"}, + "type": "object" + } + +The constraints carried over from the key slot are the ones applicable to JSON +Schema strings, because object keys are always strings (`JSON Schema Core +2019-09, §9.3.2.5 `_): + +* ``pattern`` -- whether written directly on the slot, resolved from a + ``structured_pattern``, or inherited from the slot's ``range`` type (for + example an identifier with ``range: ncname``, or a user-defined type that + declares a ``pattern``); +* ``equals_string_in``, emitted as ``enum``; +* a string ``equals_string``, emitted as ``const``. + +The emitted key pattern is always the same one that applies to the identifier +*inside* the value object, so a key and a redundantly repeated in-object +identifier are now validated identically. + +Numeric constraints -- ``minimum_value``/``maximum_value``, and the numeric +``const`` produced by ``equals_number`` -- are deliberately **not** carried +over: they cannot be satisfied by a string key, and a numeric ``const`` would +reject every key. The ``allOf`` produced by a ``range_expression``, and the +permissible values of an ``enum``-ranged identifier, are likewise out of scope. + +``propertyNames`` composes conjunctively with ``additionalProperties``, so keys +and values are constrained independently. It is emitted only when the key slot +actually carries one of the constraints listed above; an unconstrained key slot +produces exactly the same output as before. + +.. note:: + + Because type-level patterns are included, an identifier slot whose range is + ``ncname`` (or another pattern-bearing type) gains a ``propertyNames`` + entry even if the slot itself declares no constraint. The generated schema + becomes stricter, but only in ways the model already required: data whose + keys satisfy the declared identifier type is unaffected. + + Rules ^^^^^ diff --git a/packages/linkml/src/linkml/generators/jsonschemagen.py b/packages/linkml/src/linkml/generators/jsonschemagen.py index 9cc776f5e3..ef0ea170fb 100644 --- a/packages/linkml/src/linkml/generators/jsonschemagen.py +++ b/packages/linkml/src/linkml/generators/jsonschemagen.py @@ -859,6 +859,40 @@ def get_value_constraints_for_slot(self, slot: SlotDefinition | AnonymousSlotExp return constraints + def get_key_constraints_for_slot(self, slot: SlotDefinition | None) -> JsonSchema: + """Constraints applicable to the *keys* of an inlined-as-dict slot. + + In the inlined-dict form the mapping key *is* the value of the range class's + identifier/key slot (https://linkml.io/linkml/schemas/inlining.html) and is not + repeated inside the value object, so constraints declared on that slot -- or + inherited from its type -- are constraints on the object keys. The result is + intended for JSON Schema ``propertyNames``, which composes conjunctively with + ``additionalProperties``. + + JSON object keys are always strings (JSON Schema Core 2019-09, 9.3.2.5), so only + the string-applicable subset of :meth:`get_value_constraints_for_slot` is + returned: ``pattern`` (including a resolved ``structured_pattern`` and a pattern + inherited from the slot's type), a string ``const`` (``equals_string``) and a + string ``enum`` (``equals_string_in``). Numeric constraints -- ``minimum`` and + ``maximum``, and the numeric ``const`` produced by ``equals_number`` -- and the + ``allOf`` produced by ``range_expression`` are excluded: they cannot be satisfied + by a string key, and a numeric ``const`` would reject *every* key. + + :param slot: the identifier or key slot of the range class + :return: a schema for ``propertyNames``; empty when the key is unconstrained + """ + constraints = self.get_value_constraints_for_slot(slot) + + key_constraints = JsonSchema() + for keyword in ("pattern", "const"): + value = constraints.get(keyword) + if isinstance(value, str): + key_constraints[keyword] = value + enum_values = constraints.get("enum") + if isinstance(enum_values, list) and all(isinstance(value, str) for value in enum_values): + key_constraints["enum"] = enum_values + return key_constraints + def _subschema_for_boolean_member( self, parent_slot: SlotDefinition | AnonymousSlotExpression, member: AnonymousSlotExpression ) -> JsonSchema: @@ -935,6 +969,11 @@ def get_subschema_for_slot( else: typ = ["object", "null"] prop = JsonSchema({"type": typ, "additionalProperties": additionalProps}) + # The dict keys are the range's identifier/key values, so that + # slot's string-applicable constraints constrain the keys. + key_constraints = self.get_key_constraints_for_slot(range_id_slot) + if key_constraints: + prop["propertyNames"] = key_constraints self.top_level_schema.add_lax_def(reference, self.aliased_slot_name(range_id_slot)) else: prop = JsonSchema.array_of(JsonSchema.ref_for(reference), include_null, required=slot.required) diff --git a/tests/linkml/test_generators/test_jsonschemagen.py b/tests/linkml/test_generators/test_jsonschemagen.py index 3e83ef242d..6d8e0b32ee 100644 --- a/tests/linkml/test_generators/test_jsonschemagen.py +++ b/tests/linkml/test_generators/test_jsonschemagen.py @@ -2079,3 +2079,165 @@ def test_type_without_base_defaults_to_string(tmp_path): generated = json.loads(JsonSchemaGenerator(str(schema_file)).serialize()) assert generated["$defs"]["MyClass"]["properties"]["value"]["type"] == ["string", "null"] + + +def _inlined_dict_schema( + key_slot_yaml: str, + key_decl: str = "identifier: true", + key_range: str = "string", + extra_yaml: str = "", +) -> str: + """Build a schema with an inlined-as-dict slot whose key slot is configured by + ``key_decl`` (``identifier: true`` or ``key: true``), ``key_range`` (the key slot + range), and ``key_slot_yaml`` (extra YAML lines for the key slot). ``extra_yaml`` is + appended at the top level, for declaring extra ``types``/``enums``.""" + return f""" +id: https://example.org/test-key-constraints +name: test-key-constraints +prefixes: + linkml: https://w3id.org/linkml/ +default_range: string +imports: + - linkml:types +{extra_yaml} +classes: + Container: + tree_root: true + attributes: + entries: + range: Entry + multivalued: true + inlined: true + inlined_as_list: false + Entry: + attributes: + key: + {key_decl} + range: {key_range} +{key_slot_yaml} + val: + range: string +""" + + +@pytest.mark.parametrize("key_decl", ["identifier: true", "key: true"]) +def test_inlined_dict_key_pattern_emits_property_names(key_decl): + """A literal ``pattern`` on the inlined-dict key slot (identifier or key) must be + rendered onto ``propertyNames``.""" + schema = _inlined_dict_schema(' pattern: "^[0-9]+$"', key_decl=key_decl) + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert generated["properties"]["entries"]["propertyNames"] == {"pattern": "^[0-9]+$"} + + +def test_inlined_dict_key_enum_emits_property_names(): + """``equals_string_in`` on the key slot becomes an ``enum`` constraint on keys.""" + schema = _inlined_dict_schema(" equals_string_in:\n - a\n - b") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert generated["properties"]["entries"]["propertyNames"] == {"enum": ["a", "b"]} + + +def test_inlined_dict_no_key_constraint_emits_no_property_names(): + """No constraint on the key slot -> no ``propertyNames`` (unchanged behavior).""" + schema = _inlined_dict_schema("") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + + +def test_inlined_dict_key_structured_pattern_emits_property_names(): + """``structured_pattern`` on the key slot is resolved and rendered onto + ``propertyNames`` -- identical to how value patterns are handled.""" + schema = _inlined_dict_schema(" structured_pattern:\n syntax: '[0-9]+'") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + + key_pattern = generated["$defs"]["Entry"]["properties"]["key"]["pattern"] + assert generated["properties"]["entries"]["propertyNames"] == {"pattern": key_pattern} + jsonschema.validate({"entries": {"12": {"val": "x"}}}, generated) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"entries": {"bad-key": {"val": "x"}}}, generated) + + +def test_inlined_dict_property_names_rejects_nonmatching_keys(): + """Behavioral check: keys matching the pattern validate; non-matching keys fail.""" + schema = _inlined_dict_schema(' pattern: "^[0-9]+$"') + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + + jsonschema.validate({"entries": {"0": {"val": "x"}}}, generated) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"entries": {"bad-key": {"val": "x"}}}, generated) + + +def test_inlined_dict_key_string_const_emits_property_names(): + """A string ``const`` (``equals_string``) on the key slot becomes a key const.""" + schema = _inlined_dict_schema(" equals_string: fixed") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert generated["properties"]["entries"]["propertyNames"] == {"const": "fixed"} + jsonschema.validate({"entries": {"fixed": {"val": "x"}}}, generated) + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate({"entries": {"other": {"val": "x"}}}, generated) + + +def test_inlined_dict_key_numeric_const_is_not_emitted(): + """A numeric ``const`` (``equals_number``) must NOT be emitted onto propertyNames: + keys are always strings, so a numeric const would reject every key. The keys are + left unconstrained instead.""" + schema = _inlined_dict_schema(" equals_number: 5", key_range="integer") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + # numeric-looking string keys still validate (unconstrained) + jsonschema.validate({"entries": {"5": {"val": "x"}}}, generated) + jsonschema.validate({"entries": {"anything": {"val": "x"}}}, generated) + + +def test_inlined_dict_key_numeric_bounds_are_not_emitted(): + """Numeric ``minimum``/``maximum`` on the key slot are no-ops on string keys and + must not be emitted (they would be misleading clutter).""" + schema = _inlined_dict_schema(" minimum_value: 1\n maximum_value: 10", key_range="integer") + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + + +@pytest.mark.parametrize( + ("key_range", "extra_yaml"), + [ + ("ncname", ""), + ("DigitString", "types:\n DigitString:\n typeof: string\n pattern: '^[0-9]+$'"), + ], + ids=["base-implied-pattern", "user-defined-type-pattern"], +) +def test_inlined_dict_key_type_pattern_emits_property_names(key_range, extra_yaml): + """A pattern inherited from the key slot's *type* constrains the identifier value just + as a slot-level pattern does, so it must reach ``propertyNames`` too. The emitted key + pattern is exactly the one applied to the identifier inside the value object, so keys + and the (optional) in-object identifier are validated identically.""" + schema = _inlined_dict_schema("", key_range=key_range, extra_yaml=extra_yaml) + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + + value_key_pattern = generated["$defs"]["Entry__identifier_optional"]["properties"]["key"]["pattern"] + assert generated["properties"]["entries"]["propertyNames"] == {"pattern": value_key_pattern} + + +def test_inlined_dict_key_enum_range_emits_no_property_names(): + """A key slot whose range is a LinkML *enum* is compiled to a ``$ref`` on the value + side; ``get_value_constraints_for_slot`` reports no string-applicable constraint for + it, so no ``propertyNames`` is emitted and the keys stay unconstrained.""" + schema = _inlined_dict_schema( + "", key_range="Colour", extra_yaml="enums:\n Colour:\n permissible_values:\n red:\n green:" + ) + generated = json.loads(JsonSchemaGenerator(schema).serialize()) + assert "propertyNames" not in generated["properties"]["entries"] + + +def test_inlined_dict_key_constraints_helper_drops_non_string_values(): + """``get_key_constraints_for_slot`` keeps only string-applicable keywords, regardless + of which upstream constraint produced them: a numeric ``const`` or a non-string + ``enum`` would reject every key, so both are dropped.""" + schema = _inlined_dict_schema("") + generator = JsonSchemaGenerator(schema) + generator.generate() + + slot = SlotDefinition("key", pattern="^[0-9]+$") + assert generator.get_key_constraints_for_slot(slot) == {"pattern": "^[0-9]+$"} + + assert generator.get_key_constraints_for_slot(SlotDefinition("key", equals_number=5)) == {} + assert generator.get_key_constraints_for_slot(SlotDefinition("key", minimum_value=1)) == {} + assert generator.get_key_constraints_for_slot(None) == {} From 846750ba78fa387bae18e609d447937a75d397bd Mon Sep 17 00:00:00 2001 From: jdsika Date: Thu, 24 Sep 2026 13:07:39 +0200 Subject: [PATCH 02/17] feat(generators): translate class-level boolean expressions alike to SHACL and JSON Schema gen-shacl dropped class-level any_of, all_of, exactly_one_of and none_of, so a class stating "a code or a name is required" generated shapes that accepted everything. gen-json-schema translated them, but only for the declaring class, and read an absent slot differently depending on where an expression was nested. Both generators now give class expressions one semantics, shared in linkml.generators.common.class_expression: - A class expression constrains every instance of its class, so a class carries the expressions of its ancestors and mixins, translated in its own context. - The specification does not say what a slot condition means for an absent slot. As an SQL CHECK constraint (ISO/IEC 9075) reads a condition on null, a condition that doesn't decide whether its slot may be absent is unknown for an absent slot. An instance is invalid only when an expression is definitely false, and exactly_one_of holds when exactly one member is definitely true. The operators compose, and gen-json-schema no longer rejects an instance meeting only the first member of an exactly_one_of while accepting one meeting none. gen-shacl emits sh:or, sh:and, sh:xone and sh:not (the operators' exact_mappings, SHACL 4.6) over anonymous member shapes, each in its "not false" form or, under sh:not, its "definitely true" form, where such a condition also has sh:minCount 1. Conditions translate required, value_presence, the cardinalities, minimum/maximum_value, pattern, equals_string(_in), equals_number and range; a parameter SHACL allows once per shape that one condition needs twice moves into an sh:and member. An operator whose members use anything else is skipped whole, with one warning naming the shapes it is missing from. gen-json-schema builds the same forms with anyOf, allOf, oneOf and not, resolves conditions on the induced slot, and also enforces cardinalities (minItems / maxItems, or minProperties / maxProperties for a slot inlined as a dict, whose values are the values of the dict) and range in class-level conditions. value_presence: ABSENT on several slots of one expression now requires each to be absent rather than not all of them present, in rules too. An is_a member names a class that every instance of the class carrying the expression belongs to (the class or an ancestor), that only some belong to (through a descendant), or that none belongs to, since an instance belongs to its class and that class's ancestors. Only the second case checks the instance: sh:class in SHACL, and in JSON Schema, which carries no type, a reference to the class's definition. Before, a class inherited its parent's any_of over its own subclasses as sh:class and $ref to itself, which recursed without end in JSON Schema and, in SHACL, rejected a node typed only as a subclass. The slot loop's range dispatch and sh:path computation move into _add_range and _slot_iri; output for schemas without class expressions is unchanged. The compliance tests test_class_any_of and test_class_any_of_with_required run for SHACL. Signed-off-by: jdsika --- docs/generators/json-schema.rst | 10 + docs/generators/shacl.rst | 92 + docs/schemas/advanced.md | 30 + .../generators/common/class_expression.py | 150 ++ .../src/linkml/generators/jsonschemagen.py | 174 +- .../linkml/src/linkml/generators/shaclgen.py | 430 ++++- .../test_boolean_slot_compliance.py | 22 +- .../test_generators/test_jsonschemagen.py | 243 +++ tests/linkml/test_generators/test_shaclgen.py | 1531 ++++++++++++++++- .../test_validator/test_validation_context.py | 20 +- 10 files changed, 2625 insertions(+), 77 deletions(-) create mode 100644 packages/linkml/src/linkml/generators/common/class_expression.py diff --git a/docs/generators/json-schema.rst b/docs/generators/json-schema.rst index be72b938bd..c1c68072b9 100644 --- a/docs/generators/json-schema.rst +++ b/docs/generators/json-schema.rst @@ -172,6 +172,16 @@ LinkML supports analogous elements: Use of these elements will be translated into the appropriate JSON-Schema construct. +At the class level, a class carries the expressions of its ancestors and mixins. +An ``is_a`` member names a class every instance already belongs to, or none can, +unless only some do, through a descendant: then the instance must match that +class's definition, since a JSON object carries no type. +A slot condition is unknown for an absent slot unless it decides whether the slot +may be absent, and an instance is invalid only when an expression is definitely +false, as described under "Class-level expressions and absent slots" in +:doc:`Advanced features `. +The SHACL generator reads them the same way. + Inlining ^^^^^^^^ diff --git a/docs/generators/shacl.rst b/docs/generators/shacl.rst index 3e88f0090f..9322186e44 100644 --- a/docs/generators/shacl.rst +++ b/docs/generators/shacl.rst @@ -84,6 +84,98 @@ Example Output: shacl:targetClass . +Class Expressions +^^^^^^^^^^^^^^^^^ + +Class-level boolean expressions become the SHACL logical constraint components +their metamodel definitions map to (`SHACL §4.6 +`__): + +================== ===================================================== +LinkML SHACL, on the class's ``sh:NodeShape`` +================== ===================================================== +``any_of`` ``sh:or`` over the member shapes +``all_of`` ``sh:and`` over the member shapes +``exactly_one_of`` ``sh:xone`` over the member shapes +``none_of`` one ``sh:not`` per member +================== ===================================================== + +Each member becomes an anonymous node shape. ``is_a`` names a class that every +instance of the shape's class belongs to (the class itself or an ancestor), and +then adds nothing; one that only some instances belong to, because a descendant +of the class has it among its ancestors, and then gives ``sh:class``; or one that +none belongs to, and then gives a shape nothing conforms to. An instance belongs +to its class and that class's ancestors, by ``is_a`` and by ``mixins``, so a +class that may also be another is declared as a subclass with that mixin. +Nested expressions recurse. Each entry of ``slot_conditions`` gives an +``sh:property`` whose path is that of the slot as induced for the class, so +``slot_usage`` applies: + +* ``required``, ``value_presence`` and the cardinalities give ``sh:minCount`` / + ``sh:maxCount``; +* ``minimum_value`` / ``maximum_value`` give ``sh:minInclusive`` / + ``sh:maxInclusive``, and ``equals_number`` gives both, so that ``5`` also + matches ``5.0``; +* ``pattern`` gives ``sh:pattern``; +* ``equals_string`` and ``equals_string_in`` give ``sh:in``; on an enum slot the + values are the permissible values as the enum renders them, the IRI of their + ``meaning`` where they have one; +* ``range`` gives the same class, type or enum constraint as a slot's range. + +A shape may have at most one value of ``sh:minInclusive``, ``sh:maxInclusive`` +or ``sh:in``, and of ``sh:pattern``, whose component also takes ``sh:flags`` +(`SHACL §2.1.1 `__). Where one condition +needs one of them twice, for example ``minimum_value`` next to +``equals_number``, the second value goes into an ``sh:and`` member of the +property shape, where it applies to the same values. + +A slot condition is unknown for an absent slot unless it decides whether the slot +may be absent, and an instance violates an expression only when the expression is +definitely false, as described under "Class-level expressions and absent slots" in +:doc:`Advanced features `. +The JSON Schema generator reads them the same way. In SHACL, each expression gets +its "not false" form, and, under ``sh:not``, its "definitely true" form, where +such a condition also requires its slot (``sh:minCount 1``). ``exactly_one_of`` +becomes ``sh:xone`` over the "definitely true" forms of its members. + +.. code-block:: yaml + + GeodeticReferenceSystem: + slots: [code, name] + any_of: + - slot_conditions: + code: + required: true + - slot_conditions: + name: + required: true + +.. code-block:: turtle + + ex:GeodeticReferenceSystem a sh:NodeShape ; + sh:or ( [ sh:property [ sh:path ex:code ; sh:minCount 1 ] ] + [ sh:property [ sh:path ex:name ; sh:minCount 1 ] ] ) ; + ... + +A class expression constrains every instance of its class, so the shape of a +class carries the expressions of its ancestors and mixins as well, as it carries +their slots. Each is translated in the context of that class, where +``slot_usage`` applies. A node typed only with a subclass, as ``rdflib_dumper`` +and the JSON-LD context produce it, is therefore checked without an +``rdfs:subClassOf`` triple in the data. The ``sh:class`` that ``is_a`` gives +recognises instances of subclasses only where the data graph states the +``rdfs:subClassOf`` (`SHACL §4.1.1 +`__), as it does for a +slot's range. + +An operator whose members use anything else is skipped as a whole and logged as +a warning, because leaving out one member would change what the operator +admits. That covers, for example, ``has_member`` or a slot-level ``any_of`` +inside a slot condition, a condition on a name that is not a slot, a condition +on the identifier slot (the node's IRI rather than a property), and +``equals_string`` on a slot whose range does not hold strings. + + Command Line ^^^^^^^^^^^^ diff --git a/docs/schemas/advanced.md b/docs/schemas/advanced.md index c56a808379..2f173f2443 100644 --- a/docs/schemas/advanced.md +++ b/docs/schemas/advanced.md @@ -64,6 +64,36 @@ The following LinkML constructs can be used to express boolean constraints: These can be applied at the class or slot level. The range of each of these is an array of *expressions*. +### Class-level expressions and absent slots + +At the class level, each member of `any_of`, `all_of`, `exactly_one_of` and `none_of` is a class expression whose `slot_conditions` constrain the instance's slots. The expressions of a class also constrain the instances of its subclasses, and of the classes that use it as a mixin. + +A slot condition needs a meaning when its slot is absent. The JSON Schema and SHACL generators read it as an SQL `CHECK` constraint reads a condition on a null value: an instance is invalid only when an expression is definitely false. + +- A condition that decides whether its slot may be absent is true or false as usual. Such a condition sets `value_presence: PRESENT` or `ABSENT`, `required: true`, a minimum or exact cardinality of at least 1, or a maximum or exact cardinality of 0. +- Any other condition is *unknown* when its slot is absent. +- `any_of`, `all_of` and `none_of` combine these as "or", "and" and "not". An unknown member doesn't make `any_of` true, nor `none_of` false. +- `exactly_one_of` holds when exactly one member is definitely true. + +```yaml +classes: + Sample: + none_of: + - slot_conditions: + status: + equals_string: retracted +``` + +A `Sample` without `status` is valid: the condition is unknown, so `none_of` isn't false. To require the slot, state it with `status: {required: true}`. + +| Expression | `{}` | `{label: A}` | `{label: B}` | +|---|---|---|---| +| `any_of: [label = A]` | valid | valid | invalid | +| `none_of: [label = A]` | valid | invalid | valid | +| `exactly_one_of: [label = A, note = B]` | invalid | valid | invalid | + +[Rules](#rules) are read differently: their preconditions require their slots, and so do their postconditions unless the rule is `open_world`. + ### Unions as ranges [any_of](https://w3id.org/linkml/any_of) can be used to express that a range must satisfy any of a set of ranges. diff --git a/packages/linkml/src/linkml/generators/common/class_expression.py b/packages/linkml/src/linkml/generators/common/class_expression.py new file mode 100644 index 0000000000..ba4bd04860 --- /dev/null +++ b/packages/linkml/src/linkml/generators/common/class_expression.py @@ -0,0 +1,150 @@ +"""Shared semantics of slot conditions in class-level boolean expressions. + +The class-level operators ``any_of``, ``all_of``, ``exactly_one_of`` and +``none_of`` combine class expressions whose slot conditions constrain slots of +an instance. The specification does not say what a condition means for an +absent slot. Generators that translate class expressions read it the same +way, as an SQL CHECK constraint (ISO/IEC 9075) reads a condition on a null +value: a condition that doesn't state presence is *unknown* for an absent +slot, and an expression is violated only when it is definitely false. + +Each expression therefore has two forms: + +* its **"not false" form**, which the instance must satisfy, and in which a + condition holds for an absent slot unless it states presence; +* its **"definitely true" form**, used under a negation, in which a condition + that doesn't state presence also requires its slot. + +``none_of`` takes its members in the opposite form, ``exactly_one_of`` counts +the members that are definitely true, and ``any_of`` / ``all_of`` keep the +form. + +>>> from linkml_runtime.linkml_model.meta import SlotDefinition +>>> value_bounds(SlotDefinition("label", equals_string="A"), definite=False) +(0, None) +>>> value_bounds(SlotDefinition("label", equals_string="A"), definite=True) +(1, None) +>>> value_bounds(SlotDefinition("label", required=True, value_presence="ABSENT"), definite=True) +(0, 0) +>>> value_bounds(SlotDefinition("tags", minimum_cardinality=2, maximum_cardinality=3), definite=False) +(2, 3) +""" + +from linkml_runtime.linkml_model.meta import PresenceEnum, SlotDefinition +from linkml_runtime.utils.schemaview import SchemaView + +_PRESENT = PresenceEnum(PresenceEnum.PRESENT) +_ABSENT = PresenceEnum(PresenceEnum.ABSENT) + + +def states_presence(condition: SlotDefinition) -> bool: + """Whether *condition* decides whether its slot may be absent. + + Such a condition is definitely true or false for an absent slot: + ``value_presence: PRESENT`` or ``ABSENT``; ``required: true``, unless + ``value_presence`` overrides it; a minimum or exact cardinality of at least + 1, which an absent slot fails; and a maximum or exact cardinality of 0, + which it satisfies. Other bounds, ``required: false`` and ``UNCOMMITTED`` + leave absence open, so adding one never changes the verdict on an absent + slot. + + >>> states_presence(SlotDefinition("label", equals_string="A")) + False + >>> states_presence(SlotDefinition("label", required=False, equals_string="A")) + False + >>> states_presence(SlotDefinition("tags", maximum_cardinality=5)) + False + >>> states_presence(SlotDefinition("tags", maximum_cardinality=0)) + True + >>> states_presence(SlotDefinition("tags", minimum_cardinality=1)) + True + """ + if condition.value_presence is not None: + if condition.value_presence in (_PRESENT, _ABSENT): + return True + elif condition.required: + return True + lower = (condition.minimum_cardinality, condition.exact_cardinality) + upper = (condition.maximum_cardinality, condition.exact_cardinality) + return any(bound is not None and bound >= 1 for bound in lower) or 0 in upper + + +def value_bounds(condition: SlotDefinition, definite: bool) -> tuple[int, int | None]: + """The least and the greatest number of values *condition* allows its slot. + + ``value_presence`` takes precedence over ``required``. In the "definitely + true" form (*definite*), a condition that doesn't state presence also + requires the slot. The greatest number is ``None`` when unbounded. + """ + lower, upper = [0], [] + if condition.value_presence is not None: + if condition.value_presence == _PRESENT: + lower.append(1) + elif condition.value_presence == _ABSENT: + upper.append(0) + elif condition.required: + lower.append(1) + if definite and not states_presence(condition): + lower.append(1) + for bound, target in ( + (condition.minimum_cardinality, lower), + (condition.exact_cardinality, lower), + (condition.maximum_cardinality, upper), + (condition.exact_cardinality, upper), + ): + if bound is not None: + target.append(int(bound)) + return max(lower), min(upper) if upper else None + + +def class_membership(sv: SchemaView, cls_name: str, target: str) -> bool | None: + """Whether an instance of the class *cls_name* is an instance of the class *target*. + + A LinkML instance belongs to its class and to that class's ancestors, by + ``is_a`` and by ``mixins``. So every instance of *cls_name* is one when + *target* is *cls_name* or an ancestor (``True``); some may be when *target* + is an ancestor of a descendant of *cls_name*, which the instance may belong + to (``None``); and none is otherwise (``False``). + + A class carries the expressions of its ancestors, which may name any of + these: a ``Person`` carries its parent's ``any_of: [{is_a: Person}, + {is_a: Organization}]``, whose first member it satisfies and whose second + it cannot. A value of ``linkml:Any`` may be an instance of any class, and + every instance is one of ``linkml:Any``. + + >>> sv = SchemaView(''' + ... id: https://example.org/membership + ... name: membership + ... classes: + ... Any: {class_uri: linkml:Any} + ... Agent: {} + ... Person: {is_a: Agent} + ... Student: {is_a: Person} + ... Organization: {is_a: Agent} + ... ''') + >>> class_membership(sv, "Student", "Person") + True + >>> print(class_membership(sv, "Agent", "Person")) + None + >>> class_membership(sv, "Person", "Organization") + False + >>> print(class_membership(sv, "Any", "Person")) + None + >>> class_membership(sv, "Person", "Any") + True + """ + if target in sv.class_ancestors(cls_name) or _is_any(sv, target): + return True + if _is_any(sv, cls_name): + return None + if any(target in sv.class_ancestors(descendant) for descendant in sv.class_descendants(cls_name)): + return None + return False + + +def _is_any(sv: SchemaView, cls_name: str) -> bool: + """Whether the class *cls_name* is ``linkml:Any``, the class of every value.""" + cls = sv.get_class(cls_name) + if cls is None or not cls.class_uri: + return False + return cls.class_uri == "linkml:Any" or sv.expand_curie(cls.class_uri) == "https://w3id.org/linkml/Any" diff --git a/packages/linkml/src/linkml/generators/jsonschemagen.py b/packages/linkml/src/linkml/generators/jsonschemagen.py index ef0ea170fb..4a33c3ccf9 100644 --- a/packages/linkml/src/linkml/generators/jsonschemagen.py +++ b/packages/linkml/src/linkml/generators/jsonschemagen.py @@ -14,6 +14,7 @@ from linkml.generators.common import build from linkml.generators.common.array import ArrayRangeGenerator, ArrayRepresentation from linkml.generators.common.build import RangeResult +from linkml.generators.common.class_expression import class_membership, value_bounds from linkml.generators.common.lifecycle import LifecycleMixin from linkml.generators.common.subproperty import get_subproperty_values from linkml.generators.common.type_designators import ( @@ -275,24 +276,12 @@ def add_property( self["required"].append(canonical_name) # JSON Schema does not have a very natural way to express that a property cannot be present. - # The apparent best way to do it is to use: - # { - # properties: { - # foo: ... - # }, - # not: { - # required: ['foo'] - # } - # } - # The {required: [foo]} subschema evaluates to true if the foo property is present with any - # value. Wrapping that in a `not` keyword inverts that condition. + # The apparent best way to do it is `not: {required: [foo]}`: the {required: [foo]} + # subschema evaluates to true if the foo property is present with any value, and `not` + # inverts that. Each absent property gets its own `not`, in `allOf`, since + # `not: {required: [foo, bar]}` would only forbid foo and bar together. if value_disallowed: - if "not" not in self: - self["not"] = {} - if "required" not in self["not"]: - self["not"]["required"] = [] - - self["not"]["required"].append(canonical_name) + self.setdefault("allOf", []).append({"not": {"required": [canonical_name]}}) def add_keyword(self, keyword: str, value: Any): if value is None: @@ -626,29 +615,20 @@ def handle_class(self, cls: ClassDefinition) -> None: class_subschema["allOf"] = [] class_subschema["allOf"].extend(rule_subschemas) - if cls.any_of is not None and len(cls.any_of) > 0: - class_subschema["anyOf"] = [self.get_subschema_for_anonymous_class(c, False) for c in cls.any_of] - - if cls.all_of is not None and len(cls.all_of) > 0: - if "allOf" not in class_subschema: - class_subschema["allOf"] = [] - class_subschema["allOf"].extend([self.get_subschema_for_anonymous_class(c, False) for c in cls.all_of]) - - if cls.exactly_one_of is not None and len(cls.exactly_one_of) > 0: - class_subschema["oneOf"] = [self.get_subschema_for_anonymous_class(c, False) for c in cls.exactly_one_of] - - if cls.none_of is not None and len(cls.none_of) > 0: - # properties_required=True so absent slots make their branch fail; otherwise - # properties is vacuously true and `not(anyOf)` rejects instances missing the slot. - new_not = {"anyOf": [self.get_subschema_for_anonymous_class(c, True) for c in cls.none_of]} - if "not" in class_subschema: - existing_not = class_subschema.pop("not") - if "allOf" not in class_subschema: - class_subschema["allOf"] = [] - class_subschema["allOf"].append({"not": existing_not}) - class_subschema["allOf"].append({"not": new_not}) - else: - class_subschema["not"] = new_not + # A class expression constrains every instance of its class, so a class carries the + # expressions of its ancestors and mixins, translated in its own context. + for owner in self.schemaview.class_ancestors(cls.name): + for operator in self.CLASS_EXPRESSION_OPERATORS: + members = getattr(self.schemaview.get_class(owner), operator) or [] + if members: + expression = self.get_subschema_for_class_operator(cls, operator, members, definite=False) + for keyword, value in expression.items(): + if keyword == "allOf": + class_subschema.setdefault("allOf", []).extend(value) + elif keyword not in class_subschema: + class_subschema[keyword] = value + else: + class_subschema.setdefault("allOf", []).append({keyword: value}) class_subschema = self.after_generate_class( ClassResult.model_construct(schema_=class_subschema, source=cls), self.schemaview @@ -674,6 +654,120 @@ def handle_class(self, cls: ClassDefinition) -> None: if key not in self.top_level_schema: self.top_level_schema[key] = value + CLASS_EXPRESSION_OPERATORS = ("any_of", "all_of", "exactly_one_of", "none_of") + _CLASS_OPERATOR_KEYWORDS = {"any_of": "anyOf", "all_of": "allOf", "exactly_one_of": "oneOf"} + + def get_subschema_for_class_operator( + self, cls: ClassDefinition, operator: str, members: list[AnonymousClassExpression], definite: bool + ) -> JsonSchema: + """The subschema of the class-level boolean expression *operator* over *members*, for *cls*. + + A condition that doesn't state presence is unknown for an absent slot, + and an expression is violated only when it is definitely false (see + :mod:`linkml.generators.common.class_expression`). With *definite* the + subschema accepts the instances for which the expression is definitely + true, otherwise those for which it is not false. ``none_of`` takes its + members in the opposite form, ``exactly_one_of`` counts the members + that are definitely true, and ``any_of`` / ``all_of`` keep the form. + """ + if operator == "none_of": + return JsonSchema( + {"not": {"anyOf": [self.get_subschema_for_class_expression(cls, m, not definite) for m in members]}} + ) + form = True if operator == "exactly_one_of" else definite + return JsonSchema( + { + self._CLASS_OPERATOR_KEYWORDS[operator]: [ + self.get_subschema_for_class_expression(cls, m, form) for m in members + ] + } + ) + + def get_subschema_for_class_expression( + self, cls: ClassDefinition, expr: AnonymousClassExpression, definite: bool + ) -> JsonSchema: + """The subschema of one member *expr* of a class-level boolean expression of *cls*. + + Each slot condition constrains the slot as induced for *cls*, so + ``slot_usage`` applies. Its values must satisfy its value operators + and its ``range``; the number of values must lie within + :func:`~linkml.generators.common.class_expression.value_bounds`, where + an empty list or dict counts as no value. *definite* selects the form, as in + :meth:`get_subschema_for_class_operator`. + """ + subschema = JsonSchema() + conjuncts: list[JsonSchema] = [] + for slot_name, condition in expr.slot_conditions.items(): + slot = self._class_expression_slot(cls, slot_name) or condition + if self.use_curies: + prop_name = self._curie(slot) + else: + prop_name = self.aliased_slot_name(slot) + values = self.get_subschema_for_slot(condition, omit_type=True, include_null=False) + lower, upper = value_bounds(condition, definite) + if slot.multivalued and self._inlined_as_dict(slot): + # the values of a slot inlined as a dict are the values of the dict + prop = JsonSchema({"type": "object", "additionalProperties": values}) + prop.add_keyword("minProperties", lower or None) + prop.add_keyword("maxProperties", upper) + elif slot.multivalued: + prop = JsonSchema.array_of(values, include_null=False, required=False) + prop.add_keyword("minItems", lower or None) + prop.add_keyword("maxItems", upper) + else: + prop = values + if condition.range is not None: + # the values in the representation of the slot, as a list, a dict or a single value + typed = self.get_subschema_for_slot( + SlotDefinition( + slot.name, + range=condition.range, + multivalued=slot.multivalued, + inlined=slot.inlined, + inlined_as_list=slot.inlined_as_list, + ), + include_null=False, + ) + prop = JsonSchema({"allOf": [prop, typed]}) if prop else typed + if slot.multivalued: + subschema.add_property(prop_name, prop, value_required=lower > 0) + else: + subschema.add_property(prop_name, prop, value_required=lower > 0, value_disallowed=upper == 0) + if lower > 1: + # a single-valued slot holds at most one value + conjuncts.append(JsonSchema({"not": {}})) + for operator in self.CLASS_EXPRESSION_OPERATORS: + members = getattr(expr, operator) or [] + if members: + conjuncts.append(self.get_subschema_for_class_operator(cls, operator, members, definite)) + if expr.is_a is not None: + # `is_a: ` requires an instance of the expression to be an instance of . Every + # instance of *cls* is one, or none is, unless is a descendant of *cls*: then, in + # JSON, only the instance's structure tells, so it must match the definition of . + membership = class_membership(self.schemaview, cls.name, expr.is_a) + if membership is None: + conjuncts.append(self.get_subschema_for_slot(AnonymousSlotExpression(range=expr.is_a, inlined=True))) + elif not membership: + conjuncts.append(JsonSchema({"not": {}})) + if conjuncts: + subschema.setdefault("allOf", []).extend(conjuncts) + return subschema + + def _inlined_as_dict(self, slot: SlotDefinition) -> bool: + """Whether the values of multivalued *slot* are inlined as a dict, keyed by their identifier.""" + return ( + self.schemaview.is_inlined(slot) + and not slot.inlined_as_list + and get_range_associated_slots(self.schemaview, slot.range)[0] is not None + ) + + def _class_expression_slot(self, cls: ClassDefinition, slot_name: str) -> SlotDefinition | None: + """The slot a condition of a class expression of *cls* names, as induced for *cls*, if any.""" + sv = self.schemaview + if slot_name in sv.class_slots(cls.name): + return sv.induced_slot(slot_name, cls.name) + return sv.get_slot(slot_name) + def get_subschema_for_anonymous_class( self, cls: AnonymousClassExpression, properties_required: bool = False ) -> None | JsonSchema: diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index eb6b48e9d6..f1ee237952 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -2,7 +2,7 @@ import os import string from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, fields import click from jsonasobj2 import JsonObj, as_dict @@ -11,12 +11,23 @@ from rdflib.namespace import RDF, RDFS, SH, XSD from linkml._version import __version__ +from linkml.generators.common.class_expression import class_membership, value_bounds from linkml.generators.common.subproperty import get_subproperty_values, is_uri_range from linkml.generators.shacl.shacl_data_type import ShaclDataType from linkml.generators.shacl.shacl_ifabsent_processor import ShaclIfAbsentProcessor from linkml.utils.generator import Generator, shared_arguments from linkml.utils.language_tags import LanguageTagResolver -from linkml_runtime.linkml_model.meta import ClassDefinition, ElementName, PresenceEnum +from linkml_runtime.linkml_model.meta import ( + AnonymousClassExpression, + AnonymousSlotExpression, + ClassDefinition, + ClassExpression, + Element, + ElementName, + PresenceEnum, + SlotDefinition, + SlotExpression, +) from linkml_runtime.utils.formatutils import underscore from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph from linkml_runtime.utils.yamlutils import TypedNode, extended_float, extended_int, extended_str @@ -211,6 +222,8 @@ def as_graph(self) -> Graph: for pfx in self.schema.prefixes.values(): g.bind(str(pfx.prefix_prefix), pfx.prefix_reference) + self._class_expressions_added: set[tuple[URIRef, str, str]] = set() + self._class_expression_problems: dict[tuple[str, str, str], list[str]] = {} for c in sv.all_classes(imports=not self.exclude_imports).values(): def shape_pv(p, v): @@ -251,12 +264,7 @@ def shape_pv(p, v): self._add_annotations(shape_pv, c) order = 0 for s in sv.class_induced_slots(c.name): - # fixed in linkml-runtime 1.1.3 - if s.name in sv.element_by_schema_map(): - slot_uri = URIRef(sv.get_uri(s, expand=True)) - else: - pfx = sv.schema.default_prefix - slot_uri = URIRef(sv.expand_curie(f"{pfx}:{underscore(s.name)}")) + slot_uri = URIRef(self._slot_iri(s)) pnode = BNode() shape_pv(SH.property, pnode) @@ -404,21 +412,7 @@ def st_node_pv(p, v): f" require range 'string' and not '{r}'" ) - if r in all_classes: - cls_def = sv.get_class(r) - is_any = cls_def and getattr(cls_def, "class_uri", None) == "linkml:Any" - self._add_class(prop_pv, r) - if not is_any: - if sv.get_identifier_slot(r) is not None: - prop_pv(SH.nodeKind, SH.IRI) - else: - prop_pv(SH.nodeKind, SH.BlankNodeOrIRI) - elif r in sv.all_types(): - self._add_type(prop_pv, r) - elif r in sv.all_enums(): - self._add_enum(g, prop_pv, r) - else: - add_simple_data_type(prop_pv, r) + self._add_range(g, prop_pv, r) if s.equals_string: # Map equal_string and equal_string_in to sh:in self._and_equals_string(g, prop_pv, [s.equals_string]) @@ -439,13 +433,372 @@ def st_node_pv(p, v): if default_value: prop_pv(SH.defaultValue, default_value) + self._add_class_expressions(g, class_uri_with_suffix, c) + if self.emit_rules: self._add_rules(g, class_uri_with_suffix, c) + self._report_class_expression_problems() return g LINKML_ANY_URI = "https://w3id.org/linkml/Any" + # ------------------------------------------------------------------- + # Class expressions → sh:or / sh:and / sh:xone / sh:not + # ------------------------------------------------------------------- + + # The SHACL logical constraint component, taking a list, of each list operator. + _LIST_OPERATORS = {"any_of": SH["or"], "all_of": SH["and"], "exactly_one_of": SH.xone} + _CLASS_EXPRESSION_OPERATORS = ("any_of", "all_of", "exactly_one_of", "none_of") + + # The fields of an anonymous class expression that carry meaning, derived from + # the metamodel, so that a new semantic field is reported as untranslatable + # instead of being ignored. Every other field is metadata. + _CLASS_EXPRESSION_FIELDS = frozenset(f.name for f in fields(ClassExpression)) | {"is_a"} + + # Slot-condition fields translated by _slot_condition_shape: those deciding + # whether the slot is present, and those constraining its values. + _SLOT_CONDITION_PRESENCE_FIELDS = frozenset( + {"required", "value_presence", "minimum_cardinality", "maximum_cardinality", "exact_cardinality"} + ) + _SLOT_CONDITION_VALUE_FIELDS = frozenset( + {"minimum_value", "maximum_value", "pattern", "equals_string", "equals_string_in", "equals_number", "range"} + ) + _SLOT_CONDITION_FIELDS = _SLOT_CONDITION_PRESENCE_FIELDS | _SLOT_CONDITION_VALUE_FIELDS + + # Parameters a shape may have at most one value of: sh:minInclusive and + # sh:maxInclusive (SHACL §4.3), sh:in (§4.8.3), sh:datatype and sh:nodeKind + # (§4.1), and sh:pattern, as a parameter of a component with more than one + # parameter (§2.1.1). A condition that needs one of them twice gets the + # second value in an sh:and member. + _SINGLE_VALUE_PARAMETERS = frozenset( + {SH.minInclusive, SH.maxInclusive, SH["in"], SH.pattern, SH.datatype, SH.nodeKind} + ) + + # Fields on a slot condition / class expression that carry no constraint + # semantics: they never change which instances satisfy the condition, so + # they are ignored by the operator accounting below. Anything set on a + # condition that is neither here nor explicitly translated by a converter + # makes the rule untranslatable — the converters must SKIP such a rule + # rather than emit a query that silently drops a conjunct (which would + # widen the trigger or narrow the check: a mis-translation, not a skip). + # Derived from the metamodel: the metadata every ``element`` carries, minus + # anything that is a ``slot_expression`` operator. + _NON_OPERATOR_FIELDS = frozenset(f.name for f in fields(Element)) - frozenset( + f.name for f in fields(SlotExpression) + ) + + @classmethod + def _set_operator_fields( + cls, condition: SlotDefinition | AnonymousSlotExpression | AnonymousClassExpression + ) -> set[str]: + """Return the names of the constraint-bearing fields actually set on a + rule condition or class expression. + + A field counts as *set* when it is not ``None`` and not an empty + collection (SchemaView materialises unset multivalued fields as empty + lists / dicts). Scalars are never judged by truthiness, so legitimate + falsy constraints such as ``minimum_value: 0`` or + ``equals_string: ""`` still count as set. Metadata fields + (:data:`_NON_OPERATOR_FIELDS`) are excluded. + + The converters compare this set against the exact operator set they + translate and skip the rule on any mismatch, so an unrecognised (or + future-metamodel) operator can never be silently dropped. + """ + return { + name + for name, value in vars(condition).items() + if not name.startswith("_") + and name not in cls._NON_OPERATOR_FIELDS + and value is not None + and not (isinstance(value, list | dict) and not value) + } + + def _add_class_expressions(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: + """Emit the class-level boolean expressions of *cls* as SHACL logical constraints. + + Each operator is mapped to the SHACL logical constraint component with the + same semantics (`SHACL §4.6 `_): + + * ``any_of`` → ``sh:or``, ``all_of`` → ``sh:and``, ``exactly_one_of`` → + ``sh:xone``, each over a list of the member shapes; + * ``none_of`` → one ``sh:not`` per member. A shape's values of ``sh:not`` + are separate constraints that all apply (SHACL §2.1.1), so the node must + conform to none of the members. + + Every member becomes an anonymous node shape: ``is_a`` gives ``sh:class`` + for a class that some instances of *cls* belong to through a descendant, + nothing for *cls* or an ancestor, which every instance belongs to, and a + shape nothing conforms to for any other class + (:func:`~linkml.generators.common.class_expression.class_membership`); + each slot condition gives an ``sh:property`` on the path of the slot as + induced for *cls* (so ``slot_usage`` applies), and nested expressions + recurse. + + The specification doesn't say what a slot condition means for an absent + slot. As with an SQL CHECK constraint (ISO/IEC 9075), which is + satisfied unless its condition is false, a class expression is violated + only when it is definitely false, and a condition that doesn't state + presence (:func:`~linkml.generators.common.class_expression.states_presence`) is unknown + for an absent slot. + Each operator is therefore translated in its "not false" form, and, + under a negation, in its "definitely true" form, where such a condition + requires its slot. ``exactly_one_of`` holds when exactly one member is + definitely true. + + A class expression constrains every instance of its class, so the shape + of *cls* carries the expressions of its ancestors and mixins as well, as + it carries their slots. Each is translated in the context of *cls*, + where ``slot_usage`` may refine a slot it names. Classes that share a + ``class_uri`` share one shape, which carries each expression once. + + An operator whose members use anything that cannot be translated is + skipped as a whole, with a warning: dropping one member would change what + the operator admits. + """ + sv = self.schemaview + for owner in sv.class_ancestors(cls.name): + for operator in self._CLASS_EXPRESSION_OPERATORS: + members = getattr(sv.get_class(owner), operator, None) or [] + if not members or (shape_uri, owner, operator) in self._class_expressions_added: + continue + self._class_expressions_added.add((shape_uri, owner, operator)) + reason = next(filter(None, (self._untranslatable(cls, m) for m in members)), None) + if reason is not None: + classes = self._class_expression_problems.setdefault((owner, operator, reason), []) + if cls.name not in classes: + classes.append(cls.name) + continue + self._add_logical_constraint(g, shape_uri, cls, operator, members, definite=False) + + def _report_class_expression_problems(self) -> None: + """Warn once about each class expression that is not translated, naming the + class shapes it is missing from unless that is only the declaring class.""" + for (owner, operator, reason), classes in self._class_expression_problems.items(): + shapes = "" if classes == [owner] else f" (in the shapes of {', '.join(map(repr, classes))})" + logger.warning( + "Class %r: %s is not translated to SHACL, because it uses %s%s.", owner, operator, reason, shapes + ) + + def _add_logical_constraint( + self, + g: Graph, + subject: URIRef | BNode, + cls: ClassDefinition, + operator: str, + members: list[AnonymousClassExpression], + definite: bool, + ) -> None: + """Add the logical constraint for *operator* over *members* to *subject*. + + With *definite* the constraint holds when the expression is definitely + true, otherwise when it is not false. ``none_of`` gives one ``sh:not`` + per member, in the opposite form, since an expression is not false + exactly when its negation is not definitely true. ``exactly_one_of`` + counts the members that are definitely true, in either form. The + other list operators keep the form. + """ + if operator == "none_of": + for member in members: + g.add((subject, SH["not"], self._class_expression_shape(g, cls, member, not definite))) + return + predicate = self._LIST_OPERATORS[operator] + member_form = True if operator == "exactly_one_of" else definite + shapes = [self._class_expression_shape(g, cls, m, member_form) for m in members] + list_node = BNode() + Collection(g, list_node, shapes) + g.add((subject, predicate, list_node)) + + def _class_expression_shape( + self, g: Graph, cls: ClassDefinition, expr: AnonymousClassExpression, definite: bool + ) -> BNode: + """Build the anonymous node shape for one class expression *expr*, in its + "definitely true" form with *definite*, otherwise its "not false" form.""" + node = BNode() + + def node_pv(p, v): + if v is not None: + g.add((node, p, v)) + + if expr.title is not None: + node_pv(RDFS.label, Literal(expr.title, lang=self._resolve_language(expr))) + if expr.description is not None: + node_pv(RDFS.comment, Literal(expr.description, lang=self._resolve_language(expr))) + if expr.is_a is not None: + # Every instance of *cls* is an instance of ``is_a``, or none is, unless ``is_a`` is a + # descendant of *cls*: only then does the node's type decide. + membership = class_membership(self.schemaview, cls.name, expr.is_a) + if membership is None: + self._add_class(node_pv, expr.is_a) + elif not membership: + node_pv(SH["not"], BNode()) # a shape nothing conforms to + for slot_name, condition in expr.slot_conditions.items(): + node_pv(SH.property, self._slot_condition_shape(g, cls, slot_name, condition, definite)) + for operator in self._CLASS_EXPRESSION_OPERATORS: + members = getattr(expr, operator) or [] + if members: + self._add_logical_constraint(g, node, cls, operator, members, definite) + return node + + def _slot_condition_shape( + self, g: Graph, cls: ClassDefinition, slot_name: str, condition: SlotDefinition, definite: bool + ) -> BNode: + """Build the property shape for the condition on *slot_name*, in its + "definitely true" form with *definite*, otherwise its "not false" form.""" + slot = self._condition_slot(cls, slot_name) + pnode = BNode() + repeated = [] + + def prop_pv(p, v): + if v is None: + return + if p in self._SINGLE_VALUE_PARAMETERS and (pnode, p, None) in g: + repeated.append((p, v)) + else: + g.add((pnode, p, v)) + + prop_pv(SH.path, URIRef(self._slot_iri(slot))) + if condition.title is not None: + prop_pv(SH.name, Literal(condition.title, lang=self._resolve_language(condition))) + if condition.description is not None: + prop_pv(SH.description, Literal(condition.description, lang=self._resolve_language(condition))) + + lower, upper = value_bounds(condition, definite) + if lower: + prop_pv(SH.minCount, Literal(lower)) + if upper is not None: + prop_pv(SH.maxCount, Literal(upper)) + + if condition.minimum_value is not None: + prop_pv(SH.minInclusive, Literal(condition.minimum_value)) + if condition.maximum_value is not None: + prop_pv(SH.maxInclusive, Literal(condition.maximum_value)) + if condition.pattern is not None: + prop_pv(SH.pattern, Literal(condition.pattern)) + value_range = condition.range or slot.range + for values in ( + [condition.equals_string] if condition.equals_string is not None else [], + condition.equals_string_in, + ): + if values: + in_node = BNode() + Collection(g, in_node, self._string_value_terms(value_range, values)) + prop_pv(SH["in"], in_node) + if condition.equals_number is not None: + # A value comparison, unlike the slot loop's sh:hasValue: 5 matches 5.0, + # and like every other value constraint in a condition it holds when the + # slot is absent. + prop_pv(SH.minInclusive, Literal(condition.equals_number)) + prop_pv(SH.maxInclusive, Literal(condition.equals_number)) + if condition.range is not None: + self._add_range(g, prop_pv, condition.range) + if repeated: + # Each repeated parameter in a member shape of its own: all of them hold + # for every value, as they would on the property shape itself. + members = [] + for p, v in repeated: + member = BNode() + g.add((member, p, v)) + members.append(member) + and_node = BNode() + Collection(g, and_node, members) + g.add((pnode, SH["and"], and_node)) + return pnode + + def _string_value_terms(self, r: ElementName | None, values: list[str]) -> list[URIRef | Literal]: + """The RDF terms of the ``equals_string`` / ``equals_string_in`` *values* of a slot with range *r*. + + Permissible values of an enum are rendered as :meth:`_add_enum` renders them, as + the IRI of their ``meaning`` where they have one; anything else is a plain literal. + """ + sv = self.schemaview + if r in sv.all_enums(): + pvs = sv.get_enum(r).permissible_values + return [ + URIRef(sv.expand_curie(pvs[v].meaning)) if v in pvs and pvs[v].meaning else Literal(v) for v in values + ] + return [Literal(v) for v in values] + + def _condition_slot(self, cls: ClassDefinition, slot_name: str) -> SlotDefinition | None: + """The slot a condition of *cls* names, as induced for *cls*, or ``None`` if there is none.""" + try: + return self.schemaview.induced_slot(slot_name, cls.name) + except ValueError: + return None + + def _type_uri(self, r: ElementName | None) -> str | None: + """The expanded datatype IRI of type range *r*, or ``None`` when *r* is not a type. + + Resolved through the induced type, so a type derived with ``typeof`` + inherits the ``uri`` of its ancestor. A built-in type name in a schema + that does not import ``linkml:types`` resolves as the main slot loop + resolves it (:class:`ShaclDataType`). + """ + sv = self.schemaview + if r in sv.all_types(): + return sv.get_uri(sv.induced_type(r), expand=True) + builtin = next((t for t in ShaclDataType if t.linkml_type == r), None) + return str(builtin.uri_ref) if builtin is not None else None + + def _is_string_range(self, r: ElementName | None) -> bool: + """Whether a slot with range *r* holds strings, which ``equals_string`` compares against. + + True for an enum, whose permissible values are rendered as their + ``meaning`` IRI or as a plain literal (as :meth:`_add_enum` renders + them); for a type whose datatype is ``xsd:string``, whose values are + plain literals; and for no range at all, whose values are untyped and + compared as strings, as the JSON Schema generator compares them. A + type with any other datatype, including one derived from ``string`` + (``xsd:anyURI``, ``xsd:token``, ...), holds typed literals or IRIs + that a string literal does not match. + """ + if r is None or r in self.schemaview.all_enums(): + return True + return self._type_uri(r) == str(XSD.string) + + def _untranslatable(self, cls: ClassDefinition, expr: AnonymousClassExpression) -> str | None: + """Return what in class expression *expr* of *cls* cannot be translated, or ``None``.""" + sv = self.schemaview + unknown = self._set_operator_fields(expr) - self._CLASS_EXPRESSION_FIELDS + if unknown: + return f"'{sorted(unknown)[0]}'" + if expr.is_a is not None and expr.is_a not in sv.all_classes(): + return f"is_a '{expr.is_a}', which is not a class" + for slot_name, condition in expr.slot_conditions.items(): + unknown = self._set_operator_fields(condition) - self._SLOT_CONDITION_FIELDS + if unknown: + return f"'{sorted(unknown)[0]}' in the condition on slot '{slot_name}'" + slot = self._condition_slot(cls, slot_name) + if slot is None: + return f"a condition on '{slot_name}', which is not a slot" + if slot.identifier: + # An identifier is the node's IRI, not a property arc. + return f"a condition on the identifier slot '{slot_name}'" + if condition.range is not None and not self._is_known_range(condition.range): + return f"the unknown range '{condition.range}' in the condition on slot '{slot_name}'" + value_range = condition.range or slot.range + if (condition.equals_string is not None or condition.equals_string_in) and not self._is_string_range( + value_range + ): + return f"equals_string on slot '{slot_name}', whose range '{value_range}' does not hold strings" + for operator in self._CLASS_EXPRESSION_OPERATORS: + for member in getattr(expr, operator) or []: + reason = self._untranslatable(cls, member) + if reason is not None: + return reason + return None + + def _is_known_range(self, r: ElementName) -> bool: + """Whether *r* names a class, type or enum of the schema, or a built-in type.""" + sv = self.schemaview + return ( + r in sv.all_classes() + or r in sv.all_types() + or r in sv.all_enums() + or any(datatype.linkml_type == r for datatype in ShaclDataType) + ) + # ------------------------------------------------------------------- # Rules → sh:sparql # ------------------------------------------------------------------- @@ -705,6 +1058,37 @@ def _add_class(self, func: Callable, r: ElementName) -> None: range_ref += self.suffix func(SH["node"], URIRef(range_ref)) + def _slot_iri(self, slot: SlotDefinition) -> str: + """The full IRI of *slot*, exactly as ``sh:path`` in the main slot loop renders it. + + An induced slot carries its ``slot_usage`` overrides, so an overridden + ``slot_uri`` yields the same IRI as ``sh:path``; otherwise the query + would use a property the data never uses and never fire. + """ + sv = self.schemaview + if slot.name in sv.element_by_schema_map(): + return sv.get_uri(slot, expand=True) + return sv.expand_curie(f"{sv.schema.default_prefix}:{underscore(slot.name)}") + + def _add_range(self, g: Graph, func: Callable, r: ElementName) -> None: + """Add the value-type constraint for range *r*: a class, type, enum or built-in datatype.""" + sv = self.schemaview + if r in sv.all_classes(): + cls_def = sv.get_class(r) + is_any = cls_def and getattr(cls_def, "class_uri", None) == "linkml:Any" + self._add_class(func, r) + if not is_any: + if sv.get_identifier_slot(r) is not None: + func(SH.nodeKind, SH.IRI) + else: + func(SH.nodeKind, SH.BlankNodeOrIRI) + elif r in sv.all_types(): + self._add_type(func, r) + elif r in sv.all_enums(): + self._add_enum(g, func, r) + else: + add_simple_data_type(func, r) + def _add_enum(self, g: Graph, func: Callable, r: ElementName) -> None: sv = self.schemaview enum = sv.get_enum(r) diff --git a/tests/linkml/test_compliance/test_boolean_slot_compliance.py b/tests/linkml/test_compliance/test_boolean_slot_compliance.py index fd900bf938..dc04f6f2f8 100644 --- a/tests/linkml/test_compliance/test_boolean_slot_compliance.py +++ b/tests/linkml/test_compliance/test_boolean_slot_compliance.py @@ -505,7 +505,7 @@ def test_class_any_of(framework, data_name, s1value, s2value, is_valid): core_elements=["any_of", "ClassDefinition"], ) expected_behavior = ValidationBehavior.IMPLEMENTS - if framework not in [OWL]: + if framework not in [OWL, SHACL]: # TODO: rdflib transformer has issues around ranges expected_behavior = ValidationBehavior.INCOMPLETE # TODO: rdflib transformer has issues around ranges @@ -632,8 +632,22 @@ def test_class_any_of_with_required(framework, nest, op, name, family_name, give core_elements=[op, "ClassDefinition"], ) expected_behavior = ValidationBehavior.IMPLEMENTS - if framework not in [JSON_SCHEMA]: + if framework not in [JSON_SCHEMA, SHACL]: expected_behavior = ValidationBehavior.INCOMPLETE + elif framework == SHACL and 5 in (name, family_name, given_name): + # SHACL validation makes its instances through python dataclasses, which coerce + # the integer to a string, so the range violation never reaches the shapes. A + # row can then only be detected through the operator itself. + present = [value is not None for value in (name, family_name, given_name)] + members = [present[0], present[1] and present[2]] + operator_holds = { + "any_of": any(members), + "all_of": all(members), + "exactly_one_of": sum(members) == 1, + "none_of": not any(members), + }[op] + if operator_holds: + expected_behavior = ValidationBehavior.INCOMPLETE data = {SLOT_S1: name, SLOT_S2: family_name, SLOT_S3: given_name} if nest: @@ -1435,8 +1449,8 @@ def test_class_boolean_with_expressions( """ if (s1_range == CLASS_ANY or s2_range == CLASS_ANY) and framework not in [PYDANTIC]: pytest.skip("Class Any not supported in this test") - if framework == SHACL: - pytest.skip("shaclgen does not support boolean expressions yet") + if framework == SHACL and CLASS_D in (s1_range, s2_range): + pytest.skip("the harness writes a reference as an untyped node, which the slot's sh:class rejects") if framework == PANDERA_POLARS_CLASS: pytest.skip("panderagen does not support boolean expressions yet") if s1_range.endswith("*"): diff --git a/tests/linkml/test_generators/test_jsonschemagen.py b/tests/linkml/test_generators/test_jsonschemagen.py index 6d8e0b32ee..1c0948243e 100644 --- a/tests/linkml/test_generators/test_jsonschemagen.py +++ b/tests/linkml/test_generators/test_jsonschemagen.py @@ -2241,3 +2241,246 @@ def test_inlined_dict_key_constraints_helper_drops_non_string_values(): assert generator.get_key_constraints_for_slot(SlotDefinition("key", equals_number=5)) == {} assert generator.get_key_constraints_for_slot(SlotDefinition("key", minimum_value=1)) == {} assert generator.get_key_constraints_for_slot(None) == {} + + +_CLASS_EXPRESSION_INHERITANCE_SCHEMA = """ +id: https://example.org/class-expressions +name: class_expressions +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/class-expressions/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + a: {} + b: {} +classes: + Marker: + mixin: true + none_of: + - slot_conditions: + a: + equals_string: forbidden + Parent: + slots: [a, b] + any_of: + - slot_conditions: + a: + required: true + - slot_conditions: + b: + required: true + Child: + is_a: Parent + mixins: [Marker] + GrandChild: + is_a: Child +""" + + +@pytest.mark.parametrize("target_class", ["Parent", "Child", "GrandChild"]) +@pytest.mark.parametrize( + "instance,valid_for_parent,valid_for_marker_users", + [ + pytest.param({"a": "1"}, True, True, id="a"), + pytest.param({"b": "2"}, True, True, id="b"), + pytest.param({}, False, False, id="neither"), + pytest.param({"a": "forbidden"}, True, False, id="forbidden-by-mixin"), + ], +) +def test_class_expressions_apply_to_subclasses(target_class, instance, valid_for_parent, valid_for_marker_users): + """A class expression constrains every instance of its class, so the + definitions of its subclasses, and of the classes using a mixin, carry it.""" + json_schema = json.loads( + JsonSchemaGenerator(_CLASS_EXPRESSION_INHERITANCE_SCHEMA, top_class=target_class).serialize() + ) + expected = valid_for_parent if target_class == "Parent" else valid_for_marker_users + assert jsonschema.Draft7Validator(json_schema).is_valid(instance) == expected + + +_TWO_ABSENT_RULE_SCHEMA = """ +id: https://example.org/absent +name: absent +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string +classes: + Thing: + attributes: + trigger: {} + a: {} + b: {} + rules: + - preconditions: + slot_conditions: + trigger: + value_presence: PRESENT + postconditions: + slot_conditions: + a: + value_presence: ABSENT + b: + value_presence: ABSENT +""" + + +@pytest.mark.parametrize( + "instance,valid", + [ + pytest.param({"trigger": "t"}, True, id="neither"), + pytest.param({"trigger": "t", "a": "x"}, False, id="one"), + pytest.param({"trigger": "t", "a": "x", "b": "y"}, False, id="both"), + pytest.param({"a": "x", "b": "y"}, True, id="not-triggered"), + ], +) +def test_several_absent_slots_each_must_be_absent(instance, valid): + """value_presence: ABSENT on several slots of one expression requires each + to be absent, not merely that they aren't all present.""" + json_schema = json.loads(JsonSchemaGenerator(_TWO_ABSENT_RULE_SCHEMA, top_class="Thing").serialize()) + assert jsonschema.Draft7Validator(json_schema).is_valid(instance) == valid + + +@pytest.mark.parametrize("tags,valid", [(["A"], True), (["A", "B"], False), ([], True)]) +def test_class_expression_condition_uses_the_induced_slot(tags, valid): + """A condition constrains the slot as induced for the class, so a + slot_usage that makes it multivalued applies the condition to every value.""" + schema = """ +id: https://example.org/induced +name: induced +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string +slots: + tags: {} +classes: + Thing: + slots: [tags] + slot_usage: + tags: + multivalued: true + all_of: + - slot_conditions: + tags: + equals_string: A +""" + json_schema = json.loads(JsonSchemaGenerator(schema, top_class="Thing").serialize()) + assert jsonschema.Draft7Validator(json_schema).is_valid({"tags": tags}) == valid + + +_DICT_INLINED_CONDITION_SCHEMA = """ +id: https://example.org/dict-inlined +name: dict_inlined +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string +classes: + Resource: + attributes: + id: {identifier: true} + code: {range: integer} + SpecialResource: + is_a: Resource + slot_usage: + code: {required: true} + Document: + attributes: + licenses: + range: Resource + multivalued: true + inlined: true + all_of: + - slot_conditions: + licenses: CONDITION +""" + + +@pytest.mark.parametrize( + "condition,licenses,valid", + [ + pytest.param({"minimum_cardinality": 1}, {"r1": {"code": 1}}, True, id="min-met"), + pytest.param({"minimum_cardinality": 1}, {}, False, id="min-empty-dict"), + pytest.param({"maximum_cardinality": 1}, {"r1": {}, "r2": {}}, False, id="max-exceeded"), + pytest.param({"range": "SpecialResource"}, {"r1": {"code": 1}}, True, id="range-met"), + pytest.param({"range": "SpecialResource"}, {"r1": {}}, False, id="range-violated"), + ], +) +def test_class_expression_condition_on_a_slot_inlined_as_dict(condition, licenses, valid): + """A condition on a slot inlined as a dict counts and constrains the values + of the dict, which is how the values of such a slot appear in JSON.""" + schema = _DICT_INLINED_CONDITION_SCHEMA.replace("CONDITION", json.dumps(condition)) + json_schema = json.loads(JsonSchemaGenerator(schema, top_class="Document").serialize()) + assert jsonschema.Draft7Validator(json_schema).is_valid({"licenses": licenses}) == valid + + +_SUBCLASS_UNION_SCHEMA = """ +id: https://example.org/union +name: union +prefixes: + linkml: https://w3id.org/linkml/ +imports: + - linkml:types +default_range: string +classes: + Agent: + attributes: + agent_id: {} + OPERATOR: + - is_a: Person + - is_a: Organization + Person: + is_a: Agent + attributes: + name: {required: true} + Student: + is_a: Person + attributes: + school: {} + Organization: + is_a: Agent + attributes: + id: {identifier: true} + org_name: {} +""" + + +@pytest.mark.parametrize("operator", ["any_of", "exactly_one_of"]) +@pytest.mark.parametrize( + "target_class,instance,valid", + [ + pytest.param("Person", {"name": "n"}, True, id="person"), + pytest.param("Student", {"name": "n", "school": "s"}, True, id="student"), + pytest.param("Organization", {"id": "o", "org_name": "o"}, True, id="organization"), + pytest.param("Agent", {"agent_id": "1"}, False, id="agent-of-neither-structure"), + ], +) +def test_class_expression_is_a_inherited_by_the_classes_it_names(operator, target_class, instance, valid): + """A class carries its parent's expression over its own subclasses, such as + ``any_of: [{is_a: Person}, {is_a: Organization}]``. An instance of Person satisfies + the first member and cannot satisfy the second, so its definition doesn't refer to + itself. An Agent is a Person in JSON when its structure matches Person's definition.""" + schema = _SUBCLASS_UNION_SCHEMA.replace("OPERATOR", operator) + json_schema = json.loads(JsonSchemaGenerator(schema, top_class=target_class).serialize()) + assert jsonschema.Draft7Validator(json_schema).is_valid(instance) == valid + + +@pytest.mark.parametrize( + "target_class,instance,valid", + [ + pytest.param("Person", {"name": "n"}, True, id="person-is-no-organization"), + pytest.param("Organization", {"id": "o"}, False, id="organization"), + pytest.param("Agent", {"agent_id": "1"}, True, id="agent-of-another-structure"), + ], +) +def test_class_expression_is_a_of_a_class_no_instance_belongs_to(target_class, instance, valid): + """``none_of: [{is_a: Organization}]`` holds for every Person, since no Person is an Organization.""" + schema = _SUBCLASS_UNION_SCHEMA.replace("OPERATOR", "none_of").replace(" - is_a: Person\n", "") + json_schema = json.loads(JsonSchemaGenerator(schema, top_class=target_class).serialize()) + assert jsonschema.Draft7Validator(json_schema).is_valid(instance) == valid diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index cf6764ad6d..020f756d71 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -1,9 +1,12 @@ +import json +import logging from collections import Counter from typing import Any import pytest import rdflib -from rdflib import RDF, RDFS, SH, Literal, URIRef +import yaml +from rdflib import RDF, RDFS, SH, XSD, Literal, URIRef from rdflib.collection import Collection from linkml.generators.shacl.shacl_data_type import ShaclDataType @@ -2972,3 +2975,1529 @@ def conforms(data_ttl: str) -> tuple[bool, str]: for value in ('"x1"', '"za"'): ok, _ = conforms(f"{prefixes}\nex:t2 a ex:SlotPatternWithAnyOf ; ex:tag {value} .") assert not ok, f"tag {value} violates the slot pattern or matches no branch and must be rejected" + + +# Class expressions → sh:or / sh:and / sh:xone / sh:not +# --------------------------------------------------------------------------- + +EX_CE = rdflib.Namespace("https://example.org/class-expressions/") + +_CLASS_EXPRESSION_HEADER = """ +id: https://example.org/class-expressions +name: class_expressions +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/class-expressions/ +imports: + - linkml:types +default_prefix: ex +default_range: string +""" + +_REFERENCE_SYSTEM_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + codeEPSG: + range: integer + coordinateSystemName: {{}} +classes: + ReferenceSystem: + class_uri: ex:ReferenceSystem + slots: [codeEPSG, coordinateSystemName] + {operator}: + - slot_conditions: + codeEPSG: + required: true + - slot_conditions: + coordinateSystemName: + required: true +""" +) + +_LOGICAL_PREDICATES = (SH["or"], SH["and"], SH.xone, SH["not"]) + + +def _list_members(g, shape, predicate): + """Members of the single SHACL list that *shape* has for *predicate*.""" + lists = list(g.objects(shape, predicate)) + assert len(lists) == 1, f"expected one {predicate} list on {shape}, got {len(lists)}" + return list(Collection(g, lists[0])) + + +def _condition(g, member, path): + """The property shape for *path* inside the member shape *member*.""" + shapes = [p for p in g.objects(member, SH.property) if (p, SH.path, path) in g] + assert len(shapes) == 1, f"expected one condition on {path}, got {len(shapes)}" + return shapes[0] + + +def _conforms(shacl_ttl: str, data_ttl: str) -> bool: + """Validate *data_ttl*; meta_shacl makes pyshacl fail on an ill-formed shapes graph.""" + import pyshacl + + conforms, _, _ = pyshacl.validate( + data_graph=data_ttl, + shacl_graph=shacl_ttl, + data_graph_format="turtle", + shacl_graph_format="turtle", + advanced=True, + meta_shacl=True, + ) + return conforms + + +@pytest.mark.parametrize( + "operator,predicate", + [("any_of", SH["or"]), ("all_of", SH["and"]), ("exactly_one_of", SH.xone)], +) +def test_class_expression_list_operator_generates_logical_constraint(operator, predicate): + """any_of, all_of and exactly_one_of become sh:or, sh:and and sh:xone over the member shapes.""" + g = _parse_shacl(_REFERENCE_SYSTEM_SCHEMA.format(operator=operator)) + + members = _list_members(g, EX_CE.ReferenceSystem, predicate) + assert len(members) == 2 + for member, path in zip(members, (EX_CE.codeEPSG, EX_CE.coordinateSystemName)): + condition = _condition(g, member, path) + assert (condition, SH.minCount, Literal(1)) in g + assert (member, SH.targetClass, None) not in g + others = set(_LOGICAL_PREDICATES) - {predicate} + assert not any((EX_CE.ReferenceSystem, p, None) in g for p in others) + + +def test_class_expression_none_of_generates_one_sh_not_per_member(): + """none_of becomes one sh:not per member; the negated constraints all apply (SHACL §2.1.1).""" + g = _parse_shacl(_REFERENCE_SYSTEM_SCHEMA.format(operator="none_of")) + + negated = list(g.objects(EX_CE.ReferenceSystem, SH["not"])) + assert len(negated) == 2 + paths = {path for member in negated for p in g.objects(member, SH.property) for path in g.objects(p, SH.path)} + assert paths == {EX_CE.codeEPSG, EX_CE.coordinateSystemName} + + +@pytest.mark.parametrize( + "operator,properties,expected", + [ + ("any_of", 'ex:codeEPSG 4326 ; ex:coordinateSystemName "WGS 84"', True), + ("any_of", "ex:codeEPSG 4326", True), + ("any_of", "", False), + ("exactly_one_of", "ex:codeEPSG 4326", True), + ("exactly_one_of", 'ex:codeEPSG 4326 ; ex:coordinateSystemName "WGS 84"', False), + ("exactly_one_of", "", False), + ("all_of", 'ex:codeEPSG 4326 ; ex:coordinateSystemName "WGS 84"', True), + ("all_of", "ex:codeEPSG 4326", False), + ("none_of", "", True), + ("none_of", "ex:codeEPSG 4326", False), + ], +) +def test_class_expression_pyshacl_end_to_end(operator, properties, expected): + """End-to-end: each operator admits exactly the instances the metamodel says it holds for.""" + shacl_ttl = ShaclGenerator(_REFERENCE_SYSTEM_SCHEMA.format(operator=operator), mergeimports=False).serialize() + data = f""" + @prefix ex: . + ex:rs a ex:ReferenceSystem {";" if properties else ""} {properties} . + """ + assert _conforms(shacl_ttl, data) is expected + + +_FORMAT_PROFILES_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + fileFormat: {} + formatType: {} + version: {} + hasChannel: + range: Channel + multivalued: true + inlined: true +classes: + Channel: + class_uri: ex:Channel + Format: + class_uri: ex:Format + slots: [fileFormat, formatType, version, hasChannel] + any_of: + - description: A single-channel trace. + slot_conditions: + fileFormat: + equals_string_in: [OSI, TXTH] + formatType: + required: true + version: + required: true + - description: A multi-channel container. + slot_conditions: + fileFormat: + required: true + equals_string: MCAP + hasChannel: + required: true +""" +) + + +@pytest.mark.parametrize( + "properties,expected", + [ + ('ex:fileFormat "OSI" ; ex:formatType "SensorView" ; ex:version "3.7.0"', True), + ('ex:fileFormat "MCAP" ; ex:hasChannel ex:ch', True), + ('ex:fileFormat "MCAP" ; ex:formatType "SensorView" ; ex:version "3.7.0"', False), + ('ex:fileFormat "OSI" ; ex:formatType "SensorView" ; ex:hasChannel ex:ch', False), + ], +) +def test_class_expression_any_of_profiles_pyshacl_end_to_end(properties, expected): + """End-to-end: a class-level any_of selects between two complete profiles of a class.""" + shacl_ttl = ShaclGenerator(_FORMAT_PROFILES_SCHEMA, mergeimports=False).serialize() + data = f""" + @prefix ex: . + ex:ch a ex:Channel . + ex:f a ex:Format ; {properties} . + """ + assert _conforms(shacl_ttl, data) is expected + + +def test_class_expression_member_metadata(): + """A member's title and description annotate its shape, as they do for a class's NodeShape.""" + g = _parse_shacl(_FORMAT_PROFILES_SCHEMA) + + comments = {str(c) for m in _list_members(g, EX_CE.Format, SH["or"]) for c in g.objects(m, RDFS.comment)} + assert comments == {"A single-channel trace.", "A multi-channel container."} + + +_CONDITIONS_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +enums: + ColourEnum: + permissible_values: + red: {} + blue: {} +slots: + label: {} + size: + range: integer + count: + multivalued: true + exact: + multivalued: true + colour: {} + target: {} + note: {} +classes: + Target: + class_uri: ex:Target + Thing: + class_uri: ex:Thing + slots: [label, size, count, exact, colour, target, note] + all_of: + - title: every condition + slot_conditions: + label: + description: Starts upper case. + required: true + pattern: "^[A-Z]" + equals_string_in: [Alpha, Beta] + size: + minimum_value: 1 + maximum_value: 10 + equals_number: 5 + count: + required: true + minimum_cardinality: 2 + maximum_cardinality: 4 + exact: + exact_cardinality: 3 + colour: + range: ColourEnum + target: + value_presence: PRESENT + range: Target + note: + value_presence: ABSENT +""" +) + + +def test_class_expression_slot_condition_fields(): + """Each supported slot-condition field maps to the SHACL constraint the slot loop uses for it.""" + g = _parse_shacl(_CONDITIONS_SCHEMA) + (member,) = _list_members(g, EX_CE.Thing, SH["and"]) + assert (member, RDFS.label, Literal("every condition")) in g + + def values(path, predicate): + return set(g.objects(_condition(g, member, path), predicate)) + + def in_list(path): + (node,) = values(path, SH["in"]) + return list(Collection(g, node)) + + assert values(EX_CE.label, SH.minCount) == {Literal(1)} + assert values(EX_CE.label, SH.pattern) == {Literal("^[A-Z]")} + assert values(EX_CE.label, SH.description) == {Literal("Starts upper case.")} + assert in_list(EX_CE.label) == [Literal("Alpha"), Literal("Beta")] + + # equals_number is a value comparison; SHACL allows one sh:minInclusive and one + # sh:maxInclusive per shape, so next to the bounds it moves into an sh:and member + assert values(EX_CE.size, SH.minInclusive) == {Literal(1)} + assert values(EX_CE.size, SH.maxInclusive) == {Literal(10)} + (and_node,) = values(EX_CE.size, SH["and"]) + repeated = {(p, o) for m in Collection(g, and_node) for p, o in g.predicate_objects(m)} + assert repeated == {(SH.minInclusive, Literal(5)), (SH.maxInclusive, Literal(5))} + assert values(EX_CE.size, SH["in"]) == set() + assert values(EX_CE.size, SH.minCount) == set() + assert values(EX_CE.size, SH.hasValue) == set() + + # required and a cardinality give one sh:minCount, the stricter of the two + assert values(EX_CE["count"], SH.minCount) == {Literal(2)} + assert values(EX_CE["count"], SH.maxCount) == {Literal(4)} + assert values(EX_CE.exact, SH.minCount) == {Literal(3)} + assert values(EX_CE.exact, SH.maxCount) == {Literal(3)} + + assert in_list(EX_CE.colour) == [Literal("red"), Literal("blue")] + + assert values(EX_CE.target, SH.minCount) == {Literal(1)} + assert values(EX_CE.target, SH["class"]) == {EX_CE.Target} + assert values(EX_CE.target, SH.nodeKind) == {SH.BlankNodeOrIRI} + + assert values(EX_CE.note, SH.maxCount) == {Literal(0)} + assert values(EX_CE.note, SH.minCount) == set() + + +@pytest.mark.parametrize( + "condition,properties,expected", + [ + # outside none_of a value constraint says nothing about presence + ("any_of", "", True), + ("any_of", 'ex:label "b"', False), + # inside none_of it requires the slot, so an absent slot is not rejected + ("none_of", "", True), + ("none_of", 'ex:label "A"', False), + ("none_of", 'ex:label "B"', True), + ], +) +def test_class_expression_presence_semantics(condition, properties, expected): + """A condition holds vacuously for an absent slot, except under none_of, as in the JSON Schema generator.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + f""" +slots: + label: {{}} +classes: + Thing: + class_uri: ex:Thing + slots: [label] + {condition}: + - slot_conditions: + label: + {"pattern: '^[A-Z]'" if condition == "any_of" else "equals_string: A"} +""" + ) + shacl_ttl = ShaclGenerator(schema, mergeimports=False).serialize() + data = f""" + @prefix ex: . + ex:t a ex:Thing {";" if properties else ""} {properties} . + """ + assert _conforms(shacl_ttl, data) is expected + + +def test_class_expression_nested_and_is_a(): + """Nested expressions recurse into the member shape; is_a gives sh:class.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + a: {} + b: {} + c: {} +classes: + Marker: + class_uri: ex:Marker + mixin: true + Thing: + class_uri: ex:Thing + slots: [a, b, c] + any_of: + - all_of: + - slot_conditions: + a: + required: true + - slot_conditions: + b: + required: true + - is_a: Marker + slot_conditions: + c: + required: true + MarkedThing: + is_a: Thing + mixins: [Marker] +""" + ) + # a Thing may be a Marker, since MarkedThing is one, so Thing's shape checks the type; + # open shapes: the instance is also a Marker, whose own shape declares no slots + shacl_ttl = ShaclGenerator(schema, mergeimports=False, closed=False).serialize() + g = rdflib.Graph().parse(data=shacl_ttl) + first, second = _list_members(g, EX_CE.Thing, SH["or"]) + assert len(_list_members(g, first, SH["and"])) == 2 + assert (second, SH["class"], EX_CE.Marker) in g + + prefix = "@prefix ex: ." + assert _conforms(shacl_ttl, f'{prefix} ex:t a ex:Thing ; ex:a "1" ; ex:b "2" .') + assert not _conforms(shacl_ttl, f'{prefix} ex:t a ex:Thing ; ex:a "1" .') + assert _conforms(shacl_ttl, f'{prefix} ex:t a ex:Thing, ex:Marker ; ex:c "3" .') + assert not _conforms(shacl_ttl, f'{prefix} ex:t a ex:Thing ; ex:c "3" .') + + +_SUBCLASS_UNION_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +classes: + Agent: + attributes: + agent_id: {} + any_of: + - is_a: Person + - is_a: Organization + Person: + is_a: Agent + attributes: + name: {} + Student: + is_a: Person + attributes: + school: {} + Organization: + is_a: Agent + attributes: + org_name: {} +""" +) + + +def test_class_expression_is_a_inherited_by_the_classes_it_names(): + """A class carries its parent's ``any_of: [{is_a: Person}, {is_a: Organization}]``. + Every Person is a Person and none is an Organization, so Person's shape needs no + ``sh:class`` for the first member and nothing conforms to the second; only Agent's + shape, whose instances may be either, checks the node's type.""" + shacl_ttl = ShaclGenerator(_SUBCLASS_UNION_SCHEMA, mergeimports=False, closed=False).serialize() + g = rdflib.Graph().parse(data=shacl_ttl) + person_member, organization_member = _list_members(g, EX_CE.Person, SH["or"]) + assert (person_member, SH["class"], None) not in g + assert (organization_member, SH["not"], None) in g + agent_members = _list_members(g, EX_CE.Agent, SH["or"]) + assert {g.value(m, SH["class"]) for m in agent_members} == {EX_CE.Person, EX_CE.Organization} + + prefix = "@prefix ex: ." + assert _conforms(shacl_ttl, f'{prefix} ex:p a ex:Person ; ex:name "n" .') + # a Student is a Person without carrying the type: the shape doesn't need the subclass triple + assert _conforms(shacl_ttl, f'{prefix} ex:s a ex:Student ; ex:school "x" .') + assert _conforms(shacl_ttl, f'{prefix} ex:o a ex:Organization ; ex:org_name "o" .') + assert not _conforms(shacl_ttl, f'{prefix} ex:a a ex:Agent ; ex:agent_id "1" .') + assert _conforms(shacl_ttl, f'{prefix} ex:a a ex:Agent, ex:Person ; ex:agent_id "1" .') + + +def test_class_expression_is_a_of_a_class_no_instance_belongs_to(): + """``none_of: [{is_a: Organization}]`` holds for every Person, since no Person is an Organization.""" + schema = _SUBCLASS_UNION_SCHEMA.replace("any_of:\n - is_a: Person\n", "none_of:\n") + shacl_ttl = ShaclGenerator(schema, mergeimports=False, closed=False).serialize() + prefix = "@prefix ex: ." + assert _conforms(shacl_ttl, f'{prefix} ex:p a ex:Person ; ex:name "n" .') + assert not _conforms(shacl_ttl, f'{prefix} ex:o a ex:Organization ; ex:org_name "o" .') + assert _conforms(shacl_ttl, f'{prefix} ex:a a ex:Agent ; ex:agent_id "1" .') + + +_INHERITED_CLASS_EXPRESSION_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + a: {} + b: {} +classes: + Marker: + class_uri: ex:Marker + mixin: true + none_of: + - slot_conditions: + a: + equals_string: forbidden + Parent: + class_uri: ex:Parent + slots: [a, b] + any_of: + - slot_conditions: + a: + required: true + - slot_conditions: + b: + required: true + Child: + class_uri: ex:Child + is_a: Parent + mixins: [Marker] + GrandChild: + class_uri: ex:GrandChild + is_a: Child +""" +) +_CE_PREFIXES = ( + "@prefix ex: .\n@prefix rdfs: .\n" +) + + +@pytest.mark.parametrize("subclass_axioms", [False, True], ids=["typed-only", "with-rdfs-subClassOf"]) +@pytest.mark.parametrize("class_name", ["Parent", "Child", "GrandChild"]) +@pytest.mark.parametrize( + "values,conforms", + [ + pytest.param(' ; ex:a "1"', True, id="a"), + pytest.param(' ; ex:b "2"', True, id="b"), + pytest.param("", False, id="neither"), + ], +) +def test_class_expression_applies_to_subclass_instances(subclass_axioms, class_name, values, conforms): + """A class expression constrains every instance of its class, so the shapes + of its subclasses carry it too. A node typed only with a subclass, as + rdflib_dumper and the JSON-LD context produce it, is checked without an + rdfs:subClassOf triple in the data; with one, the verdict is the same.""" + shacl_ttl = ShaclGenerator(_INHERITED_CLASS_EXPRESSION_SCHEMA, mergeimports=False, closed=False).serialize() + axioms = ( + "ex:Child rdfs:subClassOf ex:Parent . ex:GrandChild rdfs:subClassOf ex:Child .\n" if subclass_axioms else "" + ) + assert _conforms(shacl_ttl, f"{_CE_PREFIXES}{axioms}ex:x a ex:{class_name}{values} .") == conforms + + +@pytest.mark.parametrize("class_name,conforms", [("Parent", True), ("Child", False), ("GrandChild", False)]) +def test_class_expression_of_a_mixin_applies_to_the_classes_using_it(class_name, conforms): + """A mixin's class expression constrains the classes that use the mixin and + their subclasses, and no other class.""" + shacl_ttl = ShaclGenerator(_INHERITED_CLASS_EXPRESSION_SCHEMA, mergeimports=False, closed=False).serialize() + assert _conforms(shacl_ttl, f'{_CE_PREFIXES}ex:x a ex:{class_name} ; ex:a "forbidden" .') == conforms + + +def test_inherited_class_expression_follows_the_subclass_slot_usage(): + """An inherited class expression is translated in the subclass's context, so + its conditions use the subclass's slot_usage, as the subclass's own + property shapes do.""" + schema = yaml.safe_load(_INHERITED_CLASS_EXPRESSION_SCHEMA) + schema["classes"]["Child"]["slot_usage"] = {"a": {"slot_uri": "ex:childA"}} + g = rdflib.Graph().parse(data=ShaclGenerator(json.dumps(schema), mergeimports=False).serialize()) + for shape, path in ((EX_CE.Parent, EX_CE.a), (EX_CE.Child, EX_CE.childA), (EX_CE.GrandChild, EX_CE.childA)): + (or_list,) = g.objects(shape, SH["or"]) + paths = { + p + for member in Collection(g, or_list) + for prop in g.objects(member, SH.property) + for p in g.objects(prop, SH.path) + } + assert paths == {path, EX_CE.b}, (shape, paths) + + +def test_classes_sharing_a_class_uri_carry_an_inherited_expression_once(): + """Classes with the same class_uri share one shape, which carries an + expression they inherit once.""" + schema = yaml.safe_load(_INHERITED_CLASS_EXPRESSION_SCHEMA) + schema["classes"]["Child"]["class_uri"] = "ex:Shared" + schema["classes"]["GrandChild"]["class_uri"] = "ex:Shared" + g = rdflib.Graph().parse(data=ShaclGenerator(json.dumps(schema), mergeimports=False).serialize()) + assert len(list(g.objects(EX_CE.Shared, SH["or"]))) == 1 + assert len(list(g.objects(EX_CE.Shared, SH["not"]))) == 1 + + +@pytest.mark.parametrize( + "child_usage,expected_shapes,warning", + [ + pytest.param( + None, + {EX_CE.Parent: False, EX_CE.Child: False, EX_CE.GrandChild: False}, + "Class 'Parent': any_of is not translated to SHACL, because it uses 'has_member' in the condition on " + "slot 'a' (in the shapes of 'Parent', 'Child', 'GrandChild').", + id="untranslatable-everywhere", + ), + pytest.param( + {"a": {"range": "integer"}}, + {EX_CE.Parent: True, EX_CE.Child: False, EX_CE.GrandChild: False}, + "Class 'Parent': any_of is not translated to SHACL, because it uses equals_string on slot 'a', whose " + "range 'integer' does not hold strings (in the shapes of 'Child', 'GrandChild').", + id="untranslatable-in-subclasses", + ), + ], +) +def test_untranslatable_inherited_class_expression_warned_once(caplog, child_usage, expected_shapes, warning): + """An inherited expression that cannot be translated is skipped in the + shapes where it cannot, and reported once, naming them.""" + schema = yaml.safe_load(_INHERITED_CLASS_EXPRESSION_SCHEMA) + schema["classes"]["Parent"]["any_of"] = ( + [{"slot_conditions": {"a": {"has_member": {"equals_string": "x"}}}}] + if child_usage is None + else [{"slot_conditions": {"a": {"equals_string": "x"}}}] + ) + if child_usage is not None: + schema["classes"]["Child"]["slot_usage"] = child_usage + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + g = rdflib.Graph().parse(data=ShaclGenerator(json.dumps(schema), mergeimports=False).serialize()) + for shape, translated in expected_shapes.items(): + assert ((shape, SH["or"], None) in g) == translated, shape + messages = [rec.message for rec in caplog.records if "any_of is not translated" in rec.message] + assert messages == [warning] + + +def test_class_expression_untranslatable_operator_skipped_with_warning(caplog): + """An operator with an untranslatable member is skipped whole, with a warning; the others are kept.""" + import logging + + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + tags: + multivalued: true + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [tags, a] + any_of: + - slot_conditions: + tags: + has_member: + equals_string: x + - slot_conditions: + a: + required: true + none_of: + - slot_conditions: + a: + equals_string: forbidden +""" + ) + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + g = _parse_shacl(schema) + + assert (EX_CE.Thing, SH["or"], None) not in g + assert any( + "any_of" in rec.message and "has_member" in rec.message and "tags" in rec.message for rec in caplog.records + ) + # the translatable none_of is kept, and enforced + shacl_ttl = g.serialize(format="turtle") + assert not _conforms(shacl_ttl, f'{_CE_PREFIXES}ex:t a ex:Thing ; ex:a "forbidden" .') + assert _conforms(shacl_ttl, f'{_CE_PREFIXES}ex:t a ex:Thing ; ex:a "ok" .') + + +def test_class_expression_condition_on_identifier_skipped_with_warning(caplog): + """An identifier is the node's IRI, not a property arc, so a condition on it is not translated.""" + import logging + + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + id: + identifier: true + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [id, a] + exactly_one_of: + - slot_conditions: + id: + pattern: "^ex:" + - slot_conditions: + a: + required: true +""" + ) + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + g = _parse_shacl(schema) + + assert (EX_CE.Thing, SH.xone, None) not in g + assert any("exactly_one_of" in rec.message and "identifier" in rec.message for rec in caplog.records) + + +def test_class_expression_absent_leaves_node_shapes_unchanged(): + """Without class-level expressions no node shape gets a logical constraint; slot-level any_of is unaffected.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + value: + any_of: + - range: integer + - range: string +classes: + Thing: + class_uri: ex:Thing + slots: [value] +""" + ) + g = _parse_shacl(schema) + + for shape in g.subjects(SH.targetClass, None): + assert not any((shape, p, None) in g for p in _LOGICAL_PREDICATES) + (value_shape,) = [p for p in g.objects(EX_CE.Thing, SH.property) if (p, SH.path, EX_CE.value) in g] + assert (value_shape, SH["or"], None) in g + + +def test_class_expression_condition_path_is_the_slot_induced_for_the_class(): + """A condition's sh:path is the one the class's own property shape uses, slot_usage and attributes included.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + code: + slot_uri: ex:baseCode + exact mappings: + slot_uri: ex:exactMatch +classes: + Thing: + class_uri: ex:Thing + slots: [code, exact mappings] + slot_usage: + code: + slot_uri: ex:thingCode + attributes: + loc: + slot_uri: ex:thingLoc + any_of: + - slot_conditions: + code: + required: true + - slot_conditions: + exact_mappings: + required: true + - slot_conditions: + loc: + required: true + Other: + class_uri: ex:Other + attributes: + loc: + slot_uri: ex:otherLoc +""" + ) + g = _parse_shacl(schema) + + class_paths = {path for p in g.objects(EX_CE.Thing, SH.property) for path in g.objects(p, SH.path)} + condition_paths = [ + path + for member in _list_members(g, EX_CE.Thing, SH["or"]) + for p in g.objects(member, SH.property) + for path in g.objects(p, SH.path) + ] + assert condition_paths and set(condition_paths) <= class_paths + assert set(condition_paths) == {EX_CE.thingCode, EX_CE.exactMatch, EX_CE.thingLoc} + + +_PRESENCE_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +enums: + Color: + permissible_values: + red: {} + green: {} +slots: + label: {} + note: {} + tags: + multivalued: true +classes: + Thing: + class_uri: ex:Thing + tree_root: true + slots: [label, note, tags] +""" +) + + +def _class_expression_verdicts(expression: dict, obj: dict) -> tuple[bool, bool]: + """Whether the JSON Schema and the SHACL shapes generated for ``Thing`` with + the class-level *expression* accept *obj*. For SHACL, *obj* is loaded + through the generated JSON-LD context.""" + import jsonschema + import pyshacl + + from linkml.generators.jsonldcontextgen import ContextGenerator + from linkml.generators.jsonschemagen import JsonSchemaGenerator + + schema = yaml.safe_load(_PRESENCE_SCHEMA) + schema["classes"]["Thing"].update(expression) + schema = yaml.safe_dump(schema) + json_schema = json.loads(JsonSchemaGenerator(schema, top_class="Thing").serialize()) + context = json.loads(ContextGenerator(schema).serialize())["@context"] + data = rdflib.Graph().parse(data=json.dumps({"@context": context, "@type": "Thing", **obj}), format="json-ld") + shapes = rdflib.Graph().parse(data=ShaclGenerator(schema, mergeimports=False, closed=False).serialize()) + conforms, _, _ = pyshacl.validate(data, shacl_graph=shapes, meta_shacl=True) + return jsonschema.Draft7Validator(json_schema).is_valid(obj), conforms + + +def _is_a(value: str) -> dict: + """A class expression requiring ``label`` to equal *value*.""" + return {"slot_conditions": {"label": {"equals_string": value}}} + + +def _tags(**condition) -> dict: + """A class expression with *condition* on the multivalued ``tags``.""" + return {"slot_conditions": {"tags": condition}} + + +_LABEL_A_OR_NOTE_B = [_is_a("A"), {"slot_conditions": {"note": {"equals_string": "B"}}}] + + +@pytest.mark.parametrize( + "expression,obj,valid", + [ + # a condition is unknown for an absent slot, and only a definitely false expression is a violation + pytest.param({"any_of": [_is_a("A")]}, {}, True, id="any_of-absent"), + pytest.param({"any_of": [_is_a("A")]}, {"label": "Z"}, False, id="any_of-other"), + pytest.param({"any_of": _LABEL_A_OR_NOTE_B}, {"label": "Z"}, True, id="any_of-one-false-one-unknown"), + pytest.param({"none_of": [_is_a("A")]}, {}, True, id="none_of-absent"), + pytest.param({"none_of": [_is_a("A")]}, {"label": "A"}, False, id="none_of-match"), + pytest.param({"none_of": [_is_a("A")]}, {"label": "B"}, True, id="none_of-other"), + pytest.param({"none_of": [{"any_of": [_is_a("A"), _is_a("B")]}]}, {}, True, id="nested-in-none_of-absent"), + pytest.param( + {"none_of": [{"any_of": [_is_a("A"), _is_a("B")]}]}, {"label": "B"}, False, id="nested-in-none_of" + ), + pytest.param({"none_of": [{"any_of": [_is_a("A"), _is_a("B")]}]}, {"label": "C"}, True, id="nested-other"), + # the reading composes: wrapping in a one-member all_of, or negating twice, changes nothing + pytest.param({"all_of": [{"none_of": [_is_a("A")]}]}, {}, True, id="none_of-in-all_of-absent"), + pytest.param({"all_of": [{"none_of": [_is_a("A")]}]}, {"label": "A"}, False, id="none_of-in-all_of-match"), + pytest.param({"none_of": [{"none_of": [_is_a("A")]}]}, {}, True, id="double-negation-absent"), + pytest.param({"none_of": [{"none_of": [_is_a("A")]}]}, {"label": "A"}, True, id="double-negation-match"), + pytest.param({"none_of": [{"none_of": [_is_a("A")]}]}, {"label": "B"}, False, id="double-negation-other"), + # stated presence is definite; required: false only restates the default + pytest.param( + {"none_of": [{"slot_conditions": {"label": {"required": False, "equals_string": "A"}}}]}, + {}, + True, + id="restated-default", + ), + pytest.param( + {"none_of": [{"slot_conditions": {"label": {"value_presence": "ABSENT", "equals_string": "A"}}}]}, + {}, + False, + id="stated-absent", + ), + pytest.param( + {"none_of": [{"slot_conditions": {"label": {"value_presence": "ABSENT", "equals_string": "A"}}}]}, + {"label": "A"}, + True, + id="stated-absent-present", + ), + pytest.param( + {"all_of": [{"slot_conditions": {"note": {"required": True, "value_presence": "ABSENT"}}}]}, + {}, + True, + id="value_presence-over-required", + ), + pytest.param( + {"all_of": [{"slot_conditions": {"note": {"required": True, "value_presence": "ABSENT"}}}]}, + {"note": "x"}, + False, + id="value_presence-over-required-present", + ), + # an empty condition is unknown for an absent slot, and definitely true for a present one + pytest.param({"none_of": [{"slot_conditions": {"label": {}}}]}, {}, True, id="empty-condition-absent"), + pytest.param({"none_of": [{"slot_conditions": {"label": {}}}]}, {"label": "x"}, False, id="empty-condition"), + # exactly_one_of holds when exactly one member is definitely true + pytest.param({"exactly_one_of": _LABEL_A_OR_NOTE_B}, {}, False, id="exactly_one_of-none-known"), + pytest.param({"exactly_one_of": _LABEL_A_OR_NOTE_B}, {"label": "A"}, True, id="exactly_one_of-first"), + pytest.param({"exactly_one_of": _LABEL_A_OR_NOTE_B}, {"label": "Z"}, False, id="exactly_one_of-none-true"), + pytest.param( + {"exactly_one_of": _LABEL_A_OR_NOTE_B}, {"label": "A", "note": "X"}, True, id="exactly_one_of-one-of-two" + ), + pytest.param( + {"exactly_one_of": _LABEL_A_OR_NOTE_B}, {"label": "A", "note": "B"}, False, id="exactly_one_of-both" + ), + # cardinalities count the values of a list, of which an absent slot has none + pytest.param({"all_of": [_tags(minimum_cardinality=2)]}, {"tags": ["a"]}, False, id="minimum-cardinality"), + pytest.param({"all_of": [_tags(minimum_cardinality=2)]}, {"tags": ["a", "b"]}, True, id="minimum-met"), + pytest.param({"all_of": [_tags(minimum_cardinality=2)]}, {}, False, id="minimum-absent"), + pytest.param({"all_of": [_tags(maximum_cardinality=1)]}, {"tags": ["a", "b"]}, False, id="maximum-cardinality"), + pytest.param({"none_of": [_tags(maximum_cardinality=1)]}, {}, True, id="none_of-maximum-absent"), + pytest.param({"none_of": [_tags(maximum_cardinality=1)]}, {"tags": ["a"]}, False, id="none_of-maximum"), + pytest.param( + {"none_of": [_tags(maximum_cardinality=1)]}, {"tags": ["a", "b"]}, True, id="none_of-maximum-over" + ), + pytest.param({"any_of": [_tags(maximum_cardinality=0), _is_a("A")]}, {}, True, id="maximum-zero-absent"), + # a single-valued slot holds at most one value + pytest.param( + {"all_of": [{"slot_conditions": {"label": {"minimum_cardinality": 2}}}]}, + {"label": "x"}, + False, + id="single-valued-minimum-two", + ), + # a range in a condition narrows the slot's values + pytest.param( + {"all_of": [{"slot_conditions": {"label": {"range": "Color"}}}]}, {"label": "red"}, True, id="range" + ), + pytest.param( + {"all_of": [{"slot_conditions": {"label": {"range": "Color"}}}]}, {"label": "blue"}, False, id="range-other" + ), + pytest.param({"all_of": [{"slot_conditions": {"label": {"range": "Color"}}}]}, {}, True, id="range-absent"), + pytest.param( + {"none_of": [{"slot_conditions": {"label": {"range": "Color"}}}]}, + {"label": "red"}, + False, + id="none_of-range", + ), + ], +) +def test_class_expression_absent_slot_semantics(expression, obj, valid): + """The specification doesn't say what a slot condition means for an absent + slot. As with an SQL CHECK constraint, an expression is violated only when + it is definitely false: a condition that doesn't state presence is unknown + for an absent slot. exactly_one_of holds when exactly one member is + definitely true. The JSON Schema and SHACL generators decide alike.""" + assert _class_expression_verdicts(expression, obj) == (valid, valid) + + +@pytest.mark.parametrize( + "condition,tags,conforms", + [ + # a bound an absent slot satisfies and a present one may too leaves absence open + pytest.param({"maximum_cardinality": 1}, [], True, id="at-most-one-absent"), + pytest.param({"maximum_cardinality": 1}, ["a"], False, id="at-most-one"), + pytest.param({"maximum_cardinality": 1}, ["a", "b"], True, id="more-than-one"), + # a maximum of 0 decides that the slot is absent + pytest.param({"maximum_cardinality": 0, "pattern": "^A"}, [], False, id="at-most-zero-absent"), + pytest.param({"maximum_cardinality": 0, "pattern": "^A"}, ["A"], True, id="at-most-zero-present"), + ], +) +def test_class_expression_none_of_cardinality(condition, tags, conforms): + """Within none_of, a condition is definitely true or false for an absent + slot only when it decides presence, such as a maximum cardinality of 0; + otherwise it requires the slot in its "definitely true" form.""" + schema = yaml.safe_load(_PRESENCE_SCHEMA) + schema["classes"]["Thing"]["none_of"] = [{"slot_conditions": {"tags": condition}}] + shacl_ttl = ShaclGenerator(yaml.safe_dump(schema), mergeimports=False, closed=False).serialize() + values = "" if not tags else " ; ex:tags " + ", ".join(f'"{t}"' for t in tags) + assert _conforms(shacl_ttl, f"{_CE_PREFIXES}ex:x a ex:Thing{values} .") == conforms + + +@pytest.mark.parametrize( + "condition,properties,expected", + [ + # a cardinality is taken literally inside none_of: "not at most zero values" means present + ("maximum_cardinality: 0", "", False), + ("maximum_cardinality: 0", 'ex:tag "x"', True), + ("maximum_cardinality: 1", 'ex:tag "x"', False), + ("maximum_cardinality: 1", 'ex:tag "x", "y"', True), + ], +) +def test_class_expression_none_of_takes_cardinality_literally(condition, properties, expected): + """Inside none_of only a condition silent on presence and cardinality is made to require the slot.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + f""" +slots: + tag: + multivalued: true +classes: + Thing: + class_uri: ex:Thing + slots: [tag] + none_of: + - slot_conditions: + tag: + {condition} +""" + ) + shacl_ttl = ShaclGenerator(schema, mergeimports=False).serialize() + data = f""" + @prefix ex: . + ex:t a ex:Thing {";" if properties else ""} {properties} . + """ + assert _conforms(shacl_ttl, data) is expected + + +def test_class_expression_equals_string_on_enum_uses_the_meaning(): + """equals_string on an enum slot compares against the permissible value as _add_enum renders it.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +enums: + FormatEnum: + permissible_values: + OSI: + meaning: ex:OSI + MCAP: {} +slots: + fileFormat: + range: FormatEnum + hasChannel: {} +classes: + Format: + class_uri: ex:Format + slots: [fileFormat, hasChannel] + any_of: + - slot_conditions: + fileFormat: + equals_string: OSI + - slot_conditions: + fileFormat: + equals_string: MCAP + hasChannel: + required: true +""" + ) + shacl_ttl = ShaclGenerator(schema, mergeimports=False).serialize() + g = rdflib.Graph().parse(data=shacl_ttl) + first, second = _list_members(g, EX_CE.Format, SH["or"]) + (osi_in,) = g.objects(_condition(g, first, EX_CE.fileFormat), SH["in"]) + assert list(Collection(g, osi_in)) == [EX_CE.OSI] + (mcap_in,) = g.objects(_condition(g, second, EX_CE.fileFormat), SH["in"]) + assert list(Collection(g, mcap_in)) == [Literal("MCAP")] + + prefix = "@prefix ex: ." + assert _conforms(shacl_ttl, f"{prefix} ex:f a ex:Format ; ex:fileFormat ex:OSI .") + assert not _conforms(shacl_ttl, f'{prefix} ex:f a ex:Format ; ex:fileFormat "MCAP" .') + + +@pytest.mark.parametrize("value,expected", [("5.0", True), ("5", True), ("6.0", False)]) +def test_class_expression_equals_number_compares_values(value, expected): + """equals_number matches the value, whatever numeric datatype carries it.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + size: + range: float + label: {} +classes: + Thing: + class_uri: ex:Thing + slots: [size, label] + any_of: + - slot_conditions: + size: + equals_number: 5 + - slot_conditions: + label: + required: true +""" + ) + shacl_ttl = ShaclGenerator(schema, mergeimports=False, closed=False).serialize() + # sh:datatype xsd:float on the class's own property shape would reject an xsd:decimal, + # so every value is given as xsd:float + data = f""" + @prefix ex: . + @prefix xsd: . + ex:t a ex:Thing ; ex:size "{value}"^^xsd:float . + """ + assert _conforms(shacl_ttl, data) is expected + + +_UNTRANSLATABLE_CONDITIONS = { + # an identifier is the node's IRI; here it becomes one only through slot_usage + "identifier": """ +slots: + id: {} + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [id, a] + slot_usage: + id: + identifier: true + any_of: + - slot_conditions: + id: + required: true + - slot_conditions: + a: + required: true +""", + "'undefined', which is not a slot": """ +slots: + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [a] + any_of: + - slot_conditions: + undefined: + required: true + - slot_conditions: + a: + required: true +""", + "does not hold strings": """ +slots: + count: + range: integer + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [count, a] + any_of: + - slot_conditions: + count: + equals_string: "5" + - slot_conditions: + a: + required: true +""", +} + + +@pytest.mark.parametrize("reason", list(_UNTRANSLATABLE_CONDITIONS)) +def test_class_expression_untranslatable_condition_skipped_with_warning(caplog, reason): + """A condition on an identifier, on no slot, or equals_string on a non-string range is not translated.""" + import logging + + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + g = _parse_shacl(_CLASS_EXPRESSION_HEADER + _UNTRANSLATABLE_CONDITIONS[reason]) + + assert (EX_CE.Thing, SH["or"], None) not in g + assert any("any_of" in rec.message and reason in rec.message for rec in caplog.records) + + +def test_class_expression_is_a_with_native_names_uses_sh_node(): + """With native names, is_a references the member class's shape, suffix included, as a range does.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + a: {} +classes: + Marker: + class_uri: ex:Marker + mixin: true + Thing: + class_uri: ex:Thing + slots: [a] + any_of: + - is_a: Marker + - slot_conditions: + a: + required: true + MarkedThing: + is_a: Thing + mixins: [Marker] +""" + ) + g = _parse_shacl(schema, use_class_uri_names=False, suffix="Shape") + + (first, _) = _list_members(g, EX_CE.ThingShape, SH["or"]) + assert set(g.objects(first, SH.node)) == {EX_CE.MarkerShape} + assert (first, SH["class"], None) not in g + assert (EX_CE.MarkerShape, RDF.type, SH.NodeShape) in g + + +def _condition_property_shapes(g: rdflib.Graph, shape: URIRef, path: URIRef) -> list: + """The property shapes on *path* that the logical constraints of *shape* reach.""" + own = set(g.objects(shape, SH.property)) + return [node for node in g.transitive_objects(shape, None) if (node, SH.path, path) in g and node not in own] + + +def _class_expression_shacl(classes: str, **kwargs) -> rdflib.Graph: + """The SHACL shapes graph of a class-expression schema with the given ``slots:`` / ``classes:`` YAML.""" + return _parse_shacl(_CLASS_EXPRESSION_HEADER + classes, **kwargs) + + +def test_class_expression_condition_title_names_the_property_shape(): + """A condition's title becomes the sh:name of its property shape.""" + g = _class_expression_shacl( + """ +slots: + a: {} + b: {} +classes: + Thing: + class_uri: ex:Thing + slots: [a, b] + any_of: + - slot_conditions: + a: + title: the a condition + required: true + - slot_conditions: + b: + required: true +""" + ) + (condition,) = _condition_property_shapes(g, EX_CE.Thing, EX_CE.a) + assert set(g.objects(condition, SH.name)) == {Literal("the a condition")} + + +@pytest.mark.parametrize("values,conforms", [('"a", "b"', True), ('"a", "b", "c"', False)]) +def test_class_expression_strictest_maximum_applies(values, conforms): + """With an exact and a maximum cardinality, the stricter bound applies.""" + shacl_ttl = _class_expression_shacl( + """ +slots: + tag: + multivalued: true +classes: + Thing: + class_uri: ex:Thing + slots: [tag] + all_of: + - slot_conditions: + tag: + exact_cardinality: 2 + maximum_cardinality: 3 +""" + ).serialize(format="turtle") + assert _conforms(shacl_ttl, f"{_CE_PREFIXES}ex:t a ex:Thing ; ex:tag {values} .") == conforms + + +@pytest.mark.parametrize("ref,conforms", [(None, True), ("ex:s", False), ("ex:o", True)]) +def test_class_expression_none_of_range(ref, conforms): + """A range in a condition is a value constraint: within none_of it requires + the slot in its "definitely true" form, so only a Special value violates.""" + shacl_ttl = _class_expression_shacl( + """ +slots: + ref: + range: Target +classes: + Target: + class_uri: ex:Target + Special: + class_uri: ex:Special + Thing: + class_uri: ex:Thing + slots: [ref] + none_of: + - slot_conditions: + ref: + range: Special +""", + closed=False, + ).serialize(format="turtle") + value = f" ; ex:ref {ref}" if ref else "" + data = f"{_CE_PREFIXES}ex:s a ex:Special, ex:Target . ex:o a ex:Target . ex:t a ex:Thing{value} ." + assert _conforms(shacl_ttl, data) == conforms + + +def test_class_expression_equals_string_uses_the_condition_range(): + """equals_string is rendered for the condition's own range: an enum value + with a meaning becomes that IRI.""" + g = _class_expression_shacl( + """ +enums: + FormatEnum: + permissible_values: + OSI: + meaning: ex:OSI +slots: + fileFormat: {} + other: {} +classes: + Format: + class_uri: ex:Format + slots: [fileFormat, other] + any_of: + - slot_conditions: + fileFormat: + range: FormatEnum + equals_string: OSI + - slot_conditions: + other: + required: true +""" + ) + (condition,) = _condition_property_shapes(g, EX_CE.Format, EX_CE.fileFormat) + in_lists = [list(Collection(g, node)) for node in g.objects(condition, SH["in"])] + in_lists += [ + list(Collection(g, node)) + for and_list in g.objects(condition, SH["and"]) + for member in Collection(g, and_list) + for node in g.objects(member, SH["in"]) + ] + assert in_lists and all(values == [EX_CE.OSI] for values in in_lists) + + +def test_class_expression_equals_string_on_a_custom_string_type(): + """A type derived from string holds strings, so equals_string on it is translated.""" + g = _class_expression_shacl( + """ +types: + Code: + typeof: string +slots: + code: + range: Code + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [code, a] + any_of: + - slot_conditions: + code: + equals_string: AB + - slot_conditions: + a: + required: true +""" + ) + assert (EX_CE.Thing, SH["or"], None) in g + + +def test_class_expression_builtin_range_without_types_import(): + """A built-in range name in a schema that doesn't import linkml:types gives its datatype.""" + schema = """ +id: https://example.org/class-expressions +name: class_expressions +prefixes: + ex: https://example.org/class-expressions/ +default_prefix: ex +slots: + size: {} + a: {} +classes: + Thing: + class_uri: ex:Thing + slots: [size, a] + any_of: + - slot_conditions: + size: + range: integer + - slot_conditions: + a: + required: true +""" + g = _parse_shacl(schema) + (condition,) = _condition_property_shapes(g, EX_CE.Thing, EX_CE.size) + assert (condition, SH.datatype, XSD.integer) in g + + +def test_class_expression_range_with_identifier_is_an_iri(): + """A condition ranged on a class with an identifier expects IRIs, as a slot's range does.""" + g = _class_expression_shacl( + """ +slots: + id: + identifier: true + ref: {} + a: {} +classes: + Target: + class_uri: ex:Target + slots: [id] + Thing: + class_uri: ex:Thing + slots: [ref, a] + any_of: + - slot_conditions: + ref: + range: Target + - slot_conditions: + a: + required: true +""" + ) + (condition,) = _condition_property_shapes(g, EX_CE.Thing, EX_CE.ref) + assert set(g.objects(condition, SH.nodeKind)) == {SH.IRI} + + +_UNTRANSLATABLE_MEMBERS = { + "untranslatable-second-member": ( + "has_member", + """ + - slot_conditions: + a: + required: true + - slot_conditions: + tags: + has_member: + equals_string: x""", + ), + "unknown-condition-range": ( + "unknown range", + """ + - slot_conditions: + a: + range: Undefined + - slot_conditions: + b: + required: true""", + ), + "untranslatable-nested-member": ( + "has_member", + """ + - all_of: + - slot_conditions: + tags: + has_member: + equals_string: x + - slot_conditions: + a: + required: true""", + ), + "is_a-not-a-class": ( + "not a class", + """ + - is_a: Undefined + - slot_conditions: + a: + required: true""", + ), + "equals_string-on-class-range": ( + "does not hold strings", + """ + - slot_conditions: + ref: + equals_string: x + - slot_conditions: + a: + required: true""", + ), + "equals_string_in-on-integer": ( + "does not hold strings", + """ + - slot_conditions: + count: + equals_string_in: ["5"] + - slot_conditions: + a: + required: true""", + ), +} + + +@pytest.mark.parametrize("case", list(_UNTRANSLATABLE_MEMBERS)) +def test_class_expression_untranslatable_member_skips_the_operator(caplog, case): + """An operator with an untranslatable member anywhere, at any depth or + position, is skipped as a whole, with a warning naming the reason.""" + reason, members = _UNTRANSLATABLE_MEMBERS[case] + schema = ( + _CLASS_EXPRESSION_HEADER + + """ +slots: + tags: + multivalued: true + ref: + range: Target + count: + range: integer + a: {} + b: {} +classes: + Target: + class_uri: ex:Target + Thing: + class_uri: ex:Thing + slots: [tags, ref, count, a, b] + any_of:""" + + members + + "\n" + ) + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + g = _parse_shacl(schema) + assert (EX_CE.Thing, SH["or"], None) not in g + assert any("any_of" in rec.message and reason in rec.message for rec in caplog.records), caplog.text + + +_REPEATED_PARAMETERS_SCHEMA = ( + _CLASS_EXPRESSION_HEADER + + """ +types: + Code: + typeof: string + pattern: "^[A-Z]+$" +enums: + LetterEnum: + permissible_values: + A: {} + B: {} + C: {} +slots: + size: + range: integer + label: {} + letter: {} + code: {} + other: {} +classes: + Thing: + class_uri: ex:Thing + slots: [size, label, letter, code, other] + any_of: + - slot_conditions: + size: + minimum_value: 3 + equals_number: 5 + label: + equals_string: A + equals_string_in: [A, B] + letter: + range: LetterEnum + equals_string: B + code: + range: Code + pattern: "^AB" + - slot_conditions: + other: + required: true +""" +) + + +@pytest.mark.parametrize( + "properties,expected", + [ + ('ex:size 5 ; ex:label "A" ; ex:letter "B" ; ex:code "ABC"', True), + ("ex:size 4", False), + ('ex:label "B"', False), + ('ex:letter "A"', False), + ('ex:code "ABc"', False), + ('ex:code "XY"', False), + ('ex:size 4 ; ex:other "x"', True), + ], +) +def test_class_expression_repeated_parameters_stay_well_formed(properties, expected): + """A parameter SHACL allows once per shape, needed twice by one condition, goes into sh:and. + + _conforms runs pyshacl with meta_shacl, which fails on an ill-formed shapes graph. + """ + shacl_ttl = ShaclGenerator(_REPEATED_PARAMETERS_SCHEMA, mergeimports=False).serialize() + data = f""" + @prefix ex: . + ex:t a ex:Thing ; {properties} . + """ + assert _conforms(shacl_ttl, data) is expected + + +@pytest.mark.parametrize( + "extra,properties,expected", + [ + ("", "", True), + ("maximum_cardinality: 5", "", True), + ("minimum_cardinality: 0", "", True), + ("maximum_cardinality: 5", 'ex:tag "A"', False), + ("maximum_cardinality: 5", 'ex:tag "B"', True), + ], +) +def test_class_expression_none_of_presence_is_monotonic(extra, properties, expected): + """Inside none_of, a cardinality that an absent slot satisfies does not flip an absent slot to rejected.""" + schema = ( + _CLASS_EXPRESSION_HEADER + + f""" +slots: + tag: + multivalued: true +classes: + Thing: + class_uri: ex:Thing + slots: [tag] + none_of: + - slot_conditions: + tag: + equals_string: A + {extra} +""" + ) + shacl_ttl = ShaclGenerator(schema, mergeimports=False).serialize() + data = f""" + @prefix ex: . + ex:t a ex:Thing {";" if properties else ""} {properties} . + """ + assert _conforms(shacl_ttl, data) is expected diff --git a/tests/linkml/test_validator/test_validation_context.py b/tests/linkml/test_validator/test_validation_context.py index a674048952..e19c1bda48 100644 --- a/tests/linkml/test_validator/test_validation_context.py +++ b/tests/linkml/test_validator/test_validation_context.py @@ -647,9 +647,9 @@ def _schema_with_value_disallowed() -> SchemaDefinition: def test_cache_hit_preserves_value_disallowed_not_keyword(): - """A slot with value_presence=ABSENT produces a root-level `not` keyword. - Cache-hit must carry it through from defs_class so the validator rejects - instances that include the forbidden field.""" + """A slot with value_presence=ABSENT produces a root-level `not: {required}` + in `allOf`. Cache-hit must carry it through from defs_class so the + validator rejects instances that include the forbidden field.""" schema = _schema_with_value_disallowed() cold = ValidationContext(schema, "ForbidsX").json_schema_validator( @@ -661,8 +661,9 @@ def test_cache_hit_preserves_value_disallowed_not_keyword(): closed=False, include_range_class_descendants=False ) - assert "not" in cold.schema, cold.schema - assert "not" in warm.schema, "value_disallowed `not` keyword must survive cache-hit" + forbids = {"not": {"required": ["forbidden_field"]}} + assert forbids in cold.schema.get("allOf", []), cold.schema + assert forbids in warm.schema.get("allOf", []), "value_disallowed `not` keyword must survive cache-hit" # Instance includes the forbidden field -> both validators reject bad = {"forbidden_field": "anything", "other_field": "ok"} @@ -677,18 +678,19 @@ def test_cache_hit_preserves_value_disallowed_not_keyword(): def test_cache_hit_does_not_leak_value_disallowed_not_to_target_without(): - """Warming with ForbidsX places `not` at root in cached. Hitting cache for - Plain (which has no value_disallowed slots) must NOT inherit `not`.""" + """Warming with ForbidsX places `not: {required}` at root in cached. Hitting + cache for Plain (which has no value_disallowed slots) must NOT inherit it.""" schema = _schema_with_value_disallowed() cache_key = _make_cache_key(schema, include_range_class_descendants=False) ValidationContext(schema, "ForbidsX").json_schema_validator(closed=False, include_range_class_descendants=False) - assert "not" in _json_schema_cache[cache_key] + forbids = {"not": {"required": ["forbidden_field"]}} + assert forbids in _json_schema_cache[cache_key].get("allOf", []) validator = ValidationContext(schema, "Plain").json_schema_validator( closed=False, include_range_class_descendants=False ) - assert "not" not in validator.schema, validator.schema.get("not") + assert forbids not in validator.schema.get("allOf", []), validator.schema.get("allOf") # Plain has no constraint against `forbidden_field` — the leak would falsely # reject this. Confirm it doesn't. From 13f52a268fb7f55f327e15be4ca7da1cda3a3cc0 Mon Sep 17 00:00:00 2001 From: jdsika Date: Mon, 5 Oct 2026 11:12:35 +0200 Subject: [PATCH 03/17] fix(generators): enforce range_expression on each value alike in SHACL and JSON Schema gen-shacl ignored a slot's range_expression, and gen-json-schema read it with the two-valued builder of rules, which ignores cardinalities, the range of a condition, and the slot_usage and aliases of the value class. Both generators now read it as a class-level expression: each value must satisfy it as an instance of the range class satisfies a class-level expression. In SHACL it is an sh:node next to the range's sh:class. On a class alternative of any_of it applies to that alternative, and in a slot condition it takes the form of the condition. SHACL skips an expression it cannot express with a warning, as it skips a class-level expression. gen-json-schema applied the value constraints of a slot inlined as a dict to the dict instead of to each value, which only a range_expression showed. The compact forms of such a dict, {id: value} and {id: null}, stay out of its reach. Signed-off-by: jdsika --- docs/generators/json-schema.rst | 5 + docs/generators/shacl.rst | 71 ++- docs/schemas/advanced.md | 2 + .../src/linkml/generators/jsonschemagen.py | 66 ++- .../linkml/src/linkml/generators/shaclgen.py | 159 ++++-- .../test_boolean_slot_compliance.py | 2 +- .../test_generators/test_range_expression.py | 500 ++++++++++++++++++ tests/linkml/test_generators/test_shaclgen.py | 2 +- 8 files changed, 737 insertions(+), 70 deletions(-) create mode 100644 tests/linkml/test_generators/test_range_expression.py diff --git a/docs/generators/json-schema.rst b/docs/generators/json-schema.rst index c1c68072b9..9298dd6316 100644 --- a/docs/generators/json-schema.rst +++ b/docs/generators/json-schema.rst @@ -180,6 +180,11 @@ A slot condition is unknown for an absent slot unless it decides whether the slo may be absent, and an instance is invalid only when an expression is definitely false, as described under "Class-level expressions and absent slots" in :doc:`Advanced features `. +A slot's `range_expression `_ +constrains each of its values in the same way, with its conditions on the slots +of the value. For a slot inlined as a dict these are the values of the dict, +except in its compact forms ``{id: value}`` and ``{id: null}``, which the +expression can't reach. The SHACL generator reads them the same way. Inlining diff --git a/docs/generators/shacl.rst b/docs/generators/shacl.rst index 9322186e44..cd5c3565db 100644 --- a/docs/generators/shacl.rst +++ b/docs/generators/shacl.rst @@ -106,8 +106,9 @@ then adds nothing; one that only some instances belong to, because a descendant of the class has it among its ancestors, and then gives ``sh:class``; or one that none belongs to, and then gives a shape nothing conforms to. An instance belongs to its class and that class's ancestors, by ``is_a`` and by ``mixins``, so a -class that may also be another is declared as a subclass with that mixin. -Nested expressions recurse. Each entry of ``slot_conditions`` gives an +class that may also be another is declared as a subclass with that mixin. For a +type or an enum, ``is_a`` gives its value constraint. Nested expressions recurse. +Each entry of ``slot_conditions`` gives an ``sh:property`` whose path is that of the slot as induced for the class, so ``slot_usage`` applies: @@ -120,7 +121,9 @@ Nested expressions recurse. Each entry of ``slot_conditions`` gives an * ``equals_string`` and ``equals_string_in`` give ``sh:in``; on an enum slot the values are the permissible values as the enum renders them, the IRI of their ``meaning`` where they have one; -* ``range`` gives the same class, type or enum constraint as a slot's range. +* ``range`` gives the same class, type or enum constraint as a slot's range; +* ``range_expression`` gives an ``sh:node``, as described under + `Range Expressions`_. A shape may have at most one value of ``sh:minInclusive``, ``sh:maxInclusive`` or ``sh:in``, and of ``sh:pattern``, whose component also takes ``sh:flags`` @@ -170,12 +173,70 @@ slot's range. An operator whose members use anything else is skipped as a whole and logged as a warning, because leaving out one member would change what the operator -admits. That covers, for example, ``has_member`` or a slot-level ``any_of`` -inside a slot condition, a condition on a name that is not a slot, a condition +admits. That covers, for example, ``has_member``, ``equals_expression`` or a +slot-level ``any_of`` inside a slot condition, a condition on a name that is not a slot, a condition on the identifier slot (the node's IRI rather than a property), and ``equals_string`` on a slot whose range does not hold strings. +Range Expressions +^^^^^^^^^^^^^^^^^ + +A slot's `range_expression `__ +constrains each of its values as a class-level expression constrains an +instance: its conditions constrain the slots of the value as induced for the +range class, and a value violates it only when it is definitely false. It +becomes an ``sh:node`` (`SHACL §4.7.1 +`__) on the property shape, +next to the range's ``sh:class``, so it constrains only the values of this slot, +and matching it doesn't make a node an instance of the range. The JSON Schema +generator reads it the same way. + +.. code-block:: yaml + + classes: + Document: + slots: [license] + slot_usage: + license: + range_expression: + slot_conditions: + category: + required: true + equals_string: license + Resource: + slots: [category] + slots: + license: + range: Resource + category: + range: string + +.. code-block:: turtle + + ex:Document a sh:NodeShape ; + sh:property [ sh:path ex:license ; + sh:class ex:Resource ; + sh:node [ sh:property [ sh:path ex:category ; + sh:minCount 1 ; + sh:in ( "license" ) ] ] ; + ... ] ; + ... + +A class alternative in a slot's ``any_of`` can carry its own +``range_expression``, and so can a slot condition, where the expression takes +the form of the condition: under ``sh:not``, its "definitely true" form. For +values that aren't instances of a class, an expression can combine types and +enums with ``is_a``, for example ``any_of: [{is_a: OpenLicense}, {is_a: +ProprietaryLicense}]``; such values have no slots for a condition to constrain. +An expression that uses anything SHACL can't express (see above) is skipped and +logged as a warning. + +A value that is a reference is checked against its description in the data +graph, as ``sh:class`` is. The JSON Schema generator can't check it, because +there the value is just an identifier. + + Command Line ^^^^^^^^^^^^ diff --git a/docs/schemas/advanced.md b/docs/schemas/advanced.md index 2f173f2443..052b01ed21 100644 --- a/docs/schemas/advanced.md +++ b/docs/schemas/advanced.md @@ -92,6 +92,8 @@ A `Sample` without `status` is valid: the condition is unknown, so `none_of` isn | `none_of: [label = A]` | valid | invalid | valid | | `exactly_one_of: [label = A, note = B]` | invalid | valid | invalid | +A slot's [range_expression](https://w3id.org/linkml/range_expression) constrains each value of the slot in the same way: its conditions constrain the slots of the value, as induced for the slot's range class. In a slot condition, a `range_expression` is part of the condition, so under `none_of` it must hold definitely. + [Rules](#rules) are read differently: their preconditions require their slots, and so do their postconditions unless the rule is `open_world`. ### Unions as ranges diff --git a/packages/linkml/src/linkml/generators/jsonschemagen.py b/packages/linkml/src/linkml/generators/jsonschemagen.py index 4a33c3ccf9..7c6ad48a7e 100644 --- a/packages/linkml/src/linkml/generators/jsonschemagen.py +++ b/packages/linkml/src/linkml/generators/jsonschemagen.py @@ -2,7 +2,7 @@ import json import logging import os -from copy import deepcopy +from copy import copy, deepcopy from dataclasses import dataclass, field from typing import Any, cast @@ -658,7 +658,7 @@ def handle_class(self, cls: ClassDefinition) -> None: _CLASS_OPERATOR_KEYWORDS = {"any_of": "anyOf", "all_of": "allOf", "exactly_one_of": "oneOf"} def get_subschema_for_class_operator( - self, cls: ClassDefinition, operator: str, members: list[AnonymousClassExpression], definite: bool + self, cls: ClassDefinition | None, operator: str, members: list[AnonymousClassExpression], definite: bool ) -> JsonSchema: """The subschema of the class-level boolean expression *operator* over *members*, for *cls*. @@ -684,16 +684,18 @@ def get_subschema_for_class_operator( ) def get_subschema_for_class_expression( - self, cls: ClassDefinition, expr: AnonymousClassExpression, definite: bool + self, cls: ClassDefinition | None, expr: AnonymousClassExpression, definite: bool ) -> JsonSchema: """The subschema of one member *expr* of a class-level boolean expression of *cls*. Each slot condition constrains the slot as induced for *cls*, so - ``slot_usage`` applies. Its values must satisfy its value operators - and its ``range``; the number of values must lie within + ``slot_usage`` applies. Its values must satisfy its value operators, + its ``range`` and its ``range_expression``, the latter in the form of + *expr*; the number of values must lie within :func:`~linkml.generators.common.class_expression.value_bounds`, where an empty list or dict counts as no value. *definite* selects the form, as in - :meth:`get_subschema_for_class_operator`. + :meth:`get_subschema_for_class_operator`. *cls* is ``None`` for values + that are not instances of a class. """ subschema = JsonSchema() conjuncts: list[JsonSchema] = [] @@ -703,7 +705,17 @@ def get_subschema_for_class_expression( prop_name = self._curie(slot) else: prop_name = self.aliased_slot_name(slot) - values = self.get_subschema_for_slot(condition, omit_type=True, include_null=False) + value_condition = condition + if condition.range_expression is not None: + # translated below, in the form of *expr* + value_condition = copy(condition) + value_condition.range_expression = None + values = self.get_subschema_for_slot(value_condition, omit_type=True, include_null=False) + if condition.range_expression is not None: + expression = self.get_subschema_for_range_expression( + condition.range or slot.range, condition.range_expression, definite + ) + values = JsonSchema({"allOf": [values, expression]}) if values else expression lower, upper = value_bounds(condition, definite) if slot.multivalued and self._inlined_as_dict(slot): # the values of a slot inlined as a dict are the values of the dict @@ -742,11 +754,17 @@ def get_subschema_for_class_expression( conjuncts.append(self.get_subschema_for_class_operator(cls, operator, members, definite)) if expr.is_a is not None: # `is_a: ` requires an instance of the expression to be an instance of . Every - # instance of *cls* is one, or none is, unless is a descendant of *cls*: then, in - # JSON, only the instance's structure tells, so it must match the definition of . - membership = class_membership(self.schemaview, cls.name, expr.is_a) + # instance of *cls* is one, or none is, unless some are through a descendant of *cls*: + # then, in JSON, only the instance's structure tells, so it must match the definition + # of . For a type or an enum, or a value that is no class instance, the value + # constraint applies. + sv = self.schemaview + is_class = expr.is_a in sv.all_classes() + membership = class_membership(sv, cls.name, expr.is_a) if is_class and cls is not None else None if membership is None: - conjuncts.append(self.get_subschema_for_slot(AnonymousSlotExpression(range=expr.is_a, inlined=True))) + conjuncts.append( + self.get_subschema_for_slot(AnonymousSlotExpression(range=expr.is_a, inlined=is_class)) + ) elif not membership: conjuncts.append(JsonSchema({"not": {}})) if conjuncts: @@ -761,13 +779,27 @@ def _inlined_as_dict(self, slot: SlotDefinition) -> bool: and get_range_associated_slots(self.schemaview, slot.range)[0] is not None ) - def _class_expression_slot(self, cls: ClassDefinition, slot_name: str) -> SlotDefinition | None: + def _class_expression_slot(self, cls: ClassDefinition | None, slot_name: str) -> SlotDefinition | None: """The slot a condition of a class expression of *cls* names, as induced for *cls*, if any.""" sv = self.schemaview - if slot_name in sv.class_slots(cls.name): + if cls is not None and slot_name in sv.class_slots(cls.name): return sv.induced_slot(slot_name, cls.name) return sv.get_slot(slot_name) + def get_subschema_for_range_expression( + self, value_range: str | None, expression: AnonymousClassExpression, definite: bool + ) -> JsonSchema: + """The subschema a value of range *value_range* must satisfy under *expression*. + + A ``range_expression`` constrains each value as a class-level expression + constrains an instance (:meth:`get_subschema_for_class_expression`): its + conditions constrain the slots of the value as induced for the class + *value_range*, and *definite* selects the form. + """ + sv = self.schemaview + value_class = sv.get_class(value_range) if value_range in sv.all_classes() else None + return self.get_subschema_for_class_expression(value_class, expression, definite) + def get_subschema_for_anonymous_class( self, cls: AnonymousClassExpression, properties_required: bool = False ) -> None | JsonSchema: @@ -938,7 +970,7 @@ def get_value_constraints_for_slot(self, slot: SlotDefinition | AnonymousSlotExp constraints.add_keyword("enum", slot.equals_string_in) if slot.range_expression: - subschema = self.get_subschema_for_anonymous_class(slot.range_expression) + subschema = self.get_subschema_for_range_expression(slot.range, slot.range_expression, definite=False) if subschema: if "allOf" not in constraints: constraints["allOf"] = [] @@ -1024,6 +1056,7 @@ def get_subschema_for_slot( slot_is_boolean = any([slot.any_of, slot.all_of, slot.exactly_one_of, slot.none_of]) typ = None + inlined_as_dict = False if not omit_type: typ, fmt, reference = self.get_type_info_for_slot_subschema(slot) if slot_is_inlined: @@ -1037,6 +1070,7 @@ def get_subschema_for_slot( # if the range class has an ID and the slot is not inlined as a list, then we need to consider # various inlined as dict formats if range_id_slot is not None and not slot.inlined_as_list: + inlined_as_dict = True # At a minimum, the inlined dict can have keys (additionalProps) that are IDs # and the values are the range class but possibly omitting the ID. additionalProps = [JsonSchema.ref_for(reference, identifier_optional=True)] @@ -1117,6 +1151,10 @@ def get_subschema_for_slot( prop["items"].update(all_element_constraints) if any_element_constraints: prop["contains"] = any_element_constraints + elif inlined_as_dict: + # The values of the slot are the values of the dict. + if own_constraints: + prop["additionalProperties"] = JsonSchema({"allOf": [prop["additionalProperties"], own_constraints]}) else: prop.update(own_constraints) diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index f1ee237952..11e670e5c5 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -224,6 +224,7 @@ def as_graph(self) -> Graph: self._class_expressions_added: set[tuple[URIRef, str, str]] = set() self._class_expression_problems: dict[tuple[str, str, str], list[str]] = {} + self._range_expression_problems: dict[tuple[str, str], list[str]] = {} for c in sv.all_classes(imports=not self.exclude_imports).values(): def shape_pv(p, v): @@ -359,47 +360,28 @@ def prop_pv_text(p, v): range_list = [] for any in s.any_of: r = any.range - if r in all_classes: - class_node = BNode() + branch = BNode() - def cl_node_pv(p, v): - if v is not None: - g.add((class_node, p, v)) + def branch_pv(p, v): + if v is not None: + g.add((branch, p, v)) - self._add_class(cl_node_pv, r) - range_list.append(class_node) + if r in all_classes: + self._add_class(branch_pv, r) elif r in sv.all_types(): - t_node = BNode() - - def t_node_pv(p, v): - if v is not None: - g.add((t_node, p, v)) - - self._add_type(t_node_pv, r) - range_list.append(t_node) + self._add_type(branch_pv, r) elif r in sv.all_enums(): - en_node = BNode() - - def en_node_pv(p, v): - if v is not None: - g.add((en_node, p, v)) - - self._add_enum(g, en_node_pv, r) - range_list.append(en_node) + self._add_enum(g, branch_pv, r) else: - st_node = BNode() - - def st_node_pv(p, v): - if v is not None: - g.add((st_node, p, v)) - - add_simple_data_type(st_node_pv, r) - range_list.append(st_node) + add_simple_data_type(branch_pv, r) # Propagate pattern constraint to the branch node. # A branch may combine range + pattern (e.g. range: string # with pattern: "^...") or specify pattern alone (no range). if any.pattern: - g.add((range_list[-1], SH.pattern, Literal(any.pattern))) + branch_pv(SH.pattern, Literal(any.pattern)) + if any.range_expression is not None: + self._add_range_expression(g, branch_pv, c, s, r, any.range_expression) + range_list.append(branch) Collection(g, or_node, range_list) else: prop_pv_literal(SH.hasValue, s.equals_number) @@ -425,6 +407,8 @@ def st_node_pv(p, v): if s.pattern: prop_pv(SH.pattern, Literal(s.pattern)) + if s.range_expression is not None: + self._add_range_expression(g, prop_pv, c, s, s.range, s.range_expression) if s.annotations and self.include_annotations: self._add_annotations(prop_pv, s) @@ -439,6 +423,7 @@ def st_node_pv(p, v): self._add_rules(g, class_uri_with_suffix, c) self._report_class_expression_problems() + self._report_range_expression_problems() return g LINKML_ANY_URI = "https://w3id.org/linkml/Any" @@ -462,7 +447,16 @@ def st_node_pv(p, v): {"required", "value_presence", "minimum_cardinality", "maximum_cardinality", "exact_cardinality"} ) _SLOT_CONDITION_VALUE_FIELDS = frozenset( - {"minimum_value", "maximum_value", "pattern", "equals_string", "equals_string_in", "equals_number", "range"} + { + "minimum_value", + "maximum_value", + "pattern", + "equals_string", + "equals_string_in", + "equals_number", + "range", + "range_expression", + } ) _SLOT_CONDITION_FIELDS = _SLOT_CONDITION_PRESENCE_FIELDS | _SLOT_CONDITION_VALUE_FIELDS @@ -529,9 +523,10 @@ def _add_class_expressions(self, g: Graph, shape_uri: URIRef, cls: ClassDefiniti Every member becomes an anonymous node shape: ``is_a`` gives ``sh:class`` for a class that some instances of *cls* belong to through a descendant, - nothing for *cls* or an ancestor, which every instance belongs to, and a + nothing for *cls* or an ancestor, which every instance belongs to, a shape nothing conforms to for any other class - (:func:`~linkml.generators.common.class_expression.class_membership`); + (:func:`~linkml.generators.common.class_expression.class_membership`), + and the value constraint of a type or an enum; each slot condition gives an ``sh:property`` on the path of the slot as induced for *cls* (so ``slot_usage`` applies), and nested expressions recurse. @@ -585,7 +580,7 @@ def _add_logical_constraint( self, g: Graph, subject: URIRef | BNode, - cls: ClassDefinition, + cls: ClassDefinition | None, operator: str, members: list[AnonymousClassExpression], definite: bool, @@ -611,7 +606,7 @@ def _add_logical_constraint( g.add((subject, predicate, list_node)) def _class_expression_shape( - self, g: Graph, cls: ClassDefinition, expr: AnonymousClassExpression, definite: bool + self, g: Graph, cls: ClassDefinition | None, expr: AnonymousClassExpression, definite: bool ) -> BNode: """Build the anonymous node shape for one class expression *expr*, in its "definitely true" form with *definite*, otherwise its "not false" form.""" @@ -626,13 +621,16 @@ def node_pv(p, v): if expr.description is not None: node_pv(RDFS.comment, Literal(expr.description, lang=self._resolve_language(expr))) if expr.is_a is not None: - # Every instance of *cls* is an instance of ``is_a``, or none is, unless ``is_a`` is a - # descendant of *cls*: only then does the node's type decide. - membership = class_membership(self.schemaview, cls.name, expr.is_a) - if membership is None: - self._add_class(node_pv, expr.is_a) - elif not membership: - node_pv(SH["not"], BNode()) # a shape nothing conforms to + if expr.is_a in self.schemaview.all_classes(): + # Every instance of *cls* is an instance of ``is_a``, or none is, unless some are + # through a descendant of *cls*: only then does the node's type decide. + membership = class_membership(self.schemaview, cls.name, expr.is_a) if cls is not None else None + if membership is None: + self._add_class(node_pv, expr.is_a) + elif not membership: + node_pv(SH["not"], BNode()) # a shape nothing conforms to + else: + self._add_range(g, node_pv, expr.is_a) for slot_name, condition in expr.slot_conditions.items(): node_pv(SH.property, self._slot_condition_shape(g, cls, slot_name, condition, definite)) for operator in self._CLASS_EXPRESSION_OPERATORS: @@ -693,6 +691,8 @@ def prop_pv(p, v): prop_pv(SH.maxInclusive, Literal(condition.equals_number)) if condition.range is not None: self._add_range(g, prop_pv, condition.range) + if condition.range_expression is not None: + prop_pv(SH.node, self._range_expression_shape(g, value_range, condition.range_expression, definite)) if repeated: # Each repeated parameter in a member shape of its own: all of them hold # for every value, as they would on the property shape itself. @@ -757,18 +757,23 @@ def _is_string_range(self, r: ElementName | None) -> bool: return True return self._type_uri(r) == str(XSD.string) - def _untranslatable(self, cls: ClassDefinition, expr: AnonymousClassExpression) -> str | None: - """Return what in class expression *expr* of *cls* cannot be translated, or ``None``.""" - sv = self.schemaview + def _untranslatable(self, cls: ClassDefinition | None, expr: AnonymousClassExpression) -> str | None: + """Return what in class expression *expr* of *cls* cannot be translated, or ``None``. + + *cls* is ``None`` when *expr* constrains values that are not instances of + a class, which have no slots for a condition to constrain. + """ unknown = self._set_operator_fields(expr) - self._CLASS_EXPRESSION_FIELDS if unknown: return f"'{sorted(unknown)[0]}'" - if expr.is_a is not None and expr.is_a not in sv.all_classes(): - return f"is_a '{expr.is_a}', which is not a class" + if expr.is_a is not None and not self._is_known_range(expr.is_a): + return f"is_a '{expr.is_a}', which is not a known class, type or enum" for slot_name, condition in expr.slot_conditions.items(): unknown = self._set_operator_fields(condition) - self._SLOT_CONDITION_FIELDS if unknown: return f"'{sorted(unknown)[0]}' in the condition on slot '{slot_name}'" + if cls is None: + return f"a condition on slot '{slot_name}' of values that are not class instances" slot = self._condition_slot(cls, slot_name) if slot is None: return f"a condition on '{slot_name}', which is not a slot" @@ -778,6 +783,10 @@ def _untranslatable(self, cls: ClassDefinition, expr: AnonymousClassExpression) if condition.range is not None and not self._is_known_range(condition.range): return f"the unknown range '{condition.range}' in the condition on slot '{slot_name}'" value_range = condition.range or slot.range + if condition.range_expression is not None: + reason = self._untranslatable(self._value_class(value_range), condition.range_expression) + if reason is not None: + return reason if (condition.equals_string is not None or condition.equals_string_in) and not self._is_string_range( value_range ): @@ -789,6 +798,58 @@ def _untranslatable(self, cls: ClassDefinition, expr: AnonymousClassExpression) return reason return None + def _value_class(self, value_range: ElementName | None) -> ClassDefinition | None: + """The class whose instances a range *value_range* holds, or ``None`` for any other range.""" + sv = self.schemaview + return sv.get_class(value_range) if value_range in sv.all_classes() else None + + def _range_expression_shape( + self, g: Graph, value_range: ElementName | None, expression: AnonymousClassExpression, definite: bool + ) -> BNode: + """Build the node shape a value of range *value_range* must conform to under *expression*. + + A ``range_expression`` constrains each value as a class-level expression + constrains an instance: its conditions constrain the slots of the value + as induced for the class *value_range*, and *definite* selects the form, + as in :meth:`_class_expression_shape`. ``sh:node`` (SHACL §4.7.1) + applies the shape to each value alongside the range's own constraint, + and only to the values of this slot. + """ + return self._class_expression_shape(g, self._value_class(value_range), expression, definite) + + def _add_range_expression( + self, + g: Graph, + prop_pv: Callable, + cls: ClassDefinition, + slot: SlotDefinition, + value_range: ElementName | None, + expression: AnonymousClassExpression, + ) -> None: + """Constrain each value of range *value_range* of *slot*, in the shape of *cls*, by *expression*. + + The expression is read in its "not false" form. One that cannot be + translated is skipped with a warning, as a class-level expression is. + """ + reason = self._untranslatable(self._value_class(value_range), expression) + if reason is not None: + classes = self._range_expression_problems.setdefault((slot.name, reason), []) + if cls.name not in classes: + classes.append(cls.name) + return + prop_pv(SH.node, self._range_expression_shape(g, value_range, expression, definite=False)) + + def _report_range_expression_problems(self) -> None: + """Warn once about each ``range_expression`` of a slot that is not translated, + naming the class shapes it is missing from.""" + for (slot_name, reason), classes in self._range_expression_problems.items(): + logger.warning( + "Slot %r: range_expression is not translated to SHACL, because it uses %s (in the shapes of %s).", + slot_name, + reason, + ", ".join(map(repr, classes)), + ) + def _is_known_range(self, r: ElementName) -> bool: """Whether *r* names a class, type or enum of the schema, or a built-in type.""" sv = self.schemaview diff --git a/tests/linkml/test_compliance/test_boolean_slot_compliance.py b/tests/linkml/test_compliance/test_boolean_slot_compliance.py index dc04f6f2f8..abe307c1fa 100644 --- a/tests/linkml/test_compliance/test_boolean_slot_compliance.py +++ b/tests/linkml/test_compliance/test_boolean_slot_compliance.py @@ -2903,7 +2903,7 @@ def test_range_expression_nesting(framework, data_name, instance, is_valid): core_elements=["range_expression"], ) expected_behavior = ValidationBehavior.IMPLEMENTS - if framework not in [JSON_SCHEMA, OWL]: + if framework not in [JSON_SCHEMA, OWL, SHACL]: if not is_valid: expected_behavior = ValidationBehavior.INCOMPLETE check_data( diff --git a/tests/linkml/test_generators/test_range_expression.py b/tests/linkml/test_generators/test_range_expression.py new file mode 100644 index 0000000000..71a3ee71bb --- /dev/null +++ b/tests/linkml/test_generators/test_range_expression.py @@ -0,0 +1,500 @@ +"""A slot's ``range_expression`` constrains each of its values alike in the JSON Schema and SHACL generators. + +Each value must satisfy the expression as an instance of its range class +satisfies a class-level expression: its conditions constrain the slots of the +value as induced for that class, and an expression is violated only when it is +definitely false. The agreement tests validate the same instance with both +generated artifacts, loading it as RDF through the generated JSON-LD context. +""" + +import json +import logging +from pathlib import Path +from typing import Any + +import pytest +import yaml +from click.testing import CliRunner +from jsonschema import Draft201909Validator +from pyshacl import validate +from rdflib import Graph, Namespace, URIRef +from rdflib.namespace import SH + +from linkml.generators.jsonldcontextgen import ContextGenerator +from linkml.generators.jsonschemagen import JsonSchemaGenerator +from linkml.generators.shaclgen import ShaclGenerator, cli +from linkml_runtime import SchemaView + +EX = Namespace("https://example.org/context/") +PREFIX = f"@prefix ex: <{EX}> . " + + +@pytest.fixture +def schema() -> dict[str, Any]: + """A resource is constrained differently when used as a document's license.""" + return yaml.safe_load(""" +id: https://example.org/context +name: context +prefixes: + ex: https://example.org/context/ + linkml: https://w3id.org/linkml/ +default_prefix: ex +imports: [linkml:types] +classes: + Document: + tree_root: true + slots: [license] + slot_usage: + license: + range_expression: + slot_conditions: + category: {equals_string: license, required: true} + Resource: + slots: [category, code, tags, details] + Details: + slots: [code] +slots: + license: {range: Resource, inlined: true} + category: {range: string} + code: {range: integer} + tags: {range: string, multivalued: true} + details: {range: Details, inlined: true} +enums: + Category: + permissible_values: + license: {} +""") + + +def _as_json_ld(sv: SchemaView, obj: dict[str, Any], class_name: str) -> dict[str, Any]: + """*obj* with an ``@type`` on it and on each inlined object: the range of its slot, unless it states one.""" + class_name = obj.get("@type", class_name) + typed = {**obj, "@type": class_name} + for slot in sv.class_induced_slots(class_name): + value = obj.get(slot.name) + if slot.range in sv.all_classes() and value is not None: + values = [_as_json_ld(sv, v, slot.range) for v in (value if isinstance(value, list) else [value])] + typed[slot.name] = values if isinstance(value, list) else values[0] + return typed + + +def _without_types(obj: Any) -> Any: + """*obj* without its ``@type`` keys.""" + if isinstance(obj, dict): + return {k: _without_types(v) for k, v in obj.items() if k != "@type"} + if isinstance(obj, list): + return [_without_types(v) for v in obj] + return obj + + +def verdicts(schema: dict[str, Any], instance: dict[str, Any], target: str = "Document") -> tuple[bool, bool]: + """Whether the JSON Schema and the SHACL shapes generated for *schema* accept *instance* of *target*.""" + text = yaml.safe_dump(schema) + json_schema = json.loads(JsonSchemaGenerator(text, top_class=target).serialize()) + context = json.loads(ContextGenerator(text).serialize())["@context"] + data = {"@context": context, **_as_json_ld(SchemaView(text), instance, target)} + shapes = ShaclGenerator(text, closed=False).serialize() + conforms, _, _ = validate( + Graph().parse(data=json.dumps(data), format="json-ld"), + shacl_graph=Graph().parse(data=shapes, format="turtle"), + meta_shacl=True, + ) + return Draft201909Validator(json_schema).is_valid(_without_types(instance)), conforms + + +def conforms(schema: dict[str, Any], data: str, **options: Any) -> bool: + """Whether Turtle *data* conforms to the SHACL shapes generated for *schema*, which must pass meta-SHACL.""" + shapes = ShaclGenerator(yaml.safe_dump(schema), closed=False, **options).serialize() + return validate( + data_graph=PREFIX + data, + data_graph_format="turtle", + shacl_graph=shapes, + shacl_graph_format="turtle", + meta_shacl=True, + )[0] + + +def _equals(slot: str, value: str | int) -> dict[str, Any]: + """A class expression requiring *slot* to equal *value*, unknown when *slot* is absent.""" + key = "equals_number" if isinstance(value, int) else "equals_string" + return {"slot_conditions": {slot: {key: value}}} + + +_REQUIRED_LICENSE = {"slot_conditions": {"category": {"equals_string": "license", "required": True}}} +_LICENSE_OR_CODE = [_equals("category", "license"), _equals("code", 5)] +_NO_TAGS_AND_LICENSE = { + "slot_conditions": {"tags": {"maximum_cardinality": 0}}, + "all_of": [_equals("category", "license")], +} +_TWO_TAGS = {"slot_conditions": {"tags": {"minimum_cardinality": 2}}} +_CATEGORY_RANGE = {"slot_conditions": {"category": {"range": "Category"}}} +_CODE_FROM_TWO = {"slot_conditions": {"code": {"minimum_value": 2}}} +_REQUIRED_DETAILS = {"slot_conditions": {"details": {"required": True, "range_expression": _CODE_FROM_TWO}}} +_NO_DETAILS_CODE = {"none_of": [{"slot_conditions": {"details": {"range_expression": _CODE_FROM_TWO}}}]} + + +@pytest.mark.parametrize( + "expression,value,valid", + [ + pytest.param(_REQUIRED_LICENSE, {"category": "license"}, True, id="required-match"), + pytest.param(_REQUIRED_LICENSE, {"category": "other"}, False, id="required-other"), + pytest.param(_REQUIRED_LICENSE, {}, False, id="required-absent"), + # a condition that doesn't state presence is unknown for an absent slot + pytest.param(_equals("category", "license"), {}, True, id="optional-absent"), + pytest.param(_equals("category", "license"), {"category": "other"}, False, id="optional-other"), + pytest.param({"none_of": [_equals("category", "license")]}, {}, True, id="none_of-absent"), + pytest.param({"none_of": [_equals("category", "license")]}, {"category": "license"}, False, id="none_of-match"), + pytest.param({"none_of": [_equals("category", "license")]}, {"category": "other"}, True, id="none_of-other"), + pytest.param({"any_of": _LICENSE_OR_CODE}, {}, True, id="any_of-absent"), + pytest.param({"any_of": _LICENSE_OR_CODE}, {"category": "other"}, True, id="any_of-false-or-unknown"), + pytest.param({"any_of": _LICENSE_OR_CODE}, {"category": "other", "code": 1}, False, id="any_of-false"), + # exactly_one_of counts the members that are definitely true + pytest.param({"exactly_one_of": _LICENSE_OR_CODE}, {}, False, id="exactly_one_of-absent"), + pytest.param({"exactly_one_of": _LICENSE_OR_CODE}, {"category": "license"}, True, id="exactly_one_of-first"), + pytest.param({"exactly_one_of": _LICENSE_OR_CODE}, {"code": 5}, True, id="exactly_one_of-second"), + pytest.param( + {"exactly_one_of": _LICENSE_OR_CODE}, {"category": "license", "code": 5}, False, id="exactly_one_of-both" + ), + pytest.param({"exactly_one_of": _LICENSE_OR_CODE}, {"category": "other"}, False, id="exactly_one_of-none"), + pytest.param(_NO_TAGS_AND_LICENSE, {"tags": ["a"]}, False, id="absent-and-all_of-tags"), + pytest.param(_NO_TAGS_AND_LICENSE, {"category": "other"}, False, id="absent-and-all_of-other"), + pytest.param(_NO_TAGS_AND_LICENSE, {"category": "license"}, True, id="absent-and-all_of-match"), + pytest.param(_TWO_TAGS, {}, False, id="cardinality-absent"), + pytest.param(_TWO_TAGS, {"tags": ["a"]}, False, id="cardinality-too-few"), + pytest.param(_TWO_TAGS, {"tags": ["a", "b"]}, True, id="cardinality-enough"), + pytest.param(_CATEGORY_RANGE, {"category": "license"}, True, id="condition-range"), + pytest.param(_CATEGORY_RANGE, {"category": "other"}, False, id="condition-range-other"), + # a nested expression constrains the slots of the value's value, as induced for its range class + pytest.param(_REQUIRED_DETAILS, {"details": {"code": 2}}, True, id="nested"), + pytest.param(_REQUIRED_DETAILS, {"details": {"code": 1}}, False, id="nested-other"), + pytest.param(_REQUIRED_DETAILS, {"details": {}}, True, id="nested-absent"), + pytest.param(_REQUIRED_DETAILS, {}, False, id="nested-required"), + # under a negation, a nested expression takes its "definitely true" form + pytest.param(_NO_DETAILS_CODE, {"details": {}}, True, id="nested-in-none_of-absent"), + pytest.param(_NO_DETAILS_CODE, {"details": {"code": 3}}, False, id="nested-in-none_of-match"), + pytest.param(_NO_DETAILS_CODE, {"details": {"code": 1}}, True, id="nested-in-none_of-other"), + ], +) +@pytest.mark.parametrize("multivalued", [False, True]) +def test_range_expression_constrains_each_value( + schema: dict[str, Any], expression: dict[str, Any], value: dict[str, Any], valid: bool, multivalued: bool +) -> None: + """Both generators read a range expression alike, on a single value and on a list of values.""" + schema["classes"]["Document"]["slot_usage"]["license"]["range_expression"] = expression + schema["slots"]["license"]["multivalued"] = multivalued + assert verdicts(schema, {"license": [value] if multivalued else value}) == (valid, valid) + + +@pytest.mark.parametrize( + "values,valid", + [ + ([{"category": "license"}, {"category": "license"}], True), + ([{"category": "license"}, {"category": "other"}], False), + ([{"category": "license"}, {}], False), + ], +) +def test_range_expression_holds_for_every_value(schema: dict[str, Any], values: list[dict], valid: bool) -> None: + """Every value of a multivalued slot must satisfy the expression.""" + schema["slots"]["license"]["multivalued"] = True + assert verdicts(schema, {"license": values}) == (valid, valid) + + +@pytest.mark.parametrize( + "operator,value,valid", + [ + ("all_of", {}, True), + ("all_of", {"category": "license"}, True), + ("all_of", {"category": "other"}, False), + ("none_of", {}, True), + ("none_of", {"category": "license"}, False), + ("none_of", {"category": "other"}, True), + ], +) +def test_range_expression_in_class_expression( + schema: dict[str, Any], operator: str, value: dict[str, Any], valid: bool +) -> None: + """In a condition of a class-level expression, a range expression takes the form of the condition.""" + del schema["classes"]["Document"]["slot_usage"] + schema["classes"]["Document"][operator] = [ + {"slot_conditions": {"license": {"range_expression": _equals("category", "license")}}} + ] + assert verdicts(schema, {"license": value}) == (valid, valid) + + +@pytest.mark.parametrize("value,valid", [({"category": "license"}, True), ({"category": "other"}, False)]) +def test_range_expression_on_union_branch(schema: dict[str, Any], value: dict[str, Any], valid: bool) -> None: + """A class alternative of a slot's ``any_of`` carries its own range expression.""" + expression = schema["classes"]["Document"].pop("slot_usage")["license"]["range_expression"] + schema["classes"]["Note"] = {"slots": ["text"]} + schema["slots"]["text"] = {"range": "string", "required": True} + schema["slots"]["license"] = { + "inlined": True, + "any_of": [{"range": "Resource", "range_expression": expression}, {"range": "Note"}], + } + # the RDF type of a value of a union is the alternative it is an instance of + assert verdicts(schema, {"license": {"@type": "Resource", **value}}) == (valid, valid) + assert verdicts(schema, {"license": {"@type": "Note", "text": "explanation"}}) == (True, True) + + +def test_union_branch_keeps_its_class(schema: dict[str, Any]) -> None: + """Satisfying the expression of a class alternative doesn't make a value an instance of that class.""" + expression = schema["classes"]["Document"].pop("slot_usage")["license"]["range_expression"] + schema["slots"]["license"] = {"inlined": True, "any_of": [{"range": "Resource", "range_expression": expression}]} + assert conforms(schema, 'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "license"] .') + assert not conforms(schema, 'ex:doc a ex:Document; ex:license [a ex:Details; ex:category "license"] .') + + +@pytest.mark.parametrize( + "value,valid", [({"category": "mit"}, True), ({"category": "eula"}, True), ({"category": "gpl"}, False)] +) +def test_range_expression_combining_enums(schema: dict[str, Any], value: dict[str, Any], valid: bool) -> None: + """A range expression on values that aren't class instances can combine enums.""" + del schema["classes"]["Document"]["slot_usage"] + schema["enums"] = { + "Open": {"permissible_values": {"mit": {}, "bsd": {}}}, + "Proprietary": {"permissible_values": {"eula": {}}}, + } + schema["slots"]["category"]["range_expression"] = {"any_of": [{"is_a": "Open"}, {"is_a": "Proprietary"}]} + assert verdicts(schema, {"license": value}) == (valid, valid) + + +@pytest.mark.parametrize("in_class_expression", [False, True]) +@pytest.mark.parametrize("value,valid", [({"category": ["license"]}, True), ({"category": ["license", "x"]}, False)]) +def test_range_expression_uses_the_slots_of_the_range_class( + schema: dict[str, Any], value: dict[str, Any], valid: bool, in_class_expression: bool +) -> None: + """A condition constrains the slot as induced for the range class, whose ``slot_usage`` applies.""" + schema["classes"]["SpecialResource"] = {"is_a": "Resource", "slot_usage": {"category": {"multivalued": True}}} + schema["slots"]["license"]["range"] = "SpecialResource" + if in_class_expression: + del schema["classes"]["Document"]["slot_usage"] + condition = {"range_expression": _equals("category", "license")} + schema["classes"]["Document"]["all_of"] = [{"slot_conditions": {"license": condition}}] + else: + schema["classes"]["Document"]["slot_usage"]["license"]["range_expression"] = _equals("category", "license") + assert verdicts(schema, {"license": value}) == (valid, valid) + + +@pytest.mark.parametrize("target", ["Document", "SpecialDocument"]) +@pytest.mark.parametrize("value,valid", [({"category": "license"}, True), ({"category": "other"}, False)]) +def test_range_expression_is_inherited(schema: dict[str, Any], target: str, value: dict[str, Any], valid: bool) -> None: + """A subclass keeps the range expression its parent places on a slot.""" + schema["classes"]["SpecialDocument"] = {"is_a": "Document"} + assert verdicts(schema, {"license": value}, target=target) == (valid, valid) + + +@pytest.mark.parametrize("category,valid", [("license", True), ("other", False)]) +def test_range_expression_on_values_inlined_as_dict(schema: dict[str, Any], category: str, valid: bool) -> None: + """The values of a slot inlined as a dict are the values of the dict, not the dict itself.""" + schema["classes"]["Resource"]["slots"].insert(0, "id") + schema["slots"]["id"] = {"identifier": True, "range": "string"} + schema["slots"]["license"]["multivalued"] = True + json_schema = json.loads(JsonSchemaGenerator(yaml.safe_dump(schema), top_class="Document").serialize()) + instance = {"license": {"r1": {"category": "license"}, "r2": {"category": category}}} + assert Draft201909Validator(json_schema).is_valid(instance) is valid + data = 'ex:doc a ex:Document; ex:license ex:r1, ex:r2 . ex:r1 a ex:Resource; ex:category "license" . ' + assert conforms(schema, data + f'ex:r2 a ex:Resource; ex:category "{category}" .') is valid + + +@pytest.mark.parametrize("category,valid", [("license", True), ("other", False)]) +def test_range_expression_in_condition_on_values_inlined_as_dict( + schema: dict[str, Any], category: str, valid: bool +) -> None: + """In a condition of a class-level expression, a range expression constrains the values of a dict.""" + del schema["classes"]["Document"]["slot_usage"] + schema["classes"]["Resource"]["slots"].insert(0, "id") + schema["slots"]["id"] = {"identifier": True, "range": "string"} + schema["slots"]["license"]["multivalued"] = True + condition = {"range_expression": _equals("category", "license")} + schema["classes"]["Document"]["all_of"] = [{"slot_conditions": {"license": condition}}] + json_schema = json.loads(JsonSchemaGenerator(yaml.safe_dump(schema), top_class="Document").serialize()) + assert Draft201909Validator(json_schema).is_valid({"license": {"r1": {"category": category}}}) is valid + + +_COMPACT_DICT_SCHEMA = """ +id: https://example.org/compact +name: compact +prefixes: {ex: https://example.org/compact/, linkml: https://w3id.org/linkml/} +default_prefix: ex +imports: [linkml:types] +classes: + Resource: + attributes: + id: {identifier: true} + category: {range: string} + Document: + attributes: + license: + range: Resource + multivalued: true + inlined: true + range_expression: + slot_conditions: + category: {equals_string: license, required: true} +""" + + +@pytest.mark.xfail(strict=True, reason="JSON Schema can't apply a range expression to the compact forms of a dict") +@pytest.mark.parametrize("value", ["other", None], ids=["simple-dict", "null"]) +def test_range_expression_on_compact_dict_forms(value: str | None) -> None: + """The compact forms ``{id: value}`` and ``{id: null}`` of a slot inlined as a dict escape the + expression in JSON Schema, while SHACL rejects the resources they denote.""" + json_schema = json.loads(JsonSchemaGenerator(_COMPACT_DICT_SCHEMA, top_class="Document").serialize()) + assert not Draft201909Validator(json_schema).is_valid({"license": {"r1": value}}) + + +def test_range_expression_follows_the_alias(schema: dict[str, Any]) -> None: + """A condition names a slot of the value, whose alias is its JSON key and whose IRI is its RDF property.""" + schema["slots"]["category"]["alias"] = "kind" + validator = Draft201909Validator( + json.loads(JsonSchemaGenerator(yaml.safe_dump(schema), top_class="Document").serialize()) + ) + assert validator.is_valid({"license": {"kind": "license"}}) + assert not validator.is_valid({"license": {"kind": "other"}}) + assert conforms(schema, 'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "license"] .') + assert not conforms(schema, 'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "other"] .') + + +@pytest.mark.parametrize("type_statement,expected", [("a ex:Resource;", True), ("", False), ("a ex:Details;", False)]) +def test_range_expression_preserves_class_membership( + schema: dict[str, Any], type_statement: str, expected: bool +) -> None: + """Matching content cannot substitute for membership in the declared range.""" + assert conforms(schema, f'ex:doc a ex:Document; ex:license [{type_statement} ex:category "license"] .') is expected + + +@pytest.mark.parametrize("category,expected", [("license", True), ("other", False)]) +@pytest.mark.parametrize("suffix", [None, "Shape"]) +@pytest.mark.parametrize("use_class_uri_names", [False, True]) +def test_range_expression_independent_of_shape_names( + schema: dict[str, Any], category: str, expected: bool, suffix: str | None, use_class_uri_names: bool +) -> None: + """The anonymous shape of a range expression works with either naming mode and a suffix.""" + data = f'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "{category}"] .' + assert conforms(schema, data, suffix=suffix, use_class_uri_names=use_class_uri_names) is expected + + +def test_range_expression_does_not_constrain_every_instance_of_the_range(schema: dict[str, Any]) -> None: + """Only the values of the constrained slot get the extra requirement.""" + assert conforms( + schema, + 'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "license"] . ' + 'ex:other a ex:Resource; ex:category "other" .', + ) + + +@pytest.mark.parametrize("category,expected", [("license", True), ("other", False)]) +def test_range_expression_on_referenced_values(schema: dict[str, Any], category: str, expected: bool) -> None: + """In RDF, a value that is a reference is constrained by its description in the data graph.""" + schema["classes"]["Resource"]["slots"].append("id") + schema["slots"]["id"] = {"identifier": True, "range": "uriorcurie"} + schema["slots"]["license"]["inlined"] = False + data = f'ex:doc a ex:Document; ex:license ex:resource . ex:resource a ex:Resource; ex:category "{category}" .' + assert conforms(schema, data) is expected + + +def test_range_expression_uses_the_condition_range(schema: dict[str, Any]) -> None: + """A range expression in a condition with a ``range`` resolves its conditions in that range class.""" + schema["classes"]["SpecialResource"] = {"is_a": "Resource", "attributes": {"grade": {"range": "integer"}}} + del schema["classes"]["Document"]["slot_usage"] + expression = {"slot_conditions": {"grade": {"minimum_value": 2}}} + condition = {"range": "SpecialResource", "range_expression": expression} + schema["classes"]["Document"]["all_of"] = [{"slot_conditions": {"license": condition}}] + data = "ex:doc a ex:Document; ex:license [a ex:Resource, ex:SpecialResource; ex:grade {}] ." + assert conforms(schema, data.format(2)) + assert not conforms(schema, data.format(1)) + + +@pytest.mark.parametrize("category,expected", [("license", True), ("other", False)]) +def test_cli_enforces_range_expression(schema: dict[str, Any], tmp_path: Path, category: str, expected: bool) -> None: + """The command line generates the same shapes, without an option.""" + path = tmp_path / "context.yaml" + path.write_text(yaml.safe_dump(schema)) + result = CliRunner().invoke(cli, [str(path), "--non-closed"]) + assert result.exit_code == 0, result.output + data = PREFIX + f'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "{category}"] .' + report = validate( + data_graph=data, data_graph_format="turtle", shacl_graph=result.output, shacl_graph_format="turtle" + ) + assert report[0] is expected + + +@pytest.mark.parametrize("category,expected", [("license", True), ("other", False)]) +def test_range_expression_on_imported_slots( + schema: dict[str, Any], tmp_path: Path, category: str, expected: bool +) -> None: + """With imported shapes excluded, a condition still resolves the IRI of an imported slot.""" + imported = { + "id": "https://example.org/imported", + "name": "imported", + "default_prefix": "imp", + "prefixes": {"imp": "https://example.org/imported/"}, + "classes": {"Resource": {"class_uri": "imp:Resource", "slots": ["category"]}}, + "slots": {"category": {"slot_uri": "imp:category", "range": "string"}}, + } + (tmp_path / "imported.yaml").write_text(yaml.safe_dump(imported)) + del schema["classes"]["Resource"] + del schema["slots"]["category"] + schema["imports"].append("imported") + path = tmp_path / "context.yaml" + path.write_text(yaml.safe_dump(schema)) + shapes = ShaclGenerator(str(path), closed=False, exclude_imports=True).serialize() + graph = Graph().parse(data=shapes, format="turtle") + assert (None, SH.targetClass, URIRef("https://example.org/imported/Resource")) not in graph + data = ( + f"{PREFIX} @prefix imp: . " + f'ex:doc a ex:Document; ex:license [a imp:Resource; imp:category "{category}"] .' + ) + report = validate(data_graph=data, data_graph_format="turtle", shacl_graph=graph, meta_shacl=True) + assert report[0] is expected + + +@pytest.mark.parametrize( + "condition,reason", + [ + ({"unit": {"ucum_code": "m"}}, "'unit' in the condition on slot 'category'"), + ({"equals_expression": "{code} * 2"}, "'equals_expression' in the condition on slot 'category'"), + ], +) +def test_untranslatable_range_expression_skipped_with_warning( + schema: dict[str, Any], caplog: pytest.LogCaptureFixture, condition: dict[str, Any], reason: str +) -> None: + """A range expression SHACL cannot express is left out of the shapes of each class, with one warning.""" + expression = schema["classes"]["Document"]["slot_usage"]["license"]["range_expression"] + expression["slot_conditions"]["category"].update(condition) + schema["classes"]["SpecialDocument"] = {"is_a": "Document"} + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + assert conforms(schema, 'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "other"] .') + assert [r.getMessage() for r in caplog.records] == [ + f"Slot 'license': range_expression is not translated to SHACL, because it uses {reason} " + "(in the shapes of 'Document', 'SpecialDocument')." + ] + + +def test_range_expression_with_conditions_on_values_that_are_not_class_instances( + schema: dict[str, Any], caplog: pytest.LogCaptureFixture +) -> None: + """Values of a type or an enum have no slots for a condition to constrain.""" + del schema["classes"]["Document"]["slot_usage"] + schema["slots"]["category"]["range_expression"] = {"slot_conditions": {"code": {"required": True}}} + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + assert verdicts(schema, {"license": {"category": "other"}}) == (True, True) + assert [r.getMessage() for r in caplog.records] == [ + "Slot 'category': range_expression is not translated to SHACL, because it uses a condition on slot 'code' " + "of values that are not class instances (in the shapes of 'Resource')." + ] + + +def test_untranslatable_range_expression_in_class_expression_skips_it( + schema: dict[str, Any], caplog: pytest.LogCaptureFixture +) -> None: + """A class-level expression whose range expression cannot be translated is skipped as a whole.""" + del schema["classes"]["Document"]["slot_usage"] + expression = {"slot_conditions": {"category": {"equals_string": "license", "unit": {"ucum_code": "m"}}}} + schema["classes"]["Document"]["all_of"] = [{"slot_conditions": {"license": {"range_expression": expression}}}] + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + assert conforms(schema, 'ex:doc a ex:Document; ex:license [a ex:Resource; ex:category "other"] .') + assert [r.getMessage() for r in caplog.records] == [ + "Class 'Document': all_of is not translated to SHACL, because it uses " + "'unit' in the condition on slot 'category'." + ] diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index 020f756d71..e07e35e443 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -4333,7 +4333,7 @@ def test_class_expression_range_with_identifier_is_an_iri(): required: true""", ), "is_a-not-a-class": ( - "not a class", + "not a known class, type or enum", """ - is_a: Undefined - slot_conditions: From 1372936a073661da70850d57f0a74e6ba63a13f6 Mon Sep 17 00:00:00 2001 From: jdsika Date: Wed, 7 Oct 2026 23:09:30 +0200 Subject: [PATCH 04/17] feat(generators): enforce has_member alike in SHACL and JSON Schema has_member asks for at least one value of a multivalued slot satisfying an expression. gen-shacl ignored it. gen-json-schema translated it as contains, which an absent slot passes but an empty one fails, applied only the value operators of the expression, and ignored it in class-level expressions. Both generators now read a member as one value, which must satisfy the value operators, range and range_expression of the expression. A slot without a value has no member, so has_member decides presence. In SHACL it is an sh:qualifiedValueShape with sh:qualifiedMinCount 1; in JSON Schema, contains on a required slot, or some value of a dict inlined by key. In a slot condition the member takes the form of the condition. SHACL skips a has_member it cannot express with a warning, and both generators warn about one on a slot that is not multivalued. Signed-off-by: jdsika --- docs/generators/json-schema.rst | 3 + docs/generators/shacl.rst | 35 +- docs/schemas/advanced.md | 4 +- .../generators/common/class_expression.py | 24 +- .../src/linkml/generators/jsonschemagen.py | 106 +++++- .../linkml/src/linkml/generators/shaclgen.py | 193 ++++++++--- .../test_boolean_slot_compliance.py | 3 + ...chema_multivalued_element_constraints.yaml | 44 --- .../jsonschema_multivalued_has_member.yaml | 80 +++++ .../linkml/test_generators/test_has_member.py | 306 ++++++++++++++++++ .../test_generators/test_jsonschemagen.py | 6 + .../test_generators/test_range_expression.py | 36 +-- tests/linkml/test_generators/test_shaclgen.py | 20 +- tests/linkml/utils/generator_agreement.py | 54 ++++ 14 files changed, 750 insertions(+), 164 deletions(-) create mode 100644 tests/linkml/test_generators/input/jsonschema_multivalued_has_member.yaml create mode 100644 tests/linkml/test_generators/test_has_member.py create mode 100644 tests/linkml/utils/generator_agreement.py diff --git a/docs/generators/json-schema.rst b/docs/generators/json-schema.rst index 9298dd6316..b2eb6955f7 100644 --- a/docs/generators/json-schema.rst +++ b/docs/generators/json-schema.rst @@ -185,6 +185,9 @@ constrains each of its values in the same way, with its conditions on the slots of the value. For a slot inlined as a dict these are the values of the dict, except in its compact forms ``{id: value}`` and ``{id: null}``, which the expression can't reach. +A multivalued slot's `has_member `_ becomes +``contains``, and the slot is required, because a slot without a value has no +member; on a slot that isn't multivalued it is ignored, with a warning. The SHACL generator reads them the same way. Inlining diff --git a/docs/generators/shacl.rst b/docs/generators/shacl.rst index cd5c3565db..4c5031cc35 100644 --- a/docs/generators/shacl.rst +++ b/docs/generators/shacl.rst @@ -123,7 +123,9 @@ Each entry of ``slot_conditions`` gives an ``meaning`` where they have one; * ``range`` gives the same class, type or enum constraint as a slot's range; * ``range_expression`` gives an ``sh:node``, as described under - `Range Expressions`_. + `Range Expressions`_; +* ``has_member`` gives an ``sh:qualifiedValueShape``, as described under + `Members`_. A shape may have at most one value of ``sh:minInclusive``, ``sh:maxInclusive`` or ``sh:in``, and of ``sh:pattern``, whose component also takes ``sh:flags`` @@ -173,7 +175,7 @@ slot's range. An operator whose members use anything else is skipped as a whole and logged as a warning, because leaving out one member would change what the operator -admits. That covers, for example, ``has_member``, ``equals_expression`` or a +admits. That covers, for example, ``all_members``, ``equals_expression`` or a slot-level ``any_of`` inside a slot condition, a condition on a name that is not a slot, a condition on the identifier slot (the node's IRI rather than a property), and ``equals_string`` on a slot whose range does not hold strings. @@ -237,6 +239,35 @@ graph, as ``sh:class`` is. The JSON Schema generator can't check it, because there the value is just an identifier. +Members +^^^^^^^ + +A multivalued slot's `has_member `__ asks +for at least one value that satisfies an expression. It becomes an +``sh:qualifiedValueShape`` with ``sh:qualifiedMinCount 1`` (`SHACL §4.7.3 +`__), so +the other values need only satisfy the slot's own constraints. The member +expression constrains one value: its value operators, ``range`` and +``range_expression`` apply. + +.. code-block:: yaml + + slots: + tags: + range: string + multivalued: true + has_member: + equals_string: reviewed + +This accepts ``[reviewed, pending]`` and rejects ``[pending]``. A slot without a +value, absent or empty, has no member, so it fails ``has_member``, as the JSON +Schema generator also reads it. In a slot condition of a class-level expression, +``has_member`` therefore decides whether the slot may be absent, and the member +takes the form of the condition. Several conditions with ``has_member`` may be +satisfied by the same value. A ``has_member`` on a slot that isn't multivalued, +or one that uses anything else, is skipped and logged as a warning. + + Command Line ^^^^^^^^^^^^ diff --git a/docs/schemas/advanced.md b/docs/schemas/advanced.md index 052b01ed21..4e6f901450 100644 --- a/docs/schemas/advanced.md +++ b/docs/schemas/advanced.md @@ -70,7 +70,7 @@ At the class level, each member of `any_of`, `all_of`, `exactly_one_of` and `non A slot condition needs a meaning when its slot is absent. The JSON Schema and SHACL generators read it as an SQL `CHECK` constraint reads a condition on a null value: an instance is invalid only when an expression is definitely false. -- A condition that decides whether its slot may be absent is true or false as usual. Such a condition sets `value_presence: PRESENT` or `ABSENT`, `required: true`, a minimum or exact cardinality of at least 1, or a maximum or exact cardinality of 0. +- A condition that decides whether its slot may be absent is true or false as usual. Such a condition sets `value_presence: PRESENT` or `ABSENT`, `required: true`, a minimum or exact cardinality of at least 1, `has_member`, or a maximum or exact cardinality of 0. - Any other condition is *unknown* when its slot is absent. - `any_of`, `all_of` and `none_of` combine these as "or", "and" and "not". An unknown member doesn't make `any_of` true, nor `none_of` false. - `exactly_one_of` holds when exactly one member is definitely true. @@ -94,6 +94,8 @@ A `Sample` without `status` is valid: the condition is unknown, so `none_of` isn A slot's [range_expression](https://w3id.org/linkml/range_expression) constrains each value of the slot in the same way: its conditions constrain the slots of the value, as induced for the slot's range class. In a slot condition, a `range_expression` is part of the condition, so under `none_of` it must hold definitely. +A slot's [has_member](https://w3id.org/linkml/has_member) asks for at least one value satisfying its expression. A slot without a value, absent or empty, has no member, so it fails `has_member`. + [Rules](#rules) are read differently: their preconditions require their slots, and so do their postconditions unless the rule is `open_world`. ### Unions as ranges diff --git a/packages/linkml/src/linkml/generators/common/class_expression.py b/packages/linkml/src/linkml/generators/common/class_expression.py index ba4bd04860..89693c5c9d 100644 --- a/packages/linkml/src/linkml/generators/common/class_expression.py +++ b/packages/linkml/src/linkml/generators/common/class_expression.py @@ -43,10 +43,10 @@ def states_presence(condition: SlotDefinition) -> bool: Such a condition is definitely true or false for an absent slot: ``value_presence: PRESENT`` or ``ABSENT``; ``required: true``, unless ``value_presence`` overrides it; a minimum or exact cardinality of at least - 1, which an absent slot fails; and a maximum or exact cardinality of 0, - which it satisfies. Other bounds, ``required: false`` and ``UNCOMMITTED`` - leave absence open, so adding one never changes the verdict on an absent - slot. + 1, and ``has_member``, which asks for at least one value, all of which an + absent slot fails; and a maximum or exact cardinality of 0, which it + satisfies. Other bounds, ``required: false`` and ``UNCOMMITTED`` leave + absence open, so adding one never changes the verdict on an absent slot. >>> states_presence(SlotDefinition("label", equals_string="A")) False @@ -58,12 +58,16 @@ def states_presence(condition: SlotDefinition) -> bool: True >>> states_presence(SlotDefinition("tags", minimum_cardinality=1)) True + >>> states_presence(SlotDefinition("tags", has_member={"equals_string": "A"})) + True """ if condition.value_presence is not None: if condition.value_presence in (_PRESENT, _ABSENT): return True elif condition.required: return True + if condition.has_member is not None: + return True lower = (condition.minimum_cardinality, condition.exact_cardinality) upper = (condition.maximum_cardinality, condition.exact_cardinality) return any(bound is not None and bound >= 1 for bound in lower) or 0 in upper @@ -72,9 +76,13 @@ def states_presence(condition: SlotDefinition) -> bool: def value_bounds(condition: SlotDefinition, definite: bool) -> tuple[int, int | None]: """The least and the greatest number of values *condition* allows its slot. - ``value_presence`` takes precedence over ``required``. In the "definitely - true" form (*definite*), a condition that doesn't state presence also - requires the slot. The greatest number is ``None`` when unbounded. + ``value_presence`` takes precedence over ``required``, and ``has_member`` + asks for at least one value. In the "definitely true" form (*definite*), a + condition that doesn't state presence also requires the slot. The greatest + number is ``None`` when unbounded. + + >>> value_bounds(SlotDefinition("tags", has_member={"equals_string": "A"}), definite=False) + (1, None) """ lower, upper = [0], [] if condition.value_presence is not None: @@ -84,6 +92,8 @@ def value_bounds(condition: SlotDefinition, definite: bool) -> tuple[int, int | upper.append(0) elif condition.required: lower.append(1) + if condition.has_member is not None: + lower.append(1) if definite and not states_presence(condition): lower.append(1) for bound, target in ( diff --git a/packages/linkml/src/linkml/generators/jsonschemagen.py b/packages/linkml/src/linkml/generators/jsonschemagen.py index 7c6ad48a7e..b8a6210d2f 100644 --- a/packages/linkml/src/linkml/generators/jsonschemagen.py +++ b/packages/linkml/src/linkml/generators/jsonschemagen.py @@ -691,7 +691,8 @@ def get_subschema_for_class_expression( Each slot condition constrains the slot as induced for *cls*, so ``slot_usage`` applies. Its values must satisfy its value operators, its ``range`` and its ``range_expression``, the latter in the form of - *expr*; the number of values must lie within + *expr*, and one of them its ``has_member``; the number of values must + lie within :func:`~linkml.generators.common.class_expression.value_bounds`, where an empty list or dict counts as no value. *definite* selects the form, as in :meth:`get_subschema_for_class_operator`. *cls* is ``None`` for values @@ -705,27 +706,26 @@ def get_subschema_for_class_expression( prop_name = self._curie(slot) else: prop_name = self.aliased_slot_name(slot) - value_condition = condition - if condition.range_expression is not None: - # translated below, in the form of *expr* - value_condition = copy(condition) - value_condition.range_expression = None - values = self.get_subschema_for_slot(value_condition, omit_type=True, include_null=False) - if condition.range_expression is not None: - expression = self.get_subschema_for_range_expression( - condition.range or slot.range, condition.range_expression, definite - ) - values = JsonSchema({"allOf": [values, expression]}) if values else expression + values = self._value_subschema(slot, condition, definite) lower, upper = value_bounds(condition, definite) + member = None + if condition.has_member is not None: + if slot.multivalued: + member = self.get_subschema_for_member(slot, condition.has_member, definite) + else: + self._warn_has_member_on_single_valued(slot) if slot.multivalued and self._inlined_as_dict(slot): # the values of a slot inlined as a dict are the values of the dict prop = JsonSchema({"type": "object", "additionalProperties": values}) prop.add_keyword("minProperties", lower or None) prop.add_keyword("maxProperties", upper) + if member is not None: + prop.setdefault("allOf", []).append(self._some_dict_value(member)) elif slot.multivalued: prop = JsonSchema.array_of(values, include_null=False, required=False) prop.add_keyword("minItems", lower or None) prop.add_keyword("maxItems", upper) + prop.add_keyword("contains", member) else: prop = values if condition.range is not None: @@ -779,6 +779,69 @@ def _inlined_as_dict(self, slot: SlotDefinition) -> bool: and get_range_associated_slots(self.schemaview, slot.range)[0] is not None ) + def _value_subschema( + self, + slot: SlotDefinition | AnonymousSlotExpression, + condition: SlotDefinition | AnonymousSlotExpression, + definite: bool, + ) -> JsonSchema: + """The subschema each value of *slot* must satisfy under the value operators of *condition*. + + These include its ``range_expression``, in the form *definite*, but not + its ``range``, which depends on how the values of *slot* are represented. + """ + value_condition = condition + if condition.range_expression is not None: + # translated below, in the form *definite* + value_condition = copy(condition) + value_condition.range_expression = None + values = self.get_subschema_for_slot(value_condition, omit_type=True, include_null=False) + if condition.range_expression is not None: + expression = self.get_subschema_for_range_expression( + condition.range or slot.range, condition.range_expression, definite + ) + values = JsonSchema({"allOf": [values, expression]}) if values else expression + return values + + def get_subschema_for_member( + self, slot: SlotDefinition | AnonymousSlotExpression, member: AnonymousSlotExpression, definite: bool + ) -> JsonSchema: + """The subschema a value of multivalued *slot* satisfies as a member of its ``has_member`` *member*. + + A member is one value, which must satisfy the value operators, the + ``range`` and the ``range_expression`` of *member*, the latter in the + form *definite*. + """ + conjuncts = [self._value_subschema(slot, member, definite)] + if member.range is not None: + conjuncts.append(self._value_type_subschema(slot, member.range)) + conjuncts = [conjunct for conjunct in conjuncts if conjunct] + if len(conjuncts) == 1: + return conjuncts[0] + return JsonSchema({"allOf": conjuncts} if conjuncts else {}) + + def _value_type_subschema(self, slot: SlotDefinition | AnonymousSlotExpression, value_range: str) -> JsonSchema: + """The subschema of one value of range *value_range* as multivalued *slot* represents its values.""" + collection = self.get_subschema_for_slot( + AnonymousSlotExpression( + range=value_range, multivalued=True, inlined=slot.inlined, inlined_as_list=slot.inlined_as_list + ), + include_null=False, + ) + return collection["additionalProperties"] if collection.is_object else collection["items"] + + @staticmethod + def _warn_has_member_on_single_valued(slot: SlotDefinition | AnonymousSlotExpression) -> None: + """Warn that the ``has_member`` of *slot* is ignored: a member is a value of a multivalued slot.""" + logger.warning( + "Slot %r: has_member is not translated to JSON Schema, because the slot is not multivalued.", slot.name + ) + + @staticmethod + def _some_dict_value(member: JsonSchema) -> JsonSchema: + """The subschema of a dict with a value satisfying *member*: not every value fails it.""" + return JsonSchema({"not": {"additionalProperties": {"not": member}}}) + def _class_expression_slot(self, cls: ClassDefinition | None, slot_name: str) -> SlotDefinition | None: """The slot a condition of a class expression of *cls* names, as induced for *cls*, if any.""" sv = self.schemaview @@ -1144,19 +1207,26 @@ def get_subschema_for_slot( if prop.is_array: all_element_constraints = self.get_value_constraints_for_slot(slot.all_members) - any_element_constraints = self.get_value_constraints_for_slot(slot.has_member) prop.add_keyword("minItems", slot.minimum_cardinality) prop.add_keyword("maxItems", slot.maximum_cardinality) prop["items"].update(own_constraints) prop["items"].update(all_element_constraints) - if any_element_constraints: - prop["contains"] = any_element_constraints + if slot.has_member is not None: + # At least one value, so not null. + prop["type"] = "array" + prop["contains"] = self.get_subschema_for_member(slot, slot.has_member, definite=False) elif inlined_as_dict: # The values of the slot are the values of the dict. if own_constraints: prop["additionalProperties"] = JsonSchema({"allOf": [prop["additionalProperties"], own_constraints]}) + if slot.has_member is not None: + member = self.get_subschema_for_member(slot, slot.has_member, definite=False) + prop.setdefault("allOf", []).append(self._some_dict_value(member)) else: prop.update(own_constraints) + if slot.has_member is not None and not omit_type: + # with omit_type, the caller translates has_member for the slot the condition names + self._warn_has_member_on_single_valued(slot) if prop.is_object: prop.add_keyword("minProperties", slot.minimum_cardinality) @@ -1216,7 +1286,11 @@ def handle_class_slot(self, subschema: JsonSchema, cls: ClassDefinition, slot: S slot = self.before_generate_class_slot(slot, cls, self.schemaview) class_id_slot = self.schemaview.get_identifier_slot(cls.name, use_key=True) value_required = ( - slot.required or slot == class_id_slot or slot.value_presence == PresenceEnum(PresenceEnum.PRESENT) + slot.required + or slot == class_id_slot + or slot.value_presence == PresenceEnum(PresenceEnum.PRESENT) + # a member is a value + or (slot.multivalued and slot.has_member is not None) ) value_disallowed = slot.value_presence == PresenceEnum(PresenceEnum.ABSENT) diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 11e670e5c5..4149bd07c5 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -224,7 +224,7 @@ def as_graph(self) -> Graph: self._class_expressions_added: set[tuple[URIRef, str, str]] = set() self._class_expression_problems: dict[tuple[str, str, str], list[str]] = {} - self._range_expression_problems: dict[tuple[str, str], list[str]] = {} + self._slot_expression_problems: dict[tuple[str, str, str], list[str]] = {} for c in sv.all_classes(imports=not self.exclude_imports).values(): def shape_pv(p, v): @@ -409,6 +409,8 @@ def branch_pv(p, v): prop_pv(SH.pattern, Literal(s.pattern)) if s.range_expression is not None: self._add_range_expression(g, prop_pv, c, s, s.range, s.range_expression) + if s.has_member is not None: + self._add_member(g, prop_pv, c, s) if s.annotations and self.include_annotations: self._add_annotations(prop_pv, s) @@ -423,7 +425,7 @@ def branch_pv(p, v): self._add_rules(g, class_uri_with_suffix, c) self._report_class_expression_problems() - self._report_range_expression_problems() + self._report_slot_expression_problems() return g LINKML_ANY_URI = "https://w3id.org/linkml/Any" @@ -458,7 +460,9 @@ def branch_pv(p, v): "range_expression", } ) - _SLOT_CONDITION_FIELDS = _SLOT_CONDITION_PRESENCE_FIELDS | _SLOT_CONDITION_VALUE_FIELDS + # has_member, which asks for at least one value satisfying a value expression, + # does both. + _SLOT_CONDITION_FIELDS = _SLOT_CONDITION_PRESENCE_FIELDS | _SLOT_CONDITION_VALUE_FIELDS | {"has_member"} # Parameters a shape may have at most one value of: sh:minInclusive and # sh:maxInclusive (SHACL §4.3), sh:in (§4.8.3), sh:datatype and sh:nodeKind @@ -646,15 +650,7 @@ def _slot_condition_shape( "definitely true" form with *definite*, otherwise its "not false" form.""" slot = self._condition_slot(cls, slot_name) pnode = BNode() - repeated = [] - - def prop_pv(p, v): - if v is None: - return - if p in self._SINGLE_VALUE_PARAMETERS and (pnode, p, None) in g: - repeated.append((p, v)) - else: - g.add((pnode, p, v)) + prop_pv, finish = self._shape_parameters(g, pnode) prop_pv(SH.path, URIRef(self._slot_iri(slot))) if condition.title is not None: @@ -668,13 +664,62 @@ def prop_pv(p, v): if upper is not None: prop_pv(SH.maxCount, Literal(upper)) + self._add_value_constraints(g, prop_pv, condition, condition.range or slot.range, definite) + if condition.has_member is not None: + prop_pv(SH.qualifiedValueShape, self._member_shape(g, slot, condition.has_member, definite)) + prop_pv(SH.qualifiedMinCount, Literal(1)) + finish() + return pnode + + def _shape_parameters(self, g: Graph, node: BNode) -> tuple[Callable, Callable[[], None]]: + """A function adding a parameter to the shape *node*, and one to call once all are added. + + A shape may have at most one value of each of :attr:`_SINGLE_VALUE_PARAMETERS`. + A further value goes into an ``sh:and`` member shape of its own, where it + applies to the same values, as it would on *node* itself. + """ + repeated = [] + + def add(p, v): + if v is None: + return + if p in self._SINGLE_VALUE_PARAMETERS and (node, p, None) in g: + repeated.append((p, v)) + else: + g.add((node, p, v)) + + def finish(): + if repeated: + members = [] + for p, v in repeated: + member = BNode() + g.add((member, p, v)) + members.append(member) + and_node = BNode() + Collection(g, and_node, members) + g.add((node, SH["and"], and_node)) + + return add, finish + + def _add_value_constraints( + self, + g: Graph, + pv: Callable, + condition: SlotDefinition | AnonymousSlotExpression, + value_range: ElementName | None, + definite: bool, + ) -> None: + """Add the constraints that *condition* places on each value of range *value_range*. + + These are the value operators, ``range`` and ``range_expression``, the + latter in its "definitely true" form with *definite*. + """ if condition.minimum_value is not None: - prop_pv(SH.minInclusive, Literal(condition.minimum_value)) + pv(SH.minInclusive, Literal(condition.minimum_value)) if condition.maximum_value is not None: - prop_pv(SH.maxInclusive, Literal(condition.maximum_value)) + pv(SH.maxInclusive, Literal(condition.maximum_value)) if condition.pattern is not None: - prop_pv(SH.pattern, Literal(condition.pattern)) - value_range = condition.range or slot.range + pv(SH.pattern, Literal(condition.pattern)) for values in ( [condition.equals_string] if condition.equals_string is not None else [], condition.equals_string_in, @@ -682,29 +727,31 @@ def prop_pv(p, v): if values: in_node = BNode() Collection(g, in_node, self._string_value_terms(value_range, values)) - prop_pv(SH["in"], in_node) + pv(SH["in"], in_node) if condition.equals_number is not None: # A value comparison, unlike the slot loop's sh:hasValue: 5 matches 5.0, # and like every other value constraint in a condition it holds when the # slot is absent. - prop_pv(SH.minInclusive, Literal(condition.equals_number)) - prop_pv(SH.maxInclusive, Literal(condition.equals_number)) + pv(SH.minInclusive, Literal(condition.equals_number)) + pv(SH.maxInclusive, Literal(condition.equals_number)) if condition.range is not None: - self._add_range(g, prop_pv, condition.range) + self._add_range(g, pv, condition.range) if condition.range_expression is not None: - prop_pv(SH.node, self._range_expression_shape(g, value_range, condition.range_expression, definite)) - if repeated: - # Each repeated parameter in a member shape of its own: all of them hold - # for every value, as they would on the property shape itself. - members = [] - for p, v in repeated: - member = BNode() - g.add((member, p, v)) - members.append(member) - and_node = BNode() - Collection(g, and_node, members) - g.add((pnode, SH["and"], and_node)) - return pnode + pv(SH.node, self._range_expression_shape(g, value_range, condition.range_expression, definite)) + + def _member_shape(self, g: Graph, slot: SlotDefinition, member: AnonymousSlotExpression, definite: bool) -> BNode: + """Build the node shape a value of *slot* must conform to as a member of its ``has_member``. + + ``sh:qualifiedValueShape`` with ``sh:qualifiedMinCount 1`` (SHACL §4.7.3) + then asks for at least one such value, while the other values need only + satisfy the slot's own constraints. *definite* selects the form of a + ``range_expression`` of *member*. + """ + node = BNode() + pv, finish = self._shape_parameters(g, node) + self._add_value_constraints(g, pv, member, member.range or slot.range, definite) + finish() + return node def _string_value_terms(self, r: ElementName | None, values: list[str]) -> list[URIRef | Literal]: """The RDF terms of the ``equals_string`` / ``equals_string_in`` *values* of a slot with range *r*. @@ -780,17 +827,13 @@ def _untranslatable(self, cls: ClassDefinition | None, expr: AnonymousClassExpre if slot.identifier: # An identifier is the node's IRI, not a property arc. return f"a condition on the identifier slot '{slot_name}'" - if condition.range is not None and not self._is_known_range(condition.range): - return f"the unknown range '{condition.range}' in the condition on slot '{slot_name}'" - value_range = condition.range or slot.range - if condition.range_expression is not None: - reason = self._untranslatable(self._value_class(value_range), condition.range_expression) + reason = self._value_untranslatable(condition, slot, f"the condition on slot '{slot_name}'") + if reason is not None: + return reason + if condition.has_member is not None: + reason = self._member_untranslatable(slot, condition.has_member) if reason is not None: return reason - if (condition.equals_string is not None or condition.equals_string_in) and not self._is_string_range( - value_range - ): - return f"equals_string on slot '{slot_name}', whose range '{value_range}' does not hold strings" for operator in self._CLASS_EXPRESSION_OPERATORS: for member in getattr(expr, operator) or []: reason = self._untranslatable(cls, member) @@ -798,6 +841,38 @@ def _untranslatable(self, cls: ClassDefinition | None, expr: AnonymousClassExpre return reason return None + def _value_untranslatable( + self, condition: SlotDefinition | AnonymousSlotExpression, slot: SlotDefinition, where: str + ) -> str | None: + """Return what in the value constraints of *condition* on *slot* cannot be translated, or ``None``. + + *where* names *condition* in the explanation. + """ + if condition.range is not None and not self._is_known_range(condition.range): + return f"the unknown range '{condition.range}' in {where}" + value_range = condition.range or slot.range + if condition.range_expression is not None: + reason = self._untranslatable(self._value_class(value_range), condition.range_expression) + if reason is not None: + return reason + if (condition.equals_string is not None or condition.equals_string_in) and not self._is_string_range( + value_range + ): + return f"equals_string in {where}, whose range '{value_range}' does not hold strings" + return None + + def _member_untranslatable(self, slot: SlotDefinition, member: AnonymousSlotExpression) -> str | None: + """Return what in the ``has_member`` *member* of *slot* cannot be translated, or ``None``. + + A member is one value, so only value constraints apply to it. + """ + if not slot.multivalued: + return f"has_member on slot '{slot.name}', which is not multivalued" + unknown = self._set_operator_fields(member) - self._SLOT_CONDITION_VALUE_FIELDS + if unknown: + return f"'{sorted(unknown)[0]}' in the has_member of slot '{slot.name}'" + return self._value_untranslatable(member, slot, f"the has_member of slot '{slot.name}'") + def _value_class(self, value_range: ElementName | None) -> ClassDefinition | None: """The class whose instances a range *value_range* holds, or ``None`` for any other range.""" sv = self.schemaview @@ -833,19 +908,37 @@ def _add_range_expression( """ reason = self._untranslatable(self._value_class(value_range), expression) if reason is not None: - classes = self._range_expression_problems.setdefault((slot.name, reason), []) - if cls.name not in classes: - classes.append(cls.name) + self._slot_expression_problem(cls, slot, "range_expression", reason) return prop_pv(SH.node, self._range_expression_shape(g, value_range, expression, definite=False)) - def _report_range_expression_problems(self) -> None: - """Warn once about each ``range_expression`` of a slot that is not translated, - naming the class shapes it is missing from.""" - for (slot_name, reason), classes in self._range_expression_problems.items(): + def _add_member(self, g: Graph, prop_pv: Callable, cls: ClassDefinition, slot: SlotDefinition) -> None: + """Ask for at least one value of *slot*, in the shape of *cls*, satisfying its ``has_member``. + + Having no value, an absent slot fails it. A ``has_member`` that cannot be + translated is skipped with a warning. + """ + reason = self._member_untranslatable(slot, slot.has_member) + if reason is not None: + self._slot_expression_problem(cls, slot, "has_member", reason) + return + prop_pv(SH.qualifiedValueShape, self._member_shape(g, slot, slot.has_member, definite=False)) + prop_pv(SH.qualifiedMinCount, Literal(1)) + + def _slot_expression_problem(self, cls: ClassDefinition, slot: SlotDefinition, field: str, reason: str) -> None: + """Record that *field* of *slot* is left out of the shape of *cls* because of *reason*.""" + classes = self._slot_expression_problems.setdefault((slot.name, field, reason), []) + if cls.name not in classes: + classes.append(cls.name) + + def _report_slot_expression_problems(self) -> None: + """Warn once about each ``range_expression`` or ``has_member`` of a slot that is + not translated, naming the class shapes it is missing from.""" + for (slot_name, field, reason), classes in self._slot_expression_problems.items(): logger.warning( - "Slot %r: range_expression is not translated to SHACL, because it uses %s (in the shapes of %s).", + "Slot %r: %s is not translated to SHACL, because it uses %s (in the shapes of %s).", slot_name, + field, reason, ", ".join(map(repr, classes)), ) diff --git a/tests/linkml/test_compliance/test_boolean_slot_compliance.py b/tests/linkml/test_compliance/test_boolean_slot_compliance.py index abe307c1fa..c62d9774d2 100644 --- a/tests/linkml/test_compliance/test_boolean_slot_compliance.py +++ b/tests/linkml/test_compliance/test_boolean_slot_compliance.py @@ -2823,6 +2823,9 @@ def test_membership(framework, name, quantification, expression, instance, is_va expected_behavior = ValidationBehavior.INCOMPLETE if framework in [SHACL, SQL_DDL_SQLITE, PANDERA_POLARS_CLASS]: expected_behavior = ValidationBehavior.INCOMPLETE + if framework == SHACL and quantification == "has_member" and s1_range != CLASS_D: + # sh:qualifiedValueShape; equals_string on a reference is not translated + expected_behavior = ValidationBehavior.IMPLEMENTS if framework == OWL and name == "all_obj_members_equals_string" and not is_valid: # This test case relies on punning, as s1 is used as both an OP and DP, # so we do not expect a DL-reasoner to be able to handle it diff --git a/tests/linkml/test_generators/input/jsonschema_multivalued_element_constraints.yaml b/tests/linkml/test_generators/input/jsonschema_multivalued_element_constraints.yaml index d9a4667ace..78b7c26703 100644 --- a/tests/linkml/test_generators/input/jsonschema_multivalued_element_constraints.yaml +++ b/tests/linkml/test_generators/input/jsonschema_multivalued_element_constraints.yaml @@ -24,25 +24,12 @@ schema: minimum_value: 2 maximum_value: 5 - int_list_with_has_member: - range: integer - multivalued: true - has_member: - minimum_value: 2 - maximum_value: 5 - string_list_with_all_members: range: string multivalued: true all_members: pattern: e.* - string_list_with_has_member: - range: string - multivalued: true - has_member: - pattern: e.* - classes: Test: tree_root: true @@ -50,9 +37,7 @@ schema: - int_list - string_list - int_list_with_all_members - - int_list_with_has_member - string_list_with_all_members - - string_list_with_has_member json_schema: properties: int_list: @@ -69,19 +54,10 @@ json_schema: minimum: 2 maximum: 5 type: array - int_list_with_has_member: - contains: - minimum: 2 - maximum: 5 - type: array string_list_with_all_members: items: pattern: e.* type: array - string_list_with_has_member: - contains: - pattern: e.* - type: array data_cases: - data: int_list: [2, 3, 4, 5] @@ -105,13 +81,6 @@ data_cases: - data: int_list_with_all_members: [1, 2, 3] error_message: Failed validating 'minimum' - - data: - int_list_with_has_member: [2, 3, 4, 5] - - data: - int_list_with_has_member: [0, 1, 2] - - data: - int_list_with_has_member: [6, 7, 8] - error_message: Failed validating 'contains' - data: string_list_with_all_members: - echo @@ -121,16 +90,3 @@ data_cases: - echo - foxtrot error_message: Failed validating 'pattern' - - data: - string_list_with_has_member: - - echo - - elephant - - data: - string_list_with_has_member: - - echo - - foxtrot - - data: - string_list_with_has_member: - - foxtrot - - golf - error_message: Failed validating 'contains' diff --git a/tests/linkml/test_generators/input/jsonschema_multivalued_has_member.yaml b/tests/linkml/test_generators/input/jsonschema_multivalued_has_member.yaml new file mode 100644 index 0000000000..ce52cf7a38 --- /dev/null +++ b/tests/linkml/test_generators/input/jsonschema_multivalued_has_member.yaml @@ -0,0 +1,80 @@ +schema: + id: http://example.org/jsonschema_multivalued_has_member + name: jsonschema_multivalued_has_member + + imports: + - https://w3id.org/linkml/types + + slots: + int_list_with_has_member: + range: integer + multivalued: true + has_member: + minimum_value: 2 + maximum_value: 5 + + string_list_with_has_member: + range: string + multivalued: true + has_member: + pattern: e.* + + classes: + Test: + tree_root: true + slots: + - int_list_with_has_member + - string_list_with_has_member +json_schema: + properties: + int_list_with_has_member: + contains: + minimum: 2 + maximum: 5 + type: array + string_list_with_has_member: + contains: + pattern: e.* + type: array + required: + - int_list_with_has_member + - string_list_with_has_member +data_cases: + - data: + int_list_with_has_member: [2, 3, 4, 5] + string_list_with_has_member: [echo] + - data: + int_list_with_has_member: [0, 1, 2] + string_list_with_has_member: [echo] + - data: + int_list_with_has_member: [6, 7, 8] + string_list_with_has_member: [echo] + error_message: Failed validating 'contains' + - data: + int_list_with_has_member: [2] + string_list_with_has_member: + - echo + - elephant + - data: + int_list_with_has_member: [2] + string_list_with_has_member: + - echo + - foxtrot + - data: + int_list_with_has_member: [2] + string_list_with_has_member: + - foxtrot + - golf + error_message: Failed validating 'contains' + # a member is a value, so a list without a value has no member + - data: + int_list_with_has_member: [] + string_list_with_has_member: [echo] + error_message: Failed validating 'contains' + - data: + int_list_with_has_member: null + string_list_with_has_member: [echo] + error_message: Failed validating 'type' + - data: + string_list_with_has_member: [echo] + error_message: Failed validating 'required' diff --git a/tests/linkml/test_generators/test_has_member.py b/tests/linkml/test_generators/test_has_member.py new file mode 100644 index 0000000000..575d8e909d --- /dev/null +++ b/tests/linkml/test_generators/test_has_member.py @@ -0,0 +1,306 @@ +"""``has_member`` asks for a value satisfying an expression alike in the JSON Schema and SHACL generators. + +A member is one value of a multivalued slot, so a slot without a value, absent +or empty, has no member. In a condition of a class-level expression, +``has_member`` therefore decides whether the slot may be absent, and its member +takes the form of the condition. The agreement tests validate the same +instance with both generated artifacts, loading it as RDF through the +generated JSON-LD context. +""" + +import json +import logging +from typing import Any + +import pytest +import yaml +from jsonschema import Draft201909Validator +from pyshacl import validate + +from linkml.generators.jsonschemagen import JsonSchemaGenerator +from linkml.generators.shaclgen import ShaclGenerator +from tests.linkml.utils import generator_agreement + +PREFIX = "@prefix ex: . " + + +@pytest.fixture +def schema() -> dict[str, Any]: + """A collection of tags, scores and inlined items.""" + return yaml.safe_load(""" +id: https://example.org/membership +name: membership +prefixes: + ex: https://example.org/membership/ + linkml: https://w3id.org/linkml/ +default_prefix: ex +imports: [linkml:types] +classes: + Collection: + tree_root: true + slots: [tags, scores, items] + Item: + slots: [status] +slots: + tags: {range: string, multivalued: true} + scores: {range: integer, multivalued: true} + items: {range: Item, multivalued: true, inlined_as_list: true} + status: {range: string} +""") + + +def verdicts(schema: dict[str, Any], instance: dict[str, Any]) -> tuple[bool, bool]: + """Whether the JSON Schema and the SHACL shapes generated for *schema* accept *instance* of ``Collection``.""" + return generator_agreement.verdicts(schema, instance, "Collection") + + +def conforms(schema: dict[str, Any], data: str) -> bool: + """Whether Turtle *data* conforms to the SHACL shapes generated for *schema*, which must pass meta-SHACL.""" + shapes = ShaclGenerator(yaml.safe_dump(schema), closed=False).serialize() + return validate( + data_graph=PREFIX + data, + data_graph_format="turtle", + shacl_graph=shapes, + shacl_graph_format="turtle", + meta_shacl=True, + )[0] + + +def json_valid(schema: dict[str, Any], instance: dict[str, Any]) -> bool: + """Whether the JSON Schema generated for *schema* accepts *instance* of ``Collection``.""" + json_schema = json.loads(JsonSchemaGenerator(yaml.safe_dump(schema), top_class="Collection").serialize()) + return Draft201909Validator(json_schema).is_valid(instance) + + +_READY = {"range_expression": {"slot_conditions": {"status": {"equals_string": "ready"}}}} + + +@pytest.mark.parametrize( + "slot,member,values,valid", + [ + pytest.param("tags", {"equals_string": "reviewed"}, ["reviewed", "pending"], True, id="match"), + pytest.param("tags", {"equals_string": "reviewed"}, ["pending"], False, id="no-match"), + # a slot without a value has no member + pytest.param("tags", {"equals_string": "reviewed"}, [], False, id="empty"), + pytest.param("tags", {"equals_string": "reviewed"}, None, False, id="absent"), + pytest.param("tags", {}, ["anything"], True, id="any-value"), + pytest.param("tags", {}, None, False, id="any-value-absent"), + pytest.param("tags", {"equals_string_in": ["a", "b"], "pattern": "^a"}, ["a", "b"], True, id="conjunction"), + pytest.param("tags", {"equals_string_in": ["a", "b"], "pattern": "^a"}, ["b"], False, id="conjunction-other"), + pytest.param("scores", {"minimum_value": 10}, [9, 10], True, id="bound"), + pytest.param("scores", {"minimum_value": 10}, [8, 9], False, id="bound-other"), + # two values of one SHACL parameter hold together + pytest.param("scores", {"minimum_value": 0, "equals_number": 0}, [-1, 0], True, id="repeated"), + pytest.param("scores", {"minimum_value": 0, "equals_number": 0}, [-1, 1], False, id="repeated-other"), + pytest.param("scores", {"minimum_value": 5, "equals_number": 3}, [3, 5], False, id="contradiction"), + pytest.param("scores", {"minimum_value": 5, "equals_number": 7}, [6], False, id="repeated-second"), + pytest.param("items", _READY, [{"status": "draft"}, {"status": "ready"}], True, id="range_expression"), + pytest.param("items", _READY, [{"status": "draft"}], False, id="range_expression-other"), + # a value whose status is unknown is not definitely no member + pytest.param("items", _READY, [{"status": "draft"}, {}], True, id="range_expression-unknown"), + ], +) +def test_slot_has_member( + schema: dict[str, Any], slot: str, member: dict[str, Any], values: list | None, valid: bool +) -> None: + """At least one value of the slot must satisfy the member expression; the others need not.""" + schema["slots"][slot]["has_member"] = member + assert verdicts(schema, {} if values is None else {slot: values}) == (valid, valid) + + +_REVIEWED = {"has_member": {"equals_string": "reviewed"}} + + +@pytest.mark.parametrize( + "expression,values,valid", + [ + pytest.param({"all_of": [{"slot_conditions": {"tags": _REVIEWED}}]}, None, False, id="all_of-absent"), + pytest.param({"all_of": [{"slot_conditions": {"tags": _REVIEWED}}]}, ["reviewed"], True, id="all_of"), + pytest.param({"all_of": [{"slot_conditions": {"tags": _REVIEWED}}]}, ["pending"], False, id="all_of-other"), + # has_member decides presence, so it is false, not unknown, for an absent slot + pytest.param({"none_of": [{"slot_conditions": {"tags": _REVIEWED}}]}, None, True, id="none_of-absent"), + pytest.param({"none_of": [{"slot_conditions": {"tags": _REVIEWED}}]}, ["pending"], True, id="none_of-other"), + pytest.param( + {"none_of": [{"slot_conditions": {"tags": _REVIEWED}}]}, ["pending", "reviewed"], False, id="none_of" + ), + pytest.param( + {"any_of": [{"slot_conditions": {"tags": _REVIEWED}}, {"slot_conditions": {"scores": {"required": True}}}]}, + None, + False, + id="any_of-absent", + ), + ], +) +def test_has_member_in_class_expression( + schema: dict[str, Any], expression: dict[str, Any], values: list | None, valid: bool +) -> None: + """In a condition of a class-level expression, ``has_member`` is false for a slot without a value.""" + schema["classes"]["Collection"].update(expression) + assert verdicts(schema, {} if values is None else {"tags": values}) == (valid, valid) + + +@pytest.mark.parametrize( + "items,valid", + [ + pytest.param([{}], True, id="unknown"), + pytest.param([{"status": "draft"}], True, id="other"), + pytest.param([{"status": "draft"}, {"status": "ready"}], False, id="match"), + ], +) +def test_member_under_negation_takes_the_definite_form(schema: dict[str, Any], items: list, valid: bool) -> None: + """Under ``none_of``, a member must definitely satisfy its expression to count.""" + schema["classes"]["Collection"]["none_of"] = [{"slot_conditions": {"items": {"has_member": _READY}}}] + assert verdicts(schema, {"items": items}) == (valid, valid) + + +@pytest.mark.parametrize("scores,valid", [([5], True), ([-1, 11], True), ([11], False), ([-1], False)]) +def test_members_of_two_conditions_may_be_one_value(schema: dict[str, Any], scores: list[int], valid: bool) -> None: + """Each ``has_member`` asks for a value of its own expression, which may or may not be the same value.""" + schema["classes"]["Collection"]["all_of"] = [ + {"slot_conditions": {"scores": {"has_member": {"minimum_value": 0}}}}, + {"slot_conditions": {"scores": {"has_member": {"maximum_value": 10}}}}, + ] + assert verdicts(schema, {"scores": scores}) == (valid, valid) + + +@pytest.mark.parametrize("statuses,valid", [(["draft", "ready"], True), (["draft"], False), ([], False)]) +@pytest.mark.parametrize("in_class_expression", [False, True]) +def test_has_member_on_values_inlined_as_dict( + schema: dict[str, Any], statuses: list[str], valid: bool, in_class_expression: bool +) -> None: + """The members of a slot inlined as a dict are the values of the dict.""" + schema["classes"]["Item"]["slots"].insert(0, "id") + schema["slots"]["id"] = {"identifier": True, "range": "string"} + del schema["slots"]["items"]["inlined_as_list"] + schema["slots"]["items"]["inlined"] = True + if in_class_expression: + schema["classes"]["Collection"]["all_of"] = [{"slot_conditions": {"items": {"has_member": _READY}}}] + else: + schema["slots"]["items"]["has_member"] = _READY + items = {f"i{n}": {"status": status} for n, status in enumerate(statuses)} + assert json_valid(schema, {"items": items}) is valid + data = "ex:c a ex:Collection" + "".join(f"; ex:items ex:{key}" for key in items) + " . " + data += "".join(f'ex:{key} a ex:Item; ex:status "{value["status"]}" . ' for key, value in items.items()) + assert conforms(schema, data) is valid + + +@pytest.mark.parametrize("target", ["Collection", "SpecialCollection"]) +@pytest.mark.parametrize( + "values,valid", + [(["reviewed", "pending"], True), (["pending"], False), (None, False)], + ids=["match", "other", "absent"], +) +def test_has_member_is_inherited(schema: dict[str, Any], target: str, values: list | None, valid: bool) -> None: + """A subclass keeps the ``has_member`` its parent places on a slot through ``slot_usage``.""" + schema["classes"]["Collection"]["slot_usage"] = {"tags": _REVIEWED} + schema["classes"]["SpecialCollection"] = {"is_a": "Collection"} + instance = {} if values is None else {"tags": values} + assert generator_agreement.verdicts(schema, instance, target) == (valid, valid) + + +def test_has_member_on_a_single_valued_slot_is_ignored_with_a_warning( + schema: dict[str, Any], caplog: pytest.LogCaptureFixture +) -> None: + """A member is a value of a multivalued slot, so both generators ignore ``has_member`` on a + single-valued slot and say so.""" + schema["slots"]["tags"] = {"range": "string", "has_member": {"equals_string": "reviewed"}} + with caplog.at_level(logging.WARNING): + assert verdicts(schema, {"tags": "pending"}) == (True, True) + assert sorted({r.getMessage() for r in caplog.records if r.name.startswith("linkml.generators.")}) == [ + "Slot 'tags': has_member is not translated to JSON Schema, because the slot is not multivalued.", + "Slot 'tags': has_member is not translated to SHACL, because it uses has_member on slot 'tags', " + "which is not multivalued (in the shapes of 'Collection').", + ] + + +@pytest.mark.parametrize("typed,valid", [(True, True), (False, False)]) +def test_member_range_needs_the_class(schema: dict[str, Any], typed: bool, valid: bool) -> None: + """A member with a class ``range`` must be an instance of that class.""" + schema["classes"]["Special"] = {"is_a": "Item", "slot_usage": {"status": {"required": True}}} + schema["slots"]["items"]["has_member"] = {"range": "Special"} + special = "ex:Special, " if typed else "" + assert conforms(schema, f'ex:c a ex:Collection; ex:items [a {special}ex:Item; ex:status "ready"] .') is valid + assert json_valid(schema, {"items": [{"status": "ready"}]}) + assert not json_valid(schema, {"items": [{}]}) + + +@pytest.mark.parametrize("dict_inlined", [False, True]) +def test_null_has_no_member(schema: dict[str, Any], dict_inlined: bool) -> None: + """A JSON ``null`` is no value, so it has no member, although the generator otherwise allows ``null``.""" + if dict_inlined: + schema["classes"]["Item"]["slots"].insert(0, "id") + schema["slots"]["id"] = {"identifier": True, "range": "string"} + del schema["slots"]["items"]["inlined_as_list"] + schema["slots"]["items"]["inlined"] = True + schema["slots"]["items"]["has_member"] = {} + assert not json_valid(schema, {"items": None}) + del schema["slots"]["items"]["has_member"] + assert json_valid(schema, {"items": None}) + + +@pytest.mark.parametrize("grade,valid", [(3, True), (1, False)]) +def test_member_range_resolves_its_range_expression(schema: dict[str, Any], grade: int, valid: bool) -> None: + """The ``range_expression`` of a member with a ``range`` constrains the slots of that range class.""" + schema["classes"]["Special"] = {"is_a": "Item", "attributes": {"grade": {"range": "integer"}}} + expression = {"slot_conditions": {"grade": {"minimum_value": 2}}} + schema["slots"]["items"]["has_member"] = {"range": "Special", "range_expression": expression} + assert conforms(schema, f"ex:c a ex:Collection; ex:items [a ex:Item, ex:Special; ex:grade {grade}] .") is valid + + +@pytest.mark.parametrize( + "slot_change,member,reason", + [ + pytest.param( + {"multivalued": False}, + {"equals_string": "reviewed"}, + "has_member on slot 'tags', which is not multivalued", + id="single-valued", + ), + pytest.param( + {}, + {"minimum_cardinality": 1}, + "'minimum_cardinality' in the has_member of slot 'tags'", + id="cardinality-of-a-value", + ), + pytest.param( + {}, + {"any_of": [{"equals_string": "reviewed"}]}, + "'any_of' in the has_member of slot 'tags'", + id="slot-operator", + ), + pytest.param( + {"range": "integer"}, + {"equals_string": "5"}, + "equals_string in the has_member of slot 'tags', whose range 'integer' does not hold strings", + id="string-on-integer", + ), + ], +) +def test_untranslatable_has_member_skipped_with_warning( + schema: dict[str, Any], + caplog: pytest.LogCaptureFixture, + slot_change: dict[str, Any], + member: dict[str, Any], + reason: str, +) -> None: + """A ``has_member`` SHACL cannot express is left out of the shapes, with one warning.""" + schema["slots"]["tags"].update(slot_change, has_member=member) + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + assert conforms(schema, "ex:c a ex:Collection .") + assert [r.getMessage() for r in caplog.records] == [ + f"Slot 'tags': has_member is not translated to SHACL, because it uses {reason} (in the shapes of 'Collection')." + ] + + +def test_untranslatable_has_member_in_class_expression_skips_it( + schema: dict[str, Any], caplog: pytest.LogCaptureFixture +) -> None: + """A class-level expression whose ``has_member`` cannot be translated is skipped as a whole.""" + member = {"any_of": [{"equals_string": "reviewed"}]} + schema["classes"]["Collection"]["all_of"] = [{"slot_conditions": {"tags": {"has_member": member}}}] + with caplog.at_level(logging.WARNING, logger="linkml.generators.shaclgen"): + assert conforms(schema, "ex:c a ex:Collection .") + assert [r.getMessage() for r in caplog.records] == [ + "Class 'Collection': all_of is not translated to SHACL, because it uses " + "'any_of' in the has_member of slot 'tags'." + ] diff --git a/tests/linkml/test_generators/test_jsonschemagen.py b/tests/linkml/test_generators/test_jsonschemagen.py index 1c0948243e..e7301194e7 100644 --- a/tests/linkml/test_generators/test_jsonschemagen.py +++ b/tests/linkml/test_generators/test_jsonschemagen.py @@ -490,6 +490,12 @@ def test_multivalued_element_constraints(subtests, input_path): external_file_test(subtests, input_path("jsonschema_multivalued_element_constraints.yaml")) +def test_multivalued_has_member(subtests, input_path): + """Tests that has_member asks for a value satisfying its expression, which an empty or absent list lacks.""" + + external_file_test(subtests, input_path("jsonschema_multivalued_has_member.yaml")) + + def test_collection_forms(subtests, input_path): """Tests that expanded, compact, and simple dicts can be validated""" diff --git a/tests/linkml/test_generators/test_range_expression.py b/tests/linkml/test_generators/test_range_expression.py index 71a3ee71bb..3f6681bc67 100644 --- a/tests/linkml/test_generators/test_range_expression.py +++ b/tests/linkml/test_generators/test_range_expression.py @@ -20,10 +20,9 @@ from rdflib import Graph, Namespace, URIRef from rdflib.namespace import SH -from linkml.generators.jsonldcontextgen import ContextGenerator from linkml.generators.jsonschemagen import JsonSchemaGenerator from linkml.generators.shaclgen import ShaclGenerator, cli -from linkml_runtime import SchemaView +from tests.linkml.utils import generator_agreement EX = Namespace("https://example.org/context/") PREFIX = f"@prefix ex: <{EX}> . " @@ -66,40 +65,9 @@ def schema() -> dict[str, Any]: """) -def _as_json_ld(sv: SchemaView, obj: dict[str, Any], class_name: str) -> dict[str, Any]: - """*obj* with an ``@type`` on it and on each inlined object: the range of its slot, unless it states one.""" - class_name = obj.get("@type", class_name) - typed = {**obj, "@type": class_name} - for slot in sv.class_induced_slots(class_name): - value = obj.get(slot.name) - if slot.range in sv.all_classes() and value is not None: - values = [_as_json_ld(sv, v, slot.range) for v in (value if isinstance(value, list) else [value])] - typed[slot.name] = values if isinstance(value, list) else values[0] - return typed - - -def _without_types(obj: Any) -> Any: - """*obj* without its ``@type`` keys.""" - if isinstance(obj, dict): - return {k: _without_types(v) for k, v in obj.items() if k != "@type"} - if isinstance(obj, list): - return [_without_types(v) for v in obj] - return obj - - def verdicts(schema: dict[str, Any], instance: dict[str, Any], target: str = "Document") -> tuple[bool, bool]: """Whether the JSON Schema and the SHACL shapes generated for *schema* accept *instance* of *target*.""" - text = yaml.safe_dump(schema) - json_schema = json.loads(JsonSchemaGenerator(text, top_class=target).serialize()) - context = json.loads(ContextGenerator(text).serialize())["@context"] - data = {"@context": context, **_as_json_ld(SchemaView(text), instance, target)} - shapes = ShaclGenerator(text, closed=False).serialize() - conforms, _, _ = validate( - Graph().parse(data=json.dumps(data), format="json-ld"), - shacl_graph=Graph().parse(data=shapes, format="turtle"), - meta_shacl=True, - ) - return Draft201909Validator(json_schema).is_valid(_without_types(instance)), conforms + return generator_agreement.verdicts(schema, instance, target) def conforms(schema: dict[str, Any], data: str, **options: Any) -> bool: diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index e07e35e443..17725b6180 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -3510,15 +3510,15 @@ def test_classes_sharing_a_class_uri_carry_an_inherited_expression_once(): pytest.param( None, {EX_CE.Parent: False, EX_CE.Child: False, EX_CE.GrandChild: False}, - "Class 'Parent': any_of is not translated to SHACL, because it uses 'has_member' in the condition on " + "Class 'Parent': any_of is not translated to SHACL, because it uses 'all_members' in the condition on " "slot 'a' (in the shapes of 'Parent', 'Child', 'GrandChild').", id="untranslatable-everywhere", ), pytest.param( {"a": {"range": "integer"}}, {EX_CE.Parent: True, EX_CE.Child: False, EX_CE.GrandChild: False}, - "Class 'Parent': any_of is not translated to SHACL, because it uses equals_string on slot 'a', whose " - "range 'integer' does not hold strings (in the shapes of 'Child', 'GrandChild').", + "Class 'Parent': any_of is not translated to SHACL, because it uses equals_string in the condition on " + "slot 'a', whose range 'integer' does not hold strings (in the shapes of 'Child', 'GrandChild').", id="untranslatable-in-subclasses", ), ], @@ -3528,7 +3528,7 @@ def test_untranslatable_inherited_class_expression_warned_once(caplog, child_usa shapes where it cannot, and reported once, naming them.""" schema = yaml.safe_load(_INHERITED_CLASS_EXPRESSION_SCHEMA) schema["classes"]["Parent"]["any_of"] = ( - [{"slot_conditions": {"a": {"has_member": {"equals_string": "x"}}}}] + [{"slot_conditions": {"a": {"all_members": {"equals_string": "x"}}}}] if child_usage is None else [{"slot_conditions": {"a": {"equals_string": "x"}}}] ) @@ -3560,7 +3560,7 @@ def test_class_expression_untranslatable_operator_skipped_with_warning(caplog): any_of: - slot_conditions: tags: - has_member: + all_members: equals_string: x - slot_conditions: a: @@ -3576,7 +3576,7 @@ def test_class_expression_untranslatable_operator_skipped_with_warning(caplog): assert (EX_CE.Thing, SH["or"], None) not in g assert any( - "any_of" in rec.message and "has_member" in rec.message and "tags" in rec.message for rec in caplog.records + "any_of" in rec.message and "all_members" in rec.message and "tags" in rec.message for rec in caplog.records ) # the translatable none_of is kept, and enforced shacl_ttl = g.serialize(format="turtle") @@ -4300,14 +4300,14 @@ def test_class_expression_range_with_identifier_is_an_iri(): _UNTRANSLATABLE_MEMBERS = { "untranslatable-second-member": ( - "has_member", + "all_members", """ - slot_conditions: a: required: true - slot_conditions: tags: - has_member: + all_members: equals_string: x""", ), "unknown-condition-range": ( @@ -4321,12 +4321,12 @@ def test_class_expression_range_with_identifier_is_an_iri(): required: true""", ), "untranslatable-nested-member": ( - "has_member", + "all_members", """ - all_of: - slot_conditions: tags: - has_member: + all_members: equals_string: x - slot_conditions: a: diff --git a/tests/linkml/utils/generator_agreement.py b/tests/linkml/utils/generator_agreement.py new file mode 100644 index 0000000000..92a885b0e8 --- /dev/null +++ b/tests/linkml/utils/generator_agreement.py @@ -0,0 +1,54 @@ +"""Validate the same instance against the JSON Schema and the SHACL shapes generated for a schema.""" + +import json +from typing import Any + +import yaml +from jsonschema import Draft201909Validator +from pyshacl import validate +from rdflib import Graph + +from linkml.generators.jsonldcontextgen import ContextGenerator +from linkml.generators.jsonschemagen import JsonSchemaGenerator +from linkml.generators.shaclgen import ShaclGenerator +from linkml_runtime import SchemaView + + +def as_json_ld(sv: SchemaView, obj: dict[str, Any], class_name: str) -> dict[str, Any]: + """*obj* with an ``@type`` on it and on each inlined object: the range of its slot, unless it states one.""" + class_name = obj.get("@type", class_name) + typed = {**obj, "@type": class_name} + for slot in sv.class_induced_slots(class_name): + value = obj.get(slot.name) + if slot.range in sv.all_classes() and value is not None: + values = [as_json_ld(sv, v, slot.range) for v in (value if isinstance(value, list) else [value])] + typed[slot.name] = values if isinstance(value, list) else values[0] + return typed + + +def without_types(obj: Any) -> Any: + """*obj* without its ``@type`` keys.""" + if isinstance(obj, dict): + return {k: without_types(v) for k, v in obj.items() if k != "@type"} + if isinstance(obj, list): + return [without_types(v) for v in obj] + return obj + + +def verdicts(schema: dict[str, Any], instance: dict[str, Any], target: str) -> tuple[bool, bool]: + """Whether the JSON Schema and the SHACL shapes generated for *schema* accept *instance* of *target*. + + For SHACL, *instance* is loaded as RDF through the generated JSON-LD context, + and the shapes must pass meta-SHACL. + """ + text = yaml.safe_dump(schema) + json_schema = json.loads(JsonSchemaGenerator(text, top_class=target).serialize()) + context = json.loads(ContextGenerator(text).serialize())["@context"] + data = {"@context": context, **as_json_ld(SchemaView(text), instance, target)} + shapes = ShaclGenerator(text, closed=False).serialize() + conforms, _, _ = validate( + Graph().parse(data=json.dumps(data), format="json-ld"), + shacl_graph=Graph().parse(data=shapes, format="turtle"), + meta_shacl=True, + ) + return Draft201909Validator(json_schema).is_valid(without_types(instance)), conforms From 91d0f4744ce68156b17b272c1223da398d0bf2e4 Mon Sep 17 00:00:00 2001 From: Rayene Messaoud Date: Fri, 11 Sep 2026 14:45:34 +0200 Subject: [PATCH 05/17] feat(gen-shacl): translate presence-implies-value rules to SHACL-SPARQL The rules-to-SHACL-SPARQL converter recognised two named patterns. This adds the presence-implies-value pattern: a precondition asserting `value_presence: PRESENT` on one slot, and a postcondition constraining another slot with `equals_string` or `equals_string_in`. It reads as "if the guard slot is present, the target slot must be present and hold one of the allowed values", and generalises the boolean guard to arbitrary enum values. The boolean guard becomes the case `equals_string: "true"` of this pattern on an xsd:boolean-typed flag. `equals_string` / `equals_string_in` compare strings. The metamodel defines them for slots of range string; enums and types with datatype xsd:string are treated alike, and a slot without a range takes the schema's default_range. On an xsd:boolean-typed slot each value must be a lexical form of xsd:boolean (XML Schema 1.1 Part 2 section 3.3.2.2: true, false, 1, 0) and denotes that boolean. A rule applying them to any other range is skipped with a warning: its RDF values are typed literals or IRIs that equal no string. Built-in type names resolve without importing linkml:types, as in the main slot loop; the built-in name of `curie` was misspelt there and is corrected. Values are compared with `=`, spelled out as the disjunction SPARQL 1.1 section 17.4.1.9 defines `IN` to be, so the RDF 1.1-identical plain and xsd:string literal forms both match and booleans compare by value; rdflib evaluates `IN` and triple-pattern constants by term identity. COALESCE turns the type error that RDFterm-equal raises for incomparable literals into false, so a spec-conformant engine reports such a value instead of dropping the row. Queries are DISTINCT and project ?path (and ?value where there is one), so each result names the property and the offending value. A value containing a backslash followed by "u" or "U" is split with CONCAT, since codepoint escapes are replaced before a query is parsed. The exclusive-value pattern takes the precondition `has_member: {equals_string: V}`, which states "V is one of the values". A bare `equals_string: V` applies to every value of the slot, as the specification applies a slot constraint to all members of a collection and as gen-json-schema reads it, so it no longer matches this pattern: such a rule is skipped with a warning naming has_member. The pattern gets the same range check and value comparison, and its maximum_cardinality > 1 form tests HAVING on the COUNT aggregate: expressions projected by SELECT are not visible to HAVING (SPARQL 1.1 section 18.2.4.2), so `HAVING (?count > N)` never held and the constraint never fired. A rule applies to all members of its class, so a class shape now carries the rules of its ancestors and mixins, as the JSON Schema generator applies them. Each rule is translated in the inheriting class's context, where slot_usage may refine the slots it references. Classes sharing a class_uri share one shape, which carries each rule once. With `open_world: true` "the postconditions may be omitted in instance data", so an absent target is no violation. Skip warnings name the rule by its position and its title, or its description without one. A rule whose conditions carry any operator beyond what a pattern translates, or name an unknown or identifier slot, is skipped rather than partially translated. Each problem with a rule is logged once as a warning naming the declaring class, the rule's position and the class shapes it affects. The metadata fields the operator accounting ignores are derived from the metamodel. Slot resolution goes through induced slots, so slot_usage overrides, slot_uri overrides and alias-form keys resolve to the IRI sh:path emits. Behaviour changes for existing schemas: an exclusive-value rule with a bare equals_string precondition is skipped (write has_member: {equals_string: V}); the boolean guard rejects a string "true" on an xsd:boolean flag and skips a flag whose type has another datatype IRI (main compared with str()); exclusive-value rules with maximum_cardinality > 1 now fire; inherited rules are now enforced on subclass shapes; problems with rules are logged at WARNING instead of DEBUG; rule results carry sh:resultPath and sh:value; sh:message follows the rule's in_language or default_language. Co-authored-by: jdsika Signed-off-by: jdsika --- .../generators/shacl/shacl_data_type.py | 2 +- .../linkml/src/linkml/generators/shaclgen.py | 612 ++++-- tests/linkml/test_generators/test_shaclgen.py | 1765 ++++++++++++++++- .../test_shacl_validation_plugin.py | 88 + 4 files changed, 2257 insertions(+), 210 deletions(-) diff --git a/packages/linkml/src/linkml/generators/shacl/shacl_data_type.py b/packages/linkml/src/linkml/generators/shacl/shacl_data_type.py index 1210a31be2..7ffd11d45d 100644 --- a/packages/linkml/src/linkml/generators/shacl/shacl_data_type.py +++ b/packages/linkml/src/linkml/generators/shacl/shacl_data_type.py @@ -20,7 +20,7 @@ class ShaclDataType(DataType, Enum): FLOAT = ("float", URIRef("http://www.w3.org/2001/XMLSchema#float")) DOUBLE = ("double", URIRef("http://www.w3.org/2001/XMLSchema#double")) URI = ("uri", URIRef("http://www.w3.org/2001/XMLSchema#anyURI")) - CURIE = ("curi", URIRef("http://www.w3.org/2001/XMLSchema#string")) + CURIE = ("curie", URIRef("http://www.w3.org/2001/XMLSchema#string")) NCNAME = ("ncname", URIRef("http://www.w3.org/2001/XMLSchema#string")) OBJECT_IDENTIFIER = ("objectidentifier", URIRef("http://www.w3.org/ns/shex#iri")) NODE_IDENTIFIER = ("nodeidentifier", URIRef("http://www.w3.org/ns/shex#nonLiteral")) diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 4149bd07c5..00fedfc779 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -1,5 +1,6 @@ import logging import os +import re import string from collections.abc import Callable from dataclasses import dataclass, fields @@ -22,6 +23,7 @@ AnonymousSlotExpression, ClassDefinition, ClassExpression, + ClassRule, Element, ElementName, PresenceEnum, @@ -70,6 +72,28 @@ def _validate_message_template(template: str) -> None: ) +@dataclass +class _RuleSite: + """A rule as it is translated for the shape of a class. + + *cls* is the class whose shape the rule constrains, *owner* the class + that declares it, and *index* its 1-based position among the owner's + rules; *cls* differs from *owner* when the rule is inherited. + """ + + cls: ClassDefinition + owner: str + index: int + rule: ClassRule + + def __str__(self) -> str: + text = f"Rule {self.index} of class {self.owner!r}" + label = self.rule.title or self.rule.description + if label: + text += f" ({label})" + return text + + @dataclass class ShaclGenerator(Generator): """Generate SHACL (Shapes Constraint Language) shapes from a LinkML schema. @@ -163,11 +187,17 @@ class ShaclGenerator(Generator): SHACL-SPARQL constraints (``sh:SPARQLConstraint``) on the corresponding ``sh:NodeShape``. Currently two patterns are recognised: - * *Boolean guard* — a precondition with ``value_presence: PRESENT`` on a - value slot and a postcondition with ``equals_string: "true"`` on a - boolean flag slot. - * *Exclusive value* — a precondition with ``equals_string`` on a slot and - a postcondition with ``maximum_cardinality`` on the *same* slot. + * *Presence implies value* — a precondition with ``value_presence: PRESENT`` + on a guard slot and a postcondition with ``equals_string`` or + ``equals_string_in`` on a target slot holding strings (an enum, or a type + with datatype ``xsd:string``) or booleans. The *boolean guard* is the + case ``equals_string: "true"`` on an ``xsd:boolean``-typed flag. + * *Exclusive value* — a precondition with ``has_member: {equals_string: V}`` + on a slot and a postcondition with ``maximum_cardinality`` on the *same* + slot. + + A rule that cannot be translated exactly is skipped with a warning. A + class shape carries the rules of the class's ancestors and mixins too. See `W3C SHACL §5 `_ and `linkml/linkml#2464 `_. @@ -214,6 +244,9 @@ def serialize(self, **args) -> str: def as_graph(self) -> Graph: sv = self.schemaview + # Problems with rules, collected while the rules are translated for + # every class that has them and reported once each at the end. + self._rule_problems: dict[tuple[str, int, str], tuple[str, list[str]]] = {} g = Graph() g.bind("sh", SH) @@ -426,6 +459,7 @@ def branch_pv(p, v): self._report_class_expression_problems() self._report_slot_expression_problems() + self._report_rule_problems() return g LINKML_ANY_URI = "https://w3id.org/linkml/Any" @@ -958,227 +992,430 @@ def _is_known_range(self, r: ElementName) -> bool: # ------------------------------------------------------------------- def _add_rules(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: - """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. + """Emit ``sh:sparql`` constraints from the LinkML ``rules`` that apply to *cls*. Each recognised rule is converted into an ``sh:SPARQLConstraint`` - attached to *shape_uri*. Unrecognised patterns are logged at - ``DEBUG`` level and silently skipped. - - Currently recognised patterns: - - * **Boolean guard** — a *precondition* with - ``value_presence: PRESENT`` on a value slot and a *postcondition* - with ``equals_string: "true"`` on a boolean flag slot. - - * **Exclusive value** — a *precondition* with ``equals_string`` on - a slot and a *postcondition* with ``maximum_cardinality`` on the - *same* slot. Enforces that when a specific value is present in a - multivalued slot, the total number of values must not exceed the - given cardinality (typically 1 for mutual exclusion). + attached to *shape_uri*. Currently recognised patterns: + + * **Presence implies value** — a *precondition* with + ``value_presence: PRESENT`` on a guard slot and a *postcondition* + with ``equals_string`` or ``equals_string_in`` on a target slot. + When the guard is present, the target must be present and hold one + of the allowed values. The target must hold strings (an enum, or a + type with datatype ``xsd:string``) or booleans (``xsd:boolean``). + The **boolean guard** is the case ``equals_string: "true"`` on a + boolean flag. + + * **Exclusive value** — a *precondition* with ``has_member: + {equals_string: V}`` on a slot and a *postcondition* with + ``maximum_cardinality`` on the *same* slot. When V is one of the + slot's values, the slot holds at most that many values (typically 1 + for mutual exclusion). A bare ``equals_string: V`` precondition + applies to every value of the slot (``05validation.md``), so it is + not this pattern. + + Every emitted constraint translates its rule exactly. A rule that + cannot be translated exactly is skipped with a warning naming the + reason: an operator combination outside these patterns, a condition on + an unknown or identifier slot, a value the target's range cannot hold + (see :meth:`_value_terms`), or ``bidirectional``. Of a rule with ``elseconditions`` the forward + (if/then) direction is emitted, exactly, and a warning reports the + else branch as not enforced. + + A rule applies to "all members of this class" (metamodel ``rules``), + so the shape of a class carries the rules of its ancestors and mixins + as well, as it carries their slots, and as the JSON Schema generator + applies them. Each rule is translated in the context of *cls*, where + ``slot_usage`` may refine a slot the rule references. See `W3C SHACL §5 `_. """ - if not cls.rules: - return - sv = self.schemaview - for rule in cls.rules: - if getattr(rule, "deactivated", False): - continue - - if getattr(rule, "bidirectional", False): - logger.warning( - "Rule in class %r has bidirectional=true; " - "SHACL-SPARQL generation does not support bidirectional rules. " - "Skipping this rule entirely.", - cls.name, - ) - continue - - if getattr(rule, "open_world", False): - logger.warning( - "Rule in class %r has open_world=true; " - "SHACL operates under closed-world assumption. " - "The constraint is emitted but may not match open-world semantics.", - cls.name, - ) - - if getattr(rule, "elseconditions", None): - logger.warning( - "Rule in class %r has elseconditions; " - "only the forward (if/then) branch is emitted as sh:sparql. " - "The else branch cannot be represented in SHACL-SPARQL.", - cls.name, - ) + for owner in sv.class_ancestors(cls.name): + for index, rule in enumerate(sv.get_class(owner).rules, start=1): + self._add_rule(g, shape_uri, _RuleSite(cls, owner, index, rule)) - sparql_query = self._rule_to_sparql(sv, cls, rule) - if sparql_query is None: - logger.debug( - "Skipping unsupported rule pattern in class %r: %s", - cls.name, - getattr(rule, "description", "(no description)"), - ) - continue + def _add_rule(self, g: Graph, shape_uri: URIRef, site: _RuleSite) -> None: + """Emit the ``sh:SPARQLConstraint`` of the rule at *site* on *shape_uri*, unless it is skipped.""" + rule = site.rule + if rule.deactivated: + return + if rule.bidirectional: + self._skip_rule(site, "bidirectional rules are not supported") + return + sparql_query = self._rule_to_sparql(site) + if sparql_query is None: + return + if rule.elseconditions is not None: + self._warn_rule( + site, "its elseconditions are not enforced; SHACL-SPARQL generation emits the if/then direction only" + ) + message = Literal(rule.description, lang=self._resolve_language(rule)) if rule.description else None + if self._has_sparql_constraint(g, shape_uri, Literal(sparql_query), message): + return - constraint = BNode() - g.add((shape_uri, SH.sparql, constraint)) - g.add((constraint, RDF.type, SH.SPARQLConstraint)) + constraint = BNode() + g.add((shape_uri, SH.sparql, constraint)) + g.add((constraint, RDF.type, SH.SPARQLConstraint)) + if message is not None: + g.add((constraint, SH.message, message)) + g.add((constraint, SH.select, Literal(sparql_query))) - message = getattr(rule, "description", None) - if message: - g.add((constraint, SH.message, Literal(message))) + @staticmethod + def _has_sparql_constraint(g: Graph, shape_uri: URIRef, query: Literal, message: Literal | None) -> bool: + """Whether the shape *shape_uri* already carries the constraint with *query* and *message*. - g.add((constraint, SH.select, Literal(sparql_query))) + Classes that share a ``class_uri`` share one shape when shapes are named + by ``class_uri`` (the default), so a rule they all inherit would + otherwise be added once per class and reported twice. + """ + return any( + (constraint, SH.select, query) in g and set(g.objects(constraint, SH.message)) == {message} - {None} + for constraint in g.objects(shape_uri, SH.sparql) + ) - def _rule_to_sparql(self, sv, cls: ClassDefinition, rule) -> str | None: - """Convert a ``ClassRule`` to a SPARQL SELECT query string. + def _warn_rule(self, site: _RuleSite, problem: str) -> None: + """Record *problem* with the rule at *site* for the shape of ``site.cls``. - Returns ``None`` when the rule does not match any supported pattern. + A rule is translated once for every class that has it, declared or + inherited, so the same problem can arise several times; it is reported + once, by :meth:`_report_rule_problems`. """ - pre = getattr(rule, "preconditions", None) - post = getattr(rule, "postconditions", None) - if not pre or not post: + _, classes = self._rule_problems.setdefault((site.owner, site.index, problem), (str(site), [])) + if site.cls.name not in classes: + classes.append(site.cls.name) + + def _report_rule_problems(self) -> None: + """Log each recorded rule problem once, naming the class shapes it affects + unless that is only the class declaring the rule.""" + for (owner, _, problem), (rule, classes) in self._rule_problems.items(): + shapes = "" if classes == [owner] else f" (in the shapes of {', '.join(map(repr, classes))})" + logger.warning("%s: %s%s.", rule, problem, shapes) + + def _skip_rule(self, site: _RuleSite, reason: str) -> None: + """Log that the rule at *site* is not translated, and why.""" + self._warn_rule(site, f"skipped, because {reason}") + + # The lexical space of xsd:boolean and the value each lexical form maps to + # (XML Schema 1.1 Part 2 §3.3.2.2, ), + # as SPARQL boolean literals. + _XSD_BOOLEAN_LEXICAL = {"true": "true", "1": "true", "false": "false", "0": "false"} + + _NO_PATTERN = "its conditions match none of the translated patterns (presence implies value, exclusive value)" + + def _rule_to_sparql(self, site: _RuleSite) -> str | None: + """Translate the rule at *site* to a SPARQL SELECT query. + + Returns ``None``, after a warning naming the reason, when the rule + matches no supported pattern exactly. Each pattern requires its + conditions to set **exactly** the operators it translates; a rule whose + pre/postconditions carry anything more (extra scalar operators, + expression-level ``any_of``/``all_of``/``none_of``/``exactly_one_of``, + ...) is skipped rather than partially translated, since dropping a term + would widen the precondition (false positives) or weaken the + postcondition (false negatives). + """ + pre, post = site.rule.preconditions, site.rule.postconditions + if pre is None or post is None: + self._skip_rule(site, "only rules with preconditions and postconditions are translated") + return None + for side, expression in (("preconditions", pre), ("postconditions", post)): + untranslated = self._set_operator_fields(expression) - {"slot_conditions"} + if untranslated: + self._skip_rule(site, f"its {side} use {', '.join(sorted(untranslated))}") + return None + if len(pre.slot_conditions) != 1 or len(post.slot_conditions) != 1: + self._skip_rule(site, self._NO_PATTERN) return None - pre_slots = getattr(pre, "slot_conditions", None) or {} - post_slots = getattr(post, "slot_conditions", None) or {} + ((pre_name, pre_cond),) = pre.slot_conditions.items() + ((post_name, post_cond),) = post.slot_conditions.items() + pre_slot = self._rule_condition_slot(site, pre_name) + post_slot = self._rule_condition_slot(site, post_name) if pre_slot is not None else None + if pre_slot is None or post_slot is None: + return None + pre_ops = self._set_operator_fields(pre_cond) + post_ops = self._set_operator_fields(post_cond) + + # Presence implies value: "if the guard is present, the target must be + # present and hold one of the allowed values". The boolean guard is + # the case `equals_string: "true"` on a boolean flag. + if ( + pre_ops == {"value_presence"} + and pre_cond.value_presence == PresenceEnum(PresenceEnum.PRESENT) + and post_ops in ({"equals_string"}, {"equals_string_in"}) + ): + values = [post_cond.equals_string] if post_ops == {"equals_string"} else list(post_cond.equals_string_in) + terms = self._value_terms(site, post_slot, values) + if terms is None: + return None + return self._build_presence_implies_value_sparql(pre_slot, post_slot, terms, bool(site.rule.open_world)) + + # Exclusive value: "if V is one of the values of slot X, then X has at + # most N values". + if post_ops == {"maximum_cardinality"} and pre_slot.name == post_slot.name: + if pre_ops == {"equals_string"}: + self._skip_rule( + site, + f"its precondition equals_string on slot {pre_slot.name!r} applies to every value of the slot; " + f"to state that one of the values equals it, use has_member: {{equals_string: ...}}", + ) + return None + exclusive = self._exclusive_value(pre_cond, pre_ops) + if exclusive is not None: + terms = self._value_terms(site, pre_slot, [exclusive]) + if terms is None: + return None + return self._build_exclusive_value_sparql(pre_slot, terms[0], int(post_cond.maximum_cardinality)) + + self._skip_rule(site, self._NO_PATTERN) + return None - # Pattern: boolean guard - # preconditions: exactly one slot with value_presence PRESENT - # postconditions: exactly one slot with equals_string "true" - if len(pre_slots) == 1 and len(post_slots) == 1: - pre_slot_name = next(iter(pre_slots)) - post_slot_name = next(iter(post_slots)) + def _exclusive_value(self, condition: SlotDefinition, operators: set[str]) -> str | None: + """The value V of an exclusive-value precondition, or ``None`` if *condition* is not one. - pre_cond = pre_slots[pre_slot_name] - post_cond = post_slots[post_slot_name] + Only ``has_member: {equals_string: V}`` states "V is one of the values" + (metamodel ``has_member``: "at least one member satisfying the + condition"). + """ + if operators == {"has_member"} and self._set_operator_fields(condition.has_member) == {"equals_string"}: + return condition.has_member.equals_string + return None - # Note: PresenceEnum.PRESENT is a PermissibleValue, but parsed schemas - # return PresenceEnum instances — wrapping ensures type-compatible comparison. - is_value_present = getattr(pre_cond, "value_presence", None) == PresenceEnum(PresenceEnum.PRESENT) - is_flag_true = getattr(post_cond, "equals_string", None) == "true" + def _rule_slot(self, cls: ClassDefinition, slot_name: str) -> SlotDefinition | None: + """Resolve a rule condition's slot key to the slot it names, or ``None`` + when no such slot exists. - if is_value_present and is_flag_true: - return self._build_boolean_guard_sparql(sv, cls, post_slot_name, pre_slot_name) + Resolution order mirrors ``sh:path`` in the main slot loop: the induced + (class-specific) slot when the key names one of the class's slots, then + the underscored alias form (a rule key ``my_slot`` for a slot named + ``my slot`` — SchemaView normalises names the same way elsewhere), then + the base slot. + """ + sv = self.schemaview + class_slot_names = sv.class_slots(cls.name) + if slot_name in class_slot_names: + return sv.induced_slot(slot_name, cls.name) + canonical = next((s for s in class_slot_names if underscore(s) == underscore(slot_name)), None) + if canonical is not None: + return sv.induced_slot(canonical, cls.name) + return sv.get_slot(slot_name) + + def _rule_condition_slot(self, site: _RuleSite, slot_name: str) -> SlotDefinition | None: + """The slot a condition of the rule at *site* names, or ``None`` (rule skipped) when it cannot be queried. + + An unknown name would make the query use a predicate no shape or data + uses. An identifier slot is the node's IRI, not a property arc (the + main slot loop does not require it either), so a condition on it can + never match. + """ + slot = self._rule_slot(site.cls, slot_name) + if slot is None: + self._skip_rule(site, f"its condition names {slot_name!r}, which is not a slot") + return None + if slot.identifier: + self._skip_rule( + site, f"its condition is on the identifier slot {slot_name!r}, which is the node IRI, not a property" + ) + return None + return slot - # Pattern: exclusive value - # preconditions: slot X has equals_string (a specific enum value) - # postconditions: same slot X has maximum_cardinality N - # Semantics: "If value V is present in slot X, then X has at most N values." - pre_equals = getattr(pre_cond, "equals_string", None) - post_max_card = getattr(post_cond, "maximum_cardinality", None) + def _slot_range(self, slot: SlotDefinition) -> ElementName | None: + """The range of *slot*, defaulting to the schema's ``default_range`` as an induced slot does.""" + return slot.range or self.schemaview.schema.default_range - if pre_equals is not None and post_max_card is not None and pre_slot_name == post_slot_name: - return self._build_exclusive_value_sparql(sv, cls, pre_slot_name, pre_equals, int(post_max_card)) + def _slot_iri(self, slot: SlotDefinition) -> str: + """The full IRI of *slot*, exactly as ``sh:path`` in the main slot loop renders it. + An induced slot carries its ``slot_usage`` overrides, so an overridden + ``slot_uri`` yields the same IRI as ``sh:path``; otherwise the query + would use a property the data never uses and never fire. + """ + sv = self.schemaview + if slot.name in sv.element_by_schema_map(): + return sv.get_uri(slot, expand=True) + return sv.expand_curie(f"{sv.schema.default_prefix}:{underscore(slot.name)}") + + def _value_terms(self, site: _RuleSite, slot: SlotDefinition, values: list[str]) -> list[str] | None: + """The SPARQL terms of the equals_string(_in) *values* on *slot*, or ``None`` (rule skipped). + + The metamodel defines both operators for slots of range ``string``. + Enums, whose values are strings in instance data, and types with + datatype ``xsd:string`` are treated alike (:meth:`_string_value_term`). + A slot whose range is ``xsd:boolean``-typed holds booleans, as the + boolean guard compares them: each value must be a lexical form of + ``xsd:boolean`` and becomes the boolean it denotes, compared by value. + On any other range the RDF data holds typed literals + (``"3"^^xsd:integer``) or IRIs that no string equals, so the + constraint would report conforming data or never fire. + """ + r = self._slot_range(slot) + if self._is_string_range(r): + return [self._string_value_term(site, slot, value) for value in values] + if self._type_uri(r) == str(XSD.boolean): + terms = [self._XSD_BOOLEAN_LEXICAL.get(value) for value in values] + if None not in terms: + return terms + self._skip_rule( + site, + f"equals_string(_in) on the boolean slot {slot.name!r} with {values!r}, " + "which are not all xsd:boolean lexical forms (true, false, 1, 0)", + ) + return None + self._skip_rule( + site, + f"equals_string(_in) on slot {slot.name!r}, whose range {r!r} is neither an enum " + "nor a type with datatype xsd:string or xsd:boolean", + ) return None - def _build_boolean_guard_sparql(self, sv, cls: ClassDefinition, flag_slot_name: str, value_slot_name: str) -> str: - """Build a SPARQL SELECT query for the boolean-guard pattern. + def _string_value_term(self, site: _RuleSite, slot: SlotDefinition, value: str) -> str: + """The SPARQL term of the ``equals_string`` *value* of string-valued *slot*. - The query detects violations where the value property is present - but the boolean flag is absent or not ``true``. + A permissible value of an enum range is rendered as :meth:`_add_enum` + renders it, as the IRI of its ``meaning`` where it has one; anything + else is a string literal. A value that is not a permissible value of + an enum with static permissible values is reported: no valid value of + the slot can equal it. + """ + sv = self.schemaview + r = self._slot_range(slot) + if r in sv.all_enums(): + permissible_values = sv.get_enum(r).permissible_values + pv = permissible_values.get(value) + if pv is not None and pv.meaning: + return f"<{sv.expand_curie(pv.meaning)}>" + if pv is None and permissible_values: + self._warn_rule( + site, + f"it compares slot {slot.name!r} with {value!r}, which is not a permissible value of enum {r!r}", + ) + return self._sparql_string_literal(value) + + @staticmethod + def _sparql_string_literal(value: str) -> str: + """Render *value* as a SPARQL expression for that string. + + The characters the grammar forbids raw are escaped (`SPARQL 1.1 §19.7 + `_), so a value + with a double quote, backslash or newline neither breaks the + ``sh:select`` query nor injects into it. Codepoint escapes are replaced + before parsing (`§19.2 `_), + so an escaped backslash followed by ``u`` / ``U`` would be read as one: + the literal is split there and rejoined with ``CONCAT``. + """ + escaped = ( + str(value) + .replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + parts = re.split(r"(?<=\\)(?=[uU])", escaped) + if len(parts) == 1: + return f'"{escaped}"' + return "CONCAT(" + ", ".join(f'"{part}"' for part in parts) + ")" + + @staticmethod + def _sparql_is_one_of(var: str, terms: list[str]) -> str: + """A SPARQL expression that is true when *var* equals one of *terms*, and false otherwise. + + The test is spelled out as ``var = t1 || var = t2 || ...``, which is how + `SPARQL 1.1 §17.4.1.9 `_ + defines ``IN``. The ``=`` operator compares values, so a plain literal + also matches its RDF 1.1-identical ``xsd:string`` form (`RDF 1.1 + Concepts §3.3 `_), + which JSON-LD produces under ``"@type": "xsd:string"`` coercion; some + engines, rdflib among them, match ``IN`` and triple-pattern constants by + term identity and miss that form. ``COALESCE`` (§17.4.1.3) turns the + type error that ``RDFterm-equal`` (§17.4.1.7) raises for incomparable + literals into false: such a value equals none of *terms*, and an + unguarded error would drop the row and hide the violation. An unbound + *var* is false as well. + """ + disjunction = " || ".join(f"{var} = {term}" for term in terms) + return f"COALESCE( {disjunction}, false )" + + def _build_presence_implies_value_sparql( + self, guard: SlotDefinition, target: SlotDefinition, terms: list[str], open_world: bool + ) -> str: + """Build the SPARQL SELECT query of the presence-implies-value pattern. + + A focus node violates the rule when the *guard* slot is present and the + *target* slot is absent or holds a value that equals none of the SPARQL + *terms* (see :meth:`_sparql_is_one_of`). The boolean guard is this + pattern with the boolean ``true`` as the only term. With *open_world*, + "the postconditions may be omitted in instance data" (metamodel + ``open_world``), so an absent target is no violation. Conforms to `SHACL §5.3.1 - `_: - ``$this`` is pre-bound to each focus node. + `_ (``$this`` + is pre-bound to each focus node) and projects ``?path`` and ``?value`` + as result variables (`§5.3.2 + `_), so each + result names the target property and the offending value. """ - flag_uri = self._slot_uri(sv, flag_slot_name, cls) - value_uri = self._slot_uri(sv, value_slot_name, cls) - + guard_iri = self._slot_iri(guard) + target_iri = self._slot_iri(target) + is_allowed = self._sparql_is_one_of("?value", terms) + if open_world: + target_pattern = f"$this <{target_iri}> ?value ." + violation = f"!{is_allowed}" + else: + target_pattern = f"OPTIONAL {{ $this <{target_iri}> ?value . }}" + violation = f"!BOUND(?value) || !{is_allowed}" return ( - f"SELECT $this WHERE {{\n" - f" OPTIONAL {{ $this <{flag_uri}> ?flag . }}\n" - f" OPTIONAL {{ $this <{value_uri}> ?value . }}\n" - f" FILTER (\n" - f' ( !BOUND(?flag) || str(?flag) != "true" ) &&\n' - f" BOUND(?value)\n" - f" )\n" + f"SELECT DISTINCT $this (<{target_iri}> AS ?path) ?value WHERE {{\n" + f" $this <{guard_iri}> ?guard .\n" + f" {target_pattern}\n" + f" FILTER ( {violation} )\n" f"}}" ) - def _build_exclusive_value_sparql( - self, - sv, - cls: ClassDefinition, - slot_name: str, - value_name: str, - max_card: int, - ) -> str | None: - """Build a SPARQL SELECT query for the exclusive-value pattern. - - Detects violations where a specific value is present in a multivalued - slot but the total number of values exceeds *max_card*. - - For the common case ``max_card == 1``, the query checks whether the - exclusive value coexists with any other value (simple existence test). - For ``max_card > 1``, a subquery counts all values and checks against - the limit. + def _build_exclusive_value_sparql(self, slot: SlotDefinition, term: str, max_card: int) -> str: + """Build the SPARQL SELECT query of the exclusive-value pattern. - The exclusive value is resolved to its full IRI via the slot's enum - ``meaning`` field. If the slot is not an enum or the value has no - ``meaning``, the value is compared as a plain literal. + A focus node violates the rule when *slot* holds a value equal to the + SPARQL *term* (see :meth:`_sparql_is_one_of`) and more than *max_card* + values in total. For ``max_card == 1`` the query reports each other + value that coexists with the exclusive one. For ``max_card > 1`` a + subquery counts all values, testing ``HAVING`` on the ``COUNT`` + aggregate itself: expressions projected by ``SELECT`` are not visible + to ``HAVING`` (`SPARQL 1.1 §18.2.4.2 + `_), so a + projected alias would be unbound there and the constraint never fire. Conforms to `SHACL §5.3.1 - `_: - ``$this`` is pre-bound to each focus node. + `_ and + projects ``?path`` (and, for ``max_card == 1``, the coexisting value as + ``?value``) as result variables (`§5.3.2 + `_). """ - slot_uri = self._slot_uri(sv, slot_name, cls) - value_ref = self._resolve_enum_value_ref(sv, slot_name, value_name) - + iri = self._slot_iri(slot) + is_exclusive = self._sparql_is_one_of("?exclusive", [term]) if max_card == 1: + is_other_exclusive = self._sparql_is_one_of("?other", [term]) return ( - f"SELECT $this WHERE {{\n" - f" $this <{slot_uri}> {value_ref} .\n" - f" $this <{slot_uri}> ?other .\n" - f" FILTER (?other != {value_ref})\n" + f"SELECT DISTINCT $this (<{iri}> AS ?path) (?other AS ?value) WHERE {{\n" + f" $this <{iri}> ?exclusive .\n" + f" $this <{iri}> ?other .\n" + f" FILTER ( {is_exclusive} && !{is_other_exclusive} )\n" f"}}" ) - return ( - f"SELECT $this WHERE {{\n" - f" $this <{slot_uri}> {value_ref} .\n" + f"SELECT DISTINCT $this (<{iri}> AS ?path) WHERE {{\n" + f" $this <{iri}> ?exclusive .\n" + f" FILTER ( {is_exclusive} )\n" f" {{\n" - f" SELECT $this (COUNT(?val) AS ?count)\n" - f" WHERE {{ $this <{slot_uri}> ?val . }}\n" + f" SELECT $this\n" + f" WHERE {{ $this <{iri}> ?item . }}\n" f" GROUP BY $this\n" - f" HAVING (?count > {max_card})\n" + f" HAVING ( COUNT(?item) > {max_card} )\n" f" }}\n" f"}}" ) - def _resolve_enum_value_ref(self, sv, slot_name: str, value_name: str) -> str: - """Resolve an enum value name to a SPARQL term (IRI or literal). - - Looks up the slot's range as an enum, finds the permissible value - matching *value_name*, and returns its ``meaning`` as a full IRI - wrapped in angle brackets. Falls back to a quoted literal if the - slot is not an enum or the value lacks a ``meaning``. - """ - slot = sv.get_slot(slot_name) - if slot: - range_name = slot.range - if range_name and range_name in sv.all_enums(): - enum = sv.get_enum(range_name) - pv = enum.permissible_values.get(value_name) - if pv and pv.meaning: - iri = sv.expand_curie(pv.meaning) - return f"<{iri}>" - return f'"{value_name}"' - - def _slot_uri(self, sv, slot_name: str, cls: ClassDefinition) -> str: - """Resolve a slot name to a full IRI string for use in SPARQL queries. - - Mirrors the resolution logic used for ``sh:path`` in the main slot loop: - prefer ``sv.get_uri()`` for slots registered in the schema map, fall - back to ``default_prefix:underscored_name``. - """ - slot = sv.get_slot(slot_name) - if slot and slot_name in sv.element_by_schema_map(): - return sv.get_uri(slot, expand=True) - pfx = sv.schema.default_prefix - return sv.expand_curie(f"{pfx}:{underscore(slot_name)}") - def _add_class(self, func: Callable, r: ElementName) -> None: """Add a class/shape constraint for range class *r*. @@ -1212,18 +1449,6 @@ def _add_class(self, func: Callable, r: ElementName) -> None: range_ref += self.suffix func(SH["node"], URIRef(range_ref)) - def _slot_iri(self, slot: SlotDefinition) -> str: - """The full IRI of *slot*, exactly as ``sh:path`` in the main slot loop renders it. - - An induced slot carries its ``slot_usage`` overrides, so an overridden - ``slot_uri`` yields the same IRI as ``sh:path``; otherwise the query - would use a property the data never uses and never fire. - """ - sv = self.schemaview - if slot.name in sv.element_by_schema_map(): - return sv.get_uri(slot, expand=True) - return sv.expand_curie(f"{sv.schema.default_prefix}:{underscore(slot.name)}") - def _add_range(self, g: Graph, func: Callable, r: ElementName) -> None: """Add the value-type constraint for range *r*: a class, type, enum or built-in datatype.""" sv = self.schemaview @@ -1496,8 +1721,9 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: show_default=True, help=( "Emit sh:sparql constraints from LinkML rules: blocks. " - "When enabled (default), recognised rule patterns (e.g. boolean-guard) " - "are translated into SHACL-SPARQL constraints on the corresponding " + "When enabled (default), recognised rule patterns (boolean-guard, " + "presence-implies-value, exclusive-value) are translated into " + "SHACL-SPARQL constraints on the corresponding " "sh:NodeShape. Use --no-emit-rules to suppress rule generation." ), ) diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index 17725b6180..40eb3c402c 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -1,5 +1,6 @@ import json import logging +import re from collections import Counter from typing import Any @@ -2111,7 +2112,7 @@ def test_rule_deactivated_skipped(): def test_rule_unsupported_pattern_skipped(): - """Unrecognised rule patterns are silently skipped (no sh:sparql emitted).""" + """Unrecognised rule patterns are skipped (no sh:sparql emitted).""" g = _parse_shacl(_UNSUPPORTED_RULE_SCHEMA_YAML) shape = URIRef("https://example.org/unsupported-test/TestClass") @@ -2426,7 +2427,7 @@ def test_rule_sparql_syntax_valid(): # =========================================================================== # # The "exclusive value" pattern translates a LinkML rule where: -# - preconditions: slot X has equals_string (a specific enum value name) +# - preconditions: slot X has has_member: {equals_string: V} # - postconditions: same slot X has maximum_cardinality N # # Semantics: "If value V is present in multivalued slot X, then X has at most @@ -2497,7 +2498,8 @@ def test_rule_sparql_syntax_valid(): preconditions: slot_conditions: edgeType: - equals_string: "EdgeNone" + has_member: + equals_string: "EdgeNone" postconditions: slot_conditions: edgeType: @@ -2513,7 +2515,8 @@ def test_rule_sparql_syntax_valid(): preconditions: slot_conditions: edgeType: - equals_string: "EdgeNone" + has_member: + equals_string: "EdgeNone" postconditions: slot_conditions: edgeType: @@ -2529,7 +2532,8 @@ def test_rule_sparql_syntax_valid(): preconditions: slot_conditions: priority: - equals_string: "High" + has_member: + equals_string: "High" postconditions: slot_conditions: priority: @@ -2546,7 +2550,8 @@ def test_rule_sparql_syntax_valid(): preconditions: slot_conditions: edgeType: - equals_string: "EdgeNone" + has_member: + equals_string: "EdgeNone" postconditions: slot_conditions: otherSlot: @@ -2588,13 +2593,14 @@ def test_exclusive_value_sparql_uses_enum_iri(): def test_exclusive_value_max_card_1_sparql_structure(): - """For maximum_cardinality: 1, SPARQL uses FILTER(?other != ). + """For maximum_cardinality: 1, SPARQL reports each ?other value coexisting with . The query pattern for N=1 is: - SELECT $this WHERE { - $this . + SELECT DISTINCT $this ( AS ?path) (?other AS ?value) WHERE { + $this ?exclusive . $this ?other . - FILTER (?other != ) + FILTER ( COALESCE( ?exclusive = , false ) && + !COALESCE( ?other = , false ) ) } This is more efficient than the COUNT-based approach for the common @@ -2619,13 +2625,14 @@ def test_exclusive_value_max_card_gt1_sparql_structure(): """For maximum_cardinality > 1, SPARQL uses COUNT-based subquery. The query pattern for N>1 is: - SELECT $this WHERE { - $this . + SELECT DISTINCT $this ( AS ?path) WHERE { + $this ?exclusive . + FILTER ( COALESCE( ?exclusive = , false ) ) { - SELECT $this (COUNT(?val) AS ?count) - WHERE { $this ?val . } + SELECT $this + WHERE { $this ?item . } GROUP BY $this - HAVING (?count > N) + HAVING ( COUNT(?item) > N ) } } """ @@ -2758,7 +2765,8 @@ def test_exclusive_value_coexists_with_boolean_guard(): preconditions: slot_conditions: status: - equals_string: "None" + has_member: + equals_string: "None" postconditions: slot_conditions: status: @@ -4501,3 +4509,1728 @@ def test_class_expression_none_of_presence_is_monotonic(extra, properties, expec ex:t a ex:Thing {";" if properties else ""} {properties} . """ assert _conforms(shacl_ttl, data) is expected + + +# =========================================================================== +# Presence-implies-value pattern tests (enum guard) +# =========================================================================== +# +# The "presence implies value" pattern generalises the boolean guard to +# enum-valued targets. It translates a LinkML rule where: +# - preconditions: a guard slot has value_presence: PRESENT +# - postconditions: a target slot has equals_string (single required value) +# or equals_string_in (a set of acceptable values) +# +# Semantics: "If the guard slot is present, the target slot must be present +# and hold one of the allowed values", e.g. "if a document has a signature, +# its status must be Published". +# +# References: +# - W3C SHACL §5 +# - W3C SHACL §5.3.1 +# - W3C SHACL §5.3.2 +# =========================================================================== + +_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML = """ +id: https://example.org/presence-implies-value +name: presence_implies_value_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/presence-implies-value/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + StatusEnum: + permissible_values: + Draft: + meaning: ex:Draft + Reviewed: + meaning: ex:Reviewed + Approved: + meaning: ex:Approved + Published: + meaning: ex:Published + + ModeEnum: + permissible_values: + Auto: + description: Automatic mode (no meaning IRI). + Manual: + description: Manual mode (no meaning IRI). + +slots: + status: + range: StatusEnum + slot_uri: ex:status + signature: + range: string + slot_uri: ex:signature + review_score: + range: float + slot_uri: ex:review_score + mode: + range: ModeEnum + slot_uri: ex:mode + manual_value: + range: decimal + slot_uri: ex:manual_value + +classes: + Document: + class_uri: ex:Document + slots: + - status + - signature + - review_score + rules: + - description: If a document has a signature, its status must be Published. + preconditions: + slot_conditions: + signature: + value_presence: PRESENT + postconditions: + slot_conditions: + status: + equals_string: "Published" + - description: If a document has a review score, its status must be Reviewed or Approved. + preconditions: + slot_conditions: + review_score: + value_presence: PRESENT + postconditions: + slot_conditions: + status: + equals_string_in: + - Reviewed + - Approved + + Device: + class_uri: ex:Device + slots: + - mode + - manual_value + rules: + - description: If manual_value is provided, mode must be Manual (literal fallback). + preconditions: + slot_conditions: + manual_value: + value_presence: PRESENT + postconditions: + slot_conditions: + mode: + equals_string: "Manual" +""" + +EX_PIV = rdflib.Namespace("https://example.org/presence-implies-value/") + + +def _rule_results(schema: str, data: str | rdflib.Graph) -> tuple[bool, list[tuple]]: + """Validate *data* against the shapes generated for *schema*. + + *data* is Turtle or a graph. Returns whether the data conforms overall, + and the ``(focus node, result path, value)`` of each result that a rule + constraint (``sh:sparql``) reports, so that a property-shape result + (datatype, cardinality, ...) can neither mask nor stand in for what a rule + decides. + """ + import pyshacl + + shapes = rdflib.Graph().parse(data=ShaclGenerator(schema, mergeimports=False).serialize(), format="turtle") + data_graph = data if isinstance(data, rdflib.Graph) else rdflib.Graph().parse(data=data, format="turtle") + conforms, report, _ = pyshacl.validate(data_graph, shacl_graph=shapes, advanced=True) + results = [ + (report.value(result, SH.focusNode), report.value(result, SH.resultPath), report.value(result, SH.value)) + for result in report.subjects(SH.sourceConstraintComponent, SH.SPARQLConstraintComponent) + ] + return conforms, results + + +def _validate_rules(schema: str, data: str | rdflib.Graph) -> tuple[bool, set]: + """Whether *data* conforms to the shapes of *schema*, and the focus nodes its rule constraints report.""" + conforms, results = _rule_results(schema, data) + return conforms, {focus_node for focus_node, _, _ in results} + + +def _sparql_queries(g: rdflib.Graph, shape: URIRef) -> list[str]: + """The ``sh:select`` query of every ``sh:sparql`` constraint on *shape*.""" + return [str(query) for node in g.objects(shape, SH.sparql) for query in g.objects(node, SH.select)] + + +def test_presence_implies_value_generates_sparql(): + """Presence-implies-value rules produce sh:sparql constraints on the NodeShape.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + shape = EX_PIV.Document + sparql_nodes = list(g.objects(shape, SH.sparql)) + assert len(sparql_nodes) == 2, f"Expected 2 sh:sparql constraints, got {len(sparql_nodes)}" + + for node in sparql_nodes: + assert (node, RDF.type, SH.SPARQLConstraint) in g + selects = list(g.objects(node, SH.select)) + assert len(selects) == 1, "Each constraint must have exactly one sh:select" + query = str(selects[0]) + assert "$this" in query, "SPARQL must use $this pre-bound variable" + assert "?value = " in query, "presence-implies-value SPARQL must compare the target by value" + assert "FILTER" in query, "SPARQL must have a FILTER clause" + + +def test_presence_implies_value_single_uses_enum_iri(): + """A single equals_string target resolves to the enum meaning IRI.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + signature_query = [q for q in _sparql_queries(g, EX_PIV.Document) if str(EX_PIV.signature) in q] + assert len(signature_query) == 1, "Expected exactly one signature rule" + query = signature_query[0] + + assert str(EX_PIV.status) in query + assert f"<{EX_PIV.Published}>" in query, f"Expected Published IRI, got:\n{query}" + + +def test_presence_implies_value_set_uses_all_iris(): + """equals_string_in resolves every allowed value to its enum meaning IRI.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + review_query = [q for q in _sparql_queries(g, EX_PIV.Document) if str(EX_PIV.review_score) in q] + assert len(review_query) == 1, "Expected exactly one review_score rule" + query = review_query[0] + + assert f"<{EX_PIV.Reviewed}>" in query, f"Expected Reviewed IRI, got:\n{query}" + assert f"<{EX_PIV.Approved}>" in query, f"Expected Approved IRI, got:\n{query}" + + +def test_presence_implies_value_no_meaning_falls_back_to_literal(): + """When the target enum value lacks a meaning IRI, it is compared as a literal.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + queries = _sparql_queries(g, EX_PIV.Device) + assert len(queries) == 1 + query = queries[0] + assert '"Manual"' in query, f"No-meaning enum should use literal '\"Manual\"', got:\n{query}" + assert f"<{EX_PIV}Manual>" not in query, "Should not emit as IRI when meaning is absent" + + +def test_presence_implies_value_message_from_description(): + """Rule description is emitted as sh:message on the SPARQLConstraint.""" + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + sparql_nodes = list(g.objects(EX_PIV.Document, SH.sparql)) + messages = [str(m) for node in sparql_nodes for m in g.objects(node, SH.message)] + + assert any("status must be Published" in m for m in messages), f"Expected message about Published, got: {messages}" + + +def test_presence_implies_value_sparql_syntax_valid(): + """Generated SPARQL for presence-implies-value rules must be syntactically valid.""" + from rdflib.plugins.sparql import prepareQuery + + g = _parse_shacl(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML) + + for shape in (EX_PIV.Document, EX_PIV.Device): + for query in _sparql_queries(g, shape): + prepareQuery(query) + + +@pytest.mark.parametrize( + "instance,violates", + [ + pytest.param('ex:x a ex:Document ; ex:signature "s" ; ex:status ex:Published .', False, id="single-ok"), + pytest.param( + 'ex:x a ex:Document ; ex:review_score "4.5"^^xsd:float ; ex:status ex:Reviewed .', False, id="set-ok" + ), + pytest.param( + 'ex:x a ex:Document ; ex:review_score "3.0"^^xsd:float ; ex:status ex:Approved .', + False, + id="set-other-member-ok", + ), + pytest.param("ex:x a ex:Document ; ex:status ex:Draft .", False, id="unguarded"), + pytest.param('ex:x a ex:Document ; ex:signature "s" ; ex:status ex:Draft .', True, id="wrong-value"), + pytest.param( + 'ex:x a ex:Document ; ex:review_score "4.5"^^xsd:float ; ex:status ex:Published .', + True, + id="not-in-set", + ), + pytest.param('ex:x a ex:Document ; ex:signature "s" .', True, id="target-absent"), + pytest.param('ex:x a ex:Device ; ex:manual_value 1.5 ; ex:mode "Manual" .', False, id="literal-ok"), + pytest.param('ex:x a ex:Device ; ex:manual_value 1.5 ; ex:mode "Auto" .', True, id="literal-wrong"), + ], +) +def test_presence_implies_value_pyshacl_end_to_end(instance: str, violates: bool): + """End-to-end: pyshacl passes conforming instances and flags violations.""" + data = ( + "@prefix ex: .\n" + "@prefix xsd: .\n" + instance + ) + conforms, focus_nodes = _validate_rules(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML, data) + assert focus_nodes == ({EX_PIV.x} if violates else set()) + assert conforms is not violates + + +@pytest.mark.parametrize( + "instance,value", + [ + pytest.param('ex:signature "s" ; ex:status ex:Draft', EX_PIV.Draft, id="wrong-value"), + pytest.param('ex:signature "s"', EX_PIV.x, id="target-absent"), + ], +) +def test_presence_implies_value_result_names_path_and_value(instance, value): + """A result names the target property (``sh:resultPath``) and the + offending value (``sh:value``); when the target is absent, SHACL §5.3.2 + falls back to the focus node as the value.""" + data = f"@prefix ex: .\nex:x a ex:Document ; {instance} ." + _, results = _rule_results(_PRESENCE_IMPLIES_VALUE_SCHEMA_YAML, data) + assert results == [(EX_PIV.x, EX_PIV.status, value)] + + +# =========================================================================== +# equals_string / equals_string_in compare strings +# =========================================================================== +# +# The metamodel defines both operators for slots of range string: "the slot +# must have range string and the value of the slot must equal the specified +# value". Enums, whose values are strings in instance data, and types with +# datatype xsd:string are treated alike. A rule applying them to a slot of +# any other range is skipped with a warning: its RDF values are typed literals +# ("3"^^xsd:integer, false) or IRIs that equal no string, so a translated +# constraint would report conforming data or never fire. On a string-valued +# slot the value is compared with the SPARQL "=" operator, which matches the +# RDF 1.1-identical plain and xsd:string literal forms alike, and an +# incomparable value counts as unequal rather than as an error. +# +# References: +# - SPARQL 1.1 §17.4.1.3 +# - SPARQL 1.1 §17.4.1.9 +# - RDF 1.1 Concepts §3.3 +# =========================================================================== + +_RULE_TARGET_RANGE_SCHEMA_YAML = """ +id: https://example.org/rule-target-range +name: rule_target_range +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/rule-target-range/ + xsd: http://www.w3.org/2001/XMLSchema# +imports: + - linkml:types +default_prefix: ex +default_range: string +types: + label_string: + typeof: string + token_string: + typeof: string + uri: xsd:token + flag_boolean: + typeof: boolean +enums: + Code: + permissible_values: + alpha: + beta: + Color: + permissible_values: + Red: + meaning: ex:Red + Blue: + meaning: ex:Blue + Answer: + permissible_values: + "true": + meaning: ex:Yes + "false": + meaning: ex:No + Mixed: + permissible_values: + A: + meaning: ex:A + B: +classes: + Item: + class_uri: ex:Item + attributes: + id: + identifier: true + range: uriorcurie + Thing: + class_uri: ex:Thing + attributes: + guard: + multivalued: {guard_multivalued} + target: + range: {range} + multivalued: {multivalued} + rules: + - open_world: {open_world} + preconditions: + slot_conditions: + guard: {guard_condition} + postconditions: + slot_conditions: + target: {target_condition} + Sub: + is_a: Thing + class_uri: ex:Sub +""" + +EX_RTR = rdflib.Namespace("https://example.org/rule-target-range/") + +_RTR_PREFIXES = ( + "@prefix ex: .\n@prefix xsd: .\n" +) + + +def _rule_target_schema( + target_range: str, + value: str | list[str], + multivalued: bool = False, + *, + operator: str = "equals_string", + open_world: bool = False, + guard_multivalued: bool = False, + guard_extra: dict | None = None, + target_extra: dict | None = None, +) -> str: + """The rule-target schema: if ``guard`` is present, ``target`` (of *target_range*) must satisfy *operator*. + + *value* is the operand of *operator* (a list for ``equals_string_in``); + *guard_extra* and *target_extra* add operators to the two conditions. + ``Sub`` inherits the rule from ``Thing``. + """ + return _RULE_TARGET_RANGE_SCHEMA_YAML.format( + range=target_range, + multivalued=str(multivalued).lower(), + guard_multivalued=str(guard_multivalued).lower(), + open_world=str(open_world).lower(), + guard_condition=json.dumps({"value_presence": "PRESENT", **(guard_extra or {})}), + target_condition=json.dumps({operator: value, **(target_extra or {})}), + ) + + +@pytest.mark.parametrize( + "target_range,operator,value,conforming_term", + [ + pytest.param("integer", "equals_string", "3", "3", id="integer"), + pytest.param("integer", "equals_string_in", ["3", "4"], "3", id="integer-equals-string-in"), + pytest.param("float", "equals_string", "1.5", '"1.5"^^xsd:float', id="float"), + pytest.param("decimal", "equals_string", "1.5", "1.5", id="decimal"), + pytest.param("date", "equals_string", "2024-01-01", '"2024-01-01"^^xsd:date', id="date"), + pytest.param("uriorcurie", "equals_string", "ex:a", "ex:a", id="uriorcurie"), + pytest.param("token_string", "equals_string", "a", '"a"^^xsd:token', id="string-derived-xsd-token"), + pytest.param("Item", "equals_string", "ex:i1", "ex:i1 . ex:i1 a ex:Item", id="class"), + ], +) +def test_rule_equals_string_on_non_string_range_skipped(caplog, target_range, operator, value, conforming_term): + """equals_string on a slot that does not hold strings is skipped with a warning. + + The data holds a typed literal or an IRI, which equals no string: comparing + it with one would report every conforming instance. The conforming instance + must therefore pass, and the skip must be reported. + """ + schema = _rule_target_schema(target_range, value, operator=operator) + with caplog.at_level(logging.WARNING): + g = _parse_shacl(schema) + + assert _sparql_queries(g, EX_RTR.Thing) == [] + assert _sparql_queries(g, EX_RTR.Sub) == [] + assert any( + f"slot 'target', whose range '{target_range}' is neither an enum nor a type with datatype xsd:string or " + "xsd:boolean" in rec.message + for rec in caplog.records + ), caplog.text + conforms, focus_nodes = _validate_rules( + schema, f'{_RTR_PREFIXES}ex:x a ex:Thing ; ex:guard "g" ; ex:target {conforming_term} .' + ) + assert conforms and focus_nodes == set() + + +_STRING_TARGETS = [ + pytest.param("string", "x", '"x"', '"y"', id="string"), + pytest.param("label_string", "x", '"x"', '"y"', id="string-derived"), + pytest.param("curie", "ex:a", '"ex:a"', '"ex:b"', id="curie"), + pytest.param("Code", "alpha", '"alpha"', '"beta"', id="enum-literal"), + pytest.param("Color", "Red", "ex:Red", "ex:Blue", id="enum-meaning"), + pytest.param("Answer", "true", "ex:Yes", "ex:No", id="enum-meaning-named-true"), +] + + +@pytest.mark.parametrize("target_range,value,allowed,other", _STRING_TARGETS) +@pytest.mark.parametrize( + "body,violates", + [ + pytest.param('ex:guard "g" ; ex:target {allowed}', False, id="allowed"), + pytest.param('ex:guard "g" ; ex:target {allowed_xsd_string}', False, id="allowed-xsd-string"), + pytest.param('ex:guard "g" ; ex:target {other}', True, id="other"), + pytest.param('ex:guard "g" ; ex:target {allowed}, {other}', True, id="allowed-and-other"), + pytest.param('ex:guard "g"', True, id="target-absent"), + pytest.param("ex:target {other}", False, id="guard-absent"), + ], +) +@pytest.mark.parametrize("multivalued", [False, True]) +def test_rule_equals_string_on_string_range_compares_values( + target_range, value, allowed, other, body, violates, multivalued +): + """On a string-valued target, equals_string compares the value the range holds. + + A string type holds plain literals, an enum its meaning IRI or a plain + literal; the ``xsd:string``-typed form of a plain literal is the same RDF + 1.1 term and must also satisfy the rule. An enum value named ``true`` + with a meaning is an IRI like any other, not a boolean. + """ + allowed_xsd_string = f"{allowed}^^xsd:string" if allowed.startswith('"') else allowed + schema = _rule_target_schema(target_range, value, multivalued) + data = body.format(allowed=allowed, allowed_xsd_string=allowed_xsd_string, other=other) + + assert len(_sparql_queries(_parse_shacl(schema), EX_RTR.Thing)) == 1 + _, focus_nodes = _validate_rules(schema, f"{_RTR_PREFIXES}ex:x a ex:Thing ; {data} .") + assert focus_nodes == ({EX_RTR.x} if violates else set()) + + +_RULE_INSTANCES = [ + pytest.param({"guard": "g", "target": "{value}"}, True, True, id="allowed"), + pytest.param({"guard": "g", "target": "{other}"}, False, False, id="other"), + pytest.param({"guard": "g"}, False, True, id="target-absent"), + pytest.param({"target": "{other}"}, True, True, id="guard-absent"), +] +"""JSON instances of the rule-target schema, with their validity under closed- and open-world rules.""" + + +def _json_schema_and_shacl_verdicts( + schema: str, target_class: str, obj: dict, coerce_xsd_string: bool = False +) -> tuple[bool, bool]: + """Whether the JSON Schema and the SHACL rules generated for *schema* accept *obj*. + + *obj* is validated as is by the generated JSON Schema, and as RDF loaded + through the generated JSON-LD context by the generated SHACL shapes (rule + constraints only). With *coerce_xsd_string* the context types the + ``target`` slot ``xsd:string``, as JSON-LD 1.1 type coercion permits, + which yields the RDF 1.1-identical typed form of a plain literal. + """ + import jsonschema + + from linkml.generators.jsonldcontextgen import ContextGenerator + from linkml.generators.jsonschemagen import JsonSchemaGenerator + + json_schema = json.loads(JsonSchemaGenerator(schema, top_class=target_class).serialize()) + json_schema_valid = jsonschema.Draft7Validator(json_schema).is_valid(obj) + + context = json.loads(ContextGenerator(schema).serialize())["@context"] + if coerce_xsd_string and context["target"].get("@type") is None: + context["target"]["@type"] = "xsd:string" + document = {"@context": context, "@type": target_class, **obj} + _, focus_nodes = _validate_rules(schema, rdflib.Graph().parse(data=json.dumps(document), format="json-ld")) + return json_schema_valid, focus_nodes == set() + + +@pytest.mark.parametrize( + "target_range,value,other", + [ + pytest.param("string", "x", "y", id="string"), + pytest.param("curie", "ex:a", "ex:b", id="curie"), + pytest.param("Code", "alpha", "beta", id="enum-literal"), + pytest.param("Color", "Red", "Blue", id="enum-meaning"), + ], +) +@pytest.mark.parametrize("instance,valid,_valid_open_world", _RULE_INSTANCES) +@pytest.mark.parametrize("coerce_xsd_string", [False, True]) +def test_rule_equals_string_agrees_with_json_schema( + target_range, value, other, instance, valid, _valid_open_world, coerce_xsd_string +): + """The SHACL rule and the JSON Schema rule decide the same instance alike. + + One JSON object is checked by both generated artifacts (see + :func:`_json_schema_and_shacl_verdicts`), with and without ``xsd:string`` + coercion of the target, which the test adds to the generated context. An + enum whose values all have a meaning in one namespace is mapped to those + IRIs by the context and stays untyped. + """ + obj = {key: text.format(value=value, other=other) for key, text in instance.items()} + verdicts = _json_schema_and_shacl_verdicts( + _rule_target_schema(target_range, value), "Thing", obj, coerce_xsd_string + ) + assert verdicts == (valid, valid) + + +@pytest.mark.parametrize( + "target_range,value,other", + [ + pytest.param("string", "x", "y", id="string"), + pytest.param("Color", "Red", "Blue", id="enum-meaning"), + ], +) +@pytest.mark.parametrize("instance,valid_closed_world,valid_open_world", _RULE_INSTANCES) +@pytest.mark.parametrize("target_class", ["Thing", "Sub"]) +@pytest.mark.parametrize("open_world", [False, True]) +def test_rule_inheritance_and_open_world_agree_with_json_schema( + target_range, value, other, instance, valid_closed_world, valid_open_world, target_class, open_world +): + """A rule binds the instances of subclasses of its class, and ``open_world`` + lets the postcondition be omitted, in SHACL as in JSON Schema. + + The metamodel defines ``open_world`` as "the postconditions may be omitted + in instance data", so an absent target satisfies an open-world rule. + """ + obj = {key: text.format(value=value, other=other) for key, text in instance.items()} + schema = _rule_target_schema(target_range, value, open_world=open_world) + valid = valid_open_world if open_world else valid_closed_world + assert _json_schema_and_shacl_verdicts(schema, target_class, obj) == (valid, valid) + + +@pytest.mark.parametrize("flag_range", ["boolean", "flag_boolean"]) +@pytest.mark.parametrize("open_world", [False, True]) +@pytest.mark.parametrize("required", ["true", "1", "false", "0"]) +@pytest.mark.parametrize( + "flag,datatype", + [ + pytest.param("true", XSD.boolean, id="true"), + pytest.param("1", XSD.boolean, id="lexical-1"), + pytest.param("false", XSD.boolean, id="false"), + pytest.param("0", XSD.boolean, id="lexical-0"), + pytest.param("true", None, id="string-true"), + pytest.param(None, None, id="absent"), + ], +) +def test_rule_boolean_target_compared_by_value(flag_range, open_world, required, flag, datatype): + """equals_string on an xsd:boolean-typed slot (the boolean guard, for + "true") requires the boolean its value denotes in the lexical space of + xsd:boolean (XML Schema 1.1 Part 2 §3.3.2.2), whichever type the slot's + range derives it from. The flag is compared by value, so every lexical + form of a boolean satisfies the rule and a string "true" does not; an + open-world rule lets the flag be omitted. + + The data is built with unnormalised literals: rdflib would otherwise + rewrite ``"1"^^xsd:boolean`` to ``true`` while parsing, and a lexical + comparison would pass as well. + """ + schema = _rule_target_schema(flag_range, required, open_world=open_world) + required_value = required in ("true", "1") + queries = _sparql_queries(_parse_shacl(schema), EX_RTR.Thing) + assert len(queries) == 1 + assert f"?value = {str(required_value).lower()}" in queries[0], queries[0] + + data = rdflib.Graph() + data.add((EX_RTR.x, RDF.type, EX_RTR.Thing)) + data.add((EX_RTR.x, EX_RTR.guard, Literal("g"))) + if flag is not None: + data.add((EX_RTR.x, EX_RTR.target, Literal(flag, datatype=datatype, normalize=False))) + if flag is None: + violates = not open_world + else: + violates = datatype is None or (flag in ("true", "1")) != required_value + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == ({EX_RTR.x} if violates else set()) + + +@pytest.mark.parametrize("value", ["yes", "True", "TRUE", " true", ""]) +def test_rule_boolean_target_non_lexical_value_skipped(caplog, value): + """A value outside the lexical space of xsd:boolean on a boolean slot equals + no boolean, so the rule is skipped with a warning. The lexical space is + exactly {true, false, 1, 0}: it is case-sensitive and has no whitespace.""" + with caplog.at_level(logging.WARNING): + g = _parse_shacl(_rule_target_schema("boolean", value)) + assert _sparql_queries(g, EX_RTR.Thing) == [] + assert any("not all xsd:boolean lexical forms" in rec.message for rec in caplog.records), caplog.text + + +@pytest.mark.parametrize( + "target,violates", + [ + pytest.param("ex:A", False, id="meaning-iri"), + pytest.param('"B"', False, id="literal"), + pytest.param('"B"^^xsd:string', False, id="literal-xsd-string"), + pytest.param('"A"', True, id="literal-of-value-with-meaning"), + pytest.param("ex:B", True, id="iri-of-value-without-meaning"), + pytest.param('"C"', True, id="other"), + ], +) +def test_rule_equals_string_in_mixes_meaning_iris_and_literals(target, violates): + """``equals_string_in`` over permissible values with and without a meaning + compares each as the term it is rendered as: its meaning IRI, or a literal.""" + schema = _rule_target_schema("Mixed", ["A", "B"], operator="equals_string_in") + _, focus_nodes = _validate_rules(schema, f'{_RTR_PREFIXES}ex:x a ex:Thing ; ex:guard "g" ; ex:target {target} .') + assert focus_nodes == ({EX_RTR.x} if violates else set()) + + +@pytest.mark.parametrize( + "target,violates", + [pytest.param('"x"', False, id="allowed"), pytest.param('"y"', True, id="other")], +) +def test_rule_multivalued_guard(target, violates): + """Several guard values trigger the rule once per focus node, like one.""" + schema = _rule_target_schema("string", "x", guard_multivalued=True) + data = f'{_RTR_PREFIXES}ex:x a ex:Thing ; ex:guard "g1", "g2", "g3" ; ex:target {target} .' + _, results = _rule_results(schema, data) + assert results == ([(EX_RTR.x, EX_RTR.target, rdflib.Literal("y"))] if violates else []) + + +def test_rule_query_yields_one_solution_per_focus_node(): + """Each solution of a SPARQL-based constraint is a validation result + (SHACL §5.3.2), so several guard values must not multiply the solutions. + pyshacl merges identical results; the query itself must not rely on it.""" + schema = _rule_target_schema("string", "x", guard_multivalued=True) + (query,) = _sparql_queries(_parse_shacl(schema), EX_RTR.Thing) + data = rdflib.Graph().parse( + data=f'{_RTR_PREFIXES}ex:x a ex:Thing ; ex:guard "g1", "g2", "g3" ; ex:target "y" .', format="turtle" + ) + assert len(list(data.query(query, initBindings={"this": EX_RTR.x}))) == 1 + + +@pytest.mark.parametrize("dawg_literal_collation", [False, True]) +def test_rule_incomparable_value_is_a_violation(monkeypatch, dawg_literal_collation): + """A value no allowed value can be compared with equals none of them. + + SPARQL ``=`` raises a type error for literals it cannot compare + (RDFterm-equal, SPARQL 1.1 §17.4.1.7), and a FILTER error drops the row, so + a spec-conformant engine would miss the violation unless the comparison + turns the error into false. ``rdflib.DAWG_LITERAL_COLLATION`` switches + rdflib from its lenient comparison to that spec-conformant one. + """ + monkeypatch.setattr(rdflib, "DAWG_LITERAL_COLLATION", dawg_literal_collation) + schema = _rule_target_schema("string", "x") + data = f'{_RTR_PREFIXES}ex:x a ex:Thing ; ex:guard "g" ; ex:target "x"^^ex:custom .' + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == {EX_RTR.x} + + exclusive = _exclusive_schema("Tag", "alpha", 1) + data = ( + '@prefix ex: .\nex:x a ex:Thing ; ex:tags "alpha", "beta"^^ex:custom .' + ) + _, focus_nodes = _validate_rules(exclusive, data) + assert focus_nodes == {EX_EXR.x} + + +def test_rule_value_not_a_permissible_value_warns(caplog): + """An equals_string value that is not a permissible value of the enum is + reported: no valid value of the slot can equal it, so the rule rejects + every instance it applies to.""" + schema = _rule_target_schema("Color", "Purple") + with caplog.at_level(logging.WARNING): + queries = _sparql_queries(_parse_shacl(schema), EX_RTR.Thing) + + assert len(queries) == 1 + assert any("'Purple', which is not a permissible value of enum 'Color'" in rec.message for rec in caplog.records) + _, focus_nodes = _validate_rules(schema, f'{_RTR_PREFIXES}ex:x a ex:Thing ; ex:guard "g" ; ex:target ex:Red .') + assert focus_nodes == {EX_RTR.x} + + +@pytest.mark.parametrize( + "value", + [ + pytest.param('a"b', id="quote"), + pytest.param("a\\b", id="backslash"), + pytest.param("line\nbreak", id="newline"), + pytest.param("carriage\rreturn", id="carriage-return"), + pytest.param("tab\there", id="tab"), + pytest.param("a\\u0022b", id="backslash-u"), + pytest.param("a\\U00000022b", id="backslash-capital-u"), + pytest.param("check ✓", id="non-ascii"), + pytest.param("", id="empty"), + ], +) +def test_rule_string_value_round_trips(value): + """An equals_string value of any content is matched exactly: the query + parses, the value itself satisfies the rule, and any other value does not. + + Codepoint escapes are replaced before a SPARQL query is parsed (SPARQL 1.1 + §19.2), so a backslash followed by ``u`` in the value must not reach the + query as an escape sequence. + """ + from rdflib.plugins.sparql import prepareQuery + + schema = _rule_target_schema("string", value) + queries = _sparql_queries(_parse_shacl(schema), EX_RTR.Thing) + assert len(queries) == 1 + prepareQuery(queries[0]) + + for target, violates in ((value, False), (value + "!", True)): + data = rdflib.Graph() + data.add((EX_RTR.x, RDF.type, EX_RTR.Thing)) + data.add((EX_RTR.x, EX_RTR.guard, Literal("g"))) + data.add((EX_RTR.x, EX_RTR.target, Literal(target))) + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == ({EX_RTR.x} if violates else set()), repr(target) + + +_EXCLUSIVE_RANGE_SCHEMA_YAML = """ +id: https://example.org/exclusive-range +name: exclusive_range +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/exclusive-range/ +imports: + - linkml:types +default_prefix: ex +default_range: string +enums: + Tag: + permissible_values: + alpha: + beta: + gamma: +classes: + Thing: + class_uri: ex:Thing + attributes: + tags: + range: {range} + multivalued: true + rules: + - preconditions: + slot_conditions: + tags: {precondition} + postconditions: + slot_conditions: + tags: + maximum_cardinality: {max_card} +""" + +EX_EXR = rdflib.Namespace("https://example.org/exclusive-range/") + + +def _exclusive_schema(tag_range: str, value: str, max_card: int, *, has_member: bool = True) -> str: + """The exclusive-value schema: if *value* is one of the ``tags``, there are at most *max_card* of them. + + The precondition is ``has_member: {equals_string: value}``, or with + ``has_member=False`` the bare ``equals_string: value``. + """ + precondition = {"has_member": {"equals_string": value}} if has_member else {"equals_string": value} + return _EXCLUSIVE_RANGE_SCHEMA_YAML.format( + range=tag_range, precondition=json.dumps(precondition), max_card=max_card + ) + + +@pytest.mark.parametrize( + "max_card,values,violates", + [ + pytest.param(1, '"alpha", "beta"', True, id="max1-plain"), + pytest.param(1, '"alpha"^^xsd:string, "beta"^^xsd:string', True, id="max1-xsd-string"), + pytest.param(1, '"alpha"', False, id="max1-alone"), + pytest.param(1, '"beta", "gamma"', False, id="max1-absent"), + pytest.param(2, '"alpha", "beta", "gamma"', True, id="max2-plain"), + pytest.param(2, '"alpha"^^xsd:string, "beta", "gamma"', True, id="max2-xsd-string"), + pytest.param(2, '"alpha", "beta"', False, id="max2-within"), + ], +) +def test_exclusive_value_matches_value_not_term(max_card, values, violates): + """The exclusive value is matched by value, so its RDF 1.1-identical + ``xsd:string``-typed form triggers the rule as the plain literal does, and + a ``maximum_cardinality`` above 1 is enforced.""" + schema = _exclusive_schema("Tag", "alpha", max_card) + data = ( + "@prefix ex: .\n" + "@prefix xsd: .\n" + f"ex:x a ex:Thing ; ex:tags {values} ." + ) + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == ({EX_EXR.x} if violates else set()) + + +@pytest.mark.parametrize("max_card", [1, 2]) +def test_exclusive_value_result_names_path_and_value(max_card): + """A result names the slot. For ``maximum_cardinality: 1`` its value is + each value coexisting with the exclusive one; otherwise no value is + projected and SHACL §5.3.2 falls back to the focus node.""" + data = '@prefix ex: .\nex:x a ex:Thing ; ex:tags "alpha", "beta", "gamma" .' + _, results = _rule_results(_exclusive_schema("Tag", "alpha", max_card), data) + if max_card == 1: + assert sorted(results) == sorted( + [(EX_EXR.x, EX_EXR.tags, Literal("beta")), (EX_EXR.x, EX_EXR.tags, Literal("gamma"))] + ) + else: + assert results == [(EX_EXR.x, EX_EXR.tags, EX_EXR.x)] + + +@pytest.mark.parametrize("max_card", [1, 2]) +def test_exclusive_value_query_yields_distinct_solutions(max_card): + """The exclusive value in both RDF 1.1-identical forms binds twice; each + solution is a validation result (SHACL §5.3.2), so the query must not + multiply them.""" + (query,) = _sparql_queries(_parse_shacl(_exclusive_schema("Tag", "alpha", max_card)), EX_EXR.Thing) + data = rdflib.Graph().parse( + data="@prefix ex: .\n" + "@prefix xsd: .\n" + 'ex:x a ex:Thing ; ex:tags "alpha", "alpha"^^xsd:string, "beta" .', + format="turtle", + ) + assert len(list(data.query(query, initBindings={"this": EX_EXR.x}))) == 1 + + +def test_rule_warning_names_the_rule_by_title(caplog): + """A skipped rule is reported by its position and its ``title``, or its ``description`` without one.""" + rule = _rule({"target": {"equals_string": "x"}}, {"target": {"maximum_cardinality": 1}}) + with caplog.at_level(logging.WARNING): + _parse_shacl(_single_rule_schema({**rule, "title": "one target", "description": "a longer text"})) + _parse_shacl(_single_rule_schema({**rule, "description": "a longer text"})) + messages = [rec.message for rec in caplog.records if "skipped" in rec.message] + assert [m.split(":")[0] for m in messages] == [ + "Rule 1 of class 'Thing' (one target)", + "Rule 1 of class 'Thing' (a longer text)", + ] + + +@pytest.mark.parametrize("multivalued", [True, False]) +def test_exclusive_value_bare_equals_string_skipped(caplog, multivalued): + """A bare ``equals_string`` precondition applies to every value of the slot + (``05validation.md``), so it is not the exclusive-value pattern: the rule is + skipped with a warning that names ``has_member``, which states "one of the + values equals".""" + if multivalued: + schema, shape = _exclusive_schema("Tag", "alpha", 1, has_member=False), EX_EXR.Thing + else: + schema = _single_rule_schema(_rule({"target": {"equals_string": "x"}}, {"target": {"maximum_cardinality": 1}})) + shape = EX_SINGLE.Thing + with caplog.at_level(logging.WARNING): + assert _sparql_queries(_parse_shacl(schema), shape) == [] + assert any("every value" in rec.message and "has_member" in rec.message for rec in caplog.records), caplog.text + + +@pytest.mark.parametrize( + "precondition", + [ + pytest.param({"has_member": {"equals_string": "alpha", "pattern": "^a"}}, id="inside-has_member"), + pytest.param({"has_member": {"equals_string": "alpha"}, "minimum_cardinality": 2}, id="beside-has_member"), + ], +) +def test_exclusive_value_has_member_extra_operator_skipped(caplog, precondition): + """has_member is exact too: an operator next to its equals_string, or next + to has_member itself, makes the rule untranslatable.""" + precondition = json.dumps(precondition) + schema = _EXCLUSIVE_RANGE_SCHEMA_YAML.format(range="Tag", precondition=precondition, max_card=1) + with caplog.at_level(logging.WARNING): + assert _sparql_queries(_parse_shacl(schema), EX_EXR.Thing) == [] + assert any("match none of the translated patterns" in rec.message for rec in caplog.records), caplog.text + + +def test_exclusive_value_on_non_string_range_skipped(caplog): + """equals_string on a slot that does not hold strings is skipped with a + warning in the exclusive-value pattern too: the integer ``3`` never equals + the string "3", so the constraint would never fire.""" + schema = _exclusive_schema("integer", "3", 1) + with caplog.at_level(logging.WARNING): + g = _parse_shacl(schema) + + assert _sparql_queries(g, EX_EXR.Thing) == [] + assert any("whose range 'integer' is neither an enum" in rec.message for rec in caplog.records), caplog.text + + +def test_rule_non_operator_fields_are_the_metamodel_element_metadata(): + """The fields the operator accounting ignores are exactly the metamodel's + ``element`` slots that are not ``slot_expression`` slots.""" + from linkml_runtime.utils.formatutils import underscore + from linkml_runtime.utils.introspection import package_schemaview + + metamodel = package_schemaview("linkml_runtime.linkml_model.meta") + element_slots = {underscore(s) for s in metamodel.class_slots("element")} + operator_slots = {underscore(s) for s in metamodel.class_slots("slot_expression")} + + assert ShaclGenerator._NON_OPERATOR_FIELDS == element_slots - operator_slots + assert {"description", "annotations", "title"} <= ShaclGenerator._NON_OPERATOR_FIELDS + + +def test_rule_condition_metadata_does_not_block_translation(): + """Metadata on a condition does not change which instances satisfy it, + so it does not make an otherwise recognised rule untranslatable.""" + schema = _rule_target_schema( + "string", + "x", + guard_extra={"title": "Guard set"}, + target_extra={"description": "The target must then be x.", "comments": ["Metadata only."]}, + ) + assert len(_sparql_queries(_parse_shacl(schema), EX_RTR.Thing)) == 1 + + +# =========================================================================== +# Rule inheritance +# +# A rule applies to "all members of this class" (metamodel `rules`), so a +# class shape carries the rules of its ancestors and mixins, each translated +# in the class's own context. +# =========================================================================== + +_RULE_INHERITANCE_SCHEMA_YAML = """ +id: https://example.org/rule-inheritance +name: rule_inheritance +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/rule-inheritance/ +imports: + - linkml:types +default_prefix: ex +default_range: string +enums: + Color: + permissible_values: + Red: + meaning: ex:Red + Blue: + meaning: ex:Blue + Shade: + permissible_values: + Red: + meaning: ex:ShadeRed +slots: + guard: {} + target: + range: Color + code: {} + label: {} +classes: + Base: + class_uri: ex:Base + slots: [guard, target] + rules: + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + target: + equals_string: Red + Child: + is_a: Base + class_uri: ex:Child + GrandChild: + is_a: Child + class_uri: ex:GrandChild + Shaded: + is_a: Base + class_uri: ex:Shaded + slot_usage: + target: + range: Shade + Counted: + is_a: Base + class_uri: ex:Counted + slot_usage: + target: + range: integer + Labelled: + mixin: true + class_uri: ex:Labelled + slots: [code, label] + rules: + - preconditions: + slot_conditions: + code: + value_presence: PRESENT + postconditions: + slot_conditions: + label: + equals_string: x + Mixed: + class_uri: ex:Mixed + mixins: [Labelled] + SameUri: + is_a: Base + class_uri: ex:Base +""" + +EX_RI = rdflib.Namespace("https://example.org/rule-inheritance/") + + +def test_rule_inherited_by_descendant_shapes(caplog): + """The shape of a class carries the rules of its ancestors and mixins, each + translated in the class's own context: a ``slot_usage`` that narrows the + target to another enum resolves the value there, and one that narrows it to + a non-string range skips the rule for that class only. A class sharing its + ancestor's ``class_uri`` shares its shape, which carries the rule once.""" + with caplog.at_level(logging.WARNING): + g = _parse_shacl(_RULE_INHERITANCE_SCHEMA_YAML) + + for cls in ("Base", "Child", "GrandChild", "Shaded", "Labelled", "Mixed"): + assert len(_sparql_queries(g, EX_RI[cls])) == 1, cls + assert f"<{EX_RI.ShadeRed}>" in _sparql_queries(g, EX_RI.Shaded)[0] + assert _sparql_queries(g, EX_RI.Counted) == [] + assert any("'Counted'" in rec.message and "is neither an enum" in rec.message for rec in caplog.records) + + +@pytest.mark.parametrize( + "instance,violates", + [ + pytest.param('ex:x a ex:Child ; ex:guard "g" ; ex:target ex:Red .', False, id="child-ok"), + pytest.param('ex:x a ex:Child ; ex:guard "g" ; ex:target ex:Blue .', True, id="child-violates"), + pytest.param('ex:x a ex:GrandChild ; ex:guard "g" ; ex:target ex:Blue .', True, id="grandchild-violates"), + pytest.param('ex:x a ex:Shaded ; ex:guard "g" ; ex:target ex:ShadeRed .', False, id="narrowed-ok"), + pytest.param('ex:x a ex:Shaded ; ex:guard "g" ; ex:target ex:Red .', True, id="narrowed-violates"), + pytest.param('ex:x a ex:Mixed ; ex:code "c" ; ex:label "x" .', False, id="mixin-ok"), + pytest.param('ex:x a ex:Mixed ; ex:code "c" ; ex:label "y" .', True, id="mixin-violates"), + ], +) +def test_rule_inherited_rule_enforced_on_descendant_instances(instance, violates): + """An instance typed only with a descendant class is held to the inherited rule.""" + data = f"@prefix ex: .\n{instance}" + _, focus_nodes = _validate_rules(_RULE_INHERITANCE_SCHEMA_YAML, data) + assert focus_nodes == ({EX_RI.x} if violates else set()) + + +def test_rule_shared_shape_reports_once(): + """Two classes sharing a ``class_uri`` share one shape, so an instance + violating the rule both inherit gets one result, not one per class.""" + data = '@prefix ex: .\nex:x a ex:Base ; ex:guard "g" ; ex:target ex:Blue .' + _, results = _rule_results(_RULE_INHERITANCE_SCHEMA_YAML, data) + assert results == [(EX_RI.x, EX_RI.target, EX_RI.Blue)] + + +# =========================================================================== +# Slot resolution and value rendering +# +# A rule's SPARQL body must query the same IRI that ``sh:path`` emits for the +# slot, and resolve the slot's range in the class: +# 1. a slot_usage `slot_uri` (or enum `range`) override applies to the rule; +# 2. an alias-form key (``my_slot`` for slot ``my slot``) names the slot; +# 3. a slot without a range takes the schema's `default_range`; +# 4. built-in type names resolve without importing linkml:types; +# 5. a condition on an unknown or identifier slot makes the rule untranslatable. +# =========================================================================== + + +_ENUM_NARROWING_SCHEMA_YAML = """ +id: https://example.org/enum-narrowing +name: enum_narrowing_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/enum-narrowing/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +enums: + BaseMode: + permissible_values: + Active: + meaning: ex:GLOBAL_Active + SceneMode: + permissible_values: + Active: + meaning: ex:LOCAL_Active + +slots: + activator: + range: string + slot_uri: ex:activator + mode: + range: BaseMode + slot_uri: ex:mode + +classes: + Scene: + class_uri: ex:Scene + slots: + - activator + - mode + slot_usage: + mode: + range: SceneMode + rules: + - description: If an activator is present the mode must be Active. + preconditions: + slot_conditions: + activator: + value_presence: PRESENT + postconditions: + slot_conditions: + mode: + equals_string: Active +""" + +EX_EN = rdflib.Namespace("https://example.org/enum-narrowing/") + + +def test_rule_enum_range_narrowed_by_slot_usage(): + """A slot_usage range override to a class-specific enum must resolve the + value's meaning against the induced (narrowed) enum, not the base range.""" + queries = _sparql_queries(_parse_shacl(_ENUM_NARROWING_SCHEMA_YAML), EX_EN.Scene) + assert len(queries) == 1 + query = queries[0] + assert str(EX_EN.LOCAL_Active) in query, f"must resolve the narrowed enum meaning, got:\n{query}" + assert "GLOBAL_Active" not in query, f"must not resolve the base enum meaning, got:\n{query}" + + +_SLOT_RESOLUTION_SCHEMA_YAML = """ +id: https://example.org/slot-resolution +name: slot_resolution +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/slot-resolution/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + guard: {} + my slot: {} + target: {} +classes: + Thing: + class_uri: ex:Thing + slots: [guard, my slot, target] + slot_usage: + target: + slot_uri: ex:narrowed_target + rules: + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + target: + equals_string: x + - preconditions: + slot_conditions: + my_slot: + value_presence: PRESENT + postconditions: + slot_conditions: + target: + equals_string: x +""" + +EX_SR = rdflib.Namespace("https://example.org/slot-resolution/") + + +def test_rule_slot_iris_match_sh_path(): + """Every predicate a rule queries is one that ``sh:path`` emits: a + slot_usage ``slot_uri`` override and an alias-form key resolve to the + slot's own IRI rather than a fabricated one the data never uses.""" + g = _parse_shacl(_SLOT_RESOLUTION_SCHEMA_YAML) + + paths = {str(path) for path in g.objects(None, SH.path)} + queries = _sparql_queries(g, EX_SR.Thing) + assert len(queries) == 2 + for query in queries: + assert set(re.findall(r"<([^>]+)>", query)) <= paths, query + assert f"<{EX_SR.narrowed_target}>" in query, query + assert any(f"<{EX_SR.my_slot}>" in query for query in queries) + + _, focus_nodes = _validate_rules( + _SLOT_RESOLUTION_SCHEMA_YAML, + "@prefix ex: .\n" + 'ex:x a ex:Thing ; ex:my_slot "s" ; ex:narrowed_target "y" .', + ) + assert focus_nodes == {EX_SR.x} + + +_RANGE_RESOLUTION_SCHEMA_YAML = """ +id: https://example.org/range-resolution +name: range_resolution +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/range-resolution/ +imports: + - linkml:types +default_prefix: ex +default_range: {default_range} +slots: + guard: + range: string + target: {target_slot} +classes: + Thing: + class_uri: ex:Thing + slots: {class_slots} + slot_usage: {slot_usage} + rules: + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + target: + equals_string: "3" +""" + +EX_RR = rdflib.Namespace("https://example.org/range-resolution/") + + +@pytest.mark.parametrize( + "default_range,target_slot,class_slots,slot_usage,translated", + [ + pytest.param("string", "{}", "[guard, target]", "{}", True, id="default-range-string"), + pytest.param("integer", "{}", "[guard, target]", "{}", False, id="default-range-integer"), + pytest.param( + "string", + "{range: integer}", + "[guard, target]", + "{target: {range: string}}", + True, + id="slot-usage-to-string", + ), + pytest.param( + "string", "{}", "[guard, target]", "{target: {range: integer}}", False, id="slot-usage-to-integer" + ), + pytest.param("string", "{}", "[guard]", "{}", True, id="slot-not-on-class-default-string"), + pytest.param("integer", "{}", "[guard]", "{}", False, id="slot-not-on-class-default-integer"), + ], +) +def test_rule_target_range_resolved_in_class_context(default_range, target_slot, class_slots, slot_usage, translated): + """Whether the target holds strings is decided by its range in the class: + the schema's ``default_range`` for a slot without one, as refined by + ``slot_usage``, also for a slot the class does not declare.""" + schema = _RANGE_RESOLUTION_SCHEMA_YAML.format( + default_range=default_range, target_slot=target_slot, class_slots=class_slots, slot_usage=slot_usage + ) + assert len(_sparql_queries(_parse_shacl(schema), EX_RR.Thing)) == (1 if translated else 0) + if translated: + prefix = "@prefix ex: .\n" + assert _validate_rules(schema, f'{prefix}ex:x a ex:Thing ; ex:guard "g" ; ex:target "3" .')[1] == set() + assert _validate_rules(schema, f'{prefix}ex:x a ex:Thing ; ex:guard "g" ; ex:target "4" .')[1] == {EX_RR.x} + + +EX_SINGLE = rdflib.Namespace("https://example.org/single-rule/") + + +def _single_rule_schema(rule: dict, attributes: dict | None = None, *, imports: bool = True) -> str: + """A schema whose class ``Thing`` has *attributes* and the one *rule* (JSON, which is YAML). + + By default the attributes are the string slots ``guard``, ``target`` and + ``other``, the boolean ``flag`` and the multivalued ``tags``. Without + *imports* the schema does not import ``linkml:types`` and has no + ``default_range``. + """ + schema = { + "id": "https://example.org/single-rule", + "name": "single_rule", + "prefixes": {"linkml": "https://w3id.org/linkml/", "ex": str(EX_SINGLE)}, + "default_prefix": "ex", + "classes": { + "Thing": { + "class_uri": "ex:Thing", + "attributes": attributes + or { + "guard": {"range": "string"}, + "target": {"range": "string"}, + "other": {"range": "string"}, + "flag": {"range": "boolean"}, + "tags": {"range": "string", "multivalued": True}, + }, + "rules": [rule], + } + }, + } + if imports: + schema["imports"] = ["linkml:types"] + schema["default_range"] = "string" + return json.dumps(schema) + + +def _rule(pre: dict, post: dict, **rule_fields) -> dict: + """A rule with the slot conditions *pre* and *post*.""" + return {"preconditions": {"slot_conditions": pre}, "postconditions": {"slot_conditions": post}, **rule_fields} + + +_GUARD_PRESENT = {"guard": {"value_presence": "PRESENT"}} + + +def _assert_skipped(caplog, schema: str, reason: str) -> None: + """Generating *schema* emits no rule constraint and warns that the rule was skipped for *reason*.""" + with caplog.at_level(logging.WARNING): + g = _parse_shacl(schema) + assert _sparql_queries(g, EX_SINGLE.Thing) == [] + assert any("skipped, because" in rec.message and reason in rec.message for rec in caplog.records), caplog.text + + +@pytest.mark.parametrize( + "target_range,value,allowed,other", + [ + pytest.param("boolean", "true", "true", "false", id="boolean-guard"), + pytest.param("string", "x", '"x"', '"y"', id="string"), + pytest.param("curie", "ex:a", '"ex:a"', '"ex:b"', id="curie"), + pytest.param(None, "x", '"x"', '"y"', id="no-range"), + ], +) +def test_rule_builtin_types_without_imports(target_range, value, allowed, other): + """In a schema that does not import ``linkml:types``, a built-in range name + resolves as the main slot loop resolves it, so the boolean guard and the + string comparison are both translated. A slot with no range at all (and + no ``default_range``) holds untyped values, compared as strings as the + JSON Schema generator compares them.""" + target = {} if target_range is None else {"range": target_range} + schema = _single_rule_schema( + _rule(_GUARD_PRESENT, {"target": {"equals_string": value}}), + {"guard": {"range": "string"}, "target": target}, + imports=False, + ) + assert len(_sparql_queries(_parse_shacl(schema), EX_SINGLE.Thing)) == 1 + prefix = "@prefix ex: .\n" + for term, violates in ((allowed, False), (other, True)): + data = f'{prefix}ex:x a ex:Thing ; ex:guard "g" ; ex:target {term} .' + assert _validate_rules(schema, data)[1] == ({EX_SINGLE.x} if violates else set()) + + +def test_rule_builtin_non_string_type_without_imports_skipped(caplog): + """A built-in non-string range without the ``linkml:types`` import is skipped like an imported one.""" + schema = _single_rule_schema( + _rule(_GUARD_PRESENT, {"target": {"equals_string": "3"}}), + {"guard": {"range": "string"}, "target": {"range": "integer"}}, + imports=False, + ) + _assert_skipped(caplog, schema, "is neither an enum nor a type with datatype xsd:string or xsd:boolean") + + +def test_curie_slot_without_imports_has_xsd_string_datatype(): + """A ``curie`` slot in a schema without the ``linkml:types`` import gets + ``sh:datatype xsd:string``, as with the import.""" + schema = _single_rule_schema( + _rule(_GUARD_PRESENT, {"target": {"equals_string": "ex:a"}}), + {"guard": {"range": "string"}, "target": {"range": "curie"}}, + imports=False, + ) + g = _parse_shacl(schema) + datatypes = { + o + for p in g.objects(EX_SINGLE.Thing, SH.property) + if (p, SH.path, EX_SINGLE.target) in g + for o in g.objects(p, SH.datatype) + } + assert datatypes == {XSD.string} + + +def test_rule_dynamic_enum_value_not_reported(caplog): + """An enum without static permissible values (a dynamic enum) can hold + values the schema does not list, so its equals_string value is not + reported as unknown.""" + + schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"equals_string": "GO:0008150"}}))) + schema["enums"] = {"Process": {"reachable_from": {"source_ontology": "obo:go", "source_nodes": ["GO:0008150"]}}} + schema["classes"]["Thing"]["attributes"]["target"] = {"range": "Process"} + with caplog.at_level(logging.WARNING): + queries = _sparql_queries(_parse_shacl(json.dumps(schema)), EX_SINGLE.Thing) + assert len(queries) == 1 and '"GO:0008150"' in queries[0] + assert not any("permissible value" in rec.message for rec in caplog.records), caplog.text + + +def test_rule_message_follows_default_language(): + """The rule description becomes ``sh:message``, tagged with the default + language like every other human-readable literal the generator emits.""" + rule = _rule(_GUARD_PRESENT, {"target": {"equals_string": "x"}}, description="The target must be x.") + g = _parse_shacl(_single_rule_schema(rule), default_language="en") + (node,) = g.objects(EX_SINGLE.Thing, SH.sparql) + assert set(g.objects(node, SH.message)) == {Literal("The target must be x.", lang="en")} + + +@pytest.mark.parametrize( + "descriptions,constraints", + [ + pytest.param(["Same.", "Same."], 1, id="identical-rules"), + pytest.param(["First.", "Second."], 2, id="different-messages"), + ], +) +def test_rule_identical_constraints_emitted_once(descriptions, constraints): + """A shape carries an identical constraint (query and message) once; rules + that differ in their message are kept apart.""" + schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"equals_string": "x"}}))) + rule = schema["classes"]["Thing"]["rules"][0] + schema["classes"]["Thing"]["rules"] = [{**rule, "description": text} for text in descriptions] + assert len(list(_parse_shacl(json.dumps(schema)).objects(EX_SINGLE.Thing, SH.sparql))) == constraints + + +def test_rule_problem_reported_once_for_inheriting_classes(caplog): + """A problem with a rule is reported once, naming the declaring class and + the rule's position, however many classes inherit it; and a skipped rule + with elseconditions is not also reported as partially emitted.""" + + schema = json.loads( + _single_rule_schema( + _rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}, elseconditions={"slot_conditions": {}}) + ) + ) + for name in ("C1", "C2", "C3"): + schema["classes"][name] = {"is_a": "Thing", "class_uri": f"ex:{name}"} + with caplog.at_level(logging.WARNING): + _parse_shacl(json.dumps(schema)) + messages = [rec.message for rec in caplog.records if "Rule 1 of class 'Thing'" in rec.message] + assert len(messages) == 1, messages + assert "skipped, because" in messages[0] and "(in the shapes of 'Thing', 'C1', 'C2', 'C3')" in messages[0] + assert not any("elseconditions" in rec.message for rec in caplog.records), caplog.text + + +def test_rule_problem_names_every_affected_shape_whatever_the_class_order(caplog): + """A problem with an inherited rule names every class shape it affects, so + the report does not depend on the order in which classes are declared.""" + schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}))) + schema["classes"] = {"Sub": {"is_a": "Thing", "class_uri": "ex:Sub"}, **schema["classes"]} + with caplog.at_level(logging.WARNING): + _parse_shacl(json.dumps(schema)) + messages = [rec.message for rec in caplog.records if "Rule 1 of class 'Thing'" in rec.message] + assert len(messages) == 1, messages + assert "'Sub'" in messages[0] and "'Thing'" in messages[0], messages[0] + + +def test_rule_problems_reported_per_rule_and_per_problem(caplog): + """Problems are kept apart by rule and by kind: two rules with the same + problem, and one rule with two problems, each produce their own warning. + Each warning names the rule by its position and description.""" + schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}))) + schema["enums"] = {"Color": {"permissible_values": {"Red": {}, "Blue": {}}}} + schema["classes"]["Thing"]["attributes"]["color"] = {"range": "Color"} + schema["classes"]["Thing"]["rules"] = [ + _rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}, description="First"), + _rule(_GUARD_PRESENT, {"target": {"pattern": "^y"}}, description="Second"), + _rule( + _GUARD_PRESENT, + {"color": {"equals_string": "Purple"}}, + description="Third", + elseconditions={"slot_conditions": {"color": {"equals_string": "Red"}}}, + ), + ] + with caplog.at_level(logging.WARNING): + _parse_shacl(json.dumps(schema)) + messages = [rec.message for rec in caplog.records if "of class 'Thing'" in rec.message] + assert len([m for m in messages if m.startswith("Rule 1 of class 'Thing' (First): skipped")]) == 1, messages + assert len([m for m in messages if m.startswith("Rule 2 of class 'Thing' (Second): skipped")]) == 1, messages + third = [m for m in messages if m.startswith("Rule 3 of class 'Thing' (Third)")] + assert len(third) == 2 and any("elseconditions" in m for m in third), messages + assert any("'Purple', which is not a permissible value" in m for m in third), messages + + +def test_rule_first_unknown_slot_is_the_reported_problem(caplog): + """A rule is skipped at its first untranslatable condition: with an + unknown guard and an unknown target, only the guard is reported.""" + schema = _single_rule_schema( + _rule({"no_such_guard": {"value_presence": "PRESENT"}}, {"no_such_target": {"equals_string": "x"}}) + ) + with caplog.at_level(logging.WARNING): + _parse_shacl(schema) + messages = [rec.message for rec in caplog.records if "Rule 1 of class 'Thing'" in rec.message] + assert len(messages) == 1 and "no_such_guard" in messages[0], messages + + +def test_rule_message_follows_rule_language(): + """A rule's own ``in_language`` takes precedence over the default language.""" + rule = _rule(_GUARD_PRESENT, {"target": {"equals_string": "x"}}, description="Das Ziel muss x sein.") + rule["in_language"] = "de" + g = _parse_shacl(_single_rule_schema(rule), default_language="en") + (node,) = g.objects(EX_SINGLE.Thing, SH.sparql) + assert set(g.objects(node, SH.message)) == {Literal("Das Ziel muss x sein.", lang="de")} + + +@pytest.mark.parametrize( + "rule", + [ + pytest.param( + _rule({"no_such_slot": {"value_presence": "PRESENT"}}, {"flag": {"equals_string": "true"}}), + id="boolean-guard-unknown-guard", + ), + pytest.param( + _rule({"no_such_slot": {"value_presence": "PRESENT"}}, {"target": {"equals_string": "x"}}), + id="presence-implies-value-unknown-guard", + ), + pytest.param( + _rule(_GUARD_PRESENT, {"no_such_slot": {"equals_string": "x"}}), + id="presence-implies-value-unknown-target", + ), + pytest.param( + _rule({"no_such_slot": {"equals_string": "x"}}, {"no_such_slot": {"maximum_cardinality": 1}}), + id="exclusive-value-unknown-slot", + ), + ], +) +def test_rule_unknown_slot_key_skipped(caplog, rule): + """A rule whose condition names a slot that does not exist is skipped: + a fabricated predicate would make a constraint that never fires (unknown + guard) or always fires (unknown target).""" + _assert_skipped(caplog, _single_rule_schema(rule), "'no_such_slot', which is not a slot") + + +@pytest.mark.parametrize( + "rule", + [ + pytest.param(_rule({"id": {"value_presence": "PRESENT"}}, {"target": {"equals_string": "x"}}), id="guard"), + pytest.param(_rule(_GUARD_PRESENT, {"id": {"equals_string": "x"}}), id="target"), + ], +) +def test_rule_identifier_slot_skipped(caplog, rule): + """The identifier is the node's IRI, not a property arc, so a condition on + it can never match a triple and the rule is skipped.""" + attributes = {"id": {"identifier": True, "range": "uriorcurie"}, "guard": {}, "target": {}} + _assert_skipped(caplog, _single_rule_schema(rule, attributes), "identifier slot 'id'") + + +_ESCAPING_SCHEMA_YAML = """ +id: https://example.org/escaping +name: escaping_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/escaping/ +imports: + - linkml:types +default_prefix: ex +default_range: string + +slots: + trigger: + range: string + slot_uri: ex:trigger + label: + range: string + slot_uri: ex:label + +classes: + Item: + class_uri: ex:Item + slots: + - trigger + - label + rules: + - description: If a trigger is present the label must equal the quoted marker. + preconditions: + slot_conditions: + trigger: + value_presence: PRESENT + postconditions: + slot_conditions: + label: + equals_string: 'a"b\\\\c' +""" + +EX_ESC = rdflib.Namespace("https://example.org/escaping/") + + +def test_rule_equals_string_special_chars_escaped(): + """An equals_string value with a quote and backslash must be escaped so the + generated SPARQL stays syntactically valid (no injection / broken query).""" + from rdflib.plugins.sparql import prepareQuery + + queries = _sparql_queries(_parse_shacl(_ESCAPING_SCHEMA_YAML), EX_ESC.Item) + assert len(queries) == 1 + query = queries[0] + + # Would raise ParseException on the unescaped `... = "a"b\c"` form. + prepareQuery(query) + assert '\\"' in query, f"double quote must be escaped, got:\n{query}" + assert "\\\\" in query, f"backslash must be escaped, got:\n{query}" + + +# =========================================================================== +# Operator exactness +# +# A rule is translated only when its conditions set exactly the operators a +# pattern translates. Each rule below would match a pattern if the converter +# dropped the extra operator, which would widen the precondition (false +# positives) or weaken the postcondition (false negatives). +# =========================================================================== + + +@pytest.mark.parametrize( + "extra", + [ + pytest.param({"pattern": "^x"}, id="pattern"), + pytest.param({"minimum_value": 0}, id="minimum_value"), + pytest.param({"maximum_value": 5}, id="maximum_value"), + pytest.param({"required": True}, id="required"), + pytest.param({"recommended": True}, id="recommended"), + pytest.param({"minimum_cardinality": 1}, id="minimum_cardinality"), + pytest.param({"exact_cardinality": 1}, id="exact_cardinality"), + pytest.param({"equals_number": 3}, id="equals_number"), + pytest.param({"has_member": {"equals_string": "x"}}, id="has_member"), + pytest.param({"any_of": [{"equals_string": "x"}]}, id="slot-any_of"), + ], +) +@pytest.mark.parametrize("condition", ["guard", "target"]) +def test_rule_extra_condition_operator_skipped(caplog, condition, extra): + """A guard or target condition that adds any operator to the ones a pattern + translates makes the rule untranslatable.""" + guard = {"value_presence": "PRESENT", **(extra if condition == "guard" else {})} + target = {"equals_string": "x", **(extra if condition == "target" else {})} + schema = _single_rule_schema(_rule({"guard": guard}, {"target": target})) + _assert_skipped(caplog, schema, "match none of the translated patterns") + + +@pytest.mark.parametrize("presence", ["ABSENT", "UNCOMMITTED"]) +def test_rule_guard_presence_other_than_present_skipped(caplog, presence): + """Only ``value_presence: PRESENT`` is a guard: reading ``ABSENT`` as a + guard would invert the trigger.""" + schema = _single_rule_schema(_rule({"guard": {"value_presence": presence}}, {"target": {"equals_string": "x"}})) + _assert_skipped(caplog, schema, "match none of the translated patterns") + + +@pytest.mark.parametrize("operator", ["any_of", "all_of", "exactly_one_of", "none_of"]) +@pytest.mark.parametrize("side", ["preconditions", "postconditions"]) +def test_rule_expression_level_operator_skipped(caplog, side, operator): + """An expression-level boolean operator on either side cannot be honoured + by any pattern; dropping it would widen or weaken the rule.""" + rule = _rule(_GUARD_PRESENT, {"target": {"equals_string": "x"}}) + rule[side][operator] = [{"slot_conditions": {"other": {"equals_string": "y"}}}] + _assert_skipped(caplog, _single_rule_schema(rule), f"its {side} use {operator}") + + +def test_rule_post_with_both_equals_forms_skipped(caplog): + """equals_string and equals_string_in set together is ambiguous — skip, + do not let one form silently win.""" + schema = _single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"equals_string": "a", "equals_string_in": ["b"]}})) + _assert_skipped(caplog, schema, "match none of the translated patterns") + + +def test_rule_exclusive_value_extra_operator_skipped(caplog): + """The exclusive-value pattern is exact too: a precondition mixing + equals_string with an unsupported operator is skipped.""" + schema = _single_rule_schema( + _rule({"tags": {"equals_string": "x", "pattern": "^x"}}, {"tags": {"maximum_cardinality": 1}}) + ) + _assert_skipped(caplog, schema, "match none of the translated patterns") + + +_STRING_TRUE_SCHEMA_YAML = """ +id: https://example.org/string-true +name: string_true +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/string-true/ +imports: + - linkml:types +default_prefix: ex +default_range: string +slots: + opt: + slot_uri: ex:opt + status: + range: string + slot_uri: ex:status +classes: + Conf: + class_uri: ex:Conf + slots: [opt, status] + rules: + - description: If opt is present, status must be the string "true". + preconditions: + slot_conditions: + opt: + value_presence: PRESENT + postconditions: + slot_conditions: + status: + equals_string: "true" +""" + + +def test_rule_equals_true_on_string_slot_uses_piv(): + """equals_string "true" on a NON-boolean slot is compared as the string + "true", not as the boolean of the boolean guard.""" + queries = _sparql_queries(_parse_shacl(_STRING_TRUE_SCHEMA_YAML), URIRef("https://example.org/string-true/Conf")) + assert len(queries) == 1 + assert '?value = "true"' in queries[0], f"string-range 'true' must be a string comparison, got:\n{queries[0]}" + assert "?value = true" not in queries[0], "the boolean guard must not apply to a string slot" + + +@pytest.mark.parametrize( + "status,violates", + [ + pytest.param('"true"', False, id="plain"), + pytest.param('"true"^^xsd:string', False, id="xsd-string"), + pytest.param('"other"', True, id="other"), + ], +) +def test_rule_equals_true_on_string_slot_pyshacl_end_to_end(status, violates): + """status "true" (string) satisfies the rule in either RDF 1.1-identical form.""" + data = ( + "@prefix ex: .\n" + "@prefix xsd: .\n" + f'ex:x a ex:Conf ; ex:opt "x" ; ex:status {status} .' + ) + _, focus_nodes = _validate_rules(_STRING_TRUE_SCHEMA_YAML, data) + assert focus_nodes == ({URIRef("https://example.org/string-true/x")} if violates else set()) diff --git a/tests/linkml/test_validator/test_shacl_validation_plugin.py b/tests/linkml/test_validator/test_shacl_validation_plugin.py index 19d7f5d9be..ba912166b0 100644 --- a/tests/linkml/test_validator/test_shacl_validation_plugin.py +++ b/tests/linkml/test_validator/test_shacl_validation_plugin.py @@ -11,6 +11,9 @@ # not re-exported, so that pyshacl stays an optional dependency. from linkml.validator.plugins.shacl_validation_plugin import ShaclValidationPlugin from linkml.validator.report import Severity +from linkml.validator.validation_context import ValidationContext +from linkml_runtime.linkml_model import SchemaDefinition +from linkml_runtime.loaders import yaml_loader def test_conforming_instance_yields_no_results(validation_context): @@ -87,3 +90,88 @@ def test_generated_shapes_are_cached(validation_context): list(plugin.process({"id": "P:1", "name": "One"}, validation_context)) assert len(plugin._loaded_graphs) == 1 + + +_RULES_SCHEMA_YAML = """ +id: https://example.org/plugin-rules +name: plugin_rules +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/plugin-rules/ +imports: + - linkml:types +default_prefix: ex +default_range: string +enums: + Color: + permissible_values: + Red: + meaning: ex:Red + Blue: + meaning: ex:Blue + Code: + permissible_values: + alpha: + beta: +classes: + Base: + attributes: + id: + identifier: true + guard: {} + label: {} + color: + range: Color + code: + range: Code + rules: + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + label: + equals_string: x + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + color: + equals_string: Red + - preconditions: + slot_conditions: + guard: + value_presence: PRESENT + postconditions: + slot_conditions: + code: + equals_string_in: [alpha] + Child: + is_a: Base +""" + + +@pytest.mark.parametrize("target_class", ["Base", "Child"]) +@pytest.mark.parametrize( + "instance,violates", + [ + pytest.param({"guard": "g", "label": "x", "color": "Red", "code": "alpha"}, False, id="conforming"), + pytest.param({"guard": "g", "label": "y", "color": "Red", "code": "alpha"}, True, id="string"), + pytest.param({"guard": "g", "label": "x", "color": "Blue", "code": "alpha"}, True, id="enum-meaning"), + pytest.param({"guard": "g", "label": "x", "color": "Red", "code": "beta"}, True, id="enum-literal"), + pytest.param({"guard": "g", "color": "Red", "code": "alpha"}, True, id="target-absent"), + pytest.param({"label": "y", "color": "Blue", "code": "beta"}, False, id="unguarded"), + ], +) +def test_rule_violation_is_reported(target_class, instance, violates): + """Rules translated to ``sh:sparql`` are enforced through the plugin, whose + data graph comes from the LinkML RDF dumper, on instances of the declaring + class and of its subclasses alike.""" + context = ValidationContext(yaml_loader.loads(_RULES_SCHEMA_YAML, SchemaDefinition), target_class) + results = list(ShaclValidationPlugin().process({"id": "ex:x", **instance}, context)) + + assert all(result.severity is Severity.ERROR for result in results) + assert any("SPARQLConstraintComponent" in result.message for result in results) is violates From 9510586fee99a440a195b75d73570e7f9b060de3 Mon Sep 17 00:00:00 2001 From: Rayene Messaoud Date: Tue, 6 Oct 2026 15:45:50 +0200 Subject: [PATCH 06/17] feat(gen-shacl): add a compositional fallback for rule-to-SPARQL conversion A rule that no named pattern matches is now composed from the operators it uses: each precondition becomes SPARQL filters on $this, and each postcondition slot gets its own constraint, the union of the ways to violate that slot's condition, binding the offending value to ?value (SHACL sections 5.3.1 and 5.3.2). A rule P -> A and B is violated exactly when P and not A, or P and not B, holds. A condition may use value_presence, required, equals_string, equals_string_in, equals_number, pattern, minimum_value, maximum_value, the slot-level any_of, all_of, exactly_one_of and none_of over those value tests, a range_expression on the slot's class (nested to any depth) and has_member. The semantics follow the JSON Schema generator's if/then: - presence: value_presence decides, then required, then a default: a precondition requires its slot, a postcondition unless the rule is open_world, an inner condition of a nested expression never; - value operators and range_expression apply to every value of the slot (05validation.md: "all members"), has_member to some value; - equals_string(_in) only on string, enum and boolean slots (the named patterns' range check); equals_number and the bounds only on the numeric datatypes of SPARQL 1.1 section 17.1, guarded with isNumeric, and only finite numbers; pattern as REGEX on the value's string form, as sh:pattern reads it (SHACL section 4.4.2); - a value is one term, so each member of a slot-level boolean is true or false, and the operators are those of 05validation.md: exactly_one_of counts the members that hold. Any other operator, or a rule with empty pre- or postconditions, skips the rule with one warning, so a rule is never partially translated. A rule skipped part-way no longer reports the notes of the abandoned translation. The compliance suite now runs SHACL in the rule tests, and every rule of test_preconditions and test_preconditions_combos is translated, so each instance is checked. Co-authored-by: jdsika Signed-off-by: jdsika --- .../linkml/src/linkml/generators/shaclgen.py | 496 ++++++- .../test_boolean_slot_compliance.py | 16 +- tests/linkml/test_generators/test_shaclgen.py | 1209 ++++++++++++++++- 3 files changed, 1639 insertions(+), 82 deletions(-) diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 00fedfc779..33fc69515a 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -1,4 +1,5 @@ import logging +import math import os import re import string @@ -72,6 +73,11 @@ def _validate_message_template(template: str) -> None: ) +_PRESENT = PresenceEnum(PresenceEnum.PRESENT) +_ABSENT = PresenceEnum(PresenceEnum.ABSENT) +_UNCOMMITTED = PresenceEnum(PresenceEnum.UNCOMMITTED) + + @dataclass class _RuleSite: """A rule as it is translated for the shape of a class. @@ -183,9 +189,9 @@ class ShaclGenerator(Generator): emit_rules: bool = True """Emit ``sh:sparql`` constraints from LinkML ``rules:`` blocks. - When ``True`` (default), recognised rule patterns are translated into - SHACL-SPARQL constraints (``sh:SPARQLConstraint``) on the corresponding - ``sh:NodeShape``. Currently two patterns are recognised: + When ``True`` (default), rules are translated into SHACL-SPARQL + constraints (``sh:SPARQLConstraint``) on the corresponding + ``sh:NodeShape``. Two patterns are recognised first: * *Presence implies value* — a precondition with ``value_presence: PRESENT`` on a guard slot and a postcondition with ``equals_string`` or @@ -196,8 +202,12 @@ class ShaclGenerator(Generator): on a slot and a postcondition with ``maximum_cardinality`` on the *same* slot. - A rule that cannot be translated exactly is skipped with a warning. A - class shape carries the rules of the class's ancestors and mixins too. + Any other rule is composed from the operators its conditions use + (presence, value comparisons, ``pattern``, numeric bounds, boolean + combinations of these, ``range_expression``, ``has_member``), with one + constraint per postcondition slot. A rule that + cannot be translated exactly is skipped with a warning. A class shape + carries the rules of the class's ancestors and mixins too. See `W3C SHACL §5 `_ and `linkml/linkml#2464 `_. @@ -1014,13 +1024,20 @@ def _add_rules(self, g: Graph, shape_uri: URIRef, cls: ClassDefinition) -> None: applies to every value of the slot (``05validation.md``), so it is not this pattern. - Every emitted constraint translates its rule exactly. A rule that + * **Composed** — any other rule is composed from the operators its + conditions use: presence, ``equals_string(_in)``, ``equals_number``, + ``pattern`` and numeric bounds on each value, boolean combinations + of those, a ``range_expression`` on the slot's class, and + ``has_member``, with one constraint per postcondition slot (see + :meth:`_compose_rule_sparql`). + + Every emitted constraint translates its rule exactly. A rule that cannot be translated exactly is skipped with a warning naming the - reason: an operator combination outside these patterns, a condition on - an unknown or identifier slot, a value the target's range cannot hold - (see :meth:`_value_terms`), or ``bidirectional``. Of a rule with ``elseconditions`` the forward - (if/then) direction is emitted, exactly, and a warning reports the - else branch as not enforced. + reason: an operator the translations do not support, a condition on an + unknown or identifier slot, a value the slot's range cannot hold (see + :meth:`_value_terms`), or ``bidirectional``. Of a rule with + ``elseconditions`` the forward (if/then) direction is emitted, exactly, + and a warning reports the else branch as not enforced. A rule applies to "all members of this class" (metamodel ``rules``), so the shape of a class carries the rules of its ancestors and mixins @@ -1043,23 +1060,23 @@ def _add_rule(self, g: Graph, shape_uri: URIRef, site: _RuleSite) -> None: if rule.bidirectional: self._skip_rule(site, "bidirectional rules are not supported") return - sparql_query = self._rule_to_sparql(site) - if sparql_query is None: + sparql_queries = self._rule_to_sparql(site) + if sparql_queries is None: return if rule.elseconditions is not None: self._warn_rule( site, "its elseconditions are not enforced; SHACL-SPARQL generation emits the if/then direction only" ) message = Literal(rule.description, lang=self._resolve_language(rule)) if rule.description else None - if self._has_sparql_constraint(g, shape_uri, Literal(sparql_query), message): - return - - constraint = BNode() - g.add((shape_uri, SH.sparql, constraint)) - g.add((constraint, RDF.type, SH.SPARQLConstraint)) - if message is not None: - g.add((constraint, SH.message, message)) - g.add((constraint, SH.select, Literal(sparql_query))) + for sparql_query in sparql_queries: + if self._has_sparql_constraint(g, shape_uri, Literal(sparql_query), message): + continue + constraint = BNode() + g.add((shape_uri, SH.sparql, constraint)) + g.add((constraint, RDF.type, SH.SPARQLConstraint)) + if message is not None: + g.add((constraint, SH.message, message)) + g.add((constraint, SH.select, Literal(sparql_query))) @staticmethod def _has_sparql_constraint(g: Graph, shape_uri: URIRef, query: Literal, message: Literal | None) -> bool: @@ -1093,7 +1110,17 @@ def _report_rule_problems(self) -> None: logger.warning("%s: %s%s.", rule, problem, shapes) def _skip_rule(self, site: _RuleSite, reason: str) -> None: - """Log that the rule at *site* is not translated, and why.""" + """Log that the rule at *site* is not translated, and why. + + This replaces the problems recorded so far while translating the rule + for the shape of ``site.cls``, which describe a constraint that is not + emitted. + """ + for key, (_, classes) in list(self._rule_problems.items()): + if key[:2] == (site.owner, site.index) and site.cls.name in classes: + classes.remove(site.cls.name) + if not classes: + del self._rule_problems[key] self._warn_rule(site, f"skipped, because {reason}") # The lexical space of xsd:boolean and the value each lexical form maps to @@ -1101,10 +1128,15 @@ def _skip_rule(self, site: _RuleSite, reason: str) -> None: # as SPARQL boolean literals. _XSD_BOOLEAN_LEXICAL = {"true": "true", "1": "true", "false": "false", "0": "false"} - _NO_PATTERN = "its conditions match none of the translated patterns (presence implies value, exclusive value)" + # Prefixes the reason a rule is skipped when it uses an operator, or a + # combination of conditions, that no translation supports. + _NO_PATTERN = ( + "its conditions match none of the translated patterns (presence implies value, exclusive value, " + "and the compositional translation)" + ) - def _rule_to_sparql(self, site: _RuleSite) -> str | None: - """Translate the rule at *site* to a SPARQL SELECT query. + def _rule_to_sparql(self, site: _RuleSite) -> list[str] | None: + """Translate the rule at *site* to SPARQL SELECT queries, one per constraint. Returns ``None``, after a warning naming the reason, when the rule matches no supported pattern exactly. Each pattern requires its @@ -1124,9 +1156,11 @@ def _rule_to_sparql(self, site: _RuleSite) -> str | None: if untranslated: self._skip_rule(site, f"its {side} use {', '.join(sorted(untranslated))}") return None + if not expression.slot_conditions: + self._skip_rule(site, f"its {side} constrain no slot") + return None if len(pre.slot_conditions) != 1 or len(post.slot_conditions) != 1: - self._skip_rule(site, self._NO_PATTERN) - return None + return self._compose_rule_sparql(site, pre.slot_conditions, post.slot_conditions) ((pre_name, pre_cond),) = pre.slot_conditions.items() ((post_name, post_cond),) = post.slot_conditions.items() @@ -1142,14 +1176,14 @@ def _rule_to_sparql(self, site: _RuleSite) -> str | None: # the case `equals_string: "true"` on a boolean flag. if ( pre_ops == {"value_presence"} - and pre_cond.value_presence == PresenceEnum(PresenceEnum.PRESENT) + and pre_cond.value_presence == _PRESENT and post_ops in ({"equals_string"}, {"equals_string_in"}) ): values = [post_cond.equals_string] if post_ops == {"equals_string"} else list(post_cond.equals_string_in) terms = self._value_terms(site, post_slot, values) if terms is None: return None - return self._build_presence_implies_value_sparql(pre_slot, post_slot, terms, bool(site.rule.open_world)) + return [self._build_presence_implies_value_sparql(pre_slot, post_slot, terms, bool(site.rule.open_world))] # Exclusive value: "if V is one of the values of slot X, then X has at # most N values". @@ -1166,10 +1200,9 @@ def _rule_to_sparql(self, site: _RuleSite) -> str | None: terms = self._value_terms(site, pre_slot, [exclusive]) if terms is None: return None - return self._build_exclusive_value_sparql(pre_slot, terms[0], int(post_cond.maximum_cardinality)) + return [self._build_exclusive_value_sparql(pre_slot, terms[0], int(post_cond.maximum_cardinality))] - self._skip_rule(site, self._NO_PATTERN) - return None + return self._compose_rule_sparql(site, pre.slot_conditions, post.slot_conditions) def _exclusive_value(self, condition: SlotDefinition, operators: set[str]) -> str | None: """The value V of an exclusive-value precondition, or ``None`` if *condition* is not one. @@ -1182,6 +1215,394 @@ def _exclusive_value(self, condition: SlotDefinition, operators: set[str]) -> st return condition.has_member.equals_string return None + # Operators the compositional translation supports on a condition: those a + # value is tested against (:meth:`_value_test`), including the boolean + # combinations of such tests, those a value's own slots are tested against + # (``range_expression``), and those stating whether the slot is present or + # some value satisfies a condition. + _BOOLEAN_OPERATORS = frozenset({"any_of", "all_of", "exactly_one_of", "none_of"}) + _TEST_OPERATORS = ( + frozenset({"equals_string", "equals_string_in", "equals_number", "minimum_value", "maximum_value", "pattern"}) + | _BOOLEAN_OPERATORS + ) + _VALUE_OPERATORS = _TEST_OPERATORS | {"range_expression"} + _CONDITION_OPERATORS = _VALUE_OPERATORS | {"required", "value_presence", "has_member"} + + # The datatypes SPARQL compares numerically: the numeric types and the types + # derived from them (SPARQL 1.1 §17.1, + # ). + _SPARQL_NUMERIC_DATATYPES = frozenset( + str(XSD[name]) + for name in ( + "integer", + "decimal", + "float", + "double", + "nonPositiveInteger", + "negativeInteger", + "long", + "int", + "short", + "byte", + "nonNegativeInteger", + "unsignedLong", + "unsignedInt", + "unsignedShort", + "unsignedByte", + "positiveInteger", + ) + ) + + def _compose_rule_sparql( + self, site: _RuleSite, pre: dict[str, SlotDefinition], post: dict[str, SlotDefinition] + ) -> list[str] | None: + """Compose the queries of a rule no named pattern matches from its slot conditions *pre* and *post*. + + The caller passes them after checking that the pre- and postconditions + set nothing else (:meth:`_rule_to_sparql`). Each precondition becomes + filters on ``$this``. Each postcondition slot gets its own query, the + union of the ways to violate that slot's condition: a rule + ``P -> A and B`` is violated exactly when ``P and not A`` or + ``P and not B`` holds. Each query selects the focus nodes that satisfy + every precondition and violate one postcondition (`SHACL §5.3.1 + `_), with the + slot as ``?path`` and the offending value where there is one (`SHACL + §5.3.2 `_). + If any condition cannot be translated, the whole rule is skipped. + + Whether a condition requires its slot is decided by :meth:`_presence`. + Its value operators and ``range_expression`` apply to every value of + the slot: a slot constraint applies to all members of a collection + (``05validation.md``), as the JSON Schema generator applies it through + ``items``. ``has_member`` requires some value to satisfy its + condition. A condition may use :data:`_CONDITION_OPERATORS`; any other + operator skips the rule, as does an untranslatable value. + """ + filters: list[str] = [] + for index, (slot_name, condition) in enumerate(pre.items()): + slot = self._rule_condition_slot(site, slot_name) + if slot is None: + return None + slot_filters = self._precondition_filters(site, slot, condition, f"?pre{index}") + if slot_filters is None: + return None + filters.extend(slot_filters) + queries: list[str] = [] + for post_name, post_cond in post.items(): + post_slot = self._rule_condition_slot(site, post_name) + if post_slot is None: + return None + violations = self._postcondition_violations(site, post_slot, post_cond, bool(site.rule.open_world)) + if violations is None: + return None + lines = list(filters) + if len(violations) == 1: + lines.extend(violations) + else: + lines.append(f"{{ {violations[0]} }}") + lines.extend(f"UNION {{ {violation} }}" for violation in violations[1:]) + body = "\n".join(f" {line}" for line in lines) + queries.append( + f"SELECT DISTINCT $this (<{self._slot_iri(post_slot)}> AS ?path) ?value WHERE {{\n{body}\n}}" + ) + return queries + + @staticmethod + def _presence(condition: SlotDefinition, default: PresenceEnum) -> PresenceEnum: + """Whether the rule *condition* requires its slot to be ``PRESENT``, ``ABSENT``, or neither (``UNCOMMITTED``). + + ``value_presence`` decides, then ``required``, then *default*, as the + JSON Schema generator decides whether a rule condition requires its + property: by default a precondition does, a postcondition unless the + rule is ``open_world`` ("the postconditions may be omitted in instance + data", metamodel), and an inner condition of a nested expression never. + """ + if condition.value_presence is not None: + return PresenceEnum(condition.value_presence) + if condition.required is not None: + return _PRESENT if condition.required else _UNCOMMITTED + return default + + def _precondition_filters( + self, site: _RuleSite, slot: SlotDefinition, condition: SlotDefinition, var: str + ) -> list[str] | None: + """The SPARQL filters that hold when *slot* of ``$this`` satisfies the precondition *condition*: + the slot is present (by default) or absent as :meth:`_presence` decides, and nothing violates it.""" + checked = self._condition_violations(site, slot, condition, "$this", var, _PRESENT, "precondition") + if checked is None: + return None + presence, violations = checked + value = f"$this <{self._slot_iri(slot)}> {var} ." + filters: list[str] = [] + if presence == _PRESENT: + filters.append(f"FILTER EXISTS {{ {value} }}") + elif presence == _ABSENT: + filters.append(f"FILTER NOT EXISTS {{ {value} }}") + return filters + [f"FILTER NOT EXISTS {{ {violation} }}" for violation in violations] + + def _postcondition_violations( + self, site: _RuleSite, slot: SlotDefinition, condition: SlotDefinition, open_world: bool + ) -> list[str] | None: + """The patterns by which ``$this`` violates the postcondition *condition* on *slot*. + + A pattern matching an offending value binds it to ``?value``. With + *open_world* "the postconditions may be omitted in instance data" + (metamodel ``open_world``), so the slot is not required by default. + """ + default = _UNCOMMITTED if open_world else _PRESENT + checked = self._condition_violations(site, slot, condition, "$this", "?value", default, "postcondition") + if checked is None: + return None + presence, violations = checked + presence_violation = self._presence_violation("$this", slot, "?value", presence) + violations = [presence_violation, *violations] if presence_violation else violations + if not violations: + self._skip_rule(site, f"its postcondition on slot {slot.name!r} constrains nothing") + return None + return violations + + def _presence_violation(self, subject: str, slot: SlotDefinition, var: str, presence: PresenceEnum) -> str | None: + """The pattern matching *subject* when its *slot* violates *presence*, binding *var* to a value that must + be absent; ``None`` when *presence* is ``UNCOMMITTED``.""" + value = f"{subject} <{self._slot_iri(slot)}> {var} ." + if presence == _PRESENT: + return f"FILTER NOT EXISTS {{ {value} }}" + if presence == _ABSENT: + return value + return None + + def _condition_violations( + self, + site: _RuleSite, + slot: SlotDefinition, + condition: SlotDefinition, + subject: str, + var: str, + default: PresenceEnum, + role: str, + ) -> tuple[PresenceEnum, list[str]] | None: + """The presence the *role* *condition* on *slot* of *subject* requires, and its other violations. + + A value bound to *var* violates the value operators or the + ``range_expression`` (:meth:`_value_violations`); ``has_member`` is + violated when the slot has values and none satisfies the member + condition. + """ + operators = self._set_operator_fields(condition) + if not operators <= self._CONDITION_OPERATORS: + self._skip_rule(site, self._unsupported(slot, operators, self._CONDITION_OPERATORS, role)) + return None + violations = self._value_violations(site, slot, condition, subject, var) + if violations is None: + return None + member_condition = condition.has_member + if member_condition is not None: + member_operators = self._set_operator_fields(member_condition) + if not member_operators or not member_operators <= self._VALUE_OPERATORS: + self._skip_rule(site, self._unsupported(slot, member_operators, self._VALUE_OPERATORS, "has_member")) + return None + member = f"{var}_member" + member_violations = self._value_violations(site, slot, member_condition, subject, member) + if member_violations is None: + return None + iri = self._slot_iri(slot) + satisfied = "".join(f" FILTER NOT EXISTS {{ {violation} }}" for violation in member_violations) + violations.append( + f"FILTER EXISTS {{ {subject} <{iri}> {var}_any . }} " + f"FILTER NOT EXISTS {{ {subject} <{iri}> {member} .{satisfied} }}" + ) + return self._presence(condition, default), violations + + def _value_violations( + self, site: _RuleSite, slot: SlotDefinition, condition: SlotDefinition, subject: str, var: str + ) -> list[str] | None: + """The patterns matching a value *var* of *slot* of *subject* that violates a value operator of *condition*. + + The value operators are tested on the value itself, a + ``range_expression`` on the value's own slots + (:meth:`_nested_violations`). + """ + value = f"{subject} <{self._slot_iri(slot)}> {var} ." + violations: list[str] = [] + if condition.range_expression is not None: + nested = self._nested_violations(site, slot, condition.range_expression, var) + if nested is None: + return None + violations.extend(f"{value} {violation}" for violation in nested) + test = self._value_test(site, slot, condition, var) + if test is None: + return None + if test: + violations.append(f"{value} FILTER ( !( {test} ) )") + return violations + + def _nested_violations( + self, site: _RuleSite, slot: SlotDefinition, expression: AnonymousClassExpression, node: str + ) -> list[str] | None: + """The patterns by which an object *node*, a value of *slot*, violates the class expression *expression*. + + *expression* must consist of slot conditions on the slot's range class, + resolved in that class's induced context. Each one holds for an absent + inner slot unless it requires the slot (:meth:`_presence`), as a slot + constraint applies to the values present and as the JSON Schema + generator reads a nested expression. On a reference that is not + inlined, the conditions apply to the referenced node's triples in the + data graph. + """ + sv = self.schemaview + operators = self._set_operator_fields(expression) + if operators != {"slot_conditions"}: + self._skip_rule( + site, self._unsupported(slot, operators, frozenset({"slot_conditions"}), "range_expression") + ) + return None + range_name = self._slot_range(slot) + if range_name not in sv.all_classes(): + self._skip_rule(site, f"its range_expression is on slot {slot.name!r}, whose range is not a class") + return None + range_class = sv.get_class(range_name) + violations: list[str] = [] + for index, (inner_name, condition) in enumerate(expression.slot_conditions.items()): + inner = self._rule_condition_slot(site, inner_name, range_class) + if inner is None: + return None + var = f"{node}_{index}" + checked = self._condition_violations(site, inner, condition, node, var, _UNCOMMITTED, "inner condition") + if checked is None: + return None + presence, inner_violations = checked + presence_violation = self._presence_violation(node, inner, var, presence) + if presence_violation: + violations.append(presence_violation) + violations.extend(inner_violations) + return violations + + def _value_test(self, site: _RuleSite, slot: SlotDefinition, condition: SlotDefinition, var: str) -> str | None: + """The SPARQL test that a value *var* of *slot* satisfies the value operators of *condition*. + + Returns ``""`` when *condition* sets no value operator, and ``None`` + (rule skipped) when a value cannot be translated. ``equals_string`` + and ``equals_string_in`` compare as :meth:`_value_terms` renders them. + ``pattern`` is matched against the string form of the value with + ``REGEX``, as ``sh:pattern`` is (`SHACL §4.4.2 + `_). + ``equals_number`` and the inclusive bounds ``minimum_value`` / + ``maximum_value`` (metamodel) are compared numerically, so the slot's + datatype must be one SPARQL compares as a number + (:data:`_SPARQL_NUMERIC_DATATYPES`). ``any_of``, ``all_of``, + ``exactly_one_of`` and ``none_of`` combine the tests of their members, + which may use only the operators of :data:`_TEST_OPERATORS`; a value is + a single term, so each member test is true or false and the operators + are the boolean ones of ``05validation.md``. Comparing a + value of another type with a number is a type error (`SPARQL 1.1 §17.3 + `_), which some + engines resolve as an ordering anyway, so `isNumeric + `_ guards the + comparison; an engine that takes an ill-typed literal such as + ``"abc"^^xsd:integer`` for a number still compares it. Every part is + false, rather than an error, for a value it cannot compare (see + :meth:`_sparql_is_one_of`). + """ + parts: list[str] = [] + for values in ( + [condition.equals_string] if condition.equals_string is not None else None, + list(condition.equals_string_in) if condition.equals_string_in else None, + ): + if values is None: + continue + terms = self._value_terms(site, slot, values) + if terms is None: + return None + parts.append(self._sparql_is_one_of(var, terms)) + if condition.pattern is not None: + pattern = self._sparql_string_literal(condition.pattern) + parts.append(f"COALESCE( REGEX( STR( {var} ), {pattern} ), false )") + bounds = { + name: (bound, operator) + for name, bound, operator in ( + ("equals_number", condition.equals_number, "="), + ("minimum_value", condition.minimum_value, ">="), + ("maximum_value", condition.maximum_value, "<="), + ) + if bound is not None + } + r = self._slot_range(slot) + if bounds and self._type_uri(r) not in self._SPARQL_NUMERIC_DATATYPES: + self._skip_rule( + site, + f"its {' and '.join(bounds)} on slot {slot.name!r} {'need' if len(bounds) > 1 else 'needs'} a type " + "with a numeric datatype, and " + (f"the slot's range is {r!r}" if r else "the slot has no range"), + ) + return None + comparisons: list[str] = [] + for bound, operator in bounds.values(): + number = self._sparql_number(bound) + if number is None: + self._skip_rule(site, f"its bound {bound!r} on slot {slot.name!r} is not a finite number") + return None + comparisons.append(f"{var} {operator} {number}") + if comparisons: + parts.append(f"COALESCE( {' && '.join([f'isNumeric( {var} )', *comparisons])}, false )") + for operator in sorted(self._BOOLEAN_OPERATORS): + members = getattr(condition, operator) or [] + if not members: + continue + tests: list[str] = [] + for member in members: + member_operators = self._set_operator_fields(member) + if not member_operators or not member_operators <= self._TEST_OPERATORS: + self._skip_rule( + site, self._unsupported(slot, member_operators, self._TEST_OPERATORS, f"{operator} member") + ) + return None + test = self._value_test(site, slot, member, var) + if test is None: + return None + tests.append(f"( {test} )") + parts.append(self._combine_tests(operator, tests)) + return " && ".join(parts) + + @staticmethod + def _combine_tests(operator: str, tests: list[str]) -> str: + """The SPARQL expression combining the member *tests* of the boolean *operator*. + + ``exactly_one_of`` counts the members that hold (``05validation.md``: + one member true and all others false). + """ + if operator == "any_of": + return f"( {' || '.join(tests)} )" + if operator == "all_of": + return f"( {' && '.join(tests)} )" + if operator == "none_of": + return f"!( {' || '.join(tests)} )" + return f"( ( {' + '.join(f'IF( {test}, 1, 0 )' for test in tests)} ) = 1 )" + + @staticmethod + def _sparql_number(value: object) -> str | None: + """Render a ``minimum_value`` / ``maximum_value`` bound as a SPARQL numeric literal, or ``None``. + + The metamodel range of both is ``Anything``, so strings, dates, + booleans and ``.nan`` / ``.inf`` reach the generator unchanged. + Interpolated raw, ``"abc"`` makes the query unparsable and a date such + as ``2020-01-01`` parses as arithmetic; only ``int`` and finite + ``float`` values are rendered, and their ``str`` is a plain numeric + token that SPARQL compares with numeric type promotion. + """ + if isinstance(value, bool) or not isinstance(value, int | float): + return None + if isinstance(value, float) and not math.isfinite(value): + return None + return str(value) + + @classmethod + def _unsupported(cls, slot: SlotDefinition, operators: set[str], supported: frozenset[str], role: str) -> str: + """The reason a condition with *operators* in *role* on *slot* is not translated.""" + used = ", ".join(sorted(operators)) or "no operator" + return ( + f"{cls._NO_PATTERN}; its {role} on slot {slot.name!r} uses {used}, where the compositional " + f"translation supports only {', '.join(sorted(supported))}" + ) + def _rule_slot(self, cls: ClassDefinition, slot_name: str) -> SlotDefinition | None: """Resolve a rule condition's slot key to the slot it names, or ``None`` when no such slot exists. @@ -1201,15 +1622,20 @@ def _rule_slot(self, cls: ClassDefinition, slot_name: str) -> SlotDefinition | N return sv.induced_slot(canonical, cls.name) return sv.get_slot(slot_name) - def _rule_condition_slot(self, site: _RuleSite, slot_name: str) -> SlotDefinition | None: + def _rule_condition_slot( + self, site: _RuleSite, slot_name: str, cls: ClassDefinition | None = None + ) -> SlotDefinition | None: """The slot a condition of the rule at *site* names, or ``None`` (rule skipped) when it cannot be queried. + The slot is resolved in the context of ``site.cls``, or of *cls* for an + inner condition on the range class of a nested expression. + An unknown name would make the query use a predicate no shape or data uses. An identifier slot is the node's IRI, not a property arc (the main slot loop does not require it either), so a condition on it can never match. """ - slot = self._rule_slot(site.cls, slot_name) + slot = self._rule_slot(cls if cls is not None else site.cls, slot_name) if slot is None: self._skip_rule(site, f"its condition names {slot_name!r}, which is not a slot") return None diff --git a/tests/linkml/test_compliance/test_boolean_slot_compliance.py b/tests/linkml/test_compliance/test_boolean_slot_compliance.py index c62d9774d2..740853eed5 100644 --- a/tests/linkml/test_compliance/test_boolean_slot_compliance.py +++ b/tests/linkml/test_compliance/test_boolean_slot_compliance.py @@ -2177,8 +2177,6 @@ def test_preconditions(framework, s1, s2, is_valid): :param is_valid: whether the data is valid :return: """ - if framework == SHACL: - pytest.skip("shaclgen does not support rules yet") if framework == PANDERA_POLARS_CLASS: pytest.skip("panderagen does not support rules yet") @@ -2234,8 +2232,8 @@ def test_preconditions(framework, s1, s2, is_valid): ], ) expected_behavior = ValidationBehavior.IMPLEMENTS - if framework not in [JSON_SCHEMA, OWL]: - # only JSON Schema and OWL supports rules + if framework not in [JSON_SCHEMA, OWL, SHACL]: + # only JSON Schema, OWL and SHACL support rules expected_behavior = ValidationBehavior.INCOMPLETE check_data( schema, @@ -2393,8 +2391,6 @@ def test_preconditions_combos( :param is_valid: :return: """ - if framework == SHACL: - pytest.skip("shaclgen does not support rules yet") rule = {} if preconditions: rule["preconditions"] = { @@ -2426,8 +2422,8 @@ def test_preconditions_combos( ], ) expected_behavior = ValidationBehavior.IMPLEMENTS - if framework not in [JSON_SCHEMA, OWL]: - # only JSON Schema and OWL supports rules + if framework not in [JSON_SCHEMA, OWL, SHACL]: + # only JSON Schema, OWL and SHACL support rules expected_behavior = ValidationBehavior.INCOMPLETE if framework == OWL and schema_name == "Disjunctive": # TODO: investigate why the OWLAPI does not correctly parse the resulting structure @@ -2646,8 +2642,6 @@ def test_value_presence_in_rules(framework, multivalued, data_name, instance, is :param is_valid: whether the data is valid or not :return: """ - if framework == SHACL: - pytest.skip("shaclgen does not support boolean expressions yet") if framework == PANDERA_POLARS_CLASS: pytest.skip("panderagen does not support boolean expressions yet") classes = { @@ -2704,7 +2698,7 @@ def test_value_presence_in_rules(framework, multivalued, data_name, instance, is core_elements=["preconditions", "postconditions", "slot_conditions", "value_presence"], ) expected_behavior = ValidationBehavior.IMPLEMENTS - if framework != JSON_SCHEMA: + if framework not in [JSON_SCHEMA, SHACL]: if not is_valid: expected_behavior = ValidationBehavior.INCOMPLETE if multivalued: diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index 40eb3c402c..c0dc1863b5 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -5062,17 +5062,24 @@ def test_rule_equals_string_agrees_with_json_schema( @pytest.mark.parametrize("instance,valid_closed_world,valid_open_world", _RULE_INSTANCES) @pytest.mark.parametrize("target_class", ["Thing", "Sub"]) @pytest.mark.parametrize("open_world", [False, True]) +@pytest.mark.parametrize( + "guard_extra", [None, {"required": True}], ids=["presence-implies-value", "composed-equivalent"] +) def test_rule_inheritance_and_open_world_agree_with_json_schema( - target_range, value, other, instance, valid_closed_world, valid_open_world, target_class, open_world + target_range, value, other, instance, valid_closed_world, valid_open_world, target_class, open_world, guard_extra ): """A rule binds the instances of subclasses of its class, and ``open_world`` lets the postcondition be omitted, in SHACL as in JSON Schema. The metamodel defines ``open_world`` as "the postconditions may be omitted - in instance data", so an absent target satisfies an open-world rule. + in instance data", so an absent target satisfies an open-world rule. A + guard that also states ``required: true`` means the same and is composed + rather than matched by the named pattern, with the same verdicts. """ obj = {key: text.format(value=value, other=other) for key, text in instance.items()} - schema = _rule_target_schema(target_range, value, open_world=open_world) + schema = _rule_target_schema(target_range, value, open_world=open_world, guard_extra=guard_extra) + (query,) = _sparql_queries(_parse_shacl(schema), EX_RTR[target_class]) + assert ("?pre0" in query) == (guard_extra is not None), query # the composed form filters on ?pre0 valid = valid_open_world if open_world else valid_closed_world assert _json_schema_and_shacl_verdicts(schema, target_class, obj) == (valid, valid) @@ -5824,6 +5831,8 @@ def _rule(pre: dict, post: dict, **rule_fields) -> dict: _GUARD_PRESENT = {"guard": {"value_presence": "PRESENT"}} +# a postcondition no translation supports: equals_expression compares with another slot +_TARGET_EQUALS_OTHER = {"target": {"equals_expression": "{other}"}} def _assert_skipped(caplog, schema: str, reason: str) -> None: @@ -5935,9 +5944,7 @@ def test_rule_problem_reported_once_for_inheriting_classes(caplog): with elseconditions is not also reported as partially emitted.""" schema = json.loads( - _single_rule_schema( - _rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}, elseconditions={"slot_conditions": {}}) - ) + _single_rule_schema(_rule(_GUARD_PRESENT, _TARGET_EQUALS_OTHER, elseconditions={"slot_conditions": {}})) ) for name in ("C1", "C2", "C3"): schema["classes"][name] = {"is_a": "Thing", "class_uri": f"ex:{name}"} @@ -5952,7 +5959,7 @@ def test_rule_problem_reported_once_for_inheriting_classes(caplog): def test_rule_problem_names_every_affected_shape_whatever_the_class_order(caplog): """A problem with an inherited rule names every class shape it affects, so the report does not depend on the order in which classes are declared.""" - schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}))) + schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, _TARGET_EQUALS_OTHER))) schema["classes"] = {"Sub": {"is_a": "Thing", "class_uri": "ex:Sub"}, **schema["classes"]} with caplog.at_level(logging.WARNING): _parse_shacl(json.dumps(schema)) @@ -5965,12 +5972,12 @@ def test_rule_problems_reported_per_rule_and_per_problem(caplog): """Problems are kept apart by rule and by kind: two rules with the same problem, and one rule with two problems, each produce their own warning. Each warning names the rule by its position and description.""" - schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}))) + schema = json.loads(_single_rule_schema(_rule(_GUARD_PRESENT, _TARGET_EQUALS_OTHER))) schema["enums"] = {"Color": {"permissible_values": {"Red": {}, "Blue": {}}}} schema["classes"]["Thing"]["attributes"]["color"] = {"range": "Color"} schema["classes"]["Thing"]["rules"] = [ - _rule(_GUARD_PRESENT, {"target": {"pattern": "^x"}}, description="First"), - _rule(_GUARD_PRESENT, {"target": {"pattern": "^y"}}, description="Second"), + _rule(_GUARD_PRESENT, _TARGET_EQUALS_OTHER, description="First"), + _rule(_GUARD_PRESENT, {"target": {"equals_expression": "{guard}"}}, description="Second"), _rule( _GUARD_PRESENT, {"color": {"equals_string": "Purple"}}, @@ -6117,36 +6124,75 @@ def test_rule_equals_string_special_chars_escaped(): @pytest.mark.parametrize( - "extra", + "extra,reason", [ - pytest.param({"pattern": "^x"}, id="pattern"), - pytest.param({"minimum_value": 0}, id="minimum_value"), - pytest.param({"maximum_value": 5}, id="maximum_value"), - pytest.param({"required": True}, id="required"), - pytest.param({"recommended": True}, id="recommended"), - pytest.param({"minimum_cardinality": 1}, id="minimum_cardinality"), - pytest.param({"exact_cardinality": 1}, id="exact_cardinality"), - pytest.param({"equals_number": 3}, id="equals_number"), - pytest.param({"has_member": {"equals_string": "x"}}, id="has_member"), - pytest.param({"any_of": [{"equals_string": "x"}]}, id="slot-any_of"), + pytest.param({"equals_expression": "{other}"}, "equals_expression", id="equals_expression"), + pytest.param({"minimum_value": 0}, "needs a type with a numeric datatype", id="minimum_value"), + pytest.param({"maximum_value": 5}, "needs a type with a numeric datatype", id="maximum_value"), + pytest.param({"equals_number": 3}, "needs a type with a numeric datatype", id="equals_number-on-string"), + pytest.param({"recommended": True}, "recommended", id="recommended"), + pytest.param({"minimum_cardinality": 1}, "minimum_cardinality", id="minimum_cardinality"), + pytest.param({"exact_cardinality": 1}, "exact_cardinality", id="exact_cardinality"), + pytest.param({"any_of": [{"minimum_cardinality": 1}]}, "any_of member", id="slot-any_of-member"), ], ) @pytest.mark.parametrize("condition", ["guard", "target"]) -def test_rule_extra_condition_operator_skipped(caplog, condition, extra): - """A guard or target condition that adds any operator to the ones a pattern - translates makes the rule untranslatable.""" +def test_rule_extra_condition_operator_skipped(caplog, condition, extra, reason): + """A guard or target condition that adds an operator no translation + supports, or a bound on a string slot, makes the rule untranslatable: + dropping it would widen or weaken the rule.""" guard = {"value_presence": "PRESENT", **(extra if condition == "guard" else {})} target = {"equals_string": "x", **(extra if condition == "target" else {})} schema = _single_rule_schema(_rule({"guard": guard}, {"target": target})) - _assert_skipped(caplog, schema, "match none of the translated patterns") + _assert_skipped(caplog, schema, reason) -@pytest.mark.parametrize("presence", ["ABSENT", "UNCOMMITTED"]) -def test_rule_guard_presence_other_than_present_skipped(caplog, presence): - """Only ``value_presence: PRESENT`` is a guard: reading ``ABSENT`` as a - guard would invert the trigger.""" - schema = _single_rule_schema(_rule({"guard": {"value_presence": presence}}, {"target": {"equals_string": "x"}})) - _assert_skipped(caplog, schema, "match none of the translated patterns") +@pytest.mark.parametrize( + "guard_extra,target_extra", + [ + pytest.param({"required": True}, None, id="guard-required"), + pytest.param(None, {"required": True}, id="target-required"), + ], +) +@pytest.mark.parametrize("open_world", [False, True]) +@pytest.mark.parametrize("instance,valid_closed_world,valid_open_world", _RULE_INSTANCES) +def test_rule_extra_operator_translated_exactly( + guard_extra, target_extra, open_world, instance, valid_closed_world, valid_open_world +): + """An extra operator the composed translation supports is translated, not + dropped: a redundant ``required: true`` on the guard changes nothing, and on + the target it requires the target even in an open world.""" + obj = {key: text.format(value="x", other="y") for key, text in instance.items()} + schema = _rule_target_schema( + "string", "x", open_world=open_world, guard_extra=guard_extra, target_extra=target_extra + ) + target_required = bool(target_extra and target_extra.get("required")) + valid = valid_closed_world if target_required or not open_world else valid_open_world + assert len(_sparql_queries(_parse_shacl(schema), EX_RTR.Thing)) == 1 + assert _json_schema_and_shacl_verdicts(schema, "Thing", obj) == (valid, valid) + + +@pytest.mark.parametrize( + "presence,instance,valid", + [ + pytest.param("ABSENT", {"guard": "g", "target": "y"}, True, id="absent-guard-present"), + pytest.param("ABSENT", {"target": "y"}, False, id="absent-other"), + pytest.param("ABSENT", {"target": "x"}, True, id="absent-allowed"), + pytest.param("ABSENT", {}, False, id="absent-target-absent"), + pytest.param("UNCOMMITTED", {"guard": "g", "target": "y"}, False, id="uncommitted-other"), + pytest.param("UNCOMMITTED", {"target": "x"}, True, id="uncommitted-allowed"), + pytest.param("UNCOMMITTED", {}, False, id="uncommitted-target-absent"), + ], +) +def test_rule_guard_presence_other_than_present_composed(presence, instance, valid): + """A guard with ``value_presence: ABSENT`` triggers the rule when the guard + is absent, one with ``UNCOMMITTED`` always; the named pattern, which reads + ``PRESENT``, does not match them, and the composed translation does, as in + JSON Schema.""" + schema = _rule_target_schema("string", "x", guard_extra={"value_presence": presence}) + (query,) = _sparql_queries(_parse_shacl(schema), EX_RTR.Thing) + assert "?guard" not in query, query # composed, not the named pattern + assert _json_schema_and_shacl_verdicts(schema, "Thing", instance) == (valid, valid) @pytest.mark.parametrize("operator", ["any_of", "all_of", "exactly_one_of", "none_of"]) @@ -6159,11 +6205,21 @@ def test_rule_expression_level_operator_skipped(caplog, side, operator): _assert_skipped(caplog, _single_rule_schema(rule), f"its {side} use {operator}") -def test_rule_post_with_both_equals_forms_skipped(caplog): - """equals_string and equals_string_in set together is ambiguous — skip, - do not let one form silently win.""" - schema = _single_rule_schema(_rule(_GUARD_PRESENT, {"target": {"equals_string": "a", "equals_string_in": ["b"]}})) - _assert_skipped(caplog, schema, "match none of the translated patterns") +@pytest.mark.parametrize( + "allowed,target,valid", + [ + pytest.param(["x", "y"], "x", True, id="in-both"), + pytest.param(["x", "y"], "y", False, id="in-one"), + pytest.param(["y"], "x", False, id="contradictory"), + pytest.param(["y"], "y", False, id="contradictory-other"), + ], +) +def test_rule_post_with_both_equals_forms_is_a_conjunction(allowed, target, valid): + """equals_string and equals_string_in set together must both hold, as in + JSON Schema (``const`` and ``enum``); neither form silently wins.""" + schema = _rule_target_schema("string", "x", target_extra={"equals_string_in": allowed}) + assert len(_sparql_queries(_parse_shacl(schema), EX_RTR.Thing)) == 1 + assert _json_schema_and_shacl_verdicts(schema, "Thing", {"guard": "g", "target": target}) == (valid, valid) def test_rule_exclusive_value_extra_operator_skipped(caplog): @@ -6234,3 +6290,1084 @@ def test_rule_equals_true_on_string_slot_pyshacl_end_to_end(status, violates): ) _, focus_nodes = _validate_rules(_STRING_TRUE_SCHEMA_YAML, data) assert focus_nodes == ({URIRef("https://example.org/string-true/x")} if violates else set()) + + +# =========================================================================== +# Compositional fallback +# +# A rule no named pattern matches is composed from its operators. Whether a +# condition requires its slot follows the JSON Schema generator: value_presence, +# then required, then a default (preconditions do, postconditions unless +# open_world, inner conditions of a nested expression do not). Each value of a +# slot must satisfy the condition: a slot constraint applies to all members of +# a collection (05validation.md), as the JSON Schema generator's `items` reads +# it; has_member requires some value to satisfy its condition. +# =========================================================================== + +EX_COMP = rdflib.Namespace("https://example.org/compose/") +_COMP_PREFIXES = "@prefix ex: .\n@prefix xsd: .\n" + + +def _compose_schema( + rule: dict, + *, + part_usage: dict | None = None, + thing_usage: dict | None = None, + slots: list[str] | None = None, + subclass: bool = False, +) -> str: + """A schema whose class ``Thing`` has the slots used by the compositional tests and the one *rule*. + + ``Part`` is the inlined class of ``part`` / ``parts`` and of its own + ``sub``; ``SpecialPart`` narrows its ``kind``. ``site`` references a + ``Site`` by its identifier. *part_usage* and + *thing_usage* add ``slot_usage``, *slots* adds schema slots to ``Thing``, + and *subclass* adds ``SubThing``, which inherits the rule. + """ + schema = { + "id": "https://example.org/compose", + "name": "compose", + "prefixes": {"linkml": "https://w3id.org/linkml/", "ex": str(EX_COMP), "xsd": str(XSD)}, + "imports": ["linkml:types"], + "default_prefix": "ex", + "default_range": "string", + "enums": { + "Kind": {"permissible_values": {"Plain": {}, "Special": {"meaning": "ex:Special"}}}, + "SpecialKind": {"permissible_values": {"Special": {"meaning": "ex:VerySpecial"}}}, + }, + "slots": {"kind": {"range": "Kind"}}, + "classes": { + "Part": { + "class_uri": "ex:Part", + "slots": ["kind"], + "attributes": { + "depth": {"range": "integer"}, + "label": {}, + "marks": {"range": "integer", "multivalued": True}, + "sub": {"range": "Part", "inlined": True}, + }, + "slot_usage": part_usage or {}, + }, + "SpecialPart": { + "is_a": "Part", + "class_uri": "ex:SpecialPart", + "slot_usage": {"kind": {"range": "SpecialKind"}}, + }, + "Site": { + "class_uri": "ex:Site", + "attributes": {"id": {"identifier": True, "range": "uriorcurie"}, "depth": {"range": "integer"}}, + }, + "Thing": { + "class_uri": "ex:Thing", + "slots": slots or [], + "attributes": { + "mode": {}, + "flag": {"range": "boolean"}, + "level": {"range": "integer"}, + "levels": {"range": "integer", "multivalued": True}, + "note": {}, + "part": {"range": "Part", "inlined": True}, + "parts": {"range": "Part", "inlined": True, "multivalued": True, "inlined_as_list": True}, + "site": {"range": "Site"}, + }, + "slot_usage": thing_usage or {}, + "rules": [rule], + }, + }, + } + if subclass: + schema["classes"]["SubThing"] = {"is_a": "Thing", "class_uri": "ex:SubThing"} + return json.dumps(schema) + + +_REQUIRE_NOTE = {"note": {"required": True}} +_DEPTH_AT_MOST_ZERO = {"range_expression": {"slot_conditions": {"depth": {"maximum_value": 0}}}} +_IF_MODE_M = {"mode": {"equals_string": "m"}} +_LEVEL_10_TO_20 = {"level": {"minimum_value": 10, "maximum_value": 20}} + + +def _inner(**conditions: dict) -> dict: + """A ``range_expression`` with the slot *conditions* on the slot's class.""" + return {"range_expression": {"slot_conditions": conditions}} + + +@pytest.mark.parametrize("target_class", ["Thing", "SubThing"]) +@pytest.mark.parametrize( + "pre,post,instance,valid", + [ + # equals_string, and presence in the postcondition + pytest.param(_IF_MODE_M, _REQUIRE_NOTE, {"mode": "m"}, False, id="equals-required"), + pytest.param(_IF_MODE_M, _REQUIRE_NOTE, {"mode": "m", "note": "n"}, True, id="equals-met"), + pytest.param(_IF_MODE_M, _REQUIRE_NOTE, {"mode": "x"}, True, id="equals-not-triggered"), + pytest.param(_IF_MODE_M, _REQUIRE_NOTE, {}, True, id="precondition-slot-absent"), + pytest.param(_IF_MODE_M, {"note": {"value_presence": "PRESENT"}}, {"mode": "m"}, False, id="presence-post"), + pytest.param(_IF_MODE_M, {"note": {}}, {"mode": "m"}, False, id="empty-post-required-by-default"), + pytest.param( + _IF_MODE_M, {"note": {"value_presence": "ABSENT"}}, {"mode": "m", "note": "n"}, False, id="absent-post" + ), + pytest.param(_IF_MODE_M, {"note": {"value_presence": "ABSENT"}}, {"mode": "m"}, True, id="absent-met"), + # numeric bounds are inclusive + pytest.param(_LEVEL_10_TO_20, _REQUIRE_NOTE, {"level": 15}, False, id="bounds-inside"), + pytest.param(_LEVEL_10_TO_20, _REQUIRE_NOTE, {"level": 10}, False, id="bounds-at-minimum"), + pytest.param(_LEVEL_10_TO_20, _REQUIRE_NOTE, {"level": 20}, False, id="bounds-at-maximum"), + pytest.param(_LEVEL_10_TO_20, _REQUIRE_NOTE, {"level": 25}, True, id="bounds-above"), + pytest.param(_LEVEL_10_TO_20, _REQUIRE_NOTE, {"level": 5}, True, id="bounds-below"), + # presence in the precondition + pytest.param({"level": {"value_presence": "PRESENT"}}, _REQUIRE_NOTE, {"level": 1}, False, id="presence-pre"), + pytest.param({"mode": {"required": True}}, _REQUIRE_NOTE, {"mode": "m"}, False, id="required-pre"), + pytest.param({"mode": {"required": True}}, _REQUIRE_NOTE, {}, True, id="required-pre-absent"), + pytest.param({"mode": {}}, _REQUIRE_NOTE, {"mode": "x"}, False, id="empty-pre-requires-presence"), + pytest.param({"mode": {}}, _REQUIRE_NOTE, {}, True, id="empty-pre-absent"), + pytest.param({"mode": {"value_presence": "ABSENT"}}, _REQUIRE_NOTE, {}, False, id="absent-pre"), + pytest.param( + {"mode": {"value_presence": "ABSENT"}}, _REQUIRE_NOTE, {"mode": "m"}, True, id="absent-pre-present" + ), + pytest.param( + {"mode": {"required": False, "equals_string": "m"}}, _REQUIRE_NOTE, {}, False, id="optional-pre-absent" + ), + pytest.param( + {"mode": {"required": False, "equals_string": "m"}}, _REQUIRE_NOTE, {"mode": "x"}, True, id="optional-pre" + ), + pytest.param( + {"mode": {"value_presence": "UNCOMMITTED", "equals_string": "m"}}, + _REQUIRE_NOTE, + {}, + False, + id="uncommitted-pre-absent", + ), + pytest.param( + {"mode": {"value_presence": "UNCOMMITTED", "required": True, "equals_string": "m"}}, + _REQUIRE_NOTE, + {}, + False, + id="value-presence-overrides-required", + ), + pytest.param( + {"level": {"value_presence": "PRESENT", "minimum_value": 3}}, + _REQUIRE_NOTE, + {}, + True, + id="presence-bound-absent", + ), + pytest.param( + {"level": {"value_presence": "PRESENT", "minimum_value": 3}}, + _REQUIRE_NOTE, + {"level": 1}, + True, + id="presence-bound-fails", + ), + pytest.param( + {"level": {"value_presence": "PRESENT", "minimum_value": 3}}, + _REQUIRE_NOTE, + {"level": 5}, + False, + id="presence-bound-holds", + ), + # every value of a multivalued slot + pytest.param( + {"levels": {"minimum_value": 10}}, _REQUIRE_NOTE, {"levels": [12, 15]}, False, id="every-value-holds" + ), + pytest.param({"levels": {"minimum_value": 10}}, _REQUIRE_NOTE, {"levels": [12, 3]}, True, id="one-value-fails"), + # a nested range_expression, whose inner conditions hold for an absent inner slot + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, _REQUIRE_NOTE, {"part": {"depth": -1}}, False, id="nested-holds"), + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, _REQUIRE_NOTE, {"part": {"depth": 5}}, True, id="nested-fails"), + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, _REQUIRE_NOTE, {"part": {}}, False, id="nested-inner-absent"), + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, _REQUIRE_NOTE, {}, True, id="nested-container-absent"), + pytest.param( + {"part": {"required": False, **_DEPTH_AT_MOST_ZERO}}, _REQUIRE_NOTE, {}, False, id="optional-container" + ), + pytest.param( + {"part": _inner(depth={"maximum_value": 0, "required": True})}, + _REQUIRE_NOTE, + {"part": {}}, + True, + id="nested-inner-required-absent", + ), + pytest.param( + {"part": _inner(depth={"value_presence": "PRESENT"})}, + _REQUIRE_NOTE, + {"part": {}}, + True, + id="nested-inner-present-absent", + ), + pytest.param( + {"part": _inner(depth={"value_presence": "PRESENT"})}, + _REQUIRE_NOTE, + {"part": {"depth": 1}}, + False, + id="nested-inner-present", + ), + pytest.param( + {"part": _inner(depth={"value_presence": "ABSENT"})}, + _REQUIRE_NOTE, + {"part": {}}, + False, + id="nested-inner-absent-holds", + ), + pytest.param( + {"part": _inner(depth={"value_presence": "ABSENT"})}, + _REQUIRE_NOTE, + {"part": {"depth": 1}}, + True, + id="nested-inner-absent-fails", + ), + pytest.param( + {"part": _inner(depth={"minimum_value": -5, "maximum_value": 0})}, + _REQUIRE_NOTE, + {"part": {"depth": 3}}, + True, + id="nested-bounds-above", + ), + pytest.param( + {"part": _inner(sub=_DEPTH_AT_MOST_ZERO)}, + _REQUIRE_NOTE, + {"part": {"sub": {"depth": -1}}}, + False, + id="two-hops", + ), + pytest.param( + {"part": _inner(sub=_DEPTH_AT_MOST_ZERO)}, + _REQUIRE_NOTE, + {"part": {"sub": {"depth": 5}}}, + True, + id="two-hops-fails", + ), + pytest.param( + {"parts": _DEPTH_AT_MOST_ZERO}, + _REQUIRE_NOTE, + {"parts": [{"depth": -1}, {"depth": -2}]}, + False, + id="every-member", + ), + pytest.param( + {"parts": _DEPTH_AT_MOST_ZERO}, + _REQUIRE_NOTE, + {"parts": [{"depth": -1}, {"depth": 5}]}, + True, + id="one-member-fails", + ), + # value operators in a postcondition apply to every value of the slot, which is required + pytest.param( + _IF_MODE_M, {"note": {"equals_string": "n"}}, {"mode": "m", "note": "n"}, True, id="post-equals-met" + ), + pytest.param( + _IF_MODE_M, {"note": {"equals_string": "n"}}, {"mode": "m", "note": "x"}, False, id="post-equals-other" + ), + pytest.param(_IF_MODE_M, {"note": {"equals_string": "n"}}, {"mode": "m"}, False, id="post-equals-absent"), + pytest.param( + _IF_MODE_M, {"note": {"equals_string_in": ["n", "o"]}}, {"mode": "m", "note": "o"}, True, id="post-in-met" + ), + pytest.param( + _IF_MODE_M, + {"note": {"equals_string_in": ["n", "o"]}}, + {"mode": "m", "note": "x"}, + False, + id="post-in-other", + ), + pytest.param(_IF_MODE_M, {"level": {"minimum_value": 3}}, {"mode": "m", "level": 3}, True, id="post-bound-met"), + pytest.param( + _IF_MODE_M, {"level": {"minimum_value": 3}}, {"mode": "m", "level": 2}, False, id="post-bound-fails" + ), + pytest.param( + _IF_MODE_M, {"levels": {"maximum_value": 5}}, {"mode": "m", "levels": [1, 5]}, True, id="post-every-value" + ), + pytest.param( + _IF_MODE_M, {"levels": {"maximum_value": 5}}, {"mode": "m", "levels": [1, 9]}, False, id="post-one-fails" + ), + pytest.param( + _IF_MODE_M, {"part": _DEPTH_AT_MOST_ZERO}, {"mode": "m", "part": {"depth": 0}}, True, id="post-nested-met" + ), + pytest.param( + _IF_MODE_M, + {"part": _DEPTH_AT_MOST_ZERO}, + {"mode": "m", "part": {"depth": 1}}, + False, + id="post-nested-fails", + ), + pytest.param(_IF_MODE_M, {"part": _DEPTH_AT_MOST_ZERO}, {"mode": "m"}, False, id="post-nested-absent"), + pytest.param( + _IF_MODE_M, {"part": _DEPTH_AT_MOST_ZERO}, {"mode": "m", "part": {}}, True, id="post-nested-inner-absent" + ), + pytest.param( + _IF_MODE_M, + {"parts": _DEPTH_AT_MOST_ZERO}, + {"mode": "m", "parts": [{"depth": 0}, {"depth": -1}]}, + True, + id="post-nested-every-member", + ), + pytest.param( + _IF_MODE_M, + {"parts": _DEPTH_AT_MOST_ZERO}, + {"mode": "m", "parts": [{"depth": 0}, {"depth": 5}]}, + False, + id="post-nested-one-member-fails", + ), + # equals_string_in in a precondition and an inner condition + pytest.param({"mode": {"equals_string_in": ["m", "n"]}}, _REQUIRE_NOTE, {"mode": "n"}, False, id="pre-in"), + pytest.param({"mode": {"equals_string_in": ["m", "n"]}}, _REQUIRE_NOTE, {"mode": "x"}, True, id="pre-in-other"), + pytest.param( + {"part": _inner(label={"equals_string_in": ["a", "b"]})}, + _REQUIRE_NOTE, + {"part": {"label": "b"}}, + False, + id="inner-in", + ), + # several preconditions are a conjunction + pytest.param( + {**_IF_MODE_M, "level": {"minimum_value": 10}}, + _REQUIRE_NOTE, + {"mode": "m", "level": 12}, + False, + id="two-preconditions", + ), + pytest.param( + {**_IF_MODE_M, "level": {"minimum_value": 10}}, + _REQUIRE_NOTE, + {"mode": "m", "level": 3}, + True, + id="two-preconditions-one-fails", + ), + ], +) +def test_compose_agrees_with_json_schema(pre, post, instance, valid, target_class): + """A composed rule decides each instance as the JSON Schema generator's + if/then does, in the class declaring it and in a subclass.""" + schema = _compose_schema(_rule(pre, post), subclass=True) + assert len(_sparql_queries(_parse_shacl(schema), EX_COMP[target_class])) == 1 + assert _json_schema_and_shacl_verdicts(schema, target_class, instance) == (valid, valid) + + +@pytest.mark.parametrize( + "post,instance,valid", + [ + pytest.param(_REQUIRE_NOTE, {"mode": "m"}, False, id="required-absent"), + pytest.param(_REQUIRE_NOTE, {"mode": "m", "note": "n"}, True, id="required-met"), + pytest.param({"note": {"value_presence": "PRESENT"}}, {"mode": "m"}, False, id="present-absent"), + pytest.param({"note": {"value_presence": "PRESENT"}}, {"mode": "m", "note": "n"}, True, id="present-met"), + pytest.param({"note": {"value_presence": "ABSENT"}}, {"mode": "m", "note": "n"}, False, id="absent-present"), + pytest.param({"note": {"value_presence": "ABSENT"}}, {"mode": "m"}, True, id="absent-met"), + pytest.param({"note": {"equals_string": "n"}}, {"mode": "m"}, True, id="value-omitted"), + pytest.param({"note": {"equals_string": "n"}}, {"mode": "m", "note": "x"}, False, id="value-other"), + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, {"mode": "m"}, True, id="nested-omitted"), + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, {"mode": "m", "part": {}}, True, id="nested-inner-absent"), + pytest.param({"part": _DEPTH_AT_MOST_ZERO}, {"mode": "m", "part": {"depth": 1}}, False, id="nested-other"), + ], +) +@pytest.mark.parametrize("target_class", ["Thing", "SubThing"]) +def test_compose_open_world(post, instance, valid, target_class): + """With ``open_world`` a postcondition slot may be omitted unless the + postcondition states its presence, as in JSON Schema; a value present must + still satisfy it.""" + schema = _compose_schema(_rule(_IF_MODE_M, post, open_world=True), subclass=True) + assert _json_schema_and_shacl_verdicts(schema, target_class, instance) == (valid, valid) + + +_TWO_POSTCONDITIONS = {"note": {"equals_string": "ok"}, "level": {"minimum_value": 1}} + + +@pytest.mark.parametrize( + "open_world,instance,valid", + [ + pytest.param(False, {"mode": "m", "note": "ok", "level": 2}, True, id="both-met"), + pytest.param(False, {"mode": "m", "note": "bad", "level": 2}, False, id="first-fails"), + pytest.param(False, {"mode": "m", "note": "ok", "level": 0}, False, id="second-fails"), + pytest.param(False, {"mode": "m", "note": "bad", "level": 0}, False, id="both-fail"), + pytest.param(False, {"mode": "m", "note": "ok"}, False, id="second-absent"), + pytest.param(False, {"mode": "x", "note": "bad", "level": 0}, True, id="precondition-not-met"), + pytest.param(True, {"mode": "m", "note": "ok"}, True, id="open-world-second-absent"), + pytest.param(True, {"mode": "m", "note": "ok", "level": 0}, False, id="open-world-second-fails"), + ], +) +@pytest.mark.parametrize("target_class", ["Thing", "SubThing"]) +def test_compose_several_postconditions_agree_with_json_schema(open_world, instance, valid, target_class): + """A rule whose postconditions constrain several slots is violated when any + of them is: it becomes one constraint per postcondition slot and decides + each instance as the JSON Schema generator's ``then`` does.""" + schema = _compose_schema(_rule(_IF_MODE_M, _TWO_POSTCONDITIONS, open_world=open_world), subclass=True) + assert len(_sparql_queries(_parse_shacl(schema), EX_COMP[target_class])) == 2 + assert _json_schema_and_shacl_verdicts(schema, target_class, instance) == (valid, valid) + + +def test_compose_several_postconditions_report_each_slot(): + """Each violated postcondition slot is reported with its own ``sh:resultPath`` and value.""" + schema = _compose_schema(_rule(_IF_MODE_M, _TWO_POSTCONDITIONS)) + data = _COMP_PREFIXES + 'ex:x a ex:Thing ; ex:mode "m" ; ex:note "bad" ; ex:level 0 .' + _, results = _rule_results(schema, data) + assert sorted(results) == sorted( + [(EX_COMP.x, EX_COMP.note, Literal("bad")), (EX_COMP.x, EX_COMP.level, Literal(0))] + ) + + +_SPECIAL_KIND = {"kind": {"equals_string": "Special"}} + + +@pytest.mark.parametrize( + "member,post_presence,open_world,members,violates", + [ + pytest.param(_SPECIAL_KIND, {}, False, "ex:p1 . ex:p1 ex:kind ex:Special", False, id="matching-member"), + pytest.param(_SPECIAL_KIND, {}, False, 'ex:p1 . ex:p1 ex:kind "Plain"', True, id="no-matching-member"), + pytest.param( + _SPECIAL_KIND, + {}, + False, + 'ex:p1, ex:p2 . ex:p1 ex:kind "Plain" . ex:p2 ex:kind ex:Special', + False, + id="one-matching", + ), + pytest.param(_SPECIAL_KIND, {}, False, 'ex:p1 . ex:p1 ex:label "l"', False, id="member-without-kind"), + pytest.param( + {"kind": {"equals_string": "Special", "required": True}}, + {}, + False, + 'ex:p1 . ex:p1 ex:label "l"', + True, + id="member-without-required-kind", + ), + pytest.param( + {"kind": {"value_presence": "ABSENT"}}, + {}, + False, + "ex:p1 . ex:p1 ex:kind ex:Special", + True, + id="inner-absent", + ), + pytest.param( + {"kind": {"value_presence": "ABSENT"}}, + {}, + False, + 'ex:p1 . ex:p1 ex:label "l"', + False, + id="inner-absent-met", + ), + pytest.param( + {**_SPECIAL_KIND, "label": {"equals_string": "l"}}, + {}, + False, + 'ex:p1 . ex:p1 ex:kind ex:Special ; ex:label "x"', + True, + id="two-inner-conditions-one-fails", + ), + pytest.param( + {**_SPECIAL_KIND, "label": {"equals_string": "l"}}, + {}, + False, + 'ex:p1 . ex:p1 ex:kind ex:Special ; ex:label "l"', + False, + id="two-inner-conditions-hold", + ), + pytest.param(_SPECIAL_KIND, {}, False, None, True, id="no-members"), + pytest.param(_SPECIAL_KIND, {}, True, None, False, id="no-members-open-world"), + pytest.param(_SPECIAL_KIND, {"required": True}, True, None, True, id="no-members-open-world-required"), + pytest.param(_SPECIAL_KIND, {"required": False}, False, None, False, id="no-members-not-required"), + pytest.param( + _SPECIAL_KIND, + {"value_presence": "UNCOMMITTED", "required": True}, + False, + None, + False, + id="value-presence-overrides-required", + ), + pytest.param(_SPECIAL_KIND, {"value_presence": "ABSENT"}, False, None, False, id="absent-slot"), + pytest.param( + _SPECIAL_KIND, + {"value_presence": "ABSENT"}, + False, + "ex:p1 . ex:p1 ex:kind ex:Special", + True, + id="absent-slot-present", + ), + pytest.param( + {"sub": _DEPTH_AT_MOST_ZERO}, + {}, + False, + "ex:p1 . ex:p1 ex:sub ex:s1 . ex:s1 ex:depth 0", + False, + id="two-hops", + ), + pytest.param( + {"sub": _DEPTH_AT_MOST_ZERO}, + {}, + False, + "ex:p1 . ex:p1 ex:sub ex:s1 . ex:s1 ex:depth 5", + True, + id="two-hops-fails", + ), + pytest.param( + _SPECIAL_KIND, {}, True, 'ex:p1 . ex:p1 ex:kind "Plain"', True, id="no-matching-member-open-world" + ), + ], +) +@pytest.mark.parametrize("target_class", ["Thing", "SubThing"]) +def test_compose_has_member(member, post_presence, open_world, members, violates, target_class): + """``has_member`` is violated when no member satisfies the member condition. + + A member condition holds for an absent inner slot unless it requires the + slot. Whether ``parts`` must be present is decided as for any + postcondition (``open_world``, ``required``, ``value_presence``). The JSON + Schema generator drops ``has_member`` inside rule conditions, so the + verdicts are checked against explicit expectations. + """ + post = {"parts": {"has_member": _inner(**member), **post_presence}} + schema = _compose_schema(_rule({"mode": {"value_presence": "PRESENT"}}, post, open_world=open_world), subclass=True) + body = f'ex:x a ex:{target_class} ; ex:mode "m"' + ("" if members is None else f" ; ex:parts {members}") + _, focus_nodes = _validate_rules(schema, f"{_COMP_PREFIXES}{body} .") + assert focus_nodes == ({EX_COMP.x} if violates else set()) + + +_SOME_LEVEL_10 = {"levels": {"has_member": {"minimum_value": 10}}} +_SOME_LEVEL_10_ALL_20 = {"levels": {"has_member": {"minimum_value": 10}, "maximum_value": 20}} +_PART_WITH_SOME_MARK_10 = {"part": _inner(marks={"has_member": {"minimum_value": 10}})} + + +@pytest.mark.parametrize( + "pre,post,data,violates", + [ + pytest.param(_IF_MODE_M, _SOME_LEVEL_10, " ; ex:levels 3, 12", False, id="post-some"), + pytest.param(_IF_MODE_M, _SOME_LEVEL_10, " ; ex:levels 3, 4", True, id="post-none"), + pytest.param(_IF_MODE_M, _SOME_LEVEL_10, "", True, id="post-absent"), + pytest.param(_SOME_LEVEL_10, _REQUIRE_NOTE, " ; ex:levels 3, 12", True, id="pre-some"), + pytest.param(_SOME_LEVEL_10, _REQUIRE_NOTE, " ; ex:levels 3, 4", False, id="pre-none"), + pytest.param(_SOME_LEVEL_10, _REQUIRE_NOTE, "", False, id="pre-absent"), + # with value operators on the same slot, which every value must satisfy + pytest.param(_IF_MODE_M, _SOME_LEVEL_10_ALL_20, " ; ex:levels 12, 15", False, id="post-some-and-all"), + pytest.param(_IF_MODE_M, _SOME_LEVEL_10_ALL_20, " ; ex:levels 12, 25", True, id="post-some-not-all"), + pytest.param(_IF_MODE_M, _SOME_LEVEL_10_ALL_20, " ; ex:levels 3, 4", True, id="post-all-not-some"), + # inside a nested condition, on the values of the inner slot + pytest.param( + _PART_WITH_SOME_MARK_10, _REQUIRE_NOTE, " ; ex:part ex:p1 . ex:p1 ex:marks 3, 12", True, id="inner-some" + ), + pytest.param( + _PART_WITH_SOME_MARK_10, _REQUIRE_NOTE, " ; ex:part ex:p1 . ex:p1 ex:marks 3, 4", False, id="inner-none" + ), + pytest.param( + _PART_WITH_SOME_MARK_10, _REQUIRE_NOTE, ' ; ex:part ex:p1 . ex:p1 ex:label "l"', True, id="inner-absent" + ), + pytest.param( + {**_PART_WITH_SOME_MARK_10, "levels": {"minimum_value": 0}}, + _REQUIRE_NOTE, + " ; ex:levels 1 ; ex:part ex:p1 . ex:p1 ex:marks 4 . ex:y a ex:Thing ; ex:part ex:p2 . ex:p2 ex:marks 12", + False, + id="inner-member-of-another-subject", + ), + ], +) +def test_compose_has_member_with_value_operators(pre, post, data, violates): + """``has_member`` holds when some value satisfies its value operators, in a + postcondition, a precondition and a nested condition alike, next to value + operators that every value must satisfy. An absent slot fails a + precondition, which requires its slot, violates a closed-world + postcondition, and satisfies a nested condition, which does not.""" + schema = _compose_schema(_rule(pre, post)) + _, focus_nodes = _validate_rules(schema, f'{_COMP_PREFIXES}ex:x a ex:Thing ; ex:mode "m"{data} .') + assert EX_COMP.x in focus_nodes if violates else EX_COMP.x not in focus_nodes + + +@pytest.mark.parametrize( + "rule,data,expected", + [ + pytest.param( + _rule(_IF_MODE_M, _REQUIRE_NOTE), + 'ex:x a ex:Thing ; ex:mode "m" .', + [(EX_COMP.x, EX_COMP.note, EX_COMP.x)], + id="required", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {"value_presence": "ABSENT"}}), + 'ex:x a ex:Thing ; ex:mode "m" ; ex:note "n" .', + [(EX_COMP.x, EX_COMP.note, Literal("n"))], + id="absent", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {"equals_string": "n"}}), + 'ex:x a ex:Thing ; ex:mode "m" ; ex:note "a", "n", "b" .', + [(EX_COMP.x, EX_COMP.note, Literal("a")), (EX_COMP.x, EX_COMP.note, Literal("b"))], + id="each-offending-value", + ), + pytest.param( + _rule(_IF_MODE_M, {"parts": _inner(label={"equals_string": "l"}, depth={"maximum_value": 0})}), + 'ex:x a ex:Thing ; ex:mode "m" ; ex:parts ex:p1 . ex:p1 ex:label "x" ; ex:depth 5 .', + [(EX_COMP.x, EX_COMP.parts, EX_COMP.p1)], + id="value-violating-twice-reported-once", + ), + pytest.param( + _rule(_IF_MODE_M, _SOME_LEVEL_10), + 'ex:x a ex:Thing ; ex:mode "m" ; ex:levels 3, 4 .', + [(EX_COMP.x, EX_COMP.levels, EX_COMP.x)], + id="no-member-satisfies", + ), + ], +) +def test_compose_result_names_path_and_value(rule, data, expected): + """A composed result names the postcondition's property and the offending + value, once per value, or the focus node when a value is missing (SHACL + §5.3.2).""" + _, results = _rule_results(_compose_schema(rule), _COMP_PREFIXES + data) + assert sorted(results) == sorted(expected) + + +def test_compose_query_yields_one_solution_per_offending_value(): + """A value that violates two parts of the postcondition is one solution, + so a processor that maps every solution to a result (SHACL §5.3.2) reports + it once.""" + rule = _rule(_IF_MODE_M, {"parts": _inner(label={"equals_string": "l"}, depth={"maximum_value": 0})}) + (query,) = _sparql_queries(_parse_shacl(_compose_schema(rule)), EX_COMP.Thing) + data = rdflib.Graph().parse( + data=_COMP_PREFIXES + 'ex:x a ex:Thing ; ex:mode "m" ; ex:parts ex:p1 . ex:p1 ex:label "x" ; ex:depth 5 .', + format="turtle", + ) + solutions = [(row.path, row.value) for row in data.query(query, initBindings={"this": EX_COMP.x})] + assert solutions == [(EX_COMP.parts, EX_COMP.p1)] + + +@pytest.mark.parametrize( + "flag,violates", + [ + pytest.param(Literal("true", datatype=XSD.boolean, normalize=False), True, id="true"), + pytest.param(Literal("1", datatype=XSD.boolean, normalize=False), True, id="lexical-1"), + pytest.param(Literal("false", datatype=XSD.boolean, normalize=False), False, id="false"), + pytest.param(Literal("true"), False, id="string-true"), + ], +) +def test_compose_boolean_precondition_compared_by_value(flag, violates): + """``equals_string`` on a boolean precondition slot compares the boolean it denotes, as in the named patterns.""" + schema = _compose_schema(_rule({"flag": {"equals_string": "true"}}, _REQUIRE_NOTE)) + data = rdflib.Graph() + data.add((EX_COMP.x, RDF.type, EX_COMP.Thing)) + data.add((EX_COMP.x, EX_COMP.flag, flag)) + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == ({EX_COMP.x} if violates else set()) + + +@pytest.mark.parametrize( + "kind,violates", + [ + pytest.param("ex:Special", False, id="meaning-iri"), + pytest.param('"Special"', True, id="permissible-value-text"), + pytest.param('"Plain"', True, id="other-value"), + ], +) +def test_compose_nested_postcondition_compares_enum_meaning(kind, violates): + """In a nested postcondition, an enum value with a ``meaning`` is compared as + that IRI, resolved on the range class, as ``rdflib_dumper`` writes it. The + generated JSON-LD context leaves the values of this enum, which has a value + without a meaning, as strings, so the expectations are explicit.""" + rule = _rule(_IF_MODE_M, {"part": _inner(kind={"equals_string": "Special"})}) + data = f'{_COMP_PREFIXES}ex:x a ex:Thing ; ex:mode "m" ; ex:part ex:p . ex:p ex:kind {kind} .' + _, focus_nodes = _validate_rules(_compose_schema(rule), data) + assert focus_nodes == ({EX_COMP.x} if violates else set()) + + +# The numeric datatypes of SPARQL 1.1 §17.1, . +_SPARQL_NUMERIC_DATATYPES = [ + *("integer", "decimal", "float", "double"), + *("nonPositiveInteger", "negativeInteger", "long", "int", "short", "byte"), + *("nonNegativeInteger", "unsignedLong", "unsignedInt", "unsignedShort", "unsignedByte", "positiveInteger"), +] + + +@pytest.mark.parametrize("datatype", _SPARQL_NUMERIC_DATATYPES) +def test_compose_bounds_on_every_sparql_numeric_datatype(datatype): + """Bounds are translated on a type with any numeric datatype of SPARQL 1.1 + §17.1 and compare its values numerically.""" + schema = json.loads(_compose_schema(_rule({"amount": {"minimum_value": -10, "maximum_value": 10}}, _REQUIRE_NOTE))) + base = datatype if datatype in ("decimal", "float", "double") else "integer" + schema["types"] = {"Amount": {"typeof": base, "uri": f"xsd:{datatype}"}} + schema["classes"]["Thing"]["attributes"]["amount"] = {"range": "Amount"} + sign = -1 if datatype in ("nonPositiveInteger", "negativeInteger") else 1 + for value, violates in ((5 * sign, True), (50 * sign, False)): + data = rdflib.Graph() + data.add((EX_COMP.x, RDF.type, EX_COMP.Thing)) + data.add((EX_COMP.x, EX_COMP.amount, Literal(str(value), datatype=XSD[datatype]))) + _, focus_nodes = _validate_rules(json.dumps(schema), data) + assert focus_nodes == ({EX_COMP.x} if violates else set()), value + + +@pytest.mark.parametrize( + "slot_range,translated", + [ + *(pytest.param(r, True, id=r) for r in ("integer", "decimal", "float", "double")), + pytest.param("Count", True, id="custom-nonNegativeInteger"), + *( + pytest.param(r, False, id=r) + for r in ("string", "date", "datetime", "time", "boolean", "uriorcurie", "Kind", "Part") + ), + pytest.param("Year", False, id="custom-gYear"), + pytest.param("Span", False, id="custom-duration"), + ], +) +def test_compose_bounds_only_on_numeric_ranges(caplog, slot_range, translated): + """Bounds compare numerically, so they are translated on a type with a + numeric datatype and skip the rule on any other range: SPARQL orders no + string, date, duration, boolean, IRI or node against a number (§17.3).""" + schema = json.loads(_compose_schema(_rule({"amount": {"minimum_value": 1}}, _REQUIRE_NOTE))) + schema["types"] = { + "Count": {"typeof": "integer", "uri": "xsd:nonNegativeInteger"}, + "Year": {"typeof": "string", "uri": "xsd:gYear"}, + "Span": {"typeof": "string", "uri": "xsd:duration"}, + } + schema["classes"]["Thing"]["attributes"]["amount"] = {"range": slot_range} + with caplog.at_level(logging.WARNING): + g = _parse_shacl(json.dumps(schema)) + assert len(_sparql_queries(g, EX_COMP.Thing)) == (1 if translated else 0) + skipped = "its minimum_value on slot 'amount' needs a type with a numeric datatype, and the slot's range is " + skipped += repr(slot_range) + assert any(skipped in rec.message for rec in caplog.records) != translated, caplog.text + + +_LEVEL_ONE_OR_TWO = {"level": {"any_of": [{"equals_number": 1}, {"equals_number": 2}]}} +_LEVEL_NOT_ZERO_OR_TEN = {"level": {"none_of": [{"equals_number": 0}, {"equals_number": 10}]}} +_LEVEL_EXACTLY_ONE_BOUND = {"level": {"exactly_one_of": [{"minimum_value": 1}, {"maximum_value": 10}]}} +_LEVEL_BOTH_BOUNDS = {"level": {"all_of": [{"minimum_value": 1}, {"maximum_value": 10}]}} +_NOTE_A_OR_B_WORD = { + "note": {"any_of": [{"equals_string": "a"}, {"all_of": [{"pattern": "^b"}, {"equals_string_in": ["bb", "bc"]}]}]} +} + + +@pytest.mark.parametrize("target_class", ["Thing", "SubThing"]) +@pytest.mark.parametrize( + "pre,post,instance,valid", + [ + pytest.param({"level": {"equals_number": 3}}, _REQUIRE_NOTE, {"level": 3}, False, id="pre-equals-number"), + pytest.param({"level": {"equals_number": 3}}, _REQUIRE_NOTE, {"level": 4}, True, id="pre-equals-number-other"), + pytest.param( + _IF_MODE_M, {"level": {"equals_number": 3}}, {"mode": "m", "level": 3}, True, id="post-equals-number" + ), + pytest.param( + _IF_MODE_M, {"level": {"equals_number": 3}}, {"mode": "m", "level": 4}, False, id="post-equals-number-other" + ), + pytest.param({"mode": {"pattern": "^m"}}, _REQUIRE_NOTE, {"mode": "mx"}, False, id="pre-pattern"), + pytest.param({"mode": {"pattern": "^m"}}, _REQUIRE_NOTE, {"mode": "xm"}, True, id="pre-pattern-other"), + pytest.param(_IF_MODE_M, {"note": {"pattern": "^n"}}, {"mode": "m", "note": "nx"}, True, id="post-pattern"), + pytest.param( + _IF_MODE_M, {"note": {"pattern": "^n"}}, {"mode": "m", "note": "x"}, False, id="post-pattern-other" + ), + pytest.param(_LEVEL_ONE_OR_TWO, _REQUIRE_NOTE, {"level": 2}, False, id="pre-any-of"), + pytest.param(_LEVEL_ONE_OR_TWO, _REQUIRE_NOTE, {"level": 3}, True, id="pre-any-of-other"), + pytest.param(_IF_MODE_M, _LEVEL_NOT_ZERO_OR_TEN, {"mode": "m", "level": 10}, False, id="post-none-of"), + pytest.param(_IF_MODE_M, _LEVEL_NOT_ZERO_OR_TEN, {"mode": "m", "level": 5}, True, id="post-none-of-other"), + pytest.param(_IF_MODE_M, _LEVEL_EXACTLY_ONE_BOUND, {"mode": "m", "level": 0}, True, id="post-exactly-one-low"), + pytest.param( + _IF_MODE_M, _LEVEL_EXACTLY_ONE_BOUND, {"mode": "m", "level": 5}, False, id="post-exactly-one-both" + ), + pytest.param( + _IF_MODE_M, _LEVEL_EXACTLY_ONE_BOUND, {"mode": "m", "level": 11}, True, id="post-exactly-one-high" + ), + pytest.param(_IF_MODE_M, _LEVEL_BOTH_BOUNDS, {"mode": "m", "level": 5}, True, id="post-all-of"), + pytest.param(_IF_MODE_M, _LEVEL_BOTH_BOUNDS, {"mode": "m", "level": 11}, False, id="post-all-of-fails"), + pytest.param(_IF_MODE_M, _NOTE_A_OR_B_WORD, {"mode": "m", "note": "bb"}, True, id="post-nested-boolean"), + pytest.param(_IF_MODE_M, _NOTE_A_OR_B_WORD, {"mode": "m", "note": "bx"}, False, id="post-nested-boolean-fails"), + # a boolean combination, like any value test, holds for every value of a multivalued slot + pytest.param( + _IF_MODE_M, + {"levels": {"any_of": [{"equals_number": 1}, {"equals_number": 2}]}}, + {"mode": "m", "levels": [1, 2]}, + True, + id="post-any-of-every-value", + ), + pytest.param( + _IF_MODE_M, + {"levels": {"any_of": [{"equals_number": 1}, {"equals_number": 2}]}}, + {"mode": "m", "levels": [1, 3]}, + False, + id="post-any-of-one-value-fails", + ), + ], +) +def test_compose_value_tests_agree_with_json_schema(pre, post, instance, valid, target_class): + """``equals_number``, ``pattern`` and the boolean combinations of value tests decide each instance + as the JSON Schema generator's if/then does, in the declaring class and in a subclass.""" + schema = _compose_schema(_rule(pre, post), subclass=True) + assert len(_sparql_queries(_parse_shacl(schema), EX_COMP[target_class])) == 1 + assert _json_schema_and_shacl_verdicts(schema, target_class, instance) == (valid, valid) + + +@pytest.mark.parametrize("levels,violates", [([3, 9], False), ([3, 4], True), ([], True)]) +def test_compose_has_member_with_a_boolean_member(levels, violates): + """A ``has_member`` condition may combine value tests; it holds when some value satisfies the combination.""" + member = {"any_of": [{"equals_number": 1}, {"equals_number": 9}]} + schema = _compose_schema(_rule(_IF_MODE_M, {"levels": {"has_member": member}})) + data = rdflib.Graph() + data.add((EX_COMP.x, RDF.type, EX_COMP.Thing)) + data.add((EX_COMP.x, EX_COMP.mode, Literal("m"))) + for level in levels: + data.add((EX_COMP.x, EX_COMP.levels, Literal(level))) + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == ({EX_COMP.x} if violates else set()) + + +@pytest.mark.parametrize( + "level,violates", + [ + pytest.param(Literal(15), True, id="integer"), + pytest.param(Literal("15"), False, id="string"), + pytest.param(EX_COMP.fifteen, False, id="iri"), + ], +) +def test_compose_bound_fails_on_a_value_that_is_not_a_number(level, violates): + """A value that is not a number fails a numeric bound, since comparing it + is a type error (SPARQL 1.1 §17.3); its datatype violation is reported + by the property shape, not by the rule.""" + schema = _compose_schema(_rule({"level": {"minimum_value": 10}}, _REQUIRE_NOTE)) + data = rdflib.Graph() + data.add((EX_COMP.x, RDF.type, EX_COMP.Thing)) + data.add((EX_COMP.x, EX_COMP.level, level)) + _, focus_nodes = _validate_rules(schema, data) + assert focus_nodes == ({EX_COMP.x} if violates else set()) + + +@pytest.mark.parametrize( + "rule,reason", + [ + pytest.param( + _rule({"level": {"equals_string": "3"}}, _REQUIRE_NOTE), + "whose range 'integer' is neither an enum nor a type with datatype xsd:string or xsd:boolean", + id="equals-string-on-integer", + ), + pytest.param( + _rule({"part": _inner(label={"maximum_value": 3})}, _REQUIRE_NOTE), + "its maximum_value on slot 'label' needs a type with a numeric datatype, and the slot's range is 'string'", + id="inner-bound-on-string", + ), + pytest.param( + _rule({"mode": {"equals_expression": "{note}"}}, _REQUIRE_NOTE), + "its precondition on slot 'mode' uses equals_expression", + id="pre-op", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {"equals_number": 3}}), + "its equals_number on slot 'note' needs a type with a numeric datatype, and the slot's range is 'string'", + id="post-equals-number-on-string", + ), + pytest.param( + _rule({"level": {"any_of": [{"has_member": {"equals_number": 1}}]}}, _REQUIRE_NOTE), + "its any_of member on slot 'level' uses has_member", + id="boolean-member-op", + ), + pytest.param( + _rule({"level": {"exactly_one_of": [{"equals_number": 1}, {"description": "no test"}]}}, _REQUIRE_NOTE), + "its exactly_one_of member on slot 'level' uses no operator", + id="boolean-member-empty", + ), + pytest.param( + {"preconditions": {"slot_conditions": {}}, "postconditions": {"slot_conditions": _REQUIRE_NOTE}}, + "its preconditions constrain no slot", + id="empty-preconditions", + ), + pytest.param( + {"preconditions": {"slot_conditions": _IF_MODE_M}, "postconditions": {}}, + "its postconditions constrain no slot", + id="empty-postconditions", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {"required": True, "equals_expression": "{mode}"}}), + "its postcondition on slot 'note' uses equals_expression, required", + id="post-op", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {"required": False}}), + "its postcondition on slot 'note' constrains nothing", + id="post-not-required", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {}}, open_world=True), + "its postcondition on slot 'note' constrains nothing", + id="post-open-world-empty", + ), + pytest.param( + _rule(_IF_MODE_M, {"note": {"value_presence": "UNCOMMITTED", "required": True}}), + "its postcondition on slot 'note' constrains nothing", + id="post-value-presence-overrides-required", + ), + pytest.param( + _rule(_IF_MODE_M, {"parts": {"has_member": {"equals_string": "x"}}}), + "equals_string(_in) on slot 'parts', whose range 'Part' is neither an enum", + id="has-member-value-on-class-range", + ), + pytest.param( + _rule(_IF_MODE_M, {"levels": {"has_member": {"minimum_cardinality": 1}}}), + "its has_member on slot 'levels' uses minimum_cardinality", + id="has-member-op", + ), + pytest.param( + _rule(_IF_MODE_M, {"levels": {"has_member": {"required": True}}}), + "its has_member on slot 'levels' uses required", + id="has-member-presence", + ), + pytest.param( + _rule(_IF_MODE_M, {"levels": {"has_member": {}}}), + "its has_member on slot 'levels' uses no operator", + id="has-member-empty", + ), + pytest.param( + _rule({"level": _DEPTH_AT_MOST_ZERO}, _REQUIRE_NOTE), + "its range_expression is on slot 'level', whose range is not a class", + id="range-expression-on-non-class", + ), + pytest.param( + _rule( + {"part": {"range_expression": {"any_of": [{"slot_conditions": {"depth": {"maximum_value": 0}}}]}}}, + _REQUIRE_NOTE, + ), + "its range_expression on slot 'part' uses any_of,", + id="range-expression-any-of", + ), + pytest.param( + _rule( + { + "part": { + "range_expression": { + **_DEPTH_AT_MOST_ZERO["range_expression"], + "none_of": [{"slot_conditions": {"label": {"equals_string": "x"}}}], + } + } + }, + _REQUIRE_NOTE, + ), + "its range_expression on slot 'part' uses none_of, slot_conditions", + id="range-expression-slot-conditions-and-none-of", + ), + pytest.param( + _rule({"part": {"range_expression": {"slot_conditions": {}}}}, _REQUIRE_NOTE), + "its range_expression on slot 'part' uses no operator", + id="range-expression-empty", + ), + pytest.param( + _rule({"part": _inner(depth={"minimum_cardinality": 1})}, _REQUIRE_NOTE), + "its inner condition on slot 'depth' uses minimum_cardinality", + id="inner-op", + ), + pytest.param( + _rule({"part": _inner(nope={"maximum_value": 0})}, _REQUIRE_NOTE), + "'nope', which is not a slot", + id="inner-unknown-slot", + ), + pytest.param( + _rule(_IF_MODE_M, {"site": _inner(id={"equals_string": "ex:s1"})}), + "its condition is on the identifier slot 'id'", + id="inner-identifier", + ), + ], +) +def test_compose_untranslatable_rule_skipped(caplog, rule, reason): + """The compositional fallback is exact too: any operator it does not translate skips the rule, with the reason.""" + with caplog.at_level(logging.WARNING): + g = _parse_shacl(_compose_schema(rule)) + assert _sparql_queries(g, EX_COMP.Thing) == [] + messages = [rec.message for rec in caplog.records if "Rule 1 of class 'Thing'" in rec.message] + assert len(messages) == 1 and "skipped, because" in messages[0] and reason in messages[0], caplog.text + assert "in the shapes of" not in messages[0], "a problem with an inner slot belongs to the outer rule" + + +@pytest.mark.parametrize("bound", ['"abc"', "2020-01-01", "true", ".nan", ".inf", "1e20"]) +def test_compose_non_numeric_bound_skipped(caplog, bound): + """A ``minimum_value`` that is not a finite number (its metamodel range is + ``Anything``) skips the rule: interpolated, it would make the query + unparsable or compare as arithmetic. YAML 1.1 reads ``1e20``, which has + no dot, as a string.""" + marker = "__BOUND__" + schema = _compose_schema(_rule({"level": {"minimum_value": marker}}, _REQUIRE_NOTE)) + schema = schema.replace(f'"{marker}"', bound) # a raw YAML scalar + with caplog.at_level(logging.WARNING): + g = _parse_shacl(schema) + assert _sparql_queries(g, EX_COMP.Thing) == [] + assert any("is not a finite number" in rec.message for rec in caplog.records), caplog.text + + +def test_rule_skip_replaces_problems_noted_while_translating(caplog): + """A rule skipped part-way through its translation reports only why it was + skipped: a problem noted earlier (a value no enum permits) describes a + constraint that is not emitted.""" + schema = _compose_schema( + _rule({"part": _inner(kind={"equals_string": "Bogus"}), "mode": {"equals_expression": "{note}"}}, _REQUIRE_NOTE) + ) + reason = "its precondition on slot 'mode' uses equals_expression" + with caplog.at_level(logging.WARNING): + _parse_shacl(schema) + messages = [rec.message for rec in caplog.records if "Rule 1 of class 'Thing'" in rec.message] + assert len(messages) == 1 and "skipped, because" in messages[0] and reason in messages[0], messages + + +def test_rule_skip_in_a_subclass_keeps_the_problems_of_other_shapes(caplog): + """A rule skipped for a subclass, whose ``slot_usage`` changes a slot, keeps + the problems noted where it is translated.""" + schema = json.loads( + _compose_schema(_rule({"kind": {"equals_string": "Bogus"}}, _REQUIRE_NOTE), slots=["kind"], subclass=True) + ) + schema["classes"]["SubThing"]["slot_usage"] = {"kind": {"range": "integer"}} + with caplog.at_level(logging.WARNING): + g = _parse_shacl(json.dumps(schema)) + assert len(_sparql_queries(g, EX_COMP.Thing)) == 1 and _sparql_queries(g, EX_COMP.SubThing) == [] + messages = [rec.message for rec in caplog.records if "Rule 1 of class 'Thing'" in rec.message] + assert len(messages) == 2, messages + assert any("'Bogus', which is not a permissible value" in m and "in the shapes of" not in m for m in messages) + assert any("skipped, because" in m and "(in the shapes of 'SubThing')" in m for m in messages) + + +@pytest.mark.parametrize( + "part_usage,thing_usage,expected_iris,unexpected_iris", + [ + pytest.param( + {"kind": {"slot_uri": "ex:partKind"}}, {}, [EX_COMP.partKind], [EX_COMP.kind], id="inner-slot-uri-override" + ), + pytest.param( + {"kind": {"slot_uri": "ex:partKind"}}, + {"kind": {"slot_uri": "ex:thingKind"}}, + [EX_COMP.partKind], + [EX_COMP.thingKind], + id="inner-slot-not-shadowed-by-outer", + ), + pytest.param( + {}, {"part": {"range": "SpecialPart"}}, [EX_COMP.VerySpecial], [EX_COMP.Special], id="container-narrowed" + ), + ], +) +def test_compose_inner_slot_resolved_on_range_class(part_usage, thing_usage, expected_iris, unexpected_iris): + """Inner slots of a nested expression resolve in the induced context of the + container slot's range class: their IRI and their enum values come from it.""" + rule = _rule({"part": _inner(kind={"equals_string": "Special"})}, _REQUIRE_NOTE) + schema = _compose_schema( + rule, part_usage=part_usage, thing_usage=thing_usage, slots=["kind"] if "kind" in thing_usage else None + ) + (query,) = _sparql_queries(_parse_shacl(schema), EX_COMP.Thing) + for iri in expected_iris: + assert f"<{iri}>" in query, query + for iri in unexpected_iris: + assert f"<{iri}>" not in query, query + + +@pytest.mark.parametrize( + "values,violates", + [ + pytest.param("", True, id="both-absent"), + pytest.param(' ; ex:mode "m"', False, id="one-present"), + pytest.param(' ; ex:mode "m" ; ex:level 1', False, id="both-present"), + ], +) +def test_compose_several_absent_preconditions_each_hold(values, violates): + """Preconditions are a conjunction, so ``value_presence: ABSENT`` on two + slots requires both to be absent. The JSON Schema generator merges them + into one ``not: {required: [...]}``, "not all present", so the + expectations are explicit.""" + rule = _rule({"mode": {"value_presence": "ABSENT"}, "level": {"value_presence": "ABSENT"}}, _REQUIRE_NOTE) + _, focus_nodes = _validate_rules(_compose_schema(rule), f"{_COMP_PREFIXES}ex:x a ex:Thing{values} .") + assert focus_nodes == ({EX_COMP.x} if violates else set()) + + +@pytest.mark.parametrize( + "inner,site,violates", + [ + pytest.param({"maximum_value": 0}, "ex:s1 ex:depth -1 .", True, id="referenced-node-satisfies"), + pytest.param({"maximum_value": 0}, "ex:s1 ex:depth 5 .", False, id="referenced-node-fails"), + pytest.param({"maximum_value": 0}, "", True, id="referenced-node-not-described"), + pytest.param({"maximum_value": 0, "required": True}, "", False, id="required-on-undescribed-node"), + ], +) +def test_compose_range_expression_on_a_reference(inner, site, violates): + """On a slot that references a node instead of inlining it, a nested + condition applies to the referenced node's triples in the data graph; a + node the graph does not describe has none. JSON has only the identifier + there, so the expectations are explicit.""" + rule = _rule({"site": _inner(depth=inner)}, _REQUIRE_NOTE) + data = f"{_COMP_PREFIXES}ex:x a ex:Thing ; ex:site ex:s1 . {site}" + _, focus_nodes = _validate_rules(_compose_schema(rule), data) + assert focus_nodes == ({EX_COMP.x} if violates else set()) From 6cb42b2d937e856d9f9ec9c4997419a96af554f1 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 14:48:35 +0200 Subject: [PATCH 07/17] docs(shacl): document rule-to-SHACL-SPARQL constraint generation The SHACL generator translates LinkML rules into sh:sparql constraints, but its documentation did not mention them. Describe the two named patterns and the composed translation with the operators it supports and one constraint per postcondition slot, the semantics shared with the JSON Schema generator (presence, every value versus has_member, value comparison, numeric bounds), inheritance, open_world, --no-emit-rules and the skip warnings, with an example generated by the generator. Replace the dangling "See above for implementation status" in the rules section of advanced.md with links to the JSON Schema and SHACL generator pages. Signed-off-by: jdsika --- docs/generators/shacl.rst | 108 ++++++++++++++++++++++++++++++++++++++ docs/schemas/advanced.md | 2 +- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/docs/generators/shacl.rst b/docs/generators/shacl.rst index 4c5031cc35..29f7e10501 100644 --- a/docs/generators/shacl.rst +++ b/docs/generators/shacl.rst @@ -268,6 +268,114 @@ satisfied by the same value. A ``has_member`` on a slot that isn't multivalued, or one that uses anything else, is skipped and logged as a warning. +Rule constraints (SHACL-SPARQL) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +LinkML `rules `_ state conditional +constraints ("if slot A holds X, slot B must ..."), which property shapes +cannot express. The generator translates each rule into a +`SHACL-SPARQL constraint `_ +(``sh:sparql``) on the node shape of its class, and of every subclass, since +a rule applies to all members of its class. ``--no-emit-rules`` turns this +off. The metamodel maps ``rules`` to ``sh:rule``, the inference rules of +`SHACL Advanced Features `__, which +add triples rather than report violations; the generator reads a rule as a +constraint instead, as the JSON Schema generator does with ``if`` / ``then``. + +The constraint's query selects each focus node that satisfies the +preconditions and violates the postconditions. ``$this`` is the focus node +(`SHACL §5.3.1 `_), +and each result names the postcondition's property and, where there is one, +the offending value (``sh:resultPath``, ``sh:value``). + +Two patterns are recognised first: + +* **Presence implies value**: precondition ``value_presence: PRESENT`` on a + guard slot, and postcondition ``equals_string`` or ``equals_string_in`` on a + target slot. +* **Exclusive value**: precondition ``has_member: {equals_string: V}``, and + postcondition ``maximum_cardinality`` on the same slot. A bare + ``equals_string: V`` precondition applies to every value of the slot, so it + is not this pattern; such a rule is skipped with a warning naming + ``has_member``. + +Any other rule is composed from the operators its conditions use: +``value_presence``, ``required``, ``equals_string``, ``equals_string_in``, +``equals_number``, ``pattern``, ``minimum_value``, ``maximum_value``, the +slot-level ``any_of``, ``all_of``, ``exactly_one_of`` and ``none_of`` over +those value tests, ``range_expression`` (slot conditions on the slot's class) +and ``has_member``. Each postcondition slot gets its own constraint, so a rule +is violated when any of its postconditions is. + +All translations follow the JSON Schema generator's ``if`` / ``then``: + +* **Presence:** ``value_presence`` decides, then ``required``. Otherwise a + precondition requires its slot, a postcondition requires its slot unless + the rule is ``open_world``, and a condition inside a ``range_expression`` + or ``has_member`` does not. +* **Multivalued slots:** a condition holds when *every* value satisfies it. + ``has_member`` holds when *some* value does. +* **Values:** ``equals_string`` and ``equals_string_in`` are compared as + strings on enum and ``xsd:string`` slots, where an enum value with a + ``meaning`` is compared as that IRI. On ``xsd:boolean`` slots they are + compared as booleans and must be ``true``, ``false``, ``1`` or ``0``. + ``equals_number``, ``minimum_value`` and ``maximum_value`` (inclusive) are + translated only on slots whose datatype SPARQL compares as a number + (``xsd:integer``, ``decimal``, ``float``, ``double`` and the types derived + from them). ``pattern`` is matched against the value's string form, as + ``sh:pattern`` matches it. +* **Boolean combinations:** a slot-level ``any_of``, ``all_of``, + ``exactly_one_of`` or ``none_of`` combines the value tests of its members for + each value; ``exactly_one_of`` holds when exactly one member does. + +A rule outside these forms is skipped with a warning that names the rule and +the reason. It is never partially translated. ``deactivated`` rules are +ignored and ``bidirectional`` rules are skipped. For a rule with +``elseconditions``, only the if/then direction is emitted, and a warning +says so. + +Example: + +.. code-block:: yaml + + classes: + Document: + attributes: + review_score: + range: integer + status: + range: Status # an enum: draft, approved, published + rules: + - description: A document scoring 4 or more must be approved or published. + preconditions: + slot_conditions: + review_score: + minimum_value: 4 + postconditions: + slot_conditions: + status: + equals_string_in: [approved, published] + +generates (abridged): + +.. code-block:: turtle + + ex:Document a sh:NodeShape ; + sh:sparql [ a sh:SPARQLConstraint ; + sh:message "A document scoring 4 or more must be approved or published." ; + sh:select """SELECT DISTINCT $this ( AS ?path) ?value WHERE { + FILTER EXISTS { $this ?pre0 . } + FILTER NOT EXISTS { $this ?pre0 . FILTER ( !( COALESCE( isNumeric( ?pre0 ) && ?pre0 >= 4, false ) ) ) } + { FILTER NOT EXISTS { $this ?value . } } + UNION { $this ?value . FILTER ( !( COALESCE( ?value = "approved" || ?value = "published", false ) ) ) } + }""" ] . + +A ``Document`` with a ``review_score`` of 4 or more violates the constraint +if it has no ``status``, or once for each ``status`` other than ``approved`` +or ``published``, which the result reports as ``sh:value``. SHACL processors +that support SHACL-SPARQL, such as ``pyshacl``, validate these constraints. + + Command Line ^^^^^^^^^^^^ diff --git a/docs/schemas/advanced.md b/docs/schemas/advanced.md index 4e6f901450..2b59901638 100644 --- a/docs/schemas/advanced.md +++ b/docs/schemas/advanced.md @@ -163,7 +163,7 @@ classes: description: USA and territories must have a specific regex pattern for postal codes and phone numbers ``` -See above for implementation status +The [JSON Schema generator](../generators/json-schema) translates rules into `if` / `then` subschemas, and the [SHACL generator](../generators/shacl) into SHACL-SPARQL constraints. ## Defining slots From ac27fc1bb7d3accc9efabbabd6f8df5624f71d66 Mon Sep 17 00:00:00 2001 From: jdsika Date: Fri, 2 Oct 2026 16:41:12 +0200 Subject: [PATCH 08/17] feat(gen-owl): honor instantiates on emitted schema resources Translate explicit instantiates declarations to rdf:type on the emitted schema resource through one shared helper. Apply the mapping consistently to schemas, classes, slots, types, enums, and non-literal permissible values. Literal permissible values cannot be RDF subjects and produce a warning. Membership types the schema element itself, not its ordinary data instances, and does not become subclass inheritance. Existing enum representation and OWL class/individual punning remain intact. No additional flag is needed because the schema already declares the relationship. Cover CURIEs, absolute and minted IRIs, literal values, representation modes, and the distinction between gen-rdf metamodel serialization and gen-owl ontology assertions. Document the LinkML and OWL specification references. A resource in the OWL 2 reserved vocabulary (Structural Specification section 2.4), such as a type emitted as xsd:integer without type objects, cannot be an OWL 2 DL individual; its instantiates is not asserted, with a warning. A value that is neither a CURIE nor an IRI would become a relative IRI and is skipped with a warning too. Signed-off-by: jdsika --- docs/generators/owl.rst | 48 +++++ .../linkml/src/linkml/generators/owlgen.py | 35 +++- tests/linkml/test_generators/test_owlgen.py | 182 ++++++++++++++++++ 3 files changed, 263 insertions(+), 2 deletions(-) diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 4b6f076fe8..b2daf4c291 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -135,6 +135,54 @@ You can control enum and permissible value representation directly in your schem implements: - rdfs:Literal +**Metaclass membership with ``instantiates``** - This LinkML field types a schema +element itself; it does not add slots or type that element's data instances. The OWL +generator maps each declared membership to ``rdf:type`` on the emitted resource. +The mapping applies consistently to schemas, classes, slots, locally emitted types, +enums, and non-literal permissible values. No additional option is required: the +schema already declares the relationship. + +For example, a permissible value represented as a named individual can declare +membership in an external vocabulary class: + +.. code-block:: yaml + + enums: + LinkCategory: + implements: + - owl:NamedIndividual + permissible_values: + isLicense: + meaning: ex:isLicense + instantiates: + - vocab:LicenseCategory + +.. code-block:: turtle + + ex:isLicense a owl:NamedIndividual, ex:LinkCategory, vocab:LicenseCategory . + +For an enum of named individuals, its own class still uses ``owl:oneOf``; this does +not close the external class. A permissible value rendered as an OWL class retains +that representation and also participates as an individual in the membership +assertion (OWL punning). Membership does not become ``rdfs:subClassOf`` and is not +inherited by instances of the emitted class. ``instantiates`` is ignored, with a +warning, on a permissible value rendered as a literal, because RDF literals cannot +be subjects of ``rdf:type`` triples, on a resource in the OWL 2 reserved vocabulary, +which cannot be an individual, and for a value that is neither a CURIE nor an IRI, +which would become a relative IRI. + +``gen-rdf`` and ``gen-jsonld`` serialize the schema as metamodel data and retain +``linkml:instantiates``. ``gen-owl`` translates that declaration into ontology +membership. Instance validators such as JSON Schema and SHACL must not apply the +schema element's metaclass to ordinary data instances. + +See the `LinkML instantiation guide +`_, +the `instantiates metamodel definition +`_, and the OWL 2 +specifications for `metamodeling `_ +and `mapping class assertions to RDF `_. + **Using URIs vs. text for permissible values:** .. code-block:: yaml diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 3f16c4965c..0da2849345 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -32,6 +32,7 @@ ClassDefinitionName, ClassRule, Definition, + Element, EnumDefinition, EnumDefinitionName, PermissibleValue, @@ -358,9 +359,9 @@ def serialize(self, **kwargs: Any) -> str: fmt = "turtle" if self.format in ["owl", "ttl"] else self.format return canonicalize_rdf_graph(self.graph, output_format=fmt, diff_stable=self.diff_stable) - def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: + def add_metadata(self, e: Element | PermissibleValue, uri: URIRef) -> None: """ - Add annotation properties. + Add annotation properties and explicit metaclass membership. Set the profile attribute to the appropriate OWL profile. Human-readable string literals are language-tagged when @@ -376,6 +377,8 @@ def add_metadata(self, e: Definition | PermissibleValue, uri: URIRef) -> None: sn_mappings = msv.slot_name_mappings() lang = self._resolve_language(e) + self._add_instantiates(e, uri) + # iterate through all the assigned metamodel slots for metaslot_name, metaslot_value in vars(e).items(): if not metaslot_value: @@ -1083,11 +1086,37 @@ def add_slot(self, slot: SlotDefinition, attribute: bool = False) -> None: for mixin in slot.mixins: self.graph.add((slot_uri, RDFS.subPropertyOf, self._prop_uri(mixin))) + def _add_instantiates(self, element: Element | PermissibleValue, uri: URIRef) -> None: + """Type the emitted schema resource itself, without typing its data instances. + + ``instantiates`` declares metaclass membership in LinkML. In OWL's RDF + mapping, class assertions are ``rdf:type`` triples; a resource also used + as an OWL class or property is interpreted separately as an individual. + A resource in the OWL 2 reserved vocabulary (Structural Specification + section 2.4, Table 2), such as a type emitted as ``xsd:integer``, cannot + be an individual, so its membership is not asserted. A value that is + neither a CURIE nor an IRI, such as a bare element name, would become a + relative IRI and is skipped. Both cases are logged as warnings. + """ + if element.instantiates and any(str(uri).startswith(str(ns)) for ns in (RDF, RDFS, XSD, OWL)): + logger.warning(f"Instantiates on {uri}, which is OWL 2 reserved vocabulary, is ignored") + return + for instantiated in element.instantiates: + expanded = self.schemaview.expand_curie(instantiated) + if ":" not in expanded: + logger.warning( + f"Instantiates {instantiated!r} on {uri}, which is neither a CURIE nor an IRI, is ignored" + ) + continue + self.graph.add((uri, RDF.type, URIRef(expanded))) + def add_type(self, typ: TypeDefinition) -> None: type_uri = self._type_uri(typ.name) if typ.from_schema == "https://w3id.org/linkml/types": return + self._add_instantiates(typ, type_uri) + if self.metaclasses: self.graph.add( ( @@ -1199,6 +1228,8 @@ def add_enum(self, e: EnumDefinition) -> None: pv_node = Literal(pv.text) if pv.meaning: logger.warning(f"Meaning on literal {pv.text} in {e.name} is ignored") + if pv.instantiates: + logger.warning(f"Instantiates on literal {pv.text} in {e.name} is ignored") else: pv_node = self._permissible_value_uri(pv, enum_uri, e) pv_uris.append(pv_node) diff --git a/tests/linkml/test_generators/test_owlgen.py b/tests/linkml/test_generators/test_owlgen.py index 02e21371fc..6fed5c8522 100644 --- a/tests/linkml/test_generators/test_owlgen.py +++ b/tests/linkml/test_generators/test_owlgen.py @@ -9,6 +9,7 @@ from linkml import METAMODEL_CONTEXT_URI from linkml.generators.owlgen import MetadataProfile, OwlSchemaGenerator +from linkml.generators.rdfgen import RDFGenerator from linkml_runtime.linkml_model import SlotDefinition from linkml_runtime.linkml_model.meta import ( AnonymousClassExpression, @@ -1266,3 +1267,184 @@ def test_cli_ontology_uri_suffix(kitchen_sink_path, args: list[str], expected_su g = Graph() g.parse(data=result.output, format="turtle") assert _ontology_iris(g) == {URIRef(f"https://w3id.org/linkml/tests/kitchen_sink{expected_suffix}")} + + +_INSTANTIATES_SCHEMA = """ +id: http://example.org/test-schema +name: instantiates_test +prefixes: + linkml: https://w3id.org/linkml/ + ex: http://example.org/test-schema/ + vocab: http://example.org/vocab/ +default_prefix: ex +imports: + - linkml:types +enums: + LinkCategory: + implements: + - owl:NamedIndividual + permissible_values: + isLicense: + meaning: ex:isLicense + instantiates: + - vocab:LicenseCategory + isManifest: + meaning: ex:isManifest + instantiates: + - vocab:ManifestCategory + - vocab:Category + isMedia: + meaning: ex:isMedia +""" + +VOCAB = Namespace("http://example.org/vocab/") + + +def test_permissible_value_instantiates_types_the_value() -> None: + """Each ``instantiates`` value of a permissible value becomes an rdf:type of its IRI.""" + g = Graph() + g.parse(data=OwlSchemaGenerator(_INSTANTIATES_SCHEMA, metaclasses=False).serialize(), format="turtle") + + assert (EX.isLicense, RDF.type, VOCAB.LicenseCategory) in g + assert (EX.isManifest, RDF.type, VOCAB.ManifestCategory) in g + assert (EX.isManifest, RDF.type, VOCAB.Category) in g + # the value stays an individual of the enum, which still closes its own class + for pv in (EX.isLicense, EX.isManifest, EX.isMedia): + assert (pv, RDF.type, OWL.NamedIndividual) in g + assert (pv, RDF.type, EX.LinkCategory) in g + one_of = g.value(EX.LinkCategory, OWL.oneOf) + assert set(Collection(g, one_of)) == {EX.isLicense, EX.isManifest, EX.isMedia} + # a value without instantiates gets no further type, and the instantiated classes stay open + assert set(g.objects(EX.isMedia, RDF.type)) == {OWL.NamedIndividual, EX.LinkCategory} + assert g.value(VOCAB.LicenseCategory, OWL.oneOf) is None + + +def test_permissible_value_instantiates_ignored_on_literal(caplog: pytest.LogCaptureFixture) -> None: + """A permissible value rendered as a literal cannot be typed, so instantiates is ignored with a warning.""" + schema = _INSTANTIATES_SCHEMA.replace("owl:NamedIndividual", "rdfs:Literal") + with caplog.at_level(logging.WARNING): + g = Graph() + g.parse(data=OwlSchemaGenerator(schema, metaclasses=False).serialize(), format="turtle") + assert not list(g.triples((None, RDF.type, VOCAB.LicenseCategory))) + assert any("Instantiates on literal isLicense" in rec.message for rec in caplog.records) + + +def test_instantiates_ignored_on_reserved_vocabulary(caplog: pytest.LogCaptureFixture) -> None: + """Without type objects a type is emitted as its XSD datatype, which cannot be + an OWL 2 DL individual, so its ``instantiates`` is not asserted on that IRI.""" + schema = """ +id: https://example.org/reserved +name: reserved +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/ +default_prefix: ex +types: + Count: + uri: xsd:integer + base: int + instantiates: [ex:Metadata] +""" + with caplog.at_level(logging.WARNING): + output = OwlSchemaGenerator(schema, type_objects=False, metaclasses=False).serialize() + graph = Graph().parse(data=output, format="turtle") + assert (XSD.integer, RDF.type, URIRef("https://example.org/Metadata")) not in graph + assert "reserved vocabulary" in caplog.text + + +def test_instantiates_ignored_for_a_bare_name(caplog: pytest.LogCaptureFixture) -> None: + """A value that is neither a CURIE nor an IRI would become a relative IRI, so it is skipped with a warning.""" + schema = ( + _INSTANTIATES_SCHEMA + + """ +classes: + Metadata: {} + Entry: + instantiates: [Metadata, vocab:ClassProfile] +""" + ) + with caplog.at_level(logging.WARNING): + g = OwlSchemaGenerator(schema, metaclasses=False).as_graph() + assert set(g.objects(EX.Entry, RDF.type)) == {OWL.Class, VOCAB.ClassProfile} + assert ( + "Instantiates 'Metadata' on http://example.org/test-schema/Entry, " + "which is neither a CURIE nor an IRI, is ignored" + ) in caplog.text + + +@pytest.mark.parametrize("use_native_uris", [True, False]) +def test_instantiates_types_the_emitted_class_resource(use_native_uris: bool) -> None: + """The assertion goes on the resource the class is emitted as: its native IRI, or its ``class_uri``.""" + schema = _INSTANTIATES_SCHEMA.replace( + " vocab: http://example.org/vocab/\n", " vocab: http://example.org/vocab/\n schema: http://schema.org/\n" + ) + schema += """ +classes: + Entry: + class_uri: schema:Thing + instantiates: [vocab:ClassProfile] +""" + g = OwlSchemaGenerator(schema, metaclasses=False, use_native_uris=use_native_uris).as_graph() + expected = EX.Entry if use_native_uris else URIRef("http://schema.org/Thing") + assert set(g.subjects(RDF.type, VOCAB.ClassProfile)) == {expected} + + +@pytest.mark.parametrize("metaclasses", [False, True]) +@pytest.mark.parametrize("type_objects", [False, True]) +def test_instantiates_applies_to_emitted_schema_elements(metaclasses: bool, type_objects: bool) -> None: + """Metaclass membership is explicit schema metadata, independent of generated metamodel types.""" + schema = _INSTANTIATES_SCHEMA.replace( + "name: instantiates_test", "name: instantiates_test\ninstantiates: [vocab:SchemaProfile]" + ).replace(" LinkCategory:\n", " LinkCategory:\n instantiates: [vocab:EnumProfile]\n") + schema += """ +classes: + Entry: + instantiates: [vocab:ClassProfile] + slots: [code] +slots: + code: + range: Code + instantiates: [vocab:SlotProfile] +types: + Code: + typeof: string + uri: ex:Code + instantiates: [vocab:TypeProfile] +""" + g = OwlSchemaGenerator(schema, metaclasses=metaclasses, type_objects=type_objects).as_graph() + for resource, metaclass in ( + (URIRef("http://example.org/test-schema"), VOCAB.SchemaProfile), + (EX.Entry, VOCAB.ClassProfile), + (EX.code, VOCAB.SlotProfile), + (EX.Code, VOCAB.TypeProfile), + (EX.LinkCategory, VOCAB.EnumProfile), + (EX.isLicense, VOCAB.LicenseCategory), + ): + assert (resource, RDF.type, metaclass) in g + assert (resource, RDFS.subClassOf, metaclass) not in g + assert (EX.isLicense, RDF.type, VOCAB.EnumProfile) not in g + + +@pytest.mark.parametrize("owl_type", ["owl:NamedIndividual", "owl:Class"]) +@pytest.mark.parametrize("meaning", ["ex:isLicense", "https://example.net/value", None]) +def test_instantiates_uses_the_emitted_value_resource(owl_type: str, meaning: str | None) -> None: + """Explicit and minted IRIs both carry membership, including class/individual punning.""" + schema = _INSTANTIATES_SCHEMA.replace("owl:NamedIndividual", owl_type) + schema = schema.replace(" meaning: ex:isLicense\n", f" meaning: {meaning}\n" if meaning else "") + schema = schema.replace("vocab:LicenseCategory", "https://example.net/Category") + g = OwlSchemaGenerator(schema, metaclasses=False).as_graph() + (resource,) = g.subjects(RDF.type, URIRef("https://example.net/Category")) + assert isinstance(resource, URIRef) + assert (resource, RDFS.label, Literal("isLicense")) in g + assert (resource, RDF.type, OWL[owl_type.split(":")[1]]) in g + if meaning: + assert resource == (EX.isLicense if meaning.startswith("ex:") else URIRef(meaning)) + + +def test_instantiates_rdf_schema_representation_is_distinct_from_owl_assertion() -> None: + """gen-rdf preserves the metamodel predicate; gen-owl interprets it as membership.""" + schema = _INSTANTIATES_SCHEMA.replace("imports:\n - linkml:types\n", "") + rdf = Graph().parse(data=RDFGenerator(schema).serialize(), format="turtle") + owl = OwlSchemaGenerator(schema, metaclasses=False).as_graph() + assert {str(value) for value in rdf.objects(EX.isLicense, LINKML.instantiates)} == {"vocab:LicenseCategory"} + assert (EX.isLicense, RDF.type, VOCAB.LicenseCategory) in owl From 2c7b38d3bb9bdae20a9d1dadcff6641a45281d5d Mon Sep 17 00:00:00 2001 From: jdsika Date: Wed, 21 Oct 2026 17:56:41 +0200 Subject: [PATCH 09/17] fix(rdf): give values of URI-like types one RDF term kind in every generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A value of a type is an IRI, a blank node or a literal according to one rule, linkml_runtime.utils.rdf_terms.type_term, which gen-jsonld-context, gen-owl, gen-shacl, gen-shex, gen-yarrrml and the RDF dumper and loader now share: - uri, uriorcurie and the types derived from them are IRIs; - objectidentifier is an IRI, and nodeidentifier and types with uri: rdfs:Resource are IRIs or blank nodes; - every other type is a literal of its datatype, including a type derived from string with uri: xsd:anyURI, which keeps URI references as text. The rule resolves the inherited, expanded datatype, so its spelling does not matter, and recognises uri and uriorcurie by name and definition rather than by from_schema, which merging imports rewrites. A datatype with an undeclared prefix is an error (xsd, shex and rdfs are known). The dumper writes the inherited datatype of a derived type and rejects relative values of IRI types (RDF 1.1 Concepts §3.2); the loader accepts IRIs for node-kind ranges in strict mode, keeps uri values complete, compacts uriorcurie values to CURIEs and keeps blank-node labels. gen-shacl gives sh:in values and sh:defaultValue the slot's term kind. The --xsd-anyuri-as-iri option of gen-owl and gen-jsonld-context is deprecated in 1.13.0, has no effect, and is scheduled for removal in 1.14.0. Signed-off-by: jdsika --- docs/generators/jsonld-context.rst | 6 + docs/generators/owl.rst | 3 + docs/schemas/advanced.md | 56 +++ .../linkml/generators/common/subproperty.py | 33 -- .../src/linkml/generators/jsonldcontextgen.py | 75 ++- .../linkml/src/linkml/generators/owlgen.py | 60 +-- .../src/linkml/generators/prefixmapgen.py | 5 +- .../linkml/src/linkml/generators/shaclgen.py | 56 +-- .../linkml/src/linkml/generators/shexgen.py | 17 +- .../src/linkml/generators/yarrrmlgen.py | 13 +- .../linkml/src/linkml/utils/deprecation.py | 16 + packages/linkml/src/linkml/utils/generator.py | 26 +- .../linkml/src/linkml/utils/schemaloader.py | 5 +- .../linkml_runtime/dumpers/rdflib_dumper.py | 19 +- .../linkml_runtime/loaders/rdflib_loader.py | 17 +- .../src/linkml_runtime/utils/rdf_terms.py | 207 ++++++++ .../__snapshots__/annotations.context.jsonld | 2 +- .../test_base/__snapshots__/annotations.json | 2 +- .../__snapshots__/extensions.context.jsonld | 2 +- .../test_base/__snapshots__/extensions.json | 2 +- .../__snapshots__/mappings.context.jsonld | 16 +- .../test_base/__snapshots__/mappings.json | 16 +- .../__snapshots__/meta.context.jsonld | 78 ++-- .../linkml/test_base/__snapshots__/meta.json | 78 ++-- .../__snapshots__/biolink.context.jsonld | 26 +- .../__snapshots__/biolink.json | 26 +- .../test_generators/test_jsonldcontextgen.py | 442 ++++-------------- tests/linkml/test_generators/test_shaclgen.py | 6 +- .../test_generators/test_uri_term_kinds.py | 323 +++++++++++++ .../simple_uri_test.merge.context.jsonld | 2 +- ...e_uri_test.merge.model_only.context.jsonld | 2 +- .../simple_uri_test.no_merge.context.jsonld | 2 +- ...ri_test.no_merge.model_only.context.jsonld | 2 +- .../genjsonld/simple_uri_test.context.jsonld | 2 +- .../test_utils/__snapshots__/uriandcurie.json | 2 +- .../__snapshots__/uriandcurie.jsonld | 2 +- .../test_utils/test_rdf_terms.py | 202 ++++++++ 37 files changed, 1190 insertions(+), 659 deletions(-) create mode 100644 packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py create mode 100644 tests/linkml/test_generators/test_uri_term_kinds.py create mode 100644 tests/linkml_runtime/test_utils/test_rdf_terms.py diff --git a/docs/generators/jsonld-context.rst b/docs/generators/jsonld-context.rst index da5e4df794..18cec34644 100644 --- a/docs/generators/jsonld-context.rst +++ b/docs/generators/jsonld-context.rst @@ -47,6 +47,12 @@ declarations and Any JSON that conforms to the derived JSON Schema (see above) can be converted to RDF using this context. +Values of ``uri``, ``uriorcurie``, the types derived from them, and the node +types ``objectidentifier`` and ``nodeidentifier`` are coerced to ``@id``; a type +derived from ``string`` with ``uri: xsd:anyURI`` keeps URI references as +``xsd:anyURI`` literals (see +:ref:`links-and-uri-references`). + CURIE usage ^^^^^^^^^^^ diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index b2daf4c291..da32043267 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -33,6 +33,9 @@ Mapping * Each LinkML slot maps to an OWL property - if the range of the slot is class, then an ObjectProperty is used + - an ObjectProperty is also used when the range is a type whose values are IRIs or blank + nodes: ``uri``, ``uriorcurie`` and the types derived from them, ``objectidentifier`` and + ``nodeidentifier`` (see :ref:`links-and-uri-references`) - otherwise DataProperty is used - Exception to the above: if ``type_objects`` is set then ObjectProperties are always used; instead of having ranges of type be mapped to literals, these map to class-shadows of diff --git a/docs/schemas/advanced.md b/docs/schemas/advanced.md index 2b59901638..ef1cccbfd3 100644 --- a/docs/schemas/advanced.md +++ b/docs/schemas/advanced.md @@ -245,3 +245,59 @@ details of how to use this in code. In some cases, you may want to use a URI or CURIE as a range. To do this, create a class with `class_uri` set to the URI or CURIE and use that class as the range. + +(links-and-uri-references)= +## Links and URI references + +A value of a URI-like type is either a link, which becomes an RDF IRI, or a URI +reference held as text, which becomes an `xsd:anyURI` literal. The type decides, +in the same way for every generator and for the RDF dumper and loader: + +| Type | RDF term | Use for | +|---|---|---| +| `uri`, `uriorcurie` and types derived from them | IRI (CURIEs are expanded) | links to resources | +| `objectidentifier` | IRI | references to objects | +| `nodeidentifier`, or a type with `uri: rdfs:Resource` | IRI or blank node (`_:b1`) | references to nodes | +| a type derived from `string` with `uri: xsd:anyURI` | `xsd:anyURI` literal, as written | URI references held as data, such as relative file paths | +| `curie` | `xsd:string` literal, not expanded | compact identifiers held as text | + +```yaml +types: + FilePath: + typeof: string + uri: xsd:anyURI + description: A file location, which may be relative. +slots: + homepage: + range: uri + file_path: + range: FilePath +``` + +For `homepage: https://alice.example.org/` and `file_path: ./docs/cv.pdf` the RDF is + +```turtle +ex:alice ex:homepage ; + ex:file_path "./docs/cv.pdf"^^xsd:anyURI . +``` + +gen-jsonld-context coerces `homepage` to `@id` and `file_path` to `xsd:anyURI`, +gen-shacl requires `sh:nodeKind sh:IRI` for one and `sh:datatype xsd:anyURI` for +the other, and gen-owl makes `homepage` an object property and `file_path` a data +property. + +An RDF IRI must be absolute ([RDF 1.1 Concepts §3.2](https://www.w3.org/TR/rdf11-concepts/#section-IRIs)), +so the RDF dumper rejects a relative value of a link type, such as `./home.html` in +`homepage`, and JSON Schema validation rejects it for `uri` and the types derived +from it (`format: uri`). Through a JSON-LD context, `@id` values are resolved +against the base IRI of the document. Give values that may be relative a text type +such as `FilePath`. + +The RDF loader reads an IRI back as a complete URI for `uri` and the types derived +from it, and as a CURIE, where the schema declares the prefix, for `uriorcurie`, +`objectidentifier` and `nodeidentifier`. + +The `--xsd-anyuri-as-iri` option of gen-owl and gen-jsonld-context is deprecated +and has no effect: links are IRIs without it. A schema that relied on the earlier +default, where `uri` values were `xsd:anyURI` literals in the JSON-LD context and +in OWL, keeps that output by giving those slots a text type such as `FilePath`. diff --git a/packages/linkml/src/linkml/generators/common/subproperty.py b/packages/linkml/src/linkml/generators/common/subproperty.py index 9b136e2429..4687c38218 100644 --- a/packages/linkml/src/linkml/generators/common/subproperty.py +++ b/packages/linkml/src/linkml/generators/common/subproperty.py @@ -15,10 +15,6 @@ CURIE_TYPES: frozenset[str] = frozenset({"uriorcurie", "curie"}) URI_TYPES: frozenset[str] = frozenset({"uri"}) -# Types whose XSD mapping is xsd:anyURI (not xsd:string). -# ``curie`` maps to xsd:string and is deliberately excluded. -_ANYURI_TYPES: frozenset[str] = frozenset({"uri", "uriorcurie"}) - def is_uri_range(sv: SchemaView, range_type: str | None) -> bool: """ @@ -67,35 +63,6 @@ def is_curie_range(sv: SchemaView, range_type: str | None) -> bool: return False -def is_xsd_anyuri_range(sv: SchemaView, range_type: str | None) -> bool: - """Check if range type resolves to ``xsd:anyURI``. - - Returns True for ``uri``, ``uriorcurie``, and types that inherit from them. - Returns False for ``curie`` (which maps to ``xsd:string``). - - This is the correct predicate for the ``--xsd-anyuri-as-iri`` flag: only - types whose XSD representation is ``xsd:anyURI`` should be promoted from - literal to IRI semantics. ``curie`` is a compact string representation - that resolves to ``xsd:string`` and must not be affected. - - :param sv: SchemaView for type ancestry lookup - :param range_type: The range type to check - :return: True if range type maps to xsd:anyURI - """ - if range_type is None: - return False - - if range_type in _ANYURI_TYPES: - return True - - if range_type in sv.all_types(): - type_ancestors = set(sv.type_ancestors(range_type)) - if type_ancestors & _ANYURI_TYPES: - return True - - return False - - def format_slot_value_for_range(sv: SchemaView, slot_name: str, range_type: str | None) -> str: """ Format slot value according to the declared range type. diff --git a/packages/linkml/src/linkml/generators/jsonldcontextgen.py b/packages/linkml/src/linkml/generators/jsonldcontextgen.py index 5462b29f8b..4916fa07c3 100644 --- a/packages/linkml/src/linkml/generators/jsonldcontextgen.py +++ b/packages/linkml/src/linkml/generators/jsonldcontextgen.py @@ -14,19 +14,13 @@ from rdflib import SKOS, XSD, Namespace from linkml._version import __version__ -from linkml.utils.deprecation import deprecated_fields +from linkml.utils.deprecation import XSD_ANYURI_AS_IRI_OPTION, deprecated_fields, deprecation_warning from linkml.utils.generator import Generator, shared_arguments from linkml_runtime.linkml_model.meta import ClassDefinition, EnumDefinition, SlotDefinition -from linkml_runtime.linkml_model.types import SHEX from linkml_runtime.utils.formatutils import camelcase, underscore +from linkml_runtime.utils.rdf_terms import type_term from linkml_runtime.utils.schemaview import SchemaView -URI_RANGES = (SHEX.nonliteral, SHEX.bnode, SHEX.iri) - -# Extended URI_RANGES that also treats xsd:anyURI as an IRI reference (@id) -# rather than a typed literal. Opt-in via --xsd-anyuri-as-iri flag. -URI_RANGES_WITH_XSD = (*URI_RANGES, XSD.anyURI) - ENUM_CONTEXT = { "text": "skos:notation", "description": "skos:prefLabel", @@ -81,12 +75,9 @@ class ContextGenerator(Generator): context keys. Identifier slots retain their slot-name key and map to ``@id``. """ - xsd_anyuri_as_iri: bool = False - """Map xsd:anyURI-typed ranges (uri, uriorcurie) to ``@type: @id`` instead of ``@type: xsd:anyURI``. - - This aligns the JSON-LD context with the SHACL generator, which emits - ``sh:nodeKind sh:IRI`` for the same types. - """ + xsd_anyuri_as_iri: bool | None = None + """Deprecated and without effect: values of ``uri``, ``uriorcurie`` and the types + derived from them are always coerced to ``@id``.""" # Framing (opt-in via CLI flag) emit_frame: bool = False @@ -95,6 +86,8 @@ class ContextGenerator(Generator): frame_root: str | None = None def __post_init__(self) -> None: + if self.xsd_anyuri_as_iri is not None: + deprecation_warning(XSD_ANYURI_AS_IRI_OPTION, stack_level=4) super().__post_init__() if self.namespaces is None: raise TypeError("Schema text must be supplied to context generator. Preparsed schema will not work") @@ -299,19 +292,9 @@ def _literal_coercion_for_ranges(self, ranges: list[str]) -> tuple[bool, str | N and "could not resolve safely because the branches disagree". """ coercions: set[str | None] = set() - uri_ranges = URI_RANGES_WITH_XSD if self.xsd_anyuri_as_iri else URI_RANGES for range_name in ranges: - if range_name not in self.schema.types: - continue - - range_type = self.schema.types[range_name] - range_uri = self.namespaces.uri_for(range_type.uri) - if range_uri == XSD.string: - coercions.add(None) - elif range_uri in uri_ranges: - coercions.add("@id") - else: - coercions.add(range_type.uri) + if range_name in self.schema.types: + coercions.add(self._type_coercion(range_name)) if not coercions: return False, None @@ -413,14 +396,9 @@ def visit_slot(self, aliased_slot_name: str, slot: SlotDefinition) -> None: skos = "skos" self.emit_prefixes.add(skos) else: - range_type = self.schema.types[slot.range] - uri_ranges = URI_RANGES_WITH_XSD if self.xsd_anyuri_as_iri else URI_RANGES - if self.namespaces.uri_for(range_type.uri) == XSD.string: - pass - elif self.namespaces.uri_for(range_type.uri) in uri_ranges: - slot_def["@type"] = "@id" - else: - slot_def["@type"] = range_type.uri + coercion = self._type_coercion(slot.range) + if coercion is not None: + slot_def["@type"] = coercion if self.fix_multivalue_containers and slot.multivalued: if slot.inlined and not slot.inlined_as_list: @@ -457,15 +435,25 @@ def _resolve_slot_type(self, range_name: str) -> str | None: if range_name in self.schema.enums: return None if range_name in self.schema.types: - range_type = self.schema.types[range_name] - uri = self.namespaces.uri_for(range_type.uri) - if uri == XSD.string: - return None - if uri in URI_RANGES: - return "@id" - return range_type.uri + return self._type_coercion(range_name) return None + def _type_coercion(self, type_name: str) -> str | None: + """Return the JSON-LD ``@type`` coercion of values of the type *type_name*. + + Values of node types (:func:`~linkml_runtime.utils.rdf_terms.type_term`) are + coerced to ``@id``, which also turns ``_:`` labels into blank nodes. Literals + are coerced to their datatype as the schema writes it; plain strings need no + coercion, so the result is ``None``. + """ + view = self.type_schemaview() + term = type_term(view, type_name) + if term.kind.is_node: + return "@id" + if term.datatype is None or term.datatype == str(XSD.string): + return None + return view.induced_type(type_name).uri + def _build_scoped_context(self, cls: ClassDefinition) -> dict: """Build a scoped JSON-LD context for class-level slot range overrides. @@ -605,9 +593,8 @@ def serialize( ) @click.option( "--xsd-anyuri-as-iri/--no-xsd-anyuri-as-iri", - default=False, - show_default=True, - help="Map xsd:anyURI-typed ranges (uri, uriorcurie) to @type: @id instead of @type: xsd:anyURI.", + default=None, + help="Deprecated and without effect: values of uri, uriorcurie and the types derived from them are always IRIs.", ) @click.option( "--use-curies/--not-use-curies", diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 0da2849345..880e561eae 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -19,8 +19,7 @@ from linkml import METAMODEL_NAMESPACE_NAME from linkml._version import __version__ -from linkml.generators.common.subproperty import is_xsd_anyuri_range -from linkml.utils.deprecation import deprecation_warning +from linkml.utils.deprecation import XSD_ANYURI_AS_IRI_OPTION, deprecation_warning from linkml.utils.generator import Generator, shared_arguments from linkml.utils.language_tags import LanguageTagResolver from linkml_runtime import SchemaView @@ -45,6 +44,7 @@ from linkml_runtime.utils.formatutils import camelcase, underscore from linkml_runtime.utils.introspection import package_schemaview from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph +from linkml_runtime.utils.rdf_terms import type_term logger = logging.getLogger(__name__) @@ -227,23 +227,9 @@ def _present(values: Iterable[_T | None]) -> list[_T]: return [value for value in values if value is not None] - xsd_anyuri_as_iri: bool = False - """Treat ``range: uri`` / ``range: uriorcurie`` slots as ``owl:ObjectProperty`` - instead of ``owl:DatatypeProperty`` with ``rdfs:range xsd:anyURI``. - - This aligns the OWL output with the SHACL generator (which emits - ``sh:nodeKind sh:IRI``) and the JSON-LD context generator (which emits - ``@type: @id`` when its own ``--xsd-anyuri-as-iri`` flag is set). - - Without this flag, ``range: uri`` produces a semantic inconsistency: - OWL says the value is a literal (``DatatypeProperty``), while SHACL and - JSON-LD say it is an IRI node. Enabling the flag makes all three - generators consistent. - - When enabled, URI-range slots: - - become ``owl:ObjectProperty`` (not ``owl:DatatypeProperty``) - - have no ``rdfs:range`` restriction (any IRI is valid) - """ + xsd_anyuri_as_iri: bool | None = None + """Deprecated and without effect: slots whose values are IRIs or blank nodes + (:func:`~linkml_runtime.utils.rdf_terms.type_term`) are always object properties.""" default_language: str | None = None """Default BCP 47 language tag for human-readable string literals. @@ -269,6 +255,8 @@ def __post_init__(self) -> None: # once here; per-element tags are validated lazily, with at most one # warning per distinct malformed tag. self._language_resolver = LanguageTagResolver(self.default_language) + if self.xsd_anyuri_as_iri is not None: + deprecation_warning(XSD_ANYURI_AS_IRI_OPTION, stack_level=4) super().__post_init__() def _resolve_language(self, element: "Definition | PermissibleValue | None" = None) -> str | None: @@ -804,7 +792,8 @@ def slot_node_owltypes( """ Determine the OWL types of a named slot or slot expression - The OWL type is either OWL.Thing or RDFS.Datatype + The OWL type is either OWL.Thing or RDFS.Datatype. A type range gives OWL.Thing + when its values are IRIs or blank nodes (:meth:`_type_values_are_nodes`). :param slot: :param owning_class: @@ -821,7 +810,7 @@ def slot_node_owltypes( if not (range_class and range_class.class_uri == "linkml:Any"): node_types.add(OWL.Thing) if slot.range in sv.all_types(): - node_types.add(RDFS.Datatype) + node_types.add(OWL.Thing if self._type_values_are_nodes(slot.range) else RDFS.Datatype) for k in ["any_of", "all_of", "exactly_one_of", "none_of"]: subslots = getattr(slot, k, None) if not subslots: @@ -908,7 +897,7 @@ def _get_slot_nodes( this_owl_types: set[OWL_TYPE] = set() if slot_range: if slot_range in sv.all_types(imports=True): - if self.xsd_anyuri_as_iri and is_xsd_anyuri_range(sv, slot_range): + if self._type_values_are_nodes(slot_range): self.slot_is_literal_map[main_slot.name].add(False) this_owl_types.add(OWL.Thing) else: @@ -1577,12 +1566,14 @@ def _boolean_expression( graph.add((node, RDF.type, RDFS.Datatype)) return node - def _range_is_datatype(self, slot: SlotDefinition) -> bool: - if self.type_objects: - return False - if self.xsd_anyuri_as_iri and is_xsd_anyuri_range(self.schemaview, slot.range): - return False - return slot.range in self.schema.types + def _type_values_are_nodes(self, type_name: str) -> bool: + """Whether values of the type *type_name* are IRIs or blank nodes rather than literals. + + This follows :func:`~linkml_runtime.utils.rdf_terms.type_term`. With + :attr:`type_objects`, every typed value is an individual of the type's class + instead, so no type is treated differently. + """ + return not self.type_objects and type_term(self.schemaview, type_name).kind.is_node def _range_uri(self, slot: SlotDefinition) -> URIRef: if slot.range in self.schema.types: @@ -1730,7 +1721,7 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: elif slot_range in sv.all_enums(): return OWL.ObjectProperty elif slot_range in sv.all_types(): - if self.xsd_anyuri_as_iri and is_xsd_anyuri_range(sv, slot_range): + if self._type_values_are_nodes(slot_range): return OWL.ObjectProperty return OWL.DatatypeProperty else: @@ -1858,14 +1849,9 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: ) @click.option( "--xsd-anyuri-as-iri/--no-xsd-anyuri-as-iri", - default=False, - show_default=True, - help=( - "Treat range: uri / range: uriorcurie slots as owl:ObjectProperty (IRI node) " - "instead of owl:DatatypeProperty with rdfs:range xsd:anyURI (literal). " - "Aligns OWL output with the SHACL generator (sh:nodeKind sh:IRI) and " - "the JSON-LD context generator (--xsd-anyuri-as-iri → @type: @id)." - ), + default=None, + help="Deprecated and without effect: slots whose values are IRIs, such as uri and uriorcurie, are always " + "object properties.", ) @click.option( "--default-language", diff --git a/packages/linkml/src/linkml/generators/prefixmapgen.py b/packages/linkml/src/linkml/generators/prefixmapgen.py index eaf6d590e7..c635520d46 100644 --- a/packages/linkml/src/linkml/generators/prefixmapgen.py +++ b/packages/linkml/src/linkml/generators/prefixmapgen.py @@ -8,16 +8,13 @@ import click from jsonasobj2 import JsonObj, as_json -from rdflib import XSD, Namespace +from rdflib import Namespace from linkml._version import __version__ from linkml.utils.generator import Generator, shared_arguments from linkml_runtime.linkml_model.meta import ClassDefinition, SlotDefinition -from linkml_runtime.linkml_model.types import SHEX from linkml_runtime.utils.formatutils import camelcase -URI_RANGES = (XSD.anyURI, SHEX.nonliteral, SHEX.bnode, SHEX.iri) - @dataclass class PrefixGenerator(Generator): diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index 33fc69515a..d3eb008faf 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -5,6 +5,7 @@ import string from collections.abc import Callable from dataclasses import dataclass, fields +from typing import ClassVar import click from jsonasobj2 import JsonObj, as_dict @@ -14,7 +15,7 @@ from linkml._version import __version__ from linkml.generators.common.class_expression import class_membership, value_bounds -from linkml.generators.common.subproperty import get_subproperty_values, is_uri_range +from linkml.generators.common.subproperty import get_subproperty_values from linkml.generators.shacl.shacl_data_type import ShaclDataType from linkml.generators.shacl.shacl_ifabsent_processor import ShaclIfAbsentProcessor from linkml.utils.generator import Generator, shared_arguments @@ -33,6 +34,7 @@ ) from linkml_runtime.utils.formatutils import underscore from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph +from linkml_runtime.utils.rdf_terms import TermKind, type_term, value_term from linkml_runtime.utils.yamlutils import TypedNode, extended_float, extended_int, extended_str logger = logging.getLogger(__name__) @@ -460,6 +462,9 @@ def branch_pv(p, v): default_value = ifabsent_processor.process_slot(s, c) if default_value: + if s.range in sv.all_types() and type_term(sv, s.range).kind.is_node: + # The default is a value of the slot, so it has the slot's term kind. + default_value = value_term(sv, s.range, str(default_value)) prop_pv(SH.defaultValue, default_value) self._add_class_expressions(g, class_uri_with_suffix, c) @@ -1908,40 +1913,23 @@ def _add_enum(self, g: Graph, func: Callable, r: ElementName) -> None: ) func(SH["in"], pv_node) - # Type URIs denoting non-literal (IRI or blank-node) values. - # SHACL §4.8.1 - # defines sh:IRI, sh:BlankNode, and sh:BlankNodeOrIRI as valid node kinds. - # These URIs map to sh:IRI or sh:BlankNodeOrIRI constraints (never sh:Literal). - _NON_LITERAL_TYPE_URIS = frozenset( - { - "xsd:anyURI", # uri, uriorcurie → sh:IRI - "http://www.w3.org/ns/shex#nonLiteral", # nodeidentifier → sh:BlankNodeOrIRI - "http://www.w3.org/ns/shex#iri", # future-proofing → sh:IRI - } - ) - # IRI-only subset: uri/uriorcurie must be strict IRI references (sh:IRI), - # while nodeidentifier (shex:nonLiteral) allows blank nodes too (sh:BlankNodeOrIRI). - # See RDF 1.1 §3.2–3.3 . - _IRI_ONLY_TYPE_URIS = frozenset( - { - "xsd:anyURI", - } - ) + # SHACL §4.1.3 node kinds for the node term kinds of LinkML types. + _SH_NODE_KINDS: ClassVar[dict[TermKind, URIRef]] = { + TermKind.IRI: SH.IRI, + TermKind.BLANK_NODE: SH.BlankNode, + TermKind.BLANK_NODE_OR_IRI: SH.BlankNodeOrIRI, + } def _add_type(self, func: Callable, r: ElementName) -> None: sv = self.schemaview # Types can inherit URI and pattern constraints. rt = sv.induced_type(r) - type_uri = rt.uri - expanded = sv.get_uri(rt, expand=True) if type_uri else None - if type_uri and (type_uri in self._NON_LITERAL_TYPE_URIS or expanded in self._NON_LITERAL_TYPE_URIS): - if type_uri in self._IRI_ONLY_TYPE_URIS: - func(SH.nodeKind, SH.IRI) - else: - func(SH.nodeKind, SH.BlankNodeOrIRI) - elif type_uri: + term = type_term(sv, r) if rt.uri else None + if term is not None and term.kind.is_node: + func(SH.nodeKind, self._SH_NODE_KINDS[term.kind]) + elif term is not None: func(SH.nodeKind, SH.Literal) - func(SH.datatype, URIRef(sv.get_uri(rt, expand=True))) + func(SH.datatype, URIRef(term.datatype)) if rt.pattern: func(SH.pattern, Literal(rt.pattern)) if rt.annotations and self.include_annotations: @@ -1979,17 +1967,17 @@ def _get_subproperty_values(self, slot) -> list: """ Get all valid values from slot hierarchy for subproperty_of constraint. - Values are formatted according to range type: - - uri/uriorcurie: Returns URIRef objects with full URIs - - string: Returns Literal objects with slot names + Values take the term kind of the slot's range type: + - IRI types (uri, uriorcurie): URIRef objects with full URIs + - literal types (string, curie): Literal objects with slot names or CURIEs :param slot: SlotDefinition with subproperty_of set :return: List of URIRef or Literal objects for sh:in constraint """ sv = self.schemaview - # SHACL uses full URIs for URI-like ranges - use_uris = is_uri_range(sv, slot.range) + # sh:in matches terms exactly (SHACL §4.8.3), so the values take the slot's term kind. + use_uris = slot.range in sv.all_types() and type_term(sv, slot.range).kind.is_node # Get string values from shared utility # For URI ranges, get full URIs; for string ranges, get formatted names diff --git a/packages/linkml/src/linkml/generators/shexgen.py b/packages/linkml/src/linkml/generators/shexgen.py index e74e45225e..0fdddb92cb 100644 --- a/packages/linkml/src/linkml/generators/shexgen.py +++ b/packages/linkml/src/linkml/generators/shexgen.py @@ -6,7 +6,7 @@ import click from jsonasobj import as_json as as_json_1 -from rdflib import OWL, RDF, XSD, Graph, Namespace +from rdflib import OWL, RDF, Graph, Namespace, URIRef from ShExJSG import ShExC from ShExJSG.SchemaWithContext import Schema from ShExJSG.ShExJ import IRIREF, EachOf, NodeConstraint, Shape, ShapeOr, TripleConstraint @@ -23,10 +23,10 @@ SlotDefinitionName, TypeDefinition, ) -from linkml_runtime.linkml_model.types import SHEX from linkml_runtime.utils.formatutils import camelcase, sfx from linkml_runtime.utils.metamodelcore import URIorCURIE from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph +from linkml_runtime.utils.rdf_terms import TermKind, type_term @dataclass @@ -38,6 +38,8 @@ class ShExGenerator(Generator): file_extension = "shex.rdf" visit_all_class_slots = False uses_schemaloader = True + # ShEx 2.1 §5.4.2 node kinds for the node term kinds of LinkML types + _SHEX_NODE_KINDS = {TermKind.IRI: "iri", TermKind.BLANK_NODE: "bnode", TermKind.BLANK_NODE_OR_IRI: "nonliteral"} # ObjectVars diff_stable: bool = False @@ -71,16 +73,15 @@ def visit_schema(self, **_): context = self.shex["@context"] self.shex["@context"] = [context, {"@base": self.namespaces._base}] # Emit all of the type definitions + type_view = self.type_schemaview() for typ in self.schema.types.values(): model_uri = self._class_or_type_uri(typ) if typ.uri: - typ_type_uri = self.namespaces.uri_for(typ.uri) - if typ_type_uri in (XSD.anyURI, SHEX.iri): - self.shapes.append(NodeConstraint(id=model_uri, nodeKind="iri")) - elif typ_type_uri == SHEX.nonLiteral: - self.shapes.append(NodeConstraint(id=model_uri, nodeKind="nonliteral")) + term = type_term(type_view, typ.name) + if term.kind.is_node: + self.shapes.append(NodeConstraint(id=model_uri, nodeKind=self._SHEX_NODE_KINDS[term.kind])) else: - self.shapes.append(NodeConstraint(id=model_uri, datatype=self.namespaces.uri_for(typ.uri))) + self.shapes.append(NodeConstraint(id=model_uri, datatype=URIRef(term.datatype))) else: typeof_uri = self._class_or_type_uri(typ.typeof) self.shapes.append(Shape(id=model_uri, expression=typeof_uri)) diff --git a/packages/linkml/src/linkml/generators/yarrrmlgen.py b/packages/linkml/src/linkml/generators/yarrrmlgen.py index 6847dafbd1..887c0eaae3 100644 --- a/packages/linkml/src/linkml/generators/yarrrmlgen.py +++ b/packages/linkml/src/linkml/generators/yarrrmlgen.py @@ -10,6 +10,7 @@ from linkml._version import __version__ from linkml.utils.generator import Generator, shared_arguments from linkml_runtime.linkml_model.meta import ClassDefinition +from linkml_runtime.utils.rdf_terms import TermKind, type_term class FlowList(list): @@ -343,10 +344,14 @@ def _po_list_for_class(self, c: ClassDefinition, inline_owners: dict) -> list[di # Scalar values and datatype mapping datatype = None - t = sv.get_type(s.range) if s.range else None - - if t: - datatype = sv.get_uri(t, expand=False) + if s.range in sv.all_types(): + term = type_term(sv, s.range) + if term.kind.is_node: + # YARRRML term types are iri, blank and literal; a kind that admits IRIs maps to iri. + term_type = "blank" if term.kind is TermKind.BLANK_NODE else "iri" + po.append({"p": pred, "o": {"value": f"$({var})", "type": term_type}}) + continue + datatype = sv.induced_type(s.range).uri if datatype: po.append({"p": pred, "o": {"value": f"$({var})", "datatype": str(datatype)}}) diff --git a/packages/linkml/src/linkml/utils/deprecation.py b/packages/linkml/src/linkml/utils/deprecation.py index 9cded2f46b..fc34c87093 100644 --- a/packages/linkml/src/linkml/utils/deprecation.py +++ b/packages/linkml/src/linkml/utils/deprecation.py @@ -322,6 +322,21 @@ def warn(self, stack_level=3, **kwargs): recommendation="Use `package` (`--package` on the command line) instead.", issue=3780, ), + Deprecation( + name="xsd-anyuri-as-iri-option", + deprecated_in=SemVer.from_str("1.13.0"), + removed_in=SemVer.from_str("1.14.0"), + message=( + "The `xsd_anyuri_as_iri` option (`--xsd-anyuri-as-iri` on the command line) of gen-owl and " + "gen-jsonld-context has no effect: values of `uri`, `uriorcurie` and the types derived from " + "them are IRIs in every generator." + ), + recommendation=( + "Remove the option. To keep a URI reference as text (an xsd:anyURI literal), use a type " + "derived from `string` with `uri: xsd:anyURI`." + ), + issue=2495, + ), ) # type: tuple[Deprecation, ...] EMITTED = set() # type: set[str] @@ -357,6 +372,7 @@ def deprecation_warning(name: str, stack_level: int = 3): METADATA_FLAG = "metadata-flag" MATERIALIZE_PATTERNS_GENERATOR_OPTION = "materialize-patterns-generator-option" +XSD_ANYURI_AS_IRI_OPTION = "xsd-anyuri-as-iri-option" T = TypeVar("T") diff --git a/packages/linkml/src/linkml/utils/generator.py b/packages/linkml/src/linkml/utils/generator.py index 8c916ccb03..d5d660f4ae 100644 --- a/packages/linkml/src/linkml/utils/generator.py +++ b/packages/linkml/src/linkml/utils/generator.py @@ -36,7 +36,7 @@ from linkml.cli.logging import DEFAULT_LOG_LEVEL_INT, log_level_option from linkml.utils.deprecation import METADATA_FLAG, deprecation_warning from linkml.utils.mergeutils import alias_root -from linkml.utils.schemaloader import SchemaLoader +from linkml.utils.schemaloader import UNKNOWN_NAMESPACE, SchemaLoader from linkml.utils.typereferences import References from linkml_runtime import SchemaView from linkml_runtime.linkml_model.meta import ( @@ -47,6 +47,7 @@ ElementName, EnumDefinition, EnumDefinitionName, + Prefix, PrefixPrefixPrefix, SchemaDefinition, SlotDefinition, @@ -175,6 +176,8 @@ class Generator(metaclass=abc.ABCMeta): it is preserved here so external callers/subclasses relying on it are not silently broken by the switch to a property-backed field. """ + _loaded_schemaview: ClassVar[SchemaView | None] = None + """SchemaView built by :meth:`type_schemaview` for a SchemaLoader-based generator.""" @property def namespaces(self) -> Namespaces | None: @@ -214,6 +217,25 @@ def namespaces(self, value: Namespaces | None) -> None: :attr:`namespaces` property.""" self._namespaces = value + def type_schemaview(self) -> SchemaView: + """Return a SchemaView of the schema for questions about its types, on either generator path. + + SchemaLoader-based generators have no :attr:`schemaview`. Their loaded schema + already has its imports resolved, so the view is built once from a copy of it + with the loaded prefixes, without reopening source files. Placeholder namespaces + that SchemaLoader gives undeclared prefixes are left out. + """ + if self.schemaview is not None: + return self.schemaview + if self._loaded_schemaview is None: + schema = deepcopy(self.schema) + schema.imports = [] + for prefix, namespace in self.namespaces.items(): + if not prefix.startswith("@") and not str(namespace).startswith(UNKNOWN_NAMESPACE): + schema.prefixes[prefix] = Prefix(prefix, str(namespace)) + self._loaded_schemaview = SchemaView(schema) + return self._loaded_schemaview + namespaces: Namespaces | None = namespaces """Constructor kwarg backing the ``namespaces`` property (see above). @@ -985,7 +1007,7 @@ def add_prefix(self, ncname: str) -> None: """ if ncname not in self.namespaces: self.logger.warning(f"Unrecognized prefix: {ncname}") - self.namespaces[ncname] = f"http://example.org/UNKNOWN/{ncname}/" + self.namespaces[ncname] = f"{UNKNOWN_NAMESPACE}{ncname}/" self.emit_prefixes.add(ncname) def get_metamodel_slot_name(self, slot_name: str) -> str: diff --git a/packages/linkml/src/linkml/utils/schemaloader.py b/packages/linkml/src/linkml/utils/schemaloader.py index 16e9b208b2..68284fb6fe 100644 --- a/packages/linkml/src/linkml/utils/schemaloader.py +++ b/packages/linkml/src/linkml/utils/schemaloader.py @@ -32,6 +32,9 @@ from linkml_runtime.utils.namespaces import Namespaces from linkml_runtime.utils.yamlutils import TypedNode +# Namespace given to a prefix that is used but not declared; not a declaration of the schema. +UNKNOWN_NAMESPACE = "http://example.org/UNKNOWN/" + lgr = logging.getLogger(__name__) @@ -1067,7 +1070,7 @@ def check_prefix(self, prefix_or_curie_or_uri: str) -> None: if prefix: if prefix not in self.namespaces: self.logger.warning(f"{TypedNode.yaml_loc(prefix_or_curie_or_uri)}Unrecognized prefix: {prefix}") - self.namespaces[prefix] = f"http://example.org/UNKNOWN/{prefix}/" + self.namespaces[prefix] = f"{UNKNOWN_NAMESPACE}{prefix}/" else: case_adjusted_prefix = self.namespaces.prefix_for(prefix_or_curie_or_uri, case_shift=True) if case_adjusted_prefix != prefix: diff --git a/packages/linkml_runtime/src/linkml_runtime/dumpers/rdflib_dumper.py b/packages/linkml_runtime/src/linkml_runtime/dumpers/rdflib_dumper.py index d7a133f355..b426ff16f4 100644 --- a/packages/linkml_runtime/src/linkml_runtime/dumpers/rdflib_dumper.py +++ b/packages/linkml_runtime/src/linkml_runtime/dumpers/rdflib_dumper.py @@ -4,7 +4,7 @@ from curies import Converter from pydantic import BaseModel -from rdflib import XSD, Graph, URIRef +from rdflib import Graph, URIRef from rdflib.collection import Collection from rdflib.namespace import RDF from rdflib.term import BNode, Literal, Node @@ -12,6 +12,7 @@ from linkml_runtime.dumpers.dumper_root import Dumper from linkml_runtime.linkml_model import ElementName, PermissibleValue, SlotDefinition from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph +from linkml_runtime.utils.rdf_terms import value_term from linkml_runtime.utils.schemaview import SchemaView from linkml_runtime.utils.yamlutils import YAMLRoot @@ -102,20 +103,10 @@ def inject_triples( else: return Literal(element.text) if target_type in schemaview.all_types(): - t = schemaview.get_type(target_type) - dt_uri = t.uri - if dt_uri: - if dt_uri in ("rdfs:Resource", "xsd:anyURI"): - return URIRef(schemaview.expand_curie(element)) - elif dt_uri == "xsd:string": - return Literal(element) - else: - if "xsd" not in namespaces: - namespaces["xsd"] = XSD - return Literal(element, datatype=namespaces.uri_for(dt_uri)) - else: - logger.warning(f"No datatype specified for : {t.name}, using plain Literal") + if schemaview.induced_type(target_type).uri is None: + logger.warning(f"No datatype specified for : {target_type}, using plain Literal") return Literal(element) + return value_term(schemaview, target_type, element) element_vars = {k: v for k, v in vars(element).items() if not k.startswith("_")} if len(element_vars) == 0: id_slot = schemaview.get_identifier_slot(target_type) diff --git a/packages/linkml_runtime/src/linkml_runtime/loaders/rdflib_loader.py b/packages/linkml_runtime/src/linkml_runtime/loaders/rdflib_loader.py index 85c20605cc..dc3606d4cf 100644 --- a/packages/linkml_runtime/src/linkml_runtime/loaders/rdflib_loader.py +++ b/packages/linkml_runtime/src/linkml_runtime/loaders/rdflib_loader.py @@ -22,6 +22,7 @@ ) from linkml_runtime.loaders.loader_root import Loader from linkml_runtime.utils.formatutils import underscore +from linkml_runtime.utils.rdf_terms import TermKind, link_root, type_term from linkml_runtime.utils.schemaview import SchemaView from linkml_runtime.utils.yamlutils import YAMLRoot @@ -268,6 +269,8 @@ def _object_to_value( ) return o.value elif isinstance(o, BNode): + if self._range_term_kind(slot, schemaview) in (TermKind.BLANK_NODE, TermKind.BLANK_NODE_OR_IRI): + return f"_:{o}" if not is_inlined: logger.error(f"blank nodes should be inlined; {slot_name}={o} in {subject}") return Pointer(o) @@ -296,8 +299,11 @@ def _object_to_value( v = pv.text break elif TypeDefinition.class_name in range_applicable_elements: - if cast_literals: - v = namespaces.curie_for(o) + range_kind = self._range_term_kind(slot, schemaview) + if cast_literals or (range_kind is not None and range_kind.is_node): + # A uri value is a complete URI; values of other types may be written as CURIEs. + keep_iri = range_kind is not None and link_root(schemaview, slot.range) == "uri" + v = None if keep_iri else namespaces.curie_for(o) if v is None: v = str(o) logger.debug(f"Casting {o} to string") @@ -324,6 +330,13 @@ def _get_id_dict(self, node: VALID_SUBJECT, schemaview: SchemaView, cn: ClassDef raise Exception(f"Unexpected blank node {node}, type {cn} expects {id_slot.name} identifier") return {} + @staticmethod + def _range_term_kind(slot: SlotDefinition, schemaview: SchemaView) -> TermKind | None: + """Return the RDF term kind of the slot's values if its range is a type, else ``None``.""" + if slot.range in schemaview.all_types(): + return type_term(schemaview, slot.range).kind + return None + def _uri_to_id(self, node: VALID_SUBJECT, id_slot: SlotDefinition, schemaview: SchemaView) -> str: if schemaview.is_slot_percent_encoded(id_slot): return urllib.parse.unquote(node).replace(schemaview.namespaces()._base, "") diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py new file mode 100644 index 0000000000..73e8411234 --- /dev/null +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py @@ -0,0 +1,207 @@ +"""The RDF term that a value of a LinkML type becomes. + +A slot whose range is a type holds IRIs, blank nodes or literals, never a mix: +RDF keeps the three kinds of term distinct (RDF 1.1 Concepts §3.1), and OWL 2 DL +types every property as either an object or a data property (OWL 2 Structural +Specification §5.8.1). One rule decides the kind for the RDF generators and for +the RDF dumper and loader: + +1. A type whose datatype is a node kind has nodes of that kind. The node kinds + are those of ShEx (``shex:iri``, ``shex:bnode``, ``shex:nonliteral``; + ShEx 2.1 §5.4.2), which ``objectidentifier`` and ``nodeidentifier`` use, and + ``rdfs:Resource``, which marks a type whose values are nodes. +2. A type whose datatype is ``xsd:anyURI`` and that is ``uri`` or ``uriorcurie`` + from ``linkml:types``, or is derived from one of them, has IRIs. +3. Every other type has literals of its datatype. That includes a type derived + from ``string`` with ``uri: xsd:anyURI``: a URI reference held as text, which + may be relative (XSD 1.1 Part 2 §3.3.17), whereas an RDF IRI must be absolute + (RDF 1.1 Concepts §3.2). + +>>> from linkml_runtime.utils.schemaview import SchemaView +>>> sv = SchemaView(''' +... id: https://example.org/t +... name: t +... prefixes: +... linkml: https://w3id.org/linkml/ +... ex: https://example.org/t/ +... default_prefix: ex +... imports: [linkml:types] +... types: +... ImageLink: +... typeof: uri +... FilePath: +... typeof: string +... uri: xsd:anyURI +... ''') +>>> type_term(sv, "uri") +TypeTerm(kind=, datatype=None) +>>> type_term(sv, "ImageLink").kind + +>>> type_term(sv, "FilePath") +TypeTerm(kind=, datatype='http://www.w3.org/2001/XMLSchema#anyURI') +>>> type_term(sv, "nodeidentifier").kind + +>>> value_term(sv, "ImageLink", "ex:me.jpg") +rdflib.term.URIRef('https://example.org/t/me.jpg') +>>> value_term(sv, "FilePath", "./docs/cv.pdf") +rdflib.term.Literal('./docs/cv.pdf', datatype=rdflib.term.URIRef('http://www.w3.org/2001/XMLSchema#anyURI')) +""" + +import re +from dataclasses import dataclass +from enum import Enum +from functools import lru_cache +from typing import Any + +from rdflib import RDFS, XSD, BNode, Literal, URIRef +from rdflib.term import Node + +from linkml_runtime.linkml_model.types import SHEX +from linkml_runtime.utils.schemaview import SchemaView + + +class TermKind(Enum): + """The kind of RDF term a value becomes (RDF 1.1 Concepts §3.1).""" + + IRI = "iri" + BLANK_NODE = "bnode" + BLANK_NODE_OR_IRI = "nonliteral" + LITERAL = "literal" + + @property + def is_node(self) -> bool: + """Whether values of this kind are nodes (IRIs or blank nodes) rather than literals.""" + return self is not TermKind.LITERAL + + +# Datatype IRIs that name a kind of node rather than a literal datatype. ShEx +# spells the non-literal kind ``nonliteral``; ``linkml:types`` declares +# ``nodeidentifier`` with ``shex:nonLiteral``, so both spellings are listed. +NODE_KINDS: dict[str, TermKind] = { + str(SHEX.iri): TermKind.IRI, + str(SHEX.bnode): TermKind.BLANK_NODE, + str(SHEX.nonliteral): TermKind.BLANK_NODE_OR_IRI, + str(SHEX.nonLiteral): TermKind.BLANK_NODE_OR_IRI, + str(RDFS.Resource): TermKind.BLANK_NODE_OR_IRI, +} + +# The types of ``linkml:types`` whose values are IRIs, with the Python base their +# definitions declare. A link root is matched by name and definition: the name +# alone would admit a user type that happens to be called ``uri``, and +# ``from_schema`` cannot be used because merging imports rewrites it. +LINK_ROOTS: dict[str, str] = {"uri": "URI", "uriorcurie": "URIorCURIE"} + +# The vocabularies the built-in datatypes and the node kinds above come from. +# A schema may use their usual prefixes without declaring them; a declaration +# in the schema takes precedence. +VOCABULARY_PREFIXES: dict[str, str] = {"xsd": str(XSD), "shex": str(SHEX), "rdfs": str(RDFS)} + +# RFC 3986 §3.1: an absolute URI starts with a scheme followed by ":". +_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") + + +@dataclass(frozen=True) +class TypeTerm: + """The RDF term kind of a type's values and, for literals, their datatype IRI.""" + + kind: TermKind + datatype: str | None = None + + +def _expand(sv: SchemaView, uri: str, type_name: str) -> str: + """Expand the datatype *uri* of *type_name*, a CURIE or an IRI, with the schema's prefixes. + + A declared prefix is expanded even when the local part contains a colon, which + ``SchemaView.expand_curie`` leaves alone. An IRI is recognised by ``://``; one of + another form, such as ``urn:``, needs its scheme declared as a prefix. + + :raises ValueError: if the prefix is neither declared nor one of :data:`VOCABULARY_PREFIXES` + """ + if "://" in uri: + return uri + prefix, _, local = uri.partition(":") + namespaces = sv.namespaces() + if prefix in namespaces: + return str(namespaces[prefix]) + local + if prefix in VOCABULARY_PREFIXES: + return VOCABULARY_PREFIXES[prefix] + local + raise ValueError(f"Type {type_name!r} has datatype {uri!r}, but the schema does not declare prefix {prefix!r}") + + +def expanded_datatype(sv: SchemaView, type_name: str) -> str | None: + """Return the expanded IRI of the datatype that *type_name* declares or inherits. + + :raises ValueError: if the datatype is a CURIE whose prefix the schema does not declare + """ + uri = sv.induced_type(type_name).uri + return _expand(sv, uri, type_name) if uri else None + + +@lru_cache(maxsize=4096) +def link_root(sv: SchemaView, type_name: str) -> str | None: + """Return the link root of ``linkml:types`` that *type_name* is or derives from, else ``None``. + + The link roots are ``uri``, whose values are complete URIs, and ``uriorcurie``, + whose values may be written as CURIEs. A root is matched by name and definition + (:data:`LINK_ROOTS`), through the ``typeof`` chain. + """ + for ancestor_name in sv.type_ancestors(type_name, reflexive=True): + base = LINK_ROOTS.get(ancestor_name) + if base is None: + continue + ancestor = sv.get_type(ancestor_name) + if ( + ancestor.typeof is None + and ancestor.base == base + and ancestor.uri is not None + and _expand(sv, ancestor.uri, ancestor_name) == str(XSD.anyURI) + ): + return ancestor_name + return None + + +@lru_cache(maxsize=4096) +def type_term(sv: SchemaView, type_name: str) -> TypeTerm: + """Return the RDF term kind of the values of the type *type_name* (see the module rules). + + :raises ValueError: if the type's datatype uses an undeclared prefix + """ + datatype = expanded_datatype(sv, type_name) + if datatype in NODE_KINDS: + return TypeTerm(NODE_KINDS[datatype]) + if datatype == str(XSD.anyURI) and link_root(sv, type_name) is not None: + return TypeTerm(TermKind.IRI) + return TypeTerm(TermKind.LITERAL, datatype) + + +def value_term(sv: SchemaView, type_name: str, value: Any) -> Node: + """Return the RDF term for *value* of the type *type_name*. + + IRIs are expanded from CURIEs. A value of a node type may name a blank node + (``_:b1``) when its kind allows blank nodes. A literal keeps the Python value, + so rdflib writes its canonical lexical form (``True`` becomes ``"true"``); + ``xsd:string`` values become plain literals, which RDF 1.1 gives that datatype. + + :raises ValueError: if a value of an IRI type is a relative reference, which + cannot be an RDF IRI (RDF 1.1 Concepts §3.2) + """ + term = type_term(sv, type_name) + if not term.kind.is_node: + if term.datatype is None or term.datatype == str(XSD.string): + return Literal(value) + return Literal(value, datatype=URIRef(term.datatype)) + text = str(value) + if text.startswith("_:") and term.kind in (TermKind.BLANK_NODE, TermKind.BLANK_NODE_OR_IRI): + return BNode(text[2:]) + if term.kind is TermKind.BLANK_NODE: + raise ValueError(f"Value {text!r} of type {type_name!r} must be a blank node label such as '_:b1'") + if text.startswith("_:"): + raise ValueError(f"Value {text!r} of type {type_name!r} is a blank node, but values of this type are IRIs") + iri = sv.expand_curie(text) + if not _SCHEME.match(iri): + raise ValueError( + f"Value {text!r} of type {type_name!r} is a relative reference, but values of this type are " + "IRIs, which must be absolute. Use a type derived from string with uri: xsd:anyURI to keep " + "URI references as text." + ) + return URIRef(iri) diff --git a/tests/linkml/test_base/__snapshots__/annotations.context.jsonld b/tests/linkml/test_base/__snapshots__/annotations.context.jsonld index cb67923cc6..84d8f7877d 100644 --- a/tests/linkml/test_base/__snapshots__/annotations.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/annotations.context.jsonld @@ -8,7 +8,7 @@ "@id": "annotations" }, "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "tag" }, "value": { diff --git a/tests/linkml/test_base/__snapshots__/annotations.json b/tests/linkml/test_base/__snapshots__/annotations.json index bc51cec7c6..bffa4074a3 100644 --- a/tests/linkml/test_base/__snapshots__/annotations.json +++ b/tests/linkml/test_base/__snapshots__/annotations.json @@ -503,7 +503,7 @@ "@id": "annotations" }, "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "tag" }, "value": { diff --git a/tests/linkml/test_base/__snapshots__/extensions.context.jsonld b/tests/linkml/test_base/__snapshots__/extensions.context.jsonld index cebb3ee890..730a52a220 100644 --- a/tests/linkml/test_base/__snapshots__/extensions.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/extensions.context.jsonld @@ -4,7 +4,7 @@ "linkml": "https://w3id.org/linkml/", "@vocab": "https://w3id.org/linkml/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "tag" }, "value": { diff --git a/tests/linkml/test_base/__snapshots__/extensions.json b/tests/linkml/test_base/__snapshots__/extensions.json index f9604e0ea6..6f7d53ac3a 100644 --- a/tests/linkml/test_base/__snapshots__/extensions.json +++ b/tests/linkml/test_base/__snapshots__/extensions.json @@ -442,7 +442,7 @@ "linkml": "https://w3id.org/linkml/", "@vocab": "https://w3id.org/linkml/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "tag" }, "value": { diff --git a/tests/linkml/test_base/__snapshots__/mappings.context.jsonld b/tests/linkml/test_base/__snapshots__/mappings.context.jsonld index b209ca8a52..e14c87bb34 100644 --- a/tests/linkml/test_base/__snapshots__/mappings.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/mappings.context.jsonld @@ -12,35 +12,35 @@ "skos": "http://www.w3.org/2004/02/skos/core#", "@vocab": "https://w3id.org/linkml/", "broad_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:broadMatch" }, "close_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:closeMatch" }, "deprecated_element_has_exact_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_exact_replacement" }, "deprecated_element_has_possible_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_possible_replacement" }, "exact_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:exactMatch" }, "mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:mappingRelation" }, "narrow_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:narrowMatch" }, "related_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:relatedMatch" }, "@base": "https://w3id.org/linkml/" diff --git a/tests/linkml/test_base/__snapshots__/mappings.json b/tests/linkml/test_base/__snapshots__/mappings.json index babcbe09bd..02354810c1 100644 --- a/tests/linkml/test_base/__snapshots__/mappings.json +++ b/tests/linkml/test_base/__snapshots__/mappings.json @@ -486,35 +486,35 @@ "skos": "http://www.w3.org/2004/02/skos/core#", "@vocab": "https://w3id.org/linkml/", "broad_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:broadMatch" }, "close_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:closeMatch" }, "deprecated_element_has_exact_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_exact_replacement" }, "deprecated_element_has_possible_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_possible_replacement" }, "exact_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:exactMatch" }, "mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:mappingRelation" }, "narrow_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:narrowMatch" }, "related_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:relatedMatch" } }, diff --git a/tests/linkml/test_base/__snapshots__/meta.context.jsonld b/tests/linkml/test_base/__snapshots__/meta.context.jsonld index 9a33f0f10b..d51c9c8bf3 100644 --- a/tests/linkml/test_base/__snapshots__/meta.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/meta.context.jsonld @@ -41,7 +41,7 @@ "@id": "skos:prefLabel" }, "contexts": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "contexts" }, "predicate": { @@ -68,7 +68,7 @@ "@id": "allowed" }, "source": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:source" }, "description": { @@ -121,11 +121,11 @@ "@id": "boolean_slot" }, "broad_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:broadMatch" }, "categories": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:subject" }, "children_are_mutually_disjoint": { @@ -133,7 +133,7 @@ "@id": "children_are_mutually_disjoint" }, "class_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "class_uri" }, "classes": { @@ -145,11 +145,11 @@ "@id": "classification_rules" }, "close_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:closeMatch" }, "code_set": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "code_set" }, "code_set_tag": { @@ -162,7 +162,7 @@ "@id": "skos:note" }, "concepts": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "concepts" }, "conforms_to": { @@ -173,11 +173,11 @@ "@id": "consider_nulls_inequal" }, "contributors": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:contributor" }, "created_by": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "pav:createdBy" }, "created_on": { @@ -203,18 +203,18 @@ "@id": "defining_slots" }, "definition_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "definition_uri" }, "deprecated": { "@id": "deprecated" }, "deprecated_element_has_exact_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_exact_replacement" }, "deprecated_element_has_possible_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_possible_replacement" }, "derivation": { @@ -255,7 +255,7 @@ "@id": "enum_range" }, "enum_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "enum_uri" }, "enums": { @@ -280,7 +280,7 @@ "@id": "equals_string_in" }, "exact_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:exactMatch" }, "exact_cardinality": { @@ -300,7 +300,7 @@ "@id": "examples" }, "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "tag" }, "value": { @@ -322,7 +322,7 @@ "@id": "framework" }, "from_schema": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:inScheme" }, "generation_date": { @@ -334,11 +334,11 @@ "@id": "has_member" }, "has_quantity_kind": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "qudt:hasQuantityKind" }, "id": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "id" }, "id_prefixes": { @@ -362,7 +362,7 @@ "@id": "ifabsent" }, "implements": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "implements" }, "implicit_prefix": { @@ -372,7 +372,7 @@ "@id": "import_as" }, "import_from": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "import_from" }, "import_map": { @@ -383,7 +383,7 @@ "@id": "imported_from" }, "imports": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "imports" }, "in_language": { @@ -426,7 +426,7 @@ "@id": "inlined_as_simple_dict" }, "instantiates": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "instantiates" }, "interpolated": { @@ -508,7 +508,7 @@ "@id": "type" }, "mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:mappingRelation" }, "matches": { @@ -527,7 +527,7 @@ "@id": "maximum_value" }, "meaning": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "meaning" }, "metamodel_version": { @@ -558,7 +558,7 @@ "@id": "mixins" }, "modified_by": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "oslc:modifiedBy" }, "multivalued": { @@ -567,7 +567,7 @@ }, "name": "@id", "narrow_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:narrowMatch" }, "none_of": { @@ -590,7 +590,7 @@ "@id": "open_world" }, "owned_by": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "owned_by" }, "owner": { @@ -624,7 +624,7 @@ "@id": "sh:prefix" }, "prefix_reference": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "sh:namespace" }, "prefixes": { @@ -632,7 +632,7 @@ "@id": "sh:declare" }, "publisher": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:publisher" }, "pv_formula": { @@ -675,7 +675,7 @@ "@id": "reflexive_transitive_form_of" }, "related_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:relatedMatch" }, "relational_logical_characteristic": { @@ -691,7 +691,7 @@ "@id": "relational_role" }, "relationship_types": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "relationship_types" }, "repr": { @@ -717,7 +717,7 @@ "@id": "sh:rule" }, "see_also": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "rdfs:seeAlso" }, "setting_key": { @@ -754,7 +754,7 @@ "@id": "slot_names_unique" }, "slot_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "slot_uri" }, "slot_usage": { @@ -773,11 +773,11 @@ "@id": "source_file_size" }, "source_nodes": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "source_nodes" }, "source_ontology": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "source_ontology" }, "status": { @@ -799,7 +799,7 @@ "@id": "structured_pattern" }, "subclass_of": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "subclass_of" }, "subproperty_of": { @@ -852,7 +852,7 @@ "@id": "type_mappings" }, "uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "uri" }, "typeof": { @@ -904,7 +904,7 @@ "@id": "value_specification_constant" }, "values_from": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "values_from" }, "version": { diff --git a/tests/linkml/test_base/__snapshots__/meta.json b/tests/linkml/test_base/__snapshots__/meta.json index acda6769d8..98b6fe7d0d 100644 --- a/tests/linkml/test_base/__snapshots__/meta.json +++ b/tests/linkml/test_base/__snapshots__/meta.json @@ -8951,7 +8951,7 @@ "@id": "skos:prefLabel" }, "contexts": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "contexts" }, "predicate": { @@ -8978,7 +8978,7 @@ "@id": "allowed" }, "source": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:source" }, "description": { @@ -9031,11 +9031,11 @@ "@id": "boolean_slot" }, "broad_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:broadMatch" }, "categories": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:subject" }, "children_are_mutually_disjoint": { @@ -9043,7 +9043,7 @@ "@id": "children_are_mutually_disjoint" }, "class_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "class_uri" }, "classes": { @@ -9055,11 +9055,11 @@ "@id": "classification_rules" }, "close_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:closeMatch" }, "code_set": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "code_set" }, "code_set_tag": { @@ -9072,7 +9072,7 @@ "@id": "skos:note" }, "concepts": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "concepts" }, "conforms_to": { @@ -9083,11 +9083,11 @@ "@id": "consider_nulls_inequal" }, "contributors": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:contributor" }, "created_by": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "pav:createdBy" }, "created_on": { @@ -9113,18 +9113,18 @@ "@id": "defining_slots" }, "definition_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "definition_uri" }, "deprecated": { "@id": "deprecated" }, "deprecated_element_has_exact_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_exact_replacement" }, "deprecated_element_has_possible_replacement": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "deprecated_element_has_possible_replacement" }, "derivation": { @@ -9165,7 +9165,7 @@ "@id": "enum_range" }, "enum_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "enum_uri" }, "enums": { @@ -9190,7 +9190,7 @@ "@id": "equals_string_in" }, "exact_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:exactMatch" }, "exact_cardinality": { @@ -9210,7 +9210,7 @@ "@id": "examples" }, "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "tag" }, "value": { @@ -9232,7 +9232,7 @@ "@id": "framework" }, "from_schema": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:inScheme" }, "generation_date": { @@ -9244,11 +9244,11 @@ "@id": "has_member" }, "has_quantity_kind": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "qudt:hasQuantityKind" }, "id": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "id" }, "id_prefixes": { @@ -9272,7 +9272,7 @@ "@id": "ifabsent" }, "implements": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "implements" }, "implicit_prefix": { @@ -9282,7 +9282,7 @@ "@id": "import_as" }, "import_from": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "import_from" }, "import_map": { @@ -9293,7 +9293,7 @@ "@id": "imported_from" }, "imports": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "imports" }, "in_language": { @@ -9336,7 +9336,7 @@ "@id": "inlined_as_simple_dict" }, "instantiates": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "instantiates" }, "interpolated": { @@ -9418,7 +9418,7 @@ "@id": "type" }, "mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:mappingRelation" }, "matches": { @@ -9437,7 +9437,7 @@ "@id": "maximum_value" }, "meaning": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "meaning" }, "metamodel_version": { @@ -9468,7 +9468,7 @@ "@id": "mixins" }, "modified_by": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "oslc:modifiedBy" }, "multivalued": { @@ -9477,7 +9477,7 @@ }, "name": "@id", "narrow_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:narrowMatch" }, "none_of": { @@ -9500,7 +9500,7 @@ "@id": "open_world" }, "owned_by": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "owned_by" }, "owner": { @@ -9534,7 +9534,7 @@ "@id": "sh:prefix" }, "prefix_reference": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "sh:namespace" }, "prefixes": { @@ -9542,7 +9542,7 @@ "@id": "sh:declare" }, "publisher": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "dcterms:publisher" }, "pv_formula": { @@ -9585,7 +9585,7 @@ "@id": "reflexive_transitive_form_of" }, "related_mappings": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "skos:relatedMatch" }, "relational_logical_characteristic": { @@ -9601,7 +9601,7 @@ "@id": "relational_role" }, "relationship_types": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "relationship_types" }, "repr": { @@ -9627,7 +9627,7 @@ "@id": "sh:rule" }, "see_also": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "rdfs:seeAlso" }, "setting_key": { @@ -9664,7 +9664,7 @@ "@id": "slot_names_unique" }, "slot_uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "slot_uri" }, "slot_usage": { @@ -9683,11 +9683,11 @@ "@id": "source_file_size" }, "source_nodes": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "source_nodes" }, "source_ontology": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "source_ontology" }, "status": { @@ -9709,7 +9709,7 @@ "@id": "structured_pattern" }, "subclass_of": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "subclass_of" }, "subproperty_of": { @@ -9762,7 +9762,7 @@ "@id": "type_mappings" }, "uri": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "uri" }, "typeof": { @@ -9814,7 +9814,7 @@ "@id": "value_specification_constant" }, "values_from": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "values_from" }, "version": { diff --git a/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld b/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld index 509e297b35..16621443df 100644 --- a/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld +++ b/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld @@ -639,7 +639,7 @@ "@id": "affects_response_to" }, "affiliation": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "affiliation" }, "agent_type": { @@ -783,7 +783,7 @@ "@id": "catalyzes" }, "category": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "category" }, "causal_mechanism_qualifier": { @@ -1737,7 +1737,7 @@ "@id": "mentions" }, "mesh_terms": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "mesh_terms" }, "missing_from": { @@ -1841,7 +1841,7 @@ "@id": "object_part_qualifier" }, "object_specialization_qualifier": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "object_specialization_qualifier" }, "occurs_in": { @@ -1868,7 +1868,7 @@ "@id": "original_object" }, "original_predicate": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "original_predicate" }, "original_subject": { @@ -1993,7 +1993,7 @@ "@id": "publications" }, "published_in": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "published_in" }, "publisher": { @@ -2082,7 +2082,7 @@ "@id": "resistance_associated_with" }, "resource_id": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "resource_id" }, "resource_role": { @@ -2260,7 +2260,7 @@ "@id": "subject_part_qualifier" }, "subject_specialization_qualifier": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "subject_specialization_qualifier" }, "summary": { @@ -2287,7 +2287,7 @@ "@id": "supporting_document_year" }, "supporting_documents": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "supporting_documents" }, "supporting_study_cohort": { @@ -2303,11 +2303,11 @@ "@id": "supporting_study_metadata" }, "supporting_study_method_description": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "supporting_study_method_description" }, "supporting_study_method_type": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "supporting_study_method_type" }, "supporting_study_size": { @@ -2412,7 +2412,7 @@ "@id": "update_date" }, "upstream_resource_ids": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "upstream_resource_ids" }, "url": { @@ -2437,7 +2437,7 @@ "@id": "xenologous_to" }, "xref": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "xref" }, "AccessibleDnaRegion": { diff --git a/tests/linkml/test_biolink_model/__snapshots__/biolink.json b/tests/linkml/test_biolink_model/__snapshots__/biolink.json index 3080ce2608..02bc375a8f 100644 --- a/tests/linkml/test_biolink_model/__snapshots__/biolink.json +++ b/tests/linkml/test_biolink_model/__snapshots__/biolink.json @@ -36634,7 +36634,7 @@ "@id": "affects_response_to" }, "affiliation": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "affiliation" }, "agent_type": { @@ -36778,7 +36778,7 @@ "@id": "catalyzes" }, "category": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "category" }, "causal_mechanism_qualifier": { @@ -37732,7 +37732,7 @@ "@id": "mentions" }, "mesh_terms": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "mesh_terms" }, "missing_from": { @@ -37836,7 +37836,7 @@ "@id": "object_part_qualifier" }, "object_specialization_qualifier": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "object_specialization_qualifier" }, "occurs_in": { @@ -37863,7 +37863,7 @@ "@id": "original_object" }, "original_predicate": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "original_predicate" }, "original_subject": { @@ -37988,7 +37988,7 @@ "@id": "publications" }, "published_in": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "published_in" }, "publisher": { @@ -38077,7 +38077,7 @@ "@id": "resistance_associated_with" }, "resource_id": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "resource_id" }, "resource_role": { @@ -38255,7 +38255,7 @@ "@id": "subject_part_qualifier" }, "subject_specialization_qualifier": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "subject_specialization_qualifier" }, "summary": { @@ -38282,7 +38282,7 @@ "@id": "supporting_document_year" }, "supporting_documents": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "supporting_documents" }, "supporting_study_cohort": { @@ -38298,11 +38298,11 @@ "@id": "supporting_study_metadata" }, "supporting_study_method_description": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "supporting_study_method_description" }, "supporting_study_method_type": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "supporting_study_method_type" }, "supporting_study_size": { @@ -38407,7 +38407,7 @@ "@id": "update_date" }, "upstream_resource_ids": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "upstream_resource_ids" }, "url": { @@ -38432,7 +38432,7 @@ "@id": "xenologous_to" }, "xref": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "xref" }, "AccessibleDnaRegion": { diff --git a/tests/linkml/test_generators/test_jsonldcontextgen.py b/tests/linkml/test_generators/test_jsonldcontextgen.py index 72089eba44..6c56ab3fc5 100644 --- a/tests/linkml/test_generators/test_jsonldcontextgen.py +++ b/tests/linkml/test_generators/test_jsonldcontextgen.py @@ -4,11 +4,13 @@ import pytest from click.testing import CliRunner -from rdflib import Graph, Literal, URIRef +from rdflib import OWL, RDF, XSD, Graph, Literal, URIRef -from linkml.generators import ContextGenerator, JSONLDGenerator +from linkml.generators import ContextGenerator, JSONLDGenerator, OwlSchemaGenerator from linkml.generators.jsonldcontextgen import ContextGenerator as FrameContextGenerator from linkml.generators.jsonldcontextgen import cli as jsonld_context_cli +from linkml.generators.owlgen import cli as owl_cli +from linkml.utils.deprecation import EMITTED, XSD_ANYURI_AS_IRI_OPTION from tests.linkml.utils.compare_jsonld_context import CompareJsonldContext from tests.linkml.utils.validate_jsonld_context import RdfExpectations @@ -1019,376 +1021,132 @@ def test_exclude_external_imports_works_with_mergeimports_false(tmp_path): assert "ExternalCredential" not in ctx, "External class should be excluded with mergeimports=False" -def test_xsd_anyuri_as_iri_flag(): - """Test that --xsd-anyuri-as-iri maps uri ranges to @type: @id. - - By default, ``range: uri`` (type_uri ``xsd:anyURI``) produces - ``@type: xsd:anyURI`` (typed literal). With ``xsd_anyuri_as_iri=True``, - it produces ``@type: @id`` (IRI node reference), aligning the JSON-LD - context with the SHACL generator which already emits ``sh:nodeKind sh:IRI`` - for the same type. - - See: - - W3C SHACL §4.8.1 sh:nodeKind (https://www.w3.org/TR/shacl/#NodeKindConstraintComponent) - - JSON-LD 1.1 §4.2.2 Type Coercion (https://www.w3.org/TR/json-ld11/#type-coercion) - - RDF 1.1 §3.3 Literals vs §3.2 IRIs (https://www.w3.org/TR/rdf11-concepts/) - """ - schema_yaml = """ -id: https://example.org/test-uri-context -name: test_uri_context - -prefixes: - ex: https://example.org/ - linkml: https://w3id.org/linkml/ - -imports: - - linkml:types - -default_prefix: ex -default_range: string - -slots: - homepage: - range: uri - slot_uri: ex:homepage - node_ref: - range: nodeidentifier - slot_uri: ex:nodeRef - name: - range: string - slot_uri: ex:name - -classes: - Thing: - slots: - - homepage - - node_ref - - name -""" - # Default behaviour: uri → xsd:anyURI (backward compatible) - ctx_default = json.loads(ContextGenerator(schema_yaml).serialize())["@context"] - assert ctx_default["homepage"]["@type"] == "xsd:anyURI" - - # Opt-in: uri → @id (aligned with SHACL sh:nodeKind sh:IRI) - ctx_iri = json.loads(ContextGenerator(schema_yaml, xsd_anyuri_as_iri=True).serialize())["@context"] - assert ctx_iri["homepage"]["@type"] == "@id", ( - f"Expected @type: @id for uri range with xsd_anyuri_as_iri=True, got {ctx_iri['homepage'].get('@type')}" - ) - - # nodeidentifier is unaffected by the flag (not xsd:anyURI-typed) - # Its default @type depends on URI_RANGES matching shex:nonLiteral; - # we only verify the flag doesn't change its behaviour. - assert ctx_default["node_ref"]["@type"] == ctx_iri["node_ref"]["@type"] - - # string → no @type regardless of flag - assert "@type" not in ctx_default.get("name", {}) - assert "@type" not in ctx_iri.get("name", {}) - - -def test_xsd_anyuri_as_iri_with_any_of(): - """The --xsd-anyuri-as-iri flag must also apply to ``any_of`` slots - whose type branches include ``uri`` mixed with class ranges. - - ``_literal_coercion_for_ranges`` resolves mixed any_of type branches - and must use the extended URI_RANGES when the flag is active. - """ - schema_yaml = """ -id: https://example.org/test-anyof-uri -name: test_anyof_uri - -prefixes: - ex: https://example.org/ - linkml: https://w3id.org/linkml/ - -imports: - - linkml:types - -default_prefix: ex -default_range: string - -classes: - Container: - slots: - - mixed_slot - Target: - class_uri: ex:Target - -slots: - mixed_slot: - slot_uri: ex:mixed - any_of: - - range: Target - - range: uri -""" - # Default: mixed class+uri any_of — uri resolves to xsd:anyURI literal, - # which disagrees with @id from the class branch → no coercion emitted - ctx_default = json.loads(ContextGenerator(schema_yaml).serialize())["@context"] - default_type = ctx_default.get("mixed_slot", {}).get("@type") - assert default_type != "@id", f"Without flag, mixed any_of should not resolve to @id, got {default_type}" - - # With flag: uri branch now also resolves to @id, matching the class branch - # → all branches agree → @id is emitted - ctx_iri = json.loads(ContextGenerator(schema_yaml, xsd_anyuri_as_iri=True).serialize())["@context"] - assert ctx_iri["mixed_slot"]["@type"] == "@id", ( - f"Expected @id for mixed any_of with flag, got {ctx_iri.get('mixed_slot', {}).get('@type')}" - ) - - -def test_xsd_anyuri_as_iri_owl(): - """OWL generator must produce owl:ObjectProperty for uri ranges when flag is set. - - Without the flag, ``range: uri`` produces ``owl:DatatypeProperty`` with - ``rdfs:range xsd:anyURI``. With ``xsd_anyuri_as_iri=True``, it should - produce ``owl:ObjectProperty`` (no rdfs:range restriction), aligning - with the SHACL generator's ``sh:nodeKind sh:IRI``. - """ - from rdflib import OWL, RDF, URIRef - - from linkml.generators.owlgen import OwlSchemaGenerator - - schema_yaml = """ -id: https://example.org/test-owl-uri -name: test_owl_uri -prefixes: - ex: https://example.org/ - linkml: https://w3id.org/linkml/ -imports: - - linkml:types -default_prefix: ex -default_range: string -slots: - homepage: - range: uri - slot_uri: ex:homepage - name: - range: string - slot_uri: ex:name -classes: - Thing: - slots: - - homepage - - name -""" - # Default: uri → DatatypeProperty (must disable type_objects which - # unconditionally returns ObjectProperty for all type-ranged slots) - gen_default = OwlSchemaGenerator(schema_yaml, type_objects=False) - g_default = gen_default.as_graph() - homepage_uri = URIRef("https://example.org/homepage") - default_rdf_type = set(g_default.objects(homepage_uri, RDF.type)) - assert OWL.DatatypeProperty in default_rdf_type, ( - f"Without flag, homepage should be DatatypeProperty, got {default_rdf_type}" - ) - - # With flag: uri → ObjectProperty - gen_iri = OwlSchemaGenerator(schema_yaml, xsd_anyuri_as_iri=True, type_objects=False) - g_iri = gen_iri.as_graph() - iri_rdf_type = set(g_iri.objects(homepage_uri, RDF.type)) - assert OWL.ObjectProperty in iri_rdf_type, f"With flag, homepage should be ObjectProperty, got {iri_rdf_type}" - assert OWL.DatatypeProperty not in iri_rdf_type, ( - f"With flag, homepage should NOT be DatatypeProperty, got {iri_rdf_type}" - ) - - # String slot must remain DatatypeProperty regardless of flag - name_uri = URIRef("https://example.org/name") - name_rdf_type = set(g_iri.objects(name_uri, RDF.type)) - assert OWL.DatatypeProperty in name_rdf_type, f"String slot should remain DatatypeProperty, got {name_rdf_type}" - - -def test_xsd_anyuri_as_iri_uriorcurie_range(): - """``uriorcurie`` also maps to ``xsd:anyURI`` and must behave identically - to ``uri`` when the ``--xsd-anyuri-as-iri`` flag is active. - - This is a high-priority coverage gap: ``uriorcurie`` is distinct from - ``uri`` at the LinkML level but shares the same XSD type. - """ - schema_yaml = """ -id: https://example.org/test-uriorcurie -name: test_uriorcurie - +URI_TERMS_SCHEMA = """ +id: https://example.org/uri-terms +name: uri_terms prefixes: ex: https://example.org/ linkml: https://w3id.org/linkml/ - imports: - linkml:types - default_prefix: ex default_range: string - +types: + ImageLink: + typeof: uri + FilePath: + typeof: string + uri: xsd:anyURI slots: - reference: - range: uriorcurie - slot_uri: ex:reference - homepage: - range: uri - slot_uri: ex:homepage - + homepage: {range: uri} + related: {range: uriorcurie} + image: {range: ImageLink} + file_path: {range: FilePath} + node: {range: objectidentifier} + any_node: {range: nodeidentifier} + code: {range: curie} + name: {} + link_or_thing: + any_of: [{range: uri}, {range: Thing}] classes: Thing: - slots: - - reference - - homepage + attributes: + id: {identifier: true} + slots: [homepage, related, image, file_path, node, any_node, code, name, link_or_thing] + Special: + slots: [name] + slot_usage: + name: {range: uri} """ - ctx_default = json.loads(ContextGenerator(schema_yaml).serialize())["@context"] - assert ctx_default["reference"]["@type"] == "xsd:anyURI" - assert ctx_default["homepage"]["@type"] == "xsd:anyURI" - ctx_iri = json.loads(ContextGenerator(schema_yaml, xsd_anyuri_as_iri=True).serialize())["@context"] - assert ctx_iri["reference"]["@type"] == "@id", "uriorcurie should map to @id with xsd_anyuri_as_iri=True" - assert ctx_iri["homepage"]["@type"] == "@id", "uri should map to @id with xsd_anyuri_as_iri=True" +@pytest.mark.parametrize( + "slot,coercion", + [ + ("homepage", "@id"), + ("related", "@id"), + ("image", "@id"), + ("file_path", "xsd:anyURI"), + ("node", "@id"), + ("any_node", "@id"), + ("code", None), + ("name", None), + ("link_or_thing", "@id"), + ], +) +def test_type_coercion_follows_term_kind(slot, coercion): + """Slots get the coercion of their values' RDF term kind (``linkml_runtime.utils.rdf_terms``). -def test_xsd_anyuri_as_iri_curie_range_unchanged(): - """``curie`` maps to ``xsd:string`` (not ``xsd:anyURI``), so the flag - must NOT affect its coercion. - - This documents the cross-type boundary: ``uri`` and ``uriorcurie`` - share ``xsd:anyURI``, but ``curie`` uses ``xsd:string``. + ``uri``, ``uriorcurie``, types derived from them and the node types are IRIs + (``@id``); a string-derived ``xsd:anyURI`` type keeps URI references as typed + literals; ``curie`` and ``string`` are plain strings. """ - schema_yaml = """ -id: https://example.org/test-curie -name: test_curie - -prefixes: - ex: https://example.org/ - linkml: https://w3id.org/linkml/ - -imports: - - linkml:types - -default_prefix: ex -default_range: string - -slots: - curie_slot: - range: curie - slot_uri: ex:curieSlot - uri_slot: - range: uri - slot_uri: ex:uriSlot - -classes: - Thing: - slots: - - curie_slot - - uri_slot -""" - ctx_default = json.loads(ContextGenerator(schema_yaml).serialize())["@context"] - ctx_iri = json.loads(ContextGenerator(schema_yaml, xsd_anyuri_as_iri=True).serialize())["@context"] + ctx = json.loads(ContextGenerator(URI_TERMS_SCHEMA).serialize())["@context"] + assert ctx[slot].get("@type") == coercion - # curie (xsd:string) must be unaffected by the flag - curie_default = ctx_default.get("curie_slot", {}).get("@type") - curie_iri = ctx_iri.get("curie_slot", {}).get("@type") - assert curie_default == curie_iri, f"curie coercion should not change with flag: {curie_default} vs {curie_iri}" - # uri (xsd:anyURI) must change — sanity check - assert ctx_iri["uri_slot"]["@type"] == "@id" +def test_type_schemaview_on_both_generator_paths(): + """A SchemaView-based generator answers type questions from its own view; a SchemaLoader-based one builds one.""" + owl = OwlSchemaGenerator(URI_TERMS_SCHEMA) + assert owl.type_schemaview() is owl.schemaview + assert ContextGenerator(URI_TERMS_SCHEMA).type_schemaview().induced_type("ImageLink").uri == "xsd:anyURI" -def test_xsd_anyuri_as_iri_owl_curie_unchanged(): - """OWL generator must keep ``range: curie`` as DatatypeProperty even with flag. +def test_class_scoped_range_override_follows_term_kind(): + """A ``slot_usage`` override to a link type is coerced to ``@id`` in the class-scoped context.""" + ctx = json.loads(ContextGenerator(URI_TERMS_SCHEMA).serialize())["@context"] + assert ctx["Special"]["@context"]["name"]["@type"] == "@id" - ``curie`` maps to ``xsd:string`` (not ``xsd:anyURI``), so the - ``--xsd-anyuri-as-iri`` flag must not promote it to ObjectProperty. - This verifies cross-generator consistency: the JSON-LD context generator - already correctly excludes ``curie`` via ``URI_RANGES_WITH_XSD``; the - OWL generator must match via ``is_xsd_anyuri_range()``. - """ - from rdflib import OWL, RDF, URIRef - - from linkml.generators.owlgen import OwlSchemaGenerator - - schema_yaml = """ -id: https://example.org/test-owl-curie -name: test_owl_curie -prefixes: - ex: https://example.org/ - linkml: https://w3id.org/linkml/ -imports: - - linkml:types -default_prefix: ex -default_range: string -slots: - compact_id: - range: curie - slot_uri: ex:compactId - homepage: - range: uri - slot_uri: ex:homepage -classes: - Thing: - slots: - - compact_id - - homepage -""" - compact_id_uri = URIRef("https://example.org/compact_id") - homepage_uri = URIRef("https://example.org/homepage") - - # With flag: curie must stay DatatypeProperty, uri must become ObjectProperty - gen = OwlSchemaGenerator(schema_yaml, xsd_anyuri_as_iri=True, type_objects=False) - g = gen.as_graph() - curie_types = set(g.objects(compact_id_uri, RDF.type)) - assert OWL.DatatypeProperty in curie_types, f"curie slot must remain DatatypeProperty with flag, got {curie_types}" - assert OWL.ObjectProperty not in curie_types, ( - f"curie slot must NOT become ObjectProperty with flag, got {curie_types}" +def test_links_are_iris_and_uri_references_stay_text(): + """Through the generated context, links become IRIs and a relative file path stays an ``xsd:anyURI`` literal.""" + ctx = json.loads(ContextGenerator(URI_TERMS_SCHEMA).serialize()) + doc = { + **ctx, + "id": "ex:t1", + "homepage": "https://alice.example/", + "related": "ex:other", + "file_path": "./docs/cv.pdf", + } + graph = Graph().parse(data=json.dumps(doc), format="json-ld", base="https://reader.example/downloads/doc.json") + subject = URIRef("https://example.org/t1") + assert graph.value(subject, URIRef("https://example.org/homepage")) == URIRef("https://alice.example/") + assert graph.value(subject, URIRef("https://example.org/related")) == URIRef("https://example.org/other") + assert graph.value(subject, URIRef("https://example.org/file_path")) == Literal( + "./docs/cv.pdf", datatype=XSD.anyURI ) - # Sanity: uri must become ObjectProperty - uri_types = set(g.objects(homepage_uri, RDF.type)) - assert OWL.ObjectProperty in uri_types, f"uri slot should be ObjectProperty with flag, got {uri_types}" +@pytest.mark.parametrize("value", [True, False]) +@pytest.mark.parametrize("generator", [ContextGenerator, OwlSchemaGenerator]) +def test_xsd_anyuri_as_iri_is_deprecated_without_effect(generator, value): + """Passing the former option reports the deprecation and changes nothing, in either polarity.""" + EMITTED.discard(XSD_ANYURI_AS_IRI_OPTION) + flagged = generator(URI_TERMS_SCHEMA, xsd_anyuri_as_iri=value).serialize() + assert XSD_ANYURI_AS_IRI_OPTION in EMITTED + assert flagged == generator(URI_TERMS_SCHEMA).serialize() -def test_xsd_anyuri_as_iri_cli_flag(): - """Verify the ``--xsd-anyuri-as-iri`` flag is wired through Click.""" - import tempfile - from pathlib import Path - from click.testing import CliRunner +def _homepage_is_a_link_in_context(output: str) -> bool: + return json.loads(output)["@context"]["homepage"]["@type"] == "@id" - from linkml.generators.jsonldcontextgen import cli - schema_yaml = """ -id: https://example.org/test-cli -name: test_cli +def _homepage_is_a_link_in_owl(output: str) -> bool: + graph = Graph().parse(data=output, format="turtle") + return (URIRef("https://example.org/homepage"), RDF.type, OWL.ObjectProperty) in graph -prefixes: - ex: https://example.org/ - linkml: https://w3id.org/linkml/ - -imports: - - linkml:types - -default_prefix: ex -default_range: string -slots: - homepage: - range: uri - slot_uri: ex:homepage - -classes: - Thing: - slots: - - homepage -""" - with tempfile.TemporaryDirectory() as tmpdir: - schema_path = Path(tmpdir) / "test.yaml" - schema_path.write_text(schema_yaml) - - runner = CliRunner() - - # Without flag - result_default = runner.invoke(cli, [str(schema_path)]) - assert result_default.exit_code == 0, result_default.output - ctx_default = json.loads(result_default.output)["@context"] - assert ctx_default["homepage"]["@type"] == "xsd:anyURI" - - # With flag - result_iri = runner.invoke(cli, [str(schema_path), "--xsd-anyuri-as-iri"]) - assert result_iri.exit_code == 0, result_iri.output - ctx_iri = json.loads(result_iri.output)["@context"] - assert ctx_iri["homepage"]["@type"] == "@id" +@pytest.mark.parametrize( + "cli,homepage_is_a_link", + [(jsonld_context_cli, _homepage_is_a_link_in_context), (owl_cli, _homepage_is_a_link_in_owl)], +) +def test_xsd_anyuri_as_iri_cli_option_is_accepted(tmp_path, cli, homepage_is_a_link): + """Both command lines still accept the option, so existing build scripts keep working.""" + schema_path = tmp_path / "schema.yaml" + schema_path.write_text(URI_TERMS_SCHEMA, encoding="utf-8") + result = CliRunner().invoke(cli, [str(schema_path), "--xsd-anyuri-as-iri"]) + assert result.exit_code == 0, result.output + assert homepage_is_a_link(result.stdout) # Enum @type: @vocab context generation (JSON-LD 1.1 §4.2.3 + §4.1.8) diff --git a/tests/linkml/test_generators/test_shaclgen.py b/tests/linkml/test_generators/test_shaclgen.py index c0dc1863b5..984edd608d 100644 --- a/tests/linkml/test_generators/test_shaclgen.py +++ b/tests/linkml/test_generators/test_shaclgen.py @@ -404,7 +404,7 @@ def check_slot_default_value(slot: URIRef, default_value: Any, datatype: str = N # pyoxigraph's RDFC-1.0 serialization drops explicit ^^xsd:string # per RDF 1.1 (plain literals and xsd:string are equivalent). # Accept either form for xsd:string typed values. - expected = Literal(default_value, datatype=datatype) + expected = default_value if isinstance(default_value, URIRef) else Literal(default_value, datatype=datatype) if (subject, SH.defaultValue, expected) in g: return if datatype and str(datatype) == "http://www.w3.org/2001/XMLSchema#string": @@ -450,10 +450,10 @@ def check_slot_default_value(slot: URIRef, default_value: Any, datatype: str = N "2024-02-08T09:39:25", datatype=ShaclDataType.DATETIME.uri_ref, ) + # A uri value is an IRI (sh:nodeKind sh:IRI), so its default is one too. check_slot_default_value( URIRef("https://w3id.org/linkml/tests/kitchen_sink/ifabsent_uri"), - "https://w3id.org/linkml/tests/kitchen_sink/ifabsent_boolean", - datatype=ShaclDataType.URI.uri_ref, + URIRef("https://w3id.org/linkml/tests/kitchen_sink/ifabsent_boolean"), ) check_slot_default_value(URIRef("https://w3id.org/linkml/tests/kitchen_sink/ifabsent_not_literal"), "heartfelt") diff --git a/tests/linkml/test_generators/test_uri_term_kinds.py b/tests/linkml/test_generators/test_uri_term_kinds.py new file mode 100644 index 0000000000..e109603ff8 --- /dev/null +++ b/tests/linkml/test_generators/test_uri_term_kinds.py @@ -0,0 +1,323 @@ +"""Every generator and the RDF dumper give a URI-like value the same RDF term kind. + +The rule is ``linkml_runtime.utils.rdf_terms.type_term``: ``uri``, ``uriorcurie``, +the types derived from them and the node types (``objectidentifier``, +``nodeidentifier``, ``rdfs:Resource``, ``shex:bnode``) are IRIs or blank nodes; a type +derived from ``string`` with ``uri: xsd:anyURI`` keeps URI references as ``xsd:anyURI`` +literals; ``curie`` stays an ``xsd:string`` literal. +""" + +import json + +import pytest +import yaml +from pyshacl import validate +from rdflib import OWL, RDF, SH, XSD, BNode, Graph, Literal, URIRef + +from linkml.generators.jsonldcontextgen import ContextGenerator +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.pythongen import PythonGenerator +from linkml.generators.shaclgen import ShaclGenerator +from linkml.generators.shexgen import ShExGenerator +from linkml.generators.yarrrmlgen import YarrrmlGenerator +from linkml_runtime import SchemaView +from linkml_runtime.dumpers import rdflib_dumper +from linkml_runtime.loaders import rdflib_loader +from linkml_runtime.utils.compile_python import compile_python + +EX = "https://example.org/terms/" + +SCHEMA = """ +id: https://example.org/terms +name: terms +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/terms/ +default_prefix: ex +default_range: string +imports: + - linkml:types +types: + ImageLink: + typeof: uri + FilePath: + typeof: string + uri: xsd:anyURI + ResourceRef: + typeof: uriorcurie + uri: rdfs:Resource + BlankOnly: + typeof: string + uri: shex:bnode +classes: + Container: + tree_root: true + attributes: + documents: + range: Document + multivalued: true + inlined_as_list: true + Document: + class_uri: ex:Document + attributes: + id: + identifier: true + homepage: + range: uri + related: + range: uriorcurie + image: + range: ImageLink + file_path: + range: FilePath + node: + range: objectidentifier + any_node: + range: nodeidentifier + code: + range: curie + cross_ref: + range: ResourceRef + anon: + range: BlankOnly + title: + range: string +""" + +# slot -> (type, kind of its values): "iri", "nonliteral" (IRI or blank node), "bnode", or a literal datatype +SLOTS = { + "homepage": ("Uri", "iri"), + "related": ("Uriorcurie", "iri"), + "image": ("ImageLink", "iri"), + "file_path": ("FilePath", XSD.anyURI), + "node": ("Objectidentifier", "iri"), + "any_node": ("Nodeidentifier", "nonliteral"), + "code": ("Curie", XSD.string), + "cross_ref": ("ResourceRef", "nonliteral"), + "anon": ("BlankOnly", "bnode"), + "title": ("String", XSD.string), +} +# SHACL node kinds (§4.1.3) of the node term kinds +SH_NODE_KINDS = {"iri": SH.IRI, "nonliteral": SH.BlankNodeOrIRI, "bnode": SH.BlankNode} + +DATA = { + "id": "ex:d1", + "homepage": "https://example.org/terms/home", + "related": "ex:other", + "image": "https://example.org/terms/me.jpg", + "file_path": "./docs/cv.pdf", + "node": "ex:target", + "any_node": "ex:target", + "code": "ex:code", + "cross_ref": "ex:ref", + "anon": "_:n1", + "title": "Alice", +} + +EXPECTED_TERMS = { + "homepage": URIRef(EX + "home"), + "related": URIRef(EX + "other"), + "image": URIRef(EX + "me.jpg"), + "file_path": Literal("./docs/cv.pdf", datatype=XSD.anyURI), + "node": URIRef(EX + "target"), + "any_node": URIRef(EX + "target"), + "code": Literal("ex:code"), + "cross_ref": URIRef(EX + "ref"), + "anon": BNode("n1"), + "title": Literal("Alice"), +} + + +@pytest.fixture(scope="module") +def shapes() -> Graph: + return Graph().parse(data=ShaclGenerator(SCHEMA).serialize(), format="turtle") + + +@pytest.fixture(scope="module") +def module(): + return compile_python(PythonGenerator(SCHEMA).serialize()) + + +def _dumped(module, document: dict) -> Graph: + """The RDF of *document* written by the runtime RDF dumper.""" + container = module.Container(documents=[module.Document(**document)]) + return rdflib_dumper.as_rdf_graph(container, SchemaView(SCHEMA)) + + +def _via_context(document: dict) -> Graph: + """The RDF of *document* read through the generated JSON-LD context.""" + context = json.loads(ContextGenerator(SCHEMA).serialize())["@context"] + doc = {"@context": context, "documents": [{"@type": "Document", **document}]} + return Graph().parse(data=json.dumps(doc), format="json-ld", base="https://reader.example/downloads/doc.json") + + +@pytest.mark.parametrize("slot", SLOTS) +def test_json_ld_context(slot): + coercion = json.loads(ContextGenerator(SCHEMA).serialize())["@context"][slot].get("@type") + kind = SLOTS[slot][1] + expected = {"iri": "@id", "nonliteral": "@id", "bnode": "@id", XSD.anyURI: "xsd:anyURI", XSD.string: None}[kind] + assert coercion == expected + + +@pytest.mark.parametrize("slot", SLOTS) +def test_shacl(slot, shapes): + prop = next(shapes.subjects(SH.path, URIRef(EX + slot))) + node_kind, datatype = shapes.value(prop, SH.nodeKind), shapes.value(prop, SH.datatype) + kind = SLOTS[slot][1] + if kind in SH_NODE_KINDS: + assert (node_kind, datatype) == (SH_NODE_KINDS[kind], None) + else: + assert (node_kind, datatype) == (SH.Literal, kind) + + +def _owl_property(slot: str, **kwargs) -> tuple[set, set]: + """The OWL property types of *slot* and the fillers of its universal restrictions.""" + graph = OwlSchemaGenerator(SCHEMA, metaclasses=False, **kwargs).as_graph() + prop = URIRef(EX + slot) + fillers = {f for r in graph.subjects(OWL.onProperty, prop) for f in graph.objects(r, OWL.allValuesFrom)} + return set(graph.objects(prop, RDF.type)), fillers + + +@pytest.mark.parametrize("slot", SLOTS) +def test_owl(slot): + """Node-valued slots are object properties ranging over owl:Thing (OWL 2 §5.3, §5.8.1).""" + types, fillers = _owl_property(slot, type_objects=False) + kind = SLOTS[slot][1] + if kind in SH_NODE_KINDS: + assert (types, fillers) == ({OWL.ObjectProperty}, {OWL.Thing}) + else: + assert (types, fillers) == ({OWL.DatatypeProperty}, {kind}) + + +@pytest.mark.parametrize("slot", SLOTS) +def test_owl_type_objects_mode_is_unchanged(slot): + """With ``type_objects``, every typed value is an individual of the type's class, whatever its term kind.""" + types, fillers = _owl_property(slot, type_objects=True) + assert types == {OWL.ObjectProperty} + assert [str(f).rsplit("/", 1)[-1] for f in fillers] == [SLOTS[slot][0]] + + +@pytest.mark.parametrize("slot", SLOTS) +def test_shex(slot): + shapes = json.loads(ShExGenerator(SCHEMA, format="json").serialize())["shapes"] + type_name, kind = SLOTS[slot] + shape = next(s for s in shapes if s["id"].rsplit("/", 1)[-1] == type_name) + if kind in SH_NODE_KINDS: + assert shape.get("nodeKind") == kind + else: + assert shape.get("datatype") == str(kind) + + +@pytest.mark.parametrize("slot", SLOTS) +def test_yarrrml(slot): + mapping = yaml.safe_load(YarrrmlGenerator(SCHEMA).serialize())["mappings"]["Document"]["po"] + obj = next(po["o"] for po in mapping if po["p"] == f"ex:{slot}") + kind = SLOTS[slot][1] + if kind in SH_NODE_KINDS: + assert obj["type"] == {"iri": "iri", "nonliteral": "iri", "bnode": "blank"}[kind] + else: + assert obj["datatype"] == {XSD.anyURI: "xsd:anyURI", XSD.string: "xsd:string"}[kind] + + +@pytest.mark.parametrize("path", ["dumper", "context"]) +def test_both_data_paths_give_the_same_terms_and_conform(module, shapes, path): + """The dumper and the generated context write the same terms, and the generated shapes accept them. + + The relative file path stays text on both paths; it is not resolved against the reader's location. + """ + graph = _dumped(module, DATA) if path == "dumper" else _via_context(DATA) + subject = URIRef(EX + "d1") + assert {slot: graph.value(subject, URIRef(EX + slot)) for slot in SLOTS} == EXPECTED_TERMS + conforms, _, report = validate(graph, shacl_graph=shapes) + assert conforms, report + + +@pytest.mark.parametrize( + "slot,wrong", + [ + ("homepage", Literal(EX + "home", datatype=XSD.anyURI)), + ("image", Literal("ex:me.jpg")), + ("file_path", URIRef("https://example.org/cv.pdf")), + ("node", BNode("b1")), + ("any_node", Literal("ex:target", datatype=URIRef("http://www.w3.org/ns/shex#nonLiteral"))), + ("code", URIRef(EX + "code")), + ("cross_ref", Literal("ex:ref", datatype=URIRef("http://www.w3.org/2000/01/rdf-schema#Resource"))), + ("anon", URIRef(EX + "n1")), + ], +) +def test_shapes_reject_the_wrong_term_kind(module, shapes, slot, wrong): + graph = _dumped(module, DATA) + subject = URIRef(EX + "d1") + graph.set((subject, URIRef(EX + slot), wrong)) + conforms, _, report = validate(graph, shacl_graph=shapes) + assert not conforms, f"{slot}={wrong!r} accepted" + + +@pytest.mark.parametrize("cast_literals", [True, False]) +def test_dumped_terms_load_back(module, cast_literals): + """The loader accepts the dumper's terms, also in strict mode; ``uri`` values stay complete, CURIEs stay CURIEs.""" + graph = _dumped(module, DATA) + [container] = rdflib_loader.from_rdf_graph( + graph, schemaview=SchemaView(SCHEMA), target_class=module.Container, cast_literals=cast_literals + ) + loaded = {slot: getattr(container.documents[0], slot) for slot in SLOTS} + assert loaded == {slot: DATA[slot] for slot in SLOTS} + + +def test_loader_keeps_blank_node_labels(module): + """A blank node in an IRI-or-blank-node slot loads as its ``_:`` label instead of an inlined object.""" + graph = Graph() + container, document = BNode(), URIRef(EX + "d1") + graph.add((container, RDF.type, URIRef(EX + "Container"))) + graph.add((container, URIRef(EX + "documents"), document)) + graph.add((document, RDF.type, URIRef(EX + "Document"))) + graph.add((document, URIRef(EX + "cross_ref"), BNode("n1"))) + [container] = rdflib_loader.from_rdf_graph(graph, schemaview=SchemaView(SCHEMA), target_class=module.Container) + assert container.documents[0].cross_ref == "_:n1" + + +@pytest.mark.parametrize("slot", ["homepage", "image"]) +def test_dumper_rejects_relative_links(module, slot): + """A relative reference is not an RDF IRI (RDF 1.1 Concepts §3.2); text needs a string-derived type.""" + with pytest.raises(ValueError, match="relative reference"): + _dumped(module, {**DATA, slot: "../img/cover.png"}) + + +def test_blank_node_values_through_the_context(shapes): + """``_:`` labels in an IRI-or-blank-node slot become blank nodes and conform.""" + graph = _via_context({**DATA, "any_node": "_:n1", "cross_ref": "_:n2"}) + subject = URIRef(EX + "d1") + assert isinstance(graph.value(subject, URIRef(EX + "any_node")), BNode) + assert isinstance(graph.value(subject, URIRef(EX + "cross_ref")), BNode) + conforms, _, report = validate(graph, shacl_graph=shapes) + assert conforms, report + + +SUBPROPERTY_SCHEMA = """ +id: https://example.org/sub +name: sub +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/sub/ +default_prefix: ex +imports: + - linkml:types +slots: + related_to: {range: curie} + part_of: {is_a: related_to} +classes: + Edge: + class_uri: ex:Edge + attributes: + predicate: + range: curie + subproperty_of: related_to +""" + + +def test_subproperty_values_take_the_slot_term_kind(): + """``sh:in`` members are compared exactly (SHACL §4.8.3), so a ``curie`` slot lists CURIE literals.""" + shapes = Graph().parse(data=ShaclGenerator(SUBPROPERTY_SCHEMA).serialize(), format="turtle") + edge = '@prefix ex: . ex:e1 a ex:Edge ; ex:predicate "{}" .' + assert validate(Graph().parse(data=edge.format("ex:part_of"), format="turtle"), shacl_graph=shapes)[0] + assert not validate(Graph().parse(data=edge.format("ex:unrelated"), format="turtle"), shacl_graph=shapes)[0] diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld index 1ce86f3a57..3092b3dd07 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld @@ -10,7 +10,7 @@ "simple_uri_test": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "linkml:tag" }, "value": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld index 4cf9c59701..9bd2b80f40 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld @@ -3,7 +3,7 @@ "xsd": "http://www.w3.org/2001/XMLSchema#", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "linkml:tag" }, "value": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld index 1ce86f3a57..3092b3dd07 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld @@ -10,7 +10,7 @@ "simple_uri_test": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "linkml:tag" }, "value": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld index 4cf9c59701..9bd2b80f40 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld @@ -3,7 +3,7 @@ "xsd": "http://www.w3.org/2001/XMLSchema#", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "linkml:tag" }, "value": { diff --git a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld index d7274380da..ad8978b81f 100644 --- a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld @@ -10,7 +10,7 @@ "simple_uri_test": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "linkml:tag" }, "value": { diff --git a/tests/linkml/test_utils/__snapshots__/uriandcurie.json b/tests/linkml/test_utils/__snapshots__/uriandcurie.json index 2c39326ff5..97198624f6 100644 --- a/tests/linkml/test_utils/__snapshots__/uriandcurie.json +++ b/tests/linkml/test_utils/__snapshots__/uriandcurie.json @@ -196,7 +196,7 @@ "@id": "hasNcName" }, "hasURI": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "hasURI" }, "id": "@id", diff --git a/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld b/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld index 47f2eb296b..f87f92da56 100644 --- a/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld +++ b/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld @@ -15,7 +15,7 @@ "@id": "hasNcName" }, "hasURI": { - "@type": "xsd:anyURI", + "@type": "@id", "@id": "hasURI" }, "id": "@id", diff --git a/tests/linkml_runtime/test_utils/test_rdf_terms.py b/tests/linkml_runtime/test_utils/test_rdf_terms.py new file mode 100644 index 0000000000..92b08f9833 --- /dev/null +++ b/tests/linkml_runtime/test_utils/test_rdf_terms.py @@ -0,0 +1,202 @@ +"""Tests for the RDF term kind of values of LinkML types (``linkml_runtime.utils.rdf_terms``).""" + +import pytest +from rdflib import XSD, BNode, Literal, URIRef + +from linkml_runtime.utils.rdf_terms import TermKind, TypeTerm, link_root, type_term, value_term +from linkml_runtime.utils.schemaview import SchemaView + +ANYURI = str(XSD.anyURI) + +SCHEMA = """ +id: https://example.org/terms +name: terms +prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/terms/ +default_prefix: ex +imports: + - linkml:types +types: + ImageLink: + typeof: uri + FilePath: + typeof: string + uri: xsd:anyURI + RootAnyUri: + uri: xsd:anyURI + base: str + ResourceRef: + typeof: uriorcurie + uri: rdfs:Resource + TextLink: + typeof: uri + uri: xsd:string + FullIriLink: + typeof: uri + uri: http://www.w3.org/2001/XMLSchema#anyURI + FullIriText: + typeof: string + uri: http://www.w3.org/2001/XMLSchema#anyURI + BlankOnly: + typeof: string + uri: shex:bnode +""" + + +@pytest.fixture(scope="module") +def sv() -> SchemaView: + return SchemaView(SCHEMA) + + +@pytest.mark.parametrize( + "type_name,expected", + [ + ("uri", TypeTerm(TermKind.IRI)), + ("uriorcurie", TypeTerm(TermKind.IRI)), + ("ImageLink", TypeTerm(TermKind.IRI)), + ("FullIriLink", TypeTerm(TermKind.IRI)), + ("objectidentifier", TypeTerm(TermKind.IRI)), + ("nodeidentifier", TypeTerm(TermKind.BLANK_NODE_OR_IRI)), + ("ResourceRef", TypeTerm(TermKind.BLANK_NODE_OR_IRI)), + ("BlankOnly", TypeTerm(TermKind.BLANK_NODE)), + ("FilePath", TypeTerm(TermKind.LITERAL, ANYURI)), + ("FullIriText", TypeTerm(TermKind.LITERAL, ANYURI)), + ("RootAnyUri", TypeTerm(TermKind.LITERAL, ANYURI)), + ("TextLink", TypeTerm(TermKind.LITERAL, str(XSD.string))), + ("curie", TypeTerm(TermKind.LITERAL, str(XSD.string))), + ("string", TypeTerm(TermKind.LITERAL, str(XSD.string))), + ("integer", TypeTerm(TermKind.LITERAL, str(XSD.integer))), + ], +) +def test_type_term(sv, type_name, expected): + """Links and node types are nodes; every other type, including text anyURI types, is a literal. + + An explicit datatype on a type derived from ``uri`` wins (``TextLink``), and the + spelling of the datatype IRI (CURIE or full IRI) does not matter. + """ + assert type_term(sv, type_name) == expected + + +def test_vocabulary_prefix_need_not_be_declared(sv): + """``ResourceRef`` uses ``rdfs:`` without declaring it; the vocabulary prefix is recognised.""" + assert "rdfs" not in sv.namespaces() + assert type_term(sv, "ResourceRef").kind is TermKind.BLANK_NODE_OR_IRI + + +def test_undeclared_prefix_fails_fast(): + """A datatype with a prefix that is neither declared nor a vocabulary prefix is an error.""" + sv = SchemaView(SCHEMA + " Odd:\n typeof: string\n uri: foo:bar\n") + with pytest.raises(ValueError, match="does not declare prefix 'foo'"): + type_term(sv, "Odd") + + +@pytest.mark.parametrize( + "prefix,uri,expected", + [ + ("ex: https://example.org/c/", "ex:a:b", "https://example.org/c/a:b"), + ("urn: 'urn:'", "urn:example:dt", "urn:example:dt"), + ], +) +def test_declared_prefix_expands_a_local_part_with_a_colon(prefix, uri, expected): + """A declared prefix expands whatever the local part, so a ``urn:`` datatype works once ``urn`` is declared.""" + sv = SchemaView( + f"id: https://example.org/c\nname: c\nprefixes:\n {prefix}\ntypes:\n Odd:\n base: str\n uri: {uri}\n" + ) + assert type_term(sv, "Odd") == TypeTerm(TermKind.LITERAL, expected) + + +@pytest.mark.parametrize( + "type_name,expected", + [ + ("uri", "uri"), + ("ImageLink", "uri"), + ("TextLink", "uri"), + ("FullIriLink", "uri"), + ("uriorcurie", "uriorcurie"), + ("ResourceRef", "uriorcurie"), + ("RootAnyUri", None), + ("FilePath", None), + ("objectidentifier", None), + ("curie", None), + ("string", None), + ], +) +def test_link_root(sv, type_name, expected): + """``uri`` values are complete URIs and ``uriorcurie`` values may be CURIEs; the root is found through ``typeof``.""" + assert link_root(sv, type_name) == expected + + +OWN_TYPES = """ +id: https://example.org/own +name: own +prefixes: + ex: https://example.org/own/ + xsd: http://www.w3.org/2001/XMLSchema# +default_prefix: ex +types: + string: + uri: xsd:string + base: str + uri: + typeof: string + uri: xsd:anyURI + uriorcurie: + uri: xsd:anyURI + base: URIorCURIE +""" + + +@pytest.mark.parametrize("type_name,kind", [("uri", TermKind.LITERAL), ("uriorcurie", TermKind.IRI)]) +def test_link_types_are_identified_by_their_definition(type_name, kind): + """A user type that is merely called ``uri`` is text; a verbatim copy of ``uriorcurie`` is a link.""" + assert type_term(SchemaView(OWN_TYPES), type_name).kind is kind + + +def test_merged_schema_keeps_link_types(): + """Merging imports rewrites ``from_schema``; the link types are still recognised.""" + sv = SchemaView(SCHEMA) + sv.merge_imports() + assert type_term(sv, "uri").kind is TermKind.IRI + assert type_term(sv, "ImageLink").kind is TermKind.IRI + + +@pytest.mark.parametrize( + "type_name,value,expected", + [ + ("uri", "https://example.org/x", URIRef("https://example.org/x")), + ("uriorcurie", "ex:x", URIRef("https://example.org/terms/x")), + ("objectidentifier", "ex:x", URIRef("https://example.org/terms/x")), + ("nodeidentifier", "_:b1", BNode("b1")), + ("ResourceRef", "_:b1", BNode("b1")), + ("FilePath", "./docs/cv.pdf", Literal("./docs/cv.pdf", datatype=XSD.anyURI)), + ("FilePath", "ex:x", Literal("ex:x", datatype=XSD.anyURI)), + ("curie", "ex:x", Literal("ex:x")), + ("string", "text", Literal("text")), + ("boolean", True, Literal(True)), + ], +) +def test_value_term(sv, type_name, value, expected): + """IRIs are expanded, text stays as written, and literals keep their native value.""" + term = value_term(sv, type_name, value) + assert term == expected + assert type(term) is type(expected) + + +def test_boolean_literal_has_canonical_lexical_form(sv): + assert value_term(sv, "boolean", True).n3() == '"true"^^' + + +@pytest.mark.parametrize( + "type_name,value,message", + [ + ("uri", "./docs/cv.pdf", "relative reference"), + ("ImageLink", "../img/a.png", "relative reference"), + ("uriorcurie", "_:b1", "is a blank node"), + ("BlankOnly", "https://example.org/x", "blank node label"), + ], +) +def test_value_term_rejects_values_of_the_wrong_kind(sv, type_name, value, message): + """A relative reference cannot be an RDF IRI (RDF 1.1 Concepts §3.2), nor a blank node an IRI.""" + with pytest.raises(ValueError, match=message): + value_term(sv, type_name, value) From 9258891e0738b4597bdab6002b3fde80ad275d08 Mon Sep 17 00:00:00 2001 From: jdsika Date: Wed, 21 Oct 2026 22:18:27 +0200 Subject: [PATCH 10/17] feat(rdf): give declared annotation values and native metadata the term of their range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An annotation whose tag a metaclass declares through instantiates gets the RDF term that the slot's range gives a slot value (linkml_runtime.utils.rdf_terms): nodeidentifier, uri and uriorcurie values are IRIs or blank nodes, string values are text with the resolved language, curie values are xsd:string literals, and an enum-ranged slot writes the permissible value's text. gen-owl and gen-shacl share the rule through linkml.generators.common.annotations; without a declaration both keep their earlier behaviour, and metaclasses that declare a tag with different terms are an error. gen-owl applies the same rule to native metadata, so see_also, source and the mappings are IRIs by the metamodel's ranges rather than by a name check, and created_on is an xsd:dateTime literal. A link value that is neither an IRI nor a CURIE, such as a bare element name in see_also, is written as a plain literal with a warning: a relative IRI would make the whole graph non-standard RDF and push it onto the canonicalizer's rdflib fallback. OWL 2 §10.1 admits IRIs, anonymous individuals and literals as annotation values. Signed-off-by: jdsika --- docs/examples/owl-dcmi-agent-annotations.yaml | 26 ++ docs/generators/owl.rst | 19 ++ docs/schemas/annotations.md | 32 +++ .../linkml/generators/common/annotations.py | 120 ++++++++ .../linkml/src/linkml/generators/owlgen.py | 42 ++- .../linkml/src/linkml/generators/shaclgen.py | 5 + .../src/linkml_runtime/utils/rdf_terms.py | 17 +- .../test_declared_annotations.py | 259 ++++++++++++++++++ 8 files changed, 509 insertions(+), 11 deletions(-) create mode 100644 docs/examples/owl-dcmi-agent-annotations.yaml create mode 100644 packages/linkml/src/linkml/generators/common/annotations.py create mode 100644 tests/linkml/test_generators/test_declared_annotations.py diff --git a/docs/examples/owl-dcmi-agent-annotations.yaml b/docs/examples/owl-dcmi-agent-annotations.yaml new file mode 100644 index 0000000000..d2156d223c --- /dev/null +++ b/docs/examples/owl-dcmi-agent-annotations.yaml @@ -0,0 +1,26 @@ +id: https://example.org/model +name: model +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + dcterms: http://purl.org/dc/terms/ + orcid: https://orcid.org/ + owl: http://www.w3.org/2002/07/owl# +imports: [linkml:types] +default_prefix: ex +instantiates: [ex:AgentMetadata] +annotations: + creator: orcid:0000-0002-1825-0097 + rights: Copyright 2026 The Example Team +classes: + AgentMetadata: + description: Declares the agent metadata that a schema may carry as annotations. + attributes: + creator: + slot_uri: dcterms:creator + range: nodeidentifier + implements: [owl:AnnotationProperty] + rights: + slot_uri: dcterms:rights + range: string + implements: [owl:AnnotationProperty] diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index da32043267..5c57aa9aba 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -70,6 +70,25 @@ Mapping .. note:: The current default settings for ``metaclasses`` and ``type-objects`` may change in the future +Annotation values +^^^^^^^^^^^^^^^^^ + +Native metadata takes the RDF term of its metamodel range: ``see_also`` and ``source`` +values are IRIs, ``license`` and ``description`` values are strings, and ``created_on`` +is an ``xsd:dateTime`` literal. A link value that is neither an IRI nor a CURIE, such as +a bare element name in ``see_also``, is written as a plain literal, with a warning. An annotation +whose tag a metaclass declares through ``instantiates`` takes the term of that slot's +range in the same way (see :ref:`annotations-in-rdf`); any other annotation value is a +string. + +.. literalinclude:: ../examples/owl-dcmi-agent-annotations.yaml + :language: yaml + +The ontology header then carries ``dcterms:creator `` +and ``dcterms:rights "Copyright 2026 The Example Team"``. ``implements: [owl:AnnotationProperty]`` +declares the two predicates as annotation properties; the restrictions that the metaclass's +own slots produce are a matter of the property role, not of the value. + Enums and PermissibleValues ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/schemas/annotations.md b/docs/schemas/annotations.md index b29d56e231..a22892ac41 100644 --- a/docs/schemas/annotations.md +++ b/docs/schemas/annotations.md @@ -130,6 +130,38 @@ first-class slots, and instantiates value was a mixin. However, this is not yet supported in the current LinkML validator, so using `instantiates` indicates your intent, but does not yet enforce validation. +(annotations-in-rdf)= +## Annotation values in RDF + +The declared range also decides how gen-owl and gen-shacl (with `--include-annotations`) write an +annotation value, by the rule that applies to slot values (see {ref}`links-and-uri-references`): a +`nodeidentifier` or `uri` value becomes an IRI, or a blank node for a `_:` label; a `string` value stays +text and carries the language of `--default-language`; a `curie` value stays a string; and the value of +an enum-ranged slot is written as the permissible value's text. + +```yaml +classes: + Reviewable: + class_uri: mymetamodel:Reviewable + attributes: + review: + range: string + reviewer: + range: nodeidentifier + Person: + instantiates: + - mymetamodel:Reviewable + annotations: + review: A very useful class that is well defined + reviewer: orcid:0000-0002-1825-0097 +``` + +gen-owl then writes `mymetamodel:review "A very useful class that is well defined"` and +`mymetamodel:reviewer ` on the class. A value that cannot be a +term of the declared kind, such as a relative reference for a `uri` slot, is an error. Without a +declaration the generators keep their earlier behaviour: gen-owl writes every annotation value as a +string, and gen-shacl writes a value containing a colon as an IRI. + ## Other uses of instantiates The `instantiates` slot can also be used constrain rather than extend. diff --git a/packages/linkml/src/linkml/generators/common/annotations.py b/packages/linkml/src/linkml/generators/common/annotations.py new file mode 100644 index 0000000000..702ace7ef8 --- /dev/null +++ b/packages/linkml/src/linkml/generators/common/annotations.py @@ -0,0 +1,120 @@ +"""RDF terms of annotation values that a metamodel extension declares. + +A schema element can ``instantiates`` a metaclass whose slots describe the +element's annotations (LinkML specification, part 5, "Metamodel refinement using +annotations"). Such a slot gives an annotation value its RDF term by the rule +that applies to slot values (:mod:`linkml_runtime.utils.rdf_terms`): a value of +``nodeidentifier`` or ``uri`` is a node, a value of ``string`` is text, a value +of a type derived from ``string`` with ``uri: xsd:anyURI`` is an ``xsd:anyURI`` +literal, and so on. OWL 2 admits IRIs, anonymous individuals and literals as +annotation values (OWL 2 Structural Specification §10.1), so no annotation value +needs a term of another kind. + +>>> from linkml_runtime.utils.schemaview import SchemaView +>>> sv = SchemaView(''' +... id: https://example.org/m +... name: m +... prefixes: {ex: https://example.org/m/, linkml: https://w3id.org/linkml/} +... default_prefix: ex +... imports: [linkml:types] +... classes: +... Reviewable: +... attributes: +... reviewer: {range: nodeidentifier} +... review: {range: string} +... Person: +... instantiates: [ex:Reviewable] +... annotations: {reviewer: ex:alice, review: well defined} +... ''') +>>> person = sv.get_class("Person") +>>> declared_annotation(sv, person, "reviewer", "ex:alice") +DeclaredAnnotation(predicate=rdflib.term.URIRef('https://example.org/m/reviewer'), value=rdflib.term.URIRef('https://example.org/m/alice')) +>>> declared_annotation(sv, person, "review", "well defined", language="en").value +rdflib.term.Literal('well defined', lang='en') +>>> declared_annotation(sv, person, "undeclared", "x") is None +True +""" + +from dataclasses import dataclass +from typing import Any + +from rdflib import BNode, Literal, URIRef + +from linkml_runtime.linkml_model.meta import Element, PermissibleValue, SlotDefinition +from linkml_runtime.utils.rdf_terms import value_term +from linkml_runtime.utils.schemaview import SchemaView + + +@dataclass(frozen=True) +class DeclaredAnnotation: + """The RDF predicate and value of an annotation that a metaclass slot declares.""" + + predicate: URIRef + value: URIRef | BNode | Literal + + +def declaring_slots(sv: SchemaView, element: Element | PermissibleValue, tag: str) -> list[SlotDefinition]: + """Return the slots of the metaclasses that *element* instantiates which declare annotation *tag*. + + A metaclass is referenced by its class URI. A tag matches a slot by name or, + as a CURIE or IRI, by the slot's URI. Inherited and imported slots count. + """ + expanded_tag = sv.expand_curie(tag) + slots: list[SlotDefinition] = [] + for reference in element.instantiates: + metaclass_uri = sv.expand_curie(reference) + for cls in sv.all_classes().values(): + if sv.get_uri(cls, expand=True) != metaclass_uri: + continue + for slot in sv.class_induced_slots(cls.name): + if slot.name == tag or sv.get_uri(slot, expand=True) == expanded_tag: + slots.append(slot) + return slots + + +def annotation_term( + sv: SchemaView, slot: SlotDefinition, value: Any, language: str | None = None +) -> URIRef | BNode | Literal | None: + """Return the RDF term of annotation *value* according to the range of the declaring *slot*. + + A type range gives the term of a slot value of that type; text, which is a + plain literal of a type derived from ``string``, carries *language* when one is + set. An enum range gives the permissible value's text as a plain literal, since + it is an identifier rather than text. Any other range, such as a class, declares + no scalar term, and the result is ``None``. + + :raises ValueError: if *value* cannot be a term of the declared kind, for + example a relative reference for a ``uri`` slot + """ + if slot.range in sv.all_types(): + term = value_term(sv, slot.range, value) + is_text = isinstance(term, Literal) and term.datatype is None and "string" in sv.type_ancestors(slot.range) + return Literal(value, lang=language) if language and is_text else term + if slot.range in sv.all_enums(): + return Literal(str(value)) + return None + + +def declared_annotation( + sv: SchemaView, element: Element | PermissibleValue, tag: str, value: Any, language: str | None = None +) -> DeclaredAnnotation | None: + """Return the predicate and term of annotation *tag* on *element* if a metaclass declares it. + + The result is ``None`` when no metaclass of *element* declares the tag, when the + value is not a scalar, or when the declaring slot has no scalar range; the + caller then keeps its undeclared-annotation behaviour. + + :raises ValueError: if the metaclasses declare the tag with different terms, or + if the value cannot be a term of the declared kind + """ + if not isinstance(value, str | bool | int | float): + return None + declared: set[DeclaredAnnotation] = set() + for slot in declaring_slots(sv, element, tag): + term = annotation_term(sv, slot, value, language) + if term is not None: + declared.add(DeclaredAnnotation(URIRef(sv.get_uri(slot, expand=True)), term)) + if len(declared) > 1: + metaclasses = ", ".join(element.instantiates) + raise ValueError(f"The metaclasses {metaclasses} declare annotation {tag!r} with different terms") + return declared.pop() if declared else None diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index 880e561eae..fbbe54054d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -19,6 +19,7 @@ from linkml import METAMODEL_NAMESPACE_NAME from linkml._version import __version__ +from linkml.generators.common.annotations import declared_annotation from linkml.utils.deprecation import XSD_ANYURI_AS_IRI_OPTION, deprecation_warning from linkml.utils.generator import Generator, shared_arguments from linkml.utils.language_tags import LanguageTagResolver @@ -44,7 +45,7 @@ from linkml_runtime.utils.formatutils import camelcase, underscore from linkml_runtime.utils.introspection import package_schemaview from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph -from linkml_runtime.utils.rdf_terms import type_term +from linkml_runtime.utils.rdf_terms import link_iri, type_term, value_term logger = logging.getLogger(__name__) @@ -386,14 +387,7 @@ def add_metadata(self, e: Element | PermissibleValue, uri: URIRef) -> None: metaslot_value = [metaslot_value] for v in metaslot_value: if metaslot_range in msv.all_types(): - if metaslot_range == "uri": - obj = URIRef(v) - elif metaslot_range == "uriorcurie": - obj = URIRef(this_sv.expand_curie(v)) - elif metaslot_range in self._LANGUAGE_TAGGABLE_RANGES and lang: - obj = Literal(v, lang=lang) - else: - obj = Literal(v) + obj = self._metaslot_term(metaslot_range, v, lang, f"{metaslot_name} of {uri}") elif metaslot_range in msv.all_subsets(): obj = Literal(v) # TODO elif metaslot_range in msv.all_classes(): @@ -418,6 +412,10 @@ def add_metadata(self, e: Element | PermissibleValue, uri: URIRef) -> None: for k, v in e.annotations.items(): if isinstance(v, dict) or isinstance(v, list): continue + declared = declared_annotation(this_sv, e, k, v.value, lang) + if declared is not None: + self.graph.add((uri, declared.predicate, declared.value)) + continue if ":" not in k: default_prefix = this_sv.schema.default_prefix if default_prefix in this_sv.schema.prefixes: @@ -435,6 +433,32 @@ def add_metadata(self, e: Element | PermissibleValue, uri: URIRef) -> None: obj = Literal(v.value) self.graph.add((uri, URIRef(k_uri), obj)) + def _metaslot_term( + self, range_name: str, value: Any, language: str | None, origin: str + ) -> URIRef | BNode | Literal: + """Return the RDF term of a native metadata value according to the metamodel slot's range. + + The term kind follows :func:`~linkml_runtime.utils.rdf_terms.value_term`, so + ``see_also`` and ``source`` values are IRIs and ``license`` values are + strings. A link's CURIE is expanded with the schema's prefixes, then with the + metamodel's. Human-readable text carries the resolved language. A link value + that is neither an IRI nor a CURIE, such as a bare element name in + ``see_also``, cannot be resolved; it is written as a plain literal, with a + warning, since a relative IRI would make the whole graph non-standard RDF + and OWL 2 admits a literal as an annotation value (Structural Specification + §10.1). + """ + msv = self.metamodel_schemaview + if range_name in self._LANGUAGE_TAGGABLE_RANGES and language: + return Literal(value, lang=language) + if not type_term(msv, range_name).kind.is_node: + return value_term(msv, range_name, value) + link = self.schemaview.expand_curie(str(value)) + if link_iri(msv, link) is None and not link.startswith("_:"): + logger.warning(f"{origin}: {value!r} is neither an IRI nor a CURIE; written as a literal") + return Literal(str(value)) + return value_term(msv, range_name, link) + def add_class(self, cls: ClassDefinition) -> None: """ Each ClassDefinition is represented as an OWL class. diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index d3eb008faf..cea9d5f3aa 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -14,6 +14,7 @@ from rdflib.namespace import RDF, RDFS, SH, XSD from linkml._version import __version__ +from linkml.generators.common.annotations import declared_annotation from linkml.generators.common.class_expression import class_membership, value_bounds from linkml.generators.common.subproperty import get_subproperty_values from linkml.generators.shacl.shacl_data_type import ShaclDataType @@ -1998,6 +1999,10 @@ def _add_annotations(self, func: Callable, item) -> None: if type(annotations) is JsonObj: annotations = as_dict(annotations) for a in annotations.values(): + declared = declared_annotation(sv, item, a["tag"], a["value"], self._resolve_language(item)) + if declared is not None: + func(declared.predicate, declared.value) + continue # If ':' is in the tag, treat it as a CURIE, otherwise string Literal if ":" in a["tag"]: N_predicate = URIRef(sv.expand_curie(a["tag"])) diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py index 73e8411234..90da49cb0a 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_terms.py @@ -45,6 +45,8 @@ rdflib.term.URIRef('https://example.org/t/me.jpg') >>> value_term(sv, "FilePath", "./docs/cv.pdf") rdflib.term.Literal('./docs/cv.pdf', datatype=rdflib.term.URIRef('http://www.w3.org/2001/XMLSchema#anyURI')) +>>> link_iri(sv, "ex:me.jpg"), link_iri(sv, "./docs/cv.pdf") +('https://example.org/t/me.jpg', None) """ import re @@ -174,6 +176,17 @@ def type_term(sv: SchemaView, type_name: str) -> TypeTerm: return TypeTerm(TermKind.LITERAL, datatype) +def link_iri(sv: SchemaView, value: str) -> str | None: + """Return the absolute IRI that link *value* denotes, or ``None`` if it is a relative reference. + + A CURIE is expanded with the schema's prefixes. A value with an undeclared prefix + passes as an absolute IRI of that scheme, as in JSON-LD; one without a scheme + (RFC 3986 §3.1) is a relative reference. + """ + iri = sv.expand_curie(value) + return iri if _SCHEME.match(iri) else None + + def value_term(sv: SchemaView, type_name: str, value: Any) -> Node: """Return the RDF term for *value* of the type *type_name*. @@ -197,8 +210,8 @@ def value_term(sv: SchemaView, type_name: str, value: Any) -> Node: raise ValueError(f"Value {text!r} of type {type_name!r} must be a blank node label such as '_:b1'") if text.startswith("_:"): raise ValueError(f"Value {text!r} of type {type_name!r} is a blank node, but values of this type are IRIs") - iri = sv.expand_curie(text) - if not _SCHEME.match(iri): + iri = link_iri(sv, text) + if iri is None: raise ValueError( f"Value {text!r} of type {type_name!r} is a relative reference, but values of this type are " "IRIs, which must be absolute. Use a type derived from string with uri: xsd:anyURI to keep " diff --git a/tests/linkml/test_generators/test_declared_annotations.py b/tests/linkml/test_generators/test_declared_annotations.py new file mode 100644 index 0000000000..34b618b672 --- /dev/null +++ b/tests/linkml/test_generators/test_declared_annotations.py @@ -0,0 +1,259 @@ +"""Annotation values declared through a metamodel extension take the RDF term of their declared range. + +The rule is ``linkml.generators.common.annotations``: a slot of a metaclass that a +schema element ``instantiates`` gives the annotation value the term a slot value of +that range gets (``linkml_runtime.utils.rdf_terms``). gen-owl and gen-shacl share it, +and gen-owl applies the same rule to native metadata such as ``see_also``. +""" + +import warnings +from pathlib import Path + +import pytest +import yaml +from pyshacl import validate +from rdflib import DCTERMS, RDF, RDFS, SH, XSD, BNode, Graph, Literal, URIRef + +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.shaclgen import ShaclGenerator +from linkml_runtime.utils.rdf_canonicalize import RDFCanonicalizationWarning + +EX = "https://example.org/" +EXAMPLE = Path(__file__).resolve().parents[3] / "docs/examples/owl-dcmi-agent-annotations.yaml" + +SCHEMA = """ +id: https://example.org/model +name: model +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ +imports: [linkml:types] +default_prefix: ex +default_range: string +types: + FilePath: + typeof: string + uri: xsd:anyURI +enums: + Status: + permissible_values: {draft: {}, final: {}} +classes: + Metadata: + attributes: + reviewer: {range: nodeidentifier} + homepage: {range: uri} + file_path: {range: FilePath} + code: {range: curie} + status: {range: Status} + review: {range: string} + note: {} + Profile: + is_a: Metadata + class_uri: ex:Profile + Thing: + instantiates: [ex:Profile] + annotations: + reviewer: ex:alice + homepage: https://alice.example.org/ + file_path: ./docs/cv.pdf + code: ex:c1 + status: draft + review: well defined + note: free text +""" + +# tag -> the term its declared range gives the value (with default_language "en") +EXPECTED = { + "reviewer": URIRef(EX + "alice"), + "homepage": URIRef("https://alice.example.org/"), + "file_path": Literal("./docs/cv.pdf", datatype=XSD.anyURI), + "code": Literal("ex:c1"), + "status": Literal("draft"), + "review": Literal("well defined", lang="en"), + "note": Literal("free text", lang="en"), +} + + +def _graph(source: str | Path, generator: str, **kwargs) -> Graph: + """The generated OWL ontology or SHACL shapes, parsed back from Turtle.""" + if generator == "owl": + text = OwlSchemaGenerator(source, type_objects=False, metaclasses=False, default_language="en", **kwargs) + else: + text = ShaclGenerator(source, include_annotations=True, default_language="en", **kwargs) + return Graph().parse(data=text.serialize(), format="turtle") + + +@pytest.mark.parametrize("generator", ["owl", "shacl"]) +@pytest.mark.parametrize("tag", EXPECTED) +def test_declared_range_gives_the_term(generator, tag): + """Nodes, typed literals, strings and enum values follow the metaclass slot's range. + + The metaclass needs no ``implements`` role for this, as in the annotations guide. + """ + graph = _graph(SCHEMA, generator) + assert set(graph.objects(URIRef(EX + "Thing"), URIRef(EX + tag))) == {EXPECTED[tag]} + + +@pytest.mark.parametrize("generator", ["owl", "shacl"]) +@pytest.mark.parametrize("form", ["name", "curie", "iri"]) +def test_tag_as_slot_name_curie_or_iri(generator, form): + schema = yaml.safe_load(SCHEMA) + prefix = {"name": "", "curie": "ex:", "iri": EX}[form] + schema["classes"]["Thing"]["annotations"] = {prefix + "reviewer": "ex:alice"} + graph = _graph(yaml.safe_dump(schema), generator) + assert set(graph.objects(URIRef(EX + "Thing"), URIRef(EX + "reviewer"))) == {URIRef(EX + "alice")} + + +@pytest.mark.parametrize( + "generator,owner", + [ + ("owl", "class"), + ("owl", "slot"), + ("owl", "enum"), + ("owl", "permissible_value"), + ("shacl", "slot"), + ("shacl", "type"), + ], +) +def test_annotated_elements(generator, owner): + """Every element that carries annotations in the output uses the same rule.""" + schema = yaml.safe_load(SCHEMA) + metadata = {"instantiates": ["ex:Profile"], "annotations": {"reviewer": "ex:alice"}} + schema["classes"]["Thing"].pop("annotations") + if owner == "class": + schema["classes"]["Thing"].update(metadata) + elif owner == "slot": + schema["classes"]["Thing"]["attributes"] = {"value": {"range": "string", **metadata}} + elif owner == "enum": + schema["enums"]["Status"].update(metadata) + elif owner == "permissible_value": + schema["enums"]["Status"]["permissible_values"]["draft"] = {"meaning": "ex:Draft", **metadata} + else: + schema["types"]["FilePath"].update(metadata) + schema["classes"]["Thing"]["attributes"] = {"value": {"range": "FilePath"}} + graph = _graph(yaml.safe_dump(schema), generator) + assert URIRef(EX + "alice") in graph.objects(None, URIRef(EX + "reviewer")) + + +@pytest.mark.parametrize("generator", ["owl", "shacl"]) +def test_imported_metaclass(tmp_path, generator): + """A metaclass from an imported schema is found by its class URI, with its inherited slots.""" + profile = yaml.safe_load(SCHEMA) + del profile["classes"]["Thing"] + (tmp_path / "profile.yaml").write_text(yaml.safe_dump(profile), encoding="utf-8") + consumer = yaml.safe_load(SCHEMA) + consumer.update({"id": EX + "consumer", "imports": ["profile"], "classes": {"Thing": consumer["classes"]["Thing"]}}) + del consumer["types"], consumer["enums"] + source = tmp_path / "consumer.yaml" + source.write_text(yaml.safe_dump(consumer), encoding="utf-8") + assert URIRef(EX + "alice") in _graph(source, generator).objects(None, URIRef(EX + "reviewer")) + + +@pytest.mark.parametrize("generator", ["owl", "shacl"]) +def test_blank_node_label_for_a_node_type(generator): + schema = yaml.safe_load(SCHEMA) + schema["classes"]["Thing"]["annotations"]["reviewer"] = "_:anonymous" + graph = _graph(yaml.safe_dump(schema), generator) + assert isinstance(next(graph.objects(URIRef(EX + "Thing"), URIRef(EX + "reviewer"))), BNode) + + +@pytest.mark.parametrize( + "generator,tag,expected", [("owl", "other", Literal("ex:x", lang="en")), ("shacl", "ex:other", URIRef(EX + "x"))] +) +def test_undeclared_tag_keeps_the_generator_behaviour(generator, tag, expected): + """Without a declaration, gen-owl writes text and gen-shacl guesses an IRI from the colon, as before.""" + schema = yaml.safe_load(SCHEMA) + schema["classes"]["Thing"]["annotations"][tag] = "ex:x" + graph = _graph(yaml.safe_dump(schema), generator) + assert set(graph.objects(URIRef(EX + "Thing"), URIRef(EX + "other"))) == {expected} + + +@pytest.mark.parametrize("generator,owner", [("owl", "class"), ("shacl", "class"), ("owl", "permissible_value")]) +def test_conflicting_declarations_fail(generator, owner): + schema = yaml.safe_load(SCHEMA) + schema["classes"]["Other"] = {"attributes": {"reviewer": {"range": "string"}}} + metadata = {"instantiates": ["ex:Profile", "ex:Other"], "annotations": {"reviewer": "ex:alice"}} + if owner == "class": + schema["classes"]["Thing"].update(metadata) + else: + schema["enums"]["Status"]["permissible_values"]["draft"] = {"meaning": "ex:Draft", **metadata} + with pytest.raises(ValueError, match="ex:Other.*different terms"): + _graph(yaml.safe_dump(schema), generator) + + +@pytest.mark.parametrize("generator", ["owl", "shacl"]) +@pytest.mark.parametrize("value,message", [("./home.html", "relative reference"), ("_:b1", "blank node")]) +def test_value_of_the_wrong_kind_fails(generator, value, message): + """A ``uri`` value must be an absolute IRI (RDF 1.1 Concepts §3.2), not text or a blank node.""" + schema = yaml.safe_load(SCHEMA) + schema["classes"]["Thing"]["annotations"]["homepage"] = value + with pytest.raises(ValueError, match=message): + _graph(yaml.safe_dump(schema), generator) + + +def test_native_metadata_follows_the_metamodel_range(): + """``see_also`` and ``source`` are ``uriorcurie`` (IRIs); ``license`` is a string; ``created_on`` a datetime.""" + schema = yaml.safe_load(SCHEMA) + schema.update( + { + "see_also": ["ex:docs"], + "source": "https://example.org/source", + "license": "https://creativecommons.org/licenses/by/4.0/", + "created_on": "2026-10-21T10:00:00", + } + ) + graph = _graph(yaml.safe_dump(schema), "owl") + ontology = URIRef(EX + "model") + assert set(graph.objects(ontology, RDFS.seeAlso)) == {URIRef(EX + "docs")} + assert set(graph.objects(ontology, DCTERMS.source)) == {URIRef("https://example.org/source")} + assert set(graph.objects(ontology, DCTERMS.license)) == {Literal(schema["license"], lang="en")} + assert set(graph.objects(ontology, URIRef("http://purl.org/pav/createdOn"))) == { + Literal("2026-10-21T10:00:00", datatype=XSD.dateTime) + } + + +def test_native_link_uses_the_schema_prefixes_without_a_types_import(): + """A schema that imports nothing still expands its own CURIEs; the term kind comes from the metamodel.""" + schema = { + "id": EX + "bare", + "name": "bare", + "prefixes": {"ex": EX, "linkml": "https://w3id.org/linkml/"}, + "default_prefix": "ex", + "classes": {"Thing": {"see_also": ["ex:docs", "skos:Concept"]}}, + } + graph = _graph(yaml.safe_dump(schema), "owl") + assert set(graph.objects(URIRef(EX + "Thing"), RDFS.seeAlso)) == { + URIRef(EX + "docs"), + URIRef("http://www.w3.org/2004/02/skos/core#Concept"), + } + + +def test_native_link_that_is_not_a_link_is_kept_as_a_literal(caplog): + """A bare element name in ``see_also`` cannot be resolved; it is kept as text, so the output stays standard RDF.""" + schema = yaml.safe_load(SCHEMA) + schema["classes"]["Thing"]["see_also"] = ["implements"] + with warnings.catch_warnings(): + warnings.simplefilter("error", RDFCanonicalizationWarning) + output = OwlSchemaGenerator(yaml.safe_dump(schema), type_objects=False, metaclasses=False).serialize() + graph = Graph().parse(data=output, format="turtle") + assert set(graph.objects(URIRef(EX + "Thing"), RDFS.seeAlso)) == {Literal("implements")} + assert "see_also of https://example.org/Thing: 'implements' is neither an IRI nor a CURIE" in caplog.text + + +def test_documented_dcmi_example(): + """The example in the OWL documentation names its creator by IRI and holds its rights as text.""" + graph = _graph(EXAMPLE, "owl") + ontology = URIRef(EX + "model") + assert set(graph.objects(ontology, DCTERMS.creator)) == {URIRef("https://orcid.org/0000-0002-1825-0097")} + assert set(graph.objects(ontology, DCTERMS.rights)) == {Literal("Copyright 2026 The Example Team", lang="en")} + + +def test_shacl_annotations_do_not_constrain_data(): + """An annotation on a node shape is metadata; instances need no such property.""" + shapes = _graph(SCHEMA, "shacl") + shape = next(shapes.subjects(SH.targetClass, URIRef(EX + "Thing"))) + assert (shape, URIRef(EX + "reviewer"), URIRef(EX + "alice")) in shapes + data = Graph() + data.add((URIRef(EX + "instance"), RDF.type, URIRef(EX + "Thing"))) + conforms, _, report = validate(data, shacl_graph=shapes) + assert conforms, report From 87a6ef89128b85a6073ee785b94489f249f345b8 Mon Sep 17 00:00:00 2001 From: jdsika Date: Wed, 21 Oct 2026 22:39:25 +0200 Subject: [PATCH 11/17] feat(owlgen): give annotation properties their OWL role and no logical axioms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A slot that implements owl:AnnotationProperty, or whose property IRI is one of the built-in annotation properties of OWL 2 (Structural Specification §5.5), is an annotation property: it is declared as such, produces no class restriction or cardinality axiom (annotations have no logical meaning, §10.1), and its domain, datatype or class range and super-properties are emitted as annotation axioms (§10.2). Built-in properties get neither a declaration nor axioms. The predicate of a declared annotation whose metaclass slot has the role is declared too. A slot that implements two roles is an error (§5.8.1), as is an inverse or a property characteristic, a unique key, a type designator, or a rule or class-expression condition on an annotation property. A super-property must have the role of its sub-property: a sub-property axiom relates two annotation properties (§10.2.1) or two logical properties (§9.2.1, §9.3.1), so an annotation property under a data or object property, or the reverse, is an error. Signed-off-by: jdsika --- docs/generators/owl.rst | 30 ++- .../linkml/generators/common/annotations.py | 7 +- .../linkml/src/linkml/generators/owlgen.py | 143 ++++++++++++- .../test_owl_annotation_properties.py | 198 ++++++++++++++++++ 4 files changed, 363 insertions(+), 15 deletions(-) create mode 100644 tests/linkml/test_generators/test_owl_annotation_properties.py diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 5c57aa9aba..123530dcef 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -85,9 +85,33 @@ string. :language: yaml The ontology header then carries ``dcterms:creator `` -and ``dcterms:rights "Copyright 2026 The Example Team"``. ``implements: [owl:AnnotationProperty]`` -declares the two predicates as annotation properties; the restrictions that the metaclass's -own slots produce are a matter of the property role, not of the value. +and ``dcterms:rights "Copyright 2026 The Example Team"``, and the two predicates are +declared as annotation properties (see below). + +Annotation properties +^^^^^^^^^^^^^^^^^^^^^ + +A slot that ``implements`` ``owl:AnnotationProperty`` is an annotation property +(`OWL 2 §5.5 `__), and so is +a slot whose property IRI is one of the nine built-in annotation properties, such as a +slot with ``slot_uri: rdfs:label`` emitted with ``--no-use-native-uris``. An annotation +property has no logical meaning (`§10.1 `__), +so the generator emits no class restriction for it: a class that uses the slot gets no +``owl:allValuesFrom`` or cardinality axiom on it. Its domain, a datatype or class range +and its super-properties are emitted as the annotation axioms of +`§10.2 `__; a range that is a node +type such as ``nodeidentifier`` gives no range axiom. Built-in annotation properties are +not declared, as OWL 2 reserves them. + +An IRI is an annotation, an object or a data property, not two of these +(`§5.8.1 `__), so a slot +that implements two roles is an error, as is a logical use of an annotation property: an +inverse or a property characteristic, a unique key, a type designator, or a condition in +a rule or class expression. A super-property (``is_a`` or a mixin) must have the same +role as its sub-property, since a sub-property axiom relates two annotation properties +or two logical properties. Slots without the role keep their object or data property +projection, also when a metaclass declares them; LinkML then puns the annotated element +as an individual, as it does for all metadata. Enums and PermissibleValues ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/packages/linkml/src/linkml/generators/common/annotations.py b/packages/linkml/src/linkml/generators/common/annotations.py index 702ace7ef8..bc585500ad 100644 --- a/packages/linkml/src/linkml/generators/common/annotations.py +++ b/packages/linkml/src/linkml/generators/common/annotations.py @@ -35,7 +35,7 @@ True """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from rdflib import BNode, Literal, URIRef @@ -47,10 +47,11 @@ @dataclass(frozen=True) class DeclaredAnnotation: - """The RDF predicate and value of an annotation that a metaclass slot declares.""" + """The RDF predicate and value of an annotation, and the metaclass slot that declares it.""" predicate: URIRef value: URIRef | BNode | Literal + slot: SlotDefinition = field(compare=False, hash=False, repr=False) def declaring_slots(sv: SchemaView, element: Element | PermissibleValue, tag: str) -> list[SlotDefinition]: @@ -113,7 +114,7 @@ def declared_annotation( for slot in declaring_slots(sv, element, tag): term = annotation_term(sv, slot, value, language) if term is not None: - declared.add(DeclaredAnnotation(URIRef(sv.get_uri(slot, expand=True)), term)) + declared.add(DeclaredAnnotation(URIRef(sv.get_uri(slot, expand=True)), term, slot)) if len(declared) > 1: metaclasses = ", ".join(element.instantiates) raise ValueError(f"The metaclasses {metaclasses} declare annotation {tag!r} with different terms") diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index fbbe54054d..b4be78ec42 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -249,6 +249,63 @@ def _present(values: Iterable[_T | None]) -> list[_T]: # is technical and must never be language-tagged. _LANGUAGE_TAGGABLE_RANGES: ClassVar[frozenset[str]] = frozenset({"string", "ncname"}) + # The built-in annotation properties of OWL 2 (Structural Specification §5.5, Table 5). + _BUILTIN_ANNOTATION_PROPERTIES: ClassVar[frozenset[URIRef]] = frozenset( + { + RDFS.label, + RDFS.comment, + RDFS.seeAlso, + RDFS.isDefinedBy, + OWL.deprecated, + OWL.versionInfo, + OWL.priorVersion, + OWL.backwardCompatibleWith, + OWL.incompatibleWith, + } + ) + # The property roles a slot can declare with ``implements`` (OWL 2 §5.3 to §5.5). + _PROPERTY_ROLES: ClassVar[frozenset[URIRef]] = frozenset( + {OWL.AnnotationProperty, OWL.ObjectProperty, OWL.DatatypeProperty} + ) + + def _declared_property_role(self, slot: SlotDefinition) -> URIRef | None: + """Return the OWL property role that *slot* declares with ``implements``, if any. + + :raises ValueError: if the slot declares more than one role; an IRI is an + annotation, an object or a data property (OWL 2 §5.8.1) + """ + roles: set[URIRef] = set() + for value in slot.implements: + uri = self.schemaview.expand_curie(value) + if uri.startswith("owl:"): + uri = str(OWL) + uri[len("owl:") :] + if URIRef(uri) in self._PROPERTY_ROLES: + roles.add(URIRef(uri)) + if len(roles) > 1: + raise ValueError(f"Slot {slot.name!r} implements more than one OWL property role: {sorted(roles)}") + return next(iter(roles), None) + + def is_annotation_property(self, slot: SlotDefinition) -> bool: + """Whether *slot* is an OWL annotation property. + + It is one when it implements ``owl:AnnotationProperty`` or when its property + IRI is a built-in annotation property (OWL 2 §5.5), as for a slot mapped to + ``rdfs:label`` and emitted with that IRI. + """ + return ( + self._declared_property_role(slot) == OWL.AnnotationProperty + or self._prop_uri(slot) in self._BUILTIN_ANNOTATION_PROPERTIES + ) + + def _resolved_slot( + self, slot: SlotDefinition, cls: "ClassDefinition | AnonymousClassExpression | None" + ) -> SlotDefinition: + """Return *slot* with its global definition and, within a class, the class's usage applied.""" + sv = self.schemaview + if isinstance(cls, ClassDefinition) and slot.name in sv.class_slots(cls.name): + return sv.induced_slot(slot.name, cls.name) + return sv.get_slot(slot.name, attributes=True) or slot + def __post_init__(self) -> None: # Resolver must be assigned before ``super().__post_init__()`` so that # any hook the parent invokes during initialisation can safely call @@ -414,6 +471,11 @@ def add_metadata(self, e: Element | PermissibleValue, uri: URIRef) -> None: continue declared = declared_annotation(this_sv, e, k, v.value, lang) if declared is not None: + if ( + self.is_annotation_property(declared.slot) + and declared.predicate not in self._BUILTIN_ANNOTATION_PROPERTIES + ): + self.graph.add((declared.predicate, RDF.type, OWL.AnnotationProperty)) self.graph.add((uri, declared.predicate, declared.value)) continue if ":" not in k: @@ -540,12 +602,19 @@ def add_class(self, cls: ClassDefinition) -> None: # then create an axiom: (t some C) subClassOf C. type_designator = sv.get_type_designator_slot(cls.name) if type_designator: + if self.is_annotation_property(self._resolved_slot(type_designator, cls)): + raise ValueError( + f"Annotation property {type_designator.name!r} cannot be the type designator of {cls.name!r}" + ) td_prop = self._prop_uri(type_designator.name) restr = self._some_values_from(td_prop, subject_expr) self.graph.add((restr, RDFS.subClassOf, subject_expr)) # unique key constraints are mapped to OWL hasKey if cls.unique_keys: for uk in cls.unique_keys.values(): + for slot_name in uk.unique_key_slots: + if self.is_annotation_property(sv.induced_slot(slot_name, cls.name)): + raise ValueError(f"Annotation property {slot_name!r} cannot be in a unique key of {cls.name!r}") uk_props = [self._prop_uri(slot) for slot in uk.unique_key_slots] uk_props_listnode = BNode() Collection(self.graph, uk_props_listnode, uk_props) @@ -719,6 +788,11 @@ def transform_class_expression( if none_of_expr: owl_exprs.append(none_of_expr) for slot in own_slots: + if self.is_annotation_property(self._resolved_slot(slot, cls)): + if isinstance(cls, AnonymousClassExpression): + raise ValueError(f"Annotation property {slot.name!r} cannot be a condition of a class expression") + # An annotation property has no logical meaning (OWL 2 §10.1), so no restriction. + continue if slot.name: owltypes = self.slot_node_owltypes(sv.get_slot(slot.name), owning_class=cls) else: @@ -1049,7 +1123,9 @@ def add_slot(self, slot: SlotDefinition, attribute: bool = False) -> None: # Slots may be modeled as Object or Datatype Properties # if type_objects is True, then ALL slots are ObjectProperties - self.graph.add((slot_uri, RDF.type, self.slot_owl_type(slot))) + role = self.slot_owl_type(slot) + if slot_uri not in self._BUILTIN_ANNOTATION_PROPERTIES: + self.graph.add((slot_uri, RDF.type, role)) if self.metaclasses: # add metaclass which this property instantiates -- induces punning self.graph.add( @@ -1075,6 +1151,10 @@ def add_slot(self, slot: SlotDefinition, attribute: bool = False) -> None: if attribute: return + if role == OWL.AnnotationProperty: + if slot_uri not in self._BUILTIN_ANNOTATION_PROPERTIES: + self._add_annotation_property_axioms(slot, slot_uri) + return range_expr = self.transform_class_slot_expression(None, slot) if range_expr: @@ -1094,10 +1174,54 @@ def add_slot(self, slot: SlotDefinition, attribute: bool = False) -> None: if getattr(slot, metaslot, False): self.graph.add((slot_uri, RDF.type, uri)) - if slot.is_a: - self.graph.add((slot_uri, RDFS.subPropertyOf, self._prop_uri(slot.is_a))) - for mixin in slot.mixins: - self.graph.add((slot_uri, RDFS.subPropertyOf, self._prop_uri(mixin))) + self._add_super_properties(slot, slot_uri) + + def _add_super_properties(self, slot: SlotDefinition, slot_uri: URIRef) -> None: + """Emit the ``is_a`` parent and the mixins of *slot* as its super-properties. + + A sub-property axiom relates two annotation properties (OWL 2 §10.2.1) or two + properties of a logical role (§9.2.1, §9.3.1), so a super-property must have + the role *slot* has: a slot whose ``is_a`` or mixin is an annotation property + while it is none, or the reverse, is an error. + + :raises ValueError: if *slot* and a super-property differ in the annotation role + """ + is_annotation = self.is_annotation_property(slot) + for parent_name in ([slot.is_a] if slot.is_a else []) + list(slot.mixins): + parent = self.schemaview.get_slot(parent_name, attributes=True) + if parent is not None and self.is_annotation_property(parent) != is_annotation: + kinds = ( + ("an annotation property", "not one") if is_annotation else ("not an annotation property", "one") + ) + raise ValueError( + f"Slot {slot.name!r} is {kinds[0]} but its super-property {parent_name!r} is {kinds[1]}; " + "a sub-property axiom needs both to have the same role (OWL 2 §10.2.1)" + ) + self.graph.add((slot_uri, RDFS.subPropertyOf, self._prop_uri(parent_name))) + + def _add_annotation_property_axioms(self, slot: SlotDefinition, slot_uri: URIRef) -> None: + """Emit the axioms OWL 2 allows for an annotation property (§10.2). + + These are sub-properties and a domain and a range given by a named IRI; a + range that is a type gives its datatype, a node type gives no range. Inverses + and property characteristics are object property axioms (§9.2), so a slot that + declares them with the annotation role is an error. + """ + for metaslot in ("inverse", "symmetric", "asymmetric", "transitive", "reflexive", "irreflexive"): + if getattr(slot, metaslot): + raise ValueError(f"Annotation property {slot.name!r} cannot declare {metaslot}") + sv = self.schemaview + if slot.domain: + self.graph.add((slot_uri, RDFS.domain, self._class_uri(slot.domain))) + if slot.range in sv.all_types(): + term = type_term(sv, slot.range) + if not term.kind.is_node: + self.graph.add((slot_uri, RDFS.range, URIRef(term.datatype))) + elif slot.range in sv.all_classes(): + self.graph.add((slot_uri, RDFS.range, self._class_uri(slot.range))) + elif slot.range in sv.all_enums(): + self.graph.add((slot_uri, RDFS.range, self._enum_uri(slot.range))) + self._add_super_properties(slot, slot_uri) def _add_instantiates(self, element: Element | PermissibleValue, uri: URIRef) -> None: """Type the emitted schema resource itself, without typing its data instances. @@ -1719,10 +1843,11 @@ def _permissible_value_uri( def slot_owl_type(self, slot: SlotDefinition) -> URIRef: sv = self.schemaview - if slot.implements: - for t in ["owl:AnnotationProperty", "owl:ObjectProperty", "owl:DatatypeProperty"]: - if t in slot.implements: - return OWL[t.replace("owl:", "")] + role = self._declared_property_role(slot) + if role is not None: + return role + if self._prop_uri(slot) in self._BUILTIN_ANNOTATION_PROPERTIES: + return OWL.AnnotationProperty if slot.range is None: slot_range = self.schemaview.schema.default_range else: diff --git a/tests/linkml/test_generators/test_owl_annotation_properties.py b/tests/linkml/test_generators/test_owl_annotation_properties.py new file mode 100644 index 0000000000..14674149c2 --- /dev/null +++ b/tests/linkml/test_generators/test_owl_annotation_properties.py @@ -0,0 +1,198 @@ +"""gen-owl gives annotation properties their OWL role and no logical axioms. + +A slot that implements ``owl:AnnotationProperty``, or whose property IRI is one of +the built-in annotation properties of OWL 2 (Structural Specification §5.5), is +declared as such and produces no class restriction (annotation properties have no +logical meaning, §10.1). Its domain, range and super-properties are emitted as the +annotation axioms of §10.2; logical uses are errors. +""" + +from pathlib import Path + +import pytest +import yaml +from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, Graph, URIRef + +from linkml.generators.owlgen import OwlSchemaGenerator + +EX = "https://example.org/m/" +EXAMPLE = Path(__file__).resolve().parents[3] / "docs/examples/owl-dcmi-agent-annotations.yaml" + +SCHEMA = """ +id: https://example.org/m +name: m +prefixes: + ex: https://example.org/m/ + linkml: https://w3id.org/linkml/ + owl: http://www.w3.org/2002/07/owl# + rdfs: http://www.w3.org/2000/01/rdf-schema# + dcterms: http://purl.org/dc/terms/ +imports: [linkml:types] +default_prefix: ex +default_range: string +slots: + comment: + slot_uri: rdfs:comment + # an annotation property with native URIs too, so that note may refine it + implements: [owl:AnnotationProperty] + note: + slot_uri: dcterms:description + range: string + implements: [owl:AnnotationProperty] + is_a: comment + domain: Thing + ref: + range: nodeidentifier + implements: [owl:AnnotationProperty] +classes: + Thing: + slots: [comment, note, ref] + attributes: + name: + slot_uri: rdfs:label + plain: + range: string +""" + + +def _graph(source: str | Path, **kwargs) -> Graph: + """The generated ontology, parsed back from Turtle.""" + generator = OwlSchemaGenerator(source, type_objects=False, metaclasses=False, **kwargs) + return Graph().parse(data=generator.serialize(), format="turtle") + + +def _restrictions(graph: Graph, prop: URIRef) -> int: + return len(list(graph.subjects(OWL.onProperty, prop))) + + +@pytest.mark.parametrize("prop", ["note", "ref"]) +def test_declared_annotation_property_has_no_restrictions(prop): + """``implements: [owl:AnnotationProperty]`` gives the declaration and suppresses the restrictions.""" + graph = _graph(SCHEMA) + assert set(graph.objects(URIRef(EX + prop), RDF.type)) == {OWL.AnnotationProperty} + assert _restrictions(graph, URIRef(EX + prop)) == 0 + assert set(graph.objects(URIRef(EX + "plain"), RDF.type)) == {OWL.DatatypeProperty} + assert _restrictions(graph, URIRef(EX + "plain")) > 0 + + +@pytest.mark.parametrize( + "native,expected", + [ + pytest.param(True, {URIRef(EX + "name"): OWL.DatatypeProperty}, id="native"), + pytest.param(False, {RDFS.label: None, RDFS.comment: None}, id="mapped"), + ], +) +def test_builtin_annotation_property_iri(native, expected): + """A slot emitted with a built-in annotation property IRI gets no declaration and no restriction. + + With native URIs the local property is an ordinary data property, as before. + """ + graph = _graph(SCHEMA, use_native_uris=native) + for prop, role in expected.items(): + assert set(graph.objects(prop, RDF.type)) == ({role} if role else set()) + assert (_restrictions(graph, prop) > 0) is (role is not None) + if role is None: + assert not set(graph.objects(prop, RDFS.range)) | set(graph.objects(prop, RDFS.domain)) + + +def test_annotation_property_axioms(): + """Domain, a datatype range and super-properties are kept; a node type gives no range (OWL 2 §10.2).""" + graph = _graph(SCHEMA, use_native_uris=False) + note = URIRef("http://purl.org/dc/terms/description") + assert set(graph.objects(note, RDFS.domain)) == {URIRef(EX + "Thing")} + assert set(graph.objects(note, RDFS.range)) == {XSD.string} + assert set(graph.objects(note, RDFS.subPropertyOf)) == {RDFS.comment} + assert set(graph.objects(URIRef(EX + "ref"), RDFS.range)) == set() + + +def test_declared_annotation_predicate_is_declared(): + """A metaclass slot with the annotation role declares the predicate its annotations use.""" + graph = _graph(EXAMPLE) + ontology = URIRef("https://example.org/model") + assert set(graph.objects(ontology, DCTERMS.creator)) == {URIRef("https://orcid.org/0000-0002-1825-0097")} + for predicate in (DCTERMS.creator, DCTERMS.rights): + assert (predicate, RDF.type, OWL.AnnotationProperty) in graph + assert _restrictions(graph, predicate) == 0 + assert not any(obj in {OWL.AnnotationProperty} for obj in graph.objects(None, OWL.onProperty)) + + +def test_contradictory_roles_fail(): + schema = yaml.safe_load(SCHEMA) + schema["slots"]["note"]["implements"] = ["owl:AnnotationProperty", "owl:DatatypeProperty"] + with pytest.raises(ValueError, match="more than one OWL property role"): + _graph(yaml.safe_dump(schema)) + + +@pytest.mark.parametrize( + "change,message", + [ + pytest.param({"slots": {"note": {"symmetric": True}}}, "cannot declare symmetric", id="characteristic"), + pytest.param({"slots": {"note": {"inverse": "comment"}}}, "cannot declare inverse", id="inverse"), + pytest.param( + {"classes": {"Thing": {"unique_keys": {"k": {"unique_key_slots": ["note"]}}}}}, "unique key", id="key" + ), + pytest.param( + {"slots": {"ref": {"designates_type": True, "range": "string"}}}, "type designator", id="designator" + ), + pytest.param( + { + "classes": { + "Thing": {"rules": [{"preconditions": {"slot_conditions": {"note": {"equals_string": "x"}}}}]} + } + }, + "condition of a class expression", + id="rule", + ), + ], +) +def test_logical_use_of_annotation_property_fails(change, message): + """An annotation property cannot carry object property axioms or appear in a class expression.""" + schema = yaml.safe_load(SCHEMA) + for section, elements in change.items(): + for name, fields in elements.items(): + schema[section][name].update(fields) + with pytest.raises(ValueError, match=message): + _graph(yaml.safe_dump(schema)) + + +@pytest.mark.parametrize( + "child,parent,message", + [ + pytest.param( + {"implements": ["owl:AnnotationProperty"]}, + {}, + "is an annotation property but its super-property 'base' is not one", + id="annotation-under-data", + ), + pytest.param( + {}, + {"implements": ["owl:AnnotationProperty"]}, + "is not an annotation property but its super-property 'base' is one", + id="data-under-annotation", + ), + ], +) +@pytest.mark.parametrize("link", ["is_a", "mixins"]) +def test_super_property_with_another_role_fails(child, parent, message, link): + """A sub-property axiom relates properties of one role (OWL 2 §9.3.1, §10.2.1).""" + schema = yaml.safe_load(SCHEMA) + schema["slots"]["base"] = {"range": "string", **parent} + schema["slots"]["derived"] = {"range": "string", **child, link: "base" if link == "is_a" else ["base"]} + schema["classes"]["Thing"]["slots"].append("derived") + with pytest.raises(ValueError, match=message): + _graph(yaml.safe_dump(schema)) + + +@pytest.mark.parametrize( + "native,sub,sup", + [ + pytest.param(True, URIRef(EX + "note"), URIRef(EX + "comment"), id="native"), + pytest.param(False, URIRef("http://purl.org/dc/terms/description"), RDFS.comment, id="mapped"), + ], +) +def test_super_property_with_the_same_role_is_emitted(native, sub, sup): + """An annotation property under an annotation property keeps its super-property, + as ``note`` under ``comment``, which is ``rdfs:comment`` with mapped URIs.""" + graph = _graph(SCHEMA, use_native_uris=native) + assert (sub, RDFS.subPropertyOf, sup) in graph + assert (sub, RDF.type, OWL.AnnotationProperty) in graph From 2f64325a279b5e739a33e956f75d3a6e3c68c4f5 Mon Sep 17 00:00:00 2001 From: jdsika Date: Thu, 15 Oct 2026 18:16:05 +0200 Subject: [PATCH 12/17] fix(contextgen): declare JSON-LD 1.1 in generated contexts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generated contexts use JSON-LD 1.1 constructs: scoped contexts on enum slots and class-level range overrides, and @prefix. A JSON-LD 1.0 processor ignores them silently, so datatype coercions are lost. Emit "@version": 1.1 first in every generated context. It makes a JSON-LD 1.0 processor stop (JSON-LD 1.1 §1.7), and a JSON-LD 1.1 processor then requires 1.1 processing (JSON-LD 1.1 API §4.1.2). gen-jsonld embeds the declaration with the generated context. Signed-off-by: jdsika --- docs/generators/jsonld-context.rst | 11 +++ .../personinfo-semantic.context.jsonld | 1 + .../src/linkml/generators/jsonldcontextgen.py | 6 +- .../__snapshots__/annotations.context.jsonld | 1 + .../test_base/__snapshots__/annotations.json | 1 + .../__snapshots__/extensions.context.jsonld | 1 + .../test_base/__snapshots__/extensions.json | 1 + .../__snapshots__/mappings.context.jsonld | 1 + .../test_base/__snapshots__/mappings.json | 1 + .../__snapshots__/meta.context.jsonld | 1 + .../linkml/test_base/__snapshots__/meta.json | 1 + .../__snapshots__/types.context.jsonld | 1 + .../linkml/test_base/__snapshots__/types.json | 1 + .../__snapshots__/biolink.context.jsonld | 1 + .../__snapshots__/biolink.json | 1 + .../__snapshots__/class_uri_prefix.jsonld | 1 + .../context_inlined_external_types.jsonld | 1 + .../__snapshots__/kitchen_sink.jsonld | 1 + .../no_default_namespace_prefix.jsonld | 1 + .../test_generators/test_jsonldcontextgen.py | 71 +++++++++++++++++++ .../__snapshots__/issue_344_context.json | 1 + .../__snapshots__/issue_368.context.jsonld | 1 + .../__snapshots__/linkml_issue_384.other.txt | 1 + tests/linkml/test_issues/test_issue_332.py | 1 + .../gencontext/meta.context.jsonld | 1 + .../gencontext/meta_context.jsonld | 1 + .../simple_uri_test.merge.context.jsonld | 1 + ...est.merge.flatprefixes_only.context.jsonld | 1 + ...e_uri_test.merge.model_only.context.jsonld | 1 + ...ri_test.merge.prefixes_only.context.jsonld | 1 + .../simple_uri_test.no_merge.context.jsonld | 1 + ....no_merge.flatprefixes_only.context.jsonld | 1 + ...ri_test.no_merge.model_only.context.jsonld | 1 + ...test.no_merge.prefixes_only.context.jsonld | 1 + .../__snapshots__/gencontext/uri_tests.jsonld | 1 + .../includes/simple_types.context.jsonld | 1 + .../genjsonld/simple_slots.context.jsonld | 1 + .../genjsonld/simple_uri_test.context.jsonld | 1 + .../genjsonld/simple_uri_test.jsonld | 1 + tests/linkml/test_scripts/test_gen_jsonld.py | 3 +- .../test_utils/__snapshots__/uriandcurie.json | 1 + .../__snapshots__/uriandcurie.jsonld | 1 + 42 files changed, 126 insertions(+), 3 deletions(-) diff --git a/docs/generators/jsonld-context.rst b/docs/generators/jsonld-context.rst index 18cec34644..362732ff41 100644 --- a/docs/generators/jsonld-context.rst +++ b/docs/generators/jsonld-context.rst @@ -53,6 +53,17 @@ derived from ``string`` with ``uri: xsd:anyURI`` keeps URI references as ``xsd:anyURI`` literals (see :ref:`links-and-uri-references`). + +JSON-LD version +^^^^^^^^^^^^^^^ + +Generated contexts start with ``"@version": 1.1``. They use JSON-LD 1.1 +constructs, such as scoped contexts for enum slots and class-level range +overrides, and ``@prefix`` (see below), which a JSON-LD 1.0 processor would +silently ignore. The number ``1.1`` makes such a processor stop instead +(`JSON-LD 1.1 §1.7 `__), and selects +JSON-LD 1.1 processing in processors that support both. + CURIE usage ^^^^^^^^^^^ diff --git a/examples/tutorial/tutorial04/personinfo-semantic.context.jsonld b/examples/tutorial/tutorial04/personinfo-semantic.context.jsonld index 69b6460d0c..48d1c0600e 100644 --- a/examples/tutorial/tutorial04/personinfo-semantic.context.jsonld +++ b/examples/tutorial/tutorial04/personinfo-semantic.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "ORCID": "https://orcid.org/", "linkml": "https://w3id.org/linkml/", diff --git a/packages/linkml/src/linkml/generators/jsonldcontextgen.py b/packages/linkml/src/linkml/generators/jsonldcontextgen.py index 4916fa07c3..60a41d0c34 100644 --- a/packages/linkml/src/linkml/generators/jsonldcontextgen.py +++ b/packages/linkml/src/linkml/generators/jsonldcontextgen.py @@ -177,7 +177,11 @@ def end_schema( comments.generation_date = self.schema.generation_date comments.source = self.schema.source_file context.comments = comments - context_content = {"xsd": "http://www.w3.org/2001/XMLSchema#"} + # The generated contexts use JSON-LD 1.1 constructs (scoped contexts, ``@prefix``) + # that a JSON-LD 1.0 processor silently ignores. ``"@version": 1.1`` makes such a + # processor stop instead (JSON-LD 1.1 §1.7), and a JSON-LD 1.1 processor then + # requires 1.1 processing (JSON-LD 1.1 API §4.1.2, step 5.5). + context_content = {"@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#"} if base: base = str(base) if "://" not in base: diff --git a/tests/linkml/test_base/__snapshots__/annotations.context.jsonld b/tests/linkml/test_base/__snapshots__/annotations.context.jsonld index 84d8f7877d..c8f5bd1d4b 100644 --- a/tests/linkml/test_base/__snapshots__/annotations.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/annotations.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "@vocab": "https://w3id.org/linkml/", diff --git a/tests/linkml/test_base/__snapshots__/annotations.json b/tests/linkml/test_base/__snapshots__/annotations.json index bffa4074a3..055b05342b 100644 --- a/tests/linkml/test_base/__snapshots__/annotations.json +++ b/tests/linkml/test_base/__snapshots__/annotations.json @@ -495,6 +495,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "@vocab": "https://w3id.org/linkml/", diff --git a/tests/linkml/test_base/__snapshots__/extensions.context.jsonld b/tests/linkml/test_base/__snapshots__/extensions.context.jsonld index 730a52a220..d2e9c08dff 100644 --- a/tests/linkml/test_base/__snapshots__/extensions.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/extensions.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "@vocab": "https://w3id.org/linkml/", diff --git a/tests/linkml/test_base/__snapshots__/extensions.json b/tests/linkml/test_base/__snapshots__/extensions.json index 6f7d53ac3a..5dce374763 100644 --- a/tests/linkml/test_base/__snapshots__/extensions.json +++ b/tests/linkml/test_base/__snapshots__/extensions.json @@ -438,6 +438,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "@vocab": "https://w3id.org/linkml/", diff --git a/tests/linkml/test_base/__snapshots__/mappings.context.jsonld b/tests/linkml/test_base/__snapshots__/mappings.context.jsonld index e14c87bb34..ffb0578dc4 100644 --- a/tests/linkml/test_base/__snapshots__/mappings.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/mappings.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "IAO": { "@id": "http://purl.obolibrary.org/obo/IAO_", diff --git a/tests/linkml/test_base/__snapshots__/mappings.json b/tests/linkml/test_base/__snapshots__/mappings.json index 02354810c1..4f1decb6dc 100644 --- a/tests/linkml/test_base/__snapshots__/mappings.json +++ b/tests/linkml/test_base/__snapshots__/mappings.json @@ -474,6 +474,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "IAO": { "@id": "http://purl.obolibrary.org/obo/IAO_", diff --git a/tests/linkml/test_base/__snapshots__/meta.context.jsonld b/tests/linkml/test_base/__snapshots__/meta.context.jsonld index d51c9c8bf3..f31068d31e 100644 --- a/tests/linkml/test_base/__snapshots__/meta.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/meta.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "IAO": { "@id": "http://purl.obolibrary.org/obo/IAO_", diff --git a/tests/linkml/test_base/__snapshots__/meta.json b/tests/linkml/test_base/__snapshots__/meta.json index 98b6fe7d0d..be3ec4b40e 100644 --- a/tests/linkml/test_base/__snapshots__/meta.json +++ b/tests/linkml/test_base/__snapshots__/meta.json @@ -8910,6 +8910,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "IAO": { "@id": "http://purl.obolibrary.org/obo/IAO_", diff --git a/tests/linkml/test_base/__snapshots__/types.context.jsonld b/tests/linkml/test_base/__snapshots__/types.context.jsonld index 7a47b4e0fb..00fd3e87fd 100644 --- a/tests/linkml/test_base/__snapshots__/types.context.jsonld +++ b/tests/linkml/test_base/__snapshots__/types.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "schema": "http://schema.org/", diff --git a/tests/linkml/test_base/__snapshots__/types.json b/tests/linkml/test_base/__snapshots__/types.json index de04d796ac..15a9d4054a 100644 --- a/tests/linkml/test_base/__snapshots__/types.json +++ b/tests/linkml/test_base/__snapshots__/types.json @@ -324,6 +324,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "schema": "http://schema.org/", diff --git a/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld b/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld index 16621443df..bae530473e 100644 --- a/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld +++ b/tests/linkml/test_biolink_model/__snapshots__/biolink.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "AGRKB": "https://www.alliancegenome.org/", "APO": { diff --git a/tests/linkml/test_biolink_model/__snapshots__/biolink.json b/tests/linkml/test_biolink_model/__snapshots__/biolink.json index 02bc375a8f..49a702e56a 100644 --- a/tests/linkml/test_biolink_model/__snapshots__/biolink.json +++ b/tests/linkml/test_biolink_model/__snapshots__/biolink.json @@ -35995,6 +35995,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "AGRKB": "https://www.alliancegenome.org/", "APO": { diff --git a/tests/linkml/test_generators/__snapshots__/class_uri_prefix.jsonld b/tests/linkml/test_generators/__snapshots__/class_uri_prefix.jsonld index da825237ae..6d5521dabf 100644 --- a/tests/linkml/test_generators/__snapshots__/class_uri_prefix.jsonld +++ b/tests/linkml/test_generators/__snapshots__/class_uri_prefix.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", diff --git a/tests/linkml/test_generators/__snapshots__/context_inlined_external_types.jsonld b/tests/linkml/test_generators/__snapshots__/context_inlined_external_types.jsonld index aa4f4f3693..217387a266 100644 --- a/tests/linkml/test_generators/__snapshots__/context_inlined_external_types.jsonld +++ b/tests/linkml/test_generators/__snapshots__/context_inlined_external_types.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "cred": "https://www.w3.org/2018/credentials#", "ex": "https://example.org/external_type#", diff --git a/tests/linkml/test_generators/__snapshots__/kitchen_sink.jsonld b/tests/linkml/test_generators/__snapshots__/kitchen_sink.jsonld index 299abdf340..836133a929 100644 --- a/tests/linkml/test_generators/__snapshots__/kitchen_sink.jsonld +++ b/tests/linkml/test_generators/__snapshots__/kitchen_sink.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "A": "http://example.org/activities/", "BFO": { diff --git a/tests/linkml/test_generators/__snapshots__/no_default_namespace_prefix.jsonld b/tests/linkml/test_generators/__snapshots__/no_default_namespace_prefix.jsonld index 73c16ac8f1..f8ddb2bf84 100644 --- a/tests/linkml/test_generators/__snapshots__/no_default_namespace_prefix.jsonld +++ b/tests/linkml/test_generators/__snapshots__/no_default_namespace_prefix.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "linkml": "https://w3id.org/linkml/", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", diff --git a/tests/linkml/test_generators/test_jsonldcontextgen.py b/tests/linkml/test_generators/test_jsonldcontextgen.py index 6c56ab3fc5..154b4c1f6a 100644 --- a/tests/linkml/test_generators/test_jsonldcontextgen.py +++ b/tests/linkml/test_generators/test_jsonldcontextgen.py @@ -1548,3 +1548,74 @@ def test_identifier_context_round_trips_to_rdf(tmp_path, use_curies): ex = "https://example.org/identifier-test/" assert set(graph) == {(URIRef(ex + "thing1"), URIRef(ex + "name"), Literal("n"))} + + +VERSION_SCHEMA = """ +id: https://example.org/version +name: version +prefixes: + ex: https://example.org/version/ + linkml: https://w3id.org/linkml/ +default_prefix: ex +default_range: string +imports: [linkml:types] +slots: + value: + slot_uri: ex:value +classes: + Quantity: + class_uri: ex:Quantity + slots: [value] + slot_usage: + value: + range: float + Token: + class_uri: ex:Token + slots: [value] + Thing: + class_uri: ex:Thing + tree_root: true + attributes: + reading: {range: Quantity, inlined: true} + label: {range: Token, inlined: true} +""" + + +def test_context_declares_jsonld_1_1(tmp_path): + """Generated contexts, also as embedded by gen-jsonld, declare ``"@version": 1.1`` first. + + JSON-LD 1.1 §1.7: the value is the number 1.1, which makes a JSON-LD 1.0 + processor stop instead of ignoring the 1.1 constructs. + """ + schema = tmp_path / "version.yaml" + schema.write_text(VERSION_SCHEMA, encoding="utf-8") + + context = json.loads(ContextGenerator(str(schema)).serialize())["@context"] + assert next(iter(context)) == "@version" + assert context["@version"] == 1.1 and isinstance(context["@version"], float) + + # gen-jsonld embeds a prefix-only generated context first, then ``@base`` on its own + embedded = json.loads(JSONLDGenerator(str(schema)).serialize())["@context"] + assert next(c for c in embedded if isinstance(c, dict))["@version"] == 1.1 + + +@pytest.mark.parametrize("version", [None, 1.0], ids=["processor-default", "json-ld-1.0-requested"]) +def test_type_scoped_coercion_survives_json_ld_1_0_processing(tmp_path, version): + """A class-level range override reaches the data even when the caller asks rdflib for JSON-LD 1.0. + + The override is a type-scoped context, a JSON-LD 1.1 construct. rdflib in + JSON-LD 1.0 mode ignores it, so ``1.5`` would become ``xsd:double``. Asked + for 1.0, rdflib switches to 1.1 when the context declares ``@version``; a + conformant JSON-LD 1.1 processor stops with a processing mode conflict + instead (JSON-LD 1.1 API §4.1.2). Either way the coercion is not silently lost. + """ + schema = tmp_path / "version.yaml" + schema.write_text(VERSION_SCHEMA, encoding="utf-8") + context = json.loads(ContextGenerator(str(schema)).serialize()) + doc = {**context, "@type": "Thing", "reading": {"@type": "Quantity", "value": 1.5}, "label": {"value": "x"}} + + options = {} if version is None else {"version": version} + graph = Graph().parse(data=json.dumps(doc), format="json-ld", **options) + + values = {o.datatype for o in graph.objects(None, URIRef("https://example.org/version/value"))} + assert values == {XSD.float, None} diff --git a/tests/linkml/test_issues/__snapshots__/issue_344_context.json b/tests/linkml/test_issues/__snapshots__/issue_344_context.json index 67f62a0b81..e5d0f534be 100644 --- a/tests/linkml/test_issues/__snapshots__/issue_344_context.json +++ b/tests/linkml/test_issues/__snapshots__/issue_344_context.json @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "GO": { "@id": "http://purl.obolibrary.org/obo/GO_", diff --git a/tests/linkml/test_issues/__snapshots__/issue_368.context.jsonld b/tests/linkml/test_issues/__snapshots__/issue_368.context.jsonld index 64d29f16df..ab40384ceb 100644 --- a/tests/linkml/test_issues/__snapshots__/issue_368.context.jsonld +++ b/tests/linkml/test_issues/__snapshots__/issue_368.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bms": "https://microbiomedata/schema/", "linkml": "https://w3id.org/linkml/", diff --git a/tests/linkml/test_issues/__snapshots__/linkml_issue_384.other.txt b/tests/linkml/test_issues/__snapshots__/linkml_issue_384.other.txt index 1755c9215a..d159a418b7 100644 --- a/tests/linkml/test_issues/__snapshots__/linkml_issue_384.other.txt +++ b/tests/linkml/test_issues/__snapshots__/linkml_issue_384.other.txt @@ -540,6 +540,7 @@ generation_date: '2000-01-01T00:00:00' # --- linkml_issue_384.context.jsonld --- { "@context": { + "@version": 1.1, "@vocab": "https://w3id.org/linkml/examples/personinfo/", "GeoAge": { "@id": "GeoAge" diff --git a/tests/linkml/test_issues/test_issue_332.py b/tests/linkml/test_issues/test_issue_332.py index a09b6e45e3..16cc6e09b9 100644 --- a/tests/linkml/test_issues/test_issue_332.py +++ b/tests/linkml/test_issues/test_issue_332.py @@ -11,6 +11,7 @@ def test_context(input_path): [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "meta": "https://w3id.org/linkml/", "test14": "https://example.com/test14/", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/meta.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/meta.context.jsonld index ce935718c5..c08c2b5c55 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/meta.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/meta.context.jsonld @@ -5,6 +5,7 @@ "source": "kitchen_sink.yaml" }, "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "A": "http://example.org/activities/", "BFO": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/meta_context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/meta_context.jsonld index ba1f5c389a..d893742c6b 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/meta_context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/meta_context.jsonld @@ -5,6 +5,7 @@ "source": "kitchen_sink.yaml" }, "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "A": "http://example.org/activities/", "BFO": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld index 3092b3dd07..2d36420bd9 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": { "@id": "http://samples.r.us/bar_", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.flatprefixes_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.flatprefixes_only.context.jsonld index 394f05ac47..db2bd4cec4 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.flatprefixes_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.flatprefixes_only.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": "http://samples.r.us/bar_", "foo": "http://samples.r.us/foo#", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld index 9bd2b80f40..a7878caff5 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.model_only.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.prefixes_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.prefixes_only.context.jsonld index e982cc2fda..cfb713c120 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.prefixes_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.merge.prefixes_only.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": { "@id": "http://samples.r.us/bar_", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld index 3092b3dd07..2d36420bd9 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": { "@id": "http://samples.r.us/bar_", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.flatprefixes_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.flatprefixes_only.context.jsonld index 394f05ac47..db2bd4cec4 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.flatprefixes_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.flatprefixes_only.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": "http://samples.r.us/bar_", "foo": "http://samples.r.us/foo#", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld index 9bd2b80f40..a7878caff5 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.model_only.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "@vocab": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_uri_test/", "tag": { diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.prefixes_only.context.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.prefixes_only.context.jsonld index e982cc2fda..cfb713c120 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.prefixes_only.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/simple_uri_test.no_merge.prefixes_only.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": { "@id": "http://samples.r.us/bar_", diff --git a/tests/linkml/test_scripts/__snapshots__/gencontext/uri_tests.jsonld b/tests/linkml/test_scripts/__snapshots__/gencontext/uri_tests.jsonld index 100a390955..73ac8e2c15 100644 --- a/tests/linkml/test_scripts/__snapshots__/gencontext/uri_tests.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/gencontext/uri_tests.jsonld @@ -5,6 +5,7 @@ "source": "uri_tests.yaml" }, "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "OIO": "http://www.geneontology.org/formats/oboInOwl#", "linkml": "https://w3id.org/linkml/", diff --git a/tests/linkml/test_scripts/__snapshots__/genjsonld/includes/simple_types.context.jsonld b/tests/linkml/test_scripts/__snapshots__/genjsonld/includes/simple_types.context.jsonld index ee59dab3a1..6b19235fb9 100644 --- a/tests/linkml/test_scripts/__snapshots__/genjsonld/includes/simple_types.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/genjsonld/includes/simple_types.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "simple_types": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/includes/simple_types/", "slotpfx": "http://samples.r.us/slot/", diff --git a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_slots.context.jsonld b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_slots.context.jsonld index 90118f1051..f54a5a5272 100644 --- a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_slots.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_slots.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "foo": "http://samples.r.us/foo#", "simple_slots": "https://raw.githubusercontent.com/linkml/linkml/main/tests/test_scripts/input/simple_slots/", diff --git a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld index ad8978b81f..b94cdbfb73 100644 --- a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.context.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": { "@id": "http://samples.r.us/bar_", diff --git a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.jsonld b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.jsonld index 97c502567c..12eafd5d03 100644 --- a/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.jsonld +++ b/tests/linkml/test_scripts/__snapshots__/genjsonld/simple_uri_test.jsonld @@ -492,6 +492,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "bar": { "@id": "http://samples.r.us/bar_", diff --git a/tests/linkml/test_scripts/test_gen_jsonld.py b/tests/linkml/test_scripts/test_gen_jsonld.py index c1b7d5c1f2..6d149f2fcf 100644 --- a/tests/linkml/test_scripts/test_gen_jsonld.py +++ b/tests/linkml/test_scripts/test_gen_jsonld.py @@ -221,5 +221,4 @@ def test_context_kwargs(input_path, prefixes: bool, flatprefixes: bool): assert isinstance(pref_dict["bar"], dict) assert pref_dict["bar"]["@prefix"] is True else: - assert len(pref_dict) == 1 - assert "xsd" in pref_dict + assert pref_dict.keys() == {"@version", "xsd"} diff --git a/tests/linkml/test_utils/__snapshots__/uriandcurie.json b/tests/linkml/test_utils/__snapshots__/uriandcurie.json index 97198624f6..32ea0a7413 100644 --- a/tests/linkml/test_utils/__snapshots__/uriandcurie.json +++ b/tests/linkml/test_utils/__snapshots__/uriandcurie.json @@ -181,6 +181,7 @@ "@context": [ "https://w3id.org/linkml/meta.context.jsonld", { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "m": { "@id": "http://example.org/test/uriandcurie", diff --git a/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld b/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld index f87f92da56..5aa031de7f 100644 --- a/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld +++ b/tests/linkml/test_utils/__snapshots__/uriandcurie.jsonld @@ -1,5 +1,6 @@ { "@context": { + "@version": 1.1, "xsd": "http://www.w3.org/2001/XMLSchema#", "m": { "@id": "http://example.org/test/uriandcurie", From 4a11a1e720131ba456139bfc79a002df3a535f12 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Fri, 11 Sep 2026 16:34:30 +0200 Subject: [PATCH 13/17] docs(owl): document deterministic serialization and --diff-stable Describe in the gen-owl documentation what RDFC-1.0 canonicalization guarantees for standard RDF, why its sequential blank-node numbering can still produce noisy diffs, and that --diff-stable reduces that churn without guaranteeing minimal diffs. Note that the option also exists on gen-rdf, gen-shacl and gen-shex --format rdf, and that graphs that are not standard RDF, such as gen-shacl --include-annotations output with a non-CURIE annotation tag, are serialized by rdflib with a warning, where --diff-stable has no effect. Signed-off-by: jdsika --- docs/generators/owl.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 123530dcef..9ff3867999 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -405,6 +405,28 @@ Other examples translation of Biolink schema to OWL +Deterministic output +^^^^^^^^^^^^^^^^^^^^ + +``gen-owl`` canonicalizes the graph with `RDFC-1.0 `_ +before serializing it. For standard RDF, repeated runs over the same schema +therefore produce byte-identical Turtle, and checked-in artifacts do not change +between runs. + +RDFC-1.0 numbers blank nodes in canonical order (``_:c14n0``, ``_:c14n1``, ...), +so adding a statement can renumber unrelated blank nodes. ``--diff-stable`` +derives labels from blank-node neighbourhoods instead, which reduces this churn +but does not guarantee minimal diffs; see :ref:`rdf-in-version-control`, which +also names the optional dependency it needs. The option is also available on +``gen-rdf``, ``gen-shacl`` and ``gen-shex --format rdf``. + +Graphs that are not standard RDF, with a relative IRI or a literal as a +predicate, cannot be canonicalized under RDFC-1.0; ``gen-shacl +--include-annotations`` writes such a predicate for an annotation whose tag is +not a CURIE. They are serialized by rdflib instead, with an +``RDFCanonicalizationWarning``, and ``--diff-stable`` has no effect on them. + + Docs ---- From fefca1804dabffbbc807bf9413d375c09f193e31 Mon Sep 17 00:00:00 2001 From: jdsika Date: Fri, 2 Oct 2026 13:22:53 +0200 Subject: [PATCH 14/17] feat(generators): add --normalize-prefixes flag for well-known prefix names Add an opt-in --normalize-prefixes flag to the OWL, SHACL, JSON-LD context and JSON-LD generators that renames non-standard prefix aliases to their well-known names. The names come from the prefixmaps linked_data context (curated) layered over its merged context, and where rdflib binds a namespace by default, from rdflib, which serializes the output (geo, schema, wgs where prefixmaps says geosparql, sc, WGS84). - Both http://schema.org/ and https://schema.org/ map to 'schema'. - OWL and SHACL share normalize_graph_prefixes(): phase 1 renames schema-declared prefixes, phase 2 cleans up runtime-injected bindings without overwriting HTTPS bindings with HTTP variants. - A standard name already declared for another namespace is skipped with a warning. The flag defaults to off, preserving existing behaviour, and only the generators that honour it offer it: gen-rdf writes the schema's class_uri and prefix declarations as literal data and gen-shex's RDF uses rdflib's names already, so neither emits an alias to rename. The documentation for the flag is in docs/generators/owl.rst. test_csv_limit set csv.field_size_limit(20) without restoring it, so any later test on the same worker that read prefixmaps' CSV data failed; it now restores the limit. Signed-off-by: jdsika --- docs/generators/owl.rst | 25 + packages/linkml/pyproject.toml | 9 +- .../src/linkml/generators/jsonldcontextgen.py | 86 ++- .../linkml/src/linkml/generators/jsonldgen.py | 9 +- .../linkml/src/linkml/generators/owlgen.py | 11 +- .../linkml/src/linkml/generators/shaclgen.py | 11 +- packages/linkml/src/linkml/utils/generator.py | 145 ++++- tests/linkml/test_data/test_sqlite.py | 10 +- .../test_generators/test_jsonldcontextgen.py | 115 ++++ .../test_normalize_prefixes.py | 589 ++++++++++++++++++ uv.lock | 10 +- 11 files changed, 1005 insertions(+), 15 deletions(-) create mode 100644 tests/linkml/test_generators/test_normalize_prefixes.py diff --git a/docs/generators/owl.rst b/docs/generators/owl.rst index 9ff3867999..513c77aa6b 100644 --- a/docs/generators/owl.rst +++ b/docs/generators/owl.rst @@ -113,6 +113,31 @@ or two logical properties. Slots without the role keep their object or data prop projection, also when a metaclass declares them; LinkML then puns the annotated element as an individual, as it does for all metadata. + +Prefix normalization +^^^^^^^^^^^^^^^^^^^^ + +Schemas sometimes declare non-standard aliases for well-known namespaces +(e.g. ``sh1:`` for the SHACL namespace, or a versioned alias for ``skos:``). +By default these aliases are carried through into the generated artifact. + +Use ``--normalize-prefixes`` to remap declared prefixes whose namespace IRI +matches a well-known vocabulary to that vocabulary's conventional name in the +output (``owl``, ``rdf``, ``rdfs``, ``skos``, ``sh``, ``xsd``, ...): + +.. code:: bash + + gen-owl --normalize-prefixes schema.yaml + +The names come from `prefixmaps `_, its +curated ``linked_data`` context over its broader ``merged`` one, and where rdflib +binds a namespace by default, from rdflib, which serializes the output. They can +therefore change with the installed prefixmaps and rdflib versions. Namespace +IRIs that none of them names are left untouched, and a well-known name that the +schema declares for another namespace is kept, with a warning. The option is +also available on ``gen-shacl``, ``gen-jsonld-context`` and ``gen-jsonld``. + + Enums and PermissibleValues ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/packages/linkml/pyproject.toml b/packages/linkml/pyproject.toml index febf545983..e8ce90f30c 100644 --- a/packages/linkml/pyproject.toml +++ b/packages/linkml/pyproject.toml @@ -50,7 +50,10 @@ dependencies = [ # Specifier syntax: https://peps.python.org/pep-0631/ "openpyxl", "parse", "prefixcommons >= 0.1.7", - "prefixmaps >= 0.2.2", + # TODO(prefixmaps-0.2.8): Replace git pin with "prefixmaps >= 0.2.8" once released, + # then remove [tool.hatch.metadata] allow-direct-references and regenerate uv.lock. + # Tracked in: https://github.com/linkml/prefixmaps/issues/82 + "prefixmaps @ git+https://github.com/linkml/prefixmaps@75435150a1b31760b9780af2b64a265943a9b263", "pydantic>=2.14.1,<3.0.0", "pyjsg>=0.12.5", "pyshex >= 0.9.0", @@ -211,6 +214,10 @@ vcs = "git" style = "pep440" fallback-version = "0.0.0" +[tool.hatch.metadata] +# TODO(prefixmaps-0.2.8): Remove this section once the git pin is replaced with >= 0.2.8 +allow-direct-references = true + [tool.hatch.version] source = "uv-dynamic-versioning" diff --git a/packages/linkml/src/linkml/generators/jsonldcontextgen.py b/packages/linkml/src/linkml/generators/jsonldcontextgen.py index 60a41d0c34..de26a9f4f0 100644 --- a/packages/linkml/src/linkml/generators/jsonldcontextgen.py +++ b/packages/linkml/src/linkml/generators/jsonldcontextgen.py @@ -15,7 +15,7 @@ from linkml._version import __version__ from linkml.utils.deprecation import XSD_ANYURI_AS_IRI_OPTION, deprecated_fields, deprecation_warning -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, normalize_prefixes_option, shared_arguments, well_known_prefix_map from linkml_runtime.linkml_model.meta import ClassDefinition, EnumDefinition, SlotDefinition from linkml_runtime.utils.formatutils import camelcase, underscore from linkml_runtime.utils.rdf_terms import type_term @@ -52,6 +52,9 @@ class ContextGenerator(Generator): prefixes: bool | None = True flatprefixes: bool | None = False fix_multivalue_containers: bool | None = False + normalize_prefixes: bool = False + """Rename non-standard prefix aliases to the well-known names of + :func:`~linkml.utils.generator.well_known_prefix_map`, e.g. ``sdo`` → ``schema``.""" exclude_imports: bool = False """If True, elements from imported schemas won't be included in the generated context""" exclude_external_imports: bool = False @@ -88,6 +91,9 @@ class ContextGenerator(Generator): def __post_init__(self) -> None: if self.xsd_anyuri_as_iri is not None: deprecation_warning(XSD_ANYURI_AS_IRI_OPTION, stack_level=4) + # Must be set before super().__post_init__() because the parent triggers + # the visitor pattern (visit_schema), which accesses _prefix_remap. + self._prefix_remap: dict[str, str] = {} super().__post_init__() if self.namespaces is None: raise TypeError("Schema text must be supplied to context generator. Preparsed schema will not work") @@ -125,8 +131,14 @@ def _collect_external_elements(sv: SchemaView) -> tuple[set[str], set[str]]: external_slots.update(schema_def.slots.keys()) return external_classes, external_slots + def add_prefix(self, ncname: str) -> None: + """Add a prefix, applying well-known prefix normalisation when enabled.""" + super().add_prefix(self._prefix_remap.get(ncname, ncname)) + def visit_schema(self, base: str | Namespace | None = None, output: str | None = None, **_): - # Add any explicitly declared prefixes + # Add any explicitly declared prefixes. + # Direct .add() is safe here: the normalisation block below explicitly + # rewrites emit_prefixes entries for any renamed prefixes (Cases 1-3). for prefix in self.schema.prefixes.values(): self.emit_prefixes.add(prefix.prefix_prefix) @@ -134,6 +146,68 @@ def visit_schema(self, base: str | Namespace | None = None, output: str | None = for pfx in self.schema.emit_prefixes: self.add_prefix(pfx) + # Normalise well-known prefix names when --normalize-prefixes is set. + # If the schema declares a non-standard alias for a namespace that has + # a well-known standard name (e.g. ``sdo`` for + # ``https://schema.org/``), replace the alias with the standard name + # so that generated JSON-LD contexts use the conventional prefix. + # + # Three cases are handled: + # 1. Standard prefix is not yet bound → just rebind from old to new. + # 2. Standard prefix is bound to a *different* URI: + # a. User-declared (in schema.prefixes) → collision, skip with warning. + # b. Runtime default (e.g. linkml-runtime's ``schema: http://…``) + # → remove stale binding, then rebind. + # 3. Standard prefix is already bound to the *same* URI (duplicate) + # → just drop the non-standard alias. + # + # A remap dict is stored for ``_build_element_id`` because + # ``prefix_suffix()`` splits CURIEs on ``:`` without looking up the + # namespace dict. + self._prefix_remap.clear() + if self.normalize_prefixes: + wk = well_known_prefix_map() + for old_pfx in list(self.namespaces): + url = str(self.namespaces[old_pfx]) + std_pfx = wk.get(url) + if not std_pfx or std_pfx == old_pfx: + continue + if std_pfx in self.namespaces: + if str(self.namespaces[std_pfx]) != url: + # Case 2: std_pfx is bound to a different URI. + # If the user explicitly declared std_pfx in the schema, + # it is intentional — skip to avoid data loss. + if std_pfx in self.schema.prefixes: + self.logger.warning( + "Prefix collision: cannot rename '%s' to '%s' because '%s' is " + "already declared for <%s>; skipping normalisation for <%s>", + old_pfx, + std_pfx, + std_pfx, + str(self.namespaces[std_pfx]), + url, + ) + continue + # Not user-declared (e.g. linkml-runtime default) — safe to remove + self.emit_prefixes.discard(std_pfx) + del self.namespaces[std_pfx] + else: + # Case 3: standard prefix already bound to same URI + # — just drop the non-standard alias + del self.namespaces[old_pfx] + if old_pfx in self.emit_prefixes: + self.emit_prefixes.discard(old_pfx) + self.emit_prefixes.add(std_pfx) + self._prefix_remap[old_pfx] = std_pfx + continue + # Case 1 (or Case 2 after stale removal): bind standard name + self.namespaces[std_pfx] = self.namespaces[old_pfx] + del self.namespaces[old_pfx] + if old_pfx in self.emit_prefixes: + self.emit_prefixes.discard(old_pfx) + self.emit_prefixes.add(std_pfx) + self._prefix_remap[old_pfx] = std_pfx + # Add the default prefix if self.schema.default_prefix: dflt = self.namespaces.prefix_for(self.schema.default_prefix) @@ -141,6 +215,8 @@ def visit_schema(self, base: str | Namespace | None = None, output: str | None = self.default_ns = dflt if self.default_ns: default_uri = self.namespaces[self.default_ns] + # Direct .add() is safe: default_ns is already resolved from + # the (possibly normalised) namespace bindings above. self.emit_prefixes.add(self.default_ns) else: default_uri = self.schema.default_prefix @@ -508,6 +584,11 @@ def _build_element_id(self, definition: Any, uri: str) -> None: @return: None """ uri_prefix, uri_suffix = self.namespaces.prefix_suffix(uri) + # Apply well-known prefix normalisation (e.g. sdo → schema). + # prefix_suffix() splits CURIEs on ':' without checking the + # namespace dict, so it may return a stale alias. + if uri_prefix and uri_prefix in self._prefix_remap: + uri_prefix = self._prefix_remap[uri_prefix] is_default_namespace = uri_prefix == self.context_body["@vocab"] or uri_prefix == self.namespaces.prefix_for( self.context_body["@vocab"] ) @@ -607,6 +688,7 @@ def serialize( help="Use class_uri/slot_uri CURIEs as class and non-identifier slot context keys; identifier slots retain their " + "slot-name key and map to @id.", ) +@normalize_prefixes_option @click.version_option(__version__, "-V", "--version") def cli(yamlfile, emit_frame, embed_context_in_frame, output, **args): """Generate jsonld @context definition from LinkML model""" diff --git a/packages/linkml/src/linkml/generators/jsonldgen.py b/packages/linkml/src/linkml/generators/jsonldgen.py index b6aafbc60a..190b025c85 100644 --- a/packages/linkml/src/linkml/generators/jsonldgen.py +++ b/packages/linkml/src/linkml/generators/jsonldgen.py @@ -14,7 +14,7 @@ from linkml._version import __version__ from linkml.generators.jsonldcontextgen import ContextGenerator from linkml.utils.deprecation import deprecated_fields -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, normalize_prefixes_option, shared_arguments from linkml_runtime.linkml_model.meta import ( ClassDefinition, ClassDefinitionName, @@ -64,6 +64,10 @@ class JSONLDGenerator(Generator): context: str | list[str] | None = None """Path to a JSONLD context file""" + normalize_prefixes: bool = False + """Rename non-standard prefix aliases to the well-known names of + :func:`~linkml.utils.generator.well_known_prefix_map`, e.g. ``sdo`` → ``schema``.""" + metamodel_context: str | None = None """Override for metamodel context URI/path. When None, uses METAMODEL_CONTEXT_URI.""" @@ -211,6 +215,8 @@ def end_schema( # directory twice. context_kwargs.setdefault("importmap", self.importmap) context_kwargs.setdefault("base_dir", self.original_base_dir) + # Forward prefix normalisation into the inline @context. + context_kwargs.setdefault("normalize_prefixes", self.normalize_prefixes) add_prefixes = ContextGenerator(self.original_schema, **context_kwargs).serialize() add_prefixes_json = loads(add_prefixes) metamodel_ctx = self.metamodel_context or METAMODEL_CONTEXT_URI @@ -276,6 +282,7 @@ def serialize(self, context: Sequence[str] | None = None, context_kwargs: dict | "The `emit_metadata` value is forced to be False.\n\n" "multiple kwargs like `-k {key} {value}` can be passed", ) +@normalize_prefixes_option @click.version_option(__version__, "-V", "--version") def cli(yamlfile, context_kwargs: tuple[tuple[str, bool], ...], context: tuple[str, ...], **kwargs): """Generate JSONLD file from LinkML schema. diff --git a/packages/linkml/src/linkml/generators/owlgen.py b/packages/linkml/src/linkml/generators/owlgen.py index b4be78ec42..dd6ed4708d 100644 --- a/packages/linkml/src/linkml/generators/owlgen.py +++ b/packages/linkml/src/linkml/generators/owlgen.py @@ -21,7 +21,7 @@ from linkml._version import __version__ from linkml.generators.common.annotations import declared_annotation from linkml.utils.deprecation import XSD_ANYURI_AS_IRI_OPTION, deprecation_warning -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, normalize_graph_prefixes, normalize_prefixes_option, shared_arguments from linkml.utils.language_tags import LanguageTagResolver from linkml_runtime import SchemaView from linkml_runtime.linkml_model.meta import ( @@ -124,6 +124,10 @@ class OwlSchemaGenerator(Generator): """Suffix to add to the schema name to create the ontology URI, e.g. .owl.ttl""" # ObjectVars + normalize_prefixes: bool = False + """Rename non-standard prefix aliases to the well-known names of + :func:`~linkml.utils.generator.well_known_prefix_map`, e.g. ``sdo`` → ``schema``.""" + diff_stable: bool = False """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" @@ -366,6 +370,10 @@ def as_graph(self) -> Graph: self.graph.bind(prefix, self.metamodel.namespaces[prefix]) for pfx in schema.prefixes.values(): self.graph.namespace_manager.bind(pfx.prefix_prefix, URIRef(pfx.prefix_reference)) + if self.normalize_prefixes: + normalize_graph_prefixes( + graph, {str(v.prefix_prefix): str(v.prefix_reference) for v in schema.prefixes.values()} + ) graph.add((base, RDF.type, OWL.Ontology)) # Add main schema elements @@ -2022,6 +2030,7 @@ def slot_owl_type(self, slot: SlotDefinition) -> URIRef: "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" ), ) +@normalize_prefixes_option @click.version_option(__version__, "-V", "--version") def cli(yamlfile: str, metadata_profile: str, **kwargs: Any) -> None: """Generate an OWL representation of a LinkML model diff --git a/packages/linkml/src/linkml/generators/shaclgen.py b/packages/linkml/src/linkml/generators/shaclgen.py index cea9d5f3aa..7cc977ad0c 100644 --- a/packages/linkml/src/linkml/generators/shaclgen.py +++ b/packages/linkml/src/linkml/generators/shaclgen.py @@ -19,7 +19,7 @@ from linkml.generators.common.subproperty import get_subproperty_values from linkml.generators.shacl.shacl_data_type import ShaclDataType from linkml.generators.shacl.shacl_ifabsent_processor import ShaclIfAbsentProcessor -from linkml.utils.generator import Generator, shared_arguments +from linkml.utils.generator import Generator, normalize_graph_prefixes, normalize_prefixes_option, shared_arguments from linkml.utils.language_tags import LanguageTagResolver from linkml_runtime.linkml_model.meta import ( AnonymousClassExpression, @@ -186,6 +186,10 @@ class ShaclGenerator(Generator): ignores any per-slot ``in_language``. """ + normalize_prefixes: bool = False + """Rename non-standard prefix aliases to the well-known names of + :func:`~linkml.utils.generator.well_known_prefix_map`, e.g. ``sdo`` → ``schema``.""" + diff_stable: bool = False """Reduce blank-node label churn. See :ref:`rdf-in-version-control`.""" @@ -267,6 +271,10 @@ def as_graph(self) -> Graph: for pfx in self.schema.prefixes.values(): g.bind(str(pfx.prefix_prefix), pfx.prefix_reference) + if self.normalize_prefixes: + normalize_graph_prefixes( + g, {str(v.prefix_prefix): str(v.prefix_reference) for v in self.schema.prefixes.values()} + ) self._class_expressions_added: set[tuple[URIRef, str, str]] = set() self._class_expression_problems: dict[tuple[str, str, str], list[str]] = {} @@ -2155,6 +2163,7 @@ def add_simple_data_type(func: Callable, r: ElementName) -> None: "https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control" ), ) +@normalize_prefixes_option @click.version_option(__version__, "-V", "--version") def cli(yamlfile, **args): """Generate SHACL turtle from a LinkML model""" diff --git a/packages/linkml/src/linkml/utils/generator.py b/packages/linkml/src/linkml/utils/generator.py index d5d660f4ae..92345e27d9 100644 --- a/packages/linkml/src/linkml/utils/generator.py +++ b/packages/linkml/src/linkml/utils/generator.py @@ -26,7 +26,7 @@ from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path -from typing import IO, Any, ClassVar, TextIO, Union, cast +from typing import IO, TYPE_CHECKING, Any, ClassVar, TextIO, Union, cast import click import yaml @@ -61,6 +61,9 @@ from linkml_runtime.utils.formatutils import camelcase, underscore from linkml_runtime.utils.namespaces import Namespaces +if TYPE_CHECKING: + from rdflib import Graph + logger = logging.getLogger(__name__) @@ -81,6 +84,136 @@ def _resolved_metamodel(mergeimports): return metamodel +def well_known_prefix_map() -> dict[str, str]: + """Return a mapping from namespace URI to standard prefix name. + + Primary source: the ``linked_data`` context from `prefixmaps + `_ — the canonical curated + registry maintained by the LinkML team. This context provides + correct, community-consensus prefix names (e.g. ``sh`` not ``shacl``, + ``schema`` not ``sdo``). + + Secondary source: the ``merged`` context from prefixmaps, which + combines prefix.cc, bioregistry, and other sources for broad coverage. + + Third source, taking precedence: the names rdflib binds by default + (``Graph(bind_namespaces="rdflib")``), which these generators serialize + with. ``http://schema.org/`` gets ``schema`` from prefixmaps and + ``https://schema.org/`` from rdflib, so both forms map to ``schema``. + + .. note:: + Requires ``prefixmaps >= 0.2.7``. For entries added in + linkml/prefixmaps#81 (W3C/OGC standard prefixes), pin to + ``prefixmaps @ git+https://github.com/linkml/prefixmaps@75435150`` + until v0.2.8 is released. + """ + return dict(_cached_well_known_prefix_map()) + + +@lru_cache(maxsize=1) +def _cached_well_known_prefix_map() -> dict[str, str]: + """Internal cached builder for well_known_prefix_map().""" + from prefixmaps import load_context + + # Layer 1: merged context (broad coverage, first-seen-wins for duplicates). + merged = load_context("merged") + ns_to_prefix: dict[str, str] = {} + for rec in merged.prefix_expansions: + if rec.namespace not in ns_to_prefix: + ns_to_prefix[rec.namespace] = rec.prefix + + # Layer 2: linked_data context (curated, correct names) overrides merged. + ld = load_context("linked_data") + for rec in ld.prefix_expansions: + ns_to_prefix[rec.namespace] = rec.prefix + + # Layer 3: the names rdflib binds by default. These generators serialize with + # rdflib, so its names are the ones an unprefixed namespace already gets, and + # they take precedence where prefixmaps differs (``geosparql`` vs ``geo``, + # ``sc`` vs ``schema``, ``WGS84`` vs ``wgs``). + from rdflib import Graph + + for prefix, namespace in Graph(bind_namespaces="rdflib").namespaces(): + if str(prefix): + ns_to_prefix[str(namespace)] = str(prefix) + + return ns_to_prefix + + +def normalize_graph_prefixes(graph: "Graph", schema_prefixes: dict[str, str]) -> None: + """Normalise non-standard prefix aliases in an rdflib Graph. + + For each prefix bound in *schema_prefixes* (mapping prefix name → + namespace URI), check whether ``well_known_prefix_map()`` knows a + standard name for that URI. If the standard name differs from the + schema-declared name, rebind the namespace to the standard name. + + This is the **shared implementation** used by the OWL and SHACL + generators, and (via a different code-path) the JSON-LD context generator, + so that all serialisation formats agree on prefix names when + ``--normalize-prefixes`` is active. + + :param graph: rdflib Graph whose namespace bindings should be adjusted. + :param schema_prefixes: mapping of prefix name → namespace URI string, + typically from ``schema.prefixes``. + """ + from rdflib import Namespace + + wk = well_known_prefix_map() + + # Phase 1: normalise schema-declared prefixes. + for old_pfx, ns_uri in schema_prefixes.items(): + ns_str = str(ns_uri) + std_pfx = wk.get(ns_str) + if not std_pfx or std_pfx == old_pfx: + continue + # Collision: the user explicitly declared std_pfx for a different + # namespace — do not clobber their binding. + if std_pfx in schema_prefixes and schema_prefixes[std_pfx] != ns_str: + logger.warning( + "Prefix collision: cannot rename '%s' to '%s' because '%s' is already " + "declared for <%s>; skipping normalisation for <%s>", + old_pfx, + std_pfx, + std_pfx, + schema_prefixes[std_pfx], + ns_str, + ) + continue + # Rebind: remove old prefix, add standard prefix. + # ``replace=True`` forces the new prefix even if the prefix name + # is already bound to a different namespace. + graph.bind(std_pfx, Namespace(ns_str), override=True, replace=True) + + # Phase 2: normalise runtime-injected bindings (e.g. metamodel defaults). + # The linkml-runtime / rdflib may inject well-known namespaces under + # non-standard prefix names. After Phase 1 rebinds schema-declared + # prefixes, orphaned runtime bindings can appear as ``schema1``, ``dc0``, + # etc. Scan the graph's current bindings and fix any that map to a + # well-known namespace under a non-standard name, provided the standard + # name isn't already claimed by the user for a different namespace. + # + # Guard: if Phase 1 already bound std_pfx to a different URI (e.g. + # ``schema`` → ``https://schema.org/``), do not clobber it with the + # HTTP variant (``http://schema.org/``). Build a snapshot of the + # current bindings after Phase 1 to detect this. + current_bindings = {str(p): str(n) for p, n in graph.namespaces()} + for pfx, ns in list(graph.namespaces()): + pfx_str, ns_str = str(pfx), str(ns) + std_pfx = wk.get(ns_str) + if not std_pfx or std_pfx == pfx_str: + continue + # Same collision check as Phase 1: respect user-declared prefixes. + if std_pfx in schema_prefixes and schema_prefixes[std_pfx] != ns_str: + continue + # Guard: if std_pfx is already bound to a different (correct) URI + # by Phase 1, do not overwrite it. This prevents the HTTP variant + # of schema.org from clobbering the HTTPS binding. + if std_pfx in current_bindings and current_bindings[std_pfx] != ns_str: + continue + graph.bind(std_pfx, Namespace(ns_str), override=True, replace=True) + + @dataclass class Generator(metaclass=abc.ABCMeta): """ @@ -1272,3 +1405,13 @@ def apply_config_defaults(ctx: click.Context, config: Mapping[str, Any], args: d # convert as click would, so e.g. `template_dir: /tmp` arrives as a Path args[key] = param.type_cast_value(ctx, value) return args + + +def normalize_prefixes_option(f: Callable) -> Callable: + """Add ``--normalize-prefixes`` to the command line of a generator that honours it.""" + return click.option( + "--normalize-prefixes/--no-normalize-prefixes", + default=False, + show_default=True, + help="Rename non-standard prefix aliases to well-known names (e.g. sdo → schema for https://schema.org/).", + )(f) diff --git a/tests/linkml/test_data/test_sqlite.py b/tests/linkml/test_data/test_sqlite.py index 38dce4ffb8..8b28979d91 100644 --- a/tests/linkml/test_data/test_sqlite.py +++ b/tests/linkml/test_data/test_sqlite.py @@ -71,9 +71,17 @@ def test_mixin(tmp_outputs): assert diff == "" +@pytest.fixture +def restore_csv_field_size_limit(): + """Restore the process-wide CSV field size limit that a test changes.""" + limit = csv.field_size_limit() + yield + csv.field_size_limit(limit) + + @pytest.mark.parametrize("diff", [1, -1]) @pytest.mark.parametrize("size", [20, 200000]) -def test_csv_limit(tmp_outputs, diff, size): +def test_csv_limit(tmp_outputs, diff, size, restore_csv_field_size_limit): """ Tests https://github.com/linkml/linkml/issues/815 """ diff --git a/tests/linkml/test_generators/test_jsonldcontextgen.py b/tests/linkml/test_generators/test_jsonldcontextgen.py index 154b4c1f6a..d99673deb4 100644 --- a/tests/linkml/test_generators/test_jsonldcontextgen.py +++ b/tests/linkml/test_generators/test_jsonldcontextgen.py @@ -1619,3 +1619,118 @@ def test_type_scoped_coercion_survives_json_ld_1_0_processing(tmp_path, version) values = {o.datatype for o in graph.objects(None, URIRef("https://example.org/version/value"))} assert values == {XSD.float, None} + + +def test_normalize_prefixes_renames_nonstandard_alias(tmp_path): + """When --normalize-prefixes is set, non-standard aliases are replaced by rdflib defaults. + + rdflib binds ``dc`` to ``http://purl.org/dc/elements/1.1/`` by default. + A schema that declares ``dce`` for the same URI should have it normalised + to ``dc`` when the flag is enabled. + + See: rdflib default namespace bindings. + """ + schema = tmp_path / "schema.yaml" + schema.write_text( + """\ +id: https://example.org/test +name: test_normalize +default_prefix: ex +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + dce: http://purl.org/dc/elements/1.1/ +imports: + - linkml:types +classes: + Record: + class_uri: ex:Record + attributes: + title: + range: string + slot_uri: dce:title +""", + encoding="utf-8", + ) + + # Flag OFF (default): non-standard alias preserved + ctx_off = json.loads(ContextGenerator(str(schema), normalize_prefixes=False).serialize())["@context"] + assert "dce" in ctx_off, "With flag off, original prefix 'dce' must be preserved" + + # Flag ON: rdflib default name used + ctx_on = json.loads(ContextGenerator(str(schema), normalize_prefixes=True).serialize())["@context"] + assert "dc" in ctx_on, "With flag on, 'dce' should be normalised to 'dc'" + assert "dce" not in ctx_on, "With flag on, original alias 'dce' should be removed" + assert ctx_on["dc"] == "http://purl.org/dc/elements/1.1/" + + +def test_normalize_prefixes_default_is_off(tmp_path): + """The --normalize-prefixes flag defaults to False — no prefix renaming. + + Ensures backward compatibility: existing schemas produce identical output. + """ + schema = tmp_path / "schema.yaml" + schema.write_text( + """\ +id: https://example.org/test +name: test_default +default_prefix: ex +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: https://schema.org/ +imports: + - linkml:types +classes: + Thing: + class_uri: sdo:Thing + attributes: + name: + range: string + slot_uri: sdo:name +""", + encoding="utf-8", + ) + + ctx = json.loads(ContextGenerator(str(schema)).serialize())["@context"] + # Without the flag, the schema's own prefix name must be preserved + assert "sdo" in ctx, "Default behavior must preserve schema-declared prefix 'sdo'" + + +def test_normalize_prefixes_curie_remapping(tmp_path): + """CURIEs in element @id values use the normalised prefix name. + + When ``sdo`` is normalised to ``schema``, slot URIs like ``sdo:name`` + must appear as ``schema:name`` in the generated context. + """ + schema = tmp_path / "schema.yaml" + schema.write_text( + """\ +id: https://example.org/test +name: test_curie +default_prefix: ex +prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: https://schema.org/ +imports: + - linkml:types +classes: + Person: + class_uri: sdo:Person + attributes: + full_name: + range: string + slot_uri: sdo:name +""", + encoding="utf-8", + ) + + ctx = json.loads(ContextGenerator(str(schema), normalize_prefixes=True).serialize())["@context"] + # The prefix declaration must use the standard name + assert "schema" in ctx, "Normalised prefix 'schema' must appear" + # Element @id must use the normalised prefix + person = ctx.get("Person", {}) + assert person.get("@id", "").startswith("schema:"), ( + f"Person @id should use normalised prefix 'schema:', got {person}" + ) diff --git a/tests/linkml/test_generators/test_normalize_prefixes.py b/tests/linkml/test_generators/test_normalize_prefixes.py new file mode 100644 index 0000000000..f1c4f2f23c --- /dev/null +++ b/tests/linkml/test_generators/test_normalize_prefixes.py @@ -0,0 +1,589 @@ +"""Tests for the --normalize-prefixes flag across all generators. + +Verifies that non-standard prefix aliases (e.g. ``sdo`` for ``https://schema.org/``) +are normalised to well-known names (e.g. ``schema``) consistently in OWL, SHACL, +and JSON-LD context output. + +References: +- prefix.cc — community consensus RDF prefix registry +- rdflib 7.x curated default namespace bindings +- W3C Turtle §2.4 — prefix declarations are syntactic sugar +""" + +import json +import logging +import re +import textwrap + +import pytest + +# ── Shared test schema ────────────────────────────────────────────── + +SCHEMA_SDO = textwrap.dedent("""\ + id: https://example.org/test + name: test_normalize + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: https://schema.org/ + imports: + - linkml:types + classes: + Person: + class_uri: sdo:Person + attributes: + full_name: + range: string + slot_uri: sdo:name +""") + +SCHEMA_DCE = textwrap.dedent("""\ + id: https://example.org/test + name: test_normalize_dce + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + dce: http://purl.org/dc/elements/1.1/ + imports: + - linkml:types + classes: + Record: + class_uri: ex:Record + attributes: + title: + range: string + slot_uri: dce:title +""") + +# HTTP variant — linkml-runtime historically binds schema: http://schema.org/ +# while rdflib (and the W3C) prefer https://schema.org/. The normalize flag +# must handle both. +SCHEMA_HTTP_SDO = textwrap.dedent("""\ + id: https://example.org/test + name: test_http_schema + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + sdo: http://schema.org/ + imports: + - linkml:types + classes: + Place: + class_uri: sdo:Place + attributes: + geo: + range: string + slot_uri: sdo:geo +""") + +# Collision scenario: user declares 'foaf' for a custom namespace AND 'myfoaf' +# for http://xmlns.com/foaf/0.1/. Normalisation must NOT clobber the user's 'foaf'. +# Uses 'foaf' instead of 'schema' because 'schema' is declared in linkml:types, +# which causes a SchemaLoader merge conflict before normalisation even runs. +SCHEMA_COLLISION = textwrap.dedent("""\ + id: https://example.org/test + name: test_collision + default_prefix: ex + prefixes: + ex: https://example.org/ + linkml: https://w3id.org/linkml/ + foaf: https://something-else.org/ + myfoaf: http://xmlns.com/foaf/0.1/ + imports: + - linkml:types + classes: + Agent: + class_uri: myfoaf:Agent + attributes: + label: + range: string + slot_uri: myfoaf:name +""") + + +def _write_schema(tmp_path, content: str, name: str = "schema.yaml") -> str: + """Write schema content to a temporary file and return its path as string.""" + p = tmp_path / name + p.write_text(content, encoding="utf-8") + return str(p) + + +def _turtle_prefixes(ttl: str) -> dict[str, str]: + """Extract @prefix declarations from Turtle output → {prefix: namespace}.""" + result = {} + for m in re.finditer(r"@prefix\s+(\w+):\s+<([^>]+)>", ttl): + result[m.group(1)] = m.group(2) + return result + + +# ── OWL Generator Tests ───────────────────────────────────────────── + + +def test_owl_sdo_normalised_to_schema(tmp_path): + """sdo → schema when --normalize-prefixes is active.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix in OWL output, got: {sorted(pfx)}" + assert pfx["schema"] == "https://schema.org/" + assert "sdo" not in pfx, "Non-standard 'sdo' prefix should be removed" + + +def test_owl_flag_off_preserves_original(tmp_path): + """Without the flag, schema-declared prefix names are preserved.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=False).serialize() + pfx = _turtle_prefixes(ttl) + assert "sdo" in pfx, "With flag off, original prefix 'sdo' must be preserved" + + +def test_owl_dce_normalised_to_dc(tmp_path): + """dce → dc for http://purl.org/dc/elements/1.1/ in graph bindings. + + Note: rdflib's Turtle serializer only emits @prefix declarations for + namespaces actually used in triples. Since the OWL generator may not + produce triples using dc:elements URIs for simple attribute schemas, + we verify the graph's namespace bindings directly. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_DCE) + gen = OwlSchemaGenerator(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "dc" in bound, f"Expected 'dc' in graph bindings, got: {sorted(bound)}" + assert bound["dc"] == "http://purl.org/dc/elements/1.1/" + + +def test_owl_custom_prefix_not_affected(tmp_path): + """Domain-specific prefixes (e.g. 'ex') are not touched by normalisation.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "ex" in pfx, "Custom prefix 'ex' must survive normalisation" + assert pfx["ex"] == "https://example.org/" + + +def test_owl_http_schema_org_normalised(tmp_path): + """http://schema.org/ (HTTP variant) also normalises to 'schema'. + + The linkml-runtime historically binds ``schema: http://schema.org/`` + while the W3C and rdflib prefer ``https://schema.org/``. Both + variants must be recognised by the static well-known prefix map. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_HTTP_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix for http://schema.org/, got: {sorted(pfx)}" + assert "sdo" not in pfx + + +def test_owl_no_schema1_from_runtime_http_binding(tmp_path): + """Runtime-injected ``schema: http://schema.org/`` must not create ``schema1``. + + The linkml metamodel (types.yaml) declares ``schema: http://schema.org/`` + (HTTP). When a user schema declares ``sdo: https://schema.org/`` (HTTPS), + normalisation must clean up *both* variants so the output never contains + auto-generated suffixed prefixes like ``schema1``. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + suffixed = [p for p in pfx if re.match(r"schema\d+", p)] + assert not suffixed, ( + f"Auto-generated suffixed prefix(es) {suffixed} found — runtime http://schema.org/ binding was not cleaned up" + ) + + +# ── SHACL Generator Tests ─────────────────────────────────────────── + + +def test_shacl_sdo_normalised_to_schema(tmp_path): + """sdo → schema when --normalize-prefixes is active.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix in SHACL output, got: {sorted(pfx)}" + assert pfx["schema"] == "https://schema.org/" + assert "sdo" not in pfx, "Non-standard 'sdo' prefix should be removed" + + +def test_shacl_flag_off_preserves_original(tmp_path): + """Without the flag, schema-declared prefix names are preserved.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=False).serialize() + pfx = _turtle_prefixes(ttl) + assert "sdo" in pfx, "With flag off, original prefix 'sdo' must be preserved" + + +def test_shacl_dce_normalised_to_dc(tmp_path): + """dce → dc for http://purl.org/dc/elements/1.1/.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_DCE) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "dc" in pfx, f"Expected 'dc' prefix in SHACL output, got: {sorted(pfx)}" + assert pfx["dc"] == "http://purl.org/dc/elements/1.1/" + assert "dce" not in pfx, "Non-standard 'dce' prefix should be removed" + + +def test_shacl_custom_prefix_not_affected(tmp_path): + """Domain-specific prefixes (e.g. 'ex') are not touched by normalisation. + + Note: rdflib only emits @prefix for namespaces used in triples. + We verify graph bindings directly. + """ + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + gen = ShaclGenerator(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "ex" in bound, f"Custom prefix 'ex' must survive in graph bindings, got: {sorted(bound)}" + assert bound["ex"] == "https://example.org/" + + +def test_shacl_http_schema_org_normalised(tmp_path): + """http://schema.org/ (HTTP variant) also normalises to 'schema'.""" + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_HTTP_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + assert "schema" in pfx, f"Expected 'schema' prefix for http://schema.org/, got: {sorted(pfx)}" + assert "sdo" not in pfx + + +def test_shacl_no_schema1_from_runtime_http_binding(tmp_path): + """Runtime-injected ``schema: http://schema.org/`` must not create ``schema1``. + + Same scenario as the OWL test: linkml:types imports bring in + ``schema: http://schema.org/`` while the user schema has + ``sdo: https://schema.org/``. Phase 2 of normalisation must + clean up the orphaned HTTP binding. + """ + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + pfx = _turtle_prefixes(ttl) + suffixed = [p for p in pfx if re.match(r"schema\d+", p)] + assert not suffixed, ( + f"Auto-generated suffixed prefix(es) {suffixed} found — runtime http://schema.org/ binding was not cleaned up" + ) + + +# ── JSON-LD Context Generator Tests ───────────────────────────────── + + +def test_context_http_schema_org_normalised(tmp_path): + """http://schema.org/ (HTTP variant) normalises to 'schema' in JSON-LD context. + + This covers the edge case where linkml-runtime's ``schema: http://schema.org/`` + conflicts with rdflib's ``schema: https://schema.org/``. The stale binding + must be removed and replaced with the correct one. + """ + from linkml.generators.jsonldcontextgen import ContextGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_HTTP_SDO) + ctx = json.loads(ContextGenerator(schema_path, normalize_prefixes=True).serialize())["@context"] + assert "schema" in ctx, "HTTP schema.org should normalise to 'schema'" + assert "sdo" not in ctx, "Non-standard 'sdo' should be removed" + # The namespace URI must match the schema-declared one (http, not https) + schema_val = ctx["schema"] + if isinstance(schema_val, dict): + schema_val = schema_val.get("@id", "") + assert schema_val == "http://schema.org/", f"Namespace URI must be preserved: got {schema_val}" + + +# ── Static Prefix Map Tests ───────────────────────────────────────── + + +def test_well_known_prefix_map_returns_dict(): + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert isinstance(wk, dict) + assert len(wk) >= 29, f"Expected ≥29 entries, got {len(wk)}" + + +def test_well_known_prefix_map_schema_https(): + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert wk["https://schema.org/"] == "schema" + + +def test_well_known_prefix_map_schema_http_variant(): + """Both http and https schema.org must map to 'schema'.""" + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert wk["http://schema.org/"] == "schema" + + +def test_well_known_prefix_map_dc_elements(): + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + assert wk["http://purl.org/dc/elements/1.1/"] == "dc" + + +def test_well_known_prefix_map_returns_copy(): + """Callers should not be able to mutate the internal map.""" + from linkml.utils.generator import well_known_prefix_map + + wk1 = well_known_prefix_map() + wk1["http://never-in-any-real-prefix-map.test/"] = "test" + wk2 = well_known_prefix_map() + assert "http://never-in-any-real-prefix-map.test/" not in wk2 + + +def test_well_known_prefix_map_uses_rdflib_default_names(): + """Where rdflib binds a namespace by default, its name wins: the generators + serialize with rdflib, so that is the name an undeclared namespace gets.""" + from rdflib import Graph as RdfGraph + + from linkml.utils.generator import well_known_prefix_map + + wk = well_known_prefix_map() + rdflib_map = {str(ns): str(pfx) for pfx, ns in RdfGraph(bind_namespaces="rdflib").namespaces() if str(pfx)} + assert {ns: wk.get(ns) for ns in rdflib_map} == rdflib_map + + +@pytest.mark.parametrize( + "namespace,prefixmaps_name,expected", + [ + ("http://www.opengis.net/ont/geosparql#", "geosparql", "geo"), + ("https://schema.org/", "sc", "schema"), + ("https://www.w3.org/2003/01/geo/wgs84_pos#", "WGS84", "wgs"), + ("http://schema.org/", "schema", "schema"), + ], +) +def test_well_known_prefix_map_layers(namespace, prefixmaps_name, expected): + """prefixmaps names a namespace; rdflib's default name replaces it where rdflib has one.""" + from prefixmaps import load_context + + from linkml.utils.generator import well_known_prefix_map + + curated = {rec.namespace: rec.prefix for rec in load_context("linked_data").prefix_expansions} + merged = {} + for rec in load_context("merged").prefix_expansions: + merged.setdefault(rec.namespace, rec.prefix) + assert curated.get(namespace, merged.get(namespace)) == prefixmaps_name + assert well_known_prefix_map()[namespace] == expected + + +# ── Cross-Generator Consistency Tests ──────────────────────────────── + + +@pytest.mark.parametrize("command", ["owl", "shacl", "jsonld-context", "jsonld"]) +def test_cli_offers_the_option(command, tmp_path): + """The generators that honour --normalize-prefixes accept it on the command line.""" + from click.testing import CliRunner + + from linkml.cli.main import linkml as linkml_cli + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + result = CliRunner().invoke(linkml_cli, ["generate", command, "--normalize-prefixes", str(schema_path)]) + assert result.exit_code == 0, result.output + + +@pytest.mark.parametrize("command", ["rdf", "shex", "python", "pydantic", "json-schema"]) +def test_cli_rejects_the_option_where_it_would_do_nothing(command, tmp_path): + """Generators that don't emit the schema's prefix labels don't offer the option. + + gen-rdf writes the schema's ``class_uri`` and prefix declarations as literal + data, and gen-shex's RDF uses rdflib's names already.""" + from click.testing import CliRunner + + from linkml.cli.main import linkml as linkml_cli + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + result = CliRunner().invoke(linkml_cli, ["generate", command, "--normalize-prefixes", str(schema_path)]) + assert result.exit_code == 2 and "No such option" in result.output + + +def test_all_generators_normalise_sdo_to_schema(tmp_path): + """OWL, SHACL, and JSON-LD context must all use 'schema' for schema.org.""" + from linkml.generators.jsonldcontextgen import ContextGenerator + from linkml.generators.owlgen import OwlSchemaGenerator + from linkml.generators.shaclgen import ShaclGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + + owl_ttl = OwlSchemaGenerator(schema_path, normalize_prefixes=True).serialize() + shacl_ttl = ShaclGenerator(schema_path, normalize_prefixes=True).serialize() + ctx = json.loads(ContextGenerator(schema_path, normalize_prefixes=True).serialize())["@context"] + + owl_pfx = _turtle_prefixes(owl_ttl) + shacl_pfx = _turtle_prefixes(shacl_ttl) + + assert "schema" in owl_pfx, "OWL must use 'schema'" + assert "schema" in shacl_pfx, "SHACL must use 'schema'" + assert "schema" in ctx, "JSON-LD context must use 'schema'" + + assert "sdo" not in owl_pfx, "OWL must not have 'sdo'" + assert "sdo" not in shacl_pfx, "SHACL must not have 'sdo'" + assert "sdo" not in ctx, "JSON-LD context must not have 'sdo'" + + +# ── Prefix Collision Tests ──────────────────────────────────────────── + + +@pytest.mark.parametrize( + "generator_cls,generator_module", + [ + ("OwlSchemaGenerator", "linkml.generators.owlgen"), + ("ShaclGenerator", "linkml.generators.shaclgen"), + ], + ids=["owl", "shacl"], +) +def test_graph_generator_collision_skips_rename(tmp_path, caplog, generator_cls, generator_module): + """Graph generators: myfoaf must NOT be renamed to 'foaf' when user claims that name.""" + import importlib + + mod = importlib.import_module(generator_module) + cls = getattr(mod, generator_cls) + + schema_path = _write_schema(tmp_path, SCHEMA_COLLISION) + with caplog.at_level(logging.WARNING): + gen = cls(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "myfoaf" in bound, "Non-standard 'myfoaf' must remain when collision prevents renaming" + assert bound["myfoaf"] == "http://xmlns.com/foaf/0.1/" + assert "collision" in caplog.text.lower(), f"Expected collision warning, got: {caplog.text}" + + +def test_context_collision_preserves_user_prefix(tmp_path, caplog): + """JSON-LD: user's 'foaf: https://something-else.org/' must survive.""" + from linkml.generators.jsonldcontextgen import ContextGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_COLLISION) + with caplog.at_level(logging.WARNING): + ctx = json.loads(ContextGenerator(schema_path, normalize_prefixes=True).serialize())["@context"] + # User's 'foaf' binding preserved + foaf_val = ctx.get("foaf") + if isinstance(foaf_val, dict): + foaf_val = foaf_val.get("@id", "") + assert foaf_val == "https://something-else.org/", f"User's 'foaf' binding must be preserved, got: {foaf_val}" + # myfoaf must remain (not renamed to foaf) + assert "myfoaf" in ctx, "Non-standard 'myfoaf' must remain when collision prevents renaming" + # Warning emitted + assert "collision" in caplog.text.lower(), f"Expected collision warning, got: {caplog.text}" + + +# ── JSONLDGenerator Flag Forwarding Tests ───────────────────────────── + + +def test_jsonld_generator_forwards_normalize_prefixes(tmp_path): + """JSONLDGenerator must pass normalize_prefixes to embedded ContextGenerator. + + Without forwarding, the inline @context in JSON-LD output would keep + non-standard prefix aliases even when --normalize-prefixes is set. + """ + from linkml.generators.jsonldgen import JSONLDGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + out = JSONLDGenerator(schema_path, normalize_prefixes=True).serialize() + parsed = json.loads(out) + # The @context may be a list; find the dict entry + ctx = parsed.get("@context", {}) + if isinstance(ctx, list): + for item in ctx: + if isinstance(item, dict): + ctx = item + break + assert "sdo" not in ctx, "normalize_prefixes not forwarded: 'sdo' still in embedded @context" + + +# ── Phase 2 HTTP/HTTPS Overwrite Bug Tests ──────────────────────────── + + +def test_phase2_does_not_overwrite_https_with_http(tmp_path): + """When Phase 1 binds schema → https://schema.org/, Phase 2 must not + overwrite it with http://schema.org/ from the runtime metamodel. + + Reproduction: linkml:types imports bring schema: http://schema.org/ + (HTTP) while the user schema has sdo: https://schema.org/ (HTTPS). + Phase 1 normalises sdo → schema (HTTPS). Phase 2 must not then + rebind schema → http://schema.org/ when it encounters the runtime + HTTP binding. + """ + from linkml.generators.owlgen import OwlSchemaGenerator + + schema_path = _write_schema(tmp_path, SCHEMA_SDO) + gen = OwlSchemaGenerator(schema_path, normalize_prefixes=True) + graph = gen.as_graph() + bound = {str(p): str(n) for p, n in graph.namespaces()} + assert "schema" in bound, f"Expected 'schema' in bindings, got: {sorted(bound)}" + # MUST be HTTPS (from the user's schema), not HTTP (from runtime) + assert bound["schema"] == "https://schema.org/", ( + f"Phase 2 overwrote HTTPS with HTTP: schema bound to {bound['schema']}" + ) + + +def test_normalize_graph_prefixes_phase2_guard(): + """Direct unit test for the Phase 2 guard in normalize_graph_prefixes. + + Simulates the exact scenario: Phase 1 binds schema → https://schema.org/, + then Phase 2 encounters schema1 → http://schema.org/ and must NOT rebind. + """ + from rdflib import Graph, Namespace, URIRef + + from linkml.utils.generator import normalize_graph_prefixes + + g = Graph(bind_namespaces="none") + # Simulate Phase 1 result + g.bind("schema", Namespace("https://schema.org/")) + # Simulate runtime-injected HTTP variant (would appear as schema1) + g.bind("schema1", Namespace("http://schema.org/")) + # Add a triple so the graph isn't empty + g.add((URIRef("https://example.org/s"), URIRef("https://schema.org/name"), URIRef("https://example.org/o"))) + + normalize_graph_prefixes(g, {"sdo": "https://schema.org/"}) + + bound = {str(p): str(n) for p, n in g.namespaces()} + assert bound.get("schema") == "https://schema.org/", f"Phase 2 guard failed: schema bound to {bound.get('schema')}" + + +def test_empty_schema_no_crash(tmp_path): + """A schema with no custom prefixes must not crash normalize_graph_prefixes.""" + from linkml.generators.owlgen import OwlSchemaGenerator + + (tmp_path / "empty.yaml").write_text( + textwrap.dedent("""\ + id: https://example.org/empty + name: empty + default_prefix: ex + prefixes: + linkml: https://w3id.org/linkml/ + ex: https://example.org/ + imports: + - linkml:types + """), + encoding="utf-8", + ) + # Should not raise + gen = OwlSchemaGenerator(str(tmp_path / "empty.yaml"), normalize_prefixes=True) + ttl = gen.serialize() + assert len(ttl) > 0 diff --git a/uv.lock b/uv.lock index d17f07d4b3..03c3c00d3c 100644 --- a/uv.lock +++ b/uv.lock @@ -2526,7 +2526,7 @@ requires-dist = [ { name = "parse" }, { name = "platformdirs", specifier = ">=4.12.2" }, { name = "prefixcommons", specifier = ">=0.1.7" }, - { name = "prefixmaps", specifier = ">=0.2.2" }, + { name = "prefixmaps", git = "https://github.com/linkml/prefixmaps?rev=75435150a1b31760b9780af2b64a265943a9b263" }, { name = "pydantic", specifier = ">=2.14.1,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.15.0" }, { name = "pyjsg", specifier = ">=0.12.5" }, @@ -3783,16 +3783,12 @@ wheels = [ [[package]] name = "prefixmaps" -version = "0.2.6" -source = { registry = "https://pypi.org/simple" } +version = "0.2.7.post2.dev0+7543515" +source = { git = "https://github.com/linkml/prefixmaps?rev=75435150a1b31760b9780af2b64a265943a9b263#75435150a1b31760b9780af2b64a265943a9b263" } dependencies = [ { name = "curies" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/cf/f588bcdfd2c841839b9d59ce219a46695da56aa2805faff937bbafb9ee2b/prefixmaps-0.2.6.tar.gz", hash = "sha256:7421e1244eea610217fa1ba96c9aebd64e8162a930dc0626207cd8bf62ecf4b9", size = 709899, upload-time = "2024-10-17T16:30:57.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/b2/2b2153173f2819e3d7d1949918612981bc6bd895b75ffa392d63d115f327/prefixmaps-0.2.6-py3-none-any.whl", hash = "sha256:f6cef28a7320fc6337cf411be212948ce570333a0ce958940ef684c7fb192a62", size = 754732, upload-time = "2024-10-17T16:30:55.731Z" }, -] [[package]] name = "prettytable" From cbfa5ca6557e6cd8e753d5e78a0e58b2b4d66187 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Tue, 13 Oct 2026 20:33:43 +0200 Subject: [PATCH 15/17] fix(rdf): delegate canonicalization to the diffable-rdf library The RDF canonicalizer in linkml_runtime was extracted into diffable-rdf, and the two copies diverged. Calling the library fixes nine defects of the in-tree copy, each asserted against both entry points in test_rdf_canonicalize_defects.py: - a base IRI ending in # no longer rewrites every IRI sharing its prefix - a shared rdf:List tail is no longer duplicated - N-Triples refuses a relative IRI instead of writing a file its own parser rejects; N-Triples allows only absolute IRIs - literals containing U+2028 and other line separators survive sorting - degraded RDF/XML, degraded Turtle and JSON-LD are byte-identical across processes - every format ends with exactly one newline - a Dataset is refused instead of silently flattened diffable-rdf becomes a required dependency of linkml-runtime. It needs only rdflib and pyoxigraph, which the runtime already requires. The diff-stable extras remain as aliases. Turtle output of the four RDF generators is unchanged. The library logs degraded output; the runtime re-emits those records as RDFCanonicalizationWarning so they stay visible without logging configuration. Signed-off-by: jdsika --- docs/howtos/collaborative-development.md | 12 +- packages/linkml_runtime/pyproject.toml | 1 + .../linkml_runtime/utils/rdf_canonicalize.py | 478 +++--------------- tests/conftest.py | 4 +- .../test_utils/test_rdf_canonicalize.py | 196 ++++++- .../test_rdf_canonicalize_defects.py | 301 +++++++++++ uv.lock | 2 + 7 files changed, 562 insertions(+), 432 deletions(-) create mode 100644 tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py diff --git a/docs/howtos/collaborative-development.md b/docs/howtos/collaborative-development.md index 221337e843..278e290523 100644 --- a/docs/howtos/collaborative-development.md +++ b/docs/howtos/collaborative-development.md @@ -49,18 +49,20 @@ but an insertion can renumber existing nodes and obscure the actual schema edit. The optional `--diff-stable` flag uses [diffable-rdf](https://github.com/ASCS-eV/diffable-rdf) to reduce this label churn. -In a uv project containing your schema, install LinkML with the extra and generate -SHACL as follows (using a release that includes this option): +The library is installed with LinkML. In a uv project containing your schema, +install LinkML and generate SHACL as follows (using a release that includes this option): ```bash -uv add 'linkml[diff-stable]' +uv add linkml uv run gen-shacl --diff-stable schema.yaml > schema.shacl.ttl ``` The flag also works with `gen-owl`, `gen-rdf`, and `gen-shex --format rdf`. Python callers of `canonicalize_rdf_graph(..., diff_stable=True)` can install -`linkml-runtime[diff-stable]`. The feature is disabled by default; enabling it -changes existing labels once, so keep that regeneration separate from schema edits. +`linkml-runtime`. The `linkml[diff-stable]` and `linkml-runtime[diff-stable]` extras +remain available for compatibility with earlier installation instructions. +The feature is disabled by default; enabling it changes existing labels once, +so keep that regeneration separate from schema edits. Labels depend on blank-node neighborhoods. Edits within connected structures or among symmetric nodes can still affect other labels; minimum diffs are not diff --git a/packages/linkml_runtime/pyproject.toml b/packages/linkml_runtime/pyproject.toml index fdfd4ef4d2..54c7c252bd 100644 --- a/packages/linkml_runtime/pyproject.toml +++ b/packages/linkml_runtime/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "prefixmaps >=0.1.4", "curies>=0.14.6", "pyoxigraph>=0.5.11", + "diffable-rdf>=0.4.0", "pydantic>=2.14.1,<3.0.0", "isodate >=0.7.2, <1.0.0; python_version < '3.11'", ] diff --git a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py index bc06a969b8..07e5229a3d 100644 --- a/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py +++ b/packages/linkml_runtime/src/linkml_runtime/utils/rdf_canonicalize.py @@ -1,48 +1,23 @@ -"""Deterministic RDF serialization via pyoxigraph RDFC-1.0 canonicalization. - -This module provides a function to canonicalize an rdflib Graph using -pyoxigraph's RDFC-1.0 implementation, producing deterministic output -with stable blank node labels and sorted triples. - -**Known limitations:** - -1. **xsd:string normalization**: pyoxigraph follows RDF 1.1, where plain - string literals and ``"text"^^xsd:string`` are identical. The output - will never contain explicit ``^^xsd:string`` annotations. Code that - re-parses the output with rdflib will see ``Literal("x")`` (datatype - ``None``) rather than ``Literal("x", datatype=XSD.string)``. - -2. **Non-standard RDF**: Graphs with literal predicates (e.g. SHACL - annotation mode) or relative IRIs (e.g. the metamodel's - ``bibo:status ``) are rejected by pyoxigraph. This function - falls back to rdflib's serializer for such graphs, but still - canonicalizes blank-node labels (and sorts line-oriented formats) so - the fallback output remains deterministic across processes. - -3. **Numeric short forms**: pyoxigraph uses Turtle short forms for - ``xsd:integer`` (``42``), ``xsd:boolean`` (``true``), and - ``xsd:decimal`` (``1.23``). rdflib parses these back with the - correct datatype, so this is lossless. - -4. **Base IRI / prefix collision**: When a graph has ``@base`` and a - prefix whose namespace equals the base IRI (e.g. rdflib's auto-bound - ``base:`` prefix), pyoxigraph emits CURIEs like ``base:label`` that - rdflib rejects. We skip such prefixes during serialization. - -5. **Trailing escaped dot in PN_LOCAL**: pyoxigraph emits CURIEs like - ``prefix:local\\.`` for IRIs whose local part ends with ``.``. This - is valid Turtle (PN_LOCAL_ESC), but rdflib's notation3 parser rejects - it because it conflicts with the statement-terminator dot. We - post-process the output to expand such CURIEs to full ```` form. +"""Deterministic RDF serialization, provided by the ``diffable-rdf`` library. + +The graph is canonicalized with RDFC-1.0, sorted, and serialized. Graphs that +pyoxigraph cannot hold -- literal predicates from SHACL annotation mode, or +relative IRIs such as the metamodel's ``bibo:status `` -- fall back to +rdflib with content-derived blank-node labels, which is still reproducible +across processes. The library's ``docs/api.md`` documents the full contract +and its limitations. + +This module adapts the library's reporting: the library logs degraded output +through :mod:`logging`, which is silent without a configured handler, while +linkml emits :class:`RDFCanonicalizationWarning` so that a user running a +generator sees it by default. """ -import io -import re +import logging import warnings -import pyoxigraph as ox import rdflib -from rdflib.compare import to_canonical_graph +from diffable_rdf import canonicalize_rdf_graph as _canonicalize_rdf_graph class RDFCanonicalizationWarning(UserWarning): @@ -55,233 +30,59 @@ class RDFCanonicalizationWarning(UserWarning): """ -# Mapping from rdflib/LinkML format strings to pyoxigraph RdfFormat objects. -_FORMAT_MAP: dict[str, ox.RdfFormat] = { - "turtle": ox.RdfFormat.TURTLE, - "ttl": ox.RdfFormat.TURTLE, - "nt": ox.RdfFormat.N_TRIPLES, - "ntriples": ox.RdfFormat.N_TRIPLES, - "n-triples": ox.RdfFormat.N_TRIPLES, - "nt11": ox.RdfFormat.N_TRIPLES, - "nquads": ox.RdfFormat.N_QUADS, - "n-quads": ox.RdfFormat.N_QUADS, - "xml": ox.RdfFormat.RDF_XML, - "rdf/xml": ox.RdfFormat.RDF_XML, - "trig": ox.RdfFormat.TRIG, - "n3": ox.RdfFormat.N3, -} - -# Formats that support prefix declarations. -_PREFIX_FORMATS = frozenset({ox.RdfFormat.TURTLE, ox.RdfFormat.TRIG, ox.RdfFormat.N3, ox.RdfFormat.RDF_XML}) - -# Line-oriented formats (one triple/quad per line) whose fallback output must -# be sorted, because rdflib does not emit them in a stable order even after -# blank-node canonicalization. -_LINE_ORIENTED_FORMATS = frozenset({"nt", "ntriples", "n-triples", "nt11", "nquads", "n-quads"}) - - -def _deterministic_fallback_serialize(graph: rdflib.Graph, output_format: str) -> str: - """Serialize a graph that pyoxigraph cannot canonicalize, deterministically. - - pyoxigraph rejects some graphs that rdflib accepts -- notably graphs - containing relative IRIs (e.g. the metamodel's ``bibo:status ``) - or literal predicates (SHACL annotation mode). A plain - ``graph.serialize()`` for such graphs is *not* reproducible across - processes: rdflib assigns blank-node labels non-deterministically, so the - structure and grouping of the output varies run to run. - - To degrade gracefully instead of silently emitting non-deterministic - output, we canonicalize blank-node labels with rdflib's own - isomorphism-based canonicalization (:func:`rdflib.compare.to_canonical_graph`, - which uses a content-derived hash, not run-local ids) and preserve the - original prefix and base bindings that the canonical graph drops. For - line-oriented formats we additionally sort the serialized lines, since - rdflib does not emit N-Triples/N-Quads in a stable order. - - Relative IRIs are preserved verbatim (not resolved against the base): the - goal is deterministic output, and silently rewriting ```` into an - absolute IRI would mask what is really a data problem in the source graph. - - :param graph: The rdflib Graph that pyoxigraph could not parse. - :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). - :return: Deterministic string serialization of the graph. - """ - canonical = to_canonical_graph(graph) - # to_canonical_graph builds a fresh graph without the source's namespace - # bindings; rebind them so the output does not fall back to rdflib's - # non-deterministic auto-generated ``ns1:``/``ns2:`` prefixes. - for prefix, namespace in graph.namespace_manager.namespaces(): - canonical.namespace_manager.bind(prefix, namespace, replace=True) - canonical.base = graph.base - serialized = canonical.serialize(format=output_format) - if output_format.lower() in _LINE_ORIENTED_FORMATS: - lines = [line for line in serialized.splitlines() if line.strip()] - return "\n".join(sorted(lines)) + "\n" - return serialized - - -# Characters that may appear escaped in a Turtle PN_LOCAL via PN_LOCAL_ESC. -_PN_LOCAL_ESC_UNESCAPE = re.compile(r"\\([_~.\-!$&'()*+,;=/?#@%])") - - -_CURIE_TRAILING_DOT_PATTERN = re.compile( - r"(?\"'\[\]]*?\\\.)" - r"(?=\s)" -) - - -def _iter_turtle_structural_spans(turtle_text: str): - """Yield ``(start, end, is_structural)`` spans of a Turtle string. - - Structural spans are regions outside of string literals, IRI refs, and - line comments — i.e. the only places where a CURIE can syntactically - appear. Non-structural spans (literal contents, IRI bodies, comments) - are yielded as-is so they round-trip unchanged. - - Handles single- and triple-quoted literals with backslash escapes for - both ``"`` and ``'`` delimiters, ``<...>`` IRI refs, and ``#...`` line - comments. - """ - n = len(turtle_text) - i = 0 - while i < n: - ch = turtle_text[i] - if ch in ('"', "'"): - # String literal — find matching delimiter, respecting triple- - # quote form and backslash escapes. - delim = ch - triple = turtle_text[i : i + 3] == delim * 3 - end_marker = delim * 3 if triple else delim - j = i + len(end_marker) - while j < n: - if turtle_text[j] == "\\" and j + 1 < n: - j += 2 - continue - if turtle_text[j : j + len(end_marker)] == end_marker: - j += len(end_marker) - break - j += 1 - yield i, j, False - i = j - elif ch == "<": - # IRI ref — find closing '>' on the same logical line. - j = turtle_text.find(">", i + 1) - if j == -1: - yield i, n, False - i = n - else: - yield i, j + 1, False - i = j + 1 - elif ch == "#": - # Line comment — to end of line. - j = turtle_text.find("\n", i) - j = n if j == -1 else j - yield i, j, False - i = j - else: - # Structural region — accumulate until the next literal / IRI / - # comment opener. Treat ``\X`` as a PN_LOCAL_ESC escape (two - # chars) so that ``\#`` and ``\.`` inside a CURIE local part - # don't trigger comment / boundary handling. - start = i - while i < n: - c = turtle_text[i] - if c == "\\" and i + 1 < n: - i += 2 - continue - if c in ('"', "'", "<", "#"): - break - i += 1 - yield start, i, True - - -def _expand_trailing_dot_curies(turtle_text: str, prefixes: dict[str, str]) -> str: - """Replace CURIEs whose local part ends in ``\\.`` with full ```` form. - - rdflib's notation3 parser rejects PN_LOCAL ending in an escaped dot - even though Turtle permits it (PN_LOCAL_ESC). pyoxigraph emits this - form for IRIs ending in ``.`` (e.g. ``biolink:StrandEnum#.``). We - rewrite each such CURIE to its expanded ```` form so the output - round-trips through rdflib. - - The rewrite is applied only to *structural* spans of the document — - string literals, IRI refs, and comments are left untouched. This is - what prevents the regex from mangling CURIE-shaped substrings that - happen to appear inside a literal value. - """ - if not prefixes: - return turtle_text - - def replace(match: re.Match[str]) -> str: - prefix = match.group(1) - local_escaped = match.group(2) - namespace = prefixes.get(prefix) - if namespace is None: - return match.group(0) - local = _PN_LOCAL_ESC_UNESCAPE.sub(r"\1", local_escaped) - return f"<{namespace}{local}>" - - parts: list[str] = [] - for start, end, is_structural in _iter_turtle_structural_spans(turtle_text): - chunk = turtle_text[start:end] - if is_structural: - chunk = _CURIE_TRAILING_DOT_PATTERN.sub(replace, chunk) - parts.append(chunk) - return "".join(parts) - +_LIBRARY_LOGGER = "diffable_rdf" -def _iri_terms(triples: list["ox.Triple"]) -> set[str]: - """Return the set of IRI strings appearing anywhere in ``triples``. - Walks subjects, predicates, non-literal objects, and literal datatypes. - Used to filter the prefix dict down to namespaces that are actually - referenced by the canonicalized graph, so the output isn't padded with - unused ``@prefix`` declarations. - """ - iris: set[str] = set() - for t in triples: - for term in (t.subject, t.predicate, t.object): - if isinstance(term, ox.NamedNode): - iris.add(term.value) - elif isinstance(term, ox.Literal): - dt = term.datatype - if dt is not None: - iris.add(dt.value) - return iris +class _DegradedPathWarnings(logging.Handler): + """Collect the library's warning logs and re-emit them as Python warnings. + Records are collected during the call and re-emitted in :meth:`__exit__` + rather than from :meth:`emit`. Warning at the moment the record arrives + would put eight ``logging`` frames plus an unknown number of library frames + between :func:`warnings.warn` and the caller, and the library's depth + differs per message, so no single ``stacklevel`` could point at the caller. + Re-emitting from ``__exit__`` makes the distance a constant. -def _filter_prefixes_to_used(prefixes: dict[str, str], used_iris: set[str]) -> dict[str, str]: - """Drop prefix bindings whose namespace is not a prefix of any used IRI. + The handler is attached to the library's package logger rather than the + root logger. The library logs under module-level children such as + ``diffable_rdf.canonicalize``, which propagate to the package logger, so + attaching there catches every module without naming any of them. - A prefix is kept if at least one IRI in ``used_iris`` starts with its - namespace string. Parent-namespace matches are honored (e.g. a prefix - bound to ``http://schema.org/`` is kept when ``http://schema.org/Person`` - appears in the graph). + ``propagate`` is left alone: a caller who *did* configure logging still + gets the record through their own handlers, and seeing the message once + per mechanism is a better trade than suppressing a handler they asked for. """ - return {prefix: ns for prefix, ns in prefixes.items() if any(iri.startswith(ns) for iri in used_iris)} - -def _is_safe_prefix_iri(iri: str) -> bool: - """Check whether a namespace IRI is safe for prefix serialization. - - pyoxigraph rejects IRIs with invalid code-points (e.g. double ``#``), - and rdflib's Turtle parser cannot round-trip CURIEs whose namespace - contains query parameters or fragments in unexpected positions. This - function returns ``False`` for such IRIs so they can be skipped during - prefix collection. - """ - # A namespace IRI should end with '/' or '#'. If '#' appears - # *before* the final character, the IRI contains an embedded - # fragment which produces unusable CURIEs. - if "#" in iri[:-1]: - return False - # Query parameters in namespace IRIs produce CURIEs that rdflib - # cannot parse back. - if "?" in iri: - return False - return True + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + self._messages: list[str] = [] + self._logger = logging.getLogger(_LIBRARY_LOGGER) + self._previous_level = logging.NOTSET + + def emit(self, record: logging.LogRecord) -> None: + """Record one formatted message for re-emission on exit.""" + self._messages.append(record.getMessage()) + + def __enter__(self) -> "_DegradedPathWarnings": + """Attach to the library's logger, raising its level if it is silent.""" + self._previous_level = self._logger.level + if not self._logger.isEnabledFor(logging.WARNING): + self._logger.setLevel(logging.WARNING) + self._logger.addHandler(self) + return self + + def __exit__(self, *exc_info: object) -> None: + """Detach, restore the logger, and re-emit what was collected. + + Runs on the exception path too, so a caller who gets an exception still + learns about the degradation that preceded it. + """ + self._logger.removeHandler(self) + self._logger.setLevel(self._previous_level) + for message in self._messages: + # 3 frames: warnings.warn, this method, and the ``with`` statement + # in canonicalize_rdf_graph -- so the warning lands on its caller. + warnings.warn(message, RDFCanonicalizationWarning, stacklevel=3) def canonicalize_rdf_graph( @@ -291,151 +92,28 @@ def canonicalize_rdf_graph( ) -> str: """Serialize an rdflib Graph deterministically using RDFC-1.0 canonicalization. - The graph is transferred to pyoxigraph via N-Triples, canonicalized - with RDFC-1.0, sorted, and serialized back to the requested format. - Prefix bindings from the rdflib Graph are preserved in the output - for formats that support them (Turtle, TriG, N3, RDF/XML). + The graph is transferred to pyoxigraph via N-Triples, canonicalized with + RDFC-1.0, sorted, and serialized back to the requested format. Prefix + bindings from the rdflib Graph are preserved in the output for formats that + support them (Turtle, TriG, N3, RDF/XML). - Falls back to plain rdflib serialization for unsupported formats or - graphs containing non-standard RDF (e.g. literal predicates). + Falls back to plain rdflib serialization for unsupported formats or graphs + containing non-standard RDF (e.g. literal predicates), warning with + :class:`RDFCanonicalizationWarning` so that the caller knows the output is + less strongly guaranteed than usual. :param graph: The rdflib Graph to serialize. :param output_format: Target serialization format (e.g. ``"turtle"``, ``"nt"``). :param diff_stable: Use diffable-rdf labels to reduce blank-node churn across - edits. Requires the ``diff-stable`` extra; disabled by default. The - non-standard RDF fallback warns when it cannot apply these labels. + edits. Disabled by default. The non-standard RDF fallback warns when it + cannot apply these labels. See https://linkml.io/linkml/howtos/collaborative-development.html#rdf-in-version-control :return: Deterministic string serialization of the graph. - :raises ImportError: If diff-stable labels are requested without diffable-rdf. + :raises ValueError: If the graph cannot be serialized to ``output_format`` + in a form that parses back -- notably a line-oriented format asked to + write a relative IRI, which N-Triples forbids. """ - if diff_stable: - try: - from diffable_rdf import wl_relabel_quads - except ImportError as exc: - raise ImportError( - "diff_stable=True requires diffable-rdf. Install " - "'linkml-runtime[diff-stable]' or 'linkml[diff-stable]'." - ) from exc - - ox_format = _FORMAT_MAP.get(output_format.lower()) - if ox_format is None: - warnings.warn( - f"pyoxigraph does not support format {output_format!r}; falling back to " - "rdflib serializer. Output will not be deterministically canonicalized.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return graph.serialize(format=output_format) - - # 1. Transfer rdflib graph to pyoxigraph via N-Triples. - nt_data = graph.serialize(format="nt") - nt_bytes = nt_data.encode("utf-8") if isinstance(nt_data, str) else nt_data - - # 2. Parse into pyoxigraph and build a Dataset for canonicalization. - # Fall back to rdflib if the graph contains non-standard RDF - # (e.g. literal predicates from annotations) that pyoxigraph rejects. - try: - triples = list(ox.parse(io.BytesIO(nt_bytes), format=ox.RdfFormat.N_TRIPLES)) - except SyntaxError: - warnings.warn( - "Graph contains non-standard RDF (e.g. relative IRIs or literal predicates) " - "that pyoxigraph cannot parse; falling back to rdflib. Output is still " - "deterministic (blank-node labels are canonicalized via rdflib) but is not " - "canonicalized with pyoxigraph RDFC-1.0.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - if diff_stable: - warnings.warn( - "diff_stable=True was requested but this graph took the rdflib fallback, " - "which cannot apply Weisfeiler-Lehman blank-node labels. Output is " - "deterministic but NOT diff-stable: an unrelated edit may still renumber " - "blank nodes. Make the offending terms standard RDF (absolute IRIs, IRI " - "predicates) to get diff-stable labels.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - return _deterministic_fallback_serialize(graph, output_format) - - dataset = ox.Dataset() - for triple in triples: - dataset.add(ox.Quad(triple.subject, triple.predicate, triple.object, ox.DefaultGraph())) - - # 3. Canonicalize blank node labels with RDFC-1.0. - dataset.canonicalize(ox.CanonicalizationAlgorithm.RDFC_1_0) - - quads = list(dataset) - - if diff_stable: - quads = wl_relabel_quads(quads) - - # 4. Sort triples for deterministic ordering. - # RDFC-1.0 stabilizes blank-node labels but pyoxigraph's Dataset - # iteration order is not sorted and varies across processes (verified - # empirically against pyoxigraph 0.5.8). The explicit string-key sort - # is load-bearing for byte-identical output across runs; see - # tests/linkml_runtime/test_utils/test_rdf_canonicalize.py::test_sort_is_load_bearing. - sorted_triples = sorted( - (ox.Triple(q.subject, q.predicate, q.object) for q in quads), - key=lambda t: (str(t.subject), str(t.predicate), str(t.object)), - ) - - # 5. Collect prefixes for formats that support them. - base_iri = str(graph.base) if graph.base else None - prefixes: dict[str, str] | None = None - if ox_format in _PREFIX_FORMATS: - prefixes = {} - for prefix, namespace in graph.namespace_manager.namespaces(): - if not prefix: # skip empty prefix (base) - continue - ns_str = str(namespace) - # Skip prefixes whose namespace matches the base IRI to avoid - # pyoxigraph emitting CURIEs like `base:label` that conflict - # with the @base directive. - if base_iri and ns_str == base_iri: - continue - # Skip namespace IRIs that pyoxigraph rejects or that produce - # CURIEs rdflib cannot round-trip. Valid namespace IRIs for - # prefix use should end with '/' or '#' and contain no query - # parameters or fragment-like characters in the middle. - if not _is_safe_prefix_iri(ns_str): - continue - prefixes[str(prefix)] = ns_str - # Drop prefix bindings whose namespace is not referenced by any IRI - # in the graph. This prevents the rdflib NamespaceManager's default - # bindings (~30 well-known vocabularies) from being emitted into - # every output file regardless of whether the schema actually uses - # them. - prefixes = _filter_prefixes_to_used(prefixes, _iri_terms(sorted_triples)) - used_prefixes = prefixes - try: - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - prefixes=prefixes, - base_iri=base_iri, - ) - except ValueError as e: - # pyoxigraph 0.5.x reports rejected prefix IRIs with a message - # that begins with "Invalid prefix" (verified empirically). Only - # swallow that case and retry without prefixes — any other - # ValueError (e.g. invalid base IRI, or an unrelated future - # serializer bug) must propagate so it surfaces as a stack trace - # rather than silently dropping all prefix declarations. - if not str(e).startswith("Invalid prefix"): - raise - warnings.warn( - f"pyoxigraph rejected one or more prefix IRIs ({e}); serializing without " - "prefix declarations. Output remains canonicalized but is more verbose.", - RDFCanonicalizationWarning, - stacklevel=2, - ) - result_bytes = ox.serialize( - sorted_triples, - format=ox_format, - ) - used_prefixes = None - result = result_bytes.decode("utf-8") - if ox_format in _PREFIX_FORMATS and used_prefixes: - result = _expand_trailing_dot_curies(result, used_prefixes) - return result + # Frames between warnings.warn and this function's caller are counted in + # _DegradedPathWarnings.__exit__, which is where the re-emission happens. + with _DegradedPathWarnings(): + return _canonicalize_rdf_graph(graph, output_format=output_format, diff_stable=diff_stable) diff --git a/tests/conftest.py b/tests/conftest.py index ba13d6f764..61e75781d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -451,8 +451,8 @@ def mock_missing_import(): dependency does not need a fixture of its own:: def test_requires_the_extra(mock_missing_import): - mock_missing_import("diffable_rdf") - with pytest.raises(ImportError, match="linkml-runtime[diff-stable]"): + mock_missing_import("google.cloud.bigquery") + with pytest.raises(ImportError, match="linkml\\[bigquery\\]"): ... See :class:`MockImportErrorFinder` for the caveat about reimporting diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py index 8da48fa8b3..4ae6acbe38 100644 --- a/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize.py @@ -1,11 +1,14 @@ """Tests for deterministic RDF serialization via pyoxigraph RDFC-1.0.""" import difflib +import json +import logging import os import re import subprocess import sys import textwrap +import warnings from pathlib import Path import pyoxigraph as ox @@ -311,13 +314,13 @@ def fake_serialize(*args, **kwargs): raise ValueError("Invalid prefix bad IRI 'http://example.com/ has space/', Invalid IRI code point ' '") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="rejected one or more prefix IRIs"): + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): ttl = canonicalize_rdf_graph(g, output_format="turtle") # Fallback retry should succeed and the output still round-trips. g2 = Graph() @@ -326,22 +329,74 @@ def fake_serialize(*args, **kwargs): def test_unsupported_format_warns_and_falls_back(): - """A format pyoxigraph doesn't know about falls back to rdflib with a warning.""" + """A format the canonicalizer doesn't know about falls back to rdflib with a warning. + + ``longturtle`` is an rdflib serializer plugin with no pyoxigraph + equivalent, so it can only be produced by delegating to rdflib, and + rdflib orders that output by graph traversal. The warning is what tells + the caller the determinism guarantee does not extend to this format. + """ g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.warns(RDFCanonicalizationWarning, match="does not support format"): - result = canonicalize_rdf_graph(g, output_format="json-ld") + with pytest.warns(RDFCanonicalizationWarning, match="not one of the formats"): + result = canonicalize_rdf_graph(g, output_format="longturtle") # The fallback should still produce valid output in the requested format. assert result +def test_json_ld_is_canonicalized_rather_than_handed_to_rdflib(): + """``json-ld`` gets the determinism guarantee, not a fallback warning. + + rdflib's JSON-LD serializer emits objects in traversal order, so this used + to be an unsupported format that warned and returned whatever rdflib felt + like. It is now produced deterministically, which is why no warning is due. + """ + g = _make_graph_with_bnodes() + with warnings.catch_warnings(): + warnings.simplefilter("error", RDFCanonicalizationWarning) + result = canonicalize_rdf_graph(g, output_format="json-ld") + assert json.loads(result) + + +def test_a_rejected_base_iri_is_recovered_rather_than_fatal(monkeypatch): + """A base pyoxigraph refuses costs the base directive, not the document. + + rdflib accepts a relative or otherwise unusable ``base``, and pyoxigraph + then refuses it at serialization time. Dropping the base and retrying + yields a correct document; failing the whole call over a directive that is + optional in every output format does not. + """ + real_serialize = ox.serialize + calls = {"n": 0} + + def fake_serialize(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + return real_serialize(*args, **kwargs) + + monkeypatch.setattr(ox, "serialize", fake_serialize) + + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) + + with pytest.warns(RDFCanonicalizationWarning, match="rejected the prefix or base IRIs"): + ttl = canonicalize_rdf_graph(g, output_format="turtle") + + g2 = Graph() + g2.parse(data=ttl, format="turtle") + assert rdflib.compare.isomorphic(g, g2) + + def test_unrelated_value_error_propagates(monkeypatch): - """Any ``ValueError`` whose message doesn't begin with ``Invalid prefix`` is re-raised. + """A ``ValueError`` that is not about prefixes or the base must be re-raised. - Guards against future regressions where a pyoxigraph serializer bug - raises a different ``ValueError`` and gets silently swallowed by the - invalid-prefix fallback path. + The retry-without-prefixes path exists for two specific pyoxigraph + complaints. Widening it to every ``ValueError`` would let a serializer bug + be papered over by a retry that happens to succeed, and the caller would + get a plausible-looking document with no indication anything went wrong. """ real_serialize = ox.serialize calls = {"count": 0} @@ -350,16 +405,16 @@ def fake_serialize(*args, **kwargs): calls["count"] += 1 # First call is the prefixed serialize — raise an unrelated ValueError. if calls["count"] == 1: - raise ValueError("Invalid base IRI 'broken', Invalid IRI code point ' '") + raise ValueError("BUG: serializer state corrupted") return real_serialize(*args, **kwargs) - monkeypatch.setattr(rdf_canon_mod.ox, "serialize", fake_serialize) + monkeypatch.setattr(ox, "serialize", fake_serialize) g = Graph() g.bind("ex", "http://example.com/") g.add((URIRef("http://example.com/a"), RDF.type, URIRef("http://example.com/Thing"))) - with pytest.raises(ValueError, match="Invalid base IRI"): + with pytest.raises(ValueError, match="BUG: serializer state corrupted"): canonicalize_rdf_graph(g, output_format="turtle") @@ -462,7 +517,7 @@ def test_fallback_preserves_relative_iri(): assert "" in result -@pytest.mark.parametrize("output_format", ["turtle", "nt"]) +@pytest.mark.parametrize("output_format", ["turtle", "xml"]) def test_fallback_is_deterministic_across_processes(output_format): """The rdflib fallback produces byte-identical output across processes. @@ -470,9 +525,15 @@ def test_fallback_is_deterministic_across_processes(output_format): parse to fail, exercising the fallback path. The graph also contains several blank nodes: a plain ``graph.serialize()`` would label them non-deterministically, so this test would fail without the blank-node - canonicalization in ``_deterministic_fallback_serialize``. Two - subprocesses with different ``PYTHONHASHSEED`` values must agree byte for - byte. + canonicalization in the fallback. Two subprocesses with different + ``PYTHONHASHSEED`` values must agree byte for byte. + + ``nt`` is not covered here because a fallback graph is by definition one + pyoxigraph refused, and for this graph the reason is a relative IRI, which + N-Triples forbids outright (N-Triples 1.1 §2.2). There is no deterministic + N-Triples document to produce, so the correct answer is a refusal -- + asserted by ``test_nt_fallback_refuses_rather_than_writing_an_unparseable_file`` + below. """ program = textwrap.dedent( f""" @@ -516,6 +577,24 @@ def run(seed: str) -> str: ) +def test_nt_fallback_refuses_rather_than_writing_an_unparseable_file(): + """N-Triples output for a graph with a relative IRI must raise, not lie. + + rdflib's N-Triples serializer reuses Turtle's term rendering and does not + enforce the absolute-IRI rule, so asking it for ``nt`` here yields a file + containing ```` that its own parser then rejects. N-Triples 1.1 + §2.2 permits only absolute IRIs, so no valid document exists for this + graph and a refusal is the only honest answer. Writing the file instead + defers the failure to whoever tries to read it. + """ + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + + with pytest.raises(ValueError, match="not an absolute IRI"): + canonicalize_rdf_graph(g, output_format="nt") + + def _shapes_graph(count: int, extra: bool = False) -> Graph: """A graph shaped like generator output: one blank node per named subject.""" g = Graph() @@ -536,14 +615,6 @@ def _changed_line_count(before: str, after: str) -> int: return sum(1 for line in diff if line[:1] in "+-" and not line.startswith(("+++", "---"))) -def test_diff_stable_without_the_extra_names_the_extra(mock_missing_import) -> None: - """Requesting the opt-in path without diffable-rdf says how to install it.""" - mock_missing_import("diffable_rdf") - - with pytest.raises(ImportError, match=r"linkml-runtime\[diff-stable\]"): - canonicalize_rdf_graph(_make_graph_with_bnodes(), diff_stable=True) - - @pytest.mark.parametrize("diff_stable", [False, pytest.param(True, marks=pytest.mark.diffable_rdf)]) @pytest.mark.parametrize("output_format", ["nt", "turtle"]) def test_diff_stable_preserves_semantics(diff_stable: bool, output_format: str) -> None: @@ -750,8 +821,83 @@ def test_diff_stable_warns_instead_of_silently_no_opping_on_the_fallback() -> No """A relative IRI requires the fallback and a warning that labels cannot apply.""" graph = _make_graph_with_bnodes() graph.add((URIRef("testing"), URIRef("http://example.com/p"), Literal("v"))) - with pytest.warns(RDFCanonicalizationWarning, match="NOT diff-stable"): + with pytest.warns(RDFCanonicalizationWarning, match="not diff-stable"): stable = canonicalize_rdf_graph(graph, diff_stable=True) with pytest.warns(RDFCanonicalizationWarning): plain = canonicalize_rdf_graph(graph, diff_stable=False) assert stable == plain + + +# -------------------------------------------------------------------------- +# The bridge from the library's logging to linkml's warnings +# -------------------------------------------------------------------------- + + +def _fallback_graph() -> Graph: + """A graph pyoxigraph refuses, so every call takes a degraded path.""" + g = Graph() + g.bind("ex", "http://example.com/") + g.add((URIRef("http://example.com/s"), URIRef("http://purl.org/ontology/bibo/status"), URIRef("testing"))) + return g + + +def test_degraded_path_warning_points_at_the_caller(): + """The warning must name the line that asked for the serialization. + + ``diffable-rdf`` reports degradation through ``logging``; linkml re-emits + it through ``warnings`` so it is visible without logging configuration. + A re-emitted warning is only actionable if it is attributed to the caller + rather than to the adapter, and the number of frames in between is not + something a reader can eyeball -- hence this test. + """ + graph = _fallback_graph() + + with pytest.warns(RDFCanonicalizationWarning) as caught: + canonicalize_rdf_graph(graph, output_format="turtle") # attribution target + + assert caught[0].filename == __file__ + source = Path(caught[0].filename).read_text(encoding="utf-8").splitlines() + assert "# attribution target" in source[caught[0].lineno - 1] + + +def test_bridging_does_not_leave_the_library_logger_modified(): + """Raising the library's log level to capture records must not be permanent. + + The adapter has to enable ``WARNING`` on the ``diffable_rdf`` logger to see + anything, which is a global mutation. Leaving it raised would silently + change logging behaviour for the rest of the process. + """ + logger = logging.getLogger("diffable_rdf") + level_before = logger.level + handlers_before = list(logger.handlers) + + with pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert logger.level == level_before + assert logger.handlers == handlers_before + + +def test_a_caller_who_configured_logging_still_receives_the_record(caplog): + """Re-emitting as a warning must not steal the record from a log handler. + + An application that deliberately configured the ``diffable_rdf`` logger is + asking for these records. The adapter adds a mechanism; it does not get to + remove one. + """ + with caplog.at_level(logging.WARNING, logger="diffable_rdf"), pytest.warns(RDFCanonicalizationWarning): + canonicalize_rdf_graph(_fallback_graph(), output_format="turtle") + + assert any("non-standard RDF" in record.getMessage() for record in caplog.records) + + +def test_warnings_survive_an_exception_from_the_library(): + """Degradation reported before a failure must still reach the caller. + + ``nt`` output for this graph degrades to rdflib and *then* refuses, because + N-Triples has no way to write the relative IRI. Dropping the warning + because the call ended in an exception would hide the first half of the + story, which is the half that explains the second. + """ + with pytest.warns(RDFCanonicalizationWarning, match="non-standard RDF"), pytest.raises(ValueError): + canonicalize_rdf_graph(_fallback_graph(), output_format="nt") diff --git a/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py new file mode 100644 index 0000000000..e72410dd18 --- /dev/null +++ b/tests/linkml_runtime/test_utils/test_rdf_canonicalize_defects.py @@ -0,0 +1,301 @@ +"""Correctness properties of RDF canonicalization, asserted through both entry points. + +``linkml_runtime.utils.rdf_canonicalize.canonicalize_rdf_graph`` delegates to +``diffable_rdf.canonicalize_rdf_graph``. Each test asserts one property -- +for example that a base IRI ending in ``#`` does not rewrite other IRIs, that a +shared ``rdf:List`` tail is not duplicated, or that output is byte-identical +across processes -- against both functions, so that the adapter cannot change +what the library guarantees. +""" + +import os +import subprocess +import sys +import textwrap +import warnings + +import diffable_rdf +import pytest +import rdflib +from rdflib import BNode, Graph, Literal, Namespace, URIRef +from rdflib.namespace import RDF + +from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph as linkml_canonicalize + +EX = Namespace("http://example.org/") + +_IMPLEMENTATIONS = ( + (linkml_canonicalize, "linkml"), + (diffable_rdf.canonicalize_rdf_graph, "diffable-rdf"), +) + + +def _impls(): + """Parametrize a test over both entry points. + + linkml's is a thin adapter over the library's, so the two agree by + construction today. Running both anyway is what turns a future re-fork, or + an adapter that quietly changes behaviour on the way through, into a test + failure. + """ + return [pytest.param(fn, id=ident) for fn, ident in _IMPLEMENTATIONS] + + +def _apply(fn, graph, output_format="turtle"): + """Call an implementation, ignoring the warnings both emit on the fallback path.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(graph, output_format) + + +# -------------------------------------------------------------------------- +# Silent data corruption +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_with_a_fragment_does_not_rewrite_every_iri(canonicalize): + """A base IRI ending in ``#`` must not change the graph's terms. + + Relativizing ``http://ex.org/d#a`` against base ``http://ex.org/d#`` gives + ``<#a>``, which is correct per RFC 3986 -- but rdflib's parser resolves a + fragment reference by concatenation and reads it back as + ``http://ex.org/d##a``. Every term of the graph silently changes, and the + output parses cleanly, so nothing reports it. + """ + graph = Graph(base="http://ex.org/d#") + graph.add((URIRef("http://ex.org/d#a"), EX.p, Literal("v"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert {str(s) for s in round_tripped.subjects()} == {"http://ex.org/d#a"} + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_a_shared_rdf_list_tail_is_not_duplicated(canonicalize): + """Two lists sharing a tail must not gain triples on the way out. + + rdflib's Turtle writer renders ``( ... )`` collection syntax per list, so a + tail referenced from two lists is written twice as two separate blank + nodes. The result asserts more than the input did. + """ + graph = Graph() + graph.bind("ex", EX) + tail = BNode() + graph.add((tail, RDF.first, Literal("shared"))) + graph.add((tail, RDF.rest, RDF.nil)) + for name in ("l1", "l2"): + head = BNode() + graph.add((EX[name], EX.list, head)) + graph.add((head, RDF.first, Literal(name))) + graph.add((head, RDF.rest, tail)) + # A relative IRI is non-standard RDF, so pyoxigraph refuses the graph and + # both implementations degrade to rdflib -- the path this defect lives on. + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + round_tripped = Graph() + round_tripped.parse(data=_apply(canonicalize, graph), format="turtle") + + assert len(round_tripped) == len(graph) + + +# -------------------------------------------------------------------------- +# Output that no parser will read +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_nt_output_either_parses_or_refuses(canonicalize): + """N-Triples admits only absolute IRIs, so a relative one must not be written. + + The fallback is taken *because* the graph holds a term pyoxigraph rejected. + Writing it out anyway produces a file that fails on line 1. Refusing with a + message naming the term and a format that can carry the graph is the fix; + returning unreadable text is not. + """ + graph = Graph() + graph.add((URIRef("testing"), EX.p, Literal("v"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused, with an explanation -- the correct outcome + + rdflib.Graph().parse(data=serialized, format="nt") + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_a_unicode_line_separator_in_a_literal_survives(canonicalize): + """Sorting N-Triples lines must split on newlines only. + + N-Triples permits U+2028 raw inside a quoted literal, but + ``str.splitlines()`` breaks on it as well as on U+2029, U+0085, U+000B, + U+000C and U+001C-1E. One statement becomes two lines, the halves sort + independently, the separator is rewritten as a newline, and the document + no longer parses. + """ + graph = Graph() + graph.add((EX.s, EX.p, Literal("a\u2028b"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + try: + serialized = _apply(canonicalize, graph, "nt") + except ValueError: + return # refused because of the relative IRI, before the sort + + assert "\u2028" in serialized + rdflib.Graph().parse(data=serialized, format="nt") + + +# -------------------------------------------------------------------------- +# Non-determinism, which is the property the module exists to provide +# -------------------------------------------------------------------------- + +_ACROSS_PROCESSES = textwrap.dedent( + """ + import sys, warnings + warnings.simplefilter("ignore") + from rdflib import BNode, Graph, Literal, Namespace, URIRef + from rdflib.namespace import RDF + + module, output_format, force_fallback = sys.argv[1], sys.argv[2], sys.argv[3] == "1" + if module == "linkml": + from linkml_runtime.utils.rdf_canonicalize import canonicalize_rdf_graph + else: + from diffable_rdf import canonicalize_rdf_graph + + EX = Namespace("http://example.org/") + g = Graph() + if force_fallback: + g.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + for i in range(12): + b = BNode() + g.add((EX["s%02d" % i], EX.child, b)) + g.add((b, EX.k, Literal("v%d" % i))) + g.add((b, RDF.type, EX.T)) + for host in "abcdefgh": + g.add((URIRef("http://%s.example/s" % host), + URIRef("http://%s.example/p" % host), + URIRef("http://%s.example/o" % host))) + sys.stdout.write(canonicalize_rdf_graph(g, output_format)) + """ +) + + +def _outputs_across_processes(module, output_format, force_fallback): + """Serialize the same graph in four processes with different hash seeds.""" + outputs = set() + for seed in ("0", "1", "12345", "999"): + completed = subprocess.run( + [sys.executable, "-c", _ACROSS_PROCESSES, module, output_format, "1" if force_fallback else "0"], + capture_output=True, + text=True, + encoding="utf-8", + env={**os.environ, "PYTHONHASHSEED": seed}, + check=False, + ) + if completed.returncode != 0: + pytest.fail(f"{module}/{output_format} failed:\n{completed.stderr[-2000:]}") + outputs.add(completed.stdout) + return outputs + + +_XML_TRAVERSAL = "degraded RDF/XML is ordered by rdflib's graph traversal, which follows set iteration order" +_NS_NAMES = "auto-generated ns1/ns2 prefix names are allocated in traversal order" +_JSONLD_UNMAPPED = "json-ld is absent from the format map, so it falls through to rdflib with no determinism guarantee" + + +@pytest.mark.parametrize( + ("module", "output_format", "force_fallback"), + [ + pytest.param("linkml", "xml", True, id="linkml-xml-fallback"), + pytest.param("diffable_rdf", "xml", True, id="diffable-rdf-xml-fallback"), + pytest.param("linkml", "turtle", True, id="linkml-turtle-fallback"), + pytest.param("diffable_rdf", "turtle", True, id="diffable-rdf-turtle-fallback"), + pytest.param("linkml", "json-ld", False, id="linkml-jsonld"), + pytest.param("diffable_rdf", "json-ld", False, id="diffable-rdf-jsonld"), + ], +) +def test_output_is_byte_identical_across_processes(module, output_format, force_fallback): + """The same graph must serialize to the same bytes in any process. + + Three separate causes used to break this in the in-tree copy, and all three + are fixed by delegating. Degraded RDF/XML was ordered by rdflib's own graph + traversal, which follows set iteration order. Auto-generated ``ns1``/``ns2`` + prefix names were allocated in traversal order too, so the *names* moved + even when the triples did not. And ``json-ld`` was not in the format map at + all, so it fell through to rdflib's serializer, which carries no + determinism guarantee. + """ + assert len(_outputs_across_processes(module, output_format, force_fallback)) == 1 + + +# -------------------------------------------------------------------------- +# Interface +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_every_format_ends_with_exactly_one_newline(canonicalize): + """A committed artifact should not depend on which format wrote it. + + Passing through whatever the serializer produced gives no trailing newline + for RDF/XML and one for Turtle, so POSIX text tools disagree about whether + the file has a last line. + """ + counts = {} + for output_format in ("turtle", "nt", "xml", "trig", "n3"): + graph = Graph() + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + serialized = _apply(canonicalize, graph, output_format) + counts[output_format] = len(serialized) - len(serialized.rstrip("\n")) + + assert counts == dict.fromkeys(counts, 1) + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_a_dataset_is_refused_rather_than_partly_serialized(canonicalize): + """A Dataset is a Graph subclass, so it is accepted and quietly flattened. + + Every named graph collapses into one document and the graph names are + dropped, which is a different dataset. The caller asked for something this + function cannot express and should be told so. + """ + dataset = rdflib.Dataset() + dataset.graph(URIRef("http://ex/g1")).add((EX.a, EX.p, Literal("in-g1"))) + dataset.graph(URIRef("http://ex/g2")).add((EX.b, EX.p, Literal("in-g2"))) + + with pytest.raises(TypeError): + _apply(canonicalize, dataset) + + +# -------------------------------------------------------------------------- +# A property both implementations now hold +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("canonicalize", _impls()) +def test_base_survives_the_degraded_path(canonicalize): + """``@base`` must not disappear just because pyoxigraph refused the graph. + + ``rdflib_dumper.dumps(..., prefix_map={"@base": ...})`` is a supported way + to ask for a document whose IRIs are written relative to a base, and the + metamodel routinely produces graphs that fall back (a bare ``status: + testing`` on a ``uriorcurie`` slot serializes to the relative ````, + which pyoxigraph rejects). Losing the directive on exactly that path means + the feature works only for graphs that never needed the fallback. + + This ran the other way until diffable-rdf 0.4.0. It was the last property + blocking delegation, because linkml held it and the library did not, so + adopting the library would have been a regression. Fixing it there rather + than keeping the in-tree copy alive is what let the other nine gaps close + at once. + """ + graph = Graph(base="http://example.org/default/") + graph.bind("ex", EX) + graph.add((EX.s, EX.p, Literal("v"))) + graph.add((URIRef("testing"), EX.p, Literal("forces-fallback"))) + + assert "@base ." in _apply(canonicalize, graph) diff --git a/uv.lock b/uv.lock index 03c3c00d3c..7600803f05 100644 --- a/uv.lock +++ b/uv.lock @@ -2633,6 +2633,7 @@ dependencies = [ { name = "click" }, { name = "curies" }, { name = "deprecated" }, + { name = "diffable-rdf" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'" }, { name = "json-flattener" }, @@ -2668,6 +2669,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'" }, { name = "curies", specifier = ">=0.14.6" }, { name = "deprecated" }, + { name = "diffable-rdf", specifier = ">=0.4.0" }, { name = "diffable-rdf", marker = "extra == 'diff-stable'", specifier = ">=0.4.0" }, { name = "hbreader" }, { name = "isodate", marker = "python_full_version < '3.11'", specifier = ">=0.7.2,<1.0.0" }, From a791f01ba66050991bd2a2a79a662011d761f38d Mon Sep 17 00:00:00 2001 From: jdsika Date: Fri, 23 Oct 2026 14:53:13 +0200 Subject: [PATCH 16/17] test(integration): verify the interactions of the integrated generator features Consumer checks for features that meet on the integration branch: an any_of branch keeps its pattern next to a nested range_expression after RDF serialization, in both RDF label modes; and one OWL resource keeps its metaclass membership (instantiates) next to its declared IRI-valued annotation, while a type keeps its membership (gen-owl writes no annotations on types). Signed-off-by: jdsika --- .../test_generator_feature_integration.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tests/linkml/test_generators/test_generator_feature_integration.py diff --git a/tests/linkml/test_generators/test_generator_feature_integration.py b/tests/linkml/test_generators/test_generator_feature_integration.py new file mode 100644 index 0000000000..0a9dce359e --- /dev/null +++ b/tests/linkml/test_generators/test_generator_feature_integration.py @@ -0,0 +1,100 @@ +"""Consumer checks for features that overlap on the integration branch.""" + +import pytest +from pyshacl import validate +from rdflib import OWL, RDF, Graph, Namespace + +from linkml.generators.owlgen import OwlSchemaGenerator +from linkml.generators.shaclgen import ShaclGenerator + +EX = Namespace("https://example.org/integration/") + + +@pytest.mark.parametrize("diff_stable", [False, True]) +@pytest.mark.parametrize( + "value,details,expected", + [ + ("ex:allowed", 'ex:allowed a ex:Resource ; ex:category "ok" .', True), + ("ex:other", 'ex:other a ex:Resource ; ex:category "ok" .', False), + ("ex:allowed", 'ex:allowed a ex:Resource ; ex:category "wrong" .', False), + ("ex:allowed", "ex:allowed a ex:Resource .", False), + ('"literal"', "", True), + ('"other"', "", False), + ], +) +def test_any_of_combines_pattern_and_range_expression( + diff_stable: bool, value: str, details: str, expected: bool +) -> None: + """Each branch enforces its pattern and nested constraints together after RDF serialization.""" + schema = """ +id: https://example.org/integration +name: integration +prefixes: + ex: https://example.org/integration/ + linkml: https://w3id.org/linkml/ +default_prefix: ex +imports: [linkml:types] +classes: + Thing: + attributes: + value: + any_of: + - range: Resource + pattern: /allowed$ + range_expression: + slot_conditions: + category: {equals_string: ok, required: true} + - range: string + pattern: ^literal$ + Resource: + attributes: + category: {range: string} +""" + shapes = ShaclGenerator(schema, closed=False, diff_stable=diff_stable).serialize() + data = f"@prefix ex: <{EX}> . ex:thing a ex:Thing ; ex:value {value} . {details}" + conforms, _, report = validate( + data_graph=data, + data_graph_format="turtle", + shacl_graph=shapes, + shacl_graph_format="turtle", + meta_shacl=True, + ) + assert conforms is expected, report + + +@pytest.mark.parametrize("diff_stable", [False, True]) +def test_instantiates_and_declared_annotation_on_one_resource(diff_stable: bool) -> None: + """A class keeps its metaclass membership (#31) next to its declared IRI-valued annotation (#58); + a type keeps its membership, and gen-owl writes no annotations on types.""" + schema = """ +id: https://example.org/integration +name: integration +prefixes: + ex: https://example.org/integration/ + linkml: https://w3id.org/linkml/ + owl: http://www.w3.org/2002/07/owl# +default_prefix: ex +imports: [linkml:types] +classes: + Metadata: + class_uri: ex:Metadata + attributes: + reference: + slot_uri: ex:reference + range: nodeidentifier + implements: [owl:AnnotationProperty] + Annotated: + class_uri: ex:Annotated + instantiates: [ex:Metadata] + annotations: + reference: ex:target +types: + Label: + typeof: string + instantiates: [ex:Metadata] +""" + graph = Graph().parse(data=OwlSchemaGenerator(schema, diff_stable=diff_stable).serialize(), format="turtle") + assert (EX.Annotated, RDF.type, EX.Metadata) in graph + assert (EX.Annotated, EX.reference, EX.target) in graph + assert (EX.reference, RDF.type, OWL.AnnotationProperty) in graph + assert (EX.Label, RDF.type, EX.Metadata) in graph From 88e3a8085326a85f42bf4f434b89a3beecb8fe1d Mon Sep 17 00:00:00 2001 From: jdsika Date: Fri, 23 Oct 2026 14:53:14 +0200 Subject: [PATCH 17/17] ci(integration): run the OWLAPI consumer on gen-owl output A separate job resolves the pinned OWLAPI 5.5.0 tree with Maven and runs a Java probe on gen-owl output: OWL 2 DL profile, no unparsed triples, and the expected annotations kept through a round trip. It checks the documented DCMI example in four modes and a schema where metaclass membership, a declared annotation, annotation properties with a super-property and a built-in IRI, and an ordinary data property meet. Both profile tests are strict expected failures: gen-owl writes metadata annotations (dcterms:title, skos:definition, skos:inScheme, linkml:permissible_values) without declaring their annotation properties, and asserts membership in external classes without declaring them, which upstream main does for any schema too. They fail loudly once the declarations exist. Signed-off-by: jdsika --- .../workflows/owl-annotation-consumer.yaml | 46 +++++ .../AnnotationProbe.java | 74 ++++++++ .../input/owl_annotation_properties/pom.xml | 15 ++ .../test_owl_annotation_consumer.py | 159 ++++++++++++++++++ 4 files changed, 294 insertions(+) create mode 100644 .github/workflows/owl-annotation-consumer.yaml create mode 100644 tests/linkml/test_generators/input/owl_annotation_properties/AnnotationProbe.java create mode 100644 tests/linkml/test_generators/input/owl_annotation_properties/pom.xml create mode 100644 tests/linkml/test_generators/test_owl_annotation_consumer.py diff --git a/.github/workflows/owl-annotation-consumer.yaml b/.github/workflows/owl-annotation-consumer.yaml new file mode 100644 index 0000000000..bbedbf3471 --- /dev/null +++ b/.github/workflows/owl-annotation-consumer.yaml @@ -0,0 +1,46 @@ +name: OWL annotation consumer + +on: + pull_request: + push: + branches: [main] + merge_group: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + owlapi: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.2.0 + with: + version: "0.11.21" + enable-cache: true + - uses: actions/setup-python@v7.0.0 + with: + python-version: "3.12" + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + cache: maven + cache-dependency-path: tests/linkml/test_generators/input/owl_annotation_properties/pom.xml + - name: Install Python dependencies + run: uv sync --all-groups + - name: Resolve the pinned OWLAPI dependency tree once + run: >- + mvn --batch-mode + -f tests/linkml/test_generators/input/owl_annotation_properties/pom.xml + org.apache.maven.plugins:maven-dependency-plugin:3.8.1:build-classpath + -Dmdep.outputFile=${{ runner.temp }}/owlapi-classpath.txt + - name: Require profile validity and annotation retention on round-trip + env: + LINKML_REQUIRE_OWLAPI: "1" + LINKML_OWLAPI_CLASSPATH_FILE: ${{ runner.temp }}/owlapi-classpath.txt + run: >- + uv run pytest tests/linkml/test_generators/test_owl_annotation_consumer.py + -q -m integration diff --git a/tests/linkml/test_generators/input/owl_annotation_properties/AnnotationProbe.java b/tests/linkml/test_generators/input/owl_annotation_properties/AnnotationProbe.java new file mode 100644 index 0000000000..529b1df1e0 --- /dev/null +++ b/tests/linkml/test_generators/input/owl_annotation_properties/AnnotationProbe.java @@ -0,0 +1,74 @@ +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.semanticweb.owlapi.apibinding.OWLManager; +import org.semanticweb.owlapi.formats.TurtleDocumentFormat; +import org.semanticweb.owlapi.io.RDFParserMetaData; +import org.semanticweb.owlapi.model.IRI; +import org.semanticweb.owlapi.model.OWLAnnotationValue; +import org.semanticweb.owlapi.model.OWLOntology; +import org.semanticweb.owlapi.profiles.OWL2DLProfile; + +/** Check generated artifacts with the pinned OWLAPI, including annotation retention on round-trip. */ +class AnnotationProbe { + private static void verify(OWLOntology ontology, List expectations) { + var report = new OWL2DLProfile().checkOntology(ontology); + if (!report.isInProfile()) { + throw new AssertionError(report.toString()); + } + if (ontology.signature().anyMatch(entity -> + entity.getIRI().toString().startsWith("http://org.semanticweb.owlapi/error#"))) { + throw new AssertionError("Parser recovery entities in ontology"); + } + var metadata = ontology.getOWLOntologyManager().getOntologyFormat(ontology).getOntologyLoaderMetaData(); + if (metadata.isEmpty() || !(metadata.get() instanceof RDFParserMetaData)) { + throw new AssertionError("Missing RDF parser diagnostics"); + } + var rdfMetadata = (RDFParserMetaData) metadata.get(); + if (rdfMetadata.getUnparsedTriples().findAny().isPresent()) { + throw new AssertionError("Unparsed RDF triples: " + rdfMetadata.getUnparsedTriples().toList()); + } + var factory = ontology.getOWLOntologyManager().getOWLDataFactory(); + for (var line : expectations) { + // subject (or 'ontology'), predicate, value kind, value, datatype, language. + var fields = line.split("\t", -1); + var property = factory.getOWLAnnotationProperty(IRI.create(fields[1])); + OWLAnnotationValue value; + if (fields[2].equals("IRI")) { + value = IRI.create(fields[3]); + } else if (!fields[5].isEmpty()) { + value = factory.getOWLLiteral(fields[3], fields[5]); + } else if (!fields[4].isEmpty()) { + value = factory.getOWLLiteral(fields[3], factory.getOWLDatatype(IRI.create(fields[4]))); + } else { + value = factory.getOWLLiteral(fields[3]); + } + boolean retained = fields[0].equals("ontology") + ? ontology.getAnnotations().contains(factory.getOWLAnnotation(property, value)) + : ontology.containsAxiom(factory.getOWLAnnotationAssertionAxiom( + property, IRI.create(fields[0]), value)); + if (!retained) { + throw new AssertionError("Expected annotation lost: " + line); + } + } + } + + public static void main(String[] args) throws Exception { + var source = OWLManager.class.getProtectionDomain().getCodeSource().getLocation().toString(); + if (!source.endsWith("/owlapi-distribution-5.5.0.jar")) { + throw new AssertionError("Unexpected OWLAPI artifact: " + source); + } + var manager = OWLManager.createOWLOntologyManager(); + var ontology = manager.loadOntologyFromOntologyDocument(new File(args[0])); + var expectations = Files.readAllLines(Path.of(args[2])); + if (expectations.isEmpty()) { + throw new AssertionError("No annotation expectations supplied"); + } + verify(ontology, expectations); + var saved = new File(args[1]); + manager.saveOntology(ontology, new TurtleDocumentFormat(), IRI.create(saved)); + verify(OWLManager.createOWLOntologyManager().loadOntologyFromOntologyDocument(saved), expectations); + System.out.println("PASS: OWLAPI 5.5.0; OWL 2 DL; expected annotations survive round-trip"); + } +} diff --git a/tests/linkml/test_generators/input/owl_annotation_properties/pom.xml b/tests/linkml/test_generators/input/owl_annotation_properties/pom.xml new file mode 100644 index 0000000000..ca84f0daab --- /dev/null +++ b/tests/linkml/test_generators/input/owl_annotation_properties/pom.xml @@ -0,0 +1,15 @@ + + 4.0.0 + org.linkml.tests + annotation-consumer-check + 1.0 + + + net.sourceforge.owlapi + owlapi-distribution + 5.5.0 + + + diff --git a/tests/linkml/test_generators/test_owl_annotation_consumer.py b/tests/linkml/test_generators/test_owl_annotation_consumer.py new file mode 100644 index 0000000000..9327b33a53 --- /dev/null +++ b/tests/linkml/test_generators/test_owl_annotation_consumer.py @@ -0,0 +1,159 @@ +"""gen-owl output that a real OWL consumer accepts: OWLAPI 5.5.0 checks the OWL 2 DL profile and keeps the annotations. + +The fork's integration branch runs these in its own CI job (``owl-annotation-consumer.yaml``), which prepares the +pinned OWLAPI classpath with Maven; elsewhere they are skipped. They cover the features that meet on OWL resources: +metaclass membership (``instantiates``), declared annotation values and annotation property roles. +""" + +import os +import subprocess +from pathlib import Path + +import pytest +from rdflib import DCTERMS, RDFS, URIRef + +from linkml.generators.owlgen import OwlSchemaGenerator + +ROOT = Path(__file__).resolve().parents[3] +HARNESS = Path(__file__).parent / "input/owl_annotation_properties" +DCMI_EXAMPLE = ROOT / "docs/examples/owl-dcmi-agent-annotations.yaml" +EX = "https://example.org/consumer/" + +SCHEMA = """ +id: https://example.org/consumer +name: consumer +prefixes: + ex: https://example.org/consumer/ + linkml: https://w3id.org/linkml/ + owl: http://www.w3.org/2002/07/owl# + rdfs: http://www.w3.org/2000/01/rdf-schema# + vocab: https://example.org/vocab/ +default_prefix: ex +default_range: string +imports: [linkml:types] +slots: + comment: + slot_uri: rdfs:comment + implements: [owl:AnnotationProperty] + note: + implements: [owl:AnnotationProperty] + is_a: comment + plain: {} +classes: + Metadata: + attributes: + reference: + range: nodeidentifier + implements: [owl:AnnotationProperty] + Thing: + instantiates: [ex:Metadata, vocab:Profile] + annotations: + reference: ex:target + slots: [note, plain] + attributes: + name: + slot_uri: rdfs:label +enums: + Choice: + implements: [owl:NamedIndividual] + permissible_values: + A: + meaning: ex:A + instantiates: [vocab:Category] +""" + + +# gen-owl writes metadata annotations (dcterms:title, skos:definition, skos:inScheme, linkml:permissible_values) +# without declaring their annotation properties, and #31 asserts membership in external classes without declaring +# them, so OWLAPI finds the output outside the OWL 2 DL profile; upstream main fails the same way for any schema. +# The earlier integration branch passed because the old #33 declared them; the re-cut #58/#59 do not. +_UNDECLARED = pytest.mark.xfail( + strict=True, + raises=subprocess.CalledProcessError, + reason="undeclared metadata annotation properties and instantiated classes put gen-owl outside OWL 2 DL", +) + + +@pytest.fixture +def owlapi_classpath() -> str: + """The OWLAPI classpath the CI job prepared; pytest never fetches it.""" + filename = os.environ.get("LINKML_OWLAPI_CLASSPATH_FILE") + if not filename: + if os.environ.get("LINKML_REQUIRE_OWLAPI") == "1": + pytest.fail("Required OWLAPI gate has no LINKML_OWLAPI_CLASSPATH_FILE") + pytest.skip("The OWLAPI consumer runs in its own CI job, with a prepared classpath") + classpath = Path(filename).read_text().strip() + assert classpath, "Empty prepared OWLAPI classpath" + return classpath + + +def _probe(source: str | Path, tmp_path: Path, classpath: str, rows: list[list[str]], **options: object) -> None: + """Run the Java consumer on the generated ontology: OWL 2 DL profile, no unparsed triples, and *rows* + (owner or ``ontology``, predicate, ``IRI`` or ``LITERAL``, value, datatype, language) kept on a round trip.""" + generated = tmp_path / "generated.ttl" + generated.write_text(OwlSchemaGenerator(str(source), **options).serialize()) + expectations = tmp_path / "expected.tsv" + expectations.write_text("\n".join("\t".join(row) for row in rows) + "\n") + subprocess.run( + [ + "java", + "--class-path", + classpath, + str(HARNESS / "AnnotationProbe.java"), + str(generated), + str(tmp_path / "roundtrip.ttl"), + str(expectations), + ], + check=True, + ) + + +@_UNDECLARED +@pytest.mark.integration +@pytest.mark.parametrize("native", [True, False]) +@pytest.mark.parametrize("type_objects", [False, True]) +def test_documented_dcmi_example(tmp_path: Path, owlapi_classpath: str, native: bool, type_objects: bool) -> None: + """The DCMI example of the OWL documentation is OWL 2 DL and keeps its creator IRI and rights text.""" + rows = [ + ["ontology", str(DCTERMS.creator), "IRI", "https://orcid.org/0000-0002-1825-0097", "", ""], + ["ontology", str(DCTERMS.rights), "LITERAL", "Copyright 2026 The Example Team", "", ""], + ] + _probe( + DCMI_EXAMPLE, + tmp_path, + owlapi_classpath, + rows, + use_native_uris=native, + type_objects=type_objects, + metaclasses=False, + ) + + +@_UNDECLARED +@pytest.mark.integration +@pytest.mark.parametrize("native", [True, False]) +def test_membership_annotation_and_roles_together( + tmp_path: Path, owlapi_classpath: str, native: bool, tmp_path_factory: pytest.TempPathFactory +) -> None: + """Metaclass membership on a class and a permissible value, a declared annotation, annotation properties + with a super-property and a built-in IRI, and an ordinary data property make one OWL 2 DL ontology.""" + schema = tmp_path_factory.mktemp("schema") / "consumer.yaml" + schema.write_text(SCHEMA) + rows = [[EX + "Thing", EX + "reference", "IRI", EX + "target", "", ""]] + _probe(schema, tmp_path, owlapi_classpath, rows, use_native_uris=native, metaclasses=False, type_objects=False) + + +def test_consumer_schema_projects_as_expected() -> None: + """The consumer schema exercises what the probe checks, so a passing probe means something.""" + from rdflib import OWL, RDF, Graph + + # the command line's defaults; the Python API defaults to type objects, which make every slot an object property + generator = OwlSchemaGenerator(SCHEMA, metaclasses=False, type_objects=False) + graph = Graph().parse(data=generator.serialize(), format="turtle") + thing, vocab = URIRef(EX + "Thing"), "https://example.org/vocab/" + assert {URIRef(EX + "Metadata"), URIRef(vocab + "Profile")} <= set(graph.objects(thing, RDF.type)) + assert (thing, URIRef(EX + "reference"), URIRef(EX + "target")) in graph + assert (URIRef(EX + "A"), RDF.type, URIRef(vocab + "Category")) in graph + assert (URIRef(EX + "note"), RDFS.subPropertyOf, URIRef(EX + "comment")) in graph + assert (URIRef(EX + "note"), RDF.type, OWL.AnnotationProperty) in graph + assert (URIRef(EX + "plain"), RDF.type, OWL.DatatypeProperty) in graph