From be86c658319ee28133575aae71dff0e44cfa7df2 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Thu, 12 Nov 2026 21:21:37 +0800 Subject: [PATCH 1/2] knowledge: move seven bodies into inferrsi.knowledge and revise the manifest Move the seven Knowledge implementations into inferrsi.knowledge, leave static exact-forwarding shells at the old root names, and add the _repo_pack_data leaf for REPO_TYPES and repository_pack. Rewire infra.inspection to that leaf. Revise the legacy manifest at the K01 base (hashes, public exports, technical leaf) and protect the new paths. Root shells and the three root protected paths stay until R01. overall isolation_complete stays false. Agent-assisted-by: Claude Code / claude-opus-5-5 effort=xhigh (author) Signed-off-by: GUOGUO --- .github/knowledge-model-policy.json | 5 +- docs/knowledge-model-policy.md | 3 + docs/legacy-knowledge-manifest.json | 45 +- docs/module-layout.md | 23 +- src/inferrsi/_repo_pack_data.py | 16 + src/inferrsi/_skill_selection.py | 237 +----- src/inferrsi/_source_snapshot.py | 110 +-- src/inferrsi/catalog.py | 199 +---- src/inferrsi/infra/inspection.py | 2 +- src/inferrsi/knowledge/__init__.py | 2 + src/inferrsi/knowledge/_skill_selection.py | 211 +++++ src/inferrsi/knowledge/_source_snapshot.py | 88 ++ src/inferrsi/knowledge/catalog.py | 168 ++++ .../knowledge/knowledge_operations.py | 735 +++++++++++++++++ src/inferrsi/knowledge/knowledge_store.py | 297 +++++++ src/inferrsi/knowledge/research.py | 127 +++ src/inferrsi/knowledge/skill_catalog.py | 291 +++++++ src/inferrsi/knowledge_operations.py | 762 +----------------- src/inferrsi/knowledge_store.py | 297 +------ src/inferrsi/research.py | 127 +-- src/inferrsi/skill_catalog.py | 336 ++------ tests/test_architecture_closed_loop.py | 3 +- tests/test_bundle_manifest.py | 4 +- tests/test_cli_catalog.py | 4 +- tests/test_collect_limit_serialization.py | 4 +- tests/test_knowledge_compatibility_shells.py | 82 ++ tests/test_knowledge_model_policy.py | 2 + tests/test_knowledge_store.py | 1 + tests/test_knowledge_tiers.py | 4 +- tests/test_research.py | 5 +- .../test_skill_eval_optional_dependencies.py | 3 + tests/test_skill_library_readers.py | 3 +- 32 files changed, 2260 insertions(+), 1936 deletions(-) create mode 100644 src/inferrsi/_repo_pack_data.py create mode 100644 src/inferrsi/knowledge/__init__.py create mode 100644 src/inferrsi/knowledge/_skill_selection.py create mode 100644 src/inferrsi/knowledge/_source_snapshot.py create mode 100644 src/inferrsi/knowledge/catalog.py create mode 100644 src/inferrsi/knowledge/knowledge_operations.py create mode 100644 src/inferrsi/knowledge/knowledge_store.py create mode 100644 src/inferrsi/knowledge/research.py create mode 100644 src/inferrsi/knowledge/skill_catalog.py create mode 100644 tests/test_knowledge_compatibility_shells.py diff --git a/.github/knowledge-model-policy.json b/.github/knowledge-model-policy.json index 8e0f3d232..47ddf72ae 100644 --- a/.github/knowledge-model-policy.json +++ b/.github/knowledge-model-policy.json @@ -12,7 +12,10 @@ "docs/knowledge-store.md", "src/inferrsi/catalog.py", "src/inferrsi/skill_catalog.py", - "src/inferrsi/knowledge_operations.py" + "src/inferrsi/knowledge_operations.py", + "src/inferrsi/knowledge/catalog.py", + "src/inferrsi/knowledge/skill_catalog.py", + "src/inferrsi/knowledge/knowledge_operations.py" ], "policy_paths": [ ".github/knowledge-model-policy.json", diff --git a/docs/knowledge-model-policy.md b/docs/knowledge-model-policy.md index 6ffc225d3..186beaa93 100644 --- a/docs/knowledge-model-policy.md +++ b/docs/knowledge-model-policy.md @@ -37,6 +37,9 @@ The exact protected scope is: - `docs/knowledge-base.md`, `docs/knowledge-operations.md`, `docs/knowledge-store.md` - `src/inferrsi/catalog.py`, `src/inferrsi/skill_catalog.py`, `src/inferrsi/knowledge_operations.py` +- `src/inferrsi/knowledge/catalog.py`, `src/inferrsi/knowledge/skill_catalog.py`, + `src/inferrsi/knowledge/knowledge_operations.py`, the implementations behind + the three root compatibility shells above, which stay protected Deletions and both rename paths count. Mechanical/generated changes have no automatic exemption. Running a generator or obtaining an approved-model review diff --git a/docs/legacy-knowledge-manifest.json b/docs/legacy-knowledge-manifest.json index da02e450d..867aa34d0 100644 --- a/docs/legacy-knowledge-manifest.json +++ b/docs/legacy-knowledge-manifest.json @@ -1,7 +1,19 @@ { "$schema": "urn:inferrsi:docs:legacy-knowledge-manifest:v1", - "description": "First-wave compatibility exceptions for seven root Knowledge leaves plus the neutral technical data leaf. This manifest is frozen at P0b formal review and is not a wildcard approval. K01 verifies each hash against its base and re-reviews if any differ. D018/D020 chose these exact edges.", - "base_commit": "0da7e2f960cd4ec994ae8eddea5b71023c5379ca", + "description": "First-wave compatibility exceptions for seven root Knowledge leaves plus the neutral technical data leaf. P0b froze the first revision at 0da7e2f960cd4ec994ae8eddea5b71023c5379ca. This K01 revision re-records every leaf hash at base_commit before the move; it is not a wildcard approval and changes only through formal review. D018/D020 chose these exact edges. file_sha256 is the old root leaf before the move, and line references are at base_commit.", + "base_commit": "e2f4cd1e39b369f06835cdd655ad5dc9f6808918", + "revision_history": [ + { + "base_commit": "0da7e2f960cd4ec994ae8eddea5b71023c5379ca", + "stage": "P0b", + "superseded_by": "K01 revision at e2f4cd1e39b369f06835cdd655ad5dc9f6808918", + "reason": "#389 (c013e7778e5d421df655b969d4c89f81ade65b44) changed two leaves after P0b. #393 changed none of the seven leaves. The five other hashes are unchanged.", + "changed_file_sha256": { + "src/inferrsi/_skill_selection.py": "acbec7c91bfa05bc2008b939e009fe600a40518051229fb7618c56b092ad36b9", + "src/inferrsi/knowledge_operations.py": "3725781bcedb308af6ad9edc0b970c63f82accf6b04ebee4008c1081b63bed9c" + } + } + ], "compatibility_pairs": [ { "importer": "catalog", @@ -29,7 +41,7 @@ "REPO_TYPES is the same tuple object through root catalog, the neutral leaf, and K.catalog", "Derived KNOWLEDGE_SCOPES and KNOWLEDGE_REPOS stay unchanged" ], - "consumer_evidence": "tests/test_cli_catalog.py (L189) and tests/test_knowledge_tiers.py (L89) patch catalog.files; tests/test_research.py (L91, L120) patch catalog.knowledge_records; infra.inspection (L10, L56) imports and calls repository_pack", + "consumer_evidence": "tests/test_cli_catalog.py (L189) and tests/test_knowledge_tiers.py (L89) patch catalog.files; tests/test_research.py (L91, L120) patch catalog.knowledge_records; infra.inspection (L10, L56) imports and calls repository_pack; arguments (L12) imports BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES; cli (L12) imports get_knowledge, search_knowledge; observability/campaign_records (L12) imports BACKENDS", "retirement_evidence": "catalog sheds repository_pack to the neutral leaf; K.catalog does not export repository_pack" }, { @@ -96,13 +108,13 @@ "audited_definitions": [ "root:Path and checkout physical resource semantics unchanged" ], - "consumer_evidence": "knowledge_operations (L270, L308, L426 lazy imports; calls at L272, L310, L430, L433, L436) imports skill_directories, skill_directory, skill_metadata, skill_reference; KnowledgeReader._skill uses all skill_* APIs; tests/test_skill_catalog.py covers skill enumeration and loading" + "consumer_evidence": "knowledge_operations (L279, L317, L435 lazy imports; calls at L281, L319, L438, L439, L442, L445) imports skill_directories, get_skill, skill_directory, skill_metadata, skill_reference; KnowledgeReader._skill uses all skill_* APIs; cli (L129) lazily imports get_skill, skill_instructions, skill_records, skill_reference, skill_resource; tools/skill_eval_snapshot.py (L47) imports _skill_name, _unique_fields, skill_metadata; tests/test_skill_catalog.py covers skill enumeration and loading" }, { "importer": "_skill_selection", "target": "inferrsi.knowledge._skill_selection", "file": "src/inferrsi/_skill_selection.py", - "file_sha256": "acbec7c91bfa05bc2008b939e009fe600a40518051229fb7618c56b092ad36b9", + "file_sha256": "7bcf1930b73e09fcadf78e7c9e384b30411ed599d83c32e6837f04ca25cbda77", "exports": { "public": [ "RELEVANCE_METHODS", @@ -112,14 +124,18 @@ "raw_string_lexical_score", "indexed_skill", "skill_score", - "context_view" + "context_view", + "TASK_RELEVANCE", + "task_score", + "record_context" ], "private_used": [] }, "audited_definitions": [ - "RELEVANCE_METHODS and scoring functions are the same objects; selection algorithm not rewritten" + "RELEVANCE_METHODS, TASK_RELEVANCE, and the scoring and context functions are the same objects; selection algorithm not rewritten", + "#389 audit: TASK_RELEVANCE is \"task-lexical-v1\" and is the third RELEVANCE_METHODS element; skill_score accepts only the two skill-only methods and rejects task-lexical-v1 with \"Unknown relevance method for skill-only ranking\"; task_score ranks skill and package records by whole task terms with a portable-or-in-scope backend factor; record_context returns the whole package record entrypoint as a context view or a context_limit omission" ], - "consumer_evidence": "knowledge_operations (L12) imports RELEVANCE_METHODS, context_view, indexed_skill, skill_score, words; collect uses these for relevance selection; tests/test_raw_string_selection.py covers scoring; tests/test_collect_selection.py covers context_view" + "consumer_evidence": "knowledge_operations (L12-L21) imports RELEVANCE_METHODS, TASK_RELEVANCE, context_view, indexed_skill, record_context, skill_score, task_score, words; collect uses them for relevance selection and context views; observability/knowledge_capture (L20) imports TASK_RELEVANCE through the root leaf and uses it at L74-L75, and cli (L335) and observability/optimization_story (L14) load that module; arguments (L11) imports RELEVANCE_METHODS; tools/evaluate_indexer.py (L138) imports SkillIndexError, indexed_skill; tests/test_raw_string_selection.py covers scoring and the skill_score rejection (L84-L86); tests/test_collect_selection.py covers context_view; tests/test_campaign_task_selection.py covers task-lexical-v1 through root KnowledgeReader" }, { "importer": "research", @@ -162,7 +178,7 @@ "importer": "knowledge_operations", "target": "inferrsi.knowledge.knowledge_operations", "file": "src/inferrsi/knowledge_operations.py", - "file_sha256": "3725781bcedb308af6ad9edc0b970c63f82accf6b04ebee4008c1081b63bed9c", + "file_sha256": "686b196e03ca0a1dba5ff5e246563f305ba2bea412172833905af41a6c6ad886", "exports": { "public": [ "AUTHORITIES", @@ -184,9 +200,10 @@ }, "audited_definitions": [ "Bundle/KnowledgeReader/RevisionConflict/IdentityError/safe_path/AUTHORITIES/IDENTITY are the same objects; signatures/defaults/Path/exceptions and used symbols preserved", - "_json and _hash are explicitly re-exported for test injection (D1, D2)" + "_json and _hash are explicitly re-exported for test injection (D1, D2)", + "#389 audit of KnowledgeReader.collect: signature, defaults, and validation order are unchanged; relevance_method=\"task-lexical-v1\" ranks every package authority record with task_score, lets a packaged skills.discovery skill pass a scenario filter only for ranked selection without ids, builds non-skill contexts with record_context, and adds selection.scope \"package-skills-and-records\" with matched_count and returned_count; the two earlier methods keep the packaged-skill-only path and selection.scope \"generated-packaged-skills\". No top-level name was added or removed." ], - "consumer_evidence": "harness/skill_evaluation (L9) imports KnowledgeReader; cli (L15, L25, L33) imports and uses KnowledgeReader; knowledge_store (L10-18) imports KO symbols; tests/test_bundle_manifest.py (L73) patches _hash; tests/test_collect_limit_serialization.py (L39-47) patches _json; tests/test_knowledge_operations.py covers Bundle/KnowledgeReader contract" + "consumer_evidence": "harness/skill_evaluation (L9) imports KnowledgeReader; cli (L15, L25, L33) imports and uses KnowledgeReader; knowledge_store (L10-L19) imports KO symbols; tests/test_bundle_manifest.py (L73) patches _hash; tests/test_collect_limit_serialization.py (L39-47) patches _json; tests/test_knowledge_operations.py covers Bundle/KnowledgeReader contract; tests/test_campaign_task_selection.py covers the #389 task-lexical-v1 collect path" }, { "importer": "infra.inspection", @@ -223,7 +240,9 @@ }, "technical_leaf": { "module": "inferrsi._repo_pack_data", - "note": "The new _repo_pack_data.py file does not exist yet; do not create it in P0b; do not lock a fabricated full-file hash. K01 will create it and lock the full-file hash.", + "note": "K01 creates src/inferrsi/_repo_pack_data.py from the recorded original fragment plus a leading comment, the future import, and the three listed stdlib imports. file_sha256 locks the complete new file; P0b recorded only the fragment.", + "file": "src/inferrsi/_repo_pack_data.py", + "file_sha256": "a14edac8283b1f7b6b5794381c44d7c2632dda5cc01f03c557dc293d05523afd", "allowed_imports": [ "json", "importlib.resources", @@ -246,7 +265,7 @@ "module": "src/inferrsi/infra/inspection.py", "current_import": "from ..catalog import repository_pack", "required_new_import": "from .._repo_pack_data import repository_pack", - "note": "K01 must change only this import line. Original functions, injected parameters, and returns stay the same. Do not edit inspection.py in P0b." + "note": "K01 changes only this import line. Original functions, injected parameters, and returns stay the same. current_import records the line at base_commit." }, "intra_closure_note": "An import of inferrsi._repo_pack_data from inside inferrsi.knowledge (for example inferrsi/knowledge/catalog.py) is intra-closure, not a tenth compatibility exception; it is governed by the leaf's no-inferrsi.*-import rule and by REPO_TYPES object identity, not by this allowlist." } diff --git a/docs/module-layout.md b/docs/module-layout.md index e29358d3c..d6f593170 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -1,7 +1,9 @@ # Target module layout -This page records the directory rule for the integration slice. It is not an -implemented migration. No package has been created or moved to satisfy it. +This page records the directory rule for the integration slice. Only the +first-wave Knowledge move (K01) is implemented: `src/inferrsi/knowledge/` holds +the seven moved Knowledge bodies, and the old root modules are static +compatibility shells. No other package has been created or moved to satisfy it. #371 remains the proposal for the Apps package path and is not accepted here. ## Target modules @@ -56,7 +58,7 @@ Several flat modules remain shared seams until their slice starts. | Current path | Tentative reading | This slice | | --- | --- | --- | | `cli.py`, `arguments.py`, `__main__.py` | Future Apps entry, after #371 | Not moved | -| `knowledge_operations.py`, `knowledge_store.py`, `catalog.py`, `research.py`, `skill_catalog.py`, `_skill_selection.py` | Knowledge behavior and corpus readers | Interfaces unchanged | +| `knowledge_operations.py`, `knowledge_store.py`, `catalog.py`, `research.py`, `skill_catalog.py`, `_skill_selection.py`, `_source_snapshot.py` | Static compatibility shells since K01; bodies in `knowledge/` | Interfaces unchanged | | `harness/` | Mixed legacy tree, not target isolation | Not redefined | | `evaluation.py`, `model/validators/` | Benchmark-leaning checks | Not moved | | `infra/`, `execution.py`, `runspec.py`, `inspection.py` | Seams used by the current tree | Not a fourth core | @@ -76,9 +78,10 @@ absolute name such as `from .bar import VALUE` inside `knowledge/`. Each forbidd importer and target pair is reported once. The only exceptions are the exact first-wave pairs below, read from `docs/legacy-knowledge-manifest.json`. -"There is no exception list of current edges" describes today's tree: the -three package names are unused, so no current edge needs an exception and the -violation list is empty. It is not a permanent ban on reviewed exceptions. +"There is no exception list of current edges" described the tree before K01, +when the three package names were unused and no current edge needed an +exception. It was never a permanent ban on reviewed exceptions; since K01 the +exact first-wave pairs below are the only exceptions. The gate scans `src/inferrsi/**` as the transfer graph. It resolves absolute, relative, and `from package import name` imports, includes the parent-package @@ -176,9 +179,11 @@ The manifest records the base commit, each old leaf's file hash at that commit, export names, audited definitions, consumer evidence, catalog's two symbol routes, the required `infra.inspection` import rewrite, and the original `REPO_TYPES` / `repository_pack` fragment from `catalog.py` with its -hash. `_repo_pack_data.py` does not exist yet; K01 creates it and locks its -full-file hash. K01 re-verifies every recorded hash against its own base and -re-reviews the manifest if any differ. +hash. K01 created `_repo_pack_data.py`, and the manifest locks its full-file +hash. K01 re-verified every recorded hash against its own base. Two leaves +had changed after P0b, so K01 re-recorded all seven hashes at its base and +keeps the P0b values in `revision_history`; that revision goes through formal +review with the move. ## Old `run` exit codes diff --git a/src/inferrsi/_repo_pack_data.py b/src/inferrsi/_repo_pack_data.py new file mode 100644 index 000000000..b31869ea9 --- /dev/null +++ b/src/inferrsi/_repo_pack_data.py @@ -0,0 +1,16 @@ +# Neutral raw repository-pack data leaf (D020). It holds only the original +# REPO_TYPES tuple and pack.json reader, and imports no inferrsi module. +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +REPO_TYPES = ("vllm", "vllm_omni", "afd_plugin", "vllm_rlt", "strata", "sglang", "sglang_omni") + + +def repository_pack(repo_type: str) -> dict[str, Any]: + if repo_type not in REPO_TYPES: + raise ValueError(f"Unknown repository type: {repo_type}") + path = files("inferrsi").joinpath("data", "repo_packs", repo_type, "pack.json") + return json.loads(path.read_text(encoding="utf-8")) diff --git a/src/inferrsi/_skill_selection.py b/src/inferrsi/_skill_selection.py index c696ef571..23309a0fc 100644 --- a/src/inferrsi/_skill_selection.py +++ b/src/inferrsi/_skill_selection.py @@ -1,211 +1,30 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge._skill_selection. from __future__ import annotations -import json -import posixpath -import re - -TASK_RELEVANCE = "task-lexical-v1" -RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE) - - -class SkillIndexError(ValueError): - pass - - -def words(cue: str) -> set[str]: - return set(re.findall(r"\w+", cue.casefold())) - - -def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: - tokens = words(cue) - return sum( - weight * sum(token in json.dumps(value, ensure_ascii=False).casefold() for token in tokens) - for weight, value in fields - ) - - -def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: - """Match original strings; retain legacy JSON matching for nonstring fields.""" - tokens = words(cue) - score = 0 - for weight, value in fields: - text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) - score += weight * sum(token in text.casefold() for token in tokens) - return score - - -def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: - """Validate the existing projection against this exact canonical bundle.""" - registry_path = "data/skill_library/skill-index.json" - registry = json.loads(content[registry_path]) - name = record["skill_metadata"]["name"] - entry = registry["skills"][name] - prefix = f"data/skill_library/{entry['path']}/" - core = prefix + "SKILL.md" - expected_id = f"inferrsi.{entry['backend']}.skills.{name.replace('-', '_')}" - if ( - registry["schema_version"] != 1 - or view["authority"] != "package" - or record["id"] != view["id"] - or record["id"] != expected_id - or record["backend"] != entry["backend"] - or record["skill_documents"] != {"original": core} - or posixpath.basename(entry["path"]) != name - ): - raise SkillIndexError("Packaged skill identity disagrees with the registry") - expected = { - "src/inferrsi/" + path: digest - for path, digest in view["resources"].items() - if path.startswith(prefix) or path == registry_path - } - if record["source_files"] != expected: - raise SkillIndexError("Stale packaged skill projection; rebuild with build_skill_index.py") - - required = [core] - metadata = record["skill_metadata"] - notices = [ - source["third_party_notices"] - for source in metadata.get("sources", []) if "third_party_notices" in source - ] - for relative in [ - *metadata.get("references", []), *metadata.get("provenance_readsets", []), *notices, - ]: - path = posixpath.normpath(prefix + relative) - if "\\" in relative or not path.startswith("data/skill_library/") or path not in content: - raise SkillIndexError(f"Unavailable declared skill reference: {relative}") - if path not in required: - required.append(path) - for path in sorted(content): - if path.startswith(prefix) and re.search( - r"(?:^|[-_])(?:LICENSE|NOTICES?)(?:\.[^.]+)?$", - posixpath.basename(path), re.IGNORECASE, - ) and path not in required: - required.append(path) - for resource in list(required): - directory = posixpath.dirname(resource) - while directory.startswith("data/skill_library/"): - license_path = directory + "/LICENSE" - if license_path in content and license_path not in required: - required.append(license_path) - directory = posixpath.dirname(directory) - optional = sorted( - path for path in content - if path.startswith(prefix) and path not in required - ) - return core, required, optional - - -def skill_score( - cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1" -) -> int: - if method not in ("weighted-lexical-v1", "weighted-raw-string-v1"): - raise ValueError("Unknown relevance method for skill-only ranking") - scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score - return scorer(cue, [ - (4, record["title"]), - (3, record["summary"]), - (2, [record.get("tags", []), record.get("applicability", {})]), - (1, core.decode("utf-8")), - ]) - - -def task_score(cue: str, record: dict, core: bytes, backend: str | None) -> int: - """Rank whole task terms across both skill and repository knowledge owners.""" - def terms(text: str) -> set[str]: - return set(re.findall(r"[^\W_]+", text.casefold())) - - tokens = terms(cue) - {"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"} - fields = ( - (4, record["title"]), - (3, record["summary"]), - (2, " ".join(record.get("tags", []))), - (2, record["repo"] + " " + record["scenario"]), - (1, json.dumps(record.get("applicability", {}), ensure_ascii=False)), - (1, core.decode("utf-8")), - ) - score = sum(weight * len(tokens & terms(text)) for weight, text in fields) - # With no platform in scope prefer portable procedures to a specialization - # that happens to repeat the same task words in its description. - platforms = { - "cuda": {"cuda", "nvidia"}, "rocm": {"rocm", "amd"}, - "ascend": {"ascend", "cann"}, "mlx": {"mlx"}, "cpu": {"cpu"}, - } - in_scope = {backend} if backend else {key for key, names in platforms.items() if names & tokens} - return score * (2 if record["backend"] == "common" or record["backend"] in in_scope else 1) - - -def record_context(record: dict, view: dict, content: dict[str, bytes], available: int) -> tuple: - """Keep the canonical package record including its sources and limits.""" - path = view["entrypoint"] - body = content[path] - if len(body) > available: - return None, {"ref": view["ref"], "reason": "context_limit", - "required_bytes": len(body), "available_bytes": available} - return { - "ref": view["ref"], "revision": view["revision"], "content_bytes": len(body), - "resources": [{"path": path, "sha256": view["resources"][path], "role": "record", - "locator": "whole-resource", "content": body.decode("utf-8")}], - "omissions": [{"path": p, "reason": "linked_document"} for p in - dict.fromkeys(record.get("skill_documents", {}).values())], - "sources": view["sources"], "attribution": view["attribution"], - "limitations": view["limitations"], - }, None - - -def context_view( - record: dict, view: dict, content: dict[str, bytes], cue: str, available: int -) -> tuple[dict | None, dict | None]: - core, required, optional = indexed_skill(record, view, content) - required_bytes = sum(len(content[path]) for path in required) - if required_bytes > available: - return None, { - "ref": view["ref"], "reason": "context_limit", - "required_bytes": required_bytes, "available_bytes": available, - } - - resources, omissions = [], [] - used = 0 - - def include(path: str) -> None: - nonlocal used - resources.append({ - "path": path, "sha256": view["resources"][path], "locator": "whole-resource", - "role": "core" if path == core else "reference", - "content": content[path].decode("utf-8"), - }) - used += len(content[path]) - - for path in required: - include(path) - candidates = [] - for path in optional: - if ( - posixpath.basename(path) == "metadata.json" - or posixpath.splitext(path)[1].lower() not in (".md", ".json", ".txt", ".csv") - ): - omissions.append({"path": path, "reason": "not_context_resource"}) - continue - try: - text = content[path].decode("utf-8") - except UnicodeDecodeError: - omissions.append({"path": path, "reason": "not_utf8"}) - continue - score = lexical_score(cue, [(2, path.rpartition("/")[2]), (1, text)]) - if words(cue) and score == 0: - omissions.append({"path": path, "reason": "not_relevant"}) - continue - candidates.append((score, path)) - for _, path in sorted(candidates, key=lambda item: (-item[0], item[1])): - if used + len(content[path]) > available: - omissions.append({"path": path, "reason": "context_limit"}) - continue - include(path) - return { - "ref": view["ref"], "revision": view["revision"], "resources": resources, - "content_bytes": used, "omissions": omissions, - "sources": view["sources"], "attribution": view["attribution"], - "limitations": view["limitations"] + [ - "Selected resources are complete files; this view is not a standalone Bundle.", - "Follow omitted or other linked resources through the exact owner revision.", - ], - }, None +from inferrsi.knowledge._skill_selection import ( + RELEVANCE_METHODS, + TASK_RELEVANCE, + SkillIndexError, + context_view, + indexed_skill, + lexical_score, + raw_string_lexical_score, + record_context, + skill_score, + task_score, + words, +) + +__all__ = [ + "RELEVANCE_METHODS", + "SkillIndexError", + "words", + "lexical_score", + "raw_string_lexical_score", + "indexed_skill", + "skill_score", + "context_view", + "TASK_RELEVANCE", + "task_score", + "record_context", +] diff --git a/src/inferrsi/_source_snapshot.py b/src/inferrsi/_source_snapshot.py index 3ced2e330..b898386dc 100644 --- a/src/inferrsi/_source_snapshot.py +++ b/src/inferrsi/_source_snapshot.py @@ -1,88 +1,26 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge._source_snapshot. from __future__ import annotations -import re -from typing import Any - -# One shared snapshot keeps the repository-wide source revision and registry digest out of -# per-owner records, so a one-owner edit regenerates only that owner's record. -MANIFEST = "_provenance.json" -REPOSITORY = "https://github.com/InferMatrixLab/InferRSI" -LIBRARY = "src/inferrsi/data/skill_library" -REGISTRY = f"{LIBRARY}/skill-index.json" -STORED_SCHEMA = 2 - - -def manifest(revision: str, registry_sha256: str) -> dict[str, Any]: - return { - "schema_version": 1, - "repository": REPOSITORY, - "source_revision": revision, - "registry": {"path": REGISTRY, "sha256": registry_sha256}, - } - - -def _snapshot(value: Any) -> tuple[str, str]: - if ( - not isinstance(value, dict) - or set(value) != {"schema_version", "repository", "source_revision", "registry"} - or type(value["schema_version"]) is not int - or value["schema_version"] != 1 - or value["repository"] != REPOSITORY - or not isinstance(value["source_revision"], str) - or not re.fullmatch(r"[0-9a-f]{40}", value["source_revision"]) - ): - raise ValueError("Invalid knowledge source snapshot") - registry = value["registry"] - if ( - not isinstance(registry, dict) - or set(registry) != {"path", "sha256"} - or registry["path"] != REGISTRY - or not isinstance(registry["sha256"], str) - or not re.fullmatch(r"[0-9a-f]{64}", registry["sha256"]) - ): - raise ValueError("Invalid knowledge source snapshot registry") - return value["source_revision"], registry["sha256"] - - -def compact(record: dict[str, Any]) -> dict[str, Any]: - """Store a generated schema-v1 record without its snapshot-wide fields.""" - stored = {key: value for key, value in record.items() if key != "sources"} - stored["schema_version"] = STORED_SCHEMA - stored["source_files"] = { - path: digest for path, digest in record["source_files"].items() if path != REGISTRY - } - return stored - - -def resolve(stored: dict[str, Any], snapshot: Any) -> dict[str, Any]: - """Rebuild the public schema-v1 record from a stored record and the shared snapshot.""" - revision, registry_sha256 = _snapshot(snapshot) - if stored.get("schema_version") != STORED_SCHEMA or "sources" in stored: - raise ValueError("Stored skill record is not a schema-v2 projection") - files = stored.get("source_files") - documents = stored.get("skill_documents") - if ( - not isinstance(files, dict) or REGISTRY in files - or not isinstance(documents, dict) or not isinstance(documents.get("original"), str) - ): - raise ValueError(f"Invalid stored skill record: {stored.get('id')}") - entry = "src/inferrsi/" + documents["original"] - if entry not in files: - raise ValueError(f"Stored skill record lacks its entry hash: {stored.get('id')}") - hashes = dict(sorted({**files, REGISTRY: registry_sha256}.items())) - source = { - "type": "git", - "url": f"{REPOSITORY}/blob/{revision}/{entry}", - "commit": revision, - "paths": list(hashes), - "content_sha256": hashes[entry], - } - record = {} - for key, value in stored.items(): - record[key] = 1 if key == "schema_version" else value - if key == "origin": - record["sources"] = [source] - if "sources" not in record: - raise ValueError(f"Stored skill record lacks origin: {stored.get('id')}") - record["source_files"] = hashes - return record +from inferrsi.knowledge._source_snapshot import ( + LIBRARY, + MANIFEST, + REGISTRY, + REPOSITORY, + STORED_SCHEMA, + _snapshot, + compact, + manifest, + resolve, +) + +__all__ = [ + "MANIFEST", + "REPOSITORY", + "LIBRARY", + "REGISTRY", + "STORED_SCHEMA", + "manifest", + "compact", + "resolve", + "_snapshot", +] diff --git a/src/inferrsi/catalog.py b/src/inferrsi/catalog.py index 3408a53d2..3ab45f6fa 100644 --- a/src/inferrsi/catalog.py +++ b/src/inferrsi/catalog.py @@ -1,175 +1,32 @@ -"""Read-only package-data catalog. Knowledge text is never executable.""" - +# Compatibility shell. Knowledge names live in inferrsi.knowledge.catalog; +# REPO_TYPES and repository_pack live in the neutral inferrsi._repo_pack_data. from __future__ import annotations -import json -import re -from importlib.resources import files -from typing import Any - -from inferrsi import _source_snapshot - -REPO_TYPES = ("vllm", "vllm_omni", "afd_plugin", "vllm_rlt", "strata", "sglang", "sglang_omni") -# Learning scope does not imply that a repository has a runnable target adapter. -# Legacy repository-pack records retain their contribution tier; an independent -# peer card can describe a different learning use case for the same source repo. -KNOWLEDGE_SCOPES = { - "contribution": (*REPO_TYPES, "vllm_ascend", "inferrsi"), - "dependency": ( - "flashinfer", "flashattention", "pytorch", "triton", "diffusers", "transformers", "mlx" - ), - "peer": ( - "sglang", "tensorrt_llm", "lmdeploy", "xdit", "cache_dit", "qwen3_tts_engine", "deepinfer" - ), -} -KNOWLEDGE_REPOS = tuple( - dict.fromkeys(repo for repos in KNOWLEDGE_SCOPES.values() for repo in repos) +from inferrsi._repo_pack_data import REPO_TYPES, repository_pack +from inferrsi.knowledge.catalog import ( + BACKENDS, + KNOWLEDGE_REPOS, + KNOWLEDGE_SCOPES, + SCENARIO_ALIASES, + _validate_record, + _walk, + files, + get_knowledge, + knowledge_records, + search_knowledge, ) -BACKENDS = ("cpu", "cuda", "rocm", "ascend", "mlx") -SCENARIO_ALIASES = { - "diffusion.t2i": "diffusion.image_generation", - "diffusion.t2v": "diffusion.video_generation", -} - - -def _walk(root, prefix=()): - for child in sorted(root.iterdir(), key=lambda item: item.name): - parts = (*prefix, child.name) - if child.is_dir(): - yield from _walk(child, parts) - else: - yield parts, child - - -def _validate_record( - record: dict, repo: str, parts: tuple[str, ...], tier: str = "contribution" -) -> None: - required = ("id", "title", "repo", "backend", "component", "scenario", "status", "summary") - if not isinstance(record, dict) or any( - not isinstance(record.get(key), str) or not record[key].strip() for key in required - ): - raise ValueError(f"Incomplete knowledge record: {repo}/{'/'.join(parts)}") - if record.get("schema_version") != 1: - raise ValueError(f"Unsupported knowledge schema: {record['id']}") - # Existing schema-v1 records retain their IDs and default to contribution. - if record.get("tier", "contribution") != tier: - raise ValueError(f"Knowledge tier disagrees with path: {record['id']}") - if len(parts) != 3: - raise ValueError("Knowledge path must be BACKEND/COMPONENT/ENTRY.json") - backend, component, filename = parts - if backend not in (*BACKENDS, "common"): - raise ValueError(f"Unknown knowledge backend: {backend}") - if (record["repo"], record["backend"], record["component"]) != (repo, backend, component): - raise ValueError(f"Knowledge scope disagrees with path: {record['id']}") - expected_id = f"{repo}.{backend}.{component}.{filename[:-5].replace('-', '_')}" - if record["id"] != expected_id: - raise ValueError(f"Knowledge ID disagrees with path: {record['id']}") - if not record["scenario"].startswith(component + "."): - raise ValueError(f"Knowledge scenario disagrees with component: {record['id']}") - sources = record.get("sources") - if not isinstance(sources, list) or not sources: - raise ValueError(f"Knowledge has no source: {record['id']}") - for source in sources: - if ( - not isinstance(source, dict) - or not isinstance(source.get("url"), str) - or not source["url"].startswith("https://") - or not isinstance(source.get("commit"), str) - or not re.fullmatch(r"[0-9a-f]{40}", source["commit"]) - ): - raise ValueError(f"Knowledge requires source attribution and revision: {record['id']}") - verification = record.get("verification", {}) - if not isinstance(verification, dict) or not isinstance( - verification.get("locally_verified"), bool - ): - raise ValueError(f"Knowledge requires an explicit verification state: {record['id']}") - if record["status"] == "imported" and verification["locally_verified"]: - raise ValueError(f"Imported knowledge must state locally_verified=false: {record['id']}") - - -def knowledge_records() -> list[dict[str, Any]]: - data = files("inferrsi").joinpath("data") - records = [] - seen = set() - for tier, repos in KNOWLEDGE_SCOPES.items(): - for repo in repos: - knowledge = ( - data.joinpath("repo_packs", repo, "knowledge") - if tier == "contribution" - else data.joinpath("knowledge_packs", tier, repo) - ) - if not knowledge.is_dir(): - continue - snapshot = knowledge.joinpath(_source_snapshot.MANIFEST) - snapshot = ( - json.loads(snapshot.read_text(encoding="utf-8")) if snapshot.is_file() else None - ) - for parts, item in _walk(knowledge): - if not item.name.endswith(".json") or parts == (_source_snapshot.MANIFEST,): - continue - record = json.loads(item.read_text(encoding="utf-8")) - # Generated skill records store schema v2; readers always receive schema v1. - if isinstance(record, dict) and record.get("schema_version") == ( - _source_snapshot.STORED_SCHEMA - ): - if snapshot is None: - raise ValueError(f"Knowledge record lacks its source snapshot: {repo}") - record = _source_snapshot.resolve(record, snapshot) - _validate_record(record, repo, parts, tier) - if record["id"] in seen: - raise ValueError(f"Duplicate knowledge ID: {record['id']}") - seen.add(record["id"]) - records.append(record) - return records - - -def search_knowledge( - query: str = "", - *, - repo: str | None = None, - tier: str | None = None, - backend: str | None = None, - scenario: str | None = None, - limit: int = 10, -) -> list[dict[str, Any]]: - if limit < 1: - raise ValueError("limit must be positive") - if tier is not None and tier not in KNOWLEDGE_SCOPES: - raise ValueError(f"Unknown knowledge tier: {tier}") - scenario = SCENARIO_ALIASES.get(scenario, scenario) - tokens = re.findall(r"[\w.-]+", query.casefold()) - scored = [] - for record in knowledge_records(): - if tier and record.get("tier", "contribution") != tier: - continue - if repo and record["repo"] != repo: - continue - if backend and record["backend"] not in (backend, "common"): - continue - if scenario: - declared = record.get("scenario", "") - if declared != scenario and not ( - declared.endswith(".*") and scenario.startswith(declared[:-1]) - ): - continue - document = json.dumps(record, ensure_ascii=False).casefold() - if any(token not in document for token in tokens): - continue - score = sum(document.count(token) for token in tokens) - scored.append((score, record["id"], record)) - scored.sort(key=lambda item: (-item[0], item[1])) - return [item[2] for item in scored[:limit]] - - -def get_knowledge(record_id: str) -> dict[str, Any]: - for record in knowledge_records(): - if record["id"] == record_id: - return record - raise ValueError(f"Unknown knowledge ID: {record_id}") - -def repository_pack(repo_type: str) -> dict[str, Any]: - if repo_type not in REPO_TYPES: - raise ValueError(f"Unknown repository type: {repo_type}") - path = files("inferrsi").joinpath("data", "repo_packs", repo_type, "pack.json") - return json.loads(path.read_text(encoding="utf-8")) +__all__ = [ + "REPO_TYPES", + "KNOWLEDGE_SCOPES", + "KNOWLEDGE_REPOS", + "BACKENDS", + "SCENARIO_ALIASES", + "knowledge_records", + "search_knowledge", + "get_knowledge", + "repository_pack", + "_walk", + "_validate_record", + "files", +] diff --git a/src/inferrsi/infra/inspection.py b/src/inferrsi/infra/inspection.py index 597d3185a..c3251aef8 100644 --- a/src/inferrsi/infra/inspection.py +++ b/src/inferrsi/infra/inspection.py @@ -7,7 +7,7 @@ from collections.abc import Callable from pathlib import Path -from ..catalog import repository_pack +from .._repo_pack_data import repository_pack from .commands import probe_command from .platforms import inspect_device from .targets import repository_identity diff --git a/src/inferrsi/knowledge/__init__.py b/src/inferrsi/knowledge/__init__.py new file mode 100644 index 000000000..76ef1bf6b --- /dev/null +++ b/src/inferrsi/knowledge/__init__.py @@ -0,0 +1,2 @@ +# Knowledge core package. Keep this init import-free so loading one submodule +# never loads another (for example the PyYAML-backed skill_catalog). diff --git a/src/inferrsi/knowledge/_skill_selection.py b/src/inferrsi/knowledge/_skill_selection.py new file mode 100644 index 000000000..c696ef571 --- /dev/null +++ b/src/inferrsi/knowledge/_skill_selection.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import json +import posixpath +import re + +TASK_RELEVANCE = "task-lexical-v1" +RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE) + + +class SkillIndexError(ValueError): + pass + + +def words(cue: str) -> set[str]: + return set(re.findall(r"\w+", cue.casefold())) + + +def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: + tokens = words(cue) + return sum( + weight * sum(token in json.dumps(value, ensure_ascii=False).casefold() for token in tokens) + for weight, value in fields + ) + + +def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: + """Match original strings; retain legacy JSON matching for nonstring fields.""" + tokens = words(cue) + score = 0 + for weight, value in fields: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + score += weight * sum(token in text.casefold() for token in tokens) + return score + + +def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: + """Validate the existing projection against this exact canonical bundle.""" + registry_path = "data/skill_library/skill-index.json" + registry = json.loads(content[registry_path]) + name = record["skill_metadata"]["name"] + entry = registry["skills"][name] + prefix = f"data/skill_library/{entry['path']}/" + core = prefix + "SKILL.md" + expected_id = f"inferrsi.{entry['backend']}.skills.{name.replace('-', '_')}" + if ( + registry["schema_version"] != 1 + or view["authority"] != "package" + or record["id"] != view["id"] + or record["id"] != expected_id + or record["backend"] != entry["backend"] + or record["skill_documents"] != {"original": core} + or posixpath.basename(entry["path"]) != name + ): + raise SkillIndexError("Packaged skill identity disagrees with the registry") + expected = { + "src/inferrsi/" + path: digest + for path, digest in view["resources"].items() + if path.startswith(prefix) or path == registry_path + } + if record["source_files"] != expected: + raise SkillIndexError("Stale packaged skill projection; rebuild with build_skill_index.py") + + required = [core] + metadata = record["skill_metadata"] + notices = [ + source["third_party_notices"] + for source in metadata.get("sources", []) if "third_party_notices" in source + ] + for relative in [ + *metadata.get("references", []), *metadata.get("provenance_readsets", []), *notices, + ]: + path = posixpath.normpath(prefix + relative) + if "\\" in relative or not path.startswith("data/skill_library/") or path not in content: + raise SkillIndexError(f"Unavailable declared skill reference: {relative}") + if path not in required: + required.append(path) + for path in sorted(content): + if path.startswith(prefix) and re.search( + r"(?:^|[-_])(?:LICENSE|NOTICES?)(?:\.[^.]+)?$", + posixpath.basename(path), re.IGNORECASE, + ) and path not in required: + required.append(path) + for resource in list(required): + directory = posixpath.dirname(resource) + while directory.startswith("data/skill_library/"): + license_path = directory + "/LICENSE" + if license_path in content and license_path not in required: + required.append(license_path) + directory = posixpath.dirname(directory) + optional = sorted( + path for path in content + if path.startswith(prefix) and path not in required + ) + return core, required, optional + + +def skill_score( + cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1" +) -> int: + if method not in ("weighted-lexical-v1", "weighted-raw-string-v1"): + raise ValueError("Unknown relevance method for skill-only ranking") + scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score + return scorer(cue, [ + (4, record["title"]), + (3, record["summary"]), + (2, [record.get("tags", []), record.get("applicability", {})]), + (1, core.decode("utf-8")), + ]) + + +def task_score(cue: str, record: dict, core: bytes, backend: str | None) -> int: + """Rank whole task terms across both skill and repository knowledge owners.""" + def terms(text: str) -> set[str]: + return set(re.findall(r"[^\W_]+", text.casefold())) + + tokens = terms(cue) - {"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"} + fields = ( + (4, record["title"]), + (3, record["summary"]), + (2, " ".join(record.get("tags", []))), + (2, record["repo"] + " " + record["scenario"]), + (1, json.dumps(record.get("applicability", {}), ensure_ascii=False)), + (1, core.decode("utf-8")), + ) + score = sum(weight * len(tokens & terms(text)) for weight, text in fields) + # With no platform in scope prefer portable procedures to a specialization + # that happens to repeat the same task words in its description. + platforms = { + "cuda": {"cuda", "nvidia"}, "rocm": {"rocm", "amd"}, + "ascend": {"ascend", "cann"}, "mlx": {"mlx"}, "cpu": {"cpu"}, + } + in_scope = {backend} if backend else {key for key, names in platforms.items() if names & tokens} + return score * (2 if record["backend"] == "common" or record["backend"] in in_scope else 1) + + +def record_context(record: dict, view: dict, content: dict[str, bytes], available: int) -> tuple: + """Keep the canonical package record including its sources and limits.""" + path = view["entrypoint"] + body = content[path] + if len(body) > available: + return None, {"ref": view["ref"], "reason": "context_limit", + "required_bytes": len(body), "available_bytes": available} + return { + "ref": view["ref"], "revision": view["revision"], "content_bytes": len(body), + "resources": [{"path": path, "sha256": view["resources"][path], "role": "record", + "locator": "whole-resource", "content": body.decode("utf-8")}], + "omissions": [{"path": p, "reason": "linked_document"} for p in + dict.fromkeys(record.get("skill_documents", {}).values())], + "sources": view["sources"], "attribution": view["attribution"], + "limitations": view["limitations"], + }, None + + +def context_view( + record: dict, view: dict, content: dict[str, bytes], cue: str, available: int +) -> tuple[dict | None, dict | None]: + core, required, optional = indexed_skill(record, view, content) + required_bytes = sum(len(content[path]) for path in required) + if required_bytes > available: + return None, { + "ref": view["ref"], "reason": "context_limit", + "required_bytes": required_bytes, "available_bytes": available, + } + + resources, omissions = [], [] + used = 0 + + def include(path: str) -> None: + nonlocal used + resources.append({ + "path": path, "sha256": view["resources"][path], "locator": "whole-resource", + "role": "core" if path == core else "reference", + "content": content[path].decode("utf-8"), + }) + used += len(content[path]) + + for path in required: + include(path) + candidates = [] + for path in optional: + if ( + posixpath.basename(path) == "metadata.json" + or posixpath.splitext(path)[1].lower() not in (".md", ".json", ".txt", ".csv") + ): + omissions.append({"path": path, "reason": "not_context_resource"}) + continue + try: + text = content[path].decode("utf-8") + except UnicodeDecodeError: + omissions.append({"path": path, "reason": "not_utf8"}) + continue + score = lexical_score(cue, [(2, path.rpartition("/")[2]), (1, text)]) + if words(cue) and score == 0: + omissions.append({"path": path, "reason": "not_relevant"}) + continue + candidates.append((score, path)) + for _, path in sorted(candidates, key=lambda item: (-item[0], item[1])): + if used + len(content[path]) > available: + omissions.append({"path": path, "reason": "context_limit"}) + continue + include(path) + return { + "ref": view["ref"], "revision": view["revision"], "resources": resources, + "content_bytes": used, "omissions": omissions, + "sources": view["sources"], "attribution": view["attribution"], + "limitations": view["limitations"] + [ + "Selected resources are complete files; this view is not a standalone Bundle.", + "Follow omitted or other linked resources through the exact owner revision.", + ], + }, None diff --git a/src/inferrsi/knowledge/_source_snapshot.py b/src/inferrsi/knowledge/_source_snapshot.py new file mode 100644 index 000000000..3ced2e330 --- /dev/null +++ b/src/inferrsi/knowledge/_source_snapshot.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import re +from typing import Any + +# One shared snapshot keeps the repository-wide source revision and registry digest out of +# per-owner records, so a one-owner edit regenerates only that owner's record. +MANIFEST = "_provenance.json" +REPOSITORY = "https://github.com/InferMatrixLab/InferRSI" +LIBRARY = "src/inferrsi/data/skill_library" +REGISTRY = f"{LIBRARY}/skill-index.json" +STORED_SCHEMA = 2 + + +def manifest(revision: str, registry_sha256: str) -> dict[str, Any]: + return { + "schema_version": 1, + "repository": REPOSITORY, + "source_revision": revision, + "registry": {"path": REGISTRY, "sha256": registry_sha256}, + } + + +def _snapshot(value: Any) -> tuple[str, str]: + if ( + not isinstance(value, dict) + or set(value) != {"schema_version", "repository", "source_revision", "registry"} + or type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or value["repository"] != REPOSITORY + or not isinstance(value["source_revision"], str) + or not re.fullmatch(r"[0-9a-f]{40}", value["source_revision"]) + ): + raise ValueError("Invalid knowledge source snapshot") + registry = value["registry"] + if ( + not isinstance(registry, dict) + or set(registry) != {"path", "sha256"} + or registry["path"] != REGISTRY + or not isinstance(registry["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", registry["sha256"]) + ): + raise ValueError("Invalid knowledge source snapshot registry") + return value["source_revision"], registry["sha256"] + + +def compact(record: dict[str, Any]) -> dict[str, Any]: + """Store a generated schema-v1 record without its snapshot-wide fields.""" + stored = {key: value for key, value in record.items() if key != "sources"} + stored["schema_version"] = STORED_SCHEMA + stored["source_files"] = { + path: digest for path, digest in record["source_files"].items() if path != REGISTRY + } + return stored + + +def resolve(stored: dict[str, Any], snapshot: Any) -> dict[str, Any]: + """Rebuild the public schema-v1 record from a stored record and the shared snapshot.""" + revision, registry_sha256 = _snapshot(snapshot) + if stored.get("schema_version") != STORED_SCHEMA or "sources" in stored: + raise ValueError("Stored skill record is not a schema-v2 projection") + files = stored.get("source_files") + documents = stored.get("skill_documents") + if ( + not isinstance(files, dict) or REGISTRY in files + or not isinstance(documents, dict) or not isinstance(documents.get("original"), str) + ): + raise ValueError(f"Invalid stored skill record: {stored.get('id')}") + entry = "src/inferrsi/" + documents["original"] + if entry not in files: + raise ValueError(f"Stored skill record lacks its entry hash: {stored.get('id')}") + hashes = dict(sorted({**files, REGISTRY: registry_sha256}.items())) + source = { + "type": "git", + "url": f"{REPOSITORY}/blob/{revision}/{entry}", + "commit": revision, + "paths": list(hashes), + "content_sha256": hashes[entry], + } + record = {} + for key, value in stored.items(): + record[key] = 1 if key == "schema_version" else value + if key == "origin": + record["sources"] = [source] + if "sources" not in record: + raise ValueError(f"Stored skill record lacks origin: {stored.get('id')}") + record["source_files"] = hashes + return record diff --git a/src/inferrsi/knowledge/catalog.py b/src/inferrsi/knowledge/catalog.py new file mode 100644 index 000000000..da5f417fe --- /dev/null +++ b/src/inferrsi/knowledge/catalog.py @@ -0,0 +1,168 @@ +"""Read-only package-data catalog. Knowledge text is never executable.""" + +from __future__ import annotations + +import json +import re +from importlib.resources import files +from typing import Any + +from .._repo_pack_data import REPO_TYPES +from . import _source_snapshot + +# Learning scope does not imply that a repository has a runnable target adapter. +# Legacy repository-pack records retain their contribution tier; an independent +# peer card can describe a different learning use case for the same source repo. +KNOWLEDGE_SCOPES = { + "contribution": (*REPO_TYPES, "vllm_ascend", "inferrsi"), + "dependency": ( + "flashinfer", "flashattention", "pytorch", "triton", "diffusers", "transformers", "mlx" + ), + "peer": ( + "sglang", "tensorrt_llm", "lmdeploy", "xdit", "cache_dit", "qwen3_tts_engine", "deepinfer" + ), +} +KNOWLEDGE_REPOS = tuple( + dict.fromkeys(repo for repos in KNOWLEDGE_SCOPES.values() for repo in repos) +) +BACKENDS = ("cpu", "cuda", "rocm", "ascend", "mlx") +SCENARIO_ALIASES = { + "diffusion.t2i": "diffusion.image_generation", + "diffusion.t2v": "diffusion.video_generation", +} + + +def _walk(root, prefix=()): + for child in sorted(root.iterdir(), key=lambda item: item.name): + parts = (*prefix, child.name) + if child.is_dir(): + yield from _walk(child, parts) + else: + yield parts, child + + +def _validate_record( + record: dict, repo: str, parts: tuple[str, ...], tier: str = "contribution" +) -> None: + required = ("id", "title", "repo", "backend", "component", "scenario", "status", "summary") + if not isinstance(record, dict) or any( + not isinstance(record.get(key), str) or not record[key].strip() for key in required + ): + raise ValueError(f"Incomplete knowledge record: {repo}/{'/'.join(parts)}") + if record.get("schema_version") != 1: + raise ValueError(f"Unsupported knowledge schema: {record['id']}") + # Existing schema-v1 records retain their IDs and default to contribution. + if record.get("tier", "contribution") != tier: + raise ValueError(f"Knowledge tier disagrees with path: {record['id']}") + if len(parts) != 3: + raise ValueError("Knowledge path must be BACKEND/COMPONENT/ENTRY.json") + backend, component, filename = parts + if backend not in (*BACKENDS, "common"): + raise ValueError(f"Unknown knowledge backend: {backend}") + if (record["repo"], record["backend"], record["component"]) != (repo, backend, component): + raise ValueError(f"Knowledge scope disagrees with path: {record['id']}") + expected_id = f"{repo}.{backend}.{component}.{filename[:-5].replace('-', '_')}" + if record["id"] != expected_id: + raise ValueError(f"Knowledge ID disagrees with path: {record['id']}") + if not record["scenario"].startswith(component + "."): + raise ValueError(f"Knowledge scenario disagrees with component: {record['id']}") + sources = record.get("sources") + if not isinstance(sources, list) or not sources: + raise ValueError(f"Knowledge has no source: {record['id']}") + for source in sources: + if ( + not isinstance(source, dict) + or not isinstance(source.get("url"), str) + or not source["url"].startswith("https://") + or not isinstance(source.get("commit"), str) + or not re.fullmatch(r"[0-9a-f]{40}", source["commit"]) + ): + raise ValueError(f"Knowledge requires source attribution and revision: {record['id']}") + verification = record.get("verification", {}) + if not isinstance(verification, dict) or not isinstance( + verification.get("locally_verified"), bool + ): + raise ValueError(f"Knowledge requires an explicit verification state: {record['id']}") + if record["status"] == "imported" and verification["locally_verified"]: + raise ValueError(f"Imported knowledge must state locally_verified=false: {record['id']}") + + +def knowledge_records() -> list[dict[str, Any]]: + data = files("inferrsi").joinpath("data") + records = [] + seen = set() + for tier, repos in KNOWLEDGE_SCOPES.items(): + for repo in repos: + knowledge = ( + data.joinpath("repo_packs", repo, "knowledge") + if tier == "contribution" + else data.joinpath("knowledge_packs", tier, repo) + ) + if not knowledge.is_dir(): + continue + snapshot = knowledge.joinpath(_source_snapshot.MANIFEST) + snapshot = ( + json.loads(snapshot.read_text(encoding="utf-8")) if snapshot.is_file() else None + ) + for parts, item in _walk(knowledge): + if not item.name.endswith(".json") or parts == (_source_snapshot.MANIFEST,): + continue + record = json.loads(item.read_text(encoding="utf-8")) + # Generated skill records store schema v2; readers always receive schema v1. + if isinstance(record, dict) and record.get("schema_version") == ( + _source_snapshot.STORED_SCHEMA + ): + if snapshot is None: + raise ValueError(f"Knowledge record lacks its source snapshot: {repo}") + record = _source_snapshot.resolve(record, snapshot) + _validate_record(record, repo, parts, tier) + if record["id"] in seen: + raise ValueError(f"Duplicate knowledge ID: {record['id']}") + seen.add(record["id"]) + records.append(record) + return records + + +def search_knowledge( + query: str = "", + *, + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, +) -> list[dict[str, Any]]: + if limit < 1: + raise ValueError("limit must be positive") + if tier is not None and tier not in KNOWLEDGE_SCOPES: + raise ValueError(f"Unknown knowledge tier: {tier}") + scenario = SCENARIO_ALIASES.get(scenario, scenario) + tokens = re.findall(r"[\w.-]+", query.casefold()) + scored = [] + for record in knowledge_records(): + if tier and record.get("tier", "contribution") != tier: + continue + if repo and record["repo"] != repo: + continue + if backend and record["backend"] not in (backend, "common"): + continue + if scenario: + declared = record.get("scenario", "") + if declared != scenario and not ( + declared.endswith(".*") and scenario.startswith(declared[:-1]) + ): + continue + document = json.dumps(record, ensure_ascii=False).casefold() + if any(token not in document for token in tokens): + continue + score = sum(document.count(token) for token in tokens) + scored.append((score, record["id"], record)) + scored.sort(key=lambda item: (-item[0], item[1])) + return [item[2] for item in scored[:limit]] + + +def get_knowledge(record_id: str) -> dict[str, Any]: + for record in knowledge_records(): + if record["id"] == record_id: + return record + raise ValueError(f"Unknown knowledge ID: {record_id}") diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py new file mode 100644 index 000000000..cff710c26 --- /dev/null +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -0,0 +1,735 @@ +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from dataclasses import dataclass +from datetime import date, datetime +from importlib.resources import files +from pathlib import Path, PurePosixPath + +from ._skill_selection import ( + RELEVANCE_METHODS, + TASK_RELEVANCE, + context_view, + indexed_skill, + record_context, + skill_score, + task_score, + words, +) +from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records + +AUTHORITIES = ("package", "checkout", "store") +IDENTITY = r"[a-z0-9][a-z0-9._-]*" + + +class RevisionConflict(ValueError): + pass + + +class IdentityError(ValueError): + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + +def safe_path(value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or "\\" in value + or ":" in value + or path.is_absolute() + or any(part in ("", ".", "..") for part in value.split("/")) + ): + raise ValueError(f"Unsafe bundle path: {value}") + return value + + +def _json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + + +def _normalized(value: object, field: str, ancestors: frozenset[int] = frozenset()) -> object: + # Bundle identity is JSON, so YAML scalars without a JSON form are rejected + # here instead of failing inside revision serialization. + if value is None or isinstance(value, (str, bool, int, float)): + return value + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, (list, dict)): + # A YAML alias can make a container its own descendant. Only the active path + # is tracked, so the same object shared by sibling fields stays accepted. + identity = id(value) + if identity in ancestors: + raise ValueError( + f"Field {field} contains a reference cycle: {type(value).__name__}" + ) + ancestors = ancestors | {identity} + if isinstance(value, list): + return [ + _normalized(item, f"{field}[{index}]", ancestors) + for index, item in enumerate(value) + ] + result = {} + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"Field {field} has a non-string key: {key!r}") + result[key] = _normalized(item, f"{field}.{key}" if field else key, ancestors) + return result + raise ValueError( + f"Field {field or ''} cannot be represented in the bundle identity: " + f"{type(value).__name__}" + ) + + +def _hash(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _sources(records: list[dict]) -> list[dict]: + result = [] + for record in records: + source = dict(record) + kind = source.get("type") + if kind in ("source", "source_code", "technical_document") and "commit" in source: + source["original_type"] = kind + kind = "git" + if kind is None and "commit" in source: + kind = "git" + if kind not in ("git", "paper", "document", "internal", "unknown"): + # Legacy types keep their original label; nothing is inferred from them. + source["original_type"] = kind + kind = "unknown" + source["type"] = kind + if kind == "git": + commit = source.get("commit") + if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{7,40}", commit): + raise ValueError("Git source requires a real commit or explicit historical prefix") + source["revision_precision"] = "full" if len(commit) == 40 else "prefix" + elif kind == "paper": + if not source.get("id") or "version" not in source: + raise ValueError("Paper source requires ID and explicit version (null if unknown)") + if "commit" in source: + raise ValueError("Paper versions are not Git commits") + elif kind == "document": + if not source.get("url") or not any(k in source for k in ("version", "sha256")): + raise ValueError("Document source requires URL and version or snapshot identity") + elif kind == "internal": + if not source.get("artifact") or not any(k in source for k in ("issue", "run")): + raise ValueError("Internal source requires artifact and issue/run identity") + result.append(source) + return result + + +@dataclass(frozen=True) +class Bundle: + metadata: dict + content: dict[str, bytes] + dependencies: dict[str, str] + + def __post_init__(self) -> None: + # Normalize once at ingestion so manifest, revision and views agree. + object.__setattr__(self, "metadata", _normalized(self.metadata, "metadata")) + + def manifest(self) -> dict: + record = self.metadata + if not isinstance(record.get("id"), str) or not re.fullmatch(IDENTITY, record["id"]): + raise ValueError("Invalid bundle identity") + if record.get("kind") not in ("skill", "knowledge", "case"): + raise ValueError("Invalid bundle kind") + for field in ( + "sources", + "applicability", + "state", + "limitations", + "attribution", + "entrypoint", + ): + if field not in record: + raise ValueError(f"Missing normalized field: {field}") + for field in ("state", "entrypoint"): + if not isinstance(record[field], str) or not record[field]: + raise ValueError(f"Normalized {field} must be a nonempty string") + if not isinstance(record["limitations"], list): + raise ValueError("Normalized limitations must be a list") + if not isinstance(record["sources"], list) or not record["sources"]: + raise ValueError("Bundle requires sources, including explicit unknown provenance") + _sources(record["sources"]) + for path, data in self.content.items(): + safe_path(path) + if not isinstance(data, bytes): + raise ValueError("Bundle content must be bytes") + if record["entrypoint"] not in self.content: + raise ValueError("Missing entrypoint in complete bundle") + for required in record.get("required_resources", []): + # Present string keys already passed the content path validation. + if isinstance(required, str) and required in self.content: + continue + if safe_path(required) not in self.content: + raise ValueError(f"Missing required resource: {required}") + resources = {path: _hash(data) for path, data in sorted(self.content.items())} + for dependency, revision in self.dependencies.items(): + safe_path(dependency) + if not re.fullmatch(r"sha256:[0-9a-f]{64}", revision): + raise ValueError("Dependency requires a resolved content revision") + if dependency in self.content: + actual = "sha256:" + resources[dependency] + else: + members = { + path: digest + for path, digest in resources.items() + if path.startswith(dependency + "/") + } + if not members: + raise ValueError(f"Missing dependency: {dependency}") + actual = "sha256:" + _hash(_json(members)) + if actual != revision: + raise RevisionConflict(f"Dependency bytes disagree with revision: {dependency}") + return { + "metadata": record, + "resources": resources, + "dependencies": dict(sorted(self.dependencies.items())), + } + + @property + def revision(self) -> str: + return "sha256:" + _hash(_json(self.manifest())) + + def view(self, *, material: bool = False) -> dict: + manifest = self.manifest() + result = { + **self.metadata, + "revision": "sha256:" + _hash(_json(manifest)), + "resources": manifest["resources"], + "dependencies": manifest["dependencies"], + } + if material: + result["content"] = self.content[self.metadata["entrypoint"]].decode("utf-8") + return result + + +INCIDENTAL = ("__pycache__", ".DS_Store") + + +def _tree( + node, prefix: str, *, root: Path | None = None, ancestors: frozenset[Path] = frozenset() +) -> dict[str, bytes]: + if isinstance(node, Path): + # A junction or symlink resolves away from where it appears; every descended + # directory must still land inside the root that owns this walk. + directory = node.resolve() + if root is None: + root = directory + if directory in ancestors: + raise ValueError(f"Resource cycle inside the skills root: {prefix}") + ancestors = ancestors | {directory} + result = {} + for child in sorted(node.iterdir(), key=lambda p: p.name): + if child.name in INCIDENTAL or child.name.endswith(".pyc"): + continue + path = safe_path(f"{prefix}/{child.name}") + if isinstance(child, Path) and child.is_symlink(): + raise ValueError(f"Symlink resource is not supported: {path}") + if child.is_dir(): + if isinstance(child, Path) and root is not None and not child.resolve().is_relative_to( + root + ): + raise ValueError(f"Resource escapes the skills root: {path}") + result.update(_tree(child, path, root=root, ancestors=ancestors)) + else: + result[path] = child.read_bytes() + return result + + +class KnowledgeReader: + def __init__(self, *, skill_root: Path | None = None, store=None): + self.skill_root = skill_root.resolve() if skill_root is not None else None + if store is not None and store.skill_root != self.skill_root: + raise ValueError("Store must reserve identities from the same checkout") + self.store = store + self._records: dict[str, dict] | None = None + self._library: tuple[dict[str, bytes], str] | None = None + + def records(self) -> dict[str, dict]: + """Installed catalog records, read once per reader (one package snapshot).""" + if self._records is None: + self._records = {record["id"]: record for record in knowledge_records()} + return self._records + + def _skill_library(self) -> tuple[dict[str, bytes], str]: + if self._library is None: + root = files("inferrsi").joinpath("data", "skill_library") + library = _tree(root, "data/skill_library") + digest = _hash(_json({path: _hash(data) for path, data in sorted(library.items())})) + self._library = (library, "sha256:" + digest) + return self._library + + def owners(self) -> dict[str, list[str]]: + """Map each public ID to every authority that owns it.""" + result: dict[str, list[str]] = {} + for identity in self.records(): + result.setdefault(identity, []).append("package") + if self.skill_root is not None: + # Enumeration reads registered names only. Validating sidecar content + # here would abort the whole collection over one malformed, possibly + # unrequested, skill; Collect reports that per item through _read. + from .skill_catalog import skill_directories + + local = list(skill_directories(self.skill_root)) + local.extend(self._cases()) + if len(set(local)) != len(local): + raise IdentityError("collision", "Duplicate identity inside the checkout") + for identity in local: + result.setdefault(identity, []).append("checkout") + if self.store is not None: + for identity in self.store.identities(): + result.setdefault(identity, []).append("store") + return result + + def identities(self) -> list[str]: + return sorted(f"{a}:{i}" for i, authorities in self.owners().items() for a in authorities) + + def resolve(self, reference: str, owners: dict[str, list[str]] | None = None) -> str: + """Return the qualified `authority:id`; ambiguity is an error, never precedence.""" + authority, qualified, identity = reference.rpartition(":") + if not re.fullmatch(IDENTITY, identity) or (qualified and authority not in AUTHORITIES): + raise IdentityError("invalid_identity", f"Invalid identity: {reference}") + found = (self.owners() if owners is None else owners).get(identity, []) + if qualified: + if authority not in found: + raise IdentityError("unknown_identity", f"Unknown identity: {reference}") + return reference + if not found: + raise IdentityError("unknown_identity", f"Unknown identity: {identity}") + if len(found) > 1: + choices = ", ".join(f"{a}:{identity}" for a in found) + raise IdentityError( + "ambiguous_identity", f"Ambiguous identity {identity}; qualify one of: {choices}" + ) + return f"{found[0]}:{identity}" + + def _cases(self) -> dict[str, Path]: + result = {} + if self.skill_root is not None: + from .skill_catalog import skill_directories + + for directory in skill_directories(self.skill_root).values(): + for path in sorted(directory.glob("cases/*.json")): + if path.is_symlink() or not path.resolve().is_relative_to(self.skill_root): + raise ValueError("Case must stay inside the selected skills root") + record = json.loads(path.read_text(encoding="utf-8")) + if record.get("kind") == "case": + identity = record["id"] + if not isinstance(identity, str) or not re.fullmatch(IDENTITY, identity): + raise ValueError(f"Invalid case identity in {path.name}: {identity!r}") + if identity in result: + raise ValueError(f"Duplicate case identity: {identity}") + result[identity] = path + return result + + def read(self, reference: str, *, revision: str | None = None) -> Bundle: + return self._read(self.resolve(reference), revision) + + def _read(self, qualified: str, revision: str | None) -> Bundle: + authority, _, identity = qualified.partition(":") + if authority == "store": + return self.store.read(identity, revision=revision) + bundle = self._checkout(identity) if authority == "checkout" else self._knowledge(identity) + if revision is not None and bundle.revision != revision: + raise RevisionConflict( + f"Revision unavailable in this snapshot: {qualified} at {revision}" + ) + return bundle + + def _checkout(self, identity: str) -> Bundle: + cases = self._cases() + if identity in cases: + path = cases[identity] + record = json.loads(path.read_text(encoding="utf-8")) + for field in ("references", "dependencies"): + if field in record: + raise ValueError(f"Case {field} are not supported; declare required_resources") + declared = record.get("required_resources", []) + if not isinstance(declared, list) or any(not isinstance(p, str) for p in declared): + raise ValueError("Case required_resources must be a string list") + if len(set(declared)) != len(declared): + raise ValueError("Case required_resources must not repeat a path") + relative = safe_path(record["content"]) + required = [relative, "case.json", *(p for p in declared if p != relative)] + content = {"case.json": path.read_bytes()} + for resource in [relative, *required[2:]]: + safe_path(resource) + if resource == "case.json": + raise ValueError("Case resource collides with its case.json record") + resource_path = path.parent / resource + if resource_path.is_symlink() or not resource_path.resolve().is_relative_to( + self.skill_root + ): + raise ValueError(f"Case resource escapes the skills root: {resource}") + if not resource_path.is_file(): + raise ValueError(f"Missing case resource: {resource}") + content[resource] = resource_path.read_bytes() + return Bundle( + { + **record, + "entrypoint": relative, + "sources": _sources(record["sources"]), + "required_resources": required, + }, + content, + {}, + ) + return self._skill(identity) + + def _knowledge(self, identity: str) -> Bundle: + record = self.records().get(identity) + if record is None: + raise IdentityError("unknown_identity", f"Unknown knowledge ID: {identity}") + content = {"record.json": _json(record)} + dependencies = {} + if record.get("skill_documents"): + # The whole imported library is one explicit dependency so bilingual + # references and licenses stay a complete closure. + library, digest = self._skill_library() + content.update(library) + dependencies["data/skill_library"] = digest + documents = record["skill_documents"] + if not isinstance(documents, dict): + raise ValueError(f"skill_documents must map editions to paths: {identity}") + for target in documents.values(): + if not isinstance(target, str) or safe_path(target) not in library: + raise ValueError(f"Missing declared skill document: {target}") + return Bundle( + { + "id": identity, + "kind": "knowledge", + "entrypoint": "record.json", + "origin": record.get("origin", ["external"]), + "sources": _sources(record["sources"]), + "applicability": record.get("applicability", {}), + "state": record["status"], + "limitations": record.get("limitations", []) + + [ + "Imported guidance; reading does not validate or execute it.", + "Historical revisions require the matching installed snapshot " + "or a retained store.", + ], + "attribution": record["sources"], + "verification": record["verification"], + "tier": record.get("tier", "contribution"), + "repo": record["repo"], + "backend": record["backend"], + "scenario": record["scenario"], + "title": record["title"], + "skill_documents": record.get("skill_documents", {}), + "required_resources": sorted(content), + }, + content, + dependencies, + ) + + def _skill(self, identity: str) -> Bundle: + from .skill_catalog import get_skill, skill_directory, skill_metadata, skill_reference + + root = self.skill_root + record = get_skill(root, identity) + directory = skill_directory(root, identity) + prefix = directory.relative_to(root).as_posix() + content = _tree(directory, prefix, root=root) + metadata = skill_metadata(root, identity) + dependencies = {} + for reference in metadata.get("references", []): + path = skill_reference(root, identity, reference) + key = path.relative_to(root).as_posix() + content[key] = path.read_bytes() + dependencies[key] = "sha256:" + _hash(content[key]) + return Bundle( + { + "id": identity, + "kind": "skill", + "entrypoint": f"{prefix}/SKILL.md", + "title": record["description"], + "origin": metadata.get("origin", ["unknown"]), + "sources": _sources(metadata.get("sources", [])) + or [ + { + "type": "unknown", + "note": "Legacy provenance remains in procedure/readsets", + } + ], + "applicability": metadata.get("applicability", {}), + "state": record.get("status", "unspecified"), + "limitations": metadata.get("limitations", []) + + [ + "Checkout-only local skill; no automatic execution or promotion.", + "Only declared references are required; " + "other cross-skill links are navigation.", + "Historical revisions require a matching checkout or a retained store.", + ], + "attribution": metadata.get("sources", []), + "empirical_validation": metadata.get("empirical_validation", "unknown"), + "required_resources": sorted(content), + }, + content, + dependencies, + ) + + def resource(self, identity: str, path: str, *, revision: str) -> bytes: + safe_path(path) + bundle = self.read(identity, revision=revision) + if path not in bundle.content: + raise ValueError(f"Unknown resource: {path}") + return bundle.content[path] + + def collect( + self, + query: str = "", + *, + ids: list[str] | None = None, + revisions: dict[str, str] | None = None, + kind: str | None = None, + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, + max_bytes: int | None = None, + material: bool = False, + relevance: str | None = None, + relevance_method: str = "weighted-lexical-v1", + context_bytes: int | None = None, + ) -> dict: + if type(limit) is not int or limit < 1: + raise ValueError("limit must be a positive integer") + if max_bytes is not None and (type(max_bytes) is not int or max_bytes < 1): + raise ValueError("max_bytes must be a positive integer") + if ids is not None and ( + not isinstance(ids, list) or any(not isinstance(identity, str) for identity in ids) + ): + raise ValueError("ids must be a list of identity strings") + if kind not in (None, "skill", "knowledge", "case"): + raise ValueError("Unknown content kind") + if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): + raise ValueError("relevance requires at least one word") + if relevance_method not in RELEVANCE_METHODS: + raise ValueError("Unknown relevance method") + if relevance is None and relevance_method != "weighted-lexical-v1": + raise ValueError("relevance_method requires relevance") + if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): + raise ValueError("context_bytes must be a positive integer") + if revisions is None: + revisions = {} + if not isinstance(revisions, dict) or any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in revisions.items() + ): + raise ValueError("revisions must map identity strings to revision strings") + if ids is not None and len(set(ids)) != len(ids): + raise ValueError("Duplicate requested identity") + owners = self.owners() + requested = ids if ids is not None else self.identities() + + def qualified(reference: str) -> str | None: + # Revision keys may use any spelling the API accepts for an identity. + try: + return self.resolve(reference, owners) + except IdentityError: + return None + + selected = {identity for identity in map(qualified, requested) if identity is not None} + pinned: dict[str, str] = {} + for reference, revision in revisions.items(): + target = qualified(reference) + if target is None or target not in selected: + raise ValueError( + f"Revision requested for an unselected identity: {reference}" + ) + if pinned.setdefault(target, revision) != revision: + raise ValueError(f"Conflicting revisions for the same identity: {reference}") + pinned[target] = revision + collisions = sorted(i for i, authorities in owners.items() if len(authorities) > 1) + if tier is not None and tier not in KNOWLEDGE_SCOPES: + raise ValueError(f"Unknown knowledge tier: {tier}") + scenario = SCENARIO_ALIASES.get(scenario, scenario) + package_filters = any(value is not None for value in (repo, tier, backend, scenario)) + task_ranking = relevance_method == TASK_RELEVANCE + + def matches(metadata: dict, authority: str) -> bool: + declared = metadata.get("scenario") + declared = declared if isinstance(declared, str) else "" + # The catalog records skills.discovery as the discovery operation, + # not the workload served by a procedure. Let the task cue rank + # these procedures alongside records declaring the requested workload. + discovery_skill = ( + task_ranking and ids is None and authority == "package" + and declared == "skills.discovery" + and self.records()[metadata["id"]].get("index_type") == "packaged-skill" + ) + return not ( + (kind and metadata.get("kind") != kind) + or (package_filters and authority != "package") + or (repo and metadata.get("repo") != repo) + or (tier and metadata.get("tier", "contribution") != tier) + or (backend and metadata.get("backend") not in (backend, "common")) + or not ( + scenario is None + or declared == scenario + or (declared.endswith(".*") and scenario.startswith(declared[:-1])) + or discovery_skill + ) + ) + + cases = self._cases() if kind else {} + selected, omitted, seen = [], [], set() + used = 0 + ranked, contexts, context_omissions = [], [], [] + context_used = 0 + + def select(reference: str, bundle: Bundle, view: dict) -> None: + nonlocal used, context_used + # Normal Bundle views were serialized while computing their revision. + # Preserve custom subclasses' serialization/error behavior below. + if type(bundle) is Bundle and len(selected) >= limit: + omitted.append({"id": reference, "reason": "output_limit"}) + return + size = len(_json(view)) + if len(selected) >= limit or (max_bytes is not None and used + size > max_bytes): + omitted.append({"id": reference, "reason": "output_limit"}) + return + selected.append(view) + used += size + if context_bytes is None: + return + record = self.records().get(view["id"], {}) + is_skill = record.get("index_type") == "packaged-skill" + if view["authority"] != "package" or (not is_skill and not task_ranking): + context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) + return + try: + available = context_bytes - context_used + if is_skill: + context, omission = context_view( + record, view, bundle.content, relevance or query, available + ) + else: + context, omission = record_context(record, view, bundle.content, available) + except (ValueError, KeyError, TypeError) as exc: + context_omissions.append({ + "ref": view["ref"], "reason": "index_unavailable", "detail": str(exc), + }) + return + if omission is not None: + context_omissions.append(omission) + else: + contexts.append(context) + context_used += context["content_bytes"] + + for reference in requested: + try: + qualified = self.resolve(reference, owners) + if qualified in seen: + raise IdentityError("duplicate_identity", f"Already selected: {qualified}") + seen.add(qualified) + authority, _, identity = qualified.partition(":") + # Cheap metadata checks first, so filtered items never load resource bytes. + if authority == "package": + early = {**self.records()[identity], "kind": "knowledge"} + elif authority == "checkout": + early = {"kind": "case" if identity in cases else "skill"} + else: + early = None + if relevance is not None and ids is None and ( + authority != "package" + or (not task_ranking and early.get("index_type") != "packaged-skill") + ): + continue + if early is not None and not matches(early, authority): + if ids is not None: + omitted.append({"id": reference, "reason": "filters"}) + continue + bundle = self._read(qualified, pinned.get(qualified)) + view = {**bundle.view(material=material), "authority": authority, "ref": qualified} + entry = bundle.content[bundle.metadata["entrypoint"]] + haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() + except IdentityError as exc: + omitted.append({"id": reference, "reason": exc.code, "detail": str(exc)}) + continue + except RevisionConflict as exc: + omitted.append( + {"id": reference, "reason": "unavailable_revision", "detail": str(exc)} + ) + continue + except (ValueError, KeyError, TypeError, AttributeError, OSError, sqlite3.Error) as exc: + omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) + continue + if not matches(view, authority): + if ids is not None: + omitted.append({"id": reference, "reason": "filters"}) + continue + if any(token not in haystack for token in query.casefold().split()): + if ids is not None: + omitted.append({"id": reference, "reason": "query"}) + continue + if relevance is not None and ids is None: + record = self.records().get(identity, {}) + try: + is_skill = record.get("index_type") == "packaged-skill" + if is_skill: + core, _, _ = indexed_skill(record, view, bundle.content) + body = bundle.content[core] + else: + body = entry + if task_ranking: + score = task_score(relevance, record, body, backend) + else: + score = skill_score(relevance, record, body, method=relevance_method) + except (ValueError, KeyError, TypeError) as exc: + omitted.append({ + "id": reference, "reason": "index_unavailable", "detail": str(exc), + }) + continue + if score: + ranked.append((score, reference, bundle, view)) + continue + select(reference, bundle, view) + for _, reference, bundle, view in sorted( + ranked, key=lambda item: (-item[0], item[3]["ref"]) + ): + select(reference, bundle, view) + result = { + "items": selected, + "omissions": omitted, + "no_match": not selected, + "collisions": collisions, + "material_bytes": used, + "limitations": [ + "No crawling, ranking model or conversation state; deterministic identity order.", + "max_bytes bounds item payloads, not omission/control metadata.", + "Read references must be resolved with their returned exact revision.", + ], + } + if relevance is not None: + result["selection"] = { + "method": relevance_method, "cue": relevance, + "scope": ("package-skills-and-records" if task_ranking + else "generated-packaged-skills"), + "ranking_applied": ids is None, + } + if task_ranking and ids is None: + result["selection"].update(matched_count=len(ranked), returned_count=len(selected)) + result["limitations"][0] = ( + "Deterministic lexical selection; explicit IDs retain caller order and membership." + ) + if context_bytes is not None: + result.update({ + "contexts": contexts, "context_omissions": context_omissions, + "context_bytes": context_used, + "context_budget": { + "max_bytes": context_bytes, "accounting": "utf8-resource-content", + "excludes": "provenance, omission and control metadata", + }, + }) + return result diff --git a/src/inferrsi/knowledge/knowledge_store.py b/src/inferrsi/knowledge/knowledge_store.py new file mode 100644 index 000000000..4cb585703 --- /dev/null +++ b/src/inferrsi/knowledge/knowledge_store.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing +from importlib.resources import files +from pathlib import Path + +from .knowledge_operations import ( + AUTHORITIES, + IDENTITY, + Bundle, + IdentityError, + KnowledgeReader, + RevisionConflict, + _json, + safe_path, +) + + +def _protected(skill_root: Path | None) -> list[Path]: + package = Path(str(files("inferrsi"))).resolve() + roots = [package] + # A source, editable or exported tree (git archive, sdist) surrounds the package + # and holds evidence and both matrices; protect it even without a .git entry, + # because review snapshots carry no repository metadata. + for parent in package.parents: + if (parent / "pyproject.toml").is_file() or (parent / ".git").exists(): + roots.append(parent) + break + if skill_root is not None: + roots.append(Path(skill_root).resolve().parent) + return roots + + +class KnowledgeStore: + """One authorized writable store for store-owned candidate bundles. + + Package and checkout content stay read-only; their IDs are reserved here so a + store candidate cannot shadow them. Repository-owned amendments use PRs. + """ + + def __init__(self, workspace: Path, *, skill_root: Path | None, read_only: bool = False): + # The caller authorizes an existing dedicated directory; none is created or guessed. + workspace = Path(workspace) + if workspace.is_symlink() or not workspace.is_dir(): + raise ValueError("Store requires an existing authorized workspace directory") + self.workspace = workspace.resolve() + for root in _protected(skill_root): + if self.workspace.is_relative_to(root): + raise ValueError(f"Store workspace must be outside read-only sources: {root}") + self.skill_root = Path(skill_root).resolve() if skill_root is not None else None + self.adapters = KnowledgeReader(skill_root=self.skill_root) + self.read_only = read_only + self.database = self.workspace / "knowledge.sqlite3" + if self.database.is_symlink(): + raise ValueError("Store database cannot be a symlink") + if read_only: + # Reading an existing store must not create or migrate anything. + if not self.database.is_file(): + raise ValueError(f"Read-only store requires an existing database: {self.database}") + self._verify_reserved_skill_root() + return + with closing(self._connect()) as connection, connection: + connection.executescript( + "CREATE TABLE IF NOT EXISTS revisions (" + "id TEXT NOT NULL, revision TEXT NOT NULL, metadata TEXT NOT NULL, " + "dependencies TEXT NOT NULL, parent TEXT, reason TEXT NOT NULL, " + "evidence TEXT NOT NULL, PRIMARY KEY (id, revision));" + "CREATE TABLE IF NOT EXISTS resources (" + "id TEXT NOT NULL, revision TEXT NOT NULL, path TEXT NOT NULL, data BLOB NOT NULL, " + "PRIMARY KEY (id, revision, path), " + "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" + "CREATE TABLE IF NOT EXISTS current (id TEXT PRIMARY KEY, revision TEXT NOT NULL, " + "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" + "CREATE TABLE IF NOT EXISTS meta (" + "key TEXT PRIMARY KEY, value TEXT NOT NULL);" + ) + reserved = connection.execute( + "SELECT value FROM meta WHERE key='skill_root'" + ).fetchone() + if reserved is None: + connection.execute( + "INSERT INTO meta VALUES ('skill_root', ?)", (self._reserved_skill_root(),) + ) + elif reserved[0] != self._reserved_skill_root(): + raise ValueError( + f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " + "reopen it with the same skill_root" + ) + + def _reserved_skill_root(self) -> str: + # The reserved checkout is recorded so later opens cannot silently drop it; + # an empty value records a package-only store. + return str(self.skill_root) if self.skill_root is not None else "" + + def _verify_reserved_skill_root(self) -> None: + # Stores created before reservation recording have no meta row and stay usable. + with closing(self._connect()) as connection: + present = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'" + ).fetchone() + reserved = ( + connection.execute("SELECT value FROM meta WHERE key='skill_root'").fetchone() + if present + else None + ) + if reserved is not None and reserved[0] != self._reserved_skill_root(): + raise ValueError( + f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " + "open it with the same --skill-root, or without one for a package-only store" + ) + + def _connect(self): + if self.database.is_symlink() or self.database.resolve().parent != self.workspace: + raise ValueError("Store database escapes the authorized workspace") + if self.read_only: + # mode=ro fails every write at the SQLite level; as_uri() percent-encodes + # spaces and non-ASCII so the file: URI stays valid on Windows too. + connection = sqlite3.connect(self.database.as_uri() + "?mode=ro", uri=True, timeout=5) + else: + connection = sqlite3.connect(self.database, timeout=5) + connection.execute("PRAGMA foreign_keys=ON") + return connection + + def identities(self) -> list[str]: + with closing(self._connect()) as connection: + return [row[0] for row in connection.execute("SELECT id FROM current ORDER BY id")] + + def reader(self) -> KnowledgeReader: + return KnowledgeReader(skill_root=self.skill_root, store=self) + + def collect(self, query: str = "", **options) -> dict: + return self.reader().collect(query, **options) + + def create(self, bundle: Bundle) -> str: + if self.read_only: + raise ValueError("Store is read-only; create requires a writable store") + return self._publish(bundle, expected=None, reason="create", evidence=[]) + + def update(self, bundle: Bundle, *, expected: str, reason: str, evidence: list[dict]) -> str: + if self.read_only: + raise ValueError("Store is read-only; update requires a writable store") + if not expected or not reason.strip(): + raise ValueError("Update requires expected old revision and reason") + if not isinstance(evidence, list): + raise ValueError("Update evidence must be a list; an empty list means none supplied") + return self._publish(bundle, expected=expected, reason=reason, evidence=evidence) + + def _lineage(self, metadata: dict) -> None: + lineage = metadata.get("derived_from", []) + if not isinstance(lineage, list): + raise ValueError("derived_from must be a list of qualified source revisions") + for source in lineage: + if not isinstance(source, dict) or not isinstance(source.get("ref"), str): + raise ValueError("derived_from entries require a qualified ref") + authority, _, identity = source["ref"].partition(":") + if authority not in AUTHORITIES or not re.fullmatch(IDENTITY, identity): + raise ValueError(f"derived_from ref must be :: {source['ref']}") + revision = source.get("revision") + if revision is not None and ( + not isinstance(revision, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", revision) + ): + raise ValueError("derived_from revision must be a content revision or null") + if identity == metadata["id"]: + raise IdentityError( + "collision", "An adaptation needs a distinct identity from its source" + ) + + def _publish(self, bundle: Bundle, *, expected: str | None, reason: str, evidence: list) -> str: + # Copy caller data, validate it, then store the whole bundle and current + # pointer in one transaction. + bundle = Bundle( + json.loads(_json(bundle.metadata)), dict(bundle.content), dict(bundle.dependencies) + ) + revision = bundle.revision + identity = bundle.metadata["id"] + try: + bundle.content[bundle.metadata["entrypoint"]].decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("Bundle entrypoint must be UTF-8 text") from exc + self._lineage(bundle.metadata) + owners = self.adapters.owners().get(identity) + if owners: + raise IdentityError( + "collision", + f"Identity {identity} is owned by read-only {', '.join(owners)} content; " + "amend it through a repository PR or create an adaptation with a distinct ID", + ) + if expected is None and bundle.metadata["state"] != "candidate": + raise ValueError( + "Create registers a candidate; imported or active content is not promoted" + ) + try: + with closing(self._connect()) as connection, connection: + connection.execute("BEGIN IMMEDIATE") + current = connection.execute( + "SELECT revision FROM current WHERE id=?", (identity,) + ).fetchone() + if expected is None: + if current is not None: + raise RevisionConflict(f"Duplicate identity: {identity}") + elif current is None or current[0] != expected: + raise RevisionConflict(f"Stale or missing revision for {identity}") + if expected is not None: + previous = connection.execute( + "SELECT metadata FROM revisions WHERE id=? AND revision=?", + (identity, expected), + ).fetchone() + old = json.loads(previous[0]) + if bundle.metadata["state"] != old["state"]: + raise ValueError("Update cannot promote content state") + if revision == expected: + raise ValueError("Update must change the bundle") + if connection.execute( + "SELECT 1 FROM revisions WHERE id=? AND revision=?", (identity, revision) + ).fetchone(): + raise RevisionConflict( + f"Update reproduces retained revision {revision}; " + "change the bundle so the successor has its own revision" + ) + connection.execute( + "INSERT INTO revisions VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + identity, + revision, + _json(bundle.metadata).decode(), + _json(bundle.dependencies).decode(), + expected, + reason, + _json(evidence).decode(), + ), + ) + connection.executemany( + "INSERT INTO resources VALUES (?, ?, ?, ?)", + [(identity, revision, path, data) for path, data in bundle.content.items()], + ) + connection.execute( + "INSERT INTO current VALUES (?, ?) " + "ON CONFLICT(id) DO UPDATE SET revision=excluded.revision", + (identity, revision), + ) + except sqlite3.OperationalError as exc: + if "locked" not in str(exc) and "busy" not in str(exc): + raise + raise RevisionConflict( + f"Store is locked by another writer; retry {identity} once it is released" + ) from exc + return revision + + def read(self, identity: str, *, revision: str | None = None) -> Bundle: + with closing(self._connect()) as connection: + if revision is None: + row = connection.execute( + "SELECT revision FROM current WHERE id=?", (identity,) + ).fetchone() + if row is None: + raise IdentityError("unknown_identity", f"Unknown identity: store:{identity}") + revision = row[0] + row = connection.execute( + "SELECT metadata, dependencies FROM revisions WHERE id=? AND revision=?", + (identity, revision), + ).fetchone() + if row is None: + raise RevisionConflict(f"Unknown revision: store:{identity} at {revision}") + resources = connection.execute( + "SELECT path, data FROM resources WHERE id=? AND revision=?", (identity, revision) + ).fetchall() + bundle = Bundle(json.loads(row[0]), dict(resources), json.loads(row[1])) + if bundle.revision != revision: + raise RevisionConflict("Stored bundle integrity mismatch") + return bundle + + def resource(self, identity: str, path: str, *, revision: str) -> bytes: + safe_path(path) + bundle = self.read(identity, revision=revision) + if path not in bundle.content: + raise ValueError(f"Unknown resource: {path}") + return bundle.content[path] + + def history(self, identity: str) -> list[dict]: + with closing(self._connect()) as connection: + rows = connection.execute( + "SELECT revision, parent, reason, evidence FROM revisions " + "WHERE id=? ORDER BY rowid", + (identity,), + ).fetchall() + return [ + { + "revision": revision, + "parent": parent, + "reason": reason, + "evidence": json.loads(evidence), + } + for revision, parent, reason, evidence in rows + ] diff --git a/src/inferrsi/knowledge/research.py b/src/inferrsi/knowledge/research.py new file mode 100644 index 000000000..fc0ce3dfc --- /dev/null +++ b/src/inferrsi/knowledge/research.py @@ -0,0 +1,127 @@ +"""Read-only model-aware discovery; target annotations are not runtime support.""" + +from __future__ import annotations + +import json +import math +from collections.abc import Sequence +from importlib.resources import files + +from .catalog import knowledge_records, search_knowledge + +PROCEDURE = ( + "data/skill_library/workflows/inference-engine-research/" + "references/model-aware-preparation.md" +) + + +def _profile(model: str) -> dict | None: + root = files("inferrsi").joinpath("data", "research_profiles") + for resource in sorted(root.iterdir(), key=lambda path: path.name): + if resource.name.endswith(".json"): + profile = json.loads(resource.read_text(encoding="utf-8")) + if model.casefold() in {name.casefold() for name in profile["aliases"]}: + return profile + return None + + +def research_knowledge( + model: str, + *, + frameworks: Sequence[str] = (), + tasks: Sequence[str] = (), + modalities: Sequence[str] = (), + mechanisms: Sequence[str] = (), + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, + target_backend: str | None = None, + target_device: str | None = None, + target_memory_gib: float | None = None, +) -> dict: + """Union separate research queries, preserving every package filter and record. + + Round-robin selection gives each query a turn before another result from the + same query. It is discovery ordering, not an engine or evidence-quality rank. + """ + if not model.strip(): + raise ValueError("model must be nonempty") + if limit < 1: + raise ValueError("limit must be positive") + if target_memory_gib is not None and ( + not math.isfinite(target_memory_gib) or target_memory_gib <= 0 + ): + raise ValueError("target memory must be positive and finite GiB") + model = model.strip() + profile = _profile(model) + defaults = profile or {} + groups = { + "model": [model, *([profile["model"]] if profile else [])], + "framework": frameworks, + "task": [*defaults.get("tasks", []), *tasks], + "modality": [*defaults.get("modalities", []), *modalities], + "mechanism": [*defaults.get("mechanisms", []), *mechanisms], + } + filters = {"repo": repo, "tier": tier, "backend": backend, "scenario": scenario} + queries, streams, matches = [], [], {} + # Retrieve every scoped match before applying the shared output limit; an + # early per-query limit would lose attribution for overlapping records. + catalog_size = max(1, len(knowledge_records())) + for group, terms in groups.items(): + seen = set() + for term in terms: + term = term.strip() + if not term: + raise ValueError(f"{group} query must be nonempty") + if term.casefold() in seen: + continue + seen.add(term.casefold()) + query = {"group": group, "query": term} + records = search_knowledge(term, **filters, limit=catalog_size) + queries.append({**query, "matches": len(records)}) + streams.append(records) + for record in records: + matches.setdefault(record["id"], []).append(query) + selected = {} + for rank in range(max((len(stream) for stream in streams), default=0)): + for stream in streams: + if rank < len(stream): + record = stream[rank] + selected.setdefault(record["id"], record) + if len(selected) == limit: + break + if len(selected) == limit: + break + return { + "model": model, + "profile": profile, + "queries": queries, + "filters": filters, + "items": [ + {"record": record, "matched_queries": matches[identity]} + for identity, record in selected.items() + ], + "no_match": not selected, + "target": { + "backend": target_backend, "device": target_device, "memory_gib": target_memory_gib, + }, + "target_validation": "unverified", + "components": defaults.get("components", []), + "procedure_resource": PROCEDURE, + "worked_example_resource": defaults.get("worked_example_resource"), + "warnings": [] if profile else [ + "No packaged model profile: supply --task/--modality and verify the full component " + "inventory from the selected checkpoint before judging feasibility." + ], + "limitations": [ + "Queries search installed package records only; empty results do not rule out " + "external sources. Existing token-AND search and scope filters are unchanged.", + "Mechanism queries are research leads, not claims of model/backend support. " + "Retain each record's applicability, source revision and verification state.", + "Target annotations do not inspect hardware or add a runtime backend. Full weight " + "inventory, engine/format compatibility, peak residency and quality need evidence " + "under the linked preparation procedure; no engine is selected or executed.", + ], + } diff --git a/src/inferrsi/knowledge/skill_catalog.py b/src/inferrsi/knowledge/skill_catalog.py new file mode 100644 index 000000000..ad3ad25a2 --- /dev/null +++ b/src/inferrsi/knowledge/skill_catalog.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path, PurePosixPath +from typing import BinaryIO + +try: + import yaml +except ImportError as exc: # The base install stays PyYAML-free by contract. + raise ValueError( + "Reading checkout skills requires the 'skills' extra (PyYAML); install it " + "from your InferRSI checkout, replacing the placeholder path: " + "python -m pip install '[skills]'" + ) from exc + +from .catalog import BACKENDS + + +def _frontmatter(stream: BinaryIO) -> dict: + if stream.readline().strip() != b"---": + raise ValueError("SKILL.md must start with YAML frontmatter") + lines = [] + for line in stream: + if line.strip() == b"---": + break + lines.append(line) + else: + raise ValueError("Unclosed SKILL.md frontmatter") + try: + record = yaml.safe_load(b"".join(lines).decode("utf-8")) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid skill YAML: {exc}") from exc + if not isinstance(record, dict): + raise ValueError("Skill frontmatter must be a mapping") + return record + + +def _skill_name(name: str) -> None: + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name): + raise ValueError(f"Invalid skill name: {name}") + + +def _directory(root: Path, name: str, relative: str) -> Path: + _skill_name(name) + path = PurePosixPath(relative) + if ( + not relative + or "\\" in relative + or ":" in relative + or path.is_absolute() + or any(part in ("", ".", "..") for part in relative.split("/")) + ): + raise ValueError(f"Skill path must be a relative POSIX path: {relative}") + directory = (root / relative).resolve() + if not directory.is_relative_to(root): + raise ValueError("Skill must be inside the selected root") + if path.name != name or directory.name != name: + raise ValueError(f"Skill path basename disagrees with its name: {name}") + if not _entry(directory, "SKILL.md").is_file(): + raise ValueError(f"Unknown skill: {name}") + return directory + + +def _unique_fields(pairs: list[tuple[str, object]]) -> dict: + record = {} + for key, value in pairs: + if key in record: + raise ValueError(f"Duplicate skill index field: {key}") + record[key] = value + return record + + +def _indexed_directories(root: Path) -> dict[str, Path] | None: + index = _entry(root, "skill-index.json") + if not index.exists(): + return None + record = json.loads(index.read_text(encoding="utf-8"), object_pairs_hook=_unique_fields) + if ( + not isinstance(record, dict) + or type(record.get("schema_version")) is not int + or record["schema_version"] != 1 + or not isinstance(record.get("skills"), dict) + ): + raise ValueError("Skill index requires schema_version 1 and a skills mapping") + directories = {} + paths = set() + for name, entry in sorted(record["skills"].items()): + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("path"), str) + or not isinstance(entry.get("backend"), str) + or entry["backend"] not in (*BACKENDS, "common") + ): + raise ValueError(f"Skill index requires a path and supported backend: {name}") + directory = _directory(root, name, entry["path"]) + if directory in paths: + raise ValueError(f"Duplicate skill index path: {entry['path']}") + directories[name] = directory + paths.add(directory) + return directories + + +def skill_directories(root: Path) -> dict[str, Path]: + """Resolve registered skill locations without reading instruction bodies.""" + root = root.resolve() + if not root.is_dir(): + raise ValueError(f"Skill root is not a directory: {root}") + indexed = _indexed_directories(root) + if indexed is not None: + return indexed + return { + path.parent.name: _directory(root, path.parent.name, path.parent.name) + for path in sorted(root.glob("*/SKILL.md")) + } + + +def skill_directory(root: Path, name: str) -> Path: + """Resolve one named skill in an indexed library or a legacy flat root.""" + _skill_name(name) + root = root.resolve() + indexed = _indexed_directories(root) + if indexed is None: + return _directory(root, name, name) + if name not in indexed: + raise ValueError(f"Unknown skill: {name}") + return indexed[name] + + +def _entry(directory: Path, filename: str) -> Path: + # The resolved entry file must stay inside the selected skill directory; + # reject symlinks that point outside it. + path = (directory / filename).resolve() + if not path.is_relative_to(directory): + raise ValueError(f"Skill {filename} must stay inside the selected skill: {directory.name}") + return path + + +def _sidecar(directory: Path) -> dict | None: + # metadata.json shares the directory-boundary loading path with SKILL.md. + path = _entry(directory, "metadata.json") + if not path.is_file(): + return None + metadata = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise ValueError(f"Skill metadata must be a mapping: {directory.name}") + return metadata + + +def _references(metadata: dict, name: str) -> list[str]: + # references must be a list of strings; a string value would degrade + # membership testing to substring matching. + references = metadata.get("references", []) + if not isinstance(references, list) or any(not isinstance(item, str) for item in references): + raise ValueError(f"Skill references must be a string list: {name}") + return references + + +def _strings(record: dict, key: str, name: str) -> None: + """A declared string list must be strings; [] is a deliberate empty value.""" + if key not in record: + return + listed = record[key] + if not isinstance(listed, list) or any( + not isinstance(item, str) or not item.strip() for item in listed + ): + raise ValueError(f"Skill {key} must be a list of non-empty strings: {name}") + + +def _applicability(record: dict, name: str) -> None: + """Conditions a procedure assumes. Descriptive only; never a validation claim.""" + if "applicability" not in record: + return + declared = record["applicability"] + if not isinstance(declared, dict): + raise ValueError(f"Skill applicability must be a mapping: {name}") + for key, value in declared.items(): + if isinstance(value, str): + if not value.strip(): + raise ValueError(f"Skill applicability.{key} is empty: {name}") + elif isinstance(value, list): + if not value or any(not isinstance(item, str) or not item.strip() for item in value): + raise ValueError( + f"Skill applicability.{key} must be a non-empty string list: {name}" + ) + else: + raise ValueError(f"Skill applicability.{key} must be a string or string list: {name}") + + +def _effective(directory: Path) -> dict: + # Read bytes line by line; parse only the frontmatter, never decode the body. + with _entry(directory, "SKILL.md").open("rb") as stream: + record = _frontmatter(stream) + for key in ("name", "description"): + if not isinstance(record.get(key), str) or not record[key].strip(): + raise ValueError(f"Skill requires {key}: {directory.name}") + if record["name"] != directory.name: + raise ValueError(f"Skill name disagrees with directory: {directory.name}") + metadata = _sidecar(directory) + if metadata is not None: + for key in record.keys() & metadata.keys(): + if record[key] != metadata[key]: + raise ValueError(f"Conflicting skill metadata field {key}: {directory.name}") + if metadata.get("profile") == "local-skill-v1": + if any(key in metadata for key in ("name", "description")): + raise ValueError("Local skill name and description belong in SKILL.md") + if set(record) != {"name", "description"}: + raise ValueError("Local skill structured fields belong in metadata.json") + # Only sidecars declare these fields, and JSON cannot express a container + # cycle; validating the sidecar leaves the bundle normalizer free to report + # a cyclic legacy frontmatter value against the field that contains it. + _strings(metadata, "recommendations", directory.name) + _applicability(metadata, directory.name) + if "searchspace_not_applicable" in metadata: + reason = metadata["searchspace_not_applicable"] + if not isinstance(reason, str) or not reason.strip(): + raise ValueError( + f"Skill searchspace_not_applicable requires a reason: {directory.name}" + ) + record.update(metadata) + _references(record, directory.name) + if "status" in record and not isinstance(record["status"], str): + raise ValueError(f"Skill status must be a string: {directory.name}") + if "tags" in record and ( + not isinstance(record["tags"], list) + or any(not isinstance(tag, str) for tag in record["tags"]) + ): + raise ValueError(f"Skill tags must be a string list: {directory.name}") + return record + + +def _metadata(directory: Path) -> dict: + record = _effective(directory) + return {key: record[key] for key in ("name", "description", "status", "tags") if key in record} + + +def skill_metadata(root: Path, name: str) -> dict: + """Effective legacy frontmatter plus sidecar fields, after conflict checks.""" + return _effective(skill_directory(root, name)) + + +def skill_records(root: Path) -> list[dict]: + """Project metadata only; this projection is not an authoritative matrix.""" + return [_metadata(directory) for directory in skill_directories(root).values()] + + +def get_skill(root: Path, name: str) -> dict: + """Read selected metadata and resource names, never resource contents.""" + directory = skill_directory(root, name) + record = _metadata(directory) + record["resources"] = [ + path.relative_to(directory).as_posix() + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != "SKILL.md" + ] + return record + + +def skill_instructions(root: Path, name: str) -> str: + """Load selected instructions. Reading does not authorize their commands.""" + directory = skill_directory(root, name) + _metadata(directory) + with _entry(directory, "SKILL.md").open("rb") as stream: + _frontmatter(stream) + return stream.read().decode("utf-8").lstrip("\r\n") + + +def skill_resource(root: Path, name: str, resource: str) -> Path: + """Resolve one explicitly selected resource inside its skill directory.""" + directory = skill_directory(root, name) + path = (directory / resource).resolve() + if not path.is_relative_to(directory) or path == directory / "SKILL.md": + raise ValueError("Resource must be inside the selected skill; use show for instructions") + if not path.is_file(): + raise ValueError(f"Unknown skill resource: {resource}") + return path + + +def skill_reference(root: Path, name: str, reference: str) -> Path: + """Resolve a declared checkout reference without escaping the skills root.""" + directory = skill_directory(root, name) + if reference not in _references(_effective(directory), directory.name): + raise ValueError(f"Undeclared skill reference: {reference}") + if "\\" in reference or Path(reference).is_absolute(): + raise ValueError("Reference must use a relative POSIX path") + path = (directory / reference).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("Reference must stay inside the skills root") + if not path.is_file(): + raise ValueError(f"Missing skill reference: {reference}") + return path diff --git a/src/inferrsi/knowledge_operations.py b/src/inferrsi/knowledge_operations.py index cff710c26..fab038591 100644 --- a/src/inferrsi/knowledge_operations.py +++ b/src/inferrsi/knowledge_operations.py @@ -1,735 +1,35 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge.knowledge_operations; +# patch implementation globals such as _json and _hash on that module. from __future__ import annotations -import hashlib -import json -import re -import sqlite3 -from dataclasses import dataclass -from datetime import date, datetime -from importlib.resources import files -from pathlib import Path, PurePosixPath - -from ._skill_selection import ( - RELEVANCE_METHODS, - TASK_RELEVANCE, - context_view, - indexed_skill, - record_context, - skill_score, - task_score, - words, +from inferrsi.knowledge.knowledge_operations import ( + AUTHORITIES, + IDENTITY, + INCIDENTAL, + Bundle, + IdentityError, + KnowledgeReader, + RevisionConflict, + _hash, + _json, + _normalized, + _sources, + _tree, + safe_path, ) -from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records - -AUTHORITIES = ("package", "checkout", "store") -IDENTITY = r"[a-z0-9][a-z0-9._-]*" - - -class RevisionConflict(ValueError): - pass - - -class IdentityError(ValueError): - def __init__(self, code: str, message: str): - super().__init__(message) - self.code = code - - -def safe_path(value: str) -> str: - path = PurePosixPath(value) - if ( - not value - or "\\" in value - or ":" in value - or path.is_absolute() - or any(part in ("", ".", "..") for part in value.split("/")) - ): - raise ValueError(f"Unsafe bundle path: {value}") - return value - - -def _json(value: object) -> bytes: - return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() - - -def _normalized(value: object, field: str, ancestors: frozenset[int] = frozenset()) -> object: - # Bundle identity is JSON, so YAML scalars without a JSON form are rejected - # here instead of failing inside revision serialization. - if value is None or isinstance(value, (str, bool, int, float)): - return value - if isinstance(value, (datetime, date)): - return value.isoformat() - if isinstance(value, (list, dict)): - # A YAML alias can make a container its own descendant. Only the active path - # is tracked, so the same object shared by sibling fields stays accepted. - identity = id(value) - if identity in ancestors: - raise ValueError( - f"Field {field} contains a reference cycle: {type(value).__name__}" - ) - ancestors = ancestors | {identity} - if isinstance(value, list): - return [ - _normalized(item, f"{field}[{index}]", ancestors) - for index, item in enumerate(value) - ] - result = {} - for key, item in value.items(): - if not isinstance(key, str): - raise ValueError(f"Field {field} has a non-string key: {key!r}") - result[key] = _normalized(item, f"{field}.{key}" if field else key, ancestors) - return result - raise ValueError( - f"Field {field or ''} cannot be represented in the bundle identity: " - f"{type(value).__name__}" - ) - - -def _hash(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _sources(records: list[dict]) -> list[dict]: - result = [] - for record in records: - source = dict(record) - kind = source.get("type") - if kind in ("source", "source_code", "technical_document") and "commit" in source: - source["original_type"] = kind - kind = "git" - if kind is None and "commit" in source: - kind = "git" - if kind not in ("git", "paper", "document", "internal", "unknown"): - # Legacy types keep their original label; nothing is inferred from them. - source["original_type"] = kind - kind = "unknown" - source["type"] = kind - if kind == "git": - commit = source.get("commit") - if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{7,40}", commit): - raise ValueError("Git source requires a real commit or explicit historical prefix") - source["revision_precision"] = "full" if len(commit) == 40 else "prefix" - elif kind == "paper": - if not source.get("id") or "version" not in source: - raise ValueError("Paper source requires ID and explicit version (null if unknown)") - if "commit" in source: - raise ValueError("Paper versions are not Git commits") - elif kind == "document": - if not source.get("url") or not any(k in source for k in ("version", "sha256")): - raise ValueError("Document source requires URL and version or snapshot identity") - elif kind == "internal": - if not source.get("artifact") or not any(k in source for k in ("issue", "run")): - raise ValueError("Internal source requires artifact and issue/run identity") - result.append(source) - return result - - -@dataclass(frozen=True) -class Bundle: - metadata: dict - content: dict[str, bytes] - dependencies: dict[str, str] - - def __post_init__(self) -> None: - # Normalize once at ingestion so manifest, revision and views agree. - object.__setattr__(self, "metadata", _normalized(self.metadata, "metadata")) - - def manifest(self) -> dict: - record = self.metadata - if not isinstance(record.get("id"), str) or not re.fullmatch(IDENTITY, record["id"]): - raise ValueError("Invalid bundle identity") - if record.get("kind") not in ("skill", "knowledge", "case"): - raise ValueError("Invalid bundle kind") - for field in ( - "sources", - "applicability", - "state", - "limitations", - "attribution", - "entrypoint", - ): - if field not in record: - raise ValueError(f"Missing normalized field: {field}") - for field in ("state", "entrypoint"): - if not isinstance(record[field], str) or not record[field]: - raise ValueError(f"Normalized {field} must be a nonempty string") - if not isinstance(record["limitations"], list): - raise ValueError("Normalized limitations must be a list") - if not isinstance(record["sources"], list) or not record["sources"]: - raise ValueError("Bundle requires sources, including explicit unknown provenance") - _sources(record["sources"]) - for path, data in self.content.items(): - safe_path(path) - if not isinstance(data, bytes): - raise ValueError("Bundle content must be bytes") - if record["entrypoint"] not in self.content: - raise ValueError("Missing entrypoint in complete bundle") - for required in record.get("required_resources", []): - # Present string keys already passed the content path validation. - if isinstance(required, str) and required in self.content: - continue - if safe_path(required) not in self.content: - raise ValueError(f"Missing required resource: {required}") - resources = {path: _hash(data) for path, data in sorted(self.content.items())} - for dependency, revision in self.dependencies.items(): - safe_path(dependency) - if not re.fullmatch(r"sha256:[0-9a-f]{64}", revision): - raise ValueError("Dependency requires a resolved content revision") - if dependency in self.content: - actual = "sha256:" + resources[dependency] - else: - members = { - path: digest - for path, digest in resources.items() - if path.startswith(dependency + "/") - } - if not members: - raise ValueError(f"Missing dependency: {dependency}") - actual = "sha256:" + _hash(_json(members)) - if actual != revision: - raise RevisionConflict(f"Dependency bytes disagree with revision: {dependency}") - return { - "metadata": record, - "resources": resources, - "dependencies": dict(sorted(self.dependencies.items())), - } - - @property - def revision(self) -> str: - return "sha256:" + _hash(_json(self.manifest())) - - def view(self, *, material: bool = False) -> dict: - manifest = self.manifest() - result = { - **self.metadata, - "revision": "sha256:" + _hash(_json(manifest)), - "resources": manifest["resources"], - "dependencies": manifest["dependencies"], - } - if material: - result["content"] = self.content[self.metadata["entrypoint"]].decode("utf-8") - return result - - -INCIDENTAL = ("__pycache__", ".DS_Store") - - -def _tree( - node, prefix: str, *, root: Path | None = None, ancestors: frozenset[Path] = frozenset() -) -> dict[str, bytes]: - if isinstance(node, Path): - # A junction or symlink resolves away from where it appears; every descended - # directory must still land inside the root that owns this walk. - directory = node.resolve() - if root is None: - root = directory - if directory in ancestors: - raise ValueError(f"Resource cycle inside the skills root: {prefix}") - ancestors = ancestors | {directory} - result = {} - for child in sorted(node.iterdir(), key=lambda p: p.name): - if child.name in INCIDENTAL or child.name.endswith(".pyc"): - continue - path = safe_path(f"{prefix}/{child.name}") - if isinstance(child, Path) and child.is_symlink(): - raise ValueError(f"Symlink resource is not supported: {path}") - if child.is_dir(): - if isinstance(child, Path) and root is not None and not child.resolve().is_relative_to( - root - ): - raise ValueError(f"Resource escapes the skills root: {path}") - result.update(_tree(child, path, root=root, ancestors=ancestors)) - else: - result[path] = child.read_bytes() - return result - - -class KnowledgeReader: - def __init__(self, *, skill_root: Path | None = None, store=None): - self.skill_root = skill_root.resolve() if skill_root is not None else None - if store is not None and store.skill_root != self.skill_root: - raise ValueError("Store must reserve identities from the same checkout") - self.store = store - self._records: dict[str, dict] | None = None - self._library: tuple[dict[str, bytes], str] | None = None - - def records(self) -> dict[str, dict]: - """Installed catalog records, read once per reader (one package snapshot).""" - if self._records is None: - self._records = {record["id"]: record for record in knowledge_records()} - return self._records - - def _skill_library(self) -> tuple[dict[str, bytes], str]: - if self._library is None: - root = files("inferrsi").joinpath("data", "skill_library") - library = _tree(root, "data/skill_library") - digest = _hash(_json({path: _hash(data) for path, data in sorted(library.items())})) - self._library = (library, "sha256:" + digest) - return self._library - - def owners(self) -> dict[str, list[str]]: - """Map each public ID to every authority that owns it.""" - result: dict[str, list[str]] = {} - for identity in self.records(): - result.setdefault(identity, []).append("package") - if self.skill_root is not None: - # Enumeration reads registered names only. Validating sidecar content - # here would abort the whole collection over one malformed, possibly - # unrequested, skill; Collect reports that per item through _read. - from .skill_catalog import skill_directories - - local = list(skill_directories(self.skill_root)) - local.extend(self._cases()) - if len(set(local)) != len(local): - raise IdentityError("collision", "Duplicate identity inside the checkout") - for identity in local: - result.setdefault(identity, []).append("checkout") - if self.store is not None: - for identity in self.store.identities(): - result.setdefault(identity, []).append("store") - return result - - def identities(self) -> list[str]: - return sorted(f"{a}:{i}" for i, authorities in self.owners().items() for a in authorities) - - def resolve(self, reference: str, owners: dict[str, list[str]] | None = None) -> str: - """Return the qualified `authority:id`; ambiguity is an error, never precedence.""" - authority, qualified, identity = reference.rpartition(":") - if not re.fullmatch(IDENTITY, identity) or (qualified and authority not in AUTHORITIES): - raise IdentityError("invalid_identity", f"Invalid identity: {reference}") - found = (self.owners() if owners is None else owners).get(identity, []) - if qualified: - if authority not in found: - raise IdentityError("unknown_identity", f"Unknown identity: {reference}") - return reference - if not found: - raise IdentityError("unknown_identity", f"Unknown identity: {identity}") - if len(found) > 1: - choices = ", ".join(f"{a}:{identity}" for a in found) - raise IdentityError( - "ambiguous_identity", f"Ambiguous identity {identity}; qualify one of: {choices}" - ) - return f"{found[0]}:{identity}" - - def _cases(self) -> dict[str, Path]: - result = {} - if self.skill_root is not None: - from .skill_catalog import skill_directories - - for directory in skill_directories(self.skill_root).values(): - for path in sorted(directory.glob("cases/*.json")): - if path.is_symlink() or not path.resolve().is_relative_to(self.skill_root): - raise ValueError("Case must stay inside the selected skills root") - record = json.loads(path.read_text(encoding="utf-8")) - if record.get("kind") == "case": - identity = record["id"] - if not isinstance(identity, str) or not re.fullmatch(IDENTITY, identity): - raise ValueError(f"Invalid case identity in {path.name}: {identity!r}") - if identity in result: - raise ValueError(f"Duplicate case identity: {identity}") - result[identity] = path - return result - - def read(self, reference: str, *, revision: str | None = None) -> Bundle: - return self._read(self.resolve(reference), revision) - - def _read(self, qualified: str, revision: str | None) -> Bundle: - authority, _, identity = qualified.partition(":") - if authority == "store": - return self.store.read(identity, revision=revision) - bundle = self._checkout(identity) if authority == "checkout" else self._knowledge(identity) - if revision is not None and bundle.revision != revision: - raise RevisionConflict( - f"Revision unavailable in this snapshot: {qualified} at {revision}" - ) - return bundle - - def _checkout(self, identity: str) -> Bundle: - cases = self._cases() - if identity in cases: - path = cases[identity] - record = json.loads(path.read_text(encoding="utf-8")) - for field in ("references", "dependencies"): - if field in record: - raise ValueError(f"Case {field} are not supported; declare required_resources") - declared = record.get("required_resources", []) - if not isinstance(declared, list) or any(not isinstance(p, str) for p in declared): - raise ValueError("Case required_resources must be a string list") - if len(set(declared)) != len(declared): - raise ValueError("Case required_resources must not repeat a path") - relative = safe_path(record["content"]) - required = [relative, "case.json", *(p for p in declared if p != relative)] - content = {"case.json": path.read_bytes()} - for resource in [relative, *required[2:]]: - safe_path(resource) - if resource == "case.json": - raise ValueError("Case resource collides with its case.json record") - resource_path = path.parent / resource - if resource_path.is_symlink() or not resource_path.resolve().is_relative_to( - self.skill_root - ): - raise ValueError(f"Case resource escapes the skills root: {resource}") - if not resource_path.is_file(): - raise ValueError(f"Missing case resource: {resource}") - content[resource] = resource_path.read_bytes() - return Bundle( - { - **record, - "entrypoint": relative, - "sources": _sources(record["sources"]), - "required_resources": required, - }, - content, - {}, - ) - return self._skill(identity) - - def _knowledge(self, identity: str) -> Bundle: - record = self.records().get(identity) - if record is None: - raise IdentityError("unknown_identity", f"Unknown knowledge ID: {identity}") - content = {"record.json": _json(record)} - dependencies = {} - if record.get("skill_documents"): - # The whole imported library is one explicit dependency so bilingual - # references and licenses stay a complete closure. - library, digest = self._skill_library() - content.update(library) - dependencies["data/skill_library"] = digest - documents = record["skill_documents"] - if not isinstance(documents, dict): - raise ValueError(f"skill_documents must map editions to paths: {identity}") - for target in documents.values(): - if not isinstance(target, str) or safe_path(target) not in library: - raise ValueError(f"Missing declared skill document: {target}") - return Bundle( - { - "id": identity, - "kind": "knowledge", - "entrypoint": "record.json", - "origin": record.get("origin", ["external"]), - "sources": _sources(record["sources"]), - "applicability": record.get("applicability", {}), - "state": record["status"], - "limitations": record.get("limitations", []) - + [ - "Imported guidance; reading does not validate or execute it.", - "Historical revisions require the matching installed snapshot " - "or a retained store.", - ], - "attribution": record["sources"], - "verification": record["verification"], - "tier": record.get("tier", "contribution"), - "repo": record["repo"], - "backend": record["backend"], - "scenario": record["scenario"], - "title": record["title"], - "skill_documents": record.get("skill_documents", {}), - "required_resources": sorted(content), - }, - content, - dependencies, - ) - - def _skill(self, identity: str) -> Bundle: - from .skill_catalog import get_skill, skill_directory, skill_metadata, skill_reference - - root = self.skill_root - record = get_skill(root, identity) - directory = skill_directory(root, identity) - prefix = directory.relative_to(root).as_posix() - content = _tree(directory, prefix, root=root) - metadata = skill_metadata(root, identity) - dependencies = {} - for reference in metadata.get("references", []): - path = skill_reference(root, identity, reference) - key = path.relative_to(root).as_posix() - content[key] = path.read_bytes() - dependencies[key] = "sha256:" + _hash(content[key]) - return Bundle( - { - "id": identity, - "kind": "skill", - "entrypoint": f"{prefix}/SKILL.md", - "title": record["description"], - "origin": metadata.get("origin", ["unknown"]), - "sources": _sources(metadata.get("sources", [])) - or [ - { - "type": "unknown", - "note": "Legacy provenance remains in procedure/readsets", - } - ], - "applicability": metadata.get("applicability", {}), - "state": record.get("status", "unspecified"), - "limitations": metadata.get("limitations", []) - + [ - "Checkout-only local skill; no automatic execution or promotion.", - "Only declared references are required; " - "other cross-skill links are navigation.", - "Historical revisions require a matching checkout or a retained store.", - ], - "attribution": metadata.get("sources", []), - "empirical_validation": metadata.get("empirical_validation", "unknown"), - "required_resources": sorted(content), - }, - content, - dependencies, - ) - - def resource(self, identity: str, path: str, *, revision: str) -> bytes: - safe_path(path) - bundle = self.read(identity, revision=revision) - if path not in bundle.content: - raise ValueError(f"Unknown resource: {path}") - return bundle.content[path] - - def collect( - self, - query: str = "", - *, - ids: list[str] | None = None, - revisions: dict[str, str] | None = None, - kind: str | None = None, - repo: str | None = None, - tier: str | None = None, - backend: str | None = None, - scenario: str | None = None, - limit: int = 10, - max_bytes: int | None = None, - material: bool = False, - relevance: str | None = None, - relevance_method: str = "weighted-lexical-v1", - context_bytes: int | None = None, - ) -> dict: - if type(limit) is not int or limit < 1: - raise ValueError("limit must be a positive integer") - if max_bytes is not None and (type(max_bytes) is not int or max_bytes < 1): - raise ValueError("max_bytes must be a positive integer") - if ids is not None and ( - not isinstance(ids, list) or any(not isinstance(identity, str) for identity in ids) - ): - raise ValueError("ids must be a list of identity strings") - if kind not in (None, "skill", "knowledge", "case"): - raise ValueError("Unknown content kind") - if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): - raise ValueError("relevance requires at least one word") - if relevance_method not in RELEVANCE_METHODS: - raise ValueError("Unknown relevance method") - if relevance is None and relevance_method != "weighted-lexical-v1": - raise ValueError("relevance_method requires relevance") - if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): - raise ValueError("context_bytes must be a positive integer") - if revisions is None: - revisions = {} - if not isinstance(revisions, dict) or any( - not isinstance(key, str) or not isinstance(value, str) - for key, value in revisions.items() - ): - raise ValueError("revisions must map identity strings to revision strings") - if ids is not None and len(set(ids)) != len(ids): - raise ValueError("Duplicate requested identity") - owners = self.owners() - requested = ids if ids is not None else self.identities() - - def qualified(reference: str) -> str | None: - # Revision keys may use any spelling the API accepts for an identity. - try: - return self.resolve(reference, owners) - except IdentityError: - return None - - selected = {identity for identity in map(qualified, requested) if identity is not None} - pinned: dict[str, str] = {} - for reference, revision in revisions.items(): - target = qualified(reference) - if target is None or target not in selected: - raise ValueError( - f"Revision requested for an unselected identity: {reference}" - ) - if pinned.setdefault(target, revision) != revision: - raise ValueError(f"Conflicting revisions for the same identity: {reference}") - pinned[target] = revision - collisions = sorted(i for i, authorities in owners.items() if len(authorities) > 1) - if tier is not None and tier not in KNOWLEDGE_SCOPES: - raise ValueError(f"Unknown knowledge tier: {tier}") - scenario = SCENARIO_ALIASES.get(scenario, scenario) - package_filters = any(value is not None for value in (repo, tier, backend, scenario)) - task_ranking = relevance_method == TASK_RELEVANCE - - def matches(metadata: dict, authority: str) -> bool: - declared = metadata.get("scenario") - declared = declared if isinstance(declared, str) else "" - # The catalog records skills.discovery as the discovery operation, - # not the workload served by a procedure. Let the task cue rank - # these procedures alongside records declaring the requested workload. - discovery_skill = ( - task_ranking and ids is None and authority == "package" - and declared == "skills.discovery" - and self.records()[metadata["id"]].get("index_type") == "packaged-skill" - ) - return not ( - (kind and metadata.get("kind") != kind) - or (package_filters and authority != "package") - or (repo and metadata.get("repo") != repo) - or (tier and metadata.get("tier", "contribution") != tier) - or (backend and metadata.get("backend") not in (backend, "common")) - or not ( - scenario is None - or declared == scenario - or (declared.endswith(".*") and scenario.startswith(declared[:-1])) - or discovery_skill - ) - ) - - cases = self._cases() if kind else {} - selected, omitted, seen = [], [], set() - used = 0 - ranked, contexts, context_omissions = [], [], [] - context_used = 0 - - def select(reference: str, bundle: Bundle, view: dict) -> None: - nonlocal used, context_used - # Normal Bundle views were serialized while computing their revision. - # Preserve custom subclasses' serialization/error behavior below. - if type(bundle) is Bundle and len(selected) >= limit: - omitted.append({"id": reference, "reason": "output_limit"}) - return - size = len(_json(view)) - if len(selected) >= limit or (max_bytes is not None and used + size > max_bytes): - omitted.append({"id": reference, "reason": "output_limit"}) - return - selected.append(view) - used += size - if context_bytes is None: - return - record = self.records().get(view["id"], {}) - is_skill = record.get("index_type") == "packaged-skill" - if view["authority"] != "package" or (not is_skill and not task_ranking): - context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) - return - try: - available = context_bytes - context_used - if is_skill: - context, omission = context_view( - record, view, bundle.content, relevance or query, available - ) - else: - context, omission = record_context(record, view, bundle.content, available) - except (ValueError, KeyError, TypeError) as exc: - context_omissions.append({ - "ref": view["ref"], "reason": "index_unavailable", "detail": str(exc), - }) - return - if omission is not None: - context_omissions.append(omission) - else: - contexts.append(context) - context_used += context["content_bytes"] - for reference in requested: - try: - qualified = self.resolve(reference, owners) - if qualified in seen: - raise IdentityError("duplicate_identity", f"Already selected: {qualified}") - seen.add(qualified) - authority, _, identity = qualified.partition(":") - # Cheap metadata checks first, so filtered items never load resource bytes. - if authority == "package": - early = {**self.records()[identity], "kind": "knowledge"} - elif authority == "checkout": - early = {"kind": "case" if identity in cases else "skill"} - else: - early = None - if relevance is not None and ids is None and ( - authority != "package" - or (not task_ranking and early.get("index_type") != "packaged-skill") - ): - continue - if early is not None and not matches(early, authority): - if ids is not None: - omitted.append({"id": reference, "reason": "filters"}) - continue - bundle = self._read(qualified, pinned.get(qualified)) - view = {**bundle.view(material=material), "authority": authority, "ref": qualified} - entry = bundle.content[bundle.metadata["entrypoint"]] - haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() - except IdentityError as exc: - omitted.append({"id": reference, "reason": exc.code, "detail": str(exc)}) - continue - except RevisionConflict as exc: - omitted.append( - {"id": reference, "reason": "unavailable_revision", "detail": str(exc)} - ) - continue - except (ValueError, KeyError, TypeError, AttributeError, OSError, sqlite3.Error) as exc: - omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) - continue - if not matches(view, authority): - if ids is not None: - omitted.append({"id": reference, "reason": "filters"}) - continue - if any(token not in haystack for token in query.casefold().split()): - if ids is not None: - omitted.append({"id": reference, "reason": "query"}) - continue - if relevance is not None and ids is None: - record = self.records().get(identity, {}) - try: - is_skill = record.get("index_type") == "packaged-skill" - if is_skill: - core, _, _ = indexed_skill(record, view, bundle.content) - body = bundle.content[core] - else: - body = entry - if task_ranking: - score = task_score(relevance, record, body, backend) - else: - score = skill_score(relevance, record, body, method=relevance_method) - except (ValueError, KeyError, TypeError) as exc: - omitted.append({ - "id": reference, "reason": "index_unavailable", "detail": str(exc), - }) - continue - if score: - ranked.append((score, reference, bundle, view)) - continue - select(reference, bundle, view) - for _, reference, bundle, view in sorted( - ranked, key=lambda item: (-item[0], item[3]["ref"]) - ): - select(reference, bundle, view) - result = { - "items": selected, - "omissions": omitted, - "no_match": not selected, - "collisions": collisions, - "material_bytes": used, - "limitations": [ - "No crawling, ranking model or conversation state; deterministic identity order.", - "max_bytes bounds item payloads, not omission/control metadata.", - "Read references must be resolved with their returned exact revision.", - ], - } - if relevance is not None: - result["selection"] = { - "method": relevance_method, "cue": relevance, - "scope": ("package-skills-and-records" if task_ranking - else "generated-packaged-skills"), - "ranking_applied": ids is None, - } - if task_ranking and ids is None: - result["selection"].update(matched_count=len(ranked), returned_count=len(selected)) - result["limitations"][0] = ( - "Deterministic lexical selection; explicit IDs retain caller order and membership." - ) - if context_bytes is not None: - result.update({ - "contexts": contexts, "context_omissions": context_omissions, - "context_bytes": context_used, - "context_budget": { - "max_bytes": context_bytes, "accounting": "utf8-resource-content", - "excludes": "provenance, omission and control metadata", - }, - }) - return result +__all__ = [ + "AUTHORITIES", + "IDENTITY", + "RevisionConflict", + "IdentityError", + "safe_path", + "Bundle", + "KnowledgeReader", + "INCIDENTAL", + "_json", + "_hash", + "_normalized", + "_sources", + "_tree", +] diff --git a/src/inferrsi/knowledge_store.py b/src/inferrsi/knowledge_store.py index 4cb585703..9c699ca48 100644 --- a/src/inferrsi/knowledge_store.py +++ b/src/inferrsi/knowledge_store.py @@ -1,297 +1,6 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge.knowledge_store. from __future__ import annotations -import json -import re -import sqlite3 -from contextlib import closing -from importlib.resources import files -from pathlib import Path +from inferrsi.knowledge.knowledge_store import KnowledgeStore, _protected -from .knowledge_operations import ( - AUTHORITIES, - IDENTITY, - Bundle, - IdentityError, - KnowledgeReader, - RevisionConflict, - _json, - safe_path, -) - - -def _protected(skill_root: Path | None) -> list[Path]: - package = Path(str(files("inferrsi"))).resolve() - roots = [package] - # A source, editable or exported tree (git archive, sdist) surrounds the package - # and holds evidence and both matrices; protect it even without a .git entry, - # because review snapshots carry no repository metadata. - for parent in package.parents: - if (parent / "pyproject.toml").is_file() or (parent / ".git").exists(): - roots.append(parent) - break - if skill_root is not None: - roots.append(Path(skill_root).resolve().parent) - return roots - - -class KnowledgeStore: - """One authorized writable store for store-owned candidate bundles. - - Package and checkout content stay read-only; their IDs are reserved here so a - store candidate cannot shadow them. Repository-owned amendments use PRs. - """ - - def __init__(self, workspace: Path, *, skill_root: Path | None, read_only: bool = False): - # The caller authorizes an existing dedicated directory; none is created or guessed. - workspace = Path(workspace) - if workspace.is_symlink() or not workspace.is_dir(): - raise ValueError("Store requires an existing authorized workspace directory") - self.workspace = workspace.resolve() - for root in _protected(skill_root): - if self.workspace.is_relative_to(root): - raise ValueError(f"Store workspace must be outside read-only sources: {root}") - self.skill_root = Path(skill_root).resolve() if skill_root is not None else None - self.adapters = KnowledgeReader(skill_root=self.skill_root) - self.read_only = read_only - self.database = self.workspace / "knowledge.sqlite3" - if self.database.is_symlink(): - raise ValueError("Store database cannot be a symlink") - if read_only: - # Reading an existing store must not create or migrate anything. - if not self.database.is_file(): - raise ValueError(f"Read-only store requires an existing database: {self.database}") - self._verify_reserved_skill_root() - return - with closing(self._connect()) as connection, connection: - connection.executescript( - "CREATE TABLE IF NOT EXISTS revisions (" - "id TEXT NOT NULL, revision TEXT NOT NULL, metadata TEXT NOT NULL, " - "dependencies TEXT NOT NULL, parent TEXT, reason TEXT NOT NULL, " - "evidence TEXT NOT NULL, PRIMARY KEY (id, revision));" - "CREATE TABLE IF NOT EXISTS resources (" - "id TEXT NOT NULL, revision TEXT NOT NULL, path TEXT NOT NULL, data BLOB NOT NULL, " - "PRIMARY KEY (id, revision, path), " - "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" - "CREATE TABLE IF NOT EXISTS current (id TEXT PRIMARY KEY, revision TEXT NOT NULL, " - "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" - "CREATE TABLE IF NOT EXISTS meta (" - "key TEXT PRIMARY KEY, value TEXT NOT NULL);" - ) - reserved = connection.execute( - "SELECT value FROM meta WHERE key='skill_root'" - ).fetchone() - if reserved is None: - connection.execute( - "INSERT INTO meta VALUES ('skill_root', ?)", (self._reserved_skill_root(),) - ) - elif reserved[0] != self._reserved_skill_root(): - raise ValueError( - f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " - "reopen it with the same skill_root" - ) - - def _reserved_skill_root(self) -> str: - # The reserved checkout is recorded so later opens cannot silently drop it; - # an empty value records a package-only store. - return str(self.skill_root) if self.skill_root is not None else "" - - def _verify_reserved_skill_root(self) -> None: - # Stores created before reservation recording have no meta row and stay usable. - with closing(self._connect()) as connection: - present = connection.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'" - ).fetchone() - reserved = ( - connection.execute("SELECT value FROM meta WHERE key='skill_root'").fetchone() - if present - else None - ) - if reserved is not None and reserved[0] != self._reserved_skill_root(): - raise ValueError( - f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " - "open it with the same --skill-root, or without one for a package-only store" - ) - - def _connect(self): - if self.database.is_symlink() or self.database.resolve().parent != self.workspace: - raise ValueError("Store database escapes the authorized workspace") - if self.read_only: - # mode=ro fails every write at the SQLite level; as_uri() percent-encodes - # spaces and non-ASCII so the file: URI stays valid on Windows too. - connection = sqlite3.connect(self.database.as_uri() + "?mode=ro", uri=True, timeout=5) - else: - connection = sqlite3.connect(self.database, timeout=5) - connection.execute("PRAGMA foreign_keys=ON") - return connection - - def identities(self) -> list[str]: - with closing(self._connect()) as connection: - return [row[0] for row in connection.execute("SELECT id FROM current ORDER BY id")] - - def reader(self) -> KnowledgeReader: - return KnowledgeReader(skill_root=self.skill_root, store=self) - - def collect(self, query: str = "", **options) -> dict: - return self.reader().collect(query, **options) - - def create(self, bundle: Bundle) -> str: - if self.read_only: - raise ValueError("Store is read-only; create requires a writable store") - return self._publish(bundle, expected=None, reason="create", evidence=[]) - - def update(self, bundle: Bundle, *, expected: str, reason: str, evidence: list[dict]) -> str: - if self.read_only: - raise ValueError("Store is read-only; update requires a writable store") - if not expected or not reason.strip(): - raise ValueError("Update requires expected old revision and reason") - if not isinstance(evidence, list): - raise ValueError("Update evidence must be a list; an empty list means none supplied") - return self._publish(bundle, expected=expected, reason=reason, evidence=evidence) - - def _lineage(self, metadata: dict) -> None: - lineage = metadata.get("derived_from", []) - if not isinstance(lineage, list): - raise ValueError("derived_from must be a list of qualified source revisions") - for source in lineage: - if not isinstance(source, dict) or not isinstance(source.get("ref"), str): - raise ValueError("derived_from entries require a qualified ref") - authority, _, identity = source["ref"].partition(":") - if authority not in AUTHORITIES or not re.fullmatch(IDENTITY, identity): - raise ValueError(f"derived_from ref must be :: {source['ref']}") - revision = source.get("revision") - if revision is not None and ( - not isinstance(revision, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", revision) - ): - raise ValueError("derived_from revision must be a content revision or null") - if identity == metadata["id"]: - raise IdentityError( - "collision", "An adaptation needs a distinct identity from its source" - ) - - def _publish(self, bundle: Bundle, *, expected: str | None, reason: str, evidence: list) -> str: - # Copy caller data, validate it, then store the whole bundle and current - # pointer in one transaction. - bundle = Bundle( - json.loads(_json(bundle.metadata)), dict(bundle.content), dict(bundle.dependencies) - ) - revision = bundle.revision - identity = bundle.metadata["id"] - try: - bundle.content[bundle.metadata["entrypoint"]].decode("utf-8") - except UnicodeDecodeError as exc: - raise ValueError("Bundle entrypoint must be UTF-8 text") from exc - self._lineage(bundle.metadata) - owners = self.adapters.owners().get(identity) - if owners: - raise IdentityError( - "collision", - f"Identity {identity} is owned by read-only {', '.join(owners)} content; " - "amend it through a repository PR or create an adaptation with a distinct ID", - ) - if expected is None and bundle.metadata["state"] != "candidate": - raise ValueError( - "Create registers a candidate; imported or active content is not promoted" - ) - try: - with closing(self._connect()) as connection, connection: - connection.execute("BEGIN IMMEDIATE") - current = connection.execute( - "SELECT revision FROM current WHERE id=?", (identity,) - ).fetchone() - if expected is None: - if current is not None: - raise RevisionConflict(f"Duplicate identity: {identity}") - elif current is None or current[0] != expected: - raise RevisionConflict(f"Stale or missing revision for {identity}") - if expected is not None: - previous = connection.execute( - "SELECT metadata FROM revisions WHERE id=? AND revision=?", - (identity, expected), - ).fetchone() - old = json.loads(previous[0]) - if bundle.metadata["state"] != old["state"]: - raise ValueError("Update cannot promote content state") - if revision == expected: - raise ValueError("Update must change the bundle") - if connection.execute( - "SELECT 1 FROM revisions WHERE id=? AND revision=?", (identity, revision) - ).fetchone(): - raise RevisionConflict( - f"Update reproduces retained revision {revision}; " - "change the bundle so the successor has its own revision" - ) - connection.execute( - "INSERT INTO revisions VALUES (?, ?, ?, ?, ?, ?, ?)", - ( - identity, - revision, - _json(bundle.metadata).decode(), - _json(bundle.dependencies).decode(), - expected, - reason, - _json(evidence).decode(), - ), - ) - connection.executemany( - "INSERT INTO resources VALUES (?, ?, ?, ?)", - [(identity, revision, path, data) for path, data in bundle.content.items()], - ) - connection.execute( - "INSERT INTO current VALUES (?, ?) " - "ON CONFLICT(id) DO UPDATE SET revision=excluded.revision", - (identity, revision), - ) - except sqlite3.OperationalError as exc: - if "locked" not in str(exc) and "busy" not in str(exc): - raise - raise RevisionConflict( - f"Store is locked by another writer; retry {identity} once it is released" - ) from exc - return revision - - def read(self, identity: str, *, revision: str | None = None) -> Bundle: - with closing(self._connect()) as connection: - if revision is None: - row = connection.execute( - "SELECT revision FROM current WHERE id=?", (identity,) - ).fetchone() - if row is None: - raise IdentityError("unknown_identity", f"Unknown identity: store:{identity}") - revision = row[0] - row = connection.execute( - "SELECT metadata, dependencies FROM revisions WHERE id=? AND revision=?", - (identity, revision), - ).fetchone() - if row is None: - raise RevisionConflict(f"Unknown revision: store:{identity} at {revision}") - resources = connection.execute( - "SELECT path, data FROM resources WHERE id=? AND revision=?", (identity, revision) - ).fetchall() - bundle = Bundle(json.loads(row[0]), dict(resources), json.loads(row[1])) - if bundle.revision != revision: - raise RevisionConflict("Stored bundle integrity mismatch") - return bundle - - def resource(self, identity: str, path: str, *, revision: str) -> bytes: - safe_path(path) - bundle = self.read(identity, revision=revision) - if path not in bundle.content: - raise ValueError(f"Unknown resource: {path}") - return bundle.content[path] - - def history(self, identity: str) -> list[dict]: - with closing(self._connect()) as connection: - rows = connection.execute( - "SELECT revision, parent, reason, evidence FROM revisions " - "WHERE id=? ORDER BY rowid", - (identity,), - ).fetchall() - return [ - { - "revision": revision, - "parent": parent, - "reason": reason, - "evidence": json.loads(evidence), - } - for revision, parent, reason, evidence in rows - ] +__all__ = ["KnowledgeStore", "_protected"] diff --git a/src/inferrsi/research.py b/src/inferrsi/research.py index fc0ce3dfc..45c0852c5 100644 --- a/src/inferrsi/research.py +++ b/src/inferrsi/research.py @@ -1,127 +1,6 @@ -"""Read-only model-aware discovery; target annotations are not runtime support.""" - +# Compatibility shell. The implementation lives in inferrsi.knowledge.research. from __future__ import annotations -import json -import math -from collections.abc import Sequence -from importlib.resources import files - -from .catalog import knowledge_records, search_knowledge - -PROCEDURE = ( - "data/skill_library/workflows/inference-engine-research/" - "references/model-aware-preparation.md" -) - - -def _profile(model: str) -> dict | None: - root = files("inferrsi").joinpath("data", "research_profiles") - for resource in sorted(root.iterdir(), key=lambda path: path.name): - if resource.name.endswith(".json"): - profile = json.loads(resource.read_text(encoding="utf-8")) - if model.casefold() in {name.casefold() for name in profile["aliases"]}: - return profile - return None - - -def research_knowledge( - model: str, - *, - frameworks: Sequence[str] = (), - tasks: Sequence[str] = (), - modalities: Sequence[str] = (), - mechanisms: Sequence[str] = (), - repo: str | None = None, - tier: str | None = None, - backend: str | None = None, - scenario: str | None = None, - limit: int = 10, - target_backend: str | None = None, - target_device: str | None = None, - target_memory_gib: float | None = None, -) -> dict: - """Union separate research queries, preserving every package filter and record. +from inferrsi.knowledge.research import PROCEDURE, _profile, research_knowledge - Round-robin selection gives each query a turn before another result from the - same query. It is discovery ordering, not an engine or evidence-quality rank. - """ - if not model.strip(): - raise ValueError("model must be nonempty") - if limit < 1: - raise ValueError("limit must be positive") - if target_memory_gib is not None and ( - not math.isfinite(target_memory_gib) or target_memory_gib <= 0 - ): - raise ValueError("target memory must be positive and finite GiB") - model = model.strip() - profile = _profile(model) - defaults = profile or {} - groups = { - "model": [model, *([profile["model"]] if profile else [])], - "framework": frameworks, - "task": [*defaults.get("tasks", []), *tasks], - "modality": [*defaults.get("modalities", []), *modalities], - "mechanism": [*defaults.get("mechanisms", []), *mechanisms], - } - filters = {"repo": repo, "tier": tier, "backend": backend, "scenario": scenario} - queries, streams, matches = [], [], {} - # Retrieve every scoped match before applying the shared output limit; an - # early per-query limit would lose attribution for overlapping records. - catalog_size = max(1, len(knowledge_records())) - for group, terms in groups.items(): - seen = set() - for term in terms: - term = term.strip() - if not term: - raise ValueError(f"{group} query must be nonempty") - if term.casefold() in seen: - continue - seen.add(term.casefold()) - query = {"group": group, "query": term} - records = search_knowledge(term, **filters, limit=catalog_size) - queries.append({**query, "matches": len(records)}) - streams.append(records) - for record in records: - matches.setdefault(record["id"], []).append(query) - selected = {} - for rank in range(max((len(stream) for stream in streams), default=0)): - for stream in streams: - if rank < len(stream): - record = stream[rank] - selected.setdefault(record["id"], record) - if len(selected) == limit: - break - if len(selected) == limit: - break - return { - "model": model, - "profile": profile, - "queries": queries, - "filters": filters, - "items": [ - {"record": record, "matched_queries": matches[identity]} - for identity, record in selected.items() - ], - "no_match": not selected, - "target": { - "backend": target_backend, "device": target_device, "memory_gib": target_memory_gib, - }, - "target_validation": "unverified", - "components": defaults.get("components", []), - "procedure_resource": PROCEDURE, - "worked_example_resource": defaults.get("worked_example_resource"), - "warnings": [] if profile else [ - "No packaged model profile: supply --task/--modality and verify the full component " - "inventory from the selected checkpoint before judging feasibility." - ], - "limitations": [ - "Queries search installed package records only; empty results do not rule out " - "external sources. Existing token-AND search and scope filters are unchanged.", - "Mechanism queries are research leads, not claims of model/backend support. " - "Retain each record's applicability, source revision and verification state.", - "Target annotations do not inspect hardware or add a runtime backend. Full weight " - "inventory, engine/format compatibility, peak residency and quality need evidence " - "under the linked preparation procedure; no engine is selected or executed.", - ], - } +__all__ = ["research_knowledge", "_profile", "PROCEDURE"] diff --git a/src/inferrsi/skill_catalog.py b/src/inferrsi/skill_catalog.py index ad3ad25a2..ac38ab091 100644 --- a/src/inferrsi/skill_catalog.py +++ b/src/inferrsi/skill_catalog.py @@ -1,291 +1,49 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge.skill_catalog; +# importing it still requires the optional PyYAML 'skills' extra. from __future__ import annotations -import json -import re -from pathlib import Path, PurePosixPath -from typing import BinaryIO - -try: - import yaml -except ImportError as exc: # The base install stays PyYAML-free by contract. - raise ValueError( - "Reading checkout skills requires the 'skills' extra (PyYAML); install it " - "from your InferRSI checkout, replacing the placeholder path: " - "python -m pip install '[skills]'" - ) from exc - -from .catalog import BACKENDS - - -def _frontmatter(stream: BinaryIO) -> dict: - if stream.readline().strip() != b"---": - raise ValueError("SKILL.md must start with YAML frontmatter") - lines = [] - for line in stream: - if line.strip() == b"---": - break - lines.append(line) - else: - raise ValueError("Unclosed SKILL.md frontmatter") - try: - record = yaml.safe_load(b"".join(lines).decode("utf-8")) - except yaml.YAMLError as exc: - raise ValueError(f"Invalid skill YAML: {exc}") from exc - if not isinstance(record, dict): - raise ValueError("Skill frontmatter must be a mapping") - return record - - -def _skill_name(name: str) -> None: - if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name): - raise ValueError(f"Invalid skill name: {name}") - - -def _directory(root: Path, name: str, relative: str) -> Path: - _skill_name(name) - path = PurePosixPath(relative) - if ( - not relative - or "\\" in relative - or ":" in relative - or path.is_absolute() - or any(part in ("", ".", "..") for part in relative.split("/")) - ): - raise ValueError(f"Skill path must be a relative POSIX path: {relative}") - directory = (root / relative).resolve() - if not directory.is_relative_to(root): - raise ValueError("Skill must be inside the selected root") - if path.name != name or directory.name != name: - raise ValueError(f"Skill path basename disagrees with its name: {name}") - if not _entry(directory, "SKILL.md").is_file(): - raise ValueError(f"Unknown skill: {name}") - return directory - - -def _unique_fields(pairs: list[tuple[str, object]]) -> dict: - record = {} - for key, value in pairs: - if key in record: - raise ValueError(f"Duplicate skill index field: {key}") - record[key] = value - return record - - -def _indexed_directories(root: Path) -> dict[str, Path] | None: - index = _entry(root, "skill-index.json") - if not index.exists(): - return None - record = json.loads(index.read_text(encoding="utf-8"), object_pairs_hook=_unique_fields) - if ( - not isinstance(record, dict) - or type(record.get("schema_version")) is not int - or record["schema_version"] != 1 - or not isinstance(record.get("skills"), dict) - ): - raise ValueError("Skill index requires schema_version 1 and a skills mapping") - directories = {} - paths = set() - for name, entry in sorted(record["skills"].items()): - if ( - not isinstance(entry, dict) - or not isinstance(entry.get("path"), str) - or not isinstance(entry.get("backend"), str) - or entry["backend"] not in (*BACKENDS, "common") - ): - raise ValueError(f"Skill index requires a path and supported backend: {name}") - directory = _directory(root, name, entry["path"]) - if directory in paths: - raise ValueError(f"Duplicate skill index path: {entry['path']}") - directories[name] = directory - paths.add(directory) - return directories - - -def skill_directories(root: Path) -> dict[str, Path]: - """Resolve registered skill locations without reading instruction bodies.""" - root = root.resolve() - if not root.is_dir(): - raise ValueError(f"Skill root is not a directory: {root}") - indexed = _indexed_directories(root) - if indexed is not None: - return indexed - return { - path.parent.name: _directory(root, path.parent.name, path.parent.name) - for path in sorted(root.glob("*/SKILL.md")) - } - - -def skill_directory(root: Path, name: str) -> Path: - """Resolve one named skill in an indexed library or a legacy flat root.""" - _skill_name(name) - root = root.resolve() - indexed = _indexed_directories(root) - if indexed is None: - return _directory(root, name, name) - if name not in indexed: - raise ValueError(f"Unknown skill: {name}") - return indexed[name] - - -def _entry(directory: Path, filename: str) -> Path: - # The resolved entry file must stay inside the selected skill directory; - # reject symlinks that point outside it. - path = (directory / filename).resolve() - if not path.is_relative_to(directory): - raise ValueError(f"Skill {filename} must stay inside the selected skill: {directory.name}") - return path - - -def _sidecar(directory: Path) -> dict | None: - # metadata.json shares the directory-boundary loading path with SKILL.md. - path = _entry(directory, "metadata.json") - if not path.is_file(): - return None - metadata = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(metadata, dict): - raise ValueError(f"Skill metadata must be a mapping: {directory.name}") - return metadata - - -def _references(metadata: dict, name: str) -> list[str]: - # references must be a list of strings; a string value would degrade - # membership testing to substring matching. - references = metadata.get("references", []) - if not isinstance(references, list) or any(not isinstance(item, str) for item in references): - raise ValueError(f"Skill references must be a string list: {name}") - return references - - -def _strings(record: dict, key: str, name: str) -> None: - """A declared string list must be strings; [] is a deliberate empty value.""" - if key not in record: - return - listed = record[key] - if not isinstance(listed, list) or any( - not isinstance(item, str) or not item.strip() for item in listed - ): - raise ValueError(f"Skill {key} must be a list of non-empty strings: {name}") - - -def _applicability(record: dict, name: str) -> None: - """Conditions a procedure assumes. Descriptive only; never a validation claim.""" - if "applicability" not in record: - return - declared = record["applicability"] - if not isinstance(declared, dict): - raise ValueError(f"Skill applicability must be a mapping: {name}") - for key, value in declared.items(): - if isinstance(value, str): - if not value.strip(): - raise ValueError(f"Skill applicability.{key} is empty: {name}") - elif isinstance(value, list): - if not value or any(not isinstance(item, str) or not item.strip() for item in value): - raise ValueError( - f"Skill applicability.{key} must be a non-empty string list: {name}" - ) - else: - raise ValueError(f"Skill applicability.{key} must be a string or string list: {name}") - - -def _effective(directory: Path) -> dict: - # Read bytes line by line; parse only the frontmatter, never decode the body. - with _entry(directory, "SKILL.md").open("rb") as stream: - record = _frontmatter(stream) - for key in ("name", "description"): - if not isinstance(record.get(key), str) or not record[key].strip(): - raise ValueError(f"Skill requires {key}: {directory.name}") - if record["name"] != directory.name: - raise ValueError(f"Skill name disagrees with directory: {directory.name}") - metadata = _sidecar(directory) - if metadata is not None: - for key in record.keys() & metadata.keys(): - if record[key] != metadata[key]: - raise ValueError(f"Conflicting skill metadata field {key}: {directory.name}") - if metadata.get("profile") == "local-skill-v1": - if any(key in metadata for key in ("name", "description")): - raise ValueError("Local skill name and description belong in SKILL.md") - if set(record) != {"name", "description"}: - raise ValueError("Local skill structured fields belong in metadata.json") - # Only sidecars declare these fields, and JSON cannot express a container - # cycle; validating the sidecar leaves the bundle normalizer free to report - # a cyclic legacy frontmatter value against the field that contains it. - _strings(metadata, "recommendations", directory.name) - _applicability(metadata, directory.name) - if "searchspace_not_applicable" in metadata: - reason = metadata["searchspace_not_applicable"] - if not isinstance(reason, str) or not reason.strip(): - raise ValueError( - f"Skill searchspace_not_applicable requires a reason: {directory.name}" - ) - record.update(metadata) - _references(record, directory.name) - if "status" in record and not isinstance(record["status"], str): - raise ValueError(f"Skill status must be a string: {directory.name}") - if "tags" in record and ( - not isinstance(record["tags"], list) - or any(not isinstance(tag, str) for tag in record["tags"]) - ): - raise ValueError(f"Skill tags must be a string list: {directory.name}") - return record - - -def _metadata(directory: Path) -> dict: - record = _effective(directory) - return {key: record[key] for key in ("name", "description", "status", "tags") if key in record} - - -def skill_metadata(root: Path, name: str) -> dict: - """Effective legacy frontmatter plus sidecar fields, after conflict checks.""" - return _effective(skill_directory(root, name)) - - -def skill_records(root: Path) -> list[dict]: - """Project metadata only; this projection is not an authoritative matrix.""" - return [_metadata(directory) for directory in skill_directories(root).values()] - - -def get_skill(root: Path, name: str) -> dict: - """Read selected metadata and resource names, never resource contents.""" - directory = skill_directory(root, name) - record = _metadata(directory) - record["resources"] = [ - path.relative_to(directory).as_posix() - for path in sorted(directory.rglob("*")) - if path.is_file() and path.name != "SKILL.md" - ] - return record - - -def skill_instructions(root: Path, name: str) -> str: - """Load selected instructions. Reading does not authorize their commands.""" - directory = skill_directory(root, name) - _metadata(directory) - with _entry(directory, "SKILL.md").open("rb") as stream: - _frontmatter(stream) - return stream.read().decode("utf-8").lstrip("\r\n") - - -def skill_resource(root: Path, name: str, resource: str) -> Path: - """Resolve one explicitly selected resource inside its skill directory.""" - directory = skill_directory(root, name) - path = (directory / resource).resolve() - if not path.is_relative_to(directory) or path == directory / "SKILL.md": - raise ValueError("Resource must be inside the selected skill; use show for instructions") - if not path.is_file(): - raise ValueError(f"Unknown skill resource: {resource}") - return path - - -def skill_reference(root: Path, name: str, reference: str) -> Path: - """Resolve a declared checkout reference without escaping the skills root.""" - directory = skill_directory(root, name) - if reference not in _references(_effective(directory), directory.name): - raise ValueError(f"Undeclared skill reference: {reference}") - if "\\" in reference or Path(reference).is_absolute(): - raise ValueError("Reference must use a relative POSIX path") - path = (directory / reference).resolve() - if not path.is_relative_to(root.resolve()): - raise ValueError("Reference must stay inside the skills root") - if not path.is_file(): - raise ValueError(f"Missing skill reference: {reference}") - return path +from inferrsi.knowledge.skill_catalog import ( + _applicability, + _directory, + _effective, + _entry, + _frontmatter, + _indexed_directories, + _metadata, + _references, + _sidecar, + _skill_name, + _strings, + _unique_fields, + get_skill, + skill_directories, + skill_directory, + skill_instructions, + skill_metadata, + skill_records, + skill_reference, + skill_resource, +) + +__all__ = [ + "skill_directories", + "skill_directory", + "skill_metadata", + "skill_records", + "get_skill", + "skill_instructions", + "skill_resource", + "skill_reference", + "_frontmatter", + "_skill_name", + "_directory", + "_unique_fields", + "_indexed_directories", + "_entry", + "_sidecar", + "_references", + "_strings", + "_applicability", + "_effective", + "_metadata", +] diff --git a/tests/test_architecture_closed_loop.py b/tests/test_architecture_closed_loop.py index 98f3fe4ea..8ce862e5f 100644 --- a/tests/test_architecture_closed_loop.py +++ b/tests/test_architecture_closed_loop.py @@ -377,7 +377,8 @@ def test_real_ports_stay_on_three_existing_seams(self) -> None: self.assertIn("evaluate", calls) self.assertNotIn("search_knowledge", calls) self.assertNotIn("collect", calls) - knowledge = _import_modules(ROOT / "src/inferrsi/knowledge_operations.py") + # The root module is a static shell; check the implementation's imports. + knowledge = _import_modules(ROOT / "src/inferrsi/knowledge/knowledge_operations.py") harness = _import_modules(ROOT / "src/inferrsi/harness/scenarios/external.py") self.assertFalse(any("harness" in name or "evaluation" in name for name in knowledge)) self.assertFalse(any("knowledge" in name or "evaluation" in name for name in harness)) diff --git a/tests/test_bundle_manifest.py b/tests/test_bundle_manifest.py index 1b0b8d8dc..c3a6d2257 100644 --- a/tests/test_bundle_manifest.py +++ b/tests/test_bundle_manifest.py @@ -4,7 +4,7 @@ import pytest -from inferrsi import knowledge_operations +from inferrsi.knowledge import knowledge_operations from inferrsi.knowledge_operations import Bundle, RevisionConflict @@ -70,6 +70,8 @@ def counted(data): calls[data] += 1 return _hash(data) + # Patch implementation globals; the root module is a static re-export shell. + assert Bundle.manifest.__globals__ is vars(knowledge_operations) monkeypatch.setattr(knowledge_operations, "_hash", counted) for count in (1, 2): bundle.manifest() diff --git a/tests/test_cli_catalog.py b/tests/test_cli_catalog.py index 522f1be90..90eaa3093 100644 --- a/tests/test_cli_catalog.py +++ b/tests/test_cli_catalog.py @@ -17,6 +17,7 @@ from inferrsi.arguments import make_spec, parser from inferrsi.catalog import get_knowledge, knowledge_records, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.knowledge import catalog as knowledge_catalog PROJECT = Path(__file__).resolve().parents[1] SOURCE_REPOSITORIES = { @@ -186,7 +187,8 @@ def test_misfiled_or_overstated_record_cannot_leak_into_context( ) destination.parent.mkdir(parents=True) destination.write_text(json.dumps(record)) - monkeypatch.setattr(catalog, "files", lambda _: tmp_path) + # Patch implementation globals; the root module is a static re-export shell. + monkeypatch.setattr(knowledge_catalog, "files", lambda _: tmp_path) with pytest.raises(ValueError, match=error): search_knowledge(backend="cuda", scenario="diffusion.image_generation") diff --git a/tests/test_collect_limit_serialization.py b/tests/test_collect_limit_serialization.py index 00153e5f2..7f8ffd1d8 100644 --- a/tests/test_collect_limit_serialization.py +++ b/tests/test_collect_limit_serialization.py @@ -1,7 +1,7 @@ """Count saturation skips only redundant normal-view payload sizing.""" import pytest -from inferrsi import knowledge_operations as operations +from inferrsi.knowledge import knowledge_operations as operations from inferrsi.knowledge_operations import Bundle, KnowledgeReader @@ -44,6 +44,8 @@ def observe(value): seen.append(value["id"]) return original(value) + # Patch implementation globals; the root module is a static re-export shell. + assert KnowledgeReader.collect.__globals__ is vars(operations) monkeypatch.setattr(operations, "_json", observe) return seen diff --git a/tests/test_knowledge_compatibility_shells.py b/tests/test_knowledge_compatibility_shells.py new file mode 100644 index 000000000..354b43ae3 --- /dev/null +++ b/tests/test_knowledge_compatibility_shells.py @@ -0,0 +1,82 @@ +# K01: root Knowledge names are static re-exports of the moved implementations. + +from __future__ import annotations + +import hashlib +import importlib +import os +import subprocess +import sys + +import pytest + +from tests._gate_policy_support import REPO, ROOT, blocker_kinds, first_wave_report, load_manifest + +MANIFEST = load_manifest() +LEAF = MANIFEST["technical_leaf"] +LEAF_NAMES = MANIFEST["catalog_two_routes"]["neutral_leaf_export"]["symbols"] +MOVED = [item for item in MANIFEST["compatibility_pairs"] if "exports" in item] + + +@pytest.mark.parametrize("item", MOVED, ids=lambda item: item["importer"]) +def test_root_names_are_the_moved_objects(item): + # K01-1: identity carries signatures, defaults, exception types and Path behavior. + root = importlib.import_module(f"inferrsi.{item['importer']}") + moved = importlib.import_module(item["target"]) + names = [*item["exports"]["public"], *item["exports"]["private_used"]] + for name in names: + assert getattr(root, name) is getattr(moved, name), name + extra = LEAF_NAMES if item["importer"] == "catalog" else [] + # REPO_TYPES is listed on both catalog routes; __all__ names it once. + assert len(root.__all__) == len(set(root.__all__)) + assert set(root.__all__) == {*names, *extra} + + +def test_catalog_routes_share_one_repo_types_tuple(): + from inferrsi import _repo_pack_data, catalog + from inferrsi.knowledge import catalog as knowledge_catalog + + assert catalog.REPO_TYPES is _repo_pack_data.REPO_TYPES is knowledge_catalog.REPO_TYPES + assert catalog.repository_pack is _repo_pack_data.repository_pack + assert not hasattr(knowledge_catalog, "repository_pack") + assert knowledge_catalog.KNOWLEDGE_SCOPES["contribution"] == ( + *_repo_pack_data.REPO_TYPES, "vllm_ascend", "inferrsi" + ) + + +def test_data_leaf_holds_the_recorded_fragment_and_locked_hash(): + data = (ROOT / "_repo_pack_data.py").read_bytes() + assert hashlib.sha256(data).hexdigest() == LEAF["file_sha256"] + assert LEAF["original_fragment"]["source"] + "\n" in data.decode("utf-8") + + +def _fresh(script: str) -> None: + environment = {**os.environ, "PYTHONPATH": str(REPO / "src")} + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, env=environment, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_package_init_and_infra_inspection_load_no_knowledge_body(): + _fresh( + "import sys\n" + "import inferrsi.knowledge\n" + "assert not [m for m in sys.modules if m.startswith('inferrsi.knowledge.')]\n" + "import inferrsi.infra.inspection\n" + "assert 'inferrsi.catalog' not in sys.modules\n" + "assert not [m for m in sys.modules if m.startswith('inferrsi.knowledge.')]\n" + "assert 'inferrsi._repo_pack_data' in sys.modules\n" + ) + + +def test_knowledge_closure_is_isolated_while_overall_stays_false(): + # K01-5: the Knowledge package exists and its closure is clean, but the + # allowlist, the registered residual and harness -> model/infra remain. + report = first_wave_report() + assert report.violations == () + assert "absent: knowledge/" not in report.blockers + assert report.new_core_closure_isolated + assert {"allowlist", "residual", "harness reaches model/infra"} <= blocker_kinds(report) + assert not report.isolation_complete diff --git a/tests/test_knowledge_model_policy.py b/tests/test_knowledge_model_policy.py index 145e22ffd..46982928e 100644 --- a/tests/test_knowledge_model_policy.py +++ b/tests/test_knowledge_model_policy.py @@ -173,6 +173,8 @@ def test_stale_head_and_unsupported_schema_fail(): "docs/reference-matrix.md", "docs/searchspace-matrix.md", "docs/knowledge-base.md", "docs/knowledge-operations.md", "docs/knowledge-store.md", "src/inferrsi/catalog.py", "src/inferrsi/skill_catalog.py", "src/inferrsi/knowledge_operations.py", + "src/inferrsi/knowledge/catalog.py", "src/inferrsi/knowledge/skill_catalog.py", + "src/inferrsi/knowledge/knowledge_operations.py", ]) def test_explicit_protected_paths_require_declarations(policy, path): item = pr() diff --git a/tests/test_knowledge_store.py b/tests/test_knowledge_store.py index 3a4bb9b0f..bdc86a8c3 100644 --- a/tests/test_knowledge_store.py +++ b/tests/test_knowledge_store.py @@ -370,6 +370,7 @@ def test_package_only_collect_without_checkout_or_yaml(tmp_path): "assert item['authority'] == 'package', item\n" "assert reader.read(item['ref'], revision=item['revision']).revision == item['revision']\n" "assert 'inferrsi.skill_catalog' not in sys.modules\n" + "assert 'inferrsi.knowledge.skill_catalog' not in sys.modules\n" ) env = {**os.environ, "PYTHONPATH": str(REPO / "src"), "PYTHONUTF8": "1"} result = subprocess.run( diff --git a/tests/test_knowledge_tiers.py b/tests/test_knowledge_tiers.py index 1f2772315..163572343 100644 --- a/tests/test_knowledge_tiers.py +++ b/tests/test_knowledge_tiers.py @@ -10,6 +10,7 @@ from inferrsi.arguments import parser from inferrsi.catalog import get_knowledge, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.knowledge import catalog as knowledge_catalog ROOT = Path(__file__).resolve().parents[1] / "src/inferrsi/data/knowledge_packs" @@ -86,7 +87,8 @@ def test_misclassified_external_card_is_rejected_before_retrieval( ) target.parent.mkdir(parents=True) target.write_text(json.dumps(record)) - monkeypatch.setattr(catalog, "files", lambda _: tmp_path) + # Patch implementation globals; the root module is a static re-export shell. + monkeypatch.setattr(knowledge_catalog, "files", lambda _: tmp_path) with pytest.raises(ValueError, match="tier disagrees"): search_knowledge(tier="contribution") diff --git a/tests/test_research.py b/tests/test_research.py index c62bde6b2..da25c12af 100644 --- a/tests/test_research.py +++ b/tests/test_research.py @@ -9,6 +9,7 @@ from inferrsi import catalog from inferrsi.catalog import get_knowledge, search_knowledge from inferrsi.cli import main +from inferrsi.knowledge import catalog as knowledge_catalog from inferrsi.research import research_knowledge @@ -88,7 +89,7 @@ def test_independent_queries_are_unioned_fairly_and_exact_search_is_unchanged(mo ("f-overlap", "ExampleModel mlx diffusion video offload"), ) ] - monkeypatch.setattr(catalog, "knowledge_records", lambda: records) + monkeypatch.setattr(knowledge_catalog, "knowledge_records", lambda: records) exact = search_knowledge("ExampleModel mlx diffusion video offload") assert [record["id"] for record in exact] == ["f-overlap"] result = research_knowledge( @@ -117,7 +118,7 @@ def test_query_attribution_survives_the_shared_output_limit(monkeypatch): {"id": "model", "summary": "SyntheticModel mlx"}, {"id": "framework", "summary": "mlx mlx"}, ] - monkeypatch.setattr(catalog, "knowledge_records", lambda: records) + monkeypatch.setattr(knowledge_catalog, "knowledge_records", lambda: records) result = research_knowledge("SyntheticModel", frameworks=["mlx"], limit=1) assert result["items"] == [{ "record": records[0], diff --git a/tests/test_skill_eval_optional_dependencies.py b/tests/test_skill_eval_optional_dependencies.py index 3a243090c..ffc0ed65e 100644 --- a/tests/test_skill_eval_optional_dependencies.py +++ b/tests/test_skill_eval_optional_dependencies.py @@ -30,6 +30,7 @@ def test_comparison_reporting_without_yaml(self): assert all(item['input_tokens'] is None for item in summary.values()) assert callable(read_report) assert 'inferrsi.skill_catalog' not in sys.modules + assert 'inferrsi.knowledge.skill_catalog' not in sys.modules """) def test_checkpoint_without_yaml(self): @@ -41,6 +42,7 @@ def test_checkpoint_without_yaml(self): assert result.action == 'repair_evidence' assert result.reasons == ('missing_evidence',) assert 'inferrsi.skill_catalog' not in sys.modules + assert 'inferrsi.knowledge.skill_catalog' not in sys.modules """) def test_daily_controller_without_yaml(self): @@ -55,6 +57,7 @@ def test_daily_controller_without_yaml(self): else: raise AssertionError('Daily controller accepted missing budget configuration') assert 'inferrsi.skill_catalog' not in sys.modules + assert 'inferrsi.knowledge.skill_catalog' not in sys.modules """) def test_selected_snapshot_still_requires_yaml_before_side_effects(self): diff --git a/tests/test_skill_library_readers.py b/tests/test_skill_library_readers.py index 945deb7ee..02924e673 100644 --- a/tests/test_skill_library_readers.py +++ b/tests/test_skill_library_readers.py @@ -220,7 +220,8 @@ def test_package_catalog_and_reader_do_not_require_yaml(): "records = knowledge_records()\n" "identity = next(r['id'] for r in records if r.get('skill_documents'))\n" "assert KnowledgeReader().read(identity).content\n" - "assert 'inferrsi.skill_catalog' not in sys.modules\n", + "assert 'inferrsi.skill_catalog' not in sys.modules\n" + "assert 'inferrsi.knowledge.skill_catalog' not in sys.modules\n", ], capture_output=True, text=True, From 18271323cd4360809ed25b4cb7ac4689e794eca4 Mon Sep 17 00:00:00 2001 From: GUOGUO Date: Thu, 12 Nov 2026 22:50:10 +0800 Subject: [PATCH 2/2] docs: address K01 review comments and post-#338 isolation expectation Revise the Knowledge compatibility manifest: injection points are on inferrsi.knowledge.knowledge_operations / catalog, not the root shells; record the current test patch modules; note that OBS (#413) removes the TASK_RELEVANCE edge from observability/knowledge_capture. Update the K01-5 isolation test for #338: runtime resource loaders keep new_core_closure_isolated false, while Knowledge violations stay empty and overall isolation_complete stays false. Refs #407 Signed-off-by: GUOGUO --- docs/legacy-knowledge-manifest.json | 8 ++++---- tests/test_knowledge_compatibility_shells.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/legacy-knowledge-manifest.json b/docs/legacy-knowledge-manifest.json index 867aa34d0..ef5394b08 100644 --- a/docs/legacy-knowledge-manifest.json +++ b/docs/legacy-knowledge-manifest.json @@ -41,7 +41,7 @@ "REPO_TYPES is the same tuple object through root catalog, the neutral leaf, and K.catalog", "Derived KNOWLEDGE_SCOPES and KNOWLEDGE_REPOS stay unchanged" ], - "consumer_evidence": "tests/test_cli_catalog.py (L189) and tests/test_knowledge_tiers.py (L89) patch catalog.files; tests/test_research.py (L91, L120) patch catalog.knowledge_records; infra.inspection (L10, L56) imports and calls repository_pack; arguments (L12) imports BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES; cli (L12) imports get_knowledge, search_knowledge; observability/campaign_records (L12) imports BACKENDS", + "consumer_evidence": "tests/test_cli_catalog.py (L189) and tests/test_knowledge_tiers.py (L89) patch catalog.files; tests/test_research.py (L91, L120) patch catalog.knowledge_records; infra.inspection (L10, L56) imports and calls repository_pack; arguments (L12) imports BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES; cli (L12) imports get_knowledge, search_knowledge; observability/campaign_records (L12) imports BACKENDS. On this K01 head, tests/test_cli_catalog.py (L191), tests/test_knowledge_tiers.py (L91), and tests/test_research.py (L92, L121) patch inferrsi.knowledge.catalog; a root-shell patch is no longer effective.", "retirement_evidence": "catalog sheds repository_pack to the neutral leaf; K.catalog does not export repository_pack" }, { @@ -135,7 +135,7 @@ "RELEVANCE_METHODS, TASK_RELEVANCE, and the scoring and context functions are the same objects; selection algorithm not rewritten", "#389 audit: TASK_RELEVANCE is \"task-lexical-v1\" and is the third RELEVANCE_METHODS element; skill_score accepts only the two skill-only methods and rejects task-lexical-v1 with \"Unknown relevance method for skill-only ranking\"; task_score ranks skill and package records by whole task terms with a portable-or-in-scope backend factor; record_context returns the whole package record entrypoint as a context view or a context_limit omission" ], - "consumer_evidence": "knowledge_operations (L12-L21) imports RELEVANCE_METHODS, TASK_RELEVANCE, context_view, indexed_skill, record_context, skill_score, task_score, words; collect uses them for relevance selection and context views; observability/knowledge_capture (L20) imports TASK_RELEVANCE through the root leaf and uses it at L74-L75, and cli (L335) and observability/optimization_story (L14) load that module; arguments (L11) imports RELEVANCE_METHODS; tools/evaluate_indexer.py (L138) imports SkillIndexError, indexed_skill; tests/test_raw_string_selection.py covers scoring and the skill_score rejection (L84-L86); tests/test_collect_selection.py covers context_view; tests/test_campaign_task_selection.py covers task-lexical-v1 through root KnowledgeReader" + "consumer_evidence": "knowledge_operations (L12-L21) imports RELEVANCE_METHODS, TASK_RELEVANCE, context_view, indexed_skill, record_context, skill_score, task_score, words; collect uses them for relevance selection and context views; observability/knowledge_capture (L20) imports TASK_RELEVANCE through the root leaf and uses it at L74-L75, and cli (L335) and observability/optimization_story (L14) load that module; arguments (L11) imports RELEVANCE_METHODS; tools/evaluate_indexer.py (L138) imports SkillIndexError, indexed_skill; tests/test_raw_string_selection.py covers scoring and the skill_score rejection (L84-L86); tests/test_collect_selection.py covers context_view; tests/test_campaign_task_selection.py covers task-lexical-v1 through root KnowledgeReader. The observability/knowledge_capture import of TASK_RELEVANCE is removed by OBS (#413); K01 only keeps the root-shell pass-through of that name." }, { "importer": "research", @@ -200,10 +200,10 @@ }, "audited_definitions": [ "Bundle/KnowledgeReader/RevisionConflict/IdentityError/safe_path/AUTHORITIES/IDENTITY are the same objects; signatures/defaults/Path/exceptions and used symbols preserved", - "_json and _hash are explicitly re-exported for test injection (D1, D2)", + "The test-injection points for _json and _hash (D1, D2) are on the implementation module inferrsi.knowledge.knowledge_operations; the root shell only re-exports the same names as pass-throughs. Patching the root shell does not affect Bundle.manifest or KnowledgeReader.collect.", "#389 audit of KnowledgeReader.collect: signature, defaults, and validation order are unchanged; relevance_method=\"task-lexical-v1\" ranks every package authority record with task_score, lets a packaged skills.discovery skill pass a scenario filter only for ranked selection without ids, builds non-skill contexts with record_context, and adds selection.scope \"package-skills-and-records\" with matched_count and returned_count; the two earlier methods keep the packaged-skill-only path and selection.scope \"generated-packaged-skills\". No top-level name was added or removed." ], - "consumer_evidence": "harness/skill_evaluation (L9) imports KnowledgeReader; cli (L15, L25, L33) imports and uses KnowledgeReader; knowledge_store (L10-L19) imports KO symbols; tests/test_bundle_manifest.py (L73) patches _hash; tests/test_collect_limit_serialization.py (L39-47) patches _json; tests/test_knowledge_operations.py covers Bundle/KnowledgeReader contract; tests/test_campaign_task_selection.py covers the #389 task-lexical-v1 collect path" + "consumer_evidence": "harness/skill_evaluation (L9) imports KnowledgeReader; cli (L15, L25, L33) imports and uses KnowledgeReader; knowledge_store (L10-L19) imports KO symbols; tests/test_bundle_manifest.py (L73) patches _hash; tests/test_collect_limit_serialization.py (L39-47) patches _json; tests/test_knowledge_operations.py covers Bundle/KnowledgeReader contract; tests/test_campaign_task_selection.py covers the #389 task-lexical-v1 collect path. On this K01 head, tests/test_bundle_manifest.py (L74-L75) and tests/test_collect_limit_serialization.py (L48-L49) patch inferrsi.knowledge.knowledge_operations; a root-shell patch is no longer effective." }, { "importer": "infra.inspection", diff --git a/tests/test_knowledge_compatibility_shells.py b/tests/test_knowledge_compatibility_shells.py index 354b43ae3..4be152ef6 100644 --- a/tests/test_knowledge_compatibility_shells.py +++ b/tests/test_knowledge_compatibility_shells.py @@ -74,9 +74,16 @@ def test_package_init_and_infra_inspection_load_no_knowledge_body(): def test_knowledge_closure_is_isolated_while_overall_stays_false(): # K01-5: the Knowledge package exists and its closure is clean, but the # allowlist, the registered residual and harness -> model/infra remain. + # After #338, runtime resource loaders keep new_core_closure_isolated + # false; that is not a Knowledge-closure regression. report = first_wave_report() assert report.violations == () assert "absent: knowledge/" not in report.blockers - assert report.new_core_closure_isolated - assert {"allowlist", "residual", "harness reaches model/infra"} <= blocker_kinds(report) + assert { + "allowlist", + "residual", + "harness reaches model/infra", + "runtime resource loaders", + } <= blocker_kinds(report) + assert not report.new_core_closure_isolated assert not report.isolation_complete