diff --git a/.github/knowledge-model-policy.json b/.github/knowledge-model-policy.json index 8e0f3d232..47ddf72ae 100644 --- a/.github/knowledge-model-policy.json +++ b/.github/knowledge-model-policy.json @@ -12,7 +12,10 @@ "docs/knowledge-store.md", "src/inferrsi/catalog.py", "src/inferrsi/skill_catalog.py", - "src/inferrsi/knowledge_operations.py" + "src/inferrsi/knowledge_operations.py", + "src/inferrsi/knowledge/catalog.py", + "src/inferrsi/knowledge/skill_catalog.py", + "src/inferrsi/knowledge/knowledge_operations.py" ], "policy_paths": [ ".github/knowledge-model-policy.json", diff --git a/docs/knowledge-model-policy.md b/docs/knowledge-model-policy.md index 6ffc225d3..186beaa93 100644 --- a/docs/knowledge-model-policy.md +++ b/docs/knowledge-model-policy.md @@ -37,6 +37,9 @@ The exact protected scope is: - `docs/knowledge-base.md`, `docs/knowledge-operations.md`, `docs/knowledge-store.md` - `src/inferrsi/catalog.py`, `src/inferrsi/skill_catalog.py`, `src/inferrsi/knowledge_operations.py` +- `src/inferrsi/knowledge/catalog.py`, `src/inferrsi/knowledge/skill_catalog.py`, + `src/inferrsi/knowledge/knowledge_operations.py`, the implementations behind + the three root compatibility shells above, which stay protected Deletions and both rename paths count. Mechanical/generated changes have no automatic exemption. Running a generator or obtaining an approved-model review diff --git a/docs/legacy-knowledge-manifest.json b/docs/legacy-knowledge-manifest.json index da02e450d..ef5394b08 100644 --- a/docs/legacy-knowledge-manifest.json +++ b/docs/legacy-knowledge-manifest.json @@ -1,7 +1,19 @@ { "$schema": "urn:inferrsi:docs:legacy-knowledge-manifest:v1", - "description": "First-wave compatibility exceptions for seven root Knowledge leaves plus the neutral technical data leaf. This manifest is frozen at P0b formal review and is not a wildcard approval. K01 verifies each hash against its base and re-reviews if any differ. D018/D020 chose these exact edges.", - "base_commit": "0da7e2f960cd4ec994ae8eddea5b71023c5379ca", + "description": "First-wave compatibility exceptions for seven root Knowledge leaves plus the neutral technical data leaf. P0b froze the first revision at 0da7e2f960cd4ec994ae8eddea5b71023c5379ca. This K01 revision re-records every leaf hash at base_commit before the move; it is not a wildcard approval and changes only through formal review. D018/D020 chose these exact edges. file_sha256 is the old root leaf before the move, and line references are at base_commit.", + "base_commit": "e2f4cd1e39b369f06835cdd655ad5dc9f6808918", + "revision_history": [ + { + "base_commit": "0da7e2f960cd4ec994ae8eddea5b71023c5379ca", + "stage": "P0b", + "superseded_by": "K01 revision at e2f4cd1e39b369f06835cdd655ad5dc9f6808918", + "reason": "#389 (c013e7778e5d421df655b969d4c89f81ade65b44) changed two leaves after P0b. #393 changed none of the seven leaves. The five other hashes are unchanged.", + "changed_file_sha256": { + "src/inferrsi/_skill_selection.py": "acbec7c91bfa05bc2008b939e009fe600a40518051229fb7618c56b092ad36b9", + "src/inferrsi/knowledge_operations.py": "3725781bcedb308af6ad9edc0b970c63f82accf6b04ebee4008c1081b63bed9c" + } + } + ], "compatibility_pairs": [ { "importer": "catalog", @@ -29,7 +41,7 @@ "REPO_TYPES is the same tuple object through root catalog, the neutral leaf, and K.catalog", "Derived KNOWLEDGE_SCOPES and KNOWLEDGE_REPOS stay unchanged" ], - "consumer_evidence": "tests/test_cli_catalog.py (L189) and tests/test_knowledge_tiers.py (L89) patch catalog.files; tests/test_research.py (L91, L120) patch catalog.knowledge_records; infra.inspection (L10, L56) imports and calls repository_pack", + "consumer_evidence": "tests/test_cli_catalog.py (L189) and tests/test_knowledge_tiers.py (L89) patch catalog.files; tests/test_research.py (L91, L120) patch catalog.knowledge_records; infra.inspection (L10, L56) imports and calls repository_pack; arguments (L12) imports BACKENDS, KNOWLEDGE_REPOS, KNOWLEDGE_SCOPES, REPO_TYPES; cli (L12) imports get_knowledge, search_knowledge; observability/campaign_records (L12) imports BACKENDS. On this K01 head, tests/test_cli_catalog.py (L191), tests/test_knowledge_tiers.py (L91), and tests/test_research.py (L92, L121) patch inferrsi.knowledge.catalog; a root-shell patch is no longer effective.", "retirement_evidence": "catalog sheds repository_pack to the neutral leaf; K.catalog does not export repository_pack" }, { @@ -96,13 +108,13 @@ "audited_definitions": [ "root:Path and checkout physical resource semantics unchanged" ], - "consumer_evidence": "knowledge_operations (L270, L308, L426 lazy imports; calls at L272, L310, L430, L433, L436) imports skill_directories, skill_directory, skill_metadata, skill_reference; KnowledgeReader._skill uses all skill_* APIs; tests/test_skill_catalog.py covers skill enumeration and loading" + "consumer_evidence": "knowledge_operations (L279, L317, L435 lazy imports; calls at L281, L319, L438, L439, L442, L445) imports skill_directories, get_skill, skill_directory, skill_metadata, skill_reference; KnowledgeReader._skill uses all skill_* APIs; cli (L129) lazily imports get_skill, skill_instructions, skill_records, skill_reference, skill_resource; tools/skill_eval_snapshot.py (L47) imports _skill_name, _unique_fields, skill_metadata; tests/test_skill_catalog.py covers skill enumeration and loading" }, { "importer": "_skill_selection", "target": "inferrsi.knowledge._skill_selection", "file": "src/inferrsi/_skill_selection.py", - "file_sha256": "acbec7c91bfa05bc2008b939e009fe600a40518051229fb7618c56b092ad36b9", + "file_sha256": "7bcf1930b73e09fcadf78e7c9e384b30411ed599d83c32e6837f04ca25cbda77", "exports": { "public": [ "RELEVANCE_METHODS", @@ -112,14 +124,18 @@ "raw_string_lexical_score", "indexed_skill", "skill_score", - "context_view" + "context_view", + "TASK_RELEVANCE", + "task_score", + "record_context" ], "private_used": [] }, "audited_definitions": [ - "RELEVANCE_METHODS and scoring functions are the same objects; selection algorithm not rewritten" + "RELEVANCE_METHODS, TASK_RELEVANCE, and the scoring and context functions are the same objects; selection algorithm not rewritten", + "#389 audit: TASK_RELEVANCE is \"task-lexical-v1\" and is the third RELEVANCE_METHODS element; skill_score accepts only the two skill-only methods and rejects task-lexical-v1 with \"Unknown relevance method for skill-only ranking\"; task_score ranks skill and package records by whole task terms with a portable-or-in-scope backend factor; record_context returns the whole package record entrypoint as a context view or a context_limit omission" ], - "consumer_evidence": "knowledge_operations (L12) imports RELEVANCE_METHODS, context_view, indexed_skill, skill_score, words; collect uses these for relevance selection; tests/test_raw_string_selection.py covers scoring; tests/test_collect_selection.py covers context_view" + "consumer_evidence": "knowledge_operations (L12-L21) imports RELEVANCE_METHODS, TASK_RELEVANCE, context_view, indexed_skill, record_context, skill_score, task_score, words; collect uses them for relevance selection and context views; observability/knowledge_capture (L20) imports TASK_RELEVANCE through the root leaf and uses it at L74-L75, and cli (L335) and observability/optimization_story (L14) load that module; arguments (L11) imports RELEVANCE_METHODS; tools/evaluate_indexer.py (L138) imports SkillIndexError, indexed_skill; tests/test_raw_string_selection.py covers scoring and the skill_score rejection (L84-L86); tests/test_collect_selection.py covers context_view; tests/test_campaign_task_selection.py covers task-lexical-v1 through root KnowledgeReader. The observability/knowledge_capture import of TASK_RELEVANCE is removed by OBS (#413); K01 only keeps the root-shell pass-through of that name." }, { "importer": "research", @@ -162,7 +178,7 @@ "importer": "knowledge_operations", "target": "inferrsi.knowledge.knowledge_operations", "file": "src/inferrsi/knowledge_operations.py", - "file_sha256": "3725781bcedb308af6ad9edc0b970c63f82accf6b04ebee4008c1081b63bed9c", + "file_sha256": "686b196e03ca0a1dba5ff5e246563f305ba2bea412172833905af41a6c6ad886", "exports": { "public": [ "AUTHORITIES", @@ -184,9 +200,10 @@ }, "audited_definitions": [ "Bundle/KnowledgeReader/RevisionConflict/IdentityError/safe_path/AUTHORITIES/IDENTITY are the same objects; signatures/defaults/Path/exceptions and used symbols preserved", - "_json and _hash are explicitly re-exported for test injection (D1, D2)" + "The test-injection points for _json and _hash (D1, D2) are on the implementation module inferrsi.knowledge.knowledge_operations; the root shell only re-exports the same names as pass-throughs. Patching the root shell does not affect Bundle.manifest or KnowledgeReader.collect.", + "#389 audit of KnowledgeReader.collect: signature, defaults, and validation order are unchanged; relevance_method=\"task-lexical-v1\" ranks every package authority record with task_score, lets a packaged skills.discovery skill pass a scenario filter only for ranked selection without ids, builds non-skill contexts with record_context, and adds selection.scope \"package-skills-and-records\" with matched_count and returned_count; the two earlier methods keep the packaged-skill-only path and selection.scope \"generated-packaged-skills\". No top-level name was added or removed." ], - "consumer_evidence": "harness/skill_evaluation (L9) imports KnowledgeReader; cli (L15, L25, L33) imports and uses KnowledgeReader; knowledge_store (L10-18) imports KO symbols; tests/test_bundle_manifest.py (L73) patches _hash; tests/test_collect_limit_serialization.py (L39-47) patches _json; tests/test_knowledge_operations.py covers Bundle/KnowledgeReader contract" + "consumer_evidence": "harness/skill_evaluation (L9) imports KnowledgeReader; cli (L15, L25, L33) imports and uses KnowledgeReader; knowledge_store (L10-L19) imports KO symbols; tests/test_bundle_manifest.py (L73) patches _hash; tests/test_collect_limit_serialization.py (L39-47) patches _json; tests/test_knowledge_operations.py covers Bundle/KnowledgeReader contract; tests/test_campaign_task_selection.py covers the #389 task-lexical-v1 collect path. On this K01 head, tests/test_bundle_manifest.py (L74-L75) and tests/test_collect_limit_serialization.py (L48-L49) patch inferrsi.knowledge.knowledge_operations; a root-shell patch is no longer effective." }, { "importer": "infra.inspection", @@ -223,7 +240,9 @@ }, "technical_leaf": { "module": "inferrsi._repo_pack_data", - "note": "The new _repo_pack_data.py file does not exist yet; do not create it in P0b; do not lock a fabricated full-file hash. K01 will create it and lock the full-file hash.", + "note": "K01 creates src/inferrsi/_repo_pack_data.py from the recorded original fragment plus a leading comment, the future import, and the three listed stdlib imports. file_sha256 locks the complete new file; P0b recorded only the fragment.", + "file": "src/inferrsi/_repo_pack_data.py", + "file_sha256": "a14edac8283b1f7b6b5794381c44d7c2632dda5cc01f03c557dc293d05523afd", "allowed_imports": [ "json", "importlib.resources", @@ -246,7 +265,7 @@ "module": "src/inferrsi/infra/inspection.py", "current_import": "from ..catalog import repository_pack", "required_new_import": "from .._repo_pack_data import repository_pack", - "note": "K01 must change only this import line. Original functions, injected parameters, and returns stay the same. Do not edit inspection.py in P0b." + "note": "K01 changes only this import line. Original functions, injected parameters, and returns stay the same. current_import records the line at base_commit." }, "intra_closure_note": "An import of inferrsi._repo_pack_data from inside inferrsi.knowledge (for example inferrsi/knowledge/catalog.py) is intra-closure, not a tenth compatibility exception; it is governed by the leaf's no-inferrsi.*-import rule and by REPO_TYPES object identity, not by this allowlist." } diff --git a/docs/module-layout.md b/docs/module-layout.md index afdf415a9..d85b4e1fc 100644 --- a/docs/module-layout.md +++ b/docs/module-layout.md @@ -1,8 +1,11 @@ # Target module layout -This page records the directory rule for the integration slice. The engineering -task contribution now implements `benchmark/registry.py` and `benchmark/tasks/`. -The current flat `cli.py` and `benchmark_operations.py` assemble this slice. +This page records the directory rule for the integration slice. The first-wave +Knowledge move (K01) is implemented: `src/inferrsi/knowledge/` holds the seven +moved Knowledge bodies, and the old root modules are static compatibility +shells. The engineering task contribution now implements `benchmark/registry.py` +and `benchmark/tasks/`. The current flat `cli.py` and `benchmark_operations.py` +assemble this slice. #371 remains the proposal for the Apps package path and is not accepted here. ## Target modules @@ -62,7 +65,7 @@ Several flat modules remain shared seams until their slice starts. | `harness/optimization_session.py`, `harness/project.py`, `harness/task_protocol.py` | Shared preparation, execution and retained-result assessment | CLI supplies task resources and knowledge access; ordinary project lifecycle does not import Benchmark | | `harness/optimization.py`, `harness/candidates.py`, `harness/usage.py` | Shared project and packaged-task lifecycle | Injected execution/assessment; declared file/directory candidates, selection direction and exact/up-to rounds | | `harness/agent_sessions.py` | Native host handoff records | Freezes requests and retains native IDs and exported observations; the initiating Agent calls host tools | -| `knowledge_operations.py`, `knowledge_store.py`, `catalog.py`, `research.py`, `skill_catalog.py`, `_skill_selection.py` | Knowledge behavior and corpus readers | Interfaces unchanged | +| `knowledge_operations.py`, `knowledge_store.py`, `catalog.py`, `research.py`, `skill_catalog.py`, `_skill_selection.py`, `_source_snapshot.py` | Static compatibility shells since K01; bodies in `knowledge/` | Interfaces unchanged | | `harness/` | Mixed legacy tree, not target isolation | Not redefined | | `evaluation.py`, `model/validators/` | Benchmark-leaning checks | Not moved | | `infra/`, `execution.py`, `runspec.py`, `inspection.py` | Seams used by the current tree | Not a fourth core | @@ -87,8 +90,12 @@ first-wave pairs below, read from `docs/legacy-knowledge-manifest.json`. The Benchmark package now provides resource discovery and task scripts. The CLI and single-task application launcher assemble them with Harness; task -scripts receive execution support without importing another core. The -compatibility exceptions below remain scoped to the first-wave Knowledge move. +scripts receive execution support without importing another core. +"There is no exception list of current edges" described the tree before K01, +when the three package names were unused and no current edge needed an +exception. It was never a permanent ban on reviewed exceptions; since K01 the +exact first-wave Knowledge pairs below are the only Knowledge compatibility +exceptions, and they remain scoped to that move. The gate scans `src/inferrsi/**` as the transfer graph. It resolves absolute, relative, and `from package import name` imports, includes the parent-package @@ -206,9 +213,11 @@ The manifest records the base commit, each old leaf's file hash at that commit, export names, audited definitions, consumer evidence, catalog's two symbol routes, the required `infra.inspection` import rewrite, and the original `REPO_TYPES` / `repository_pack` fragment from `catalog.py` with its -hash. `_repo_pack_data.py` does not exist yet; K01 creates it and locks its -full-file hash. K01 re-verifies every recorded hash against its own base and -re-reviews the manifest if any differ. +hash. K01 created `_repo_pack_data.py`, and the manifest locks its full-file +hash. K01 re-verified every recorded hash against its own base. Two leaves +had changed after P0b, so K01 re-recorded all seven hashes at its base and +keeps the P0b values in `revision_history`; that revision goes through formal +review with the move. ## Old `run` exit codes diff --git a/src/inferrsi/_repo_pack_data.py b/src/inferrsi/_repo_pack_data.py new file mode 100644 index 000000000..b31869ea9 --- /dev/null +++ b/src/inferrsi/_repo_pack_data.py @@ -0,0 +1,16 @@ +# Neutral raw repository-pack data leaf (D020). It holds only the original +# REPO_TYPES tuple and pack.json reader, and imports no inferrsi module. +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +REPO_TYPES = ("vllm", "vllm_omni", "afd_plugin", "vllm_rlt", "strata", "sglang", "sglang_omni") + + +def repository_pack(repo_type: str) -> dict[str, Any]: + if repo_type not in REPO_TYPES: + raise ValueError(f"Unknown repository type: {repo_type}") + path = files("inferrsi").joinpath("data", "repo_packs", repo_type, "pack.json") + return json.loads(path.read_text(encoding="utf-8")) diff --git a/src/inferrsi/_skill_selection.py b/src/inferrsi/_skill_selection.py index c696ef571..23309a0fc 100644 --- a/src/inferrsi/_skill_selection.py +++ b/src/inferrsi/_skill_selection.py @@ -1,211 +1,30 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge._skill_selection. from __future__ import annotations -import json -import posixpath -import re - -TASK_RELEVANCE = "task-lexical-v1" -RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE) - - -class SkillIndexError(ValueError): - pass - - -def words(cue: str) -> set[str]: - return set(re.findall(r"\w+", cue.casefold())) - - -def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: - tokens = words(cue) - return sum( - weight * sum(token in json.dumps(value, ensure_ascii=False).casefold() for token in tokens) - for weight, value in fields - ) - - -def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: - """Match original strings; retain legacy JSON matching for nonstring fields.""" - tokens = words(cue) - score = 0 - for weight, value in fields: - text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) - score += weight * sum(token in text.casefold() for token in tokens) - return score - - -def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: - """Validate the existing projection against this exact canonical bundle.""" - registry_path = "data/skill_library/skill-index.json" - registry = json.loads(content[registry_path]) - name = record["skill_metadata"]["name"] - entry = registry["skills"][name] - prefix = f"data/skill_library/{entry['path']}/" - core = prefix + "SKILL.md" - expected_id = f"inferrsi.{entry['backend']}.skills.{name.replace('-', '_')}" - if ( - registry["schema_version"] != 1 - or view["authority"] != "package" - or record["id"] != view["id"] - or record["id"] != expected_id - or record["backend"] != entry["backend"] - or record["skill_documents"] != {"original": core} - or posixpath.basename(entry["path"]) != name - ): - raise SkillIndexError("Packaged skill identity disagrees with the registry") - expected = { - "src/inferrsi/" + path: digest - for path, digest in view["resources"].items() - if path.startswith(prefix) or path == registry_path - } - if record["source_files"] != expected: - raise SkillIndexError("Stale packaged skill projection; rebuild with build_skill_index.py") - - required = [core] - metadata = record["skill_metadata"] - notices = [ - source["third_party_notices"] - for source in metadata.get("sources", []) if "third_party_notices" in source - ] - for relative in [ - *metadata.get("references", []), *metadata.get("provenance_readsets", []), *notices, - ]: - path = posixpath.normpath(prefix + relative) - if "\\" in relative or not path.startswith("data/skill_library/") or path not in content: - raise SkillIndexError(f"Unavailable declared skill reference: {relative}") - if path not in required: - required.append(path) - for path in sorted(content): - if path.startswith(prefix) and re.search( - r"(?:^|[-_])(?:LICENSE|NOTICES?)(?:\.[^.]+)?$", - posixpath.basename(path), re.IGNORECASE, - ) and path not in required: - required.append(path) - for resource in list(required): - directory = posixpath.dirname(resource) - while directory.startswith("data/skill_library/"): - license_path = directory + "/LICENSE" - if license_path in content and license_path not in required: - required.append(license_path) - directory = posixpath.dirname(directory) - optional = sorted( - path for path in content - if path.startswith(prefix) and path not in required - ) - return core, required, optional - - -def skill_score( - cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1" -) -> int: - if method not in ("weighted-lexical-v1", "weighted-raw-string-v1"): - raise ValueError("Unknown relevance method for skill-only ranking") - scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score - return scorer(cue, [ - (4, record["title"]), - (3, record["summary"]), - (2, [record.get("tags", []), record.get("applicability", {})]), - (1, core.decode("utf-8")), - ]) - - -def task_score(cue: str, record: dict, core: bytes, backend: str | None) -> int: - """Rank whole task terms across both skill and repository knowledge owners.""" - def terms(text: str) -> set[str]: - return set(re.findall(r"[^\W_]+", text.casefold())) - - tokens = terms(cue) - {"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"} - fields = ( - (4, record["title"]), - (3, record["summary"]), - (2, " ".join(record.get("tags", []))), - (2, record["repo"] + " " + record["scenario"]), - (1, json.dumps(record.get("applicability", {}), ensure_ascii=False)), - (1, core.decode("utf-8")), - ) - score = sum(weight * len(tokens & terms(text)) for weight, text in fields) - # With no platform in scope prefer portable procedures to a specialization - # that happens to repeat the same task words in its description. - platforms = { - "cuda": {"cuda", "nvidia"}, "rocm": {"rocm", "amd"}, - "ascend": {"ascend", "cann"}, "mlx": {"mlx"}, "cpu": {"cpu"}, - } - in_scope = {backend} if backend else {key for key, names in platforms.items() if names & tokens} - return score * (2 if record["backend"] == "common" or record["backend"] in in_scope else 1) - - -def record_context(record: dict, view: dict, content: dict[str, bytes], available: int) -> tuple: - """Keep the canonical package record including its sources and limits.""" - path = view["entrypoint"] - body = content[path] - if len(body) > available: - return None, {"ref": view["ref"], "reason": "context_limit", - "required_bytes": len(body), "available_bytes": available} - return { - "ref": view["ref"], "revision": view["revision"], "content_bytes": len(body), - "resources": [{"path": path, "sha256": view["resources"][path], "role": "record", - "locator": "whole-resource", "content": body.decode("utf-8")}], - "omissions": [{"path": p, "reason": "linked_document"} for p in - dict.fromkeys(record.get("skill_documents", {}).values())], - "sources": view["sources"], "attribution": view["attribution"], - "limitations": view["limitations"], - }, None - - -def context_view( - record: dict, view: dict, content: dict[str, bytes], cue: str, available: int -) -> tuple[dict | None, dict | None]: - core, required, optional = indexed_skill(record, view, content) - required_bytes = sum(len(content[path]) for path in required) - if required_bytes > available: - return None, { - "ref": view["ref"], "reason": "context_limit", - "required_bytes": required_bytes, "available_bytes": available, - } - - resources, omissions = [], [] - used = 0 - - def include(path: str) -> None: - nonlocal used - resources.append({ - "path": path, "sha256": view["resources"][path], "locator": "whole-resource", - "role": "core" if path == core else "reference", - "content": content[path].decode("utf-8"), - }) - used += len(content[path]) - - for path in required: - include(path) - candidates = [] - for path in optional: - if ( - posixpath.basename(path) == "metadata.json" - or posixpath.splitext(path)[1].lower() not in (".md", ".json", ".txt", ".csv") - ): - omissions.append({"path": path, "reason": "not_context_resource"}) - continue - try: - text = content[path].decode("utf-8") - except UnicodeDecodeError: - omissions.append({"path": path, "reason": "not_utf8"}) - continue - score = lexical_score(cue, [(2, path.rpartition("/")[2]), (1, text)]) - if words(cue) and score == 0: - omissions.append({"path": path, "reason": "not_relevant"}) - continue - candidates.append((score, path)) - for _, path in sorted(candidates, key=lambda item: (-item[0], item[1])): - if used + len(content[path]) > available: - omissions.append({"path": path, "reason": "context_limit"}) - continue - include(path) - return { - "ref": view["ref"], "revision": view["revision"], "resources": resources, - "content_bytes": used, "omissions": omissions, - "sources": view["sources"], "attribution": view["attribution"], - "limitations": view["limitations"] + [ - "Selected resources are complete files; this view is not a standalone Bundle.", - "Follow omitted or other linked resources through the exact owner revision.", - ], - }, None +from inferrsi.knowledge._skill_selection import ( + RELEVANCE_METHODS, + TASK_RELEVANCE, + SkillIndexError, + context_view, + indexed_skill, + lexical_score, + raw_string_lexical_score, + record_context, + skill_score, + task_score, + words, +) + +__all__ = [ + "RELEVANCE_METHODS", + "SkillIndexError", + "words", + "lexical_score", + "raw_string_lexical_score", + "indexed_skill", + "skill_score", + "context_view", + "TASK_RELEVANCE", + "task_score", + "record_context", +] diff --git a/src/inferrsi/_source_snapshot.py b/src/inferrsi/_source_snapshot.py index 3ced2e330..b898386dc 100644 --- a/src/inferrsi/_source_snapshot.py +++ b/src/inferrsi/_source_snapshot.py @@ -1,88 +1,26 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge._source_snapshot. from __future__ import annotations -import re -from typing import Any - -# One shared snapshot keeps the repository-wide source revision and registry digest out of -# per-owner records, so a one-owner edit regenerates only that owner's record. -MANIFEST = "_provenance.json" -REPOSITORY = "https://github.com/InferMatrixLab/InferRSI" -LIBRARY = "src/inferrsi/data/skill_library" -REGISTRY = f"{LIBRARY}/skill-index.json" -STORED_SCHEMA = 2 - - -def manifest(revision: str, registry_sha256: str) -> dict[str, Any]: - return { - "schema_version": 1, - "repository": REPOSITORY, - "source_revision": revision, - "registry": {"path": REGISTRY, "sha256": registry_sha256}, - } - - -def _snapshot(value: Any) -> tuple[str, str]: - if ( - not isinstance(value, dict) - or set(value) != {"schema_version", "repository", "source_revision", "registry"} - or type(value["schema_version"]) is not int - or value["schema_version"] != 1 - or value["repository"] != REPOSITORY - or not isinstance(value["source_revision"], str) - or not re.fullmatch(r"[0-9a-f]{40}", value["source_revision"]) - ): - raise ValueError("Invalid knowledge source snapshot") - registry = value["registry"] - if ( - not isinstance(registry, dict) - or set(registry) != {"path", "sha256"} - or registry["path"] != REGISTRY - or not isinstance(registry["sha256"], str) - or not re.fullmatch(r"[0-9a-f]{64}", registry["sha256"]) - ): - raise ValueError("Invalid knowledge source snapshot registry") - return value["source_revision"], registry["sha256"] - - -def compact(record: dict[str, Any]) -> dict[str, Any]: - """Store a generated schema-v1 record without its snapshot-wide fields.""" - stored = {key: value for key, value in record.items() if key != "sources"} - stored["schema_version"] = STORED_SCHEMA - stored["source_files"] = { - path: digest for path, digest in record["source_files"].items() if path != REGISTRY - } - return stored - - -def resolve(stored: dict[str, Any], snapshot: Any) -> dict[str, Any]: - """Rebuild the public schema-v1 record from a stored record and the shared snapshot.""" - revision, registry_sha256 = _snapshot(snapshot) - if stored.get("schema_version") != STORED_SCHEMA or "sources" in stored: - raise ValueError("Stored skill record is not a schema-v2 projection") - files = stored.get("source_files") - documents = stored.get("skill_documents") - if ( - not isinstance(files, dict) or REGISTRY in files - or not isinstance(documents, dict) or not isinstance(documents.get("original"), str) - ): - raise ValueError(f"Invalid stored skill record: {stored.get('id')}") - entry = "src/inferrsi/" + documents["original"] - if entry not in files: - raise ValueError(f"Stored skill record lacks its entry hash: {stored.get('id')}") - hashes = dict(sorted({**files, REGISTRY: registry_sha256}.items())) - source = { - "type": "git", - "url": f"{REPOSITORY}/blob/{revision}/{entry}", - "commit": revision, - "paths": list(hashes), - "content_sha256": hashes[entry], - } - record = {} - for key, value in stored.items(): - record[key] = 1 if key == "schema_version" else value - if key == "origin": - record["sources"] = [source] - if "sources" not in record: - raise ValueError(f"Stored skill record lacks origin: {stored.get('id')}") - record["source_files"] = hashes - return record +from inferrsi.knowledge._source_snapshot import ( + LIBRARY, + MANIFEST, + REGISTRY, + REPOSITORY, + STORED_SCHEMA, + _snapshot, + compact, + manifest, + resolve, +) + +__all__ = [ + "MANIFEST", + "REPOSITORY", + "LIBRARY", + "REGISTRY", + "STORED_SCHEMA", + "manifest", + "compact", + "resolve", + "_snapshot", +] diff --git a/src/inferrsi/catalog.py b/src/inferrsi/catalog.py index 3408a53d2..3ab45f6fa 100644 --- a/src/inferrsi/catalog.py +++ b/src/inferrsi/catalog.py @@ -1,175 +1,32 @@ -"""Read-only package-data catalog. Knowledge text is never executable.""" - +# Compatibility shell. Knowledge names live in inferrsi.knowledge.catalog; +# REPO_TYPES and repository_pack live in the neutral inferrsi._repo_pack_data. from __future__ import annotations -import json -import re -from importlib.resources import files -from typing import Any - -from inferrsi import _source_snapshot - -REPO_TYPES = ("vllm", "vllm_omni", "afd_plugin", "vllm_rlt", "strata", "sglang", "sglang_omni") -# Learning scope does not imply that a repository has a runnable target adapter. -# Legacy repository-pack records retain their contribution tier; an independent -# peer card can describe a different learning use case for the same source repo. -KNOWLEDGE_SCOPES = { - "contribution": (*REPO_TYPES, "vllm_ascend", "inferrsi"), - "dependency": ( - "flashinfer", "flashattention", "pytorch", "triton", "diffusers", "transformers", "mlx" - ), - "peer": ( - "sglang", "tensorrt_llm", "lmdeploy", "xdit", "cache_dit", "qwen3_tts_engine", "deepinfer" - ), -} -KNOWLEDGE_REPOS = tuple( - dict.fromkeys(repo for repos in KNOWLEDGE_SCOPES.values() for repo in repos) +from inferrsi._repo_pack_data import REPO_TYPES, repository_pack +from inferrsi.knowledge.catalog import ( + BACKENDS, + KNOWLEDGE_REPOS, + KNOWLEDGE_SCOPES, + SCENARIO_ALIASES, + _validate_record, + _walk, + files, + get_knowledge, + knowledge_records, + search_knowledge, ) -BACKENDS = ("cpu", "cuda", "rocm", "ascend", "mlx") -SCENARIO_ALIASES = { - "diffusion.t2i": "diffusion.image_generation", - "diffusion.t2v": "diffusion.video_generation", -} - - -def _walk(root, prefix=()): - for child in sorted(root.iterdir(), key=lambda item: item.name): - parts = (*prefix, child.name) - if child.is_dir(): - yield from _walk(child, parts) - else: - yield parts, child - - -def _validate_record( - record: dict, repo: str, parts: tuple[str, ...], tier: str = "contribution" -) -> None: - required = ("id", "title", "repo", "backend", "component", "scenario", "status", "summary") - if not isinstance(record, dict) or any( - not isinstance(record.get(key), str) or not record[key].strip() for key in required - ): - raise ValueError(f"Incomplete knowledge record: {repo}/{'/'.join(parts)}") - if record.get("schema_version") != 1: - raise ValueError(f"Unsupported knowledge schema: {record['id']}") - # Existing schema-v1 records retain their IDs and default to contribution. - if record.get("tier", "contribution") != tier: - raise ValueError(f"Knowledge tier disagrees with path: {record['id']}") - if len(parts) != 3: - raise ValueError("Knowledge path must be BACKEND/COMPONENT/ENTRY.json") - backend, component, filename = parts - if backend not in (*BACKENDS, "common"): - raise ValueError(f"Unknown knowledge backend: {backend}") - if (record["repo"], record["backend"], record["component"]) != (repo, backend, component): - raise ValueError(f"Knowledge scope disagrees with path: {record['id']}") - expected_id = f"{repo}.{backend}.{component}.{filename[:-5].replace('-', '_')}" - if record["id"] != expected_id: - raise ValueError(f"Knowledge ID disagrees with path: {record['id']}") - if not record["scenario"].startswith(component + "."): - raise ValueError(f"Knowledge scenario disagrees with component: {record['id']}") - sources = record.get("sources") - if not isinstance(sources, list) or not sources: - raise ValueError(f"Knowledge has no source: {record['id']}") - for source in sources: - if ( - not isinstance(source, dict) - or not isinstance(source.get("url"), str) - or not source["url"].startswith("https://") - or not isinstance(source.get("commit"), str) - or not re.fullmatch(r"[0-9a-f]{40}", source["commit"]) - ): - raise ValueError(f"Knowledge requires source attribution and revision: {record['id']}") - verification = record.get("verification", {}) - if not isinstance(verification, dict) or not isinstance( - verification.get("locally_verified"), bool - ): - raise ValueError(f"Knowledge requires an explicit verification state: {record['id']}") - if record["status"] == "imported" and verification["locally_verified"]: - raise ValueError(f"Imported knowledge must state locally_verified=false: {record['id']}") - - -def knowledge_records() -> list[dict[str, Any]]: - data = files("inferrsi").joinpath("data") - records = [] - seen = set() - for tier, repos in KNOWLEDGE_SCOPES.items(): - for repo in repos: - knowledge = ( - data.joinpath("repo_packs", repo, "knowledge") - if tier == "contribution" - else data.joinpath("knowledge_packs", tier, repo) - ) - if not knowledge.is_dir(): - continue - snapshot = knowledge.joinpath(_source_snapshot.MANIFEST) - snapshot = ( - json.loads(snapshot.read_text(encoding="utf-8")) if snapshot.is_file() else None - ) - for parts, item in _walk(knowledge): - if not item.name.endswith(".json") or parts == (_source_snapshot.MANIFEST,): - continue - record = json.loads(item.read_text(encoding="utf-8")) - # Generated skill records store schema v2; readers always receive schema v1. - if isinstance(record, dict) and record.get("schema_version") == ( - _source_snapshot.STORED_SCHEMA - ): - if snapshot is None: - raise ValueError(f"Knowledge record lacks its source snapshot: {repo}") - record = _source_snapshot.resolve(record, snapshot) - _validate_record(record, repo, parts, tier) - if record["id"] in seen: - raise ValueError(f"Duplicate knowledge ID: {record['id']}") - seen.add(record["id"]) - records.append(record) - return records - - -def search_knowledge( - query: str = "", - *, - repo: str | None = None, - tier: str | None = None, - backend: str | None = None, - scenario: str | None = None, - limit: int = 10, -) -> list[dict[str, Any]]: - if limit < 1: - raise ValueError("limit must be positive") - if tier is not None and tier not in KNOWLEDGE_SCOPES: - raise ValueError(f"Unknown knowledge tier: {tier}") - scenario = SCENARIO_ALIASES.get(scenario, scenario) - tokens = re.findall(r"[\w.-]+", query.casefold()) - scored = [] - for record in knowledge_records(): - if tier and record.get("tier", "contribution") != tier: - continue - if repo and record["repo"] != repo: - continue - if backend and record["backend"] not in (backend, "common"): - continue - if scenario: - declared = record.get("scenario", "") - if declared != scenario and not ( - declared.endswith(".*") and scenario.startswith(declared[:-1]) - ): - continue - document = json.dumps(record, ensure_ascii=False).casefold() - if any(token not in document for token in tokens): - continue - score = sum(document.count(token) for token in tokens) - scored.append((score, record["id"], record)) - scored.sort(key=lambda item: (-item[0], item[1])) - return [item[2] for item in scored[:limit]] - - -def get_knowledge(record_id: str) -> dict[str, Any]: - for record in knowledge_records(): - if record["id"] == record_id: - return record - raise ValueError(f"Unknown knowledge ID: {record_id}") - -def repository_pack(repo_type: str) -> dict[str, Any]: - if repo_type not in REPO_TYPES: - raise ValueError(f"Unknown repository type: {repo_type}") - path = files("inferrsi").joinpath("data", "repo_packs", repo_type, "pack.json") - return json.loads(path.read_text(encoding="utf-8")) +__all__ = [ + "REPO_TYPES", + "KNOWLEDGE_SCOPES", + "KNOWLEDGE_REPOS", + "BACKENDS", + "SCENARIO_ALIASES", + "knowledge_records", + "search_knowledge", + "get_knowledge", + "repository_pack", + "_walk", + "_validate_record", + "files", +] diff --git a/src/inferrsi/infra/inspection.py b/src/inferrsi/infra/inspection.py index 597d3185a..c3251aef8 100644 --- a/src/inferrsi/infra/inspection.py +++ b/src/inferrsi/infra/inspection.py @@ -7,7 +7,7 @@ from collections.abc import Callable from pathlib import Path -from ..catalog import repository_pack +from .._repo_pack_data import repository_pack from .commands import probe_command from .platforms import inspect_device from .targets import repository_identity diff --git a/src/inferrsi/knowledge/__init__.py b/src/inferrsi/knowledge/__init__.py new file mode 100644 index 000000000..76ef1bf6b --- /dev/null +++ b/src/inferrsi/knowledge/__init__.py @@ -0,0 +1,2 @@ +# Knowledge core package. Keep this init import-free so loading one submodule +# never loads another (for example the PyYAML-backed skill_catalog). diff --git a/src/inferrsi/knowledge/_skill_selection.py b/src/inferrsi/knowledge/_skill_selection.py new file mode 100644 index 000000000..c696ef571 --- /dev/null +++ b/src/inferrsi/knowledge/_skill_selection.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import json +import posixpath +import re + +TASK_RELEVANCE = "task-lexical-v1" +RELEVANCE_METHODS = ("weighted-lexical-v1", "weighted-raw-string-v1", TASK_RELEVANCE) + + +class SkillIndexError(ValueError): + pass + + +def words(cue: str) -> set[str]: + return set(re.findall(r"\w+", cue.casefold())) + + +def lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: + tokens = words(cue) + return sum( + weight * sum(token in json.dumps(value, ensure_ascii=False).casefold() for token in tokens) + for weight, value in fields + ) + + +def raw_string_lexical_score(cue: str, fields: list[tuple[int, object]]) -> int: + """Match original strings; retain legacy JSON matching for nonstring fields.""" + tokens = words(cue) + score = 0 + for weight, value in fields: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + score += weight * sum(token in text.casefold() for token in tokens) + return score + + +def indexed_skill(record: dict, view: dict, content: dict[str, bytes]) -> tuple: + """Validate the existing projection against this exact canonical bundle.""" + registry_path = "data/skill_library/skill-index.json" + registry = json.loads(content[registry_path]) + name = record["skill_metadata"]["name"] + entry = registry["skills"][name] + prefix = f"data/skill_library/{entry['path']}/" + core = prefix + "SKILL.md" + expected_id = f"inferrsi.{entry['backend']}.skills.{name.replace('-', '_')}" + if ( + registry["schema_version"] != 1 + or view["authority"] != "package" + or record["id"] != view["id"] + or record["id"] != expected_id + or record["backend"] != entry["backend"] + or record["skill_documents"] != {"original": core} + or posixpath.basename(entry["path"]) != name + ): + raise SkillIndexError("Packaged skill identity disagrees with the registry") + expected = { + "src/inferrsi/" + path: digest + for path, digest in view["resources"].items() + if path.startswith(prefix) or path == registry_path + } + if record["source_files"] != expected: + raise SkillIndexError("Stale packaged skill projection; rebuild with build_skill_index.py") + + required = [core] + metadata = record["skill_metadata"] + notices = [ + source["third_party_notices"] + for source in metadata.get("sources", []) if "third_party_notices" in source + ] + for relative in [ + *metadata.get("references", []), *metadata.get("provenance_readsets", []), *notices, + ]: + path = posixpath.normpath(prefix + relative) + if "\\" in relative or not path.startswith("data/skill_library/") or path not in content: + raise SkillIndexError(f"Unavailable declared skill reference: {relative}") + if path not in required: + required.append(path) + for path in sorted(content): + if path.startswith(prefix) and re.search( + r"(?:^|[-_])(?:LICENSE|NOTICES?)(?:\.[^.]+)?$", + posixpath.basename(path), re.IGNORECASE, + ) and path not in required: + required.append(path) + for resource in list(required): + directory = posixpath.dirname(resource) + while directory.startswith("data/skill_library/"): + license_path = directory + "/LICENSE" + if license_path in content and license_path not in required: + required.append(license_path) + directory = posixpath.dirname(directory) + optional = sorted( + path for path in content + if path.startswith(prefix) and path not in required + ) + return core, required, optional + + +def skill_score( + cue: str, record: dict, core: bytes, *, method: str = "weighted-lexical-v1" +) -> int: + if method not in ("weighted-lexical-v1", "weighted-raw-string-v1"): + raise ValueError("Unknown relevance method for skill-only ranking") + scorer = raw_string_lexical_score if method == "weighted-raw-string-v1" else lexical_score + return scorer(cue, [ + (4, record["title"]), + (3, record["summary"]), + (2, [record.get("tags", []), record.get("applicability", {})]), + (1, core.decode("utf-8")), + ]) + + +def task_score(cue: str, record: dict, core: bytes, backend: str | None) -> int: + """Rank whole task terms across both skill and repository knowledge owners.""" + def terms(text: str) -> set[str]: + return set(re.findall(r"[^\W_]+", text.casefold())) + + tokens = terms(cue) - {"a", "an", "and", "for", "in", "of", "on", "the", "to", "with"} + fields = ( + (4, record["title"]), + (3, record["summary"]), + (2, " ".join(record.get("tags", []))), + (2, record["repo"] + " " + record["scenario"]), + (1, json.dumps(record.get("applicability", {}), ensure_ascii=False)), + (1, core.decode("utf-8")), + ) + score = sum(weight * len(tokens & terms(text)) for weight, text in fields) + # With no platform in scope prefer portable procedures to a specialization + # that happens to repeat the same task words in its description. + platforms = { + "cuda": {"cuda", "nvidia"}, "rocm": {"rocm", "amd"}, + "ascend": {"ascend", "cann"}, "mlx": {"mlx"}, "cpu": {"cpu"}, + } + in_scope = {backend} if backend else {key for key, names in platforms.items() if names & tokens} + return score * (2 if record["backend"] == "common" or record["backend"] in in_scope else 1) + + +def record_context(record: dict, view: dict, content: dict[str, bytes], available: int) -> tuple: + """Keep the canonical package record including its sources and limits.""" + path = view["entrypoint"] + body = content[path] + if len(body) > available: + return None, {"ref": view["ref"], "reason": "context_limit", + "required_bytes": len(body), "available_bytes": available} + return { + "ref": view["ref"], "revision": view["revision"], "content_bytes": len(body), + "resources": [{"path": path, "sha256": view["resources"][path], "role": "record", + "locator": "whole-resource", "content": body.decode("utf-8")}], + "omissions": [{"path": p, "reason": "linked_document"} for p in + dict.fromkeys(record.get("skill_documents", {}).values())], + "sources": view["sources"], "attribution": view["attribution"], + "limitations": view["limitations"], + }, None + + +def context_view( + record: dict, view: dict, content: dict[str, bytes], cue: str, available: int +) -> tuple[dict | None, dict | None]: + core, required, optional = indexed_skill(record, view, content) + required_bytes = sum(len(content[path]) for path in required) + if required_bytes > available: + return None, { + "ref": view["ref"], "reason": "context_limit", + "required_bytes": required_bytes, "available_bytes": available, + } + + resources, omissions = [], [] + used = 0 + + def include(path: str) -> None: + nonlocal used + resources.append({ + "path": path, "sha256": view["resources"][path], "locator": "whole-resource", + "role": "core" if path == core else "reference", + "content": content[path].decode("utf-8"), + }) + used += len(content[path]) + + for path in required: + include(path) + candidates = [] + for path in optional: + if ( + posixpath.basename(path) == "metadata.json" + or posixpath.splitext(path)[1].lower() not in (".md", ".json", ".txt", ".csv") + ): + omissions.append({"path": path, "reason": "not_context_resource"}) + continue + try: + text = content[path].decode("utf-8") + except UnicodeDecodeError: + omissions.append({"path": path, "reason": "not_utf8"}) + continue + score = lexical_score(cue, [(2, path.rpartition("/")[2]), (1, text)]) + if words(cue) and score == 0: + omissions.append({"path": path, "reason": "not_relevant"}) + continue + candidates.append((score, path)) + for _, path in sorted(candidates, key=lambda item: (-item[0], item[1])): + if used + len(content[path]) > available: + omissions.append({"path": path, "reason": "context_limit"}) + continue + include(path) + return { + "ref": view["ref"], "revision": view["revision"], "resources": resources, + "content_bytes": used, "omissions": omissions, + "sources": view["sources"], "attribution": view["attribution"], + "limitations": view["limitations"] + [ + "Selected resources are complete files; this view is not a standalone Bundle.", + "Follow omitted or other linked resources through the exact owner revision.", + ], + }, None diff --git a/src/inferrsi/knowledge/_source_snapshot.py b/src/inferrsi/knowledge/_source_snapshot.py new file mode 100644 index 000000000..3ced2e330 --- /dev/null +++ b/src/inferrsi/knowledge/_source_snapshot.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import re +from typing import Any + +# One shared snapshot keeps the repository-wide source revision and registry digest out of +# per-owner records, so a one-owner edit regenerates only that owner's record. +MANIFEST = "_provenance.json" +REPOSITORY = "https://github.com/InferMatrixLab/InferRSI" +LIBRARY = "src/inferrsi/data/skill_library" +REGISTRY = f"{LIBRARY}/skill-index.json" +STORED_SCHEMA = 2 + + +def manifest(revision: str, registry_sha256: str) -> dict[str, Any]: + return { + "schema_version": 1, + "repository": REPOSITORY, + "source_revision": revision, + "registry": {"path": REGISTRY, "sha256": registry_sha256}, + } + + +def _snapshot(value: Any) -> tuple[str, str]: + if ( + not isinstance(value, dict) + or set(value) != {"schema_version", "repository", "source_revision", "registry"} + or type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or value["repository"] != REPOSITORY + or not isinstance(value["source_revision"], str) + or not re.fullmatch(r"[0-9a-f]{40}", value["source_revision"]) + ): + raise ValueError("Invalid knowledge source snapshot") + registry = value["registry"] + if ( + not isinstance(registry, dict) + or set(registry) != {"path", "sha256"} + or registry["path"] != REGISTRY + or not isinstance(registry["sha256"], str) + or not re.fullmatch(r"[0-9a-f]{64}", registry["sha256"]) + ): + raise ValueError("Invalid knowledge source snapshot registry") + return value["source_revision"], registry["sha256"] + + +def compact(record: dict[str, Any]) -> dict[str, Any]: + """Store a generated schema-v1 record without its snapshot-wide fields.""" + stored = {key: value for key, value in record.items() if key != "sources"} + stored["schema_version"] = STORED_SCHEMA + stored["source_files"] = { + path: digest for path, digest in record["source_files"].items() if path != REGISTRY + } + return stored + + +def resolve(stored: dict[str, Any], snapshot: Any) -> dict[str, Any]: + """Rebuild the public schema-v1 record from a stored record and the shared snapshot.""" + revision, registry_sha256 = _snapshot(snapshot) + if stored.get("schema_version") != STORED_SCHEMA or "sources" in stored: + raise ValueError("Stored skill record is not a schema-v2 projection") + files = stored.get("source_files") + documents = stored.get("skill_documents") + if ( + not isinstance(files, dict) or REGISTRY in files + or not isinstance(documents, dict) or not isinstance(documents.get("original"), str) + ): + raise ValueError(f"Invalid stored skill record: {stored.get('id')}") + entry = "src/inferrsi/" + documents["original"] + if entry not in files: + raise ValueError(f"Stored skill record lacks its entry hash: {stored.get('id')}") + hashes = dict(sorted({**files, REGISTRY: registry_sha256}.items())) + source = { + "type": "git", + "url": f"{REPOSITORY}/blob/{revision}/{entry}", + "commit": revision, + "paths": list(hashes), + "content_sha256": hashes[entry], + } + record = {} + for key, value in stored.items(): + record[key] = 1 if key == "schema_version" else value + if key == "origin": + record["sources"] = [source] + if "sources" not in record: + raise ValueError(f"Stored skill record lacks origin: {stored.get('id')}") + record["source_files"] = hashes + return record diff --git a/src/inferrsi/knowledge/catalog.py b/src/inferrsi/knowledge/catalog.py new file mode 100644 index 000000000..da5f417fe --- /dev/null +++ b/src/inferrsi/knowledge/catalog.py @@ -0,0 +1,168 @@ +"""Read-only package-data catalog. Knowledge text is never executable.""" + +from __future__ import annotations + +import json +import re +from importlib.resources import files +from typing import Any + +from .._repo_pack_data import REPO_TYPES +from . import _source_snapshot + +# Learning scope does not imply that a repository has a runnable target adapter. +# Legacy repository-pack records retain their contribution tier; an independent +# peer card can describe a different learning use case for the same source repo. +KNOWLEDGE_SCOPES = { + "contribution": (*REPO_TYPES, "vllm_ascend", "inferrsi"), + "dependency": ( + "flashinfer", "flashattention", "pytorch", "triton", "diffusers", "transformers", "mlx" + ), + "peer": ( + "sglang", "tensorrt_llm", "lmdeploy", "xdit", "cache_dit", "qwen3_tts_engine", "deepinfer" + ), +} +KNOWLEDGE_REPOS = tuple( + dict.fromkeys(repo for repos in KNOWLEDGE_SCOPES.values() for repo in repos) +) +BACKENDS = ("cpu", "cuda", "rocm", "ascend", "mlx") +SCENARIO_ALIASES = { + "diffusion.t2i": "diffusion.image_generation", + "diffusion.t2v": "diffusion.video_generation", +} + + +def _walk(root, prefix=()): + for child in sorted(root.iterdir(), key=lambda item: item.name): + parts = (*prefix, child.name) + if child.is_dir(): + yield from _walk(child, parts) + else: + yield parts, child + + +def _validate_record( + record: dict, repo: str, parts: tuple[str, ...], tier: str = "contribution" +) -> None: + required = ("id", "title", "repo", "backend", "component", "scenario", "status", "summary") + if not isinstance(record, dict) or any( + not isinstance(record.get(key), str) or not record[key].strip() for key in required + ): + raise ValueError(f"Incomplete knowledge record: {repo}/{'/'.join(parts)}") + if record.get("schema_version") != 1: + raise ValueError(f"Unsupported knowledge schema: {record['id']}") + # Existing schema-v1 records retain their IDs and default to contribution. + if record.get("tier", "contribution") != tier: + raise ValueError(f"Knowledge tier disagrees with path: {record['id']}") + if len(parts) != 3: + raise ValueError("Knowledge path must be BACKEND/COMPONENT/ENTRY.json") + backend, component, filename = parts + if backend not in (*BACKENDS, "common"): + raise ValueError(f"Unknown knowledge backend: {backend}") + if (record["repo"], record["backend"], record["component"]) != (repo, backend, component): + raise ValueError(f"Knowledge scope disagrees with path: {record['id']}") + expected_id = f"{repo}.{backend}.{component}.{filename[:-5].replace('-', '_')}" + if record["id"] != expected_id: + raise ValueError(f"Knowledge ID disagrees with path: {record['id']}") + if not record["scenario"].startswith(component + "."): + raise ValueError(f"Knowledge scenario disagrees with component: {record['id']}") + sources = record.get("sources") + if not isinstance(sources, list) or not sources: + raise ValueError(f"Knowledge has no source: {record['id']}") + for source in sources: + if ( + not isinstance(source, dict) + or not isinstance(source.get("url"), str) + or not source["url"].startswith("https://") + or not isinstance(source.get("commit"), str) + or not re.fullmatch(r"[0-9a-f]{40}", source["commit"]) + ): + raise ValueError(f"Knowledge requires source attribution and revision: {record['id']}") + verification = record.get("verification", {}) + if not isinstance(verification, dict) or not isinstance( + verification.get("locally_verified"), bool + ): + raise ValueError(f"Knowledge requires an explicit verification state: {record['id']}") + if record["status"] == "imported" and verification["locally_verified"]: + raise ValueError(f"Imported knowledge must state locally_verified=false: {record['id']}") + + +def knowledge_records() -> list[dict[str, Any]]: + data = files("inferrsi").joinpath("data") + records = [] + seen = set() + for tier, repos in KNOWLEDGE_SCOPES.items(): + for repo in repos: + knowledge = ( + data.joinpath("repo_packs", repo, "knowledge") + if tier == "contribution" + else data.joinpath("knowledge_packs", tier, repo) + ) + if not knowledge.is_dir(): + continue + snapshot = knowledge.joinpath(_source_snapshot.MANIFEST) + snapshot = ( + json.loads(snapshot.read_text(encoding="utf-8")) if snapshot.is_file() else None + ) + for parts, item in _walk(knowledge): + if not item.name.endswith(".json") or parts == (_source_snapshot.MANIFEST,): + continue + record = json.loads(item.read_text(encoding="utf-8")) + # Generated skill records store schema v2; readers always receive schema v1. + if isinstance(record, dict) and record.get("schema_version") == ( + _source_snapshot.STORED_SCHEMA + ): + if snapshot is None: + raise ValueError(f"Knowledge record lacks its source snapshot: {repo}") + record = _source_snapshot.resolve(record, snapshot) + _validate_record(record, repo, parts, tier) + if record["id"] in seen: + raise ValueError(f"Duplicate knowledge ID: {record['id']}") + seen.add(record["id"]) + records.append(record) + return records + + +def search_knowledge( + query: str = "", + *, + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, +) -> list[dict[str, Any]]: + if limit < 1: + raise ValueError("limit must be positive") + if tier is not None and tier not in KNOWLEDGE_SCOPES: + raise ValueError(f"Unknown knowledge tier: {tier}") + scenario = SCENARIO_ALIASES.get(scenario, scenario) + tokens = re.findall(r"[\w.-]+", query.casefold()) + scored = [] + for record in knowledge_records(): + if tier and record.get("tier", "contribution") != tier: + continue + if repo and record["repo"] != repo: + continue + if backend and record["backend"] not in (backend, "common"): + continue + if scenario: + declared = record.get("scenario", "") + if declared != scenario and not ( + declared.endswith(".*") and scenario.startswith(declared[:-1]) + ): + continue + document = json.dumps(record, ensure_ascii=False).casefold() + if any(token not in document for token in tokens): + continue + score = sum(document.count(token) for token in tokens) + scored.append((score, record["id"], record)) + scored.sort(key=lambda item: (-item[0], item[1])) + return [item[2] for item in scored[:limit]] + + +def get_knowledge(record_id: str) -> dict[str, Any]: + for record in knowledge_records(): + if record["id"] == record_id: + return record + raise ValueError(f"Unknown knowledge ID: {record_id}") diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py new file mode 100644 index 000000000..cff710c26 --- /dev/null +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -0,0 +1,735 @@ +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from dataclasses import dataclass +from datetime import date, datetime +from importlib.resources import files +from pathlib import Path, PurePosixPath + +from ._skill_selection import ( + RELEVANCE_METHODS, + TASK_RELEVANCE, + context_view, + indexed_skill, + record_context, + skill_score, + task_score, + words, +) +from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records + +AUTHORITIES = ("package", "checkout", "store") +IDENTITY = r"[a-z0-9][a-z0-9._-]*" + + +class RevisionConflict(ValueError): + pass + + +class IdentityError(ValueError): + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + + +def safe_path(value: str) -> str: + path = PurePosixPath(value) + if ( + not value + or "\\" in value + or ":" in value + or path.is_absolute() + or any(part in ("", ".", "..") for part in value.split("/")) + ): + raise ValueError(f"Unsafe bundle path: {value}") + return value + + +def _json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + + +def _normalized(value: object, field: str, ancestors: frozenset[int] = frozenset()) -> object: + # Bundle identity is JSON, so YAML scalars without a JSON form are rejected + # here instead of failing inside revision serialization. + if value is None or isinstance(value, (str, bool, int, float)): + return value + if isinstance(value, (datetime, date)): + return value.isoformat() + if isinstance(value, (list, dict)): + # A YAML alias can make a container its own descendant. Only the active path + # is tracked, so the same object shared by sibling fields stays accepted. + identity = id(value) + if identity in ancestors: + raise ValueError( + f"Field {field} contains a reference cycle: {type(value).__name__}" + ) + ancestors = ancestors | {identity} + if isinstance(value, list): + return [ + _normalized(item, f"{field}[{index}]", ancestors) + for index, item in enumerate(value) + ] + result = {} + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError(f"Field {field} has a non-string key: {key!r}") + result[key] = _normalized(item, f"{field}.{key}" if field else key, ancestors) + return result + raise ValueError( + f"Field {field or ''} cannot be represented in the bundle identity: " + f"{type(value).__name__}" + ) + + +def _hash(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _sources(records: list[dict]) -> list[dict]: + result = [] + for record in records: + source = dict(record) + kind = source.get("type") + if kind in ("source", "source_code", "technical_document") and "commit" in source: + source["original_type"] = kind + kind = "git" + if kind is None and "commit" in source: + kind = "git" + if kind not in ("git", "paper", "document", "internal", "unknown"): + # Legacy types keep their original label; nothing is inferred from them. + source["original_type"] = kind + kind = "unknown" + source["type"] = kind + if kind == "git": + commit = source.get("commit") + if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{7,40}", commit): + raise ValueError("Git source requires a real commit or explicit historical prefix") + source["revision_precision"] = "full" if len(commit) == 40 else "prefix" + elif kind == "paper": + if not source.get("id") or "version" not in source: + raise ValueError("Paper source requires ID and explicit version (null if unknown)") + if "commit" in source: + raise ValueError("Paper versions are not Git commits") + elif kind == "document": + if not source.get("url") or not any(k in source for k in ("version", "sha256")): + raise ValueError("Document source requires URL and version or snapshot identity") + elif kind == "internal": + if not source.get("artifact") or not any(k in source for k in ("issue", "run")): + raise ValueError("Internal source requires artifact and issue/run identity") + result.append(source) + return result + + +@dataclass(frozen=True) +class Bundle: + metadata: dict + content: dict[str, bytes] + dependencies: dict[str, str] + + def __post_init__(self) -> None: + # Normalize once at ingestion so manifest, revision and views agree. + object.__setattr__(self, "metadata", _normalized(self.metadata, "metadata")) + + def manifest(self) -> dict: + record = self.metadata + if not isinstance(record.get("id"), str) or not re.fullmatch(IDENTITY, record["id"]): + raise ValueError("Invalid bundle identity") + if record.get("kind") not in ("skill", "knowledge", "case"): + raise ValueError("Invalid bundle kind") + for field in ( + "sources", + "applicability", + "state", + "limitations", + "attribution", + "entrypoint", + ): + if field not in record: + raise ValueError(f"Missing normalized field: {field}") + for field in ("state", "entrypoint"): + if not isinstance(record[field], str) or not record[field]: + raise ValueError(f"Normalized {field} must be a nonempty string") + if not isinstance(record["limitations"], list): + raise ValueError("Normalized limitations must be a list") + if not isinstance(record["sources"], list) or not record["sources"]: + raise ValueError("Bundle requires sources, including explicit unknown provenance") + _sources(record["sources"]) + for path, data in self.content.items(): + safe_path(path) + if not isinstance(data, bytes): + raise ValueError("Bundle content must be bytes") + if record["entrypoint"] not in self.content: + raise ValueError("Missing entrypoint in complete bundle") + for required in record.get("required_resources", []): + # Present string keys already passed the content path validation. + if isinstance(required, str) and required in self.content: + continue + if safe_path(required) not in self.content: + raise ValueError(f"Missing required resource: {required}") + resources = {path: _hash(data) for path, data in sorted(self.content.items())} + for dependency, revision in self.dependencies.items(): + safe_path(dependency) + if not re.fullmatch(r"sha256:[0-9a-f]{64}", revision): + raise ValueError("Dependency requires a resolved content revision") + if dependency in self.content: + actual = "sha256:" + resources[dependency] + else: + members = { + path: digest + for path, digest in resources.items() + if path.startswith(dependency + "/") + } + if not members: + raise ValueError(f"Missing dependency: {dependency}") + actual = "sha256:" + _hash(_json(members)) + if actual != revision: + raise RevisionConflict(f"Dependency bytes disagree with revision: {dependency}") + return { + "metadata": record, + "resources": resources, + "dependencies": dict(sorted(self.dependencies.items())), + } + + @property + def revision(self) -> str: + return "sha256:" + _hash(_json(self.manifest())) + + def view(self, *, material: bool = False) -> dict: + manifest = self.manifest() + result = { + **self.metadata, + "revision": "sha256:" + _hash(_json(manifest)), + "resources": manifest["resources"], + "dependencies": manifest["dependencies"], + } + if material: + result["content"] = self.content[self.metadata["entrypoint"]].decode("utf-8") + return result + + +INCIDENTAL = ("__pycache__", ".DS_Store") + + +def _tree( + node, prefix: str, *, root: Path | None = None, ancestors: frozenset[Path] = frozenset() +) -> dict[str, bytes]: + if isinstance(node, Path): + # A junction or symlink resolves away from where it appears; every descended + # directory must still land inside the root that owns this walk. + directory = node.resolve() + if root is None: + root = directory + if directory in ancestors: + raise ValueError(f"Resource cycle inside the skills root: {prefix}") + ancestors = ancestors | {directory} + result = {} + for child in sorted(node.iterdir(), key=lambda p: p.name): + if child.name in INCIDENTAL or child.name.endswith(".pyc"): + continue + path = safe_path(f"{prefix}/{child.name}") + if isinstance(child, Path) and child.is_symlink(): + raise ValueError(f"Symlink resource is not supported: {path}") + if child.is_dir(): + if isinstance(child, Path) and root is not None and not child.resolve().is_relative_to( + root + ): + raise ValueError(f"Resource escapes the skills root: {path}") + result.update(_tree(child, path, root=root, ancestors=ancestors)) + else: + result[path] = child.read_bytes() + return result + + +class KnowledgeReader: + def __init__(self, *, skill_root: Path | None = None, store=None): + self.skill_root = skill_root.resolve() if skill_root is not None else None + if store is not None and store.skill_root != self.skill_root: + raise ValueError("Store must reserve identities from the same checkout") + self.store = store + self._records: dict[str, dict] | None = None + self._library: tuple[dict[str, bytes], str] | None = None + + def records(self) -> dict[str, dict]: + """Installed catalog records, read once per reader (one package snapshot).""" + if self._records is None: + self._records = {record["id"]: record for record in knowledge_records()} + return self._records + + def _skill_library(self) -> tuple[dict[str, bytes], str]: + if self._library is None: + root = files("inferrsi").joinpath("data", "skill_library") + library = _tree(root, "data/skill_library") + digest = _hash(_json({path: _hash(data) for path, data in sorted(library.items())})) + self._library = (library, "sha256:" + digest) + return self._library + + def owners(self) -> dict[str, list[str]]: + """Map each public ID to every authority that owns it.""" + result: dict[str, list[str]] = {} + for identity in self.records(): + result.setdefault(identity, []).append("package") + if self.skill_root is not None: + # Enumeration reads registered names only. Validating sidecar content + # here would abort the whole collection over one malformed, possibly + # unrequested, skill; Collect reports that per item through _read. + from .skill_catalog import skill_directories + + local = list(skill_directories(self.skill_root)) + local.extend(self._cases()) + if len(set(local)) != len(local): + raise IdentityError("collision", "Duplicate identity inside the checkout") + for identity in local: + result.setdefault(identity, []).append("checkout") + if self.store is not None: + for identity in self.store.identities(): + result.setdefault(identity, []).append("store") + return result + + def identities(self) -> list[str]: + return sorted(f"{a}:{i}" for i, authorities in self.owners().items() for a in authorities) + + def resolve(self, reference: str, owners: dict[str, list[str]] | None = None) -> str: + """Return the qualified `authority:id`; ambiguity is an error, never precedence.""" + authority, qualified, identity = reference.rpartition(":") + if not re.fullmatch(IDENTITY, identity) or (qualified and authority not in AUTHORITIES): + raise IdentityError("invalid_identity", f"Invalid identity: {reference}") + found = (self.owners() if owners is None else owners).get(identity, []) + if qualified: + if authority not in found: + raise IdentityError("unknown_identity", f"Unknown identity: {reference}") + return reference + if not found: + raise IdentityError("unknown_identity", f"Unknown identity: {identity}") + if len(found) > 1: + choices = ", ".join(f"{a}:{identity}" for a in found) + raise IdentityError( + "ambiguous_identity", f"Ambiguous identity {identity}; qualify one of: {choices}" + ) + return f"{found[0]}:{identity}" + + def _cases(self) -> dict[str, Path]: + result = {} + if self.skill_root is not None: + from .skill_catalog import skill_directories + + for directory in skill_directories(self.skill_root).values(): + for path in sorted(directory.glob("cases/*.json")): + if path.is_symlink() or not path.resolve().is_relative_to(self.skill_root): + raise ValueError("Case must stay inside the selected skills root") + record = json.loads(path.read_text(encoding="utf-8")) + if record.get("kind") == "case": + identity = record["id"] + if not isinstance(identity, str) or not re.fullmatch(IDENTITY, identity): + raise ValueError(f"Invalid case identity in {path.name}: {identity!r}") + if identity in result: + raise ValueError(f"Duplicate case identity: {identity}") + result[identity] = path + return result + + def read(self, reference: str, *, revision: str | None = None) -> Bundle: + return self._read(self.resolve(reference), revision) + + def _read(self, qualified: str, revision: str | None) -> Bundle: + authority, _, identity = qualified.partition(":") + if authority == "store": + return self.store.read(identity, revision=revision) + bundle = self._checkout(identity) if authority == "checkout" else self._knowledge(identity) + if revision is not None and bundle.revision != revision: + raise RevisionConflict( + f"Revision unavailable in this snapshot: {qualified} at {revision}" + ) + return bundle + + def _checkout(self, identity: str) -> Bundle: + cases = self._cases() + if identity in cases: + path = cases[identity] + record = json.loads(path.read_text(encoding="utf-8")) + for field in ("references", "dependencies"): + if field in record: + raise ValueError(f"Case {field} are not supported; declare required_resources") + declared = record.get("required_resources", []) + if not isinstance(declared, list) or any(not isinstance(p, str) for p in declared): + raise ValueError("Case required_resources must be a string list") + if len(set(declared)) != len(declared): + raise ValueError("Case required_resources must not repeat a path") + relative = safe_path(record["content"]) + required = [relative, "case.json", *(p for p in declared if p != relative)] + content = {"case.json": path.read_bytes()} + for resource in [relative, *required[2:]]: + safe_path(resource) + if resource == "case.json": + raise ValueError("Case resource collides with its case.json record") + resource_path = path.parent / resource + if resource_path.is_symlink() or not resource_path.resolve().is_relative_to( + self.skill_root + ): + raise ValueError(f"Case resource escapes the skills root: {resource}") + if not resource_path.is_file(): + raise ValueError(f"Missing case resource: {resource}") + content[resource] = resource_path.read_bytes() + return Bundle( + { + **record, + "entrypoint": relative, + "sources": _sources(record["sources"]), + "required_resources": required, + }, + content, + {}, + ) + return self._skill(identity) + + def _knowledge(self, identity: str) -> Bundle: + record = self.records().get(identity) + if record is None: + raise IdentityError("unknown_identity", f"Unknown knowledge ID: {identity}") + content = {"record.json": _json(record)} + dependencies = {} + if record.get("skill_documents"): + # The whole imported library is one explicit dependency so bilingual + # references and licenses stay a complete closure. + library, digest = self._skill_library() + content.update(library) + dependencies["data/skill_library"] = digest + documents = record["skill_documents"] + if not isinstance(documents, dict): + raise ValueError(f"skill_documents must map editions to paths: {identity}") + for target in documents.values(): + if not isinstance(target, str) or safe_path(target) not in library: + raise ValueError(f"Missing declared skill document: {target}") + return Bundle( + { + "id": identity, + "kind": "knowledge", + "entrypoint": "record.json", + "origin": record.get("origin", ["external"]), + "sources": _sources(record["sources"]), + "applicability": record.get("applicability", {}), + "state": record["status"], + "limitations": record.get("limitations", []) + + [ + "Imported guidance; reading does not validate or execute it.", + "Historical revisions require the matching installed snapshot " + "or a retained store.", + ], + "attribution": record["sources"], + "verification": record["verification"], + "tier": record.get("tier", "contribution"), + "repo": record["repo"], + "backend": record["backend"], + "scenario": record["scenario"], + "title": record["title"], + "skill_documents": record.get("skill_documents", {}), + "required_resources": sorted(content), + }, + content, + dependencies, + ) + + def _skill(self, identity: str) -> Bundle: + from .skill_catalog import get_skill, skill_directory, skill_metadata, skill_reference + + root = self.skill_root + record = get_skill(root, identity) + directory = skill_directory(root, identity) + prefix = directory.relative_to(root).as_posix() + content = _tree(directory, prefix, root=root) + metadata = skill_metadata(root, identity) + dependencies = {} + for reference in metadata.get("references", []): + path = skill_reference(root, identity, reference) + key = path.relative_to(root).as_posix() + content[key] = path.read_bytes() + dependencies[key] = "sha256:" + _hash(content[key]) + return Bundle( + { + "id": identity, + "kind": "skill", + "entrypoint": f"{prefix}/SKILL.md", + "title": record["description"], + "origin": metadata.get("origin", ["unknown"]), + "sources": _sources(metadata.get("sources", [])) + or [ + { + "type": "unknown", + "note": "Legacy provenance remains in procedure/readsets", + } + ], + "applicability": metadata.get("applicability", {}), + "state": record.get("status", "unspecified"), + "limitations": metadata.get("limitations", []) + + [ + "Checkout-only local skill; no automatic execution or promotion.", + "Only declared references are required; " + "other cross-skill links are navigation.", + "Historical revisions require a matching checkout or a retained store.", + ], + "attribution": metadata.get("sources", []), + "empirical_validation": metadata.get("empirical_validation", "unknown"), + "required_resources": sorted(content), + }, + content, + dependencies, + ) + + def resource(self, identity: str, path: str, *, revision: str) -> bytes: + safe_path(path) + bundle = self.read(identity, revision=revision) + if path not in bundle.content: + raise ValueError(f"Unknown resource: {path}") + return bundle.content[path] + + def collect( + self, + query: str = "", + *, + ids: list[str] | None = None, + revisions: dict[str, str] | None = None, + kind: str | None = None, + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, + max_bytes: int | None = None, + material: bool = False, + relevance: str | None = None, + relevance_method: str = "weighted-lexical-v1", + context_bytes: int | None = None, + ) -> dict: + if type(limit) is not int or limit < 1: + raise ValueError("limit must be a positive integer") + if max_bytes is not None and (type(max_bytes) is not int or max_bytes < 1): + raise ValueError("max_bytes must be a positive integer") + if ids is not None and ( + not isinstance(ids, list) or any(not isinstance(identity, str) for identity in ids) + ): + raise ValueError("ids must be a list of identity strings") + if kind not in (None, "skill", "knowledge", "case"): + raise ValueError("Unknown content kind") + if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): + raise ValueError("relevance requires at least one word") + if relevance_method not in RELEVANCE_METHODS: + raise ValueError("Unknown relevance method") + if relevance is None and relevance_method != "weighted-lexical-v1": + raise ValueError("relevance_method requires relevance") + if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): + raise ValueError("context_bytes must be a positive integer") + if revisions is None: + revisions = {} + if not isinstance(revisions, dict) or any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in revisions.items() + ): + raise ValueError("revisions must map identity strings to revision strings") + if ids is not None and len(set(ids)) != len(ids): + raise ValueError("Duplicate requested identity") + owners = self.owners() + requested = ids if ids is not None else self.identities() + + def qualified(reference: str) -> str | None: + # Revision keys may use any spelling the API accepts for an identity. + try: + return self.resolve(reference, owners) + except IdentityError: + return None + + selected = {identity for identity in map(qualified, requested) if identity is not None} + pinned: dict[str, str] = {} + for reference, revision in revisions.items(): + target = qualified(reference) + if target is None or target not in selected: + raise ValueError( + f"Revision requested for an unselected identity: {reference}" + ) + if pinned.setdefault(target, revision) != revision: + raise ValueError(f"Conflicting revisions for the same identity: {reference}") + pinned[target] = revision + collisions = sorted(i for i, authorities in owners.items() if len(authorities) > 1) + if tier is not None and tier not in KNOWLEDGE_SCOPES: + raise ValueError(f"Unknown knowledge tier: {tier}") + scenario = SCENARIO_ALIASES.get(scenario, scenario) + package_filters = any(value is not None for value in (repo, tier, backend, scenario)) + task_ranking = relevance_method == TASK_RELEVANCE + + def matches(metadata: dict, authority: str) -> bool: + declared = metadata.get("scenario") + declared = declared if isinstance(declared, str) else "" + # The catalog records skills.discovery as the discovery operation, + # not the workload served by a procedure. Let the task cue rank + # these procedures alongside records declaring the requested workload. + discovery_skill = ( + task_ranking and ids is None and authority == "package" + and declared == "skills.discovery" + and self.records()[metadata["id"]].get("index_type") == "packaged-skill" + ) + return not ( + (kind and metadata.get("kind") != kind) + or (package_filters and authority != "package") + or (repo and metadata.get("repo") != repo) + or (tier and metadata.get("tier", "contribution") != tier) + or (backend and metadata.get("backend") not in (backend, "common")) + or not ( + scenario is None + or declared == scenario + or (declared.endswith(".*") and scenario.startswith(declared[:-1])) + or discovery_skill + ) + ) + + cases = self._cases() if kind else {} + selected, omitted, seen = [], [], set() + used = 0 + ranked, contexts, context_omissions = [], [], [] + context_used = 0 + + def select(reference: str, bundle: Bundle, view: dict) -> None: + nonlocal used, context_used + # Normal Bundle views were serialized while computing their revision. + # Preserve custom subclasses' serialization/error behavior below. + if type(bundle) is Bundle and len(selected) >= limit: + omitted.append({"id": reference, "reason": "output_limit"}) + return + size = len(_json(view)) + if len(selected) >= limit or (max_bytes is not None and used + size > max_bytes): + omitted.append({"id": reference, "reason": "output_limit"}) + return + selected.append(view) + used += size + if context_bytes is None: + return + record = self.records().get(view["id"], {}) + is_skill = record.get("index_type") == "packaged-skill" + if view["authority"] != "package" or (not is_skill and not task_ranking): + context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) + return + try: + available = context_bytes - context_used + if is_skill: + context, omission = context_view( + record, view, bundle.content, relevance or query, available + ) + else: + context, omission = record_context(record, view, bundle.content, available) + except (ValueError, KeyError, TypeError) as exc: + context_omissions.append({ + "ref": view["ref"], "reason": "index_unavailable", "detail": str(exc), + }) + return + if omission is not None: + context_omissions.append(omission) + else: + contexts.append(context) + context_used += context["content_bytes"] + + for reference in requested: + try: + qualified = self.resolve(reference, owners) + if qualified in seen: + raise IdentityError("duplicate_identity", f"Already selected: {qualified}") + seen.add(qualified) + authority, _, identity = qualified.partition(":") + # Cheap metadata checks first, so filtered items never load resource bytes. + if authority == "package": + early = {**self.records()[identity], "kind": "knowledge"} + elif authority == "checkout": + early = {"kind": "case" if identity in cases else "skill"} + else: + early = None + if relevance is not None and ids is None and ( + authority != "package" + or (not task_ranking and early.get("index_type") != "packaged-skill") + ): + continue + if early is not None and not matches(early, authority): + if ids is not None: + omitted.append({"id": reference, "reason": "filters"}) + continue + bundle = self._read(qualified, pinned.get(qualified)) + view = {**bundle.view(material=material), "authority": authority, "ref": qualified} + entry = bundle.content[bundle.metadata["entrypoint"]] + haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() + except IdentityError as exc: + omitted.append({"id": reference, "reason": exc.code, "detail": str(exc)}) + continue + except RevisionConflict as exc: + omitted.append( + {"id": reference, "reason": "unavailable_revision", "detail": str(exc)} + ) + continue + except (ValueError, KeyError, TypeError, AttributeError, OSError, sqlite3.Error) as exc: + omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) + continue + if not matches(view, authority): + if ids is not None: + omitted.append({"id": reference, "reason": "filters"}) + continue + if any(token not in haystack for token in query.casefold().split()): + if ids is not None: + omitted.append({"id": reference, "reason": "query"}) + continue + if relevance is not None and ids is None: + record = self.records().get(identity, {}) + try: + is_skill = record.get("index_type") == "packaged-skill" + if is_skill: + core, _, _ = indexed_skill(record, view, bundle.content) + body = bundle.content[core] + else: + body = entry + if task_ranking: + score = task_score(relevance, record, body, backend) + else: + score = skill_score(relevance, record, body, method=relevance_method) + except (ValueError, KeyError, TypeError) as exc: + omitted.append({ + "id": reference, "reason": "index_unavailable", "detail": str(exc), + }) + continue + if score: + ranked.append((score, reference, bundle, view)) + continue + select(reference, bundle, view) + for _, reference, bundle, view in sorted( + ranked, key=lambda item: (-item[0], item[3]["ref"]) + ): + select(reference, bundle, view) + result = { + "items": selected, + "omissions": omitted, + "no_match": not selected, + "collisions": collisions, + "material_bytes": used, + "limitations": [ + "No crawling, ranking model or conversation state; deterministic identity order.", + "max_bytes bounds item payloads, not omission/control metadata.", + "Read references must be resolved with their returned exact revision.", + ], + } + if relevance is not None: + result["selection"] = { + "method": relevance_method, "cue": relevance, + "scope": ("package-skills-and-records" if task_ranking + else "generated-packaged-skills"), + "ranking_applied": ids is None, + } + if task_ranking and ids is None: + result["selection"].update(matched_count=len(ranked), returned_count=len(selected)) + result["limitations"][0] = ( + "Deterministic lexical selection; explicit IDs retain caller order and membership." + ) + if context_bytes is not None: + result.update({ + "contexts": contexts, "context_omissions": context_omissions, + "context_bytes": context_used, + "context_budget": { + "max_bytes": context_bytes, "accounting": "utf8-resource-content", + "excludes": "provenance, omission and control metadata", + }, + }) + return result diff --git a/src/inferrsi/knowledge/knowledge_store.py b/src/inferrsi/knowledge/knowledge_store.py new file mode 100644 index 000000000..4cb585703 --- /dev/null +++ b/src/inferrsi/knowledge/knowledge_store.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import json +import re +import sqlite3 +from contextlib import closing +from importlib.resources import files +from pathlib import Path + +from .knowledge_operations import ( + AUTHORITIES, + IDENTITY, + Bundle, + IdentityError, + KnowledgeReader, + RevisionConflict, + _json, + safe_path, +) + + +def _protected(skill_root: Path | None) -> list[Path]: + package = Path(str(files("inferrsi"))).resolve() + roots = [package] + # A source, editable or exported tree (git archive, sdist) surrounds the package + # and holds evidence and both matrices; protect it even without a .git entry, + # because review snapshots carry no repository metadata. + for parent in package.parents: + if (parent / "pyproject.toml").is_file() or (parent / ".git").exists(): + roots.append(parent) + break + if skill_root is not None: + roots.append(Path(skill_root).resolve().parent) + return roots + + +class KnowledgeStore: + """One authorized writable store for store-owned candidate bundles. + + Package and checkout content stay read-only; their IDs are reserved here so a + store candidate cannot shadow them. Repository-owned amendments use PRs. + """ + + def __init__(self, workspace: Path, *, skill_root: Path | None, read_only: bool = False): + # The caller authorizes an existing dedicated directory; none is created or guessed. + workspace = Path(workspace) + if workspace.is_symlink() or not workspace.is_dir(): + raise ValueError("Store requires an existing authorized workspace directory") + self.workspace = workspace.resolve() + for root in _protected(skill_root): + if self.workspace.is_relative_to(root): + raise ValueError(f"Store workspace must be outside read-only sources: {root}") + self.skill_root = Path(skill_root).resolve() if skill_root is not None else None + self.adapters = KnowledgeReader(skill_root=self.skill_root) + self.read_only = read_only + self.database = self.workspace / "knowledge.sqlite3" + if self.database.is_symlink(): + raise ValueError("Store database cannot be a symlink") + if read_only: + # Reading an existing store must not create or migrate anything. + if not self.database.is_file(): + raise ValueError(f"Read-only store requires an existing database: {self.database}") + self._verify_reserved_skill_root() + return + with closing(self._connect()) as connection, connection: + connection.executescript( + "CREATE TABLE IF NOT EXISTS revisions (" + "id TEXT NOT NULL, revision TEXT NOT NULL, metadata TEXT NOT NULL, " + "dependencies TEXT NOT NULL, parent TEXT, reason TEXT NOT NULL, " + "evidence TEXT NOT NULL, PRIMARY KEY (id, revision));" + "CREATE TABLE IF NOT EXISTS resources (" + "id TEXT NOT NULL, revision TEXT NOT NULL, path TEXT NOT NULL, data BLOB NOT NULL, " + "PRIMARY KEY (id, revision, path), " + "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" + "CREATE TABLE IF NOT EXISTS current (id TEXT PRIMARY KEY, revision TEXT NOT NULL, " + "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" + "CREATE TABLE IF NOT EXISTS meta (" + "key TEXT PRIMARY KEY, value TEXT NOT NULL);" + ) + reserved = connection.execute( + "SELECT value FROM meta WHERE key='skill_root'" + ).fetchone() + if reserved is None: + connection.execute( + "INSERT INTO meta VALUES ('skill_root', ?)", (self._reserved_skill_root(),) + ) + elif reserved[0] != self._reserved_skill_root(): + raise ValueError( + f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " + "reopen it with the same skill_root" + ) + + def _reserved_skill_root(self) -> str: + # The reserved checkout is recorded so later opens cannot silently drop it; + # an empty value records a package-only store. + return str(self.skill_root) if self.skill_root is not None else "" + + def _verify_reserved_skill_root(self) -> None: + # Stores created before reservation recording have no meta row and stay usable. + with closing(self._connect()) as connection: + present = connection.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'" + ).fetchone() + reserved = ( + connection.execute("SELECT value FROM meta WHERE key='skill_root'").fetchone() + if present + else None + ) + if reserved is not None and reserved[0] != self._reserved_skill_root(): + raise ValueError( + f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " + "open it with the same --skill-root, or without one for a package-only store" + ) + + def _connect(self): + if self.database.is_symlink() or self.database.resolve().parent != self.workspace: + raise ValueError("Store database escapes the authorized workspace") + if self.read_only: + # mode=ro fails every write at the SQLite level; as_uri() percent-encodes + # spaces and non-ASCII so the file: URI stays valid on Windows too. + connection = sqlite3.connect(self.database.as_uri() + "?mode=ro", uri=True, timeout=5) + else: + connection = sqlite3.connect(self.database, timeout=5) + connection.execute("PRAGMA foreign_keys=ON") + return connection + + def identities(self) -> list[str]: + with closing(self._connect()) as connection: + return [row[0] for row in connection.execute("SELECT id FROM current ORDER BY id")] + + def reader(self) -> KnowledgeReader: + return KnowledgeReader(skill_root=self.skill_root, store=self) + + def collect(self, query: str = "", **options) -> dict: + return self.reader().collect(query, **options) + + def create(self, bundle: Bundle) -> str: + if self.read_only: + raise ValueError("Store is read-only; create requires a writable store") + return self._publish(bundle, expected=None, reason="create", evidence=[]) + + def update(self, bundle: Bundle, *, expected: str, reason: str, evidence: list[dict]) -> str: + if self.read_only: + raise ValueError("Store is read-only; update requires a writable store") + if not expected or not reason.strip(): + raise ValueError("Update requires expected old revision and reason") + if not isinstance(evidence, list): + raise ValueError("Update evidence must be a list; an empty list means none supplied") + return self._publish(bundle, expected=expected, reason=reason, evidence=evidence) + + def _lineage(self, metadata: dict) -> None: + lineage = metadata.get("derived_from", []) + if not isinstance(lineage, list): + raise ValueError("derived_from must be a list of qualified source revisions") + for source in lineage: + if not isinstance(source, dict) or not isinstance(source.get("ref"), str): + raise ValueError("derived_from entries require a qualified ref") + authority, _, identity = source["ref"].partition(":") + if authority not in AUTHORITIES or not re.fullmatch(IDENTITY, identity): + raise ValueError(f"derived_from ref must be :: {source['ref']}") + revision = source.get("revision") + if revision is not None and ( + not isinstance(revision, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", revision) + ): + raise ValueError("derived_from revision must be a content revision or null") + if identity == metadata["id"]: + raise IdentityError( + "collision", "An adaptation needs a distinct identity from its source" + ) + + def _publish(self, bundle: Bundle, *, expected: str | None, reason: str, evidence: list) -> str: + # Copy caller data, validate it, then store the whole bundle and current + # pointer in one transaction. + bundle = Bundle( + json.loads(_json(bundle.metadata)), dict(bundle.content), dict(bundle.dependencies) + ) + revision = bundle.revision + identity = bundle.metadata["id"] + try: + bundle.content[bundle.metadata["entrypoint"]].decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("Bundle entrypoint must be UTF-8 text") from exc + self._lineage(bundle.metadata) + owners = self.adapters.owners().get(identity) + if owners: + raise IdentityError( + "collision", + f"Identity {identity} is owned by read-only {', '.join(owners)} content; " + "amend it through a repository PR or create an adaptation with a distinct ID", + ) + if expected is None and bundle.metadata["state"] != "candidate": + raise ValueError( + "Create registers a candidate; imported or active content is not promoted" + ) + try: + with closing(self._connect()) as connection, connection: + connection.execute("BEGIN IMMEDIATE") + current = connection.execute( + "SELECT revision FROM current WHERE id=?", (identity,) + ).fetchone() + if expected is None: + if current is not None: + raise RevisionConflict(f"Duplicate identity: {identity}") + elif current is None or current[0] != expected: + raise RevisionConflict(f"Stale or missing revision for {identity}") + if expected is not None: + previous = connection.execute( + "SELECT metadata FROM revisions WHERE id=? AND revision=?", + (identity, expected), + ).fetchone() + old = json.loads(previous[0]) + if bundle.metadata["state"] != old["state"]: + raise ValueError("Update cannot promote content state") + if revision == expected: + raise ValueError("Update must change the bundle") + if connection.execute( + "SELECT 1 FROM revisions WHERE id=? AND revision=?", (identity, revision) + ).fetchone(): + raise RevisionConflict( + f"Update reproduces retained revision {revision}; " + "change the bundle so the successor has its own revision" + ) + connection.execute( + "INSERT INTO revisions VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + identity, + revision, + _json(bundle.metadata).decode(), + _json(bundle.dependencies).decode(), + expected, + reason, + _json(evidence).decode(), + ), + ) + connection.executemany( + "INSERT INTO resources VALUES (?, ?, ?, ?)", + [(identity, revision, path, data) for path, data in bundle.content.items()], + ) + connection.execute( + "INSERT INTO current VALUES (?, ?) " + "ON CONFLICT(id) DO UPDATE SET revision=excluded.revision", + (identity, revision), + ) + except sqlite3.OperationalError as exc: + if "locked" not in str(exc) and "busy" not in str(exc): + raise + raise RevisionConflict( + f"Store is locked by another writer; retry {identity} once it is released" + ) from exc + return revision + + def read(self, identity: str, *, revision: str | None = None) -> Bundle: + with closing(self._connect()) as connection: + if revision is None: + row = connection.execute( + "SELECT revision FROM current WHERE id=?", (identity,) + ).fetchone() + if row is None: + raise IdentityError("unknown_identity", f"Unknown identity: store:{identity}") + revision = row[0] + row = connection.execute( + "SELECT metadata, dependencies FROM revisions WHERE id=? AND revision=?", + (identity, revision), + ).fetchone() + if row is None: + raise RevisionConflict(f"Unknown revision: store:{identity} at {revision}") + resources = connection.execute( + "SELECT path, data FROM resources WHERE id=? AND revision=?", (identity, revision) + ).fetchall() + bundle = Bundle(json.loads(row[0]), dict(resources), json.loads(row[1])) + if bundle.revision != revision: + raise RevisionConflict("Stored bundle integrity mismatch") + return bundle + + def resource(self, identity: str, path: str, *, revision: str) -> bytes: + safe_path(path) + bundle = self.read(identity, revision=revision) + if path not in bundle.content: + raise ValueError(f"Unknown resource: {path}") + return bundle.content[path] + + def history(self, identity: str) -> list[dict]: + with closing(self._connect()) as connection: + rows = connection.execute( + "SELECT revision, parent, reason, evidence FROM revisions " + "WHERE id=? ORDER BY rowid", + (identity,), + ).fetchall() + return [ + { + "revision": revision, + "parent": parent, + "reason": reason, + "evidence": json.loads(evidence), + } + for revision, parent, reason, evidence in rows + ] diff --git a/src/inferrsi/knowledge/research.py b/src/inferrsi/knowledge/research.py new file mode 100644 index 000000000..fc0ce3dfc --- /dev/null +++ b/src/inferrsi/knowledge/research.py @@ -0,0 +1,127 @@ +"""Read-only model-aware discovery; target annotations are not runtime support.""" + +from __future__ import annotations + +import json +import math +from collections.abc import Sequence +from importlib.resources import files + +from .catalog import knowledge_records, search_knowledge + +PROCEDURE = ( + "data/skill_library/workflows/inference-engine-research/" + "references/model-aware-preparation.md" +) + + +def _profile(model: str) -> dict | None: + root = files("inferrsi").joinpath("data", "research_profiles") + for resource in sorted(root.iterdir(), key=lambda path: path.name): + if resource.name.endswith(".json"): + profile = json.loads(resource.read_text(encoding="utf-8")) + if model.casefold() in {name.casefold() for name in profile["aliases"]}: + return profile + return None + + +def research_knowledge( + model: str, + *, + frameworks: Sequence[str] = (), + tasks: Sequence[str] = (), + modalities: Sequence[str] = (), + mechanisms: Sequence[str] = (), + repo: str | None = None, + tier: str | None = None, + backend: str | None = None, + scenario: str | None = None, + limit: int = 10, + target_backend: str | None = None, + target_device: str | None = None, + target_memory_gib: float | None = None, +) -> dict: + """Union separate research queries, preserving every package filter and record. + + Round-robin selection gives each query a turn before another result from the + same query. It is discovery ordering, not an engine or evidence-quality rank. + """ + if not model.strip(): + raise ValueError("model must be nonempty") + if limit < 1: + raise ValueError("limit must be positive") + if target_memory_gib is not None and ( + not math.isfinite(target_memory_gib) or target_memory_gib <= 0 + ): + raise ValueError("target memory must be positive and finite GiB") + model = model.strip() + profile = _profile(model) + defaults = profile or {} + groups = { + "model": [model, *([profile["model"]] if profile else [])], + "framework": frameworks, + "task": [*defaults.get("tasks", []), *tasks], + "modality": [*defaults.get("modalities", []), *modalities], + "mechanism": [*defaults.get("mechanisms", []), *mechanisms], + } + filters = {"repo": repo, "tier": tier, "backend": backend, "scenario": scenario} + queries, streams, matches = [], [], {} + # Retrieve every scoped match before applying the shared output limit; an + # early per-query limit would lose attribution for overlapping records. + catalog_size = max(1, len(knowledge_records())) + for group, terms in groups.items(): + seen = set() + for term in terms: + term = term.strip() + if not term: + raise ValueError(f"{group} query must be nonempty") + if term.casefold() in seen: + continue + seen.add(term.casefold()) + query = {"group": group, "query": term} + records = search_knowledge(term, **filters, limit=catalog_size) + queries.append({**query, "matches": len(records)}) + streams.append(records) + for record in records: + matches.setdefault(record["id"], []).append(query) + selected = {} + for rank in range(max((len(stream) for stream in streams), default=0)): + for stream in streams: + if rank < len(stream): + record = stream[rank] + selected.setdefault(record["id"], record) + if len(selected) == limit: + break + if len(selected) == limit: + break + return { + "model": model, + "profile": profile, + "queries": queries, + "filters": filters, + "items": [ + {"record": record, "matched_queries": matches[identity]} + for identity, record in selected.items() + ], + "no_match": not selected, + "target": { + "backend": target_backend, "device": target_device, "memory_gib": target_memory_gib, + }, + "target_validation": "unverified", + "components": defaults.get("components", []), + "procedure_resource": PROCEDURE, + "worked_example_resource": defaults.get("worked_example_resource"), + "warnings": [] if profile else [ + "No packaged model profile: supply --task/--modality and verify the full component " + "inventory from the selected checkpoint before judging feasibility." + ], + "limitations": [ + "Queries search installed package records only; empty results do not rule out " + "external sources. Existing token-AND search and scope filters are unchanged.", + "Mechanism queries are research leads, not claims of model/backend support. " + "Retain each record's applicability, source revision and verification state.", + "Target annotations do not inspect hardware or add a runtime backend. Full weight " + "inventory, engine/format compatibility, peak residency and quality need evidence " + "under the linked preparation procedure; no engine is selected or executed.", + ], + } diff --git a/src/inferrsi/knowledge/skill_catalog.py b/src/inferrsi/knowledge/skill_catalog.py new file mode 100644 index 000000000..ad3ad25a2 --- /dev/null +++ b/src/inferrsi/knowledge/skill_catalog.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path, PurePosixPath +from typing import BinaryIO + +try: + import yaml +except ImportError as exc: # The base install stays PyYAML-free by contract. + raise ValueError( + "Reading checkout skills requires the 'skills' extra (PyYAML); install it " + "from your InferRSI checkout, replacing the placeholder path: " + "python -m pip install '[skills]'" + ) from exc + +from .catalog import BACKENDS + + +def _frontmatter(stream: BinaryIO) -> dict: + if stream.readline().strip() != b"---": + raise ValueError("SKILL.md must start with YAML frontmatter") + lines = [] + for line in stream: + if line.strip() == b"---": + break + lines.append(line) + else: + raise ValueError("Unclosed SKILL.md frontmatter") + try: + record = yaml.safe_load(b"".join(lines).decode("utf-8")) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid skill YAML: {exc}") from exc + if not isinstance(record, dict): + raise ValueError("Skill frontmatter must be a mapping") + return record + + +def _skill_name(name: str) -> None: + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name): + raise ValueError(f"Invalid skill name: {name}") + + +def _directory(root: Path, name: str, relative: str) -> Path: + _skill_name(name) + path = PurePosixPath(relative) + if ( + not relative + or "\\" in relative + or ":" in relative + or path.is_absolute() + or any(part in ("", ".", "..") for part in relative.split("/")) + ): + raise ValueError(f"Skill path must be a relative POSIX path: {relative}") + directory = (root / relative).resolve() + if not directory.is_relative_to(root): + raise ValueError("Skill must be inside the selected root") + if path.name != name or directory.name != name: + raise ValueError(f"Skill path basename disagrees with its name: {name}") + if not _entry(directory, "SKILL.md").is_file(): + raise ValueError(f"Unknown skill: {name}") + return directory + + +def _unique_fields(pairs: list[tuple[str, object]]) -> dict: + record = {} + for key, value in pairs: + if key in record: + raise ValueError(f"Duplicate skill index field: {key}") + record[key] = value + return record + + +def _indexed_directories(root: Path) -> dict[str, Path] | None: + index = _entry(root, "skill-index.json") + if not index.exists(): + return None + record = json.loads(index.read_text(encoding="utf-8"), object_pairs_hook=_unique_fields) + if ( + not isinstance(record, dict) + or type(record.get("schema_version")) is not int + or record["schema_version"] != 1 + or not isinstance(record.get("skills"), dict) + ): + raise ValueError("Skill index requires schema_version 1 and a skills mapping") + directories = {} + paths = set() + for name, entry in sorted(record["skills"].items()): + if ( + not isinstance(entry, dict) + or not isinstance(entry.get("path"), str) + or not isinstance(entry.get("backend"), str) + or entry["backend"] not in (*BACKENDS, "common") + ): + raise ValueError(f"Skill index requires a path and supported backend: {name}") + directory = _directory(root, name, entry["path"]) + if directory in paths: + raise ValueError(f"Duplicate skill index path: {entry['path']}") + directories[name] = directory + paths.add(directory) + return directories + + +def skill_directories(root: Path) -> dict[str, Path]: + """Resolve registered skill locations without reading instruction bodies.""" + root = root.resolve() + if not root.is_dir(): + raise ValueError(f"Skill root is not a directory: {root}") + indexed = _indexed_directories(root) + if indexed is not None: + return indexed + return { + path.parent.name: _directory(root, path.parent.name, path.parent.name) + for path in sorted(root.glob("*/SKILL.md")) + } + + +def skill_directory(root: Path, name: str) -> Path: + """Resolve one named skill in an indexed library or a legacy flat root.""" + _skill_name(name) + root = root.resolve() + indexed = _indexed_directories(root) + if indexed is None: + return _directory(root, name, name) + if name not in indexed: + raise ValueError(f"Unknown skill: {name}") + return indexed[name] + + +def _entry(directory: Path, filename: str) -> Path: + # The resolved entry file must stay inside the selected skill directory; + # reject symlinks that point outside it. + path = (directory / filename).resolve() + if not path.is_relative_to(directory): + raise ValueError(f"Skill {filename} must stay inside the selected skill: {directory.name}") + return path + + +def _sidecar(directory: Path) -> dict | None: + # metadata.json shares the directory-boundary loading path with SKILL.md. + path = _entry(directory, "metadata.json") + if not path.is_file(): + return None + metadata = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(metadata, dict): + raise ValueError(f"Skill metadata must be a mapping: {directory.name}") + return metadata + + +def _references(metadata: dict, name: str) -> list[str]: + # references must be a list of strings; a string value would degrade + # membership testing to substring matching. + references = metadata.get("references", []) + if not isinstance(references, list) or any(not isinstance(item, str) for item in references): + raise ValueError(f"Skill references must be a string list: {name}") + return references + + +def _strings(record: dict, key: str, name: str) -> None: + """A declared string list must be strings; [] is a deliberate empty value.""" + if key not in record: + return + listed = record[key] + if not isinstance(listed, list) or any( + not isinstance(item, str) or not item.strip() for item in listed + ): + raise ValueError(f"Skill {key} must be a list of non-empty strings: {name}") + + +def _applicability(record: dict, name: str) -> None: + """Conditions a procedure assumes. Descriptive only; never a validation claim.""" + if "applicability" not in record: + return + declared = record["applicability"] + if not isinstance(declared, dict): + raise ValueError(f"Skill applicability must be a mapping: {name}") + for key, value in declared.items(): + if isinstance(value, str): + if not value.strip(): + raise ValueError(f"Skill applicability.{key} is empty: {name}") + elif isinstance(value, list): + if not value or any(not isinstance(item, str) or not item.strip() for item in value): + raise ValueError( + f"Skill applicability.{key} must be a non-empty string list: {name}" + ) + else: + raise ValueError(f"Skill applicability.{key} must be a string or string list: {name}") + + +def _effective(directory: Path) -> dict: + # Read bytes line by line; parse only the frontmatter, never decode the body. + with _entry(directory, "SKILL.md").open("rb") as stream: + record = _frontmatter(stream) + for key in ("name", "description"): + if not isinstance(record.get(key), str) or not record[key].strip(): + raise ValueError(f"Skill requires {key}: {directory.name}") + if record["name"] != directory.name: + raise ValueError(f"Skill name disagrees with directory: {directory.name}") + metadata = _sidecar(directory) + if metadata is not None: + for key in record.keys() & metadata.keys(): + if record[key] != metadata[key]: + raise ValueError(f"Conflicting skill metadata field {key}: {directory.name}") + if metadata.get("profile") == "local-skill-v1": + if any(key in metadata for key in ("name", "description")): + raise ValueError("Local skill name and description belong in SKILL.md") + if set(record) != {"name", "description"}: + raise ValueError("Local skill structured fields belong in metadata.json") + # Only sidecars declare these fields, and JSON cannot express a container + # cycle; validating the sidecar leaves the bundle normalizer free to report + # a cyclic legacy frontmatter value against the field that contains it. + _strings(metadata, "recommendations", directory.name) + _applicability(metadata, directory.name) + if "searchspace_not_applicable" in metadata: + reason = metadata["searchspace_not_applicable"] + if not isinstance(reason, str) or not reason.strip(): + raise ValueError( + f"Skill searchspace_not_applicable requires a reason: {directory.name}" + ) + record.update(metadata) + _references(record, directory.name) + if "status" in record and not isinstance(record["status"], str): + raise ValueError(f"Skill status must be a string: {directory.name}") + if "tags" in record and ( + not isinstance(record["tags"], list) + or any(not isinstance(tag, str) for tag in record["tags"]) + ): + raise ValueError(f"Skill tags must be a string list: {directory.name}") + return record + + +def _metadata(directory: Path) -> dict: + record = _effective(directory) + return {key: record[key] for key in ("name", "description", "status", "tags") if key in record} + + +def skill_metadata(root: Path, name: str) -> dict: + """Effective legacy frontmatter plus sidecar fields, after conflict checks.""" + return _effective(skill_directory(root, name)) + + +def skill_records(root: Path) -> list[dict]: + """Project metadata only; this projection is not an authoritative matrix.""" + return [_metadata(directory) for directory in skill_directories(root).values()] + + +def get_skill(root: Path, name: str) -> dict: + """Read selected metadata and resource names, never resource contents.""" + directory = skill_directory(root, name) + record = _metadata(directory) + record["resources"] = [ + path.relative_to(directory).as_posix() + for path in sorted(directory.rglob("*")) + if path.is_file() and path.name != "SKILL.md" + ] + return record + + +def skill_instructions(root: Path, name: str) -> str: + """Load selected instructions. Reading does not authorize their commands.""" + directory = skill_directory(root, name) + _metadata(directory) + with _entry(directory, "SKILL.md").open("rb") as stream: + _frontmatter(stream) + return stream.read().decode("utf-8").lstrip("\r\n") + + +def skill_resource(root: Path, name: str, resource: str) -> Path: + """Resolve one explicitly selected resource inside its skill directory.""" + directory = skill_directory(root, name) + path = (directory / resource).resolve() + if not path.is_relative_to(directory) or path == directory / "SKILL.md": + raise ValueError("Resource must be inside the selected skill; use show for instructions") + if not path.is_file(): + raise ValueError(f"Unknown skill resource: {resource}") + return path + + +def skill_reference(root: Path, name: str, reference: str) -> Path: + """Resolve a declared checkout reference without escaping the skills root.""" + directory = skill_directory(root, name) + if reference not in _references(_effective(directory), directory.name): + raise ValueError(f"Undeclared skill reference: {reference}") + if "\\" in reference or Path(reference).is_absolute(): + raise ValueError("Reference must use a relative POSIX path") + path = (directory / reference).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError("Reference must stay inside the skills root") + if not path.is_file(): + raise ValueError(f"Missing skill reference: {reference}") + return path diff --git a/src/inferrsi/knowledge_operations.py b/src/inferrsi/knowledge_operations.py index cff710c26..fab038591 100644 --- a/src/inferrsi/knowledge_operations.py +++ b/src/inferrsi/knowledge_operations.py @@ -1,735 +1,35 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge.knowledge_operations; +# patch implementation globals such as _json and _hash on that module. from __future__ import annotations -import hashlib -import json -import re -import sqlite3 -from dataclasses import dataclass -from datetime import date, datetime -from importlib.resources import files -from pathlib import Path, PurePosixPath - -from ._skill_selection import ( - RELEVANCE_METHODS, - TASK_RELEVANCE, - context_view, - indexed_skill, - record_context, - skill_score, - task_score, - words, +from inferrsi.knowledge.knowledge_operations import ( + AUTHORITIES, + IDENTITY, + INCIDENTAL, + Bundle, + IdentityError, + KnowledgeReader, + RevisionConflict, + _hash, + _json, + _normalized, + _sources, + _tree, + safe_path, ) -from .catalog import KNOWLEDGE_SCOPES, SCENARIO_ALIASES, knowledge_records - -AUTHORITIES = ("package", "checkout", "store") -IDENTITY = r"[a-z0-9][a-z0-9._-]*" - - -class RevisionConflict(ValueError): - pass - - -class IdentityError(ValueError): - def __init__(self, code: str, message: str): - super().__init__(message) - self.code = code - - -def safe_path(value: str) -> str: - path = PurePosixPath(value) - if ( - not value - or "\\" in value - or ":" in value - or path.is_absolute() - or any(part in ("", ".", "..") for part in value.split("/")) - ): - raise ValueError(f"Unsafe bundle path: {value}") - return value - - -def _json(value: object) -> bytes: - return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() - - -def _normalized(value: object, field: str, ancestors: frozenset[int] = frozenset()) -> object: - # Bundle identity is JSON, so YAML scalars without a JSON form are rejected - # here instead of failing inside revision serialization. - if value is None or isinstance(value, (str, bool, int, float)): - return value - if isinstance(value, (datetime, date)): - return value.isoformat() - if isinstance(value, (list, dict)): - # A YAML alias can make a container its own descendant. Only the active path - # is tracked, so the same object shared by sibling fields stays accepted. - identity = id(value) - if identity in ancestors: - raise ValueError( - f"Field {field} contains a reference cycle: {type(value).__name__}" - ) - ancestors = ancestors | {identity} - if isinstance(value, list): - return [ - _normalized(item, f"{field}[{index}]", ancestors) - for index, item in enumerate(value) - ] - result = {} - for key, item in value.items(): - if not isinstance(key, str): - raise ValueError(f"Field {field} has a non-string key: {key!r}") - result[key] = _normalized(item, f"{field}.{key}" if field else key, ancestors) - return result - raise ValueError( - f"Field {field or ''} cannot be represented in the bundle identity: " - f"{type(value).__name__}" - ) - - -def _hash(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _sources(records: list[dict]) -> list[dict]: - result = [] - for record in records: - source = dict(record) - kind = source.get("type") - if kind in ("source", "source_code", "technical_document") and "commit" in source: - source["original_type"] = kind - kind = "git" - if kind is None and "commit" in source: - kind = "git" - if kind not in ("git", "paper", "document", "internal", "unknown"): - # Legacy types keep their original label; nothing is inferred from them. - source["original_type"] = kind - kind = "unknown" - source["type"] = kind - if kind == "git": - commit = source.get("commit") - if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{7,40}", commit): - raise ValueError("Git source requires a real commit or explicit historical prefix") - source["revision_precision"] = "full" if len(commit) == 40 else "prefix" - elif kind == "paper": - if not source.get("id") or "version" not in source: - raise ValueError("Paper source requires ID and explicit version (null if unknown)") - if "commit" in source: - raise ValueError("Paper versions are not Git commits") - elif kind == "document": - if not source.get("url") or not any(k in source for k in ("version", "sha256")): - raise ValueError("Document source requires URL and version or snapshot identity") - elif kind == "internal": - if not source.get("artifact") or not any(k in source for k in ("issue", "run")): - raise ValueError("Internal source requires artifact and issue/run identity") - result.append(source) - return result - - -@dataclass(frozen=True) -class Bundle: - metadata: dict - content: dict[str, bytes] - dependencies: dict[str, str] - - def __post_init__(self) -> None: - # Normalize once at ingestion so manifest, revision and views agree. - object.__setattr__(self, "metadata", _normalized(self.metadata, "metadata")) - - def manifest(self) -> dict: - record = self.metadata - if not isinstance(record.get("id"), str) or not re.fullmatch(IDENTITY, record["id"]): - raise ValueError("Invalid bundle identity") - if record.get("kind") not in ("skill", "knowledge", "case"): - raise ValueError("Invalid bundle kind") - for field in ( - "sources", - "applicability", - "state", - "limitations", - "attribution", - "entrypoint", - ): - if field not in record: - raise ValueError(f"Missing normalized field: {field}") - for field in ("state", "entrypoint"): - if not isinstance(record[field], str) or not record[field]: - raise ValueError(f"Normalized {field} must be a nonempty string") - if not isinstance(record["limitations"], list): - raise ValueError("Normalized limitations must be a list") - if not isinstance(record["sources"], list) or not record["sources"]: - raise ValueError("Bundle requires sources, including explicit unknown provenance") - _sources(record["sources"]) - for path, data in self.content.items(): - safe_path(path) - if not isinstance(data, bytes): - raise ValueError("Bundle content must be bytes") - if record["entrypoint"] not in self.content: - raise ValueError("Missing entrypoint in complete bundle") - for required in record.get("required_resources", []): - # Present string keys already passed the content path validation. - if isinstance(required, str) and required in self.content: - continue - if safe_path(required) not in self.content: - raise ValueError(f"Missing required resource: {required}") - resources = {path: _hash(data) for path, data in sorted(self.content.items())} - for dependency, revision in self.dependencies.items(): - safe_path(dependency) - if not re.fullmatch(r"sha256:[0-9a-f]{64}", revision): - raise ValueError("Dependency requires a resolved content revision") - if dependency in self.content: - actual = "sha256:" + resources[dependency] - else: - members = { - path: digest - for path, digest in resources.items() - if path.startswith(dependency + "/") - } - if not members: - raise ValueError(f"Missing dependency: {dependency}") - actual = "sha256:" + _hash(_json(members)) - if actual != revision: - raise RevisionConflict(f"Dependency bytes disagree with revision: {dependency}") - return { - "metadata": record, - "resources": resources, - "dependencies": dict(sorted(self.dependencies.items())), - } - - @property - def revision(self) -> str: - return "sha256:" + _hash(_json(self.manifest())) - - def view(self, *, material: bool = False) -> dict: - manifest = self.manifest() - result = { - **self.metadata, - "revision": "sha256:" + _hash(_json(manifest)), - "resources": manifest["resources"], - "dependencies": manifest["dependencies"], - } - if material: - result["content"] = self.content[self.metadata["entrypoint"]].decode("utf-8") - return result - - -INCIDENTAL = ("__pycache__", ".DS_Store") - - -def _tree( - node, prefix: str, *, root: Path | None = None, ancestors: frozenset[Path] = frozenset() -) -> dict[str, bytes]: - if isinstance(node, Path): - # A junction or symlink resolves away from where it appears; every descended - # directory must still land inside the root that owns this walk. - directory = node.resolve() - if root is None: - root = directory - if directory in ancestors: - raise ValueError(f"Resource cycle inside the skills root: {prefix}") - ancestors = ancestors | {directory} - result = {} - for child in sorted(node.iterdir(), key=lambda p: p.name): - if child.name in INCIDENTAL or child.name.endswith(".pyc"): - continue - path = safe_path(f"{prefix}/{child.name}") - if isinstance(child, Path) and child.is_symlink(): - raise ValueError(f"Symlink resource is not supported: {path}") - if child.is_dir(): - if isinstance(child, Path) and root is not None and not child.resolve().is_relative_to( - root - ): - raise ValueError(f"Resource escapes the skills root: {path}") - result.update(_tree(child, path, root=root, ancestors=ancestors)) - else: - result[path] = child.read_bytes() - return result - - -class KnowledgeReader: - def __init__(self, *, skill_root: Path | None = None, store=None): - self.skill_root = skill_root.resolve() if skill_root is not None else None - if store is not None and store.skill_root != self.skill_root: - raise ValueError("Store must reserve identities from the same checkout") - self.store = store - self._records: dict[str, dict] | None = None - self._library: tuple[dict[str, bytes], str] | None = None - - def records(self) -> dict[str, dict]: - """Installed catalog records, read once per reader (one package snapshot).""" - if self._records is None: - self._records = {record["id"]: record for record in knowledge_records()} - return self._records - - def _skill_library(self) -> tuple[dict[str, bytes], str]: - if self._library is None: - root = files("inferrsi").joinpath("data", "skill_library") - library = _tree(root, "data/skill_library") - digest = _hash(_json({path: _hash(data) for path, data in sorted(library.items())})) - self._library = (library, "sha256:" + digest) - return self._library - - def owners(self) -> dict[str, list[str]]: - """Map each public ID to every authority that owns it.""" - result: dict[str, list[str]] = {} - for identity in self.records(): - result.setdefault(identity, []).append("package") - if self.skill_root is not None: - # Enumeration reads registered names only. Validating sidecar content - # here would abort the whole collection over one malformed, possibly - # unrequested, skill; Collect reports that per item through _read. - from .skill_catalog import skill_directories - - local = list(skill_directories(self.skill_root)) - local.extend(self._cases()) - if len(set(local)) != len(local): - raise IdentityError("collision", "Duplicate identity inside the checkout") - for identity in local: - result.setdefault(identity, []).append("checkout") - if self.store is not None: - for identity in self.store.identities(): - result.setdefault(identity, []).append("store") - return result - - def identities(self) -> list[str]: - return sorted(f"{a}:{i}" for i, authorities in self.owners().items() for a in authorities) - - def resolve(self, reference: str, owners: dict[str, list[str]] | None = None) -> str: - """Return the qualified `authority:id`; ambiguity is an error, never precedence.""" - authority, qualified, identity = reference.rpartition(":") - if not re.fullmatch(IDENTITY, identity) or (qualified and authority not in AUTHORITIES): - raise IdentityError("invalid_identity", f"Invalid identity: {reference}") - found = (self.owners() if owners is None else owners).get(identity, []) - if qualified: - if authority not in found: - raise IdentityError("unknown_identity", f"Unknown identity: {reference}") - return reference - if not found: - raise IdentityError("unknown_identity", f"Unknown identity: {identity}") - if len(found) > 1: - choices = ", ".join(f"{a}:{identity}" for a in found) - raise IdentityError( - "ambiguous_identity", f"Ambiguous identity {identity}; qualify one of: {choices}" - ) - return f"{found[0]}:{identity}" - - def _cases(self) -> dict[str, Path]: - result = {} - if self.skill_root is not None: - from .skill_catalog import skill_directories - - for directory in skill_directories(self.skill_root).values(): - for path in sorted(directory.glob("cases/*.json")): - if path.is_symlink() or not path.resolve().is_relative_to(self.skill_root): - raise ValueError("Case must stay inside the selected skills root") - record = json.loads(path.read_text(encoding="utf-8")) - if record.get("kind") == "case": - identity = record["id"] - if not isinstance(identity, str) or not re.fullmatch(IDENTITY, identity): - raise ValueError(f"Invalid case identity in {path.name}: {identity!r}") - if identity in result: - raise ValueError(f"Duplicate case identity: {identity}") - result[identity] = path - return result - - def read(self, reference: str, *, revision: str | None = None) -> Bundle: - return self._read(self.resolve(reference), revision) - - def _read(self, qualified: str, revision: str | None) -> Bundle: - authority, _, identity = qualified.partition(":") - if authority == "store": - return self.store.read(identity, revision=revision) - bundle = self._checkout(identity) if authority == "checkout" else self._knowledge(identity) - if revision is not None and bundle.revision != revision: - raise RevisionConflict( - f"Revision unavailable in this snapshot: {qualified} at {revision}" - ) - return bundle - - def _checkout(self, identity: str) -> Bundle: - cases = self._cases() - if identity in cases: - path = cases[identity] - record = json.loads(path.read_text(encoding="utf-8")) - for field in ("references", "dependencies"): - if field in record: - raise ValueError(f"Case {field} are not supported; declare required_resources") - declared = record.get("required_resources", []) - if not isinstance(declared, list) or any(not isinstance(p, str) for p in declared): - raise ValueError("Case required_resources must be a string list") - if len(set(declared)) != len(declared): - raise ValueError("Case required_resources must not repeat a path") - relative = safe_path(record["content"]) - required = [relative, "case.json", *(p for p in declared if p != relative)] - content = {"case.json": path.read_bytes()} - for resource in [relative, *required[2:]]: - safe_path(resource) - if resource == "case.json": - raise ValueError("Case resource collides with its case.json record") - resource_path = path.parent / resource - if resource_path.is_symlink() or not resource_path.resolve().is_relative_to( - self.skill_root - ): - raise ValueError(f"Case resource escapes the skills root: {resource}") - if not resource_path.is_file(): - raise ValueError(f"Missing case resource: {resource}") - content[resource] = resource_path.read_bytes() - return Bundle( - { - **record, - "entrypoint": relative, - "sources": _sources(record["sources"]), - "required_resources": required, - }, - content, - {}, - ) - return self._skill(identity) - - def _knowledge(self, identity: str) -> Bundle: - record = self.records().get(identity) - if record is None: - raise IdentityError("unknown_identity", f"Unknown knowledge ID: {identity}") - content = {"record.json": _json(record)} - dependencies = {} - if record.get("skill_documents"): - # The whole imported library is one explicit dependency so bilingual - # references and licenses stay a complete closure. - library, digest = self._skill_library() - content.update(library) - dependencies["data/skill_library"] = digest - documents = record["skill_documents"] - if not isinstance(documents, dict): - raise ValueError(f"skill_documents must map editions to paths: {identity}") - for target in documents.values(): - if not isinstance(target, str) or safe_path(target) not in library: - raise ValueError(f"Missing declared skill document: {target}") - return Bundle( - { - "id": identity, - "kind": "knowledge", - "entrypoint": "record.json", - "origin": record.get("origin", ["external"]), - "sources": _sources(record["sources"]), - "applicability": record.get("applicability", {}), - "state": record["status"], - "limitations": record.get("limitations", []) - + [ - "Imported guidance; reading does not validate or execute it.", - "Historical revisions require the matching installed snapshot " - "or a retained store.", - ], - "attribution": record["sources"], - "verification": record["verification"], - "tier": record.get("tier", "contribution"), - "repo": record["repo"], - "backend": record["backend"], - "scenario": record["scenario"], - "title": record["title"], - "skill_documents": record.get("skill_documents", {}), - "required_resources": sorted(content), - }, - content, - dependencies, - ) - - def _skill(self, identity: str) -> Bundle: - from .skill_catalog import get_skill, skill_directory, skill_metadata, skill_reference - - root = self.skill_root - record = get_skill(root, identity) - directory = skill_directory(root, identity) - prefix = directory.relative_to(root).as_posix() - content = _tree(directory, prefix, root=root) - metadata = skill_metadata(root, identity) - dependencies = {} - for reference in metadata.get("references", []): - path = skill_reference(root, identity, reference) - key = path.relative_to(root).as_posix() - content[key] = path.read_bytes() - dependencies[key] = "sha256:" + _hash(content[key]) - return Bundle( - { - "id": identity, - "kind": "skill", - "entrypoint": f"{prefix}/SKILL.md", - "title": record["description"], - "origin": metadata.get("origin", ["unknown"]), - "sources": _sources(metadata.get("sources", [])) - or [ - { - "type": "unknown", - "note": "Legacy provenance remains in procedure/readsets", - } - ], - "applicability": metadata.get("applicability", {}), - "state": record.get("status", "unspecified"), - "limitations": metadata.get("limitations", []) - + [ - "Checkout-only local skill; no automatic execution or promotion.", - "Only declared references are required; " - "other cross-skill links are navigation.", - "Historical revisions require a matching checkout or a retained store.", - ], - "attribution": metadata.get("sources", []), - "empirical_validation": metadata.get("empirical_validation", "unknown"), - "required_resources": sorted(content), - }, - content, - dependencies, - ) - - def resource(self, identity: str, path: str, *, revision: str) -> bytes: - safe_path(path) - bundle = self.read(identity, revision=revision) - if path not in bundle.content: - raise ValueError(f"Unknown resource: {path}") - return bundle.content[path] - - def collect( - self, - query: str = "", - *, - ids: list[str] | None = None, - revisions: dict[str, str] | None = None, - kind: str | None = None, - repo: str | None = None, - tier: str | None = None, - backend: str | None = None, - scenario: str | None = None, - limit: int = 10, - max_bytes: int | None = None, - material: bool = False, - relevance: str | None = None, - relevance_method: str = "weighted-lexical-v1", - context_bytes: int | None = None, - ) -> dict: - if type(limit) is not int or limit < 1: - raise ValueError("limit must be a positive integer") - if max_bytes is not None and (type(max_bytes) is not int or max_bytes < 1): - raise ValueError("max_bytes must be a positive integer") - if ids is not None and ( - not isinstance(ids, list) or any(not isinstance(identity, str) for identity in ids) - ): - raise ValueError("ids must be a list of identity strings") - if kind not in (None, "skill", "knowledge", "case"): - raise ValueError("Unknown content kind") - if relevance is not None and (not isinstance(relevance, str) or not words(relevance)): - raise ValueError("relevance requires at least one word") - if relevance_method not in RELEVANCE_METHODS: - raise ValueError("Unknown relevance method") - if relevance is None and relevance_method != "weighted-lexical-v1": - raise ValueError("relevance_method requires relevance") - if context_bytes is not None and (type(context_bytes) is not int or context_bytes < 1): - raise ValueError("context_bytes must be a positive integer") - if revisions is None: - revisions = {} - if not isinstance(revisions, dict) or any( - not isinstance(key, str) or not isinstance(value, str) - for key, value in revisions.items() - ): - raise ValueError("revisions must map identity strings to revision strings") - if ids is not None and len(set(ids)) != len(ids): - raise ValueError("Duplicate requested identity") - owners = self.owners() - requested = ids if ids is not None else self.identities() - - def qualified(reference: str) -> str | None: - # Revision keys may use any spelling the API accepts for an identity. - try: - return self.resolve(reference, owners) - except IdentityError: - return None - - selected = {identity for identity in map(qualified, requested) if identity is not None} - pinned: dict[str, str] = {} - for reference, revision in revisions.items(): - target = qualified(reference) - if target is None or target not in selected: - raise ValueError( - f"Revision requested for an unselected identity: {reference}" - ) - if pinned.setdefault(target, revision) != revision: - raise ValueError(f"Conflicting revisions for the same identity: {reference}") - pinned[target] = revision - collisions = sorted(i for i, authorities in owners.items() if len(authorities) > 1) - if tier is not None and tier not in KNOWLEDGE_SCOPES: - raise ValueError(f"Unknown knowledge tier: {tier}") - scenario = SCENARIO_ALIASES.get(scenario, scenario) - package_filters = any(value is not None for value in (repo, tier, backend, scenario)) - task_ranking = relevance_method == TASK_RELEVANCE - - def matches(metadata: dict, authority: str) -> bool: - declared = metadata.get("scenario") - declared = declared if isinstance(declared, str) else "" - # The catalog records skills.discovery as the discovery operation, - # not the workload served by a procedure. Let the task cue rank - # these procedures alongside records declaring the requested workload. - discovery_skill = ( - task_ranking and ids is None and authority == "package" - and declared == "skills.discovery" - and self.records()[metadata["id"]].get("index_type") == "packaged-skill" - ) - return not ( - (kind and metadata.get("kind") != kind) - or (package_filters and authority != "package") - or (repo and metadata.get("repo") != repo) - or (tier and metadata.get("tier", "contribution") != tier) - or (backend and metadata.get("backend") not in (backend, "common")) - or not ( - scenario is None - or declared == scenario - or (declared.endswith(".*") and scenario.startswith(declared[:-1])) - or discovery_skill - ) - ) - - cases = self._cases() if kind else {} - selected, omitted, seen = [], [], set() - used = 0 - ranked, contexts, context_omissions = [], [], [] - context_used = 0 - - def select(reference: str, bundle: Bundle, view: dict) -> None: - nonlocal used, context_used - # Normal Bundle views were serialized while computing their revision. - # Preserve custom subclasses' serialization/error behavior below. - if type(bundle) is Bundle and len(selected) >= limit: - omitted.append({"id": reference, "reason": "output_limit"}) - return - size = len(_json(view)) - if len(selected) >= limit or (max_bytes is not None and used + size > max_bytes): - omitted.append({"id": reference, "reason": "output_limit"}) - return - selected.append(view) - used += size - if context_bytes is None: - return - record = self.records().get(view["id"], {}) - is_skill = record.get("index_type") == "packaged-skill" - if view["authority"] != "package" or (not is_skill and not task_ranking): - context_omissions.append({"ref": view["ref"], "reason": "unsupported_context"}) - return - try: - available = context_bytes - context_used - if is_skill: - context, omission = context_view( - record, view, bundle.content, relevance or query, available - ) - else: - context, omission = record_context(record, view, bundle.content, available) - except (ValueError, KeyError, TypeError) as exc: - context_omissions.append({ - "ref": view["ref"], "reason": "index_unavailable", "detail": str(exc), - }) - return - if omission is not None: - context_omissions.append(omission) - else: - contexts.append(context) - context_used += context["content_bytes"] - for reference in requested: - try: - qualified = self.resolve(reference, owners) - if qualified in seen: - raise IdentityError("duplicate_identity", f"Already selected: {qualified}") - seen.add(qualified) - authority, _, identity = qualified.partition(":") - # Cheap metadata checks first, so filtered items never load resource bytes. - if authority == "package": - early = {**self.records()[identity], "kind": "knowledge"} - elif authority == "checkout": - early = {"kind": "case" if identity in cases else "skill"} - else: - early = None - if relevance is not None and ids is None and ( - authority != "package" - or (not task_ranking and early.get("index_type") != "packaged-skill") - ): - continue - if early is not None and not matches(early, authority): - if ids is not None: - omitted.append({"id": reference, "reason": "filters"}) - continue - bundle = self._read(qualified, pinned.get(qualified)) - view = {**bundle.view(material=material), "authority": authority, "ref": qualified} - entry = bundle.content[bundle.metadata["entrypoint"]] - haystack = (_json(bundle.metadata) + entry).decode("utf-8", "replace").casefold() - except IdentityError as exc: - omitted.append({"id": reference, "reason": exc.code, "detail": str(exc)}) - continue - except RevisionConflict as exc: - omitted.append( - {"id": reference, "reason": "unavailable_revision", "detail": str(exc)} - ) - continue - except (ValueError, KeyError, TypeError, AttributeError, OSError, sqlite3.Error) as exc: - omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) - continue - if not matches(view, authority): - if ids is not None: - omitted.append({"id": reference, "reason": "filters"}) - continue - if any(token not in haystack for token in query.casefold().split()): - if ids is not None: - omitted.append({"id": reference, "reason": "query"}) - continue - if relevance is not None and ids is None: - record = self.records().get(identity, {}) - try: - is_skill = record.get("index_type") == "packaged-skill" - if is_skill: - core, _, _ = indexed_skill(record, view, bundle.content) - body = bundle.content[core] - else: - body = entry - if task_ranking: - score = task_score(relevance, record, body, backend) - else: - score = skill_score(relevance, record, body, method=relevance_method) - except (ValueError, KeyError, TypeError) as exc: - omitted.append({ - "id": reference, "reason": "index_unavailable", "detail": str(exc), - }) - continue - if score: - ranked.append((score, reference, bundle, view)) - continue - select(reference, bundle, view) - for _, reference, bundle, view in sorted( - ranked, key=lambda item: (-item[0], item[3]["ref"]) - ): - select(reference, bundle, view) - result = { - "items": selected, - "omissions": omitted, - "no_match": not selected, - "collisions": collisions, - "material_bytes": used, - "limitations": [ - "No crawling, ranking model or conversation state; deterministic identity order.", - "max_bytes bounds item payloads, not omission/control metadata.", - "Read references must be resolved with their returned exact revision.", - ], - } - if relevance is not None: - result["selection"] = { - "method": relevance_method, "cue": relevance, - "scope": ("package-skills-and-records" if task_ranking - else "generated-packaged-skills"), - "ranking_applied": ids is None, - } - if task_ranking and ids is None: - result["selection"].update(matched_count=len(ranked), returned_count=len(selected)) - result["limitations"][0] = ( - "Deterministic lexical selection; explicit IDs retain caller order and membership." - ) - if context_bytes is not None: - result.update({ - "contexts": contexts, "context_omissions": context_omissions, - "context_bytes": context_used, - "context_budget": { - "max_bytes": context_bytes, "accounting": "utf8-resource-content", - "excludes": "provenance, omission and control metadata", - }, - }) - return result +__all__ = [ + "AUTHORITIES", + "IDENTITY", + "RevisionConflict", + "IdentityError", + "safe_path", + "Bundle", + "KnowledgeReader", + "INCIDENTAL", + "_json", + "_hash", + "_normalized", + "_sources", + "_tree", +] diff --git a/src/inferrsi/knowledge_store.py b/src/inferrsi/knowledge_store.py index 4cb585703..9c699ca48 100644 --- a/src/inferrsi/knowledge_store.py +++ b/src/inferrsi/knowledge_store.py @@ -1,297 +1,6 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge.knowledge_store. from __future__ import annotations -import json -import re -import sqlite3 -from contextlib import closing -from importlib.resources import files -from pathlib import Path +from inferrsi.knowledge.knowledge_store import KnowledgeStore, _protected -from .knowledge_operations import ( - AUTHORITIES, - IDENTITY, - Bundle, - IdentityError, - KnowledgeReader, - RevisionConflict, - _json, - safe_path, -) - - -def _protected(skill_root: Path | None) -> list[Path]: - package = Path(str(files("inferrsi"))).resolve() - roots = [package] - # A source, editable or exported tree (git archive, sdist) surrounds the package - # and holds evidence and both matrices; protect it even without a .git entry, - # because review snapshots carry no repository metadata. - for parent in package.parents: - if (parent / "pyproject.toml").is_file() or (parent / ".git").exists(): - roots.append(parent) - break - if skill_root is not None: - roots.append(Path(skill_root).resolve().parent) - return roots - - -class KnowledgeStore: - """One authorized writable store for store-owned candidate bundles. - - Package and checkout content stay read-only; their IDs are reserved here so a - store candidate cannot shadow them. Repository-owned amendments use PRs. - """ - - def __init__(self, workspace: Path, *, skill_root: Path | None, read_only: bool = False): - # The caller authorizes an existing dedicated directory; none is created or guessed. - workspace = Path(workspace) - if workspace.is_symlink() or not workspace.is_dir(): - raise ValueError("Store requires an existing authorized workspace directory") - self.workspace = workspace.resolve() - for root in _protected(skill_root): - if self.workspace.is_relative_to(root): - raise ValueError(f"Store workspace must be outside read-only sources: {root}") - self.skill_root = Path(skill_root).resolve() if skill_root is not None else None - self.adapters = KnowledgeReader(skill_root=self.skill_root) - self.read_only = read_only - self.database = self.workspace / "knowledge.sqlite3" - if self.database.is_symlink(): - raise ValueError("Store database cannot be a symlink") - if read_only: - # Reading an existing store must not create or migrate anything. - if not self.database.is_file(): - raise ValueError(f"Read-only store requires an existing database: {self.database}") - self._verify_reserved_skill_root() - return - with closing(self._connect()) as connection, connection: - connection.executescript( - "CREATE TABLE IF NOT EXISTS revisions (" - "id TEXT NOT NULL, revision TEXT NOT NULL, metadata TEXT NOT NULL, " - "dependencies TEXT NOT NULL, parent TEXT, reason TEXT NOT NULL, " - "evidence TEXT NOT NULL, PRIMARY KEY (id, revision));" - "CREATE TABLE IF NOT EXISTS resources (" - "id TEXT NOT NULL, revision TEXT NOT NULL, path TEXT NOT NULL, data BLOB NOT NULL, " - "PRIMARY KEY (id, revision, path), " - "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" - "CREATE TABLE IF NOT EXISTS current (id TEXT PRIMARY KEY, revision TEXT NOT NULL, " - "FOREIGN KEY (id, revision) REFERENCES revisions(id, revision));" - "CREATE TABLE IF NOT EXISTS meta (" - "key TEXT PRIMARY KEY, value TEXT NOT NULL);" - ) - reserved = connection.execute( - "SELECT value FROM meta WHERE key='skill_root'" - ).fetchone() - if reserved is None: - connection.execute( - "INSERT INTO meta VALUES ('skill_root', ?)", (self._reserved_skill_root(),) - ) - elif reserved[0] != self._reserved_skill_root(): - raise ValueError( - f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " - "reopen it with the same skill_root" - ) - - def _reserved_skill_root(self) -> str: - # The reserved checkout is recorded so later opens cannot silently drop it; - # an empty value records a package-only store. - return str(self.skill_root) if self.skill_root is not None else "" - - def _verify_reserved_skill_root(self) -> None: - # Stores created before reservation recording have no meta row and stay usable. - with closing(self._connect()) as connection: - present = connection.execute( - "SELECT 1 FROM sqlite_master WHERE type='table' AND name='meta'" - ).fetchone() - reserved = ( - connection.execute("SELECT value FROM meta WHERE key='skill_root'").fetchone() - if present - else None - ) - if reserved is not None and reserved[0] != self._reserved_skill_root(): - raise ValueError( - f"Store reserves the skills checkout {reserved[0] or 'package-only'}; " - "open it with the same --skill-root, or without one for a package-only store" - ) - - def _connect(self): - if self.database.is_symlink() or self.database.resolve().parent != self.workspace: - raise ValueError("Store database escapes the authorized workspace") - if self.read_only: - # mode=ro fails every write at the SQLite level; as_uri() percent-encodes - # spaces and non-ASCII so the file: URI stays valid on Windows too. - connection = sqlite3.connect(self.database.as_uri() + "?mode=ro", uri=True, timeout=5) - else: - connection = sqlite3.connect(self.database, timeout=5) - connection.execute("PRAGMA foreign_keys=ON") - return connection - - def identities(self) -> list[str]: - with closing(self._connect()) as connection: - return [row[0] for row in connection.execute("SELECT id FROM current ORDER BY id")] - - def reader(self) -> KnowledgeReader: - return KnowledgeReader(skill_root=self.skill_root, store=self) - - def collect(self, query: str = "", **options) -> dict: - return self.reader().collect(query, **options) - - def create(self, bundle: Bundle) -> str: - if self.read_only: - raise ValueError("Store is read-only; create requires a writable store") - return self._publish(bundle, expected=None, reason="create", evidence=[]) - - def update(self, bundle: Bundle, *, expected: str, reason: str, evidence: list[dict]) -> str: - if self.read_only: - raise ValueError("Store is read-only; update requires a writable store") - if not expected or not reason.strip(): - raise ValueError("Update requires expected old revision and reason") - if not isinstance(evidence, list): - raise ValueError("Update evidence must be a list; an empty list means none supplied") - return self._publish(bundle, expected=expected, reason=reason, evidence=evidence) - - def _lineage(self, metadata: dict) -> None: - lineage = metadata.get("derived_from", []) - if not isinstance(lineage, list): - raise ValueError("derived_from must be a list of qualified source revisions") - for source in lineage: - if not isinstance(source, dict) or not isinstance(source.get("ref"), str): - raise ValueError("derived_from entries require a qualified ref") - authority, _, identity = source["ref"].partition(":") - if authority not in AUTHORITIES or not re.fullmatch(IDENTITY, identity): - raise ValueError(f"derived_from ref must be :: {source['ref']}") - revision = source.get("revision") - if revision is not None and ( - not isinstance(revision, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", revision) - ): - raise ValueError("derived_from revision must be a content revision or null") - if identity == metadata["id"]: - raise IdentityError( - "collision", "An adaptation needs a distinct identity from its source" - ) - - def _publish(self, bundle: Bundle, *, expected: str | None, reason: str, evidence: list) -> str: - # Copy caller data, validate it, then store the whole bundle and current - # pointer in one transaction. - bundle = Bundle( - json.loads(_json(bundle.metadata)), dict(bundle.content), dict(bundle.dependencies) - ) - revision = bundle.revision - identity = bundle.metadata["id"] - try: - bundle.content[bundle.metadata["entrypoint"]].decode("utf-8") - except UnicodeDecodeError as exc: - raise ValueError("Bundle entrypoint must be UTF-8 text") from exc - self._lineage(bundle.metadata) - owners = self.adapters.owners().get(identity) - if owners: - raise IdentityError( - "collision", - f"Identity {identity} is owned by read-only {', '.join(owners)} content; " - "amend it through a repository PR or create an adaptation with a distinct ID", - ) - if expected is None and bundle.metadata["state"] != "candidate": - raise ValueError( - "Create registers a candidate; imported or active content is not promoted" - ) - try: - with closing(self._connect()) as connection, connection: - connection.execute("BEGIN IMMEDIATE") - current = connection.execute( - "SELECT revision FROM current WHERE id=?", (identity,) - ).fetchone() - if expected is None: - if current is not None: - raise RevisionConflict(f"Duplicate identity: {identity}") - elif current is None or current[0] != expected: - raise RevisionConflict(f"Stale or missing revision for {identity}") - if expected is not None: - previous = connection.execute( - "SELECT metadata FROM revisions WHERE id=? AND revision=?", - (identity, expected), - ).fetchone() - old = json.loads(previous[0]) - if bundle.metadata["state"] != old["state"]: - raise ValueError("Update cannot promote content state") - if revision == expected: - raise ValueError("Update must change the bundle") - if connection.execute( - "SELECT 1 FROM revisions WHERE id=? AND revision=?", (identity, revision) - ).fetchone(): - raise RevisionConflict( - f"Update reproduces retained revision {revision}; " - "change the bundle so the successor has its own revision" - ) - connection.execute( - "INSERT INTO revisions VALUES (?, ?, ?, ?, ?, ?, ?)", - ( - identity, - revision, - _json(bundle.metadata).decode(), - _json(bundle.dependencies).decode(), - expected, - reason, - _json(evidence).decode(), - ), - ) - connection.executemany( - "INSERT INTO resources VALUES (?, ?, ?, ?)", - [(identity, revision, path, data) for path, data in bundle.content.items()], - ) - connection.execute( - "INSERT INTO current VALUES (?, ?) " - "ON CONFLICT(id) DO UPDATE SET revision=excluded.revision", - (identity, revision), - ) - except sqlite3.OperationalError as exc: - if "locked" not in str(exc) and "busy" not in str(exc): - raise - raise RevisionConflict( - f"Store is locked by another writer; retry {identity} once it is released" - ) from exc - return revision - - def read(self, identity: str, *, revision: str | None = None) -> Bundle: - with closing(self._connect()) as connection: - if revision is None: - row = connection.execute( - "SELECT revision FROM current WHERE id=?", (identity,) - ).fetchone() - if row is None: - raise IdentityError("unknown_identity", f"Unknown identity: store:{identity}") - revision = row[0] - row = connection.execute( - "SELECT metadata, dependencies FROM revisions WHERE id=? AND revision=?", - (identity, revision), - ).fetchone() - if row is None: - raise RevisionConflict(f"Unknown revision: store:{identity} at {revision}") - resources = connection.execute( - "SELECT path, data FROM resources WHERE id=? AND revision=?", (identity, revision) - ).fetchall() - bundle = Bundle(json.loads(row[0]), dict(resources), json.loads(row[1])) - if bundle.revision != revision: - raise RevisionConflict("Stored bundle integrity mismatch") - return bundle - - def resource(self, identity: str, path: str, *, revision: str) -> bytes: - safe_path(path) - bundle = self.read(identity, revision=revision) - if path not in bundle.content: - raise ValueError(f"Unknown resource: {path}") - return bundle.content[path] - - def history(self, identity: str) -> list[dict]: - with closing(self._connect()) as connection: - rows = connection.execute( - "SELECT revision, parent, reason, evidence FROM revisions " - "WHERE id=? ORDER BY rowid", - (identity,), - ).fetchall() - return [ - { - "revision": revision, - "parent": parent, - "reason": reason, - "evidence": json.loads(evidence), - } - for revision, parent, reason, evidence in rows - ] +__all__ = ["KnowledgeStore", "_protected"] diff --git a/src/inferrsi/research.py b/src/inferrsi/research.py index fc0ce3dfc..45c0852c5 100644 --- a/src/inferrsi/research.py +++ b/src/inferrsi/research.py @@ -1,127 +1,6 @@ -"""Read-only model-aware discovery; target annotations are not runtime support.""" - +# Compatibility shell. The implementation lives in inferrsi.knowledge.research. from __future__ import annotations -import json -import math -from collections.abc import Sequence -from importlib.resources import files - -from .catalog import knowledge_records, search_knowledge - -PROCEDURE = ( - "data/skill_library/workflows/inference-engine-research/" - "references/model-aware-preparation.md" -) - - -def _profile(model: str) -> dict | None: - root = files("inferrsi").joinpath("data", "research_profiles") - for resource in sorted(root.iterdir(), key=lambda path: path.name): - if resource.name.endswith(".json"): - profile = json.loads(resource.read_text(encoding="utf-8")) - if model.casefold() in {name.casefold() for name in profile["aliases"]}: - return profile - return None - - -def research_knowledge( - model: str, - *, - frameworks: Sequence[str] = (), - tasks: Sequence[str] = (), - modalities: Sequence[str] = (), - mechanisms: Sequence[str] = (), - repo: str | None = None, - tier: str | None = None, - backend: str | None = None, - scenario: str | None = None, - limit: int = 10, - target_backend: str | None = None, - target_device: str | None = None, - target_memory_gib: float | None = None, -) -> dict: - """Union separate research queries, preserving every package filter and record. +from inferrsi.knowledge.research import PROCEDURE, _profile, research_knowledge - Round-robin selection gives each query a turn before another result from the - same query. It is discovery ordering, not an engine or evidence-quality rank. - """ - if not model.strip(): - raise ValueError("model must be nonempty") - if limit < 1: - raise ValueError("limit must be positive") - if target_memory_gib is not None and ( - not math.isfinite(target_memory_gib) or target_memory_gib <= 0 - ): - raise ValueError("target memory must be positive and finite GiB") - model = model.strip() - profile = _profile(model) - defaults = profile or {} - groups = { - "model": [model, *([profile["model"]] if profile else [])], - "framework": frameworks, - "task": [*defaults.get("tasks", []), *tasks], - "modality": [*defaults.get("modalities", []), *modalities], - "mechanism": [*defaults.get("mechanisms", []), *mechanisms], - } - filters = {"repo": repo, "tier": tier, "backend": backend, "scenario": scenario} - queries, streams, matches = [], [], {} - # Retrieve every scoped match before applying the shared output limit; an - # early per-query limit would lose attribution for overlapping records. - catalog_size = max(1, len(knowledge_records())) - for group, terms in groups.items(): - seen = set() - for term in terms: - term = term.strip() - if not term: - raise ValueError(f"{group} query must be nonempty") - if term.casefold() in seen: - continue - seen.add(term.casefold()) - query = {"group": group, "query": term} - records = search_knowledge(term, **filters, limit=catalog_size) - queries.append({**query, "matches": len(records)}) - streams.append(records) - for record in records: - matches.setdefault(record["id"], []).append(query) - selected = {} - for rank in range(max((len(stream) for stream in streams), default=0)): - for stream in streams: - if rank < len(stream): - record = stream[rank] - selected.setdefault(record["id"], record) - if len(selected) == limit: - break - if len(selected) == limit: - break - return { - "model": model, - "profile": profile, - "queries": queries, - "filters": filters, - "items": [ - {"record": record, "matched_queries": matches[identity]} - for identity, record in selected.items() - ], - "no_match": not selected, - "target": { - "backend": target_backend, "device": target_device, "memory_gib": target_memory_gib, - }, - "target_validation": "unverified", - "components": defaults.get("components", []), - "procedure_resource": PROCEDURE, - "worked_example_resource": defaults.get("worked_example_resource"), - "warnings": [] if profile else [ - "No packaged model profile: supply --task/--modality and verify the full component " - "inventory from the selected checkpoint before judging feasibility." - ], - "limitations": [ - "Queries search installed package records only; empty results do not rule out " - "external sources. Existing token-AND search and scope filters are unchanged.", - "Mechanism queries are research leads, not claims of model/backend support. " - "Retain each record's applicability, source revision and verification state.", - "Target annotations do not inspect hardware or add a runtime backend. Full weight " - "inventory, engine/format compatibility, peak residency and quality need evidence " - "under the linked preparation procedure; no engine is selected or executed.", - ], - } +__all__ = ["research_knowledge", "_profile", "PROCEDURE"] diff --git a/src/inferrsi/skill_catalog.py b/src/inferrsi/skill_catalog.py index ad3ad25a2..ac38ab091 100644 --- a/src/inferrsi/skill_catalog.py +++ b/src/inferrsi/skill_catalog.py @@ -1,291 +1,49 @@ +# Compatibility shell. The implementation lives in inferrsi.knowledge.skill_catalog; +# importing it still requires the optional PyYAML 'skills' extra. from __future__ import annotations -import json -import re -from pathlib import Path, PurePosixPath -from typing import BinaryIO - -try: - import yaml -except ImportError as exc: # The base install stays PyYAML-free by contract. - raise ValueError( - "Reading checkout skills requires the 'skills' extra (PyYAML); install it " - "from your InferRSI checkout, replacing the placeholder path: " - "python -m pip install '[skills]'" - ) from exc - -from .catalog import BACKENDS - - -def _frontmatter(stream: BinaryIO) -> dict: - if stream.readline().strip() != b"---": - raise ValueError("SKILL.md must start with YAML frontmatter") - lines = [] - for line in stream: - if line.strip() == b"---": - break - lines.append(line) - else: - raise ValueError("Unclosed SKILL.md frontmatter") - try: - record = yaml.safe_load(b"".join(lines).decode("utf-8")) - except yaml.YAMLError as exc: - raise ValueError(f"Invalid skill YAML: {exc}") from exc - if not isinstance(record, dict): - raise ValueError("Skill frontmatter must be a mapping") - return record - - -def _skill_name(name: str) -> None: - if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name): - raise ValueError(f"Invalid skill name: {name}") - - -def _directory(root: Path, name: str, relative: str) -> Path: - _skill_name(name) - path = PurePosixPath(relative) - if ( - not relative - or "\\" in relative - or ":" in relative - or path.is_absolute() - or any(part in ("", ".", "..") for part in relative.split("/")) - ): - raise ValueError(f"Skill path must be a relative POSIX path: {relative}") - directory = (root / relative).resolve() - if not directory.is_relative_to(root): - raise ValueError("Skill must be inside the selected root") - if path.name != name or directory.name != name: - raise ValueError(f"Skill path basename disagrees with its name: {name}") - if not _entry(directory, "SKILL.md").is_file(): - raise ValueError(f"Unknown skill: {name}") - return directory - - -def _unique_fields(pairs: list[tuple[str, object]]) -> dict: - record = {} - for key, value in pairs: - if key in record: - raise ValueError(f"Duplicate skill index field: {key}") - record[key] = value - return record - - -def _indexed_directories(root: Path) -> dict[str, Path] | None: - index = _entry(root, "skill-index.json") - if not index.exists(): - return None - record = json.loads(index.read_text(encoding="utf-8"), object_pairs_hook=_unique_fields) - if ( - not isinstance(record, dict) - or type(record.get("schema_version")) is not int - or record["schema_version"] != 1 - or not isinstance(record.get("skills"), dict) - ): - raise ValueError("Skill index requires schema_version 1 and a skills mapping") - directories = {} - paths = set() - for name, entry in sorted(record["skills"].items()): - if ( - not isinstance(entry, dict) - or not isinstance(entry.get("path"), str) - or not isinstance(entry.get("backend"), str) - or entry["backend"] not in (*BACKENDS, "common") - ): - raise ValueError(f"Skill index requires a path and supported backend: {name}") - directory = _directory(root, name, entry["path"]) - if directory in paths: - raise ValueError(f"Duplicate skill index path: {entry['path']}") - directories[name] = directory - paths.add(directory) - return directories - - -def skill_directories(root: Path) -> dict[str, Path]: - """Resolve registered skill locations without reading instruction bodies.""" - root = root.resolve() - if not root.is_dir(): - raise ValueError(f"Skill root is not a directory: {root}") - indexed = _indexed_directories(root) - if indexed is not None: - return indexed - return { - path.parent.name: _directory(root, path.parent.name, path.parent.name) - for path in sorted(root.glob("*/SKILL.md")) - } - - -def skill_directory(root: Path, name: str) -> Path: - """Resolve one named skill in an indexed library or a legacy flat root.""" - _skill_name(name) - root = root.resolve() - indexed = _indexed_directories(root) - if indexed is None: - return _directory(root, name, name) - if name not in indexed: - raise ValueError(f"Unknown skill: {name}") - return indexed[name] - - -def _entry(directory: Path, filename: str) -> Path: - # The resolved entry file must stay inside the selected skill directory; - # reject symlinks that point outside it. - path = (directory / filename).resolve() - if not path.is_relative_to(directory): - raise ValueError(f"Skill {filename} must stay inside the selected skill: {directory.name}") - return path - - -def _sidecar(directory: Path) -> dict | None: - # metadata.json shares the directory-boundary loading path with SKILL.md. - path = _entry(directory, "metadata.json") - if not path.is_file(): - return None - metadata = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(metadata, dict): - raise ValueError(f"Skill metadata must be a mapping: {directory.name}") - return metadata - - -def _references(metadata: dict, name: str) -> list[str]: - # references must be a list of strings; a string value would degrade - # membership testing to substring matching. - references = metadata.get("references", []) - if not isinstance(references, list) or any(not isinstance(item, str) for item in references): - raise ValueError(f"Skill references must be a string list: {name}") - return references - - -def _strings(record: dict, key: str, name: str) -> None: - """A declared string list must be strings; [] is a deliberate empty value.""" - if key not in record: - return - listed = record[key] - if not isinstance(listed, list) or any( - not isinstance(item, str) or not item.strip() for item in listed - ): - raise ValueError(f"Skill {key} must be a list of non-empty strings: {name}") - - -def _applicability(record: dict, name: str) -> None: - """Conditions a procedure assumes. Descriptive only; never a validation claim.""" - if "applicability" not in record: - return - declared = record["applicability"] - if not isinstance(declared, dict): - raise ValueError(f"Skill applicability must be a mapping: {name}") - for key, value in declared.items(): - if isinstance(value, str): - if not value.strip(): - raise ValueError(f"Skill applicability.{key} is empty: {name}") - elif isinstance(value, list): - if not value or any(not isinstance(item, str) or not item.strip() for item in value): - raise ValueError( - f"Skill applicability.{key} must be a non-empty string list: {name}" - ) - else: - raise ValueError(f"Skill applicability.{key} must be a string or string list: {name}") - - -def _effective(directory: Path) -> dict: - # Read bytes line by line; parse only the frontmatter, never decode the body. - with _entry(directory, "SKILL.md").open("rb") as stream: - record = _frontmatter(stream) - for key in ("name", "description"): - if not isinstance(record.get(key), str) or not record[key].strip(): - raise ValueError(f"Skill requires {key}: {directory.name}") - if record["name"] != directory.name: - raise ValueError(f"Skill name disagrees with directory: {directory.name}") - metadata = _sidecar(directory) - if metadata is not None: - for key in record.keys() & metadata.keys(): - if record[key] != metadata[key]: - raise ValueError(f"Conflicting skill metadata field {key}: {directory.name}") - if metadata.get("profile") == "local-skill-v1": - if any(key in metadata for key in ("name", "description")): - raise ValueError("Local skill name and description belong in SKILL.md") - if set(record) != {"name", "description"}: - raise ValueError("Local skill structured fields belong in metadata.json") - # Only sidecars declare these fields, and JSON cannot express a container - # cycle; validating the sidecar leaves the bundle normalizer free to report - # a cyclic legacy frontmatter value against the field that contains it. - _strings(metadata, "recommendations", directory.name) - _applicability(metadata, directory.name) - if "searchspace_not_applicable" in metadata: - reason = metadata["searchspace_not_applicable"] - if not isinstance(reason, str) or not reason.strip(): - raise ValueError( - f"Skill searchspace_not_applicable requires a reason: {directory.name}" - ) - record.update(metadata) - _references(record, directory.name) - if "status" in record and not isinstance(record["status"], str): - raise ValueError(f"Skill status must be a string: {directory.name}") - if "tags" in record and ( - not isinstance(record["tags"], list) - or any(not isinstance(tag, str) for tag in record["tags"]) - ): - raise ValueError(f"Skill tags must be a string list: {directory.name}") - return record - - -def _metadata(directory: Path) -> dict: - record = _effective(directory) - return {key: record[key] for key in ("name", "description", "status", "tags") if key in record} - - -def skill_metadata(root: Path, name: str) -> dict: - """Effective legacy frontmatter plus sidecar fields, after conflict checks.""" - return _effective(skill_directory(root, name)) - - -def skill_records(root: Path) -> list[dict]: - """Project metadata only; this projection is not an authoritative matrix.""" - return [_metadata(directory) for directory in skill_directories(root).values()] - - -def get_skill(root: Path, name: str) -> dict: - """Read selected metadata and resource names, never resource contents.""" - directory = skill_directory(root, name) - record = _metadata(directory) - record["resources"] = [ - path.relative_to(directory).as_posix() - for path in sorted(directory.rglob("*")) - if path.is_file() and path.name != "SKILL.md" - ] - return record - - -def skill_instructions(root: Path, name: str) -> str: - """Load selected instructions. Reading does not authorize their commands.""" - directory = skill_directory(root, name) - _metadata(directory) - with _entry(directory, "SKILL.md").open("rb") as stream: - _frontmatter(stream) - return stream.read().decode("utf-8").lstrip("\r\n") - - -def skill_resource(root: Path, name: str, resource: str) -> Path: - """Resolve one explicitly selected resource inside its skill directory.""" - directory = skill_directory(root, name) - path = (directory / resource).resolve() - if not path.is_relative_to(directory) or path == directory / "SKILL.md": - raise ValueError("Resource must be inside the selected skill; use show for instructions") - if not path.is_file(): - raise ValueError(f"Unknown skill resource: {resource}") - return path - - -def skill_reference(root: Path, name: str, reference: str) -> Path: - """Resolve a declared checkout reference without escaping the skills root.""" - directory = skill_directory(root, name) - if reference not in _references(_effective(directory), directory.name): - raise ValueError(f"Undeclared skill reference: {reference}") - if "\\" in reference or Path(reference).is_absolute(): - raise ValueError("Reference must use a relative POSIX path") - path = (directory / reference).resolve() - if not path.is_relative_to(root.resolve()): - raise ValueError("Reference must stay inside the skills root") - if not path.is_file(): - raise ValueError(f"Missing skill reference: {reference}") - return path +from inferrsi.knowledge.skill_catalog import ( + _applicability, + _directory, + _effective, + _entry, + _frontmatter, + _indexed_directories, + _metadata, + _references, + _sidecar, + _skill_name, + _strings, + _unique_fields, + get_skill, + skill_directories, + skill_directory, + skill_instructions, + skill_metadata, + skill_records, + skill_reference, + skill_resource, +) + +__all__ = [ + "skill_directories", + "skill_directory", + "skill_metadata", + "skill_records", + "get_skill", + "skill_instructions", + "skill_resource", + "skill_reference", + "_frontmatter", + "_skill_name", + "_directory", + "_unique_fields", + "_indexed_directories", + "_entry", + "_sidecar", + "_references", + "_strings", + "_applicability", + "_effective", + "_metadata", +] diff --git a/tests/test_architecture_closed_loop.py b/tests/test_architecture_closed_loop.py index 98f3fe4ea..8ce862e5f 100644 --- a/tests/test_architecture_closed_loop.py +++ b/tests/test_architecture_closed_loop.py @@ -377,7 +377,8 @@ def test_real_ports_stay_on_three_existing_seams(self) -> None: self.assertIn("evaluate", calls) self.assertNotIn("search_knowledge", calls) self.assertNotIn("collect", calls) - knowledge = _import_modules(ROOT / "src/inferrsi/knowledge_operations.py") + # The root module is a static shell; check the implementation's imports. + knowledge = _import_modules(ROOT / "src/inferrsi/knowledge/knowledge_operations.py") harness = _import_modules(ROOT / "src/inferrsi/harness/scenarios/external.py") self.assertFalse(any("harness" in name or "evaluation" in name for name in knowledge)) self.assertFalse(any("knowledge" in name or "evaluation" in name for name in harness)) diff --git a/tests/test_bundle_manifest.py b/tests/test_bundle_manifest.py index 1b0b8d8dc..c3a6d2257 100644 --- a/tests/test_bundle_manifest.py +++ b/tests/test_bundle_manifest.py @@ -4,7 +4,7 @@ import pytest -from inferrsi import knowledge_operations +from inferrsi.knowledge import knowledge_operations from inferrsi.knowledge_operations import Bundle, RevisionConflict @@ -70,6 +70,8 @@ def counted(data): calls[data] += 1 return _hash(data) + # Patch implementation globals; the root module is a static re-export shell. + assert Bundle.manifest.__globals__ is vars(knowledge_operations) monkeypatch.setattr(knowledge_operations, "_hash", counted) for count in (1, 2): bundle.manifest() diff --git a/tests/test_cli_catalog.py b/tests/test_cli_catalog.py index 522f1be90..90eaa3093 100644 --- a/tests/test_cli_catalog.py +++ b/tests/test_cli_catalog.py @@ -17,6 +17,7 @@ from inferrsi.arguments import make_spec, parser from inferrsi.catalog import get_knowledge, knowledge_records, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.knowledge import catalog as knowledge_catalog PROJECT = Path(__file__).resolve().parents[1] SOURCE_REPOSITORIES = { @@ -186,7 +187,8 @@ def test_misfiled_or_overstated_record_cannot_leak_into_context( ) destination.parent.mkdir(parents=True) destination.write_text(json.dumps(record)) - monkeypatch.setattr(catalog, "files", lambda _: tmp_path) + # Patch implementation globals; the root module is a static re-export shell. + monkeypatch.setattr(knowledge_catalog, "files", lambda _: tmp_path) with pytest.raises(ValueError, match=error): search_knowledge(backend="cuda", scenario="diffusion.image_generation") diff --git a/tests/test_collect_limit_serialization.py b/tests/test_collect_limit_serialization.py index 00153e5f2..7f8ffd1d8 100644 --- a/tests/test_collect_limit_serialization.py +++ b/tests/test_collect_limit_serialization.py @@ -1,7 +1,7 @@ """Count saturation skips only redundant normal-view payload sizing.""" import pytest -from inferrsi import knowledge_operations as operations +from inferrsi.knowledge import knowledge_operations as operations from inferrsi.knowledge_operations import Bundle, KnowledgeReader @@ -44,6 +44,8 @@ def observe(value): seen.append(value["id"]) return original(value) + # Patch implementation globals; the root module is a static re-export shell. + assert KnowledgeReader.collect.__globals__ is vars(operations) monkeypatch.setattr(operations, "_json", observe) return seen diff --git a/tests/test_knowledge_compatibility_shells.py b/tests/test_knowledge_compatibility_shells.py new file mode 100644 index 000000000..4be152ef6 --- /dev/null +++ b/tests/test_knowledge_compatibility_shells.py @@ -0,0 +1,89 @@ +# K01: root Knowledge names are static re-exports of the moved implementations. + +from __future__ import annotations + +import hashlib +import importlib +import os +import subprocess +import sys + +import pytest + +from tests._gate_policy_support import REPO, ROOT, blocker_kinds, first_wave_report, load_manifest + +MANIFEST = load_manifest() +LEAF = MANIFEST["technical_leaf"] +LEAF_NAMES = MANIFEST["catalog_two_routes"]["neutral_leaf_export"]["symbols"] +MOVED = [item for item in MANIFEST["compatibility_pairs"] if "exports" in item] + + +@pytest.mark.parametrize("item", MOVED, ids=lambda item: item["importer"]) +def test_root_names_are_the_moved_objects(item): + # K01-1: identity carries signatures, defaults, exception types and Path behavior. + root = importlib.import_module(f"inferrsi.{item['importer']}") + moved = importlib.import_module(item["target"]) + names = [*item["exports"]["public"], *item["exports"]["private_used"]] + for name in names: + assert getattr(root, name) is getattr(moved, name), name + extra = LEAF_NAMES if item["importer"] == "catalog" else [] + # REPO_TYPES is listed on both catalog routes; __all__ names it once. + assert len(root.__all__) == len(set(root.__all__)) + assert set(root.__all__) == {*names, *extra} + + +def test_catalog_routes_share_one_repo_types_tuple(): + from inferrsi import _repo_pack_data, catalog + from inferrsi.knowledge import catalog as knowledge_catalog + + assert catalog.REPO_TYPES is _repo_pack_data.REPO_TYPES is knowledge_catalog.REPO_TYPES + assert catalog.repository_pack is _repo_pack_data.repository_pack + assert not hasattr(knowledge_catalog, "repository_pack") + assert knowledge_catalog.KNOWLEDGE_SCOPES["contribution"] == ( + *_repo_pack_data.REPO_TYPES, "vllm_ascend", "inferrsi" + ) + + +def test_data_leaf_holds_the_recorded_fragment_and_locked_hash(): + data = (ROOT / "_repo_pack_data.py").read_bytes() + assert hashlib.sha256(data).hexdigest() == LEAF["file_sha256"] + assert LEAF["original_fragment"]["source"] + "\n" in data.decode("utf-8") + + +def _fresh(script: str) -> None: + environment = {**os.environ, "PYTHONPATH": str(REPO / "src")} + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, env=environment, + check=False, + ) + assert result.returncode == 0, result.stderr + + +def test_package_init_and_infra_inspection_load_no_knowledge_body(): + _fresh( + "import sys\n" + "import inferrsi.knowledge\n" + "assert not [m for m in sys.modules if m.startswith('inferrsi.knowledge.')]\n" + "import inferrsi.infra.inspection\n" + "assert 'inferrsi.catalog' not in sys.modules\n" + "assert not [m for m in sys.modules if m.startswith('inferrsi.knowledge.')]\n" + "assert 'inferrsi._repo_pack_data' in sys.modules\n" + ) + + +def test_knowledge_closure_is_isolated_while_overall_stays_false(): + # K01-5: the Knowledge package exists and its closure is clean, but the + # allowlist, the registered residual and harness -> model/infra remain. + # After #338, runtime resource loaders keep new_core_closure_isolated + # false; that is not a Knowledge-closure regression. + report = first_wave_report() + assert report.violations == () + assert "absent: knowledge/" not in report.blockers + assert { + "allowlist", + "residual", + "harness reaches model/infra", + "runtime resource loaders", + } <= blocker_kinds(report) + assert not report.new_core_closure_isolated + assert not report.isolation_complete diff --git a/tests/test_knowledge_model_policy.py b/tests/test_knowledge_model_policy.py index 145e22ffd..46982928e 100644 --- a/tests/test_knowledge_model_policy.py +++ b/tests/test_knowledge_model_policy.py @@ -173,6 +173,8 @@ def test_stale_head_and_unsupported_schema_fail(): "docs/reference-matrix.md", "docs/searchspace-matrix.md", "docs/knowledge-base.md", "docs/knowledge-operations.md", "docs/knowledge-store.md", "src/inferrsi/catalog.py", "src/inferrsi/skill_catalog.py", "src/inferrsi/knowledge_operations.py", + "src/inferrsi/knowledge/catalog.py", "src/inferrsi/knowledge/skill_catalog.py", + "src/inferrsi/knowledge/knowledge_operations.py", ]) def test_explicit_protected_paths_require_declarations(policy, path): item = pr() diff --git a/tests/test_knowledge_store.py b/tests/test_knowledge_store.py index 3a4bb9b0f..bdc86a8c3 100644 --- a/tests/test_knowledge_store.py +++ b/tests/test_knowledge_store.py @@ -370,6 +370,7 @@ def test_package_only_collect_without_checkout_or_yaml(tmp_path): "assert item['authority'] == 'package', item\n" "assert reader.read(item['ref'], revision=item['revision']).revision == item['revision']\n" "assert 'inferrsi.skill_catalog' not in sys.modules\n" + "assert 'inferrsi.knowledge.skill_catalog' not in sys.modules\n" ) env = {**os.environ, "PYTHONPATH": str(REPO / "src"), "PYTHONUTF8": "1"} result = subprocess.run( diff --git a/tests/test_knowledge_tiers.py b/tests/test_knowledge_tiers.py index 1f2772315..163572343 100644 --- a/tests/test_knowledge_tiers.py +++ b/tests/test_knowledge_tiers.py @@ -10,6 +10,7 @@ from inferrsi.arguments import parser from inferrsi.catalog import get_knowledge, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.knowledge import catalog as knowledge_catalog ROOT = Path(__file__).resolve().parents[1] / "src/inferrsi/data/knowledge_packs" @@ -86,7 +87,8 @@ def test_misclassified_external_card_is_rejected_before_retrieval( ) target.parent.mkdir(parents=True) target.write_text(json.dumps(record)) - monkeypatch.setattr(catalog, "files", lambda _: tmp_path) + # Patch implementation globals; the root module is a static re-export shell. + monkeypatch.setattr(knowledge_catalog, "files", lambda _: tmp_path) with pytest.raises(ValueError, match="tier disagrees"): search_knowledge(tier="contribution") diff --git a/tests/test_research.py b/tests/test_research.py index c62bde6b2..da25c12af 100644 --- a/tests/test_research.py +++ b/tests/test_research.py @@ -9,6 +9,7 @@ from inferrsi import catalog from inferrsi.catalog import get_knowledge, search_knowledge from inferrsi.cli import main +from inferrsi.knowledge import catalog as knowledge_catalog from inferrsi.research import research_knowledge @@ -88,7 +89,7 @@ def test_independent_queries_are_unioned_fairly_and_exact_search_is_unchanged(mo ("f-overlap", "ExampleModel mlx diffusion video offload"), ) ] - monkeypatch.setattr(catalog, "knowledge_records", lambda: records) + monkeypatch.setattr(knowledge_catalog, "knowledge_records", lambda: records) exact = search_knowledge("ExampleModel mlx diffusion video offload") assert [record["id"] for record in exact] == ["f-overlap"] result = research_knowledge( @@ -117,7 +118,7 @@ def test_query_attribution_survives_the_shared_output_limit(monkeypatch): {"id": "model", "summary": "SyntheticModel mlx"}, {"id": "framework", "summary": "mlx mlx"}, ] - monkeypatch.setattr(catalog, "knowledge_records", lambda: records) + monkeypatch.setattr(knowledge_catalog, "knowledge_records", lambda: records) result = research_knowledge("SyntheticModel", frameworks=["mlx"], limit=1) assert result["items"] == [{ "record": records[0], diff --git a/tests/test_skill_eval_optional_dependencies.py b/tests/test_skill_eval_optional_dependencies.py index 3a243090c..ffc0ed65e 100644 --- a/tests/test_skill_eval_optional_dependencies.py +++ b/tests/test_skill_eval_optional_dependencies.py @@ -30,6 +30,7 @@ def test_comparison_reporting_without_yaml(self): assert all(item['input_tokens'] is None for item in summary.values()) assert callable(read_report) assert 'inferrsi.skill_catalog' not in sys.modules + assert 'inferrsi.knowledge.skill_catalog' not in sys.modules """) def test_checkpoint_without_yaml(self): @@ -41,6 +42,7 @@ def test_checkpoint_without_yaml(self): assert result.action == 'repair_evidence' assert result.reasons == ('missing_evidence',) assert 'inferrsi.skill_catalog' not in sys.modules + assert 'inferrsi.knowledge.skill_catalog' not in sys.modules """) def test_daily_controller_without_yaml(self): @@ -55,6 +57,7 @@ def test_daily_controller_without_yaml(self): else: raise AssertionError('Daily controller accepted missing budget configuration') assert 'inferrsi.skill_catalog' not in sys.modules + assert 'inferrsi.knowledge.skill_catalog' not in sys.modules """) def test_selected_snapshot_still_requires_yaml_before_side_effects(self): diff --git a/tests/test_skill_library_readers.py b/tests/test_skill_library_readers.py index 945deb7ee..02924e673 100644 --- a/tests/test_skill_library_readers.py +++ b/tests/test_skill_library_readers.py @@ -220,7 +220,8 @@ def test_package_catalog_and_reader_do_not_require_yaml(): "records = knowledge_records()\n" "identity = next(r['id'] for r in records if r.get('skill_documents'))\n" "assert KnowledgeReader().read(identity).content\n" - "assert 'inferrsi.skill_catalog' not in sys.modules\n", + "assert 'inferrsi.skill_catalog' not in sys.modules\n" + "assert 'inferrsi.knowledge.skill_catalog' not in sys.modules\n", ], capture_output=True, text=True,