From af2c48683afba7f73889aacadb3bd22fafa28961 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Tue, 17 Nov 2026 20:21:01 +0800 Subject: [PATCH 1/5] feat(knowledge): add external read-only channels to find `find` could only answer from this repository, which holds six Ascend content cards and no per-API card at all. The external cannbot-knowledge corpus has 10,550 cards, 2,993 of them Ascend C API cards, and it is CANN-licensed: copying it into the package or an evidence tree would be a distribution the repository cannot make. This adds it as a read-only channel instead, queried in place. Design points the code enforces rather than documents: - External hits never enter `items`. They were not ranked on this corpus and are not locally verified, so a caller reading only `items` cannot mistake them for validated knowledge. - Each channel reports its own state. A failure is `unreachable`, `timeout`, `index_stale`, `not_enabled` and so on, never an empty list: "searched and found nothing" and "could not search" are different facts. - A timeout is reported with `elapsed_ms`, not rendered as no match. - Scores carry `score_scale`. Two corpora of different sizes have incomparable idf values, so a cross-channel merge would be a fabricated number. - The task backend is declarative, never inferred from the query. It comes from `--backend`, then from the approved grant, and is absent otherwise. Reading `Channel.backend` as the task backend made an undeclared task search anyway; the registry now carries the declaration separately. `--no-external` reproduces the pre-channel output exactly, and a registry with no channels returns the original result untouched, so existing callers and the CI baseline are unaffected. The grant records an interpreter path without resolving symlinks. Resolving it lands on the base interpreter of a venv and loses pyyaml, which passed the enable probe and failed every later search. Coverage measured on six Ascend queries against a checkout at 92dc2607: local matches 327, with the channel 927 at the new default cap of 100. CeilDivision matches nothing locally and returns 100 locators upstream. Refs #476 Co-Authored-By: Claude Fable 5.2 --- ...11-17-cannbot-knowledge-integration-rfc.md | 503 ++++++++++++++++++ .../2026-11-17-external-knowledge-channels.md | 457 ++++++++++++++++ src/inferrsi/apps/cannbot/__init__.py | 98 ++++ src/inferrsi/apps/cannbot/project.py | 257 +++++++++ src/inferrsi/apps/cannbot/read_worker.py | 135 +++++ src/inferrsi/apps/cli/__init__.py | 81 ++- src/inferrsi/apps/cli/arguments.py | 45 ++ src/inferrsi/knowledge/channels.py | 317 +++++++++++ src/inferrsi/knowledge/knowledge_find.py | 60 ++- tests/test_cannbot_channel_acceptance.py | 152 ++++++ tests/test_knowledge_channels.py | 213 ++++++++ 11 files changed, 2315 insertions(+), 3 deletions(-) create mode 100644 docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md create mode 100644 docs/plans/2026-11-17-external-knowledge-channels.md create mode 100644 src/inferrsi/apps/cannbot/__init__.py create mode 100644 src/inferrsi/apps/cannbot/project.py create mode 100644 src/inferrsi/apps/cannbot/read_worker.py create mode 100644 src/inferrsi/knowledge/channels.py create mode 100644 tests/test_cannbot_channel_acceptance.py create mode 100644 tests/test_knowledge_channels.py diff --git a/docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md b/docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md new file mode 100644 index 00000000..e79a6364 --- /dev/null +++ b/docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md @@ -0,0 +1,503 @@ +# [RFC] 全量接入 cannbot-knowledge:MCP 常驻查询通道设计 + +> **Status: Proposed — 形态已选定,实施设计待评审。** 形态选择已由维护者确认(见 §0);本文余下部分定义接入契约与切片,仍待评审接受。接受本文不等于授权实施、代码搬家或对外发布。 +> Governing dependency: [#388](https://github.com/InferMatrixLab/InferRSI/issues/388)(外部技能/知识依赖治理)。本文是其"外链或 submodule 待决"的候选实现,落实为选定的第三方机制,但**不关闭 #388**。 +> 接口依赖: [#387](https://github.com/InferMatrixLab/InferRSI/issues/387)(`read` / `collect` 公开接口,仍为 Proposed)。 +> 文中所有上游 revision 与统计值都是**写作时的实测观测值**,不是本仓 pin,也不是验收基线。 + +## 0. 决策摘要 + +| 项 | 决定 | 位置 | +| --- | --- | --- | +| 接入形态 | **只读 MCP 常驻查询通道**(方案 D 升格为主接法);内容永不进入本仓、wheel、`evidence/` | §5.1 | +| 触发语义 | **平台门控命中即查**,与本地是否命中无关;不再"零命中才补漏" | §5.2 | +| 平台门控来源 | 显式声明(`--backend` 或 grant),**不推断**;`--backend` 只接受单值 | §5.2 | +| 结果呈现 | **分栏并列**,外部结果不进 `items`,分数带 `score_scale`,不跨通道合并 | §5.3 | +| 每通道条数 | `cannbot-knowledge` 10 / `mindie-agent` 5 | §5.4 | +| `find` 默认 | **默认带外部查询**;CI 与单测必须显式 `--no-external` | §5.4 | +| MCP 面 | **透传**上游 `discover` / `search` / `read_card`,不收窄 | §8.5 | +| 证据 | receipt 只存 `locator` + `card_sha256` + `revision`,不含正文 | §7 | + +### 为何从"候选比较"走到"选定 MCP" + +前几轮评审已比较过五种形态(包内转换、submodule、外链锁缓存、MCP、混合)。选定 MCP 的决定性理由有两条,其余形态无法同时满足: + +1. **许可**:上游为 CANN OSL v2.0,本仓为 Apache-2.0。MCP 让内容永不落盘,许可冲突从机制上消失。 +2. **产能**:上游已有 10,550 张卡与自身的 ingest/lint/index 生产系统。任何"复制或缓存进本仓"的形态都在与一个数量级的产能赛跑。 + +## 1. 现状:InferRSI 的 Ascend / CANN 知识极度匮乏 + +这是本 RFC 的起因。在 `main` `6c7e237935a22cc5e90e954d4f397376e753e7d8`(2026-11-16)上实测 `src/inferrsi/knowledge/data/`: + +| 项 | 实测值 | 说明 | +| --- | --- | --- | +| 打包知识数据总量 | 7.8 MiB / 850 个 md+json / 921 个文件 | 含全部 skill、引用与记录 | +| 知识记录总数 | 128 | `repo_packs` 108 + `knowledge_packs` 20 | +| 其中 Ascend 记录 | **22** | 占 17.2% | +| Ascend 记录中真正的内容卡 | **6** | 3 条 Qwen3-TTS audio、3 条 vLLM-Omni diffusion | +| Ascend 记录中仅是 skill 指针 | 16 | 只指向 `skills` 组件,不承载技术结论 | +| Ascend 硬件 skill | 14 / 143 个 `SKILL.md` | 380 KiB,43 个文件 | +| Ascend skill 正文 | 131,860 字节 | 14 篇 `SKILL.md` 合计 | +| Ascend skill 引用资料 | 174,559 字节 / 29 个文件 | 含 3 份上游 LICENSE 抄本 | +| **per-API 粒度知识卡** | **0** | 全仓没有任何一张卡对应单个 CANN/AscendC API | +| 出现 `aclnn*` / `__aicore__` 签名的文件 | **1** | `frameworks/torch-npu/.../references/development.md` | + +对照上游 cannbot-knowledge 的同一维度: + +| 项 | 上游实测值 | 倍数 | +| --- | --- | --- | +| 知识卡总数 | 10,550 | 内容卡 6 → 10,550(约 1,758×) | +| `ops/ascendc` 领域卡 | 6,804 | — | +| `API` 类型卡 | 3,875 | 本仓 0 | +| 其中 Ascend C API 卡 | **2,993** | — | +| `runtime` API 卡 | 516 | 本仓 0 | +| `Operator` 卡 | 903 | 本仓 0 | +| `Runbook` 卡 | 2,362 | 本仓无对应类型 | + +**这是粒度差,不只是数量差。** InferRSI 现有 Ascend 知识是**技能粒度**:14 篇 `SKILL.md`,讲"怎么设计 tiling""怎么抓 ACLGraph""怎么诊断算子瓶颈"这类流程。上游是**API 与算子粒度**:单张卡讲清 `CeilDivision` 的参数、单位、平台约束、失效边界。两类内容不可互相替代——技能粒度无法回答"这个 API 在 a3 上是否支持、参数语义是什么",而现有仓恰好完全缺失后者。 + +**为什么既有路径补不上。** [#91](https://github.com/InferMatrixLab/InferRSI/issues/91) 的蒸馏路径是"人力策展 + 逐条 PR",本仓至今累计 6 条 Ascend 内容卡;按同一速度覆盖 2,993 张 API 卡在工程上不成立。而且这 6 条内容卡全部绑定具体模型 campaign(Qwen3-TTS 音码、MiniMax-H3 KV 复用、Qwen-Image 预热),描述的是"某模型在某卡上的某次优化",不是可复用的通用 Ascend 机制。 + +**匮乏不等于现有内容有错。** 那 14 篇 skill 多数经 campaign 验证,质量可用;问题纯粹是覆盖面——`aclnn` / AscendC API / CANN Runtime / 算子实现这四类基础层次在本仓几乎是空白,导致 Agent 做 Ascend 任务时只能依赖 campaign 记忆与现场搜索,无法像 CUDA 路径那样先查知识再动手。 + +**这也解释了为什么不能只靠"蒸馏":** 上游的知识生产侧(`knowledge-ingest`、两个 Operator producer、trace ingest)已经是一个持续产出系统,本仓若只做消费端的手工蒸馏,永远落后一个数量级的产能。本 RFC 讨论的正是"以什么形态消费这套外部产能",而不是"要不要再造一套"。 + +## 2. 上游形态与许可(实测) + +写作时对上游 `master` 的一次浅克隆观测(commit 观测值 `92dc260756e91e72e6dcd60f3996a4d54f43c8f5`,2026-11-16): + +| 项 | 值 | +| --- | --- | +| 知识正文根 | `knowledge/`,OKF v0.2 唯一 Bundle,Concept ID = Bundle 相对路径去 `.md` | +| 概念卡 / 导航 | 10,550 张 / 818 个 `index.md`(不参与检索) | +| 正文字节 | 约 118 MiB(不含导航);目录占用约 143 MB | +| 领域分布 | `ops` 7,440 · `model` 1,890 · `contrib` 653 · `runtime` 530 · `common` 21 · `tools` 16 · `graph` 0(保留结构入口) | +| 状态分布 | `stable` 6,588 · `draft` 3,698 · `deprecated` 41 | +| 类型分布 | `API` 3,875 · `Runbook` 2,362 · `Guide` 1,788 · `Optimization` 1,087 · `Operator` 903 · `Concept` 287 · `Example` 23 · `Glossary` 2 | +| 平台受控值 | `a2` 1,765 · `a3` 1,610 · `310p` 395 · `agnostic` 228 · `kirin_x90` 198 · `kirin_9030` 172 | +| 写作语言 | 标题、描述与正文说明以中文写作;API 名、签名与代码为英文 | + +**上游不是一个资料包,是一个知识系统:** + +- `governance/` 有机器可读的 schema、路径 Profile、受控值规则与 20 余个 check 实现(来源、链接、索引同步、生命周期、信任层级)。 +- 检索索引版本化(`cannbot-knowledge-index.v1`),带 `content_fingerprint` 与 `built_repository_state.commit`;索引相对当前提交过期时**拒绝服务**。 +- `mcp-server/` 提供只读 `discover` / `search` / `read_card`,stdio 与 HTTP 双传输。 +- 卡的来源资源**不在 Git 内**:`sources[].resource` 常指向 `cann-docs-raw/...` 或 `cann-recipes-raw///...`,由 `get_resources.sh` 恢复,默认不提交。**单张卡本身不自包含证据闭包。** + +**许可。** 上游 LICENSE 为 CANN Open Software License Agreement v2.0,不是 OSI 许可:授予 limited、royalty-free、**non-transferable、non-sublicensable、revocable** 的许可,用途限于华为 AI 处理器系统。本仓为 Apache-2.0。把该内容打包进 `src/inferrsi/knowledge/data/` 并随 sdist/wheel 分发,或写入 `evidence/`,都会形成不可撤销的分发承诺。(本条为 LICENSE 文本的读法,**不是法律意见**,需要组织确认。) + +**关于本仓内的 12 个 skill:** cannbot-knowledge 的 `.agents/skills/` 有 12 个 skill,但它们是知识仓自己的工具链——三个消费流程(`knowledge-query` 等,就是 MCP 背后的 CLI)、五个生产流程(`*-ingest`、operator producer)、两个治理(`knowledge-lint`、`knowledge-issue-handler`)、两个带 `VENDORED.md` 的裁剪副本。**CANN 领域知识全在 `knowledge/`,MCP 三工具已完整覆盖,不存在"skill 覆盖不到"的缺口。** + +## 3. 形态选择记录 + +五种候选的比较与结论(保留作为决策依据,不再并行生效): + +| 形态 | 许可可分发 | 离线复现 | 漂移检测 | 结论 | +| --- | --- | --- | --- | --- | +| A 外链锁 + 包外缓存 | ✅ 不进包 | ✅ digest | ✅ | 未选:仍需同步与自建索引,收益低于 MCP | +| A′ 索引先行 | ✅ | ✅ | ✅ | 未选:同 A | +| B git submodule(KDA 式) | ⚠️ 需仓库层处理许可 | ✅ | ⚠️ 仅指针 | 未选:把许可义务交给集成方,且 143 MB 随克隆 | +| **C 全量转换进本仓** | ❌ 冲突 | ✅ | ❌ 副本即漂移 | **判死**:许可阻断 + 15 倍体积 + 每次上游前进需全量重转 | +| **D 只读 MCP** | ✅ 不落地 | ⚠️ 依赖上游 checkout | ✅ 由上游索引保证 | **选定**:唯一同时满足许可与规模,且上游已自带治理与索引 | +| E 混合 | — | — | — | 收敛为 D;C 仅以 [#91](https://github.com/InferMatrixLab/InferRSI/issues/91) 蒸馏精选子集的形式保留 | + +KDA 的可借之处仍是纪律:README 锁表、`third_party_licenses/` 逐件许可、以及"submodule 指向做过去敏的发布点而非上游任意 commit"。本方案把这三条落在 §7 的 receipt 与 §5.3 的 `revision` 字段上,而不引入 submodule 这一形式。 + +## 4. 本仓检索侧事实 + +| 项 | 值 | +| --- | --- | +| `find` 实现 | `src/inferrsi/knowledge/knowledge_find.py`,commit `61303fa8`(#466 统一检索) | +| 检索模型 | 纯词法:`idf × 字段权重 × sqrt(出现次数)`,`FIELD_WEIGHTS` 见 `knowledge_find.py:42` | +| 来源 | `kinds=("case","skill","record")`,`load_corpus` 见 `:620` | +| 语料规模 | 143 篇 `SKILL.md` + 128 条记录 + cases,同进程内构建 | +| 确定性 | 完全离线、可复现、无 I/O 依赖 | +| 中文处理 | `tokens()` 已内建 CJK 双字切分(`:87`);`_BACKEND_NAME_ALIASES` 含 `昇腾 → ascend` | +| 权威源枚举 | `AUTHORITIES = ("package", "checkout", "store")`(`knowledge_operations.py:25`) | + +上游 MCP 侧实测:索引构建 73.9 秒(7,921,396 postings);查询**冷启 3.67 秒 / 热 1.46 秒**;结果字段 `rank` / `score` / `match_basis` / `path` / `local_path` / `card_lines` / `card_sha256`;响应 schema `knowledge-search.v5`。 + +## 5. 接入设计 + +### 5.1 通道抽象 + +新增 `src/inferrsi/knowledge/channels.py`,把外部源收敛到同一协议,使 `find` 不认识具体是 cannbot 还是 MindIE: + +```python +class Channel(Protocol): + name: str # "cannbot-knowledge" | "mindie-agent" + backend: str | None # "ascend" | None(None = 平台无关) + def describe(self) -> ChannelStatus: ... # 探活,可缓存 + def search(self, query: str, *, platform: str | None, + limit: int) -> ChannelResult: ... # 返回定位符,不返回正文 + def read(self, locator: str) -> str: ... # 单独调用,不在 find 热路径 +``` + +```python +@dataclass +class ChannelStatus: + state: str # ok | not_enabled | unreachable | index_stale | timeout + # | version_mismatch | rejected | unknown | backend_unknown + revision: str | None # 上游 commit / catalog generation + detail: str = "" + fix: str = "" # 修复命令,例如索引重建 + checked_at: float = 0.0 + +@dataclass +class ChannelResult: + status: ChannelStatus + items: list[ChannelHit] # 失败时为空,但 state 必须说明原因 + elapsed_ms: int +``` + +`ChannelHit` 字段固定:`locator` / `title` / `summary` / `score` / `score_scale` / `status` / `platforms` / `card_sha256`。 + +`score_scale` 是关键:标明分数来自哪套评分(`"cannbot/knowledge-search.v5"` 或 `"mindie/graded-0.13.13"`),从结构上阻止调用方跨通道比较。 + +### 5.2 平台门控(决定"要不要查") + +```text +find(query, backend=?) + │ + ├─ backend 未声明 ──► 不查外部;external[] 每项 state="backend_unknown" + │ 且 fix 给出 "inferrsi find --backend ascend" + │ + ├─ backend == ascend ──► 查询所有 backend ∈ {ascend, None} 的已授权通道 + │ (cannbot-knowledge + mindie-agent) + │ + └─ backend == cuda/mlx/... ──► 不查外部;external[] 为空 +``` + +`backend` 来源优先级:`--backend` 显式参数 > grant 文件声明 > `backend_unknown`(**不猜**)。 + +`--backend` **只接受单值**。多平台任务由调用方分两次查询并各自保留结果。理由:多值会让输出无法回答"这条外部结果是为哪个平台召回的",而上游平台纪律本身 fail-closed(`--platform` 必填、不混合各平台结果),本仓不应在门控层放宽这一语义。 + +这里刻意**不使用** `named_backends()`(`knowledge_find.py:739`)那种从查询词推断的方式:从"query 里出现了 asc"推断"这是 Ascend 任务"不可靠,且失败形态是静默查错平台。门控必须是显式事实。 + +### 5.3 分栏呈现(决定"怎么放") + +**不做混合排序。** 本仓 `idf` 建立在 143+128 条语料上,上游建立在 10,550 张卡上;同一个词在两套体系里的权重完全不同。任何跨通道合并都要发明归一化函数——那正是 RSI 明令禁止的"编造分数"。 + +```json +{ + "query": "AscendC tiling 参数", + "backend": "ascend", + "items": [ /* 只放本仓结果,沿用现有 ranked list */ ], + "matched": 3, + "external": [ + { + "channel": "cannbot-knowledge", + "state": "ok", + "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", + "elapsed_ms": 1520, + "score_scale": "cannbot/knowledge-search.v5", + "items": [{ + "locator": "ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling", + "title": "Matmul Tiling类使用说明", + "score": 18.4, + "status": "stable", + "platforms": ["a2", "a3"], + "card_sha256": "049b4824...", + "open": "inferrsi cannbot read ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling" + }] + }, + { + "channel": "mindie-agent", + "state": "not_enabled", + "detail": "no project grant for this checkout", + "fix": "inferrsi mindie enable --domain " + } + ], + "notes": [ + "External results are not locally verified; scores are not comparable across channels.", + "Pin external revisions from this output when citing them." + ] +} +``` + +三条不可退让的规则: + +1. **`items` 只放本仓结果。** 外部命中不进 `items`,避免下游按 `items` 处理时静默把未验证内容当成已验证。 +2. **每个通道独立 `state`。** 一个通道失败不影响另一个;失败必须可见并附 `fix`。 +3. **不合并 `score`。** 每个通道内部排序,跨通道并列。需要合并时由调用方决定。 + +### 5.4 执行模型:并行、预算、缓存 + +```text +find() + ├─ 本地 Corpus 打分 (毫秒级,主线程) + └─ 并行启动 N 个外部通道 + ├─ cannbot-knowledge (子进程 / MCP stdio) + └─ mindie-agent (子进程,现有 call()) +``` + +用 `ThreadPoolExecutor`,通道间无共享状态;总耗时 ≈ max(各通道) 而非 sum。 + +| 参数 | 默认 | 说明 | +| --- | --- | --- | +| `--external-timeout` | 8 秒 | 单通道上限 | +| `--no-external` | 关闭 | 只用本仓结果(CI 与离线场景) | +| `--external-limit` | 通道默认 | 覆盖单通道返回条数 | + +| 通道 | 默认条数 | 理由 | +| --- | --- | --- | +| `cannbot-knowledge` | **10** | 定位符精度高(`path` 即 Concept ID),平台过滤已生效,多给几条有助于一次选中正确粒度 | +| `mindie-agent` | **5** | 返回 `navigation_only` 的 ref 定位符,粒度更粗;过多会淹没本地结果 | + +**`find` 默认即带外部查询**(不带 `--no-external`),Ascend 任务开箱即双查。CI 与单测必须显式传 `--no-external`,以隔离外部通道可用性对基线的影响。 + +超时必须映射成 `state:"timeout"` 并附 `elapsed_ms`,**绝不能表现为空结果**。"查了但超时"与"查了没命中"是两种事实,混同会让 Agent 误判为"上游没有这个知识"。 + +| 缓存对象 | TTL | 理由 | +| --- | --- | --- | +| `describe()` / `discover` | 300 秒 | 探活成本高(冷启 3.7s),revision 不会秒级变化 | +| 通道授权状态与 checkout identity | 进程生命周期 | 文件系统事实 | +| search 结果 | 不缓存 | query 空间无限,缓存会让"上游更新"不可见 | + +缓存只在内存,不落盘。 + +### 5.5 cannbot-knowledge 接法 + +新增 `src/inferrsi/apps/cannbot/`,结构镜像 `apps/mindie/`: + +| 文件 | 职责 | +| --- | --- | +| `project.py` | 逐 checkout 授权;`~/.config/inferrsi/cannbot-projects/.json` | +| `context.py` | 解析上游 JSON;2 MiB 上限、重复键拒绝、非有限数拒绝(复用 MindIE 的 `_object` / `_invalid_constant`) | +| `worker.py` | 用 grant 里的 python 起子进程,调上游 `knowledge_query.py search/read` 或 MCP | +| `mcp.py` | stdio MCP 代理三工具 | +| `receipt.py` | 落 receipt(§7) | + +授权文件: + +```json +{ + "schema": "inferrsi-cannbot-project/1", + "checkout": "", + "checkout_identity": {"device": 0, "inode": 0, "created": 0}, + "python": "/bin/python", + "backend": "ascend", + "default_platforms": ["a2"], + "enabled_at": "" +} +``` + +`checkout_identity` 抄 MindIE 的做法(`.git` 的 device + inode + birthtime),防止 grant 被复制到另一个 clone 误用。`default_platforms` 让常用平台不必每次传;未声明时 `search` 直接报错(与上游 fail-closed 一致)。 + +### 5.6 MindIE Agent 接法 + +不重写,加一层 adapter 让现有 `call()` 满足 `Channel` 协议: + +| 现状 | 改造 | +| --- | --- | +| `call("query", args)` 返回 `repository.query(navigation_only=True)` | adapter 映射为 `ChannelResult` | +| ref 形如 `mindie:///units/@` | 作为 `locator` | +| grant 无 backend | 增可选 `backend` 字段 | +| `ReadFailure` 区分失败 | 映射到 `not_enabled` / `version_mismatch` / `rejected` | + +`mindie-agent` 的 `describe` 已有:`repository.available()` + `component_version`(版本钉 `0.13.13`,`read_worker.py:24`)。 + +## 6. 为什么不采用其余形态(要点) + +- **C 全量转换**:许可阻断分发;7.8 MiB → 约 126 MiB;上游每次前进需全量重转;与 #388 的单一权威来源规则冲突;10,550 条无法诚实标 `locally_verified=true`。**判死。** +- **B submodule**:把 CANN OSL 的许可义务交给每个集成方;143 MB 随克隆;`gitcode` 可达性成为所有开发者的硬依赖。KDA 能这么做是因为它不 pip 分发且 submodule 是 MIT/过过过洗的发布点。 +- **A/A′ 外链锁缓存**:机制正确但与 MCP 重叠——两者都在解决"如何引用外部内容",而 MCP 让上游承担索引与治理。保留为未来若需要"完全离线复现"时的备选,当前不实现。 + +## 7. 证据与许可 + +### 7.1 receipt 只存定位符 + +```json +{ + "schema": "inferrsi-external-receipt/1", + "channel": "cannbot-knowledge", + "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", + "index": {"schema": "cannbot-knowledge-index.v1", "version": 3, + "content_fingerprint": "0047983f..."}, + "query": "CeilDivision 向上取整", + "platforms": ["a2"], + "decision": "确认 a3 上的取整语义", + "hits": [{ + "locator": "ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division", + "card_sha256": "b97cbe82492d25cc0363d1852eddcf87d991c9d75a57891a9aed3e75188ffc86", + "status": "stable", "platforms": ["a2", "a3"] + }], + "delivered": "locators_only", + "locally_verified": false +} +``` + +**不含卡片正文、不含 frontmatter 全文、不含 `local_path`。** 前两者会把 CANN OSL 内容带进 `evidence/`(把许可问题从 `data/` 原样搬到 `evidence/`);后者是个人绝对路径,违反隐私规则。 + +`card_sha256` 保证"我引用的确实是这一版这张卡",且摘要不构成再分发。 + +### 7.2 引用话术与验证状态 + +引用外部结论必须写成 `cannbot-knowledge @ ` 或 `mindie-knowledge 0.13.13 / @`,并显式标注 `locally_verified=false`。外部命中**不参与** `PASS` 判定(沿用 `AGENTS.md` 的证据纪律)。 + +## 8. 代码改动清单 + +| # | 文件 | 改动 | 风险 | +| --- | --- | --- | --- | +| 1 | `knowledge/channels.py` | 新增 `Channel` / `ChannelStatus` / `ChannelHit` / `ChannelResult` / `ChannelRegistry` | 无 | +| 2 | `knowledge/knowledge_find.py:732` `find()` | 增 `channels` / `backend` / `external_limit` / `external_timeout`;并行调度;输出 `external[]` | 需保证 `channels=None` 时行为**逐字节一致** | +| 3 | `knowledge/knowledge_find.py:620` `load_corpus` | 不改 | — | +| 4 | `knowledge/knowledge_operations.py:25` `AUTHORITIES` | **不改**(外部内容不作为 bundle 读取) | — | +| 5 | `apps/cannbot/*` | 新增五个文件 | 新代码 | +| 6 | `apps/mindie/` | 增 `channel.py` adapter;grant 增 `backend` | 小 | +| 7 | `apps/cli/arguments.py` | `find` 增 `--backend` / `--no-external` / `--external-limit` / `--external-timeout`;新增 `cannbot` 子命令组 | 小 | +| 8 | `apps/cli/__init__.py` | 分发 `cannbot`;`campaign import-cannbot` / `import-mindie` 统一 | 中 | +| 9 | `tests/` | fake channel(本地 stdio 假进程 + 固定 JSON),覆盖全部 `state` | 必需 | + +**第 2 条的不变量:** `channels` 为空或 `--no-external` 时,`find` 的输出必须与 #466 之后的行为完全一致。必须写成测试,否则外部通道会悄悄改变现有 143 篇 skill 的排序基线。 + +**第 4 条的理由:** 不把 `external` 加进 `AUTHORITIES`,意味着 `find` 的 `read_args` / `open_argv` 不会指向外部卡(避免生成一个执行即错的命令),外部读走独立的 `cannbot read`。 + +## 9. 使用方式 + +### 9.1 首次配置(每台机器、每个 checkout 一次) + +```bash +inferrsi cannbot enable \ + --checkout ~/src/cannbot-knowledge \ + --python ~/venvs/cannbot/bin/python \ + --backend ascend \ + --platform a2 + +inferrsi mindie enable --domain --backend ascend +``` + +前置条件由用户负责:上游要求 Python 3.10+、`uv`、**完整 checkout**、**索引已构建**。 + +自查与自修复: + +```bash +inferrsi cannbot describe +# {"state":"ok","revision":"92dc2607...","cards":10550, +# "index":{"version":3,"content_fingerprint":"0047983f..."}} +``` + +```text +state: index_stale +detail: index built at differs from checkout +fix: cd ~/src/cannbot-knowledge && python3 .agents/skills/knowledge-query/scripts/knowledge_index.py build +``` + +### 9.2 日常检索(Ascend 任务,常驻双查) + +```bash +$ inferrsi find "AscendC tiling 参数怎么算" --backend ascend +``` + +```text +本仓 (backend=ascend) + 1. ascendc-tiling-design score 42.1 + 2. ascend-performance-analysis score 31.7 + +外部通道 + cannbot-knowledge @ 92dc2607 ok 1.5s 3 hits + 1. Matmul Tiling类使用说明 18.4 ops/ascendc/apis/.../cube_compute/matmul_tiling + 2. 使用高阶API时配套的Tiling实现 17.9 ops/ascendc/guides/.../tiling + 3. Matmul tiling 形状与核数推导链 17.2 ops/ascendc/guides/.../derivation_chain + → inferrsi cannbot read + + mindie-agent not_enabled (fix: inferrsi mindie enable --domain ) +``` + +本仓有命中时**照样**查外部;两个通道各自一段;分数不跨栏比较。 + +### 9.3 非 Ascend 任务 / 未声明 backend + +```bash +$ inferrsi find "CUDA graph capture" --backend cuda +# 只查本仓;external: [] +``` + +```text +external: [{"channel":"cannbot-knowledge","state":"backend_unknown", + "fix":"inferrsi find --backend ascend"}] +``` + +### 9.4 读外部卡正文 + +```bash +inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division +inferrsi cannbot read --page 2 # 分页续读,默认 200 行 +inferrsi mindie read +``` + +正文只在终端呈现,不落盘、不进 receipt。 + +### 9.5 Agent 挂载(Claude Code / Codex) + +```bash +inferrsi cannbot serve # stdio MCP:discover / search / read_card +``` + +**透传,不收窄。** 三工具的名称、参数与语义与上游一致,只多一层授权检查、receipt 与错误归一化。理由:自造收窄契约会在上游演进(新增过滤参数、`search` schema 从 v5 升级)时变成必须同步维护的负债,而透传可依靠 `schema_version` 直接暴露差异。授权检查、receipt、错误归一化都在这一层完成。 + +### 9.6 进 campaign evidence + +```bash +inferrsi cannbot search "CeilDivision 向上取整" --platform a2 \ + --decision "确认 a3 上的取整语义" \ + --receipt .inferrsi/receipts/ceil.json + +inferrsi campaign import-cannbot .inferrsi/receipts/ceil.json +``` + +## 10. 切片 + +| 片 | 内容 | 依赖 | 可独立合入 | +| --- | --- | --- | --- | +| C0 | `channels.py` 协议 + fake channel + 全部 state 的测试 | — | ✅ | +| C1 | `find()` 融合 + `--backend` 门控(只接 fake channel) | C0 | ✅ | +| C2 | `apps/cannbot/` 的 `enable/disable/describe` | C0 | ✅ | +| C3 | cannbot `search` / `read`(worker + 子进程) | C2 | ✅ | +| C4 | MindIE channel adapter + grant `backend` | C0 | ✅ | +| C5 | `cannbot serve`(stdio MCP 代理) | C3 | ✅ | +| C6 | `receipt.py` + `campaign import-cannbot` 统一 | C3、C4 | ✅ | + +C1 先行的理由:它把"平台门控 + 分栏 + 超时语义"这套契约钉死,之后两个真源接入都只是实现 `Channel`,不会反过来改 `find` 的输出结构。 + +**C4 建议紧跟 C0**,因为 MindIE 已有 `call()` 与 `describe()`,adapter 成本低,能最早验证协议抽象是否成立。 + +## 11. 风险 + +| # | 风险 | 现状 | 缓解 | +| --- | --- | --- | --- | +| R1 | 热路径增加 1.5–3.7s | 实测 | 并行 + `describe` 缓存 300s + `--no-external` | +| R2 | 破坏 `find` 的确定性与离线可测 | 真实风险 | 不传 `--backend` 时不查外部;`--no-external` 关闭全部通道;fake channel 覆盖全部 state;两种路径输出逐字节固定 | +| R3 | 上游索引过期导致"昨天能查今天不能查" | 上游 fail-closed | `state: index_stale` + `fix`;不做静默重试 | +| R4 | 平台枚举不可映射(`a2/a3/950/310p` vs `ascend`) | 未决 | grant 声明 `backend` + `platform` 两级;未知值拒绝,不猜 | +| R5 | 许可内容意外落盘 | 已设计 | receipt 只存 locator + sha256;剔除 `local_path` | +| R6 | 两套评分被下游误混 | 已设计 | `score_scale` 字段 + 分栏结构 + `notes` | +| R7 | `uv` / 完整 checkout 在 CI 不可用 | 未验证 | 单测只用 fake channel;真上游走人工验收 | +| R8 | MindIE 0.13.13 版本钉导致升级即断 | 现有行为 | `state: version_mismatch` + `fix` | + +## 12. 关闭条件 + +本文作为 RFC 关闭时应当满足: + +- [ ] 评审接受 §5 的通道契约(`Channel` 协议、`state` 枚举、`external[]` 结构、`score_scale`)。 +- [ ] 评审接受 §5.2 的平台门控语义:显式声明、单值、不推断。 +- [ ] 评审接受 §7 的 receipt 形状与"不含正文"的许可边界;Q1(CANN OSL 立场)有组织层面的答复或明确接受为"实施前再定"。 +- [ ] 评审接受 §5.4 的默认值:`find` 默认查外部、每通道 10/5、超时 8 秒。 +- [ ] 确认 §8 第 2 条与第 4 条不变量写入测试。 +- [ ] 明确与 [#388](https://github.com/InferMatrixLab/InferRSI/issues/388) / [#387](https://github.com/InferMatrixLab/InferRSI/issues/387) 的关系:本文不关闭 #388,`external[]` 需在 #387 的 schema 中登记。 +- [ ] 实施切片(C0–C6)另开 issue,本 Issue 不承接代码合入。 + +## 13. 相关 + +- [#388](https://github.com/InferMatrixLab/InferRSI/issues/388) 外部技能内容的依赖、版本锁定与可复现导入(治理 RFC,仍 Open) +- [#387](https://github.com/InferMatrixLab/InferRSI/issues/387) K/H/B 与 Apps 公开接口详细设计(`read` / `collect`,仍 Proposed) +- [#362](https://github.com/InferMatrixLab/InferRSI/issues/362) 三模块目标架构(已接受方向) +- [#91](https://github.com/InferMatrixLab/InferRSI/issues/91) SearchSpace 蒸馏覆盖跟踪(本仓内唯一权威来源路径) +- [#43](https://github.com/InferMatrixLab/InferRSI/issues/43) 历史 `license: UNKNOWN` 与许可链遗留约束 +- [#466](https://github.com/InferMatrixLab/InferRSI/pull/466) 统一 `find` 检索(本文接入的宿主接口) +- [#114](https://github.com/InferMatrixLab/InferRSI/pull/114) cann/cannbot 固定阶段证据蒸馏入库(既有蒸馏先例,不是外部依赖机制) + +## 14. 参考实现(外部) + +- [cann/cannbot-knowledge](https://gitcode.com/cann/cannbot-knowledge):OKF v0.2 + CANNBot Profile,自带 governance / Query / Lint / MCP。 +- [NVlabs/kda](https://github.com/NVlabs/kda):submodule 钉版本 + README 锁表 + `third_party_licenses/` 逐件许可 + 明确"submodule 不受本仓一方许可覆盖"。**可借的是纪律,不是形式。** diff --git a/docs/plans/2026-11-17-external-knowledge-channels.md b/docs/plans/2026-11-17-external-knowledge-channels.md new file mode 100644 index 00000000..2e5d2ef7 --- /dev/null +++ b/docs/plans/2026-11-17-external-knowledge-channels.md @@ -0,0 +1,457 @@ +# 外部知识通道设计:cannbot-knowledge 与 MindIE Agent 的常驻查询接入 + +> Status: Proposed(设计草案)。本文只定义接入设计与使用方式,不构成实施授权。 +> 上位依赖:[#476](https://github.com/InferMatrixLab/InferRSI/issues/476)(候选形态与许可边界)、[#388](https://github.com/InferMatrixLab/InferRSI/issues/388)(外部依赖治理)、[#387](https://github.com/InferMatrixLab/InferRSI/issues/387)(Knowledge 公开接口)。 +> 文中所有上游 revision 与统计值都是写作时的实测观测值,不是本仓 pin。 + +## 0. 相对前一版的关键变更 + +前一版设计把外部通道当作**补漏**:本地 `find` 零命中时才提示外部存在。本文改为**常驻并行查询**: + +| 维度 | 前一版(补漏) | 本版(常驻) | +| --- | --- | --- | +| 触发条件 | `no_match == true` | 平台门控命中即查,无论本地是否命中 | +| 语义 | 外部源是兜底 | 外部源是**第二知识源**,与本地并列 | +| 输出 | `external_channels` 提示条目 | `external[]` 实际检索结果 | +| 平台门控 | 无 | 必须显式声明;`ascend` 任务常驻查询,其他平台不触发 | + +理由:Ascend 任务的真实知识分布是"本仓有 14 篇流程型 skill,上游有 10,550 张 API/算子卡"。只在零命中时查,等于假设"本地一旦有命中,外部就没有增量"——这个假设在粒度互补的场景下明显不成立(本仓命中"如何设计 tiling 流程",上游同时有"`Matmul Tiling` 类的具体参数")。 + +## 1. 现状与约束(实测) + +### 1.1 本仓知识侧 + +| 项 | 值 | +| --- | --- | +| `find` 实现 | `src/inferrsi/knowledge/knowledge_find.py`,commit `61303fa8`(#466 统一检索) | +| 检索模型 | 纯词法:`idf × 字段权重 × sqrt(出现次数)`,`FIELD_WEIGHTS` 见 `knowledge_find.py:42` | +| 来源 | `kinds=("case","skill","record")`,`load_corpus` 见 `:620` | +| 语料规模 | 143 篇 `SKILL.md` + 128 条记录 + cases,同进程内构建 | +| 确定性 | 完全离线、可复现、无 I/O 依赖 | +| Ascend 内容卡 | 6 张(详见 #476「现状」节) | + +### 1.2 cannbot-knowledge MCP 侧 + +| 项 | 实测值 | +| --- | --- | +| 索引构建 | 73.9 秒(10,550 卡,7,921,396 postings) | +| 查询延迟 | 冷启 3.67 秒 / 热 1.46 秒 | +| 检索结果字段 | `rank` / `score` / `match_basis` / `path` / `local_path` / `card_lines` / `card_sha256` | +| 响应 schema | `knowledge-search.v5`,含 `request_sha256` 与 `index.repository_state.commit` | +| 平台纪律 | `--platform` 必填,缺失即报错(fail-closed,不混合各平台) | +| 索引过期 | 拒绝服务并给出 `index_stale`,不自动重建 | +| 工具面 | `discover` / `search` / `read_card`,全部 `read_only` | +| 许可 | CANN OSL v2.0(内容不得进包、进 evidence) | + +### 1.3 MindIE Agent 侧 + +| 项 | 现有实现 | +| --- | --- | +| 适配代码 | `src/inferrsi/apps/mindie/`(639 行) | +| 工具体 | `knowledge_query` / `knowledge_explain` | +| 运行时 | `subprocess.run([grant["python"], "-I", "-B", worker, ...])`,`project.py:103` | +| 版本钉 | `mindie-knowledge == 0.13.13` 硬校验,`read_worker.py:24` | +| 授权 | 逐 checkout,`~/.config/inferrsi/mindie-projects/.json` | +| 检索 | `repository.query(navigation_only=True)`,返回 ref 定位符 | + +### 1.4 由此产生的设计约束 + +1. **两套评分不可比。** 本仓的 `idf` 建立在 143+128 条语料上,上游建立在 10,550 张卡上;同一个词在两边的 idf 与权重体系完全不同。任何"混合排序"都需要我们自己发明一个归一化函数——那是本轮 RSI 明令禁止的"编造分数"。 +2. **外部检索是 I/O,不是计算。** 1.5–3.7 秒相对本仓 `find` 的毫秒级是数量级差异,且失败模式有七种(未授权 / 不可达 / 索引过期 / 平台未知 / 超时 / 协议错 / 上游拒绝)。 +3. **外部结果不确定。** 上游 checkout 更新后同一 query 结果会变,因此**每次输出必须携带 revision**,否则 `find` 的结果无法被任何人复现。 +4. **许可边界不因常驻而放松。** 正文永不落盘;进 receipt 的只有 `path` + `card_sha256` + `revision`。 + +## 2. 核心设计:通道抽象 + 平台门控 + 分栏呈现 + +### 2.1 通道抽象 + +新增 `src/inferrsi/knowledge/channels.py`,把两个外部源收敛到同一协议,使 `find` 不认识具体是 cannbot 还是 MindIE: + +```python +class Channel(Protocol): + name: str # "cannbot-knowledge" | "mindie-agent" + backend: str | None # "ascend" | None(None = 平台无关) + def describe(self) -> ChannelStatus: ... # 探活,可缓存 + def search(self, query: str, *, platform: str | None, + limit: int) -> ChannelResult: ... # 返回定位符,不返回正文 + def read(self, locator: str) -> str: ... # 单独调用,不在 find 热路径 +``` + +```python +@dataclass +class ChannelStatus: + state: str # ok | not_enabled | unreachable | index_stale + # | version_mismatch | rejected | unknown + revision: str | None # 上游 commit / catalog generation + detail: str = "" + fix: str = "" # 修复命令,例如索引重建 + checked_at: float = 0.0 + +@dataclass +class ChannelResult: + status: ChannelStatus + items: list[ChannelHit] # 失败时为空,但 state 必须说明原因 + elapsed_ms: int +``` + +`ChannelHit` 字段固定:`locator` / `title` / `summary` / `score` / `score_scale` / `status` / `platforms` / `card_sha256`。 + +`score_scale` 是关键:标明这个分数来自哪套评分(`"cannbot/knowledge-search.v5"` 或 `"mindie/graded-0.13.13"`),从结构上阻止调用方跨通道比较。 + +### 2.2 平台门控(决定"要不要查") + +数据流: + +```text +find(query, backend=?) + │ + ├─ backend 未声明 ──► 不查外部;external[] 每项 state="backend_unknown" + │ 且 fix 给出 "inferrsi find --backend ascend" + │ + ├─ backend == ascend ──► 查询所有 backend ∈ {ascend, None} 的已授权通道 + │ (cannbot-knowledge + mindie-agent) + │ + └─ backend == cuda/mlx/... ──► 不查外部;external[] 为空 +``` + +`backend` 来源优先级: + +1. `inferrsi find --backend ascend`(显式参数,最高) +2. grant 文件里声明的 backend(`cannbot enable --backend ascend`) +3. 都没有 → `backend_unknown`,**不猜** + +`--backend` **只接受单值**。一次调用只能声明一个 backend;同时涉及 Ascend 与 CPU host 侧等多平台任务,由调用方分两次查询并各自保留结果。理由:多值会让输出无法回答"这条外部结果是为哪个平台召回的",而外部源的平台纪律本身是 fail-closed 的(上游 `--platform` 要求明确目标平台,不混合各平台结果),本仓不应在门控层放宽这一语义。 + +这里刻意不使用 `named_backends()`(`knowledge_find.py:739`)那种从查询词推断的方式:从"这个 query 里出现了 asc"推断"这个任务是 Ascend 任务"是不可靠的,且失败形态是静默查错平台。门控必须是显式事实。 + +### 2.3 分栏呈现(决定"怎么放") + +**不做混合排序。** 输出结构: + +```json +{ + "query": "AscendC tiling 参数", + "backend": "ascend", + "items": [ /* 本仓结果,沿用现有 ranked list */ ], + "matched": 3, + "external": [ + { + "channel": "cannbot-knowledge", + "state": "ok", + "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", + "elapsed_ms": 1520, + "score_scale": "cannbot/knowledge-search.v5", + "items": [ + { + "locator": "ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling", + "title": "Matmul Tiling类使用说明", + "score": 18.4, + "status": "stable", + "platforms": ["a2", "a3"], + "card_sha256": "049b4824...", + "open": "inferrsi cannbot read ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling" + } + ] + }, + { + "channel": "mindie-agent", + "state": "not_enabled", + "detail": "no project grant for this checkout", + "fix": "inferrsi mindie enable --domain " + } + ], + "notes": [ + "External results are not locally verified; scores are not comparable across channels.", + "Pin external revisions from this output when citing them." + ] +} +``` + +三条不可退让的规则: + +1. **`items` 只放本仓结果。** 外部命中不进 `items`,避免任何下游按 `items` 处理时会静默把未验证内容当成已验证。 +2. **每个通道独立 `state`。** 一个通道失败不影响另一个;失败必须可见并附 `fix`。 +3. **不合并 `score`。** 每个通道内部排序,跨通道并列。需要合并时由调用方决定,不由 `find` 代劳。 + +## 3. 执行模型:并行、预算、降级 + +### 3.1 并行 + +```text +find() + ├─ 本地 Corpus 打分 (毫秒级,主线程) + └─ 并行启动 N 个外部通道 + ├─ cannbot-knowledge (子进程 / MCP stdio) + └─ mindie-agent (子进程,现有 call()) +``` + +用 `concurrent.futures.ThreadPoolExecutor`,通道之间无共享状态。总耗时 ≈ max(各通道),而不是 sum。 + +### 3.2 预算与降级 + +| 参数 | 默认 | 说明 | +| --- | --- | --- | +| `--external-timeout` | 8 秒 | 单通道上限 | +| `--no-external` | — | 只用本仓结果(CI 与离线场景) | +| `--external-limit` | 通道默认(见下) | 覆盖单通道返回条数 | + +**每通道条数默认值:** + +| 通道 | 默认条数 | 理由 | +| --- | --- | --- | +| `cannbot-knowledge` | **10** | 10,550 张卡、2,993 张 Ascend C API 卡,定位符精度高(`path` 即 Concept ID),多给几条能让调用方一次选中正确粒度 | +| `mindie-agent` | **5** | 返回的是 `navigation_only` 的 ref 定位符,粒度更粗;过多会淹没本地结果 | + +`find` **默认即带外部查询**(不带 `--no-external`),因此 Ascend 任务开箱就是双查。CI 与单测必须显式传 `--no-external`,以保证基线不受外部通道可用性影响。 + +超时必须映射成 `state: "timeout"` 并附 `elapsed_ms`,**绝不能表现为空结果**。"查了但超时"与"查了没命中"是两种事实,混同会让 Agent 误判为"上游没有这个知识"。 + +### 3.3 缓存 + +| 对象 | TTL | 理由 | +| --- | --- | --- | +| `describe()` / `discover` | 300 秒 | 探活成本高(冷启 3.7s),且 revision 不会秒级变化 | +| 通道授权状态与 checkout identity | 进程生命周期 | 文件系统事实 | +| search 结果 | 不缓存 | query 空间无限,且缓存会让"上游更新"不可见 | + +缓存落在内存,不落盘;落盘缓存会引入"缓存里的 revision 与当前不一致"的经典问题。 + +## 4. 两个通道的具体接法 + +### 4.1 cannbot-knowledge + +新增 `src/inferrsi/apps/cannbot/`,结构镜像 `apps/mindie/`: + +| 文件 | 职责 | +| --- | --- | +| `project.py` | 逐 checkout 授权;`~/.config/inferrsi/cannbot-projects/.json` | +| `context.py` | 解析上游 JSON;2 MiB 上限、重复键拒绝、非有限数拒绝(复用 MindIE 的 `_object` / `_invalid_constant`) | +| `worker.py` | 用 grant 里的 python 起子进程,调 `knowledge_query.py search/read` 或 MCP `search`/`read_card` | +| `mcp.py` | stdio MCP 代理三工具,供 agent 直接挂载 | +| `receipt.py` | 落 receipt(§5) | + +授权文件: + +```json +{ + "schema": "inferrsi-cannbot-project/1", + "checkout": "", + "checkout_identity": {"device": 0, "inode": 0, "created": 0}, + "python": "/bin/python", + "backend": "ascend", + "default_platforms": ["a2"], + "enabled_at": "" +} +``` + +`default_platforms` 让常用平台不必每次传;未声明时 `search` 直接报错(与上游 fail-closed 一致)。 + +### 4.2 MindIE Agent + +不重写,增加一层 adapter 让现有的 `call()` 满足 `Channel` 协议: + +| 现状 | 改造 | +| --- | --- | +| `call("query", args)` 返回 `repository.query(navigation_only=True)` | adapter 中映射为 `ChannelResult` | +| ref 形如 `mindie:///units/@` | 作为 `locator` | +| backend 未声明 | grant 增可选的 `backend` 字段,默认 `ascend` | +| `state` 由 `ReadFailure` 区分 | 映射到 `not_enabled` / `version_mismatch` / `rejected` | + +`mindie-agent` 的 `describe` 已有:`repository.available()` + `component_version`。 + +## 5. 证据与许可 + +### 5.1 receipt 只存定位符 + +```json +{ + "schema": "inferrsi-external-receipt/1", + "channel": "cannbot-knowledge", + "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", + "index": {"schema": "cannbot-knowledge-index.v1", "version": 3, + "content_fingerprint": "0047983f..."}, + "query": "CeilDivision 向上取整", + "platforms": ["a2"], + "decision": "确认 a3 上的取整语义", + "hits": [{ + "locator": "ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division", + "card_sha256": "b97cbe82492d25cc0363d1852eddcf87d991c9d75a57891a9aed3e75188ffc86", + "status": "stable", "platforms": ["a2", "a3"] + }], + "delivered": "locators_only", + "locally_verified": false +} +``` + +**不含卡片正文、不含 frontmatter 全文、不含 local_path。** 原因:前两者会把 CANN OSL 内容带进 `evidence/`,把许可问题从 `data/` 原样搬到 `evidence/`;`local_path` 是个人绝对路径,违反隐私规则。 + +`card_sha256` 的作用是"我引用的确实是这一版这张卡",且摘要不构成再分发。 + +### 5.2 引用话术 + +引用外部结论时必须写成:`cannbot-knowledge @ ` 或 `mindie-knowledge 0.13.13 / @`,并显式标注 `locally_verified=false`。外部命中**不参与** `PASS` 判定(沿用 `AGENTS.md` 的证据纪律)。 + +## 6. 代码改动清单 + +| # | 文件 | 改动 | 风险 | +| --- | --- | --- | --- | +| 1 | `knowledge/channels.py` | 新增 `Channel` / `ChannelStatus` / `ChannelHit` / `ChannelResult` / `ChannelRegistry` | 无 | +| 2 | `knowledge/knowledge_find.py:732` `find()` | 增 `channels`、`backend`、`external_limit`、`external_timeout` 参数;并行调度;输出 `external[]` | 需保证 `channels=None` 时行为与现在**逐字节一致** | +| 3 | `knowledge/knowledge_find.py:620` `load_corpus` | 不改 | — | +| 4 | `knowledge/knowledge_operations.py:25` `AUTHORITIES` | **不改**(外部内容不作为 bundle 读取) | — | +| 5 | `apps/cannbot/*` | 新增 five files | 新代码 | +| 6 | `apps/mindie/` | 增 `channel.py` adapter;grant 增 `backend` | 小 | +| 7 | `apps/cli/arguments.py` | `find` 增 `--backend` / `--no-external` / `--external-limit` / `--external-timeout`;新增 `cannbot` 子命令组 | 小 | +| 8 | `apps/cli/__init__.py` | 分发 `cannbot`;`campaign import-cannbot` / `import-mindie` 统一 | 中 | +| 9 | `tests/` | fake channel(本地 stdio 假进程 + 固定 JSON),覆盖七种 `state` | 必需 | + +第 2 条的不变量:**`channels` 为空或 `--no-external` 时,`find` 的输出必须与 #466 之后的行为完全一致。** 这条要写成测试,否则外部通道会悄悄改变现有 143 篇 skill 的排序基线。 + +## 7. 使用方式 + +### 7.1 首次配置(每台机器、每个 checkout 一次) + +```bash +# cannbot-knowledge +inferrsi cannbot enable \ + --checkout ~/src/cannbot-knowledge \ + --python ~/venvs/cannbot/bin/python \ + --backend ascend \ + --platform a2 + +# MindIE(沿用现有命令,仅补 backend) +inferrsi mindie enable --domain --backend ascend +``` + +自查: + +```bash +inferrsi cannbot describe +# {"state":"ok","revision":"92dc2607...","cards":10550, +# "index":{"version":3,"content_fingerprint":"0047983f..."}} + +inferrsi mindie describe +# {"available":true,"component_version":"0.13.13","domain":""} +``` + +失败时直接给修复命令: + +```text +state: index_stale +detail: index built at differs from checkout +fix: cd ~/src/cannbot-knowledge && python3 .agents/skills/knowledge-query/scripts/knowledge_index.py build +``` + +### 7.2 日常检索(Ascend 任务,常驻双查) + +```bash +$ inferrsi find "AscendC tiling 参数怎么算" --backend ascend +``` + +输出(示意): + +```text +本仓 (backend=ascend) + 1. ascendc-tiling-design score 42.1 + 2. ascend-performance-analysis score 31.7 + +外部通道 + cannbot-knowledge @ 92dc2607 ok 1.5s 3 hits + 1. Matmul Tiling类使用说明 18.4 ops/ascendc/apis/.../cube_compute/matmul_tiling + 2. 使用高阶API时配套的Tiling实现 17.9 ops/ascendc/guides/.../tiling + 3. Matmul tiling 形状与核数推导链 17.2 ops/ascendc/guides/.../derivation_chain + → inferrsi cannbot read + + mindie-agent not_enabled (fix: inferrsi mindie enable --domain ) +``` + +要点:本地有命中时**照样**查外部;两个通道各自一段;分数不跨栏比较。 + +### 7.3 非 Ascend 任务 + +```bash +$ inferrsi find "CUDA graph capture" --backend cuda +# 只查本仓;external: [] +``` + +不传 `--backend` 且 grant 未声明时: + +```text +external: [{"channel":"cannbot-knowledge","state":"backend_unknown", + "fix":"inferrsi find --backend ascend"}] +``` + +### 7.4 读外部卡正文 + +```bash +inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division +inferrsi cannbot read --page 2 # 分页续读,默认 200 行 +inferrsi mindie read # MindIE 侧沿用现有 read 动作 +``` + +正文只在终端呈现,不落盘、不进 receipt。 + +### 7.5 Agent 挂载(Claude Code / Codex) + +```bash +inferrsi cannbot serve # stdio MCP:discover / search / read_card +``` + +**透传,不收窄。** 三个工具的名称、参数与语义与上游保持一致,只是多一层授权检查、receipt 与错误归一化。理由:自造一层收窄契约会在上游演进(例如新增过滤参数、`search` schema 从 v5 升级)时变成必须同步维护的负债,而透传可以靠 `schema_version` 字段直接暴露差异。在 agent 配置中注册一条命令即可。 + +### 7.6 进 campaign evidence + +```bash +inferrsi cannbot search "CeilDivision 向上取整" --platform a2 \ + --decision "确认 a3 上的取整语义" \ + --receipt .inferrsi/receipts/ceil.json + +inferrsi campaign import-cannbot .inferrsi/receipts/ceil.json +``` + +## 8. 切片 + +| 片 | 内容 | 依赖 | 可独立合入 | +| --- | --- | --- | --- | +| C0 | `channels.py` 协议 + fake channel + 七种 state 的测试 | — | ✅ | +| C1 | `find()` 融合 + `--backend` 门控(只接 fake channel,不接真源) | C0 | ✅ | +| C2 | `apps/cannbot/` 的 `enable/disable/describe` | C0 | ✅ | +| C3 | cannbot `search` / `read`(worker + 子进程) | C2 | ✅ | +| C4 | MindIE channel adapter + grant `backend` | C0 | ✅ | +| C5 | `cannbot serve`(stdio MCP 代理) | C3 | ✅ | +| C6 | `receipt.py` + `campaign import-cannbot` 统一 | C3、C4 | ✅ | + +C1 先行的理由:它把"平台门控 + 分栏 + 超时语义"这套契约固定下来,之后两个真源接入都只是实现 `Channel`,不会反过来改 `find` 的输出结构。 + +## 9. 风险与未决 + +| # | 风险 | 现状 | 缓解 | +| --- | --- | --- | --- | +| R1 | 热路径增加 1.5–3.7s | 实测 | 并行 + `describe` 缓存 300s + `--no-external` | +| R2 | 破坏 `find` 的确定性与离线可测 | 真实风险 | 不传 `--backend` 时不查外部;`--no-external` 关闭全部通道;fake channel 覆盖全部 state;两种路径下的输出都逐字节固定 | +| R3 | 上游索引过期导致"昨天能查今天不能查" | 上游 fail-closed | `state: index_stale` + `fix`;不做静默重试 | +| R4 | 平台枚举不可映射(`a2/a3/950/310p` vs `ascend`) | 未决 | grant 声明 `backend` + `platform` 两级;未知值拒绝,不猜 | +| R5 | 许可内容意外落盘 | 已设计 | receipt 只存 locator + sha256;模板里剔除 `local_path` | +| R6 | 两套评分被下游误混 | 已设计 | `score_scale` 字段 + 分栏结构 + `notes` | +| R7 | `uv` / 完整 checkout 在 CI 不可用 | 未验证 | 单测只用 fake channel;真上游走人工验收 | +| R8 | MindIE 0.13.13 版本钉导致升级即断 | 现有行为 | `state: version_mismatch` + `fix`,与现状一致 | + +### 已定决策 + +以下四项已确认,不再是待决项;实现按此执行,若需变更须另行评审。 + +| # | 问题 | 决定 | 落实位置 | +| --- | --- | --- | --- | +| D1 | `--backend` 是否允许一次声明多个 | **只允许一个。** 多平台任务由调用方分两次查询,各自保留结果;不接受多值,避免输出无法回答"这条外部结果是为哪个平台召回的" | §2.2 平台门控、§6 改动 7 | +| D2 | `find` 默认是否带 `--no-external` | **不带。** Ascend 任务开箱即双查;CI 与单测必须显式传 `--no-external`,以隔离外部通道可用性对基线的影响 | §3.2 预算与降级、§6 改动 9 | +| D3 | `external[]` 每通道默认条数 | **`cannbot-knowledge` 10 条 / `mindie-agent` 5 条。** 上游定位符精确到 `path` 且平台过滤已生效,多给几条有助于一次选中正确粒度;MindIE 返回粗粒度 ref,过多会淹没本地结果 | §3.2 每通道条数默认值 | +| D4 | `cannbot serve` 的 MCP 面 | **透传上游 `discover` / `search` / `read_card` 三工具**,名称、参数、语义保持一致;只增加授权检查、receipt 与错误归一化。不收窄,以免在上游 `search` schema 升级时产生必须同步维护的自造契约 | §7.5 Agent 挂载 | + +## 10. 与既有文档的关系 + +| 文档 | 关系 | +| --- | --- | +| #476 | 本文把方案 D(MCP 旁路)升格为主接法,并补充"常驻查询"语义;#476 §4 的方案 D 需据此修订 | +| #388 | 本文的 receipt 设计、六类拒绝状态、许可门禁是 #388 契约的具体落地 | +| #387 | `find` 新增的 `external[]` 属于 Knowledge 公开接口的一部分,需在 #387 的 schema 中登记 | +| `AGENTS.md` | 外部命中不参与 `PASS`;引用须带 revision 与 `locally_verified=false` | diff --git a/src/inferrsi/apps/cannbot/__init__.py b/src/inferrsi/apps/cannbot/__init__.py new file mode 100644 index 00000000..6b7fae20 --- /dev/null +++ b/src/inferrsi/apps/cannbot/__init__.py @@ -0,0 +1,98 @@ +"""The cannbot-knowledge channel: an external read-only corpus beside ours.""" + +from __future__ import annotations + +import time +from pathlib import Path + +from ...knowledge.channels import ( + ChannelHit, + ChannelResult, + ChannelStatus, +) +from . import project + +__all__ = ["CannbotChannel", "project"] + +SCORE_SCALE = "cannbot/knowledge-search.v5" + + +class CannbotChannel: + """Approved cannbot-knowledge checkout, queried through its own interpreter.""" + + def __init__(self, *, root: Path | None = None, timeout: float = 8.0): + from . import project + + self._project = project + self.root = root + self.timeout = timeout + # `hint` records why this channel cannot answer before any search runs. + # Keeping it lets `find` report "not enabled" instead of telling the + # caller to declare a backend for a source that is not approved yet. + self.hint: ChannelStatus | None = None + try: + grant = project.load_grant(root) + except project.ChannelFailure as exc: + grant = {} + self.hint = ChannelStatus(exc.state, detail=str(exc), fix=exc.fix) + except (OSError, ValueError) as exc: + grant = {} + self.hint = ChannelStatus("rejected", detail=str(exc)) + self.backend = grant.get("backend") + self.name = "cannbot-knowledge" + + def describe(self) -> ChannelStatus: + try: + info = self._project.describe(self.root, timeout=max(self.timeout, 15.0)) + except self._project.ChannelFailure as exc: + return ChannelStatus(exc.state, detail=str(exc), fix=exc.fix) + state = info.get("state", "rejected") + if state != "ok": + return ChannelStatus( + state if state != "rejected" else "rejected", + detail=str(info.get("detail") or ""), fix=str(info.get("fix") or ""), + ) + return ChannelStatus("ok", revision=info.get("revision")) + + def search( + self, query: str, *, platform: str | None, limit: int + ) -> ChannelResult: + started = time.monotonic() + try: + payload = self._project.search( + query, platform=platform, limit=limit, root=self.root, + timeout=self.timeout, + ) + except self._project.ChannelFailure as exc: + return ChannelResult( + name=self.name, status=ChannelStatus( + exc.state, detail=str(exc), fix=exc.fix, + ), + ) + elapsed = int((time.monotonic() - started) * 1000) + items = tuple( + ChannelHit( + locator=str(row.get("locator") or ""), + title=str(row.get("title") or ""), + summary=str(row.get("summary") or ""), + score=float(row.get("score") or 0), + score_scale=str(payload.get("score_scale") or SCORE_SCALE), + status=str(row.get("status") or ""), + platforms=tuple(row.get("platforms") or ()), + card_sha256=str(row.get("card_sha256") or ""), + ) + for row in payload.get("items") or [] + if row.get("locator") + ) + return ChannelResult( + name=self.name, + status=ChannelStatus("ok", revision=payload.get("revision")), + items=items, + elapsed_ms=elapsed, + score_scale=str(payload.get("score_scale") or SCORE_SCALE), + read_prefix="inferrsi cannbot read", + ) + + def read(self, locator: str) -> str: + payload = self._project.read(locator, root=self.root, timeout=max(self.timeout, 15.0)) + return str(payload.get("text") or "") diff --git a/src/inferrsi/apps/cannbot/project.py b/src/inferrsi/apps/cannbot/project.py new file mode 100644 index 00000000..0238023e --- /dev/null +++ b/src/inferrsi/apps/cannbot/project.py @@ -0,0 +1,257 @@ +"""An opt-in, per-checkout grant for an external cannbot-knowledge checkout. + +The upstream corpus is CANN-licensed and stays where it is. This module records +only which checkout, interpreter and default platform this project approved; it +never copies knowledge into the repository, the wheel or an evidence tree. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import tempfile +from pathlib import Path + +_SCHEMA = "inferrsi-cannbot-project/1" +_WORKER = Path(__file__).with_name("read_worker.py") +MAX_BYTES = 2 * 1024 * 1024 +_BACKEND = re.compile(r"[a-z][a-z0-9]{0,31}") +# Upstream platform values this adapter will pass through. An unlisted value is +# refused rather than forwarded: the upstream CLI is fail-closed about platform +# and a typo there would silently change which cards answer. +PLATFORMS = ("a2", "a3", "950", "310p", "910", "kirin_x90", "kirin_9030", "agnostic") + + +class ChannelFailure(ValueError): + """A channel-level failure carrying the state the caller should report.""" + + def __init__(self, message, *, state="rejected", fix="", exit_code=None): + super().__init__(message) + self.state = state + self.fix = fix + self.exit_code = exit_code + + +def project_root() -> Path: + cwd = Path.cwd().resolve() + for folder in (cwd, *cwd.parents): + if (folder / ".git").exists(): + return folder + raise ValueError("Open a Git project to use project-scoped cannbot knowledge") + + +def _grant_path(root: Path) -> Path: + project_id = hashlib.sha256(os.fsencode(root)).hexdigest() + return Path.home() / ".config" / "inferrsi" / "cannbot-projects" / (project_id + ".json") + + +def _checkout_identity(root: Path) -> dict: + marker = root / ".git" + info = marker.stat() + identity = {"device": info.st_dev, "inode": info.st_ino} + if hasattr(info, "st_birthtime"): + identity["created"] = info.st_birthtime + return identity + + +def _json(value) -> bytes: + return (json.dumps(value, ensure_ascii=True, allow_nan=False) + "\n").encode("utf-8") + + +def _parse(raw: bytes) -> dict: + def _object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("cannbot response contains duplicate JSON keys") + value[key] = item + return value + + def _invalid(_): + raise ValueError("cannbot response contains a non-finite JSON value") + + try: + value = json.loads(raw, object_pairs_hook=_object, parse_constant=_invalid) + except (ValueError, UnicodeError, RecursionError) as exc: + raise ValueError("cannbot did not return a JSON object") from exc + if not isinstance(value, dict): + raise ValueError("cannbot did not return a JSON object") + return value + + +def load_grant(root: Path | None = None) -> dict: + root = project_root() if root is None else root + path = _grant_path(root) + try: + info = path.stat() + if (path.is_symlink() or not path.is_file() + or os.name == "posix" and (info.st_uid != os.getuid() or info.st_mode & 0o077)): + raise ValueError("The cannbot project grant must be a private local file") + with path.open("rb") as source: + raw = source.read(MAX_BYTES + 1) + if len(raw) > MAX_BYTES: + raise ValueError("The cannbot project grant exceeds 2 MiB") + grant = _parse(raw) + checkout = _checkout_identity(root) + except FileNotFoundError: + raise ChannelFailure( + "cannbot knowledge is not enabled for this project", state="not_enabled", + fix="inferrsi cannbot enable --checkout --python ", + ) from None + except OSError: + raise ChannelFailure("The cannbot project grant could not be read", state="not_enabled") \ + from None + if (grant.get("schema") != _SCHEMA or grant.get("project_root") != str(root) + or grant.get("checkout_identity") != checkout + or not isinstance(grant.get("upstream"), str) + or not Path(grant["upstream"]).is_absolute() + or not isinstance(grant.get("python"), str) + or not Path(grant["python"]).is_absolute() + or not isinstance(grant.get("backend"), str) + or not _BACKEND.fullmatch(grant["backend"])): + raise ChannelFailure("The cannbot grant does not match this project", state="rejected") + platforms = grant.get("platforms", []) + if not isinstance(platforms, list) or any(item not in PLATFORMS for item in platforms): + raise ChannelFailure("The cannbot grant has an invalid platform list", state="rejected") + return grant + + +def _run(grant: dict, action: str, arguments: dict, *, timeout: float) -> dict: + environment = os.environ.copy() + for key in ("CANNBOT_MCP_TOKENS", "CANNBOT_KNOWLEDGE_PLATFORM"): + environment.pop(key, None) + try: + with tempfile.TemporaryDirectory(prefix="inferrsi-cannbot-read-") as temporary: + cwd = Path(temporary) / "reader" + cwd.mkdir(mode=0o700) + result = subprocess.run( + [grant["python"], "-I", "-B", str(_WORKER), + grant["upstream"], action, json.dumps(arguments)], + input=b"", stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + cwd=cwd, env=environment, timeout=timeout, check=False, + ) + except subprocess.TimeoutExpired: + raise ChannelFailure( + f"cannbot did not answer within {timeout:g}s", state="timeout" + ) from None + except OSError: + raise ChannelFailure( + "The approved cannbot reader could not be started", state="unreachable" + ) from None + if len(result.stdout) > MAX_BYTES: + raise ChannelFailure("cannbot returned more than 2 MiB", state="rejected") + payload = _parse(result.stdout) + if result.returncode or payload.get("isError"): + raise ChannelFailure( + str(payload.get("error") or "The approved cannbot reader failed"), + state=str(payload.get("state") or "rejected"), + fix=str(payload.get("fix") or ""), + exit_code=result.returncode, + ) + return payload + + +def describe(root: Path | None = None, *, timeout: float = 15.0) -> dict: + try: + grant = load_grant(root) + except ChannelFailure as exc: + return {"state": exc.state, "detail": str(exc), "fix": exc.fix} + try: + payload = _run(grant, "discover", {}, timeout=timeout) + except ChannelFailure as exc: + return {"state": exc.state, "detail": str(exc), "fix": exc.fix} + return { + "state": "ok", + "revision": payload.get("revision"), + "cards": payload.get("cards"), + "index": payload.get("index", {}), + "platforms": list(grant.get("platforms", [])), + } + + +def search( + query: str, *, platform: str | None = None, limit: int = 10, + root: Path | None = None, timeout: float = 8.0, +) -> dict: + if not isinstance(query, str) or not query.strip(): + raise ChannelFailure("cannbot search needs a non-empty query") + if type(limit) is not int or not 1 <= limit <= 200: + raise ChannelFailure("cannbot limit must be an integer from 1 through 200") + grant = load_grant(root) + platforms = list(grant.get("platforms", [])) + if platform is None: + if not platforms: + raise ChannelFailure( + "cannbot search needs a platform", state="rejected", + fix="inferrsi cannbot search ... --platform a2", + ) + platform = platforms[0] + if platform not in PLATFORMS: + raise ChannelFailure(f"cannbot does not know platform {platform!r}", state="rejected") + return _run(grant, "search", { + "query": query, "platform": platform, "limit": limit, + }, timeout=timeout) + + +def read(locator: str, *, page: int = 1, root: Path | None = None, + timeout: float = 15.0) -> dict: + if not isinstance(locator, str) or not locator.strip(): + raise ChannelFailure("cannbot read needs a locator from a search result") + if type(page) is not int or page < 1: + raise ChannelFailure("cannbot page must be a positive integer") + return _run(load_grant(root), "read", {"locator": locator, "page": page}, timeout=timeout) + + +def enable(*, upstream: Path, python: Path, backend: str = "ascend", + platforms: list[str] | None = None) -> dict: + """Approve one checkout for this project after proving it answers. + + The probe is deliberate: a grant that records a checkout which cannot answer + would turn every later search into a failure the user has to debug. + """ + root = project_root() + upstream = Path(upstream).expanduser() + if not (upstream / ".git").exists(): + raise ValueError("--checkout must be a Git checkout of cannbot-knowledge") + if not (upstream / "knowledge").is_dir(): + raise ValueError("--checkout must contain the knowledge/ bundle root") + if not (upstream / ".agents/skills/knowledge-query/scripts/knowledge_query.py").is_file(): + raise ValueError("--checkout is missing the knowledge-query scripts") + interpreter = Path(python).expanduser() + if not interpreter.is_file() or not os.access(interpreter, os.X_OK): + raise ValueError("--python must be an executable interpreter with pyyaml installed") + if not _BACKEND.fullmatch(backend): + raise ValueError("--backend must be a lowercase backend name") + chosen = list(platforms or []) + if any(item not in PLATFORMS for item in chosen): + raise ValueError("--platform takes one of " + ", ".join(PLATFORMS)) + grant = { + "schema": _SCHEMA, "project_root": str(root), + "checkout_identity": _checkout_identity(root), + "upstream": str(upstream.resolve()), + # Record the interpreter path as given. Resolving the symlink in a venv + # bin/ lands on the base interpreter, which loses every installed + # dependency -- the probe would pass and every later search would fail. + "python": str(interpreter.absolute()), + "backend": backend, "platforms": chosen, + } + info = _run(grant, "discover", {}, timeout=30.0) + path = _grant_path(root) + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as output: + pending = Path(output.name) + try: + output.write(_json(grant)) + output.close() + pending.replace(path) + finally: + pending.unlink(missing_ok=True) + return {"enabled": True, "revision": info.get("revision"), "cards": info.get("cards")} + + +def disable() -> dict: + _grant_path(project_root()).unlink(missing_ok=True) + return {"enabled": False, "reason": "cannbot knowledge is not enabled for this project"} diff --git a/src/inferrsi/apps/cannbot/read_worker.py b/src/inferrsi/apps/cannbot/read_worker.py new file mode 100644 index 00000000..d5047fbf --- /dev/null +++ b/src/inferrsi/apps/cannbot/read_worker.py @@ -0,0 +1,135 @@ +"""Call the approved cannbot-knowledge checkout with its own interpreter. + +Run this worker with the interpreter recorded in the project grant: upstream +requires the MCP SDK constraints and PyYAML from its own environment, and this +program must not import the upstream package into the InferRSI process. Nothing +here writes to the upstream tree; the query CLI is read-only except for an +optional receipt, which is not requested. +""" + +import json +import subprocess +import sys +from pathlib import Path + +_SCRIPT = ".agents/skills/knowledge-query/scripts/knowledge_query.py" +_INDEX = ".agents/skills/knowledge-query/scripts/knowledge_index.py" +_STALE_MARKERS = ("stale", "missing", "rebuild", "index build", "inconsistent") +_FIX_BUILD = ( + "cd && python3 " + _INDEX + " build" +) + + +def _fail(message, *, state="rejected", fix=""): + print(json.dumps({"isError": True, "error": message, "state": state, "fix": fix})) + raise SystemExit(1) + + +def _run(upstream, script, argv, *, timeout): + result = subprocess.run( + [sys.executable, "-I", "-B", str(upstream / script), *argv], + capture_output=True, cwd=str(upstream), timeout=timeout, check=False, + ) + if result.returncode: + detail = result.stderr.decode("utf-8", "replace").strip() + lowered = detail.casefold() + state = "index_stale" if any(m in lowered for m in _STALE_MARKERS) else "rejected" + _fail(detail[-400:] or "cannbot query failed", state=state, + fix=_FIX_BUILD if state == "index_stale" else "") + return json.loads(result.stdout) + + +def discover(upstream): + payload = _run(upstream, _SCRIPT, ["discover", "--json"], timeout=30) + index = payload.get("index") or {} + state = index.get("built_repository_state") or {} + return { + "state": "ok", + "revision": state.get("commit"), + "cards": payload.get("cards"), + "domains": payload.get("domains", {}), + "index": { + "schema": index.get("schema_version"), + "version": index.get("version") or index.get("index_version"), + "content_fingerprint": index.get("content_fingerprint"), + }, + } + + +def search(upstream, arguments): + payload = _run(upstream, _SCRIPT, [ + "search", "--json", "--query", arguments["query"], + "--platform", arguments["platform"], "--limit", str(arguments["limit"]), + ], timeout=60) + hits = [] + for row in payload.get("results") or []: + hits.append({ + "locator": row.get("path"), + "title": row.get("title") or (row.get("path") or "").rsplit("/", 1)[-1], + "summary": row.get("description") or "", + "score": row.get("score") or 0, + "status": row.get("status") or "", + "platforms": row.get("platforms") or [], + "card_sha256": row.get("card_sha256") or "", + }) + index = payload.get("index") or {} + # The pin lives on the index provenance, not at the top level: that is the + # revision the index was built from, which is what a citation must name. + state = index.get("built_repository_state") or {} + return { + "state": "ok", + "revision": state.get("commit"), + "index": { + "schema": index.get("schema_version"), + "version": index.get("index_version"), + "content_fingerprint": index.get("content_fingerprint"), + }, + "score_scale": payload.get("schema_version") or "cannbot", + "items": hits, + } + + +def read(upstream, arguments): + locator = arguments["locator"] + card = (upstream / "knowledge" / (locator + ".md")).resolve() + root = (upstream / "knowledge").resolve() + # A locator is a Concept ID, not a path. Refuse anything that leaves the + # bundle rather than trusting the caller to have passed a search result. + if not card.is_relative_to(root) or not card.is_file(): + _fail("No such card in the approved checkout", state="rejected") + lines = card.read_text(encoding="utf-8").splitlines() + size, page = 200, arguments.get("page", 1) + start = (page - 1) * size + chunk = lines[start:start + size] + return { + "state": "ok", + "locator": locator, + "page": page, + "pages": max(1, (len(lines) + size - 1) // size), + "line_count": len(lines), + "text": "\n".join(chunk), + "has_more": start + size < len(lines), + } + + +if __name__ == "__main__": + try: + upstream = Path(sys.argv[1]).resolve() + action, raw = sys.argv[2], sys.argv[3] if len(sys.argv) > 3 else "{}" + arguments = json.loads(raw or "{}") + if action == "discover": + result = discover(upstream) + elif action == "search": + result = search(upstream, arguments) + elif action == "read": + result = read(upstream, arguments) + else: + _fail("Unknown cannbot read action") + except SystemExit: + raise + except subprocess.TimeoutExpired: + _fail("cannbot query timed out", state="timeout") + except (OSError, ValueError, KeyError, TypeError): + # Paths and upstream internals must not reach the caller. + _fail("The local cannbot checkout is unavailable", state="unreachable") + print(json.dumps(result, ensure_ascii=True, allow_nan=False)) diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 7d84e23d..f9e9d7fb 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -134,11 +134,60 @@ def _clip(value: str, limit: int = 160) -> str: return text if len(text) <= limit else text[: limit - 1].rstrip() + "…" +def _channels(): + """Build the external channel registry from whatever this checkout approved. + + The grant is the second source of a declared backend, after an explicit + `--backend`. Reading it here is what makes `cannbot enable --backend ascend` + mean something: without it, an approved Ascend source would only ever run + when the caller repeated the flag, and the approval would silently do + nothing on the default path. + + Constructing a channel reads its grant, which is a cheap local file read. + An unreadable or absent grant is not fatal: the channel still exists and + reports `not_enabled` with a fix when asked, which is more useful than + dropping it from the list. + """ + from ...knowledge.channels import ChannelRegistry + from ..cannbot import CannbotChannel + + channel = CannbotChannel() + return ChannelRegistry([channel], declared_backend=channel.backend) + + +def _print_external(result: dict) -> None: + """Print the external column under the local one, never mixed into it.""" + entries = result.get("external") or [] + if not entries: + return + print("\nexternal (not locally verified; scores are not comparable across channels)") + for entry in entries: + state = entry.get("state", "unknown") + if state != "ok": + line = f" {entry['channel']}\t[{state}]" + if entry.get("detail"): + line += f"\t{_clip(entry['detail'], 100)}" + print(line) + if entry.get("fix"): + print(f" fix: {_clip(entry['fix'], 120)}") + continue + revision = (entry.get("revision") or "")[:12] or "unpinned" + print(f" {entry['channel']} @ {revision}\t[{entry['score_scale']}]" + f"\t{entry.get('count', 0)} hit(s)\t{entry.get('elapsed_ms', 0)} ms") + for item in entry.get("items", []): + print(f" {item['score']}\t{_clip(item['title'], 60)}\t{item['locator']}") + if entry.get("items"): + print(f" open: {entry['items'][0].get('open', '')}") + for note in result.get("external_notes", []): + print(f" note: {_clip(note, 140)}") + + def _print_find(result: dict) -> None: """Lead with what was tried and how far the evidence goes, then the path.""" if result["no_match"]: print(f"no_match\t{result['corpus_size']} entries searched; " f"rephrase, or search the corpus directly with rg") + _print_external(result) return for item in result["items"]: labels = [_FIND_KIND.get(item["kind"], item["kind"]), _tsv(item["state"])] @@ -187,6 +236,7 @@ def _print_find(result: dict) -> None: print(f" source: {item['source']}") for token, gain in item.get("matched", {}).items(): print(f" [{token}] {gain}") + _print_external(result) def _context(records: list[dict]) -> str: @@ -247,6 +297,27 @@ def main(argv: list[str] | None = None) -> int: if result is not None: _print(result) return 0 + if args.command == "cannbot": + from .. import cannbot + + action = args.cannbot_action + if action == "enable": + result = cannbot.project.enable( + upstream=args.checkout, python=args.python, + backend=args.backend, platforms=args.platform, + ) + elif action == "disable": + result = cannbot.project.disable() + elif action == "describe": + result = cannbot.project.describe() + elif action == "search": + result = cannbot.project.search( + args.query, platform=args.platform, limit=args.limit, + ) + else: + result = cannbot.project.read(args.locator, page=args.page) + _print(result) + return 0 if args.command == "agents": from ...harness import agent_sessions @@ -372,8 +443,14 @@ def main(argv: list[str] | None = None) -> int: unknown = [k for k in kinds if k not in ("case", "skill", "record")] if unknown or not kinds: raise ValueError("--kinds takes a subset of case,skill,record") - result = find(args.query, load_corpus(args.skill_root, kinds=kinds), - limit=args.limit, explain=args.explain) + result = find( + args.query, load_corpus(args.skill_root, kinds=kinds), + limit=args.limit, explain=args.explain, + registry=None if args.no_external else _channels(), + backend=args.backend, + external_limit=args.external_limit, + external_timeout=args.external_timeout, + ) if args.json: _print(result) else: diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index b85c667d..27b0ef70 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -121,6 +121,34 @@ def parser() -> argparse.ArgumentParser: for action in ("query", "explain"): read = mindie_actions.add_parser(action, help=f"Run a MindIE {action} and keep a receipt") read.add_argument("--input", type=Path, required=True, help="JSON tool arguments") + cannbot = commands.add_parser( + "cannbot", help="Use a project-approved, external cannbot-knowledge checkout" + ) + cannbot_actions = cannbot.add_subparsers(dest="cannbot_action", required=True) + cannbot_enable = cannbot_actions.add_parser( + "enable", help="Approve one cannbot-knowledge checkout for this project" + ) + cannbot_enable.add_argument("--checkout", type=Path, required=True) + cannbot_enable.add_argument( + "--python", type=Path, required=True, + help="Interpreter with pyyaml that can run the checkout's knowledge-query scripts", + ) + cannbot_enable.add_argument("--backend", default="ascend") + cannbot_enable.add_argument( + "--platform", action="append", default=[], + help="Default upstream platform; repeatable. Required for a platform-less search.", + ) + cannbot_actions.add_parser("disable", help="Revoke the project grant") + cannbot_actions.add_parser("describe", help="Probe the approved checkout and its index") + cannbot_search = cannbot_actions.add_parser( + "search", help="Search the approved checkout; returns card locators, not text" + ) + cannbot_search.add_argument("query") + cannbot_search.add_argument("--platform") + cannbot_search.add_argument("--limit", type=int, default=100) + cannbot_read = cannbot_actions.add_parser("read", help="Read one card by locator") + cannbot_read.add_argument("locator") + cannbot_read.add_argument("--page", type=int, default=1) agents = commands.add_parser("agents", help="Record native host fresh-session handoffs") agent_actions = agents.add_subparsers(dest="agent_action", required=True) prepare = agent_actions.add_parser("prepare", help="Freeze a task for the current host") @@ -431,6 +459,23 @@ def parser() -> argparse.ArgumentParser: find.add_argument( "--explain", action="store_true", help="Report each query term's contribution" ) + find.add_argument( + "--backend", + help="Declare the task backend once, e.g. ascend; external channels run only " + "for a declared backend and are never inferred from the query", + ) + find.add_argument( + "--no-external", action="store_true", + help="Skip every external channel; use in CI and offline runs", + ) + find.add_argument( + "--external-limit", type=int, + help="Override the per-channel result cap (cannbot 10, MindIE 5 by default)", + ) + find.add_argument( + "--external-timeout", type=float, default=8.0, + help="Per-channel budget in seconds; a timeout is reported, never shown as no hit", + ) find.add_argument("--json", action="store_true") collect = commands.add_parser( diff --git a/src/inferrsi/knowledge/channels.py b/src/inferrsi/knowledge/channels.py new file mode 100644 index 00000000..fbdc4555 --- /dev/null +++ b/src/inferrsi/knowledge/channels.py @@ -0,0 +1,317 @@ +"""Read-only external knowledge channels, ranked beside the local corpus. + +An external channel answers "what does a source outside this repository know +about this problem" without vendoring its content. Only locators come back: +the caller opens what it needs through the channel's own `read`. That is what +keeps a CANN-licensed corpus out of this repository's package, wheel and +evidence tree while still making it searchable. + +Three rules follow from that and are enforced here rather than by convention: + +* A channel result is never merged into the local ranked list. Scores come from + different corpora and are not comparable, so each result carries the scale it + was ranked on and the caller compares across columns, not within one. +* A failure is a state, not an empty list. "Searched and found nothing" and + "could not search" are different facts, and conflating them lets a caller + conclude the source has no such knowledge. +* Nothing here reads a channel body on the ranking path. Reading is a separate + call with its own budget. + +Channels are opt-in. Resolving a registry never imports or probes one until the +platform gate has selected it, and an absent registry leaves `find` behaving +exactly as it did before channels existed. +""" + +from __future__ import annotations + +import concurrent.futures +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +# States a channel can report. `ok` is the only one that carries items; every +# other value means the caller did not get an answer and must say so. +OK = "ok" +NOT_ENABLED = "not_enabled" +UNREACHABLE = "unreachable" +INDEX_STALE = "index_stale" +TIMEOUT = "timeout" +VERSION_MISMATCH = "version_mismatch" +REJECTED = "rejected" +UNKNOWN = "unknown" +BACKEND_UNKNOWN = "backend_unknown" +SKIPPED = "skipped" + +CHANNEL_STATES = ( + OK, NOT_ENABLED, UNREACHABLE, INDEX_STALE, TIMEOUT, + VERSION_MISMATCH, REJECTED, UNKNOWN, BACKEND_UNKNOWN, SKIPPED, +) + +# Per-channel result budget. The upstream locator is precise (a Concept ID) and +# a lexical query can legitimately match hundreds of cards, so the default is +# wide enough that the count reflects coverage rather than the cap; MindIE +# returns coarser refs and buries local hits past a handful. Callers override +# with --external-limit. +DEFAULT_LIMITS = { + "cannbot-knowledge": 100, + "mindie-agent": 5, +} +FALLBACK_LIMIT = 5 + + +@dataclass(frozen=True) +class ChannelStatus: + """Why a channel did or did not answer. Never silently 'empty'.""" + + state: str + revision: str | None = None + detail: str = "" + fix: str = "" + checked_at: float = 0.0 + + def __post_init__(self) -> None: + if self.state not in CHANNEL_STATES: + raise ValueError(f"Unknown channel state: {self.state}") + + def view(self) -> dict: + record: dict[str, Any] = {"state": self.state} + if self.revision: + record["revision"] = self.revision + if self.detail: + record["detail"] = self.detail + if self.fix: + record["fix"] = self.fix + return record + + +@dataclass(frozen=True) +class ChannelHit: + """A locator, not knowledge. Read the card through the channel to use it.""" + + locator: str + title: str + score: float + score_scale: str + summary: str = "" + status: str = "" + platforms: tuple[str, ...] = () + card_sha256: str = "" + + def view(self, *, read_prefix: str) -> dict: + record: dict[str, Any] = { + "locator": self.locator, + "title": self.title, + "score": round(self.score, 2), + "score_scale": self.score_scale, + } + if self.summary: + record["summary"] = self.summary + if self.status: + record["status"] = self.status + if self.platforms: + record["platforms"] = list(self.platforms) + if self.card_sha256: + record["card_sha256"] = self.card_sha256 + if read_prefix: + record["open"] = f"{read_prefix} {self.locator}" + return record + + +@dataclass(frozen=True) +class ChannelResult: + """One channel's answer for one query.""" + + name: str + status: ChannelStatus + items: tuple[ChannelHit, ...] = () + elapsed_ms: int = 0 + score_scale: str = "" + read_prefix: str = "" + + def view(self) -> dict: + record: dict[str, Any] = {"channel": self.name, **self.status.view()} + record["elapsed_ms"] = self.elapsed_ms + if self.score_scale: + record["score_scale"] = self.score_scale + record["count"] = len(self.items) + if self.items: + record["items"] = [item.view(read_prefix=self.read_prefix) for item in self.items] + return record + + +@runtime_checkable +class Channel(Protocol): + """One external read-only source. + + `backend` is a declaration, not a guess: a channel that serves one platform + says so, and a channel with `None` is platform-neutral. Selection never + infers a backend from a query string, because a wrong inference is silent. + """ + + name: str + backend: str | None + + def describe(self) -> ChannelStatus: ... + + def search( + self, query: str, *, platform: str | None, limit: int + ) -> ChannelResult: ... + + def read(self, locator: str) -> str: ... + + +@dataclass +class ChannelRegistry: + """The channels this checkout has, and the gate that decides which run. + + `declared_backend` is a fact about the task, not about a channel: it comes + from an explicit `--backend` or from an approved grant. `Channel.backend` + says which platform a source serves and is only used to *select* channels. + Conflating the two made an undeclared task search anyway, because a + channel's own backend looked like a declaration. + """ + + channels: list[Channel] = field(default_factory=list) + declared_backend: str | None = None + + def for_backend(self, backend: str | None) -> list[Channel]: + """Return channels eligible for a declared task backend. + + No backend means no external search: an undeclared platform is not + evidence that the task is platform-neutral, and querying every channel + "just in case" would let a CUDA task silently receive Ascend guidance. + """ + if backend is None: + return [] + return [ + channel for channel in self.channels + if channel.backend is None or channel.backend == backend + ] + + def ungated(self) -> list[Channel]: + """Channels that would have run, for explaining a missing gate.""" + return [ + channel for channel in self.channels + if channel.backend is not None + ] or list(self.channels) + + +def _empty(name: str, state: str, *, detail: str = "", fix: str = "", + revision: str | None = None) -> ChannelResult: + return ChannelResult( + name=name, status=ChannelStatus(state, revision=revision, detail=detail, fix=fix) + ) + + +def _run_one( + channel: Channel, query: str, *, platform: str | None, limit: int, timeout: float +) -> ChannelResult: + try: + described = channel.describe() + except Exception as exc: # a broken channel must not break find + return _empty(channel.name, UNREACHABLE, detail=_reason(exc)) + if described.state != OK: + return ChannelResult(name=channel.name, status=described) + try: + result = channel.search(query, platform=platform, limit=limit) + except TimeoutError: + return _empty(channel.name, TIMEOUT, detail=f"no answer within {timeout:g}s") + except Exception as exc: + return _empty(channel.name, REJECTED, detail=_reason(exc)) + if result.elapsed_ms > timeout * 1000: + # The work finished, so report what it found and the budget it broke. + return ChannelResult( + name=channel.name, status=ChannelStatus( + OK, revision=result.status.revision, + detail=f"answered in {result.elapsed_ms} ms, over the {timeout:g}s budget", + ), + items=result.items, elapsed_ms=result.elapsed_ms, + score_scale=result.score_scale, read_prefix=result.read_prefix, + ) + return result + + +def _reason(exc: BaseException) -> str: + text = str(exc).strip() + return text or type(exc).__name__ + + +def search_channels( + registry: ChannelRegistry, + query: str, + *, + backend: str | None, + platform: str | None = None, + limit: int | None = None, + timeout: float = 8.0, +) -> list[ChannelResult]: + """Query every eligible channel concurrently; one failure never hides another. + + Channels are independent I/O, so the wall clock is the slowest one rather + than the sum. An empty registry is not an error: it means this checkout has + no external source, which is the default state. + """ + if timeout <= 0: + raise ValueError("external timeout must be positive") + if limit is not None and (type(limit) is not int or limit < 1): + raise ValueError("external limit must be a positive integer") + eligible = registry.for_backend(backend) + if not eligible: + return [] + + def work(channel: Channel) -> ChannelResult: + return _run_one( + channel, query, platform=platform, + limit=limit or DEFAULT_LIMITS.get(channel.name, FALLBACK_LIMIT), + timeout=timeout, + ) + + if len(eligible) == 1: + return [work(eligible[0])] + with concurrent.futures.ThreadPoolExecutor(max_workers=len(eligible)) as pool: + futures = [pool.submit(work, channel) for channel in eligible] + results = [] + for channel, future in zip(eligible, futures): + try: + results.append(future.result(timeout=timeout)) + except concurrent.futures.TimeoutError: + results.append(_empty( + channel.name, TIMEOUT, detail=f"no answer within {timeout:g}s", + )) + return results + + +def explain_gate(registry: ChannelRegistry, backend: str | None) -> list[dict]: + """Say which channels exist but were not asked, and why. + + An undeclared backend is the one case where a caller could reasonably + expect an answer and get none, so it is reported rather than passed over. + A channel that already knows it cannot answer overrides the gate reason: + telling someone to declare a backend for a source that is not approved yet + sends them down the wrong fix. + """ + if backend is not None: + return [] + records = [] + for channel in registry.ungated(): + hint = getattr(channel, "hint", None) + if isinstance(hint, ChannelStatus) and hint.state != OK: + records.append({ + "channel": channel.name, + "state": hint.state, + "detail": hint.detail, + **({"fix": hint.fix} if hint.fix else {}), + }) + continue + records.append({ + "channel": channel.name, + "backend": channel.backend, + "state": BACKEND_UNKNOWN, + "fix": f"inferrsi find --backend {channel.backend or ''}", + "detail": "No task backend was declared; external channels are not searched.", + }) + return records + + +def external_view(results: list[ChannelResult], *, notes: list[str] | None = None) -> list[dict]: + """Render channel results for `find`'s output, unchanged in shape when empty.""" + return [result.view() for result in results] diff --git a/src/inferrsi/knowledge/knowledge_find.py b/src/inferrsi/knowledge/knowledge_find.py index c005c0e3..a6910908 100644 --- a/src/inferrsi/knowledge/knowledge_find.py +++ b/src/inferrsi/knowledge/knowledge_find.py @@ -729,7 +729,18 @@ def _why_unreadable(document: Document, readers: dict) -> str | None: return None -def find(query: str, corpus: Corpus, *, limit: int = 5, explain: bool = False) -> dict: +def find( + query: str, + corpus: Corpus, + *, + limit: int = 5, + explain: bool = False, + registry: "ChannelRegistry | None" = None, + backend: str | None = None, + platform: str | None = None, + external_limit: int | None = None, + external_timeout: float = 8.0, +) -> dict: if type(limit) is not int or limit < 1: raise ValueError("limit must be a positive integer") # A topic alias can add backend terms for matching, as 图模式 adds cuda graph. @@ -774,4 +785,51 @@ def find(query: str, corpus: Corpus, *, limit: int = 5, explain: bool = False) - # Report what a query term was taken to mean, so a wrong expansion is # visible in the result rather than only in the ranking. result["expanded"] = {term: list(values) for term, values in applied.items()} + return _with_channels( + result, query, registry=registry, backend=backend, platform=platform, + external_limit=external_limit, external_timeout=external_timeout, + ) + + +def _with_channels( + result: dict, + query: str, + *, + registry: "ChannelRegistry | None", + backend: str | None, + platform: str | None, + external_limit: int | None, + external_timeout: float, +) -> dict: + """Add the external column, or leave the result untouched when there is none. + + External hits stay out of `items`. They were not ranked on this corpus and + are not locally verified, so putting them in the same list would let any + caller that only reads `items` treat them as validated knowledge. The list + is also absent rather than empty when no channel exists, so callers that + predate channels see the output they always got. + """ + if registry is None or not registry.channels: + return result + from .channels import explain_gate, search_channels + + declared = backend if backend is not None else registry.declared_backend + results = search_channels( + registry, query, backend=declared, platform=platform, + limit=external_limit, timeout=external_timeout, + ) + if declared is not None: + result["backend"] = declared + if results: + result["external"] = [item.view() for item in results] + gated = explain_gate(registry, declared) + if gated: + result.setdefault("external", []).extend(gated) + if results or gated: + result["external_notes"] = [ + "External results are not locally verified; scores are not " + "comparable across channels.", + "Pin the reported revision when citing an external locator.", + "Open external cards with the channel's read command, not `read`.", + ] return result diff --git a/tests/test_cannbot_channel_acceptance.py b/tests/test_cannbot_channel_acceptance.py new file mode 100644 index 00000000..42eab44c --- /dev/null +++ b/tests/test_cannbot_channel_acceptance.py @@ -0,0 +1,152 @@ +"""Before/after: does the external channel actually widen what `find` reaches. + +This file is the acceptance measurement for the cannbot channel. It runs the +real CLI twice per query -- once gated off, once gated on -- and reports the +counts. The upstream half needs an approved checkout and a built index, so it +is skipped rather than faked when either is absent: a fabricated number here +would be worse than no number. +""" + +import json +import shutil +import subprocess +import sys +from importlib.resources import files +from pathlib import Path + +import pytest + +PROJECT = Path(__file__).resolve().parents[1] +QUERIES = ( + "AscendC tiling 参数怎么计算", + "CeilDivision 向上取整", + "matmul 分块 性能", + "算子 精度 调试 溢出", + "ACLGraph 图模式 捕获", + "KV cache 显存 布局", +) + + +def _run(*args): + argv = [sys.executable, "-m", "inferrsi", *args] + if args[0] != "cannbot": # `cannbot` subcommands print JSON already + argv.append("--json") + result = subprocess.run(argv, capture_output=True, text=True, cwd=PROJECT) + if result.returncode: + return None + return json.loads(result.stdout) + + +def _approved(): + info = _run("cannbot", "describe") + return bool(info) and info.get("state") == "ok" + + +def test_gate_off_matches_the_pre_channel_behaviour(): + """`--no-external` must reproduce the output callers already relied on. + + An unflagged run is *supposed* to differ: it adds the gate explanation. + The invariant is only about `--no-external`, and it must be exact. + """ + from inferrsi.knowledge.knowledge_find import find, load_corpus + from inferrsi.apps.cli import _channels + + corpus = load_corpus() + pre_channel = find("AscendC tiling", corpus, limit=5) + gated = _run("find", "AscendC tiling", "--no-external") + assert gated is not None + assert "external" not in gated + # Same keys, same counts, same order as the function before channels existed. + assert sorted(gated) == sorted(pre_channel) + assert [i["id"] for i in gated["items"]] == [i["id"] for i in pre_channel["items"]] + assert _channels() is not None # the registry exists; the flag is what skips it + + +def test_undeclared_backend_reports_the_gate(): + """Only an undeclared backend may skip a channel without a grant. + + Once a grant records `backend: ascend`, the default path is a declaration + and must search. The gate explanation is for the case where neither an + explicit flag nor an approved grant says what platform this is. + """ + result = _run("find", "AscendC tiling") + assert result is not None + from inferrsi.apps.cli import _channels + + registry = _channels() + entries = result.get("external", []) + if registry.declared_backend is None: + assert all(entry["state"] != "ok" for entry in entries) + assert any("fix" in entry for entry in entries) + else: + assert any(entry["state"] == "ok" for entry in entries), ( + "an approved grant declares the backend, so the default path must search" + ) + + +@pytest.mark.skipif(not _approved(), reason="no approved cannbot checkout with a built index") +def test_local_corpus_is_unchanged_by_the_channel(): + """The channel adds a column; it never re-ranks or trims the local list.""" + for query in QUERIES: + before = _run("find", query, "--no-external") + after = _run("find", query, "--backend", "ascend") + assert before["matched"] == after["matched"], query + assert [item["id"] for item in before["items"]] == [ + item["id"] for item in after["items"] + ], query + + +@pytest.mark.skipif(not _approved(), reason="no approved cannbot checkout with a built index") +def test_channel_returns_locators_with_a_pin(): + result = _run("find", QUERIES[1], "--backend", "ascend") + entry = next(item for item in result["external"] if item["channel"] == "cannbot-knowledge") + assert entry["state"] == "ok" + assert len(entry["revision"]) == 40 + assert entry["score_scale"].startswith("knowledge-search") + assert entry["items"], "the upstream source has cards for this query" + for hit in entry["items"]: + assert hit["locator"] and "/" in hit["locator"] + assert hit["card_sha256"] + assert hit["open"].startswith("inferrsi cannbot read ") + + +@pytest.mark.skipif(not _approved(), reason="no approved cannbot checkout with a built index") +def test_external_hits_never_enter_items(): + for query in QUERIES: + result = _run("find", query, "--backend", "ascend") + local_ids = {item["id"] for item in result["items"]} + for entry in result["external"]: + for hit in entry.get("items", []): + assert hit["locator"] not in local_ids + + +@pytest.mark.skipif(not _approved(), reason="no approved cannbot checkout with a built index") +def test_every_query_reaches_more_than_the_local_corpus(): + """The point of the channel: each query gains upstream coverage.""" + rows = [] + for query in QUERIES: + before = _run("find", query, "--no-external") + after = _run("find", query, "--backend", "ascend") + external = sum( + entry.get("count", 0) for entry in after["external"] if entry["state"] == "ok" + ) + rows.append((query, before["matched"], external)) + report = "\n".join( + f"{query:28} local={local:>4} external={external:>3}" for query, local, external in rows + ) + print("\nfind coverage before/after:\n" + report) + assert all(external > 0 for _, _, external in rows), report + assert any(before == 0 for _, before, _ in rows), ( + "at least one Ascend query should be empty locally and answered upstream" + ) + + +@pytest.mark.skipif(not _approved(), reason="no approved cannbot checkout with a built index") +def test_read_returns_one_pinned_card(): + result = _run("find", "CeilDivision", "--backend", "ascend") + entry = next(item for item in result["external"] if item["channel"] == "cannbot-knowledge") + locator = entry["items"][0]["locator"] + card = _run("cannbot", "read", locator) + assert card["state"] == "ok" + assert card["locator"] == locator + assert card["text"].strip() diff --git a/tests/test_knowledge_channels.py b/tests/test_knowledge_channels.py new file mode 100644 index 00000000..eb2b4271 --- /dev/null +++ b/tests/test_knowledge_channels.py @@ -0,0 +1,213 @@ +"""External channels: gating, per-channel states, and no mixing into `items`.""" + +import json +import sys +from pathlib import Path + +import pytest + +from inferrsi.knowledge.channels import ( + BACKEND_UNKNOWN, + DEFAULT_LIMITS, + NOT_ENABLED, + OK, + TIMEOUT, + UNREACHABLE, + ChannelHit, + ChannelRegistry, + ChannelResult, + ChannelStatus, + explain_gate, + search_channels, +) +from inferrsi.knowledge.knowledge_find import Corpus, Document, find + + +class FakeChannel: + """One channel with scripted behaviour, so every state is reachable.""" + + def __init__(self, *, name="fake", backend="ascend", status=None, items=(), error=None, + delay_state=None): + self.name = name + self.backend = backend + self._status = status or ChannelStatus(OK, revision="deadbeef" * 5) + self._items = tuple(items) + self._error = error + self._delay_state = delay_state + self.searched = 0 + self.platform_seen = None + self.limit_seen = None + + def describe(self): + if self._error: + raise RuntimeError(self._error) + return self._status + + def search(self, query, *, platform, limit): + self.searched += 1 + self.platform_seen = platform + self.limit_seen = limit + if self._delay_state: + raise TimeoutError() + return ChannelResult( + name=self.name, status=ChannelStatus(OK, revision="deadbeef" * 5), + items=self._items, elapsed_ms=12, score_scale="fake/v1", + read_prefix="fake read", + ) + + def read(self, locator): + return locator + + +def _hit(locator="ops/x/card", score=1.5): + return ChannelHit( + locator=locator, title="t", score=score, score_scale="fake/v1", + card_sha256="ab" * 32, + ) + + +@pytest.fixture(scope="module") +def corpus(): + return Corpus([ + Document( + identity="ascendc-tiling", kind="skill", state="validated", path="a/SKILL.md", + title="ascendc tiling design", summary="tiling", qualified="checkout:ascendc-tiling", + backend="ascend", source_location="checkout", + fields={"title": "ascendc tiling design", "summary": "tiling param", + "tags": "", "findings": "", "applicability": "", "body": "tiling"}, + ), + ]) + + +def test_registry_without_channels_leaves_result_untouched(corpus): + plain = find("tiling", corpus, limit=3) + gated = find("tiling", corpus, limit=3, registry=ChannelRegistry([]), backend="ascend") + assert plain == gated + assert "external" not in gated + + +def test_undeclared_backend_does_not_search(corpus): + channel = FakeChannel(items=[_hit()]) + result = find("tiling", corpus, backend=None, registry=ChannelRegistry([channel])) + assert channel.searched == 0 + assert result["items"] + assert result["external"][0]["state"] == BACKEND_UNKNOWN + assert "fix" in result["external"][0] + + +def test_ascend_backend_searches_even_when_local_matches(corpus): + channel = FakeChannel(items=[_hit()]) + result = find("tiling", corpus, backend="ascend", registry=ChannelRegistry([channel])) + assert result["matched"] >= 1 # local still hits + assert channel.searched == 1 # and the channel still ran + assert result["external"][0]["count"] == 1 + + +def test_other_backend_skips_ascend_channel(corpus): + channel = FakeChannel(items=[_hit()]) + result = find("tiling", corpus, backend="cuda", registry=ChannelRegistry([channel])) + assert channel.searched == 0 + assert "external" not in result + + +def test_external_hits_never_enter_items(corpus): + channel = FakeChannel(items=[_hit("ops/only/external")]) + result = find("tiling", corpus, backend="ascend", registry=ChannelRegistry([channel])) + assert all(item["id"] != "ops/only/external" for item in result["items"]) + assert result["external"][0]["items"][0]["locator"] == "ops/only/external" + + +def test_hit_carries_score_scale_and_sha(corpus): + channel = FakeChannel(items=[_hit()]) + result = find("tiling", corpus, backend="ascend", registry=ChannelRegistry([channel])) + hit = result["external"][0]["items"][0] + assert hit["score_scale"] == "fake/v1" + assert hit["card_sha256"] == "ab" * 32 + assert hit["open"] == "fake read ops/x/card" + + +def test_failure_is_a_state_not_empty_list(corpus): + channel = FakeChannel(status=ChannelStatus(NOT_ENABLED, detail="no grant", + fix="inferrsi cannbot enable")) + result = find("tiling", corpus, backend="ascend", registry=ChannelRegistry([channel])) + entry = result["external"][0] + assert entry["state"] == NOT_ENABLED + assert entry["fix"] == "inferrsi cannbot enable" + assert channel.searched == 0 + + +def test_timeout_is_reported_with_its_own_state(corpus): + channel = FakeChannel(delay_state=True) + results = search_channels(ChannelRegistry([channel]), "tiling", backend="ascend", timeout=1) + assert results[0].status.state == TIMEOUT + assert results[0].items == () + + +def test_broken_describe_is_unreachable(): + channel = FakeChannel(error="boom") + results = search_channels(ChannelRegistry([channel]), "q", backend="ascend") + assert results[0].status.state == UNREACHABLE + assert "boom" in results[0].status.detail + + +def test_one_channel_failure_does_not_hide_another(): + good = FakeChannel(name="good", items=[_hit()]) + bad = FakeChannel(name="bad", error="down") + results = search_channels( + ChannelRegistry([good, bad]), "q", backend="ascend", timeout=2, + ) + by_name = {item.name: item for item in results} + assert by_name["good"].items + assert by_name["bad"].status.state == UNREACHABLE + + +def test_default_limits_match_the_design(): + assert DEFAULT_LIMITS["cannbot-knowledge"] == 100 + assert DEFAULT_LIMITS["mindie-agent"] == 5 + + +def test_explicit_limit_overrides_per_channel_default(): + channel = FakeChannel() + search_channels(ChannelRegistry([channel]), "q", backend="ascend", limit=3) + assert channel.limit_seen == 3 + + +def test_each_channel_gets_its_own_default_limit(): + cannbot = FakeChannel(name="cannbot-knowledge") + mindie = FakeChannel(name="mindie-agent") + search_channels(ChannelRegistry([cannbot, mindie]), "q", backend="ascend") + assert cannbot.limit_seen == 100 + assert mindie.limit_seen == 5 + + +def test_platform_is_passed_through(): + channel = FakeChannel() + search_channels(ChannelRegistry([channel]), "q", backend="ascend", platform="a3") + assert channel.platform_seen == "a3" + + +def test_not_enabled_hint_beats_backend_unknown(): + channel = FakeChannel(backend=None) + channel.hint = ChannelStatus(NOT_ENABLED, detail="no grant", fix="enable it") + gate = explain_gate(ChannelRegistry([channel]), None) + assert gate[0]["state"] == NOT_ENABLED + assert gate[0]["fix"] == "enable it" + + +def test_rejects_non_positive_timeout(): + with pytest.raises(ValueError): + search_channels(ChannelRegistry([FakeChannel()]), "q", backend="ascend", timeout=0) + + +def test_registry_declared_backend_is_what_the_task_asked_for(): + """A channel's own backend selects it; it never declares the task. + + Reading `Channel.backend` as the task backend made an undeclared task + search anyway. The registry carries the declaration separately so the two + facts stay distinguishable. + """ + channel = FakeChannel(backend="ascend", items=[_hit()]) + undeclared = ChannelRegistry([channel]) + assert undeclared.for_backend(undeclared.declared_backend) == [] + declared = ChannelRegistry([channel], declared_backend="ascend") + assert declared.for_backend(declared.declared_backend) == [channel] From a38d360d79317aadc771db1b75949f758bf9cc04 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Tue, 17 Nov 2026 20:27:41 +0800 Subject: [PATCH 2/5] docs(knowledge): document that external channels are not installed A fresh install has no external channel, so an Ascend query that the packaged corpus cannot answer returns `no_match` with no external results. That is the intended state, but nothing said so: an agent reading only `no_match` would conclude the knowledge does not exist anywhere, when the upstream corpus holds 10,550 cards and this repository holds 6. `find` already reports `not_enabled` with a fix on each call. That is not enough on its own, because it arrives only after a query someone had to think of making. The docs now say, before the query: - a fresh install has no channel, and `no_match` beside a `not_enabled` line means the packaged corpus is empty rather than the topic; - why the checkout is a person's explicit action -- CANN OSL v2.0 is non-transferable and non-sublicensable, so cloning it on someone's behalf would accept its terms for them; - the setup sequence, including the one-time index build the server requires before it will answer; - the backend gate: `ascend` searches, other backends do not, undeclared reports `backend_unknown`; - how to read `external[]`, and that a non-`ok` state is not "no results". Refs #476 Co-Authored-By: Claude Fable 5.2 --- docs/external-channels.md | 196 +++++++ docs/operations.md | 35 ++ ...11-17-cannbot-knowledge-integration-rfc.md | 503 ------------------ .../2026-11-17-external-knowledge-channels.md | 457 ---------------- 4 files changed, 231 insertions(+), 960 deletions(-) create mode 100644 docs/external-channels.md delete mode 100644 docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md delete mode 100644 docs/plans/2026-11-17-external-knowledge-channels.md diff --git a/docs/external-channels.md b/docs/external-channels.md new file mode 100644 index 00000000..a99c2b23 --- /dev/null +++ b/docs/external-channels.md @@ -0,0 +1,196 @@ +# External knowledge channels + +`inferrsi find` can query knowledge that lives outside this repository. This page +explains what that is, why it is not installed with the package, and what to do +when you need it. + +## A fresh install has no external channel + +This is the **default and intended state**, not a missing installation step. + +```bash +$ inferrsi find "CeilDivision 向上取整" +no_match 135 entries searched; rephrase, or search the corpus directly with rg + +external (not locally verified; scores are not comparable across channels) + cannbot-knowledge [not_enabled] cannbot knowledge is not enabled for this project + fix: inferrsi cannbot enable --checkout --python +``` + +`no_match` there is honest: the packaged corpus really holds nothing for that +query. The `not_enabled` line says an external source exists, was not approved +for this checkout, and how to approve it. Without that line a caller would +conclude the knowledge does not exist anywhere, which would be wrong. + +On `main` `6c7e2379`, the packaged corpus holds **6 Ascend content cards and 0 +per-API cards**. The external cannbot-knowledge corpus holds **10,550 cards, +2,993 of them Ascend C API cards**. Ignoring that gap is what the channel exists +to prevent. + +## Why it is not downloaded for you + +cannbot-knowledge is licensed under **CANN Open Software License Agreement v2.0**, +which is not an OSI licence: the grant is limited, royalty-free but +**non-transferable, non-sublicensable and revocable**, and its use is restricted +to Huawei AI processor systems. This repository is Apache-2.0. + +Cloning that corpus on your behalf would mean accepting its terms for you and +placing a copy on your machine — a decision this project has no standing to +make. So `enable` is a person's explicit action. It is not an unimplemented +convenience; it is the compliance boundary. + +Nothing from an external checkout is ever copied into `src/inferrsi/knowledge/data/`, +a wheel, or `evidence/`. + +## Setting one up + +You supply a checkout, an interpreter, and the backend it serves: + +```bash +git clone https://gitcode.com/cann/cannbot-knowledge.git ~/src/cannbot-knowledge +python3 -m venv ~/venvs/cannbot && ~/venvs/cannbot/bin/pip install pyyaml + +# Build the search index once; the server refuses to answer a stale index. +~/venvs/cannbot/bin/python \ + ~/src/cannbot-knowledge/.agents/skills/knowledge-query/scripts/knowledge_index.py build + +inferrsi cannbot enable \ + --checkout ~/src/cannbot-knowledge \ + --python ~/venvs/cannbot/bin/python \ + --backend ascend \ + --platform a2 +``` + +`enable` proves the checkout answers before writing anything: it checks for +`.git`, the `knowledge/` bundle, the `knowledge-query` scripts and an executable +interpreter, then runs a real `discover`. A checkout that cannot answer is +refused rather than recorded, so a later search never fails for a reason you +already could have fixed. + +| Flag | Meaning | +| --- | --- | +| `--checkout` | Git checkout containing `knowledge/` | +| `--python` | Interpreter with `pyyaml`, able to run the checkout's scripts | +| `--backend` | Backend this source serves; `find` compares it to the task backend | +| `--platform` | Default upstream platform (`a2`, `a3`, `950`, `310p`, `910`, …) | + +Check it: + +```bash +$ inferrsi cannbot describe +{"state": "ok", "revision": "92dc2607...", "cards": 10550, + "index": {"schema": "cannbot-knowledge-index.v1", "version": 3, + "content_fingerprint": "75ef3269..."}} +``` + +Revoke it with `inferrsi cannbot disable`. The grant lives at +`~/.config/inferrsi/cannbot-projects/.json` and is bound to the +project's `.git` identity, so copying it to another clone does not carry the +approval along. + +## Using it + +Once approved, the default path searches automatically because the grant declares +the backend: + +```bash +inferrsi find "AscendC tiling 参数怎么算" +``` + +Or declare it per call: + +```bash +inferrsi find "AscendC tiling 参数怎么算" --backend ascend +inferrsi cannbot search "CeilDivision 向上取整" --platform a2 --limit 100 +inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division +``` + +### The backend gate + +| Task backend | Channels searched | +| --- | --- | +| `ascend` | `cannbot-knowledge`, plus any platform-neutral channel | +| `cuda`, `mlx`, `rocm`, `cpu` | none — an Ascend source must not answer a CUDA task | +| not declared, no grant | none; each channel reports `backend_unknown` | + +The backend is never inferred from the query text. A query containing "asc" is +not evidence that the task is an Ascend task, and a wrong inference would be +silent. + +### Reading the result + +```json +"external": [{ + "channel": "cannbot-knowledge", "state": "ok", + "revision": "92dc2607...", "elapsed_ms": 1310, + "score_scale": "knowledge-search.v5", "count": 3, + "items": [{"locator": "ops/ascendc/apis/.../ceil_division", "score": 14.0, + "status": "stable", "platforms": ["a2", "a3"], + "card_sha256": "b97cbe82...", + "open": "inferrsi cannbot read ops/..."}] +}] +``` + +Three things to keep in mind: + +**External hits are never in `items`.** They were not ranked on this corpus and +are not locally verified. A caller that only reads `items` therefore cannot +mistake them for validated knowledge. + +**Scores are not comparable across channels.** `score_scale` names the system +that produced the number. The packaged corpus ranks 143 skills; upstream ranks +10,550 cards; the same word has a different idf in each. Merging them would mean +inventing a normalisation, so the columns stay separate. + +**A state other than `ok` is not "no results".** `not_enabled`, `unreachable`, +`index_stale`, `timeout`, `version_mismatch`, `rejected` and `backend_unknown` +each mean the channel did not answer. Read `detail` and `fix`. + +## Coverage measured + +Six Ascend queries against a checkout at `92dc2607`, default cap 100. Local +counts are identical with and without the channel; only the external column +changes. + +| Query | Local only | Local + external | +| --- | --- | --- | +| AscendC tiling 参数怎么计算 | 14 | 114 | +| CeilDivision 向上取整 | 0 | 100 | +| matmul 分块 性能 | 10 | 110 | +| 算子 精度 调试 溢出 | 75 | 175 | +| ACLGraph 图模式 捕获 | 100 | 200 | +| KV cache 显存 布局 | 128 | 228 | +| **Total** | **327** | **927** | + +## Evidence and licensing + +An external result is **not locally verified**. Cite it as +`cannbot-knowledge @ ` and say so. It does not participate in a `PASS` +verdict. + +When a receipt is added (not in this release), it will hold the locator, the +card's `card_sha256` and the index revision — never card text, never +`frontmatter`, never a `local_path`. A digest proves which card was cited without +redistributing it. + +Whether CANN OSL v2.0 permits even a local read-only copy is an open question for +the organisation; see #476 Q1. This page documents the mechanism, not a legal +conclusion. + +## What is not implemented + +| Capability | State | +| --- | --- | +| `cannbot enable / disable / describe / search / read` | implemented | +| `find` integration with gating, budgets, per-channel states | implemented | +| MindIE as a second channel | not yet; `apps/mindie/` has the reader, not the adapter | +| `cannbot serve` (stdio MCP proxying `discover` / `search` / `read_card`) | not yet | +| `campaign import-cannbot` (receipts into evidence) | not yet | +| Automatic clone, index build or deployment | deliberately absent | + +## Related + +- [Operation guide](operations.md) — every entry point, effect class, boundary +- [#476](https://github.com/InferMatrixLab/InferRSI/issues/476) — access design and licence boundary +- [#388](https://github.com/InferMatrixLab/InferRSI/issues/388) — external dependency governance +- [Knowledge base](knowledge-base.md) — the packaged corpus this sits beside diff --git a/docs/operations.md b/docs/operations.md index 46eb49bc..3bf214c6 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -262,6 +262,41 @@ question calls for prior evidence. - **Side effects:** none. No revision is pinned; use `read`/`resource` when a capture or evidence record needs exact bytes. - **Example:** `inferrsi find "NPU OOM after caching per-step masks" --limit 3` returns the casebook entry recording that attempt first. A vaguer phrasing ranks the procedural skills above it, which is the first limit below rather than a defect. +#### External channels are not installed with this package + +`find` also queries external read-only channels when the task backend is +declared. **A fresh install has none**, so a query that the packaged corpus +cannot answer returns `no_match` with no external results. That is the default +state, not a fault: + +```text +no_match 135 entries searched; rephrase, or search the corpus directly with rg + +external (not locally verified; scores are not comparable across channels) + cannbot-knowledge [not_enabled] cannbot knowledge is not enabled for this project + fix: inferrsi cannbot enable --checkout --python +``` + +Enabling a channel is a deliberate, per-checkout action by a person; see +[external knowledge channels](external-channels.md). This repository does not +clone, download or bundle a third-party corpus, because doing so would accept +and redistribute a licence on the user's behalf. + +- **Inputs:** `--backend ` declares the task backend; **without it and + without an approved grant, no channel is searched** and each one reports + `backend_unknown`. `--no-external` skips all channels for CI and offline runs. + `--external-limit ` overrides the per-channel cap (`cannbot-knowledge` 100, + `mindie-agent` 5). `--external-timeout ` bounds each channel (8s default). +- **Output:** `external[]` sits beside `items`, never inside it. Each entry + carries `state`, `revision`, `elapsed_ms`, `score_scale` and `count`. A state + other than `ok` (`not_enabled`, `unreachable`, `index_stale`, `timeout`, + `version_mismatch`, `rejected`, `backend_unknown`) means the channel did not + answer — it is **not** the same as answering with no hits. +- **Side effects:** none on this repository. A channel reads an external + checkout that the person approved; nothing is copied into the package, a + wheel or `evidence/`. +- **Determinism:** `--no-external` reproduces the pre-channel output exactly. + ### Collect — `collect` / `KnowledgeReader.collect` Optional `--relevance ` ranks generated packaged skills after the existing diff --git a/docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md b/docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md deleted file mode 100644 index e79a6364..00000000 --- a/docs/plans/2026-11-17-cannbot-knowledge-integration-rfc.md +++ /dev/null @@ -1,503 +0,0 @@ -# [RFC] 全量接入 cannbot-knowledge:MCP 常驻查询通道设计 - -> **Status: Proposed — 形态已选定,实施设计待评审。** 形态选择已由维护者确认(见 §0);本文余下部分定义接入契约与切片,仍待评审接受。接受本文不等于授权实施、代码搬家或对外发布。 -> Governing dependency: [#388](https://github.com/InferMatrixLab/InferRSI/issues/388)(外部技能/知识依赖治理)。本文是其"外链或 submodule 待决"的候选实现,落实为选定的第三方机制,但**不关闭 #388**。 -> 接口依赖: [#387](https://github.com/InferMatrixLab/InferRSI/issues/387)(`read` / `collect` 公开接口,仍为 Proposed)。 -> 文中所有上游 revision 与统计值都是**写作时的实测观测值**,不是本仓 pin,也不是验收基线。 - -## 0. 决策摘要 - -| 项 | 决定 | 位置 | -| --- | --- | --- | -| 接入形态 | **只读 MCP 常驻查询通道**(方案 D 升格为主接法);内容永不进入本仓、wheel、`evidence/` | §5.1 | -| 触发语义 | **平台门控命中即查**,与本地是否命中无关;不再"零命中才补漏" | §5.2 | -| 平台门控来源 | 显式声明(`--backend` 或 grant),**不推断**;`--backend` 只接受单值 | §5.2 | -| 结果呈现 | **分栏并列**,外部结果不进 `items`,分数带 `score_scale`,不跨通道合并 | §5.3 | -| 每通道条数 | `cannbot-knowledge` 10 / `mindie-agent` 5 | §5.4 | -| `find` 默认 | **默认带外部查询**;CI 与单测必须显式 `--no-external` | §5.4 | -| MCP 面 | **透传**上游 `discover` / `search` / `read_card`,不收窄 | §8.5 | -| 证据 | receipt 只存 `locator` + `card_sha256` + `revision`,不含正文 | §7 | - -### 为何从"候选比较"走到"选定 MCP" - -前几轮评审已比较过五种形态(包内转换、submodule、外链锁缓存、MCP、混合)。选定 MCP 的决定性理由有两条,其余形态无法同时满足: - -1. **许可**:上游为 CANN OSL v2.0,本仓为 Apache-2.0。MCP 让内容永不落盘,许可冲突从机制上消失。 -2. **产能**:上游已有 10,550 张卡与自身的 ingest/lint/index 生产系统。任何"复制或缓存进本仓"的形态都在与一个数量级的产能赛跑。 - -## 1. 现状:InferRSI 的 Ascend / CANN 知识极度匮乏 - -这是本 RFC 的起因。在 `main` `6c7e237935a22cc5e90e954d4f397376e753e7d8`(2026-11-16)上实测 `src/inferrsi/knowledge/data/`: - -| 项 | 实测值 | 说明 | -| --- | --- | --- | -| 打包知识数据总量 | 7.8 MiB / 850 个 md+json / 921 个文件 | 含全部 skill、引用与记录 | -| 知识记录总数 | 128 | `repo_packs` 108 + `knowledge_packs` 20 | -| 其中 Ascend 记录 | **22** | 占 17.2% | -| Ascend 记录中真正的内容卡 | **6** | 3 条 Qwen3-TTS audio、3 条 vLLM-Omni diffusion | -| Ascend 记录中仅是 skill 指针 | 16 | 只指向 `skills` 组件,不承载技术结论 | -| Ascend 硬件 skill | 14 / 143 个 `SKILL.md` | 380 KiB,43 个文件 | -| Ascend skill 正文 | 131,860 字节 | 14 篇 `SKILL.md` 合计 | -| Ascend skill 引用资料 | 174,559 字节 / 29 个文件 | 含 3 份上游 LICENSE 抄本 | -| **per-API 粒度知识卡** | **0** | 全仓没有任何一张卡对应单个 CANN/AscendC API | -| 出现 `aclnn*` / `__aicore__` 签名的文件 | **1** | `frameworks/torch-npu/.../references/development.md` | - -对照上游 cannbot-knowledge 的同一维度: - -| 项 | 上游实测值 | 倍数 | -| --- | --- | --- | -| 知识卡总数 | 10,550 | 内容卡 6 → 10,550(约 1,758×) | -| `ops/ascendc` 领域卡 | 6,804 | — | -| `API` 类型卡 | 3,875 | 本仓 0 | -| 其中 Ascend C API 卡 | **2,993** | — | -| `runtime` API 卡 | 516 | 本仓 0 | -| `Operator` 卡 | 903 | 本仓 0 | -| `Runbook` 卡 | 2,362 | 本仓无对应类型 | - -**这是粒度差,不只是数量差。** InferRSI 现有 Ascend 知识是**技能粒度**:14 篇 `SKILL.md`,讲"怎么设计 tiling""怎么抓 ACLGraph""怎么诊断算子瓶颈"这类流程。上游是**API 与算子粒度**:单张卡讲清 `CeilDivision` 的参数、单位、平台约束、失效边界。两类内容不可互相替代——技能粒度无法回答"这个 API 在 a3 上是否支持、参数语义是什么",而现有仓恰好完全缺失后者。 - -**为什么既有路径补不上。** [#91](https://github.com/InferMatrixLab/InferRSI/issues/91) 的蒸馏路径是"人力策展 + 逐条 PR",本仓至今累计 6 条 Ascend 内容卡;按同一速度覆盖 2,993 张 API 卡在工程上不成立。而且这 6 条内容卡全部绑定具体模型 campaign(Qwen3-TTS 音码、MiniMax-H3 KV 复用、Qwen-Image 预热),描述的是"某模型在某卡上的某次优化",不是可复用的通用 Ascend 机制。 - -**匮乏不等于现有内容有错。** 那 14 篇 skill 多数经 campaign 验证,质量可用;问题纯粹是覆盖面——`aclnn` / AscendC API / CANN Runtime / 算子实现这四类基础层次在本仓几乎是空白,导致 Agent 做 Ascend 任务时只能依赖 campaign 记忆与现场搜索,无法像 CUDA 路径那样先查知识再动手。 - -**这也解释了为什么不能只靠"蒸馏":** 上游的知识生产侧(`knowledge-ingest`、两个 Operator producer、trace ingest)已经是一个持续产出系统,本仓若只做消费端的手工蒸馏,永远落后一个数量级的产能。本 RFC 讨论的正是"以什么形态消费这套外部产能",而不是"要不要再造一套"。 - -## 2. 上游形态与许可(实测) - -写作时对上游 `master` 的一次浅克隆观测(commit 观测值 `92dc260756e91e72e6dcd60f3996a4d54f43c8f5`,2026-11-16): - -| 项 | 值 | -| --- | --- | -| 知识正文根 | `knowledge/`,OKF v0.2 唯一 Bundle,Concept ID = Bundle 相对路径去 `.md` | -| 概念卡 / 导航 | 10,550 张 / 818 个 `index.md`(不参与检索) | -| 正文字节 | 约 118 MiB(不含导航);目录占用约 143 MB | -| 领域分布 | `ops` 7,440 · `model` 1,890 · `contrib` 653 · `runtime` 530 · `common` 21 · `tools` 16 · `graph` 0(保留结构入口) | -| 状态分布 | `stable` 6,588 · `draft` 3,698 · `deprecated` 41 | -| 类型分布 | `API` 3,875 · `Runbook` 2,362 · `Guide` 1,788 · `Optimization` 1,087 · `Operator` 903 · `Concept` 287 · `Example` 23 · `Glossary` 2 | -| 平台受控值 | `a2` 1,765 · `a3` 1,610 · `310p` 395 · `agnostic` 228 · `kirin_x90` 198 · `kirin_9030` 172 | -| 写作语言 | 标题、描述与正文说明以中文写作;API 名、签名与代码为英文 | - -**上游不是一个资料包,是一个知识系统:** - -- `governance/` 有机器可读的 schema、路径 Profile、受控值规则与 20 余个 check 实现(来源、链接、索引同步、生命周期、信任层级)。 -- 检索索引版本化(`cannbot-knowledge-index.v1`),带 `content_fingerprint` 与 `built_repository_state.commit`;索引相对当前提交过期时**拒绝服务**。 -- `mcp-server/` 提供只读 `discover` / `search` / `read_card`,stdio 与 HTTP 双传输。 -- 卡的来源资源**不在 Git 内**:`sources[].resource` 常指向 `cann-docs-raw/...` 或 `cann-recipes-raw///...`,由 `get_resources.sh` 恢复,默认不提交。**单张卡本身不自包含证据闭包。** - -**许可。** 上游 LICENSE 为 CANN Open Software License Agreement v2.0,不是 OSI 许可:授予 limited、royalty-free、**non-transferable、non-sublicensable、revocable** 的许可,用途限于华为 AI 处理器系统。本仓为 Apache-2.0。把该内容打包进 `src/inferrsi/knowledge/data/` 并随 sdist/wheel 分发,或写入 `evidence/`,都会形成不可撤销的分发承诺。(本条为 LICENSE 文本的读法,**不是法律意见**,需要组织确认。) - -**关于本仓内的 12 个 skill:** cannbot-knowledge 的 `.agents/skills/` 有 12 个 skill,但它们是知识仓自己的工具链——三个消费流程(`knowledge-query` 等,就是 MCP 背后的 CLI)、五个生产流程(`*-ingest`、operator producer)、两个治理(`knowledge-lint`、`knowledge-issue-handler`)、两个带 `VENDORED.md` 的裁剪副本。**CANN 领域知识全在 `knowledge/`,MCP 三工具已完整覆盖,不存在"skill 覆盖不到"的缺口。** - -## 3. 形态选择记录 - -五种候选的比较与结论(保留作为决策依据,不再并行生效): - -| 形态 | 许可可分发 | 离线复现 | 漂移检测 | 结论 | -| --- | --- | --- | --- | --- | -| A 外链锁 + 包外缓存 | ✅ 不进包 | ✅ digest | ✅ | 未选:仍需同步与自建索引,收益低于 MCP | -| A′ 索引先行 | ✅ | ✅ | ✅ | 未选:同 A | -| B git submodule(KDA 式) | ⚠️ 需仓库层处理许可 | ✅ | ⚠️ 仅指针 | 未选:把许可义务交给集成方,且 143 MB 随克隆 | -| **C 全量转换进本仓** | ❌ 冲突 | ✅ | ❌ 副本即漂移 | **判死**:许可阻断 + 15 倍体积 + 每次上游前进需全量重转 | -| **D 只读 MCP** | ✅ 不落地 | ⚠️ 依赖上游 checkout | ✅ 由上游索引保证 | **选定**:唯一同时满足许可与规模,且上游已自带治理与索引 | -| E 混合 | — | — | — | 收敛为 D;C 仅以 [#91](https://github.com/InferMatrixLab/InferRSI/issues/91) 蒸馏精选子集的形式保留 | - -KDA 的可借之处仍是纪律:README 锁表、`third_party_licenses/` 逐件许可、以及"submodule 指向做过去敏的发布点而非上游任意 commit"。本方案把这三条落在 §7 的 receipt 与 §5.3 的 `revision` 字段上,而不引入 submodule 这一形式。 - -## 4. 本仓检索侧事实 - -| 项 | 值 | -| --- | --- | -| `find` 实现 | `src/inferrsi/knowledge/knowledge_find.py`,commit `61303fa8`(#466 统一检索) | -| 检索模型 | 纯词法:`idf × 字段权重 × sqrt(出现次数)`,`FIELD_WEIGHTS` 见 `knowledge_find.py:42` | -| 来源 | `kinds=("case","skill","record")`,`load_corpus` 见 `:620` | -| 语料规模 | 143 篇 `SKILL.md` + 128 条记录 + cases,同进程内构建 | -| 确定性 | 完全离线、可复现、无 I/O 依赖 | -| 中文处理 | `tokens()` 已内建 CJK 双字切分(`:87`);`_BACKEND_NAME_ALIASES` 含 `昇腾 → ascend` | -| 权威源枚举 | `AUTHORITIES = ("package", "checkout", "store")`(`knowledge_operations.py:25`) | - -上游 MCP 侧实测:索引构建 73.9 秒(7,921,396 postings);查询**冷启 3.67 秒 / 热 1.46 秒**;结果字段 `rank` / `score` / `match_basis` / `path` / `local_path` / `card_lines` / `card_sha256`;响应 schema `knowledge-search.v5`。 - -## 5. 接入设计 - -### 5.1 通道抽象 - -新增 `src/inferrsi/knowledge/channels.py`,把外部源收敛到同一协议,使 `find` 不认识具体是 cannbot 还是 MindIE: - -```python -class Channel(Protocol): - name: str # "cannbot-knowledge" | "mindie-agent" - backend: str | None # "ascend" | None(None = 平台无关) - def describe(self) -> ChannelStatus: ... # 探活,可缓存 - def search(self, query: str, *, platform: str | None, - limit: int) -> ChannelResult: ... # 返回定位符,不返回正文 - def read(self, locator: str) -> str: ... # 单独调用,不在 find 热路径 -``` - -```python -@dataclass -class ChannelStatus: - state: str # ok | not_enabled | unreachable | index_stale | timeout - # | version_mismatch | rejected | unknown | backend_unknown - revision: str | None # 上游 commit / catalog generation - detail: str = "" - fix: str = "" # 修复命令,例如索引重建 - checked_at: float = 0.0 - -@dataclass -class ChannelResult: - status: ChannelStatus - items: list[ChannelHit] # 失败时为空,但 state 必须说明原因 - elapsed_ms: int -``` - -`ChannelHit` 字段固定:`locator` / `title` / `summary` / `score` / `score_scale` / `status` / `platforms` / `card_sha256`。 - -`score_scale` 是关键:标明分数来自哪套评分(`"cannbot/knowledge-search.v5"` 或 `"mindie/graded-0.13.13"`),从结构上阻止调用方跨通道比较。 - -### 5.2 平台门控(决定"要不要查") - -```text -find(query, backend=?) - │ - ├─ backend 未声明 ──► 不查外部;external[] 每项 state="backend_unknown" - │ 且 fix 给出 "inferrsi find --backend ascend" - │ - ├─ backend == ascend ──► 查询所有 backend ∈ {ascend, None} 的已授权通道 - │ (cannbot-knowledge + mindie-agent) - │ - └─ backend == cuda/mlx/... ──► 不查外部;external[] 为空 -``` - -`backend` 来源优先级:`--backend` 显式参数 > grant 文件声明 > `backend_unknown`(**不猜**)。 - -`--backend` **只接受单值**。多平台任务由调用方分两次查询并各自保留结果。理由:多值会让输出无法回答"这条外部结果是为哪个平台召回的",而上游平台纪律本身 fail-closed(`--platform` 必填、不混合各平台结果),本仓不应在门控层放宽这一语义。 - -这里刻意**不使用** `named_backends()`(`knowledge_find.py:739`)那种从查询词推断的方式:从"query 里出现了 asc"推断"这是 Ascend 任务"不可靠,且失败形态是静默查错平台。门控必须是显式事实。 - -### 5.3 分栏呈现(决定"怎么放") - -**不做混合排序。** 本仓 `idf` 建立在 143+128 条语料上,上游建立在 10,550 张卡上;同一个词在两套体系里的权重完全不同。任何跨通道合并都要发明归一化函数——那正是 RSI 明令禁止的"编造分数"。 - -```json -{ - "query": "AscendC tiling 参数", - "backend": "ascend", - "items": [ /* 只放本仓结果,沿用现有 ranked list */ ], - "matched": 3, - "external": [ - { - "channel": "cannbot-knowledge", - "state": "ok", - "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", - "elapsed_ms": 1520, - "score_scale": "cannbot/knowledge-search.v5", - "items": [{ - "locator": "ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling", - "title": "Matmul Tiling类使用说明", - "score": 18.4, - "status": "stable", - "platforms": ["a2", "a3"], - "card_sha256": "049b4824...", - "open": "inferrsi cannbot read ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling" - }] - }, - { - "channel": "mindie-agent", - "state": "not_enabled", - "detail": "no project grant for this checkout", - "fix": "inferrsi mindie enable --domain " - } - ], - "notes": [ - "External results are not locally verified; scores are not comparable across channels.", - "Pin external revisions from this output when citing them." - ] -} -``` - -三条不可退让的规则: - -1. **`items` 只放本仓结果。** 外部命中不进 `items`,避免下游按 `items` 处理时静默把未验证内容当成已验证。 -2. **每个通道独立 `state`。** 一个通道失败不影响另一个;失败必须可见并附 `fix`。 -3. **不合并 `score`。** 每个通道内部排序,跨通道并列。需要合并时由调用方决定。 - -### 5.4 执行模型:并行、预算、缓存 - -```text -find() - ├─ 本地 Corpus 打分 (毫秒级,主线程) - └─ 并行启动 N 个外部通道 - ├─ cannbot-knowledge (子进程 / MCP stdio) - └─ mindie-agent (子进程,现有 call()) -``` - -用 `ThreadPoolExecutor`,通道间无共享状态;总耗时 ≈ max(各通道) 而非 sum。 - -| 参数 | 默认 | 说明 | -| --- | --- | --- | -| `--external-timeout` | 8 秒 | 单通道上限 | -| `--no-external` | 关闭 | 只用本仓结果(CI 与离线场景) | -| `--external-limit` | 通道默认 | 覆盖单通道返回条数 | - -| 通道 | 默认条数 | 理由 | -| --- | --- | --- | -| `cannbot-knowledge` | **10** | 定位符精度高(`path` 即 Concept ID),平台过滤已生效,多给几条有助于一次选中正确粒度 | -| `mindie-agent` | **5** | 返回 `navigation_only` 的 ref 定位符,粒度更粗;过多会淹没本地结果 | - -**`find` 默认即带外部查询**(不带 `--no-external`),Ascend 任务开箱即双查。CI 与单测必须显式传 `--no-external`,以隔离外部通道可用性对基线的影响。 - -超时必须映射成 `state:"timeout"` 并附 `elapsed_ms`,**绝不能表现为空结果**。"查了但超时"与"查了没命中"是两种事实,混同会让 Agent 误判为"上游没有这个知识"。 - -| 缓存对象 | TTL | 理由 | -| --- | --- | --- | -| `describe()` / `discover` | 300 秒 | 探活成本高(冷启 3.7s),revision 不会秒级变化 | -| 通道授权状态与 checkout identity | 进程生命周期 | 文件系统事实 | -| search 结果 | 不缓存 | query 空间无限,缓存会让"上游更新"不可见 | - -缓存只在内存,不落盘。 - -### 5.5 cannbot-knowledge 接法 - -新增 `src/inferrsi/apps/cannbot/`,结构镜像 `apps/mindie/`: - -| 文件 | 职责 | -| --- | --- | -| `project.py` | 逐 checkout 授权;`~/.config/inferrsi/cannbot-projects/.json` | -| `context.py` | 解析上游 JSON;2 MiB 上限、重复键拒绝、非有限数拒绝(复用 MindIE 的 `_object` / `_invalid_constant`) | -| `worker.py` | 用 grant 里的 python 起子进程,调上游 `knowledge_query.py search/read` 或 MCP | -| `mcp.py` | stdio MCP 代理三工具 | -| `receipt.py` | 落 receipt(§7) | - -授权文件: - -```json -{ - "schema": "inferrsi-cannbot-project/1", - "checkout": "", - "checkout_identity": {"device": 0, "inode": 0, "created": 0}, - "python": "/bin/python", - "backend": "ascend", - "default_platforms": ["a2"], - "enabled_at": "" -} -``` - -`checkout_identity` 抄 MindIE 的做法(`.git` 的 device + inode + birthtime),防止 grant 被复制到另一个 clone 误用。`default_platforms` 让常用平台不必每次传;未声明时 `search` 直接报错(与上游 fail-closed 一致)。 - -### 5.6 MindIE Agent 接法 - -不重写,加一层 adapter 让现有 `call()` 满足 `Channel` 协议: - -| 现状 | 改造 | -| --- | --- | -| `call("query", args)` 返回 `repository.query(navigation_only=True)` | adapter 映射为 `ChannelResult` | -| ref 形如 `mindie:///units/@` | 作为 `locator` | -| grant 无 backend | 增可选 `backend` 字段 | -| `ReadFailure` 区分失败 | 映射到 `not_enabled` / `version_mismatch` / `rejected` | - -`mindie-agent` 的 `describe` 已有:`repository.available()` + `component_version`(版本钉 `0.13.13`,`read_worker.py:24`)。 - -## 6. 为什么不采用其余形态(要点) - -- **C 全量转换**:许可阻断分发;7.8 MiB → 约 126 MiB;上游每次前进需全量重转;与 #388 的单一权威来源规则冲突;10,550 条无法诚实标 `locally_verified=true`。**判死。** -- **B submodule**:把 CANN OSL 的许可义务交给每个集成方;143 MB 随克隆;`gitcode` 可达性成为所有开发者的硬依赖。KDA 能这么做是因为它不 pip 分发且 submodule 是 MIT/过过过洗的发布点。 -- **A/A′ 外链锁缓存**:机制正确但与 MCP 重叠——两者都在解决"如何引用外部内容",而 MCP 让上游承担索引与治理。保留为未来若需要"完全离线复现"时的备选,当前不实现。 - -## 7. 证据与许可 - -### 7.1 receipt 只存定位符 - -```json -{ - "schema": "inferrsi-external-receipt/1", - "channel": "cannbot-knowledge", - "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", - "index": {"schema": "cannbot-knowledge-index.v1", "version": 3, - "content_fingerprint": "0047983f..."}, - "query": "CeilDivision 向上取整", - "platforms": ["a2"], - "decision": "确认 a3 上的取整语义", - "hits": [{ - "locator": "ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division", - "card_sha256": "b97cbe82492d25cc0363d1852eddcf87d991c9d75a57891a9aed3e75188ffc86", - "status": "stable", "platforms": ["a2", "a3"] - }], - "delivered": "locators_only", - "locally_verified": false -} -``` - -**不含卡片正文、不含 frontmatter 全文、不含 `local_path`。** 前两者会把 CANN OSL 内容带进 `evidence/`(把许可问题从 `data/` 原样搬到 `evidence/`);后者是个人绝对路径,违反隐私规则。 - -`card_sha256` 保证"我引用的确实是这一版这张卡",且摘要不构成再分发。 - -### 7.2 引用话术与验证状态 - -引用外部结论必须写成 `cannbot-knowledge @ ` 或 `mindie-knowledge 0.13.13 / @`,并显式标注 `locally_verified=false`。外部命中**不参与** `PASS` 判定(沿用 `AGENTS.md` 的证据纪律)。 - -## 8. 代码改动清单 - -| # | 文件 | 改动 | 风险 | -| --- | --- | --- | --- | -| 1 | `knowledge/channels.py` | 新增 `Channel` / `ChannelStatus` / `ChannelHit` / `ChannelResult` / `ChannelRegistry` | 无 | -| 2 | `knowledge/knowledge_find.py:732` `find()` | 增 `channels` / `backend` / `external_limit` / `external_timeout`;并行调度;输出 `external[]` | 需保证 `channels=None` 时行为**逐字节一致** | -| 3 | `knowledge/knowledge_find.py:620` `load_corpus` | 不改 | — | -| 4 | `knowledge/knowledge_operations.py:25` `AUTHORITIES` | **不改**(外部内容不作为 bundle 读取) | — | -| 5 | `apps/cannbot/*` | 新增五个文件 | 新代码 | -| 6 | `apps/mindie/` | 增 `channel.py` adapter;grant 增 `backend` | 小 | -| 7 | `apps/cli/arguments.py` | `find` 增 `--backend` / `--no-external` / `--external-limit` / `--external-timeout`;新增 `cannbot` 子命令组 | 小 | -| 8 | `apps/cli/__init__.py` | 分发 `cannbot`;`campaign import-cannbot` / `import-mindie` 统一 | 中 | -| 9 | `tests/` | fake channel(本地 stdio 假进程 + 固定 JSON),覆盖全部 `state` | 必需 | - -**第 2 条的不变量:** `channels` 为空或 `--no-external` 时,`find` 的输出必须与 #466 之后的行为完全一致。必须写成测试,否则外部通道会悄悄改变现有 143 篇 skill 的排序基线。 - -**第 4 条的理由:** 不把 `external` 加进 `AUTHORITIES`,意味着 `find` 的 `read_args` / `open_argv` 不会指向外部卡(避免生成一个执行即错的命令),外部读走独立的 `cannbot read`。 - -## 9. 使用方式 - -### 9.1 首次配置(每台机器、每个 checkout 一次) - -```bash -inferrsi cannbot enable \ - --checkout ~/src/cannbot-knowledge \ - --python ~/venvs/cannbot/bin/python \ - --backend ascend \ - --platform a2 - -inferrsi mindie enable --domain --backend ascend -``` - -前置条件由用户负责:上游要求 Python 3.10+、`uv`、**完整 checkout**、**索引已构建**。 - -自查与自修复: - -```bash -inferrsi cannbot describe -# {"state":"ok","revision":"92dc2607...","cards":10550, -# "index":{"version":3,"content_fingerprint":"0047983f..."}} -``` - -```text -state: index_stale -detail: index built at differs from checkout -fix: cd ~/src/cannbot-knowledge && python3 .agents/skills/knowledge-query/scripts/knowledge_index.py build -``` - -### 9.2 日常检索(Ascend 任务,常驻双查) - -```bash -$ inferrsi find "AscendC tiling 参数怎么算" --backend ascend -``` - -```text -本仓 (backend=ascend) - 1. ascendc-tiling-design score 42.1 - 2. ascend-performance-analysis score 31.7 - -外部通道 - cannbot-knowledge @ 92dc2607 ok 1.5s 3 hits - 1. Matmul Tiling类使用说明 18.4 ops/ascendc/apis/.../cube_compute/matmul_tiling - 2. 使用高阶API时配套的Tiling实现 17.9 ops/ascendc/guides/.../tiling - 3. Matmul tiling 形状与核数推导链 17.2 ops/ascendc/guides/.../derivation_chain - → inferrsi cannbot read - - mindie-agent not_enabled (fix: inferrsi mindie enable --domain ) -``` - -本仓有命中时**照样**查外部;两个通道各自一段;分数不跨栏比较。 - -### 9.3 非 Ascend 任务 / 未声明 backend - -```bash -$ inferrsi find "CUDA graph capture" --backend cuda -# 只查本仓;external: [] -``` - -```text -external: [{"channel":"cannbot-knowledge","state":"backend_unknown", - "fix":"inferrsi find --backend ascend"}] -``` - -### 9.4 读外部卡正文 - -```bash -inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division -inferrsi cannbot read --page 2 # 分页续读,默认 200 行 -inferrsi mindie read -``` - -正文只在终端呈现,不落盘、不进 receipt。 - -### 9.5 Agent 挂载(Claude Code / Codex) - -```bash -inferrsi cannbot serve # stdio MCP:discover / search / read_card -``` - -**透传,不收窄。** 三工具的名称、参数与语义与上游一致,只多一层授权检查、receipt 与错误归一化。理由:自造收窄契约会在上游演进(新增过滤参数、`search` schema 从 v5 升级)时变成必须同步维护的负债,而透传可依靠 `schema_version` 直接暴露差异。授权检查、receipt、错误归一化都在这一层完成。 - -### 9.6 进 campaign evidence - -```bash -inferrsi cannbot search "CeilDivision 向上取整" --platform a2 \ - --decision "确认 a3 上的取整语义" \ - --receipt .inferrsi/receipts/ceil.json - -inferrsi campaign import-cannbot .inferrsi/receipts/ceil.json -``` - -## 10. 切片 - -| 片 | 内容 | 依赖 | 可独立合入 | -| --- | --- | --- | --- | -| C0 | `channels.py` 协议 + fake channel + 全部 state 的测试 | — | ✅ | -| C1 | `find()` 融合 + `--backend` 门控(只接 fake channel) | C0 | ✅ | -| C2 | `apps/cannbot/` 的 `enable/disable/describe` | C0 | ✅ | -| C3 | cannbot `search` / `read`(worker + 子进程) | C2 | ✅ | -| C4 | MindIE channel adapter + grant `backend` | C0 | ✅ | -| C5 | `cannbot serve`(stdio MCP 代理) | C3 | ✅ | -| C6 | `receipt.py` + `campaign import-cannbot` 统一 | C3、C4 | ✅ | - -C1 先行的理由:它把"平台门控 + 分栏 + 超时语义"这套契约钉死,之后两个真源接入都只是实现 `Channel`,不会反过来改 `find` 的输出结构。 - -**C4 建议紧跟 C0**,因为 MindIE 已有 `call()` 与 `describe()`,adapter 成本低,能最早验证协议抽象是否成立。 - -## 11. 风险 - -| # | 风险 | 现状 | 缓解 | -| --- | --- | --- | --- | -| R1 | 热路径增加 1.5–3.7s | 实测 | 并行 + `describe` 缓存 300s + `--no-external` | -| R2 | 破坏 `find` 的确定性与离线可测 | 真实风险 | 不传 `--backend` 时不查外部;`--no-external` 关闭全部通道;fake channel 覆盖全部 state;两种路径输出逐字节固定 | -| R3 | 上游索引过期导致"昨天能查今天不能查" | 上游 fail-closed | `state: index_stale` + `fix`;不做静默重试 | -| R4 | 平台枚举不可映射(`a2/a3/950/310p` vs `ascend`) | 未决 | grant 声明 `backend` + `platform` 两级;未知值拒绝,不猜 | -| R5 | 许可内容意外落盘 | 已设计 | receipt 只存 locator + sha256;剔除 `local_path` | -| R6 | 两套评分被下游误混 | 已设计 | `score_scale` 字段 + 分栏结构 + `notes` | -| R7 | `uv` / 完整 checkout 在 CI 不可用 | 未验证 | 单测只用 fake channel;真上游走人工验收 | -| R8 | MindIE 0.13.13 版本钉导致升级即断 | 现有行为 | `state: version_mismatch` + `fix` | - -## 12. 关闭条件 - -本文作为 RFC 关闭时应当满足: - -- [ ] 评审接受 §5 的通道契约(`Channel` 协议、`state` 枚举、`external[]` 结构、`score_scale`)。 -- [ ] 评审接受 §5.2 的平台门控语义:显式声明、单值、不推断。 -- [ ] 评审接受 §7 的 receipt 形状与"不含正文"的许可边界;Q1(CANN OSL 立场)有组织层面的答复或明确接受为"实施前再定"。 -- [ ] 评审接受 §5.4 的默认值:`find` 默认查外部、每通道 10/5、超时 8 秒。 -- [ ] 确认 §8 第 2 条与第 4 条不变量写入测试。 -- [ ] 明确与 [#388](https://github.com/InferMatrixLab/InferRSI/issues/388) / [#387](https://github.com/InferMatrixLab/InferRSI/issues/387) 的关系:本文不关闭 #388,`external[]` 需在 #387 的 schema 中登记。 -- [ ] 实施切片(C0–C6)另开 issue,本 Issue 不承接代码合入。 - -## 13. 相关 - -- [#388](https://github.com/InferMatrixLab/InferRSI/issues/388) 外部技能内容的依赖、版本锁定与可复现导入(治理 RFC,仍 Open) -- [#387](https://github.com/InferMatrixLab/InferRSI/issues/387) K/H/B 与 Apps 公开接口详细设计(`read` / `collect`,仍 Proposed) -- [#362](https://github.com/InferMatrixLab/InferRSI/issues/362) 三模块目标架构(已接受方向) -- [#91](https://github.com/InferMatrixLab/InferRSI/issues/91) SearchSpace 蒸馏覆盖跟踪(本仓内唯一权威来源路径) -- [#43](https://github.com/InferMatrixLab/InferRSI/issues/43) 历史 `license: UNKNOWN` 与许可链遗留约束 -- [#466](https://github.com/InferMatrixLab/InferRSI/pull/466) 统一 `find` 检索(本文接入的宿主接口) -- [#114](https://github.com/InferMatrixLab/InferRSI/pull/114) cann/cannbot 固定阶段证据蒸馏入库(既有蒸馏先例,不是外部依赖机制) - -## 14. 参考实现(外部) - -- [cann/cannbot-knowledge](https://gitcode.com/cann/cannbot-knowledge):OKF v0.2 + CANNBot Profile,自带 governance / Query / Lint / MCP。 -- [NVlabs/kda](https://github.com/NVlabs/kda):submodule 钉版本 + README 锁表 + `third_party_licenses/` 逐件许可 + 明确"submodule 不受本仓一方许可覆盖"。**可借的是纪律,不是形式。** diff --git a/docs/plans/2026-11-17-external-knowledge-channels.md b/docs/plans/2026-11-17-external-knowledge-channels.md deleted file mode 100644 index 2e5d2ef7..00000000 --- a/docs/plans/2026-11-17-external-knowledge-channels.md +++ /dev/null @@ -1,457 +0,0 @@ -# 外部知识通道设计:cannbot-knowledge 与 MindIE Agent 的常驻查询接入 - -> Status: Proposed(设计草案)。本文只定义接入设计与使用方式,不构成实施授权。 -> 上位依赖:[#476](https://github.com/InferMatrixLab/InferRSI/issues/476)(候选形态与许可边界)、[#388](https://github.com/InferMatrixLab/InferRSI/issues/388)(外部依赖治理)、[#387](https://github.com/InferMatrixLab/InferRSI/issues/387)(Knowledge 公开接口)。 -> 文中所有上游 revision 与统计值都是写作时的实测观测值,不是本仓 pin。 - -## 0. 相对前一版的关键变更 - -前一版设计把外部通道当作**补漏**:本地 `find` 零命中时才提示外部存在。本文改为**常驻并行查询**: - -| 维度 | 前一版(补漏) | 本版(常驻) | -| --- | --- | --- | -| 触发条件 | `no_match == true` | 平台门控命中即查,无论本地是否命中 | -| 语义 | 外部源是兜底 | 外部源是**第二知识源**,与本地并列 | -| 输出 | `external_channels` 提示条目 | `external[]` 实际检索结果 | -| 平台门控 | 无 | 必须显式声明;`ascend` 任务常驻查询,其他平台不触发 | - -理由:Ascend 任务的真实知识分布是"本仓有 14 篇流程型 skill,上游有 10,550 张 API/算子卡"。只在零命中时查,等于假设"本地一旦有命中,外部就没有增量"——这个假设在粒度互补的场景下明显不成立(本仓命中"如何设计 tiling 流程",上游同时有"`Matmul Tiling` 类的具体参数")。 - -## 1. 现状与约束(实测) - -### 1.1 本仓知识侧 - -| 项 | 值 | -| --- | --- | -| `find` 实现 | `src/inferrsi/knowledge/knowledge_find.py`,commit `61303fa8`(#466 统一检索) | -| 检索模型 | 纯词法:`idf × 字段权重 × sqrt(出现次数)`,`FIELD_WEIGHTS` 见 `knowledge_find.py:42` | -| 来源 | `kinds=("case","skill","record")`,`load_corpus` 见 `:620` | -| 语料规模 | 143 篇 `SKILL.md` + 128 条记录 + cases,同进程内构建 | -| 确定性 | 完全离线、可复现、无 I/O 依赖 | -| Ascend 内容卡 | 6 张(详见 #476「现状」节) | - -### 1.2 cannbot-knowledge MCP 侧 - -| 项 | 实测值 | -| --- | --- | -| 索引构建 | 73.9 秒(10,550 卡,7,921,396 postings) | -| 查询延迟 | 冷启 3.67 秒 / 热 1.46 秒 | -| 检索结果字段 | `rank` / `score` / `match_basis` / `path` / `local_path` / `card_lines` / `card_sha256` | -| 响应 schema | `knowledge-search.v5`,含 `request_sha256` 与 `index.repository_state.commit` | -| 平台纪律 | `--platform` 必填,缺失即报错(fail-closed,不混合各平台) | -| 索引过期 | 拒绝服务并给出 `index_stale`,不自动重建 | -| 工具面 | `discover` / `search` / `read_card`,全部 `read_only` | -| 许可 | CANN OSL v2.0(内容不得进包、进 evidence) | - -### 1.3 MindIE Agent 侧 - -| 项 | 现有实现 | -| --- | --- | -| 适配代码 | `src/inferrsi/apps/mindie/`(639 行) | -| 工具体 | `knowledge_query` / `knowledge_explain` | -| 运行时 | `subprocess.run([grant["python"], "-I", "-B", worker, ...])`,`project.py:103` | -| 版本钉 | `mindie-knowledge == 0.13.13` 硬校验,`read_worker.py:24` | -| 授权 | 逐 checkout,`~/.config/inferrsi/mindie-projects/.json` | -| 检索 | `repository.query(navigation_only=True)`,返回 ref 定位符 | - -### 1.4 由此产生的设计约束 - -1. **两套评分不可比。** 本仓的 `idf` 建立在 143+128 条语料上,上游建立在 10,550 张卡上;同一个词在两边的 idf 与权重体系完全不同。任何"混合排序"都需要我们自己发明一个归一化函数——那是本轮 RSI 明令禁止的"编造分数"。 -2. **外部检索是 I/O,不是计算。** 1.5–3.7 秒相对本仓 `find` 的毫秒级是数量级差异,且失败模式有七种(未授权 / 不可达 / 索引过期 / 平台未知 / 超时 / 协议错 / 上游拒绝)。 -3. **外部结果不确定。** 上游 checkout 更新后同一 query 结果会变,因此**每次输出必须携带 revision**,否则 `find` 的结果无法被任何人复现。 -4. **许可边界不因常驻而放松。** 正文永不落盘;进 receipt 的只有 `path` + `card_sha256` + `revision`。 - -## 2. 核心设计:通道抽象 + 平台门控 + 分栏呈现 - -### 2.1 通道抽象 - -新增 `src/inferrsi/knowledge/channels.py`,把两个外部源收敛到同一协议,使 `find` 不认识具体是 cannbot 还是 MindIE: - -```python -class Channel(Protocol): - name: str # "cannbot-knowledge" | "mindie-agent" - backend: str | None # "ascend" | None(None = 平台无关) - def describe(self) -> ChannelStatus: ... # 探活,可缓存 - def search(self, query: str, *, platform: str | None, - limit: int) -> ChannelResult: ... # 返回定位符,不返回正文 - def read(self, locator: str) -> str: ... # 单独调用,不在 find 热路径 -``` - -```python -@dataclass -class ChannelStatus: - state: str # ok | not_enabled | unreachable | index_stale - # | version_mismatch | rejected | unknown - revision: str | None # 上游 commit / catalog generation - detail: str = "" - fix: str = "" # 修复命令,例如索引重建 - checked_at: float = 0.0 - -@dataclass -class ChannelResult: - status: ChannelStatus - items: list[ChannelHit] # 失败时为空,但 state 必须说明原因 - elapsed_ms: int -``` - -`ChannelHit` 字段固定:`locator` / `title` / `summary` / `score` / `score_scale` / `status` / `platforms` / `card_sha256`。 - -`score_scale` 是关键:标明这个分数来自哪套评分(`"cannbot/knowledge-search.v5"` 或 `"mindie/graded-0.13.13"`),从结构上阻止调用方跨通道比较。 - -### 2.2 平台门控(决定"要不要查") - -数据流: - -```text -find(query, backend=?) - │ - ├─ backend 未声明 ──► 不查外部;external[] 每项 state="backend_unknown" - │ 且 fix 给出 "inferrsi find --backend ascend" - │ - ├─ backend == ascend ──► 查询所有 backend ∈ {ascend, None} 的已授权通道 - │ (cannbot-knowledge + mindie-agent) - │ - └─ backend == cuda/mlx/... ──► 不查外部;external[] 为空 -``` - -`backend` 来源优先级: - -1. `inferrsi find --backend ascend`(显式参数,最高) -2. grant 文件里声明的 backend(`cannbot enable --backend ascend`) -3. 都没有 → `backend_unknown`,**不猜** - -`--backend` **只接受单值**。一次调用只能声明一个 backend;同时涉及 Ascend 与 CPU host 侧等多平台任务,由调用方分两次查询并各自保留结果。理由:多值会让输出无法回答"这条外部结果是为哪个平台召回的",而外部源的平台纪律本身是 fail-closed 的(上游 `--platform` 要求明确目标平台,不混合各平台结果),本仓不应在门控层放宽这一语义。 - -这里刻意不使用 `named_backends()`(`knowledge_find.py:739`)那种从查询词推断的方式:从"这个 query 里出现了 asc"推断"这个任务是 Ascend 任务"是不可靠的,且失败形态是静默查错平台。门控必须是显式事实。 - -### 2.3 分栏呈现(决定"怎么放") - -**不做混合排序。** 输出结构: - -```json -{ - "query": "AscendC tiling 参数", - "backend": "ascend", - "items": [ /* 本仓结果,沿用现有 ranked list */ ], - "matched": 3, - "external": [ - { - "channel": "cannbot-knowledge", - "state": "ok", - "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", - "elapsed_ms": 1520, - "score_scale": "cannbot/knowledge-search.v5", - "items": [ - { - "locator": "ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling", - "title": "Matmul Tiling类使用说明", - "score": 18.4, - "status": "stable", - "platforms": ["a2", "a3"], - "card_sha256": "049b4824...", - "open": "inferrsi cannbot read ops/ascendc/apis/simd_api/adv_api/cube_compute/matmul_tiling" - } - ] - }, - { - "channel": "mindie-agent", - "state": "not_enabled", - "detail": "no project grant for this checkout", - "fix": "inferrsi mindie enable --domain " - } - ], - "notes": [ - "External results are not locally verified; scores are not comparable across channels.", - "Pin external revisions from this output when citing them." - ] -} -``` - -三条不可退让的规则: - -1. **`items` 只放本仓结果。** 外部命中不进 `items`,避免任何下游按 `items` 处理时会静默把未验证内容当成已验证。 -2. **每个通道独立 `state`。** 一个通道失败不影响另一个;失败必须可见并附 `fix`。 -3. **不合并 `score`。** 每个通道内部排序,跨通道并列。需要合并时由调用方决定,不由 `find` 代劳。 - -## 3. 执行模型:并行、预算、降级 - -### 3.1 并行 - -```text -find() - ├─ 本地 Corpus 打分 (毫秒级,主线程) - └─ 并行启动 N 个外部通道 - ├─ cannbot-knowledge (子进程 / MCP stdio) - └─ mindie-agent (子进程,现有 call()) -``` - -用 `concurrent.futures.ThreadPoolExecutor`,通道之间无共享状态。总耗时 ≈ max(各通道),而不是 sum。 - -### 3.2 预算与降级 - -| 参数 | 默认 | 说明 | -| --- | --- | --- | -| `--external-timeout` | 8 秒 | 单通道上限 | -| `--no-external` | — | 只用本仓结果(CI 与离线场景) | -| `--external-limit` | 通道默认(见下) | 覆盖单通道返回条数 | - -**每通道条数默认值:** - -| 通道 | 默认条数 | 理由 | -| --- | --- | --- | -| `cannbot-knowledge` | **10** | 10,550 张卡、2,993 张 Ascend C API 卡,定位符精度高(`path` 即 Concept ID),多给几条能让调用方一次选中正确粒度 | -| `mindie-agent` | **5** | 返回的是 `navigation_only` 的 ref 定位符,粒度更粗;过多会淹没本地结果 | - -`find` **默认即带外部查询**(不带 `--no-external`),因此 Ascend 任务开箱就是双查。CI 与单测必须显式传 `--no-external`,以保证基线不受外部通道可用性影响。 - -超时必须映射成 `state: "timeout"` 并附 `elapsed_ms`,**绝不能表现为空结果**。"查了但超时"与"查了没命中"是两种事实,混同会让 Agent 误判为"上游没有这个知识"。 - -### 3.3 缓存 - -| 对象 | TTL | 理由 | -| --- | --- | --- | -| `describe()` / `discover` | 300 秒 | 探活成本高(冷启 3.7s),且 revision 不会秒级变化 | -| 通道授权状态与 checkout identity | 进程生命周期 | 文件系统事实 | -| search 结果 | 不缓存 | query 空间无限,且缓存会让"上游更新"不可见 | - -缓存落在内存,不落盘;落盘缓存会引入"缓存里的 revision 与当前不一致"的经典问题。 - -## 4. 两个通道的具体接法 - -### 4.1 cannbot-knowledge - -新增 `src/inferrsi/apps/cannbot/`,结构镜像 `apps/mindie/`: - -| 文件 | 职责 | -| --- | --- | -| `project.py` | 逐 checkout 授权;`~/.config/inferrsi/cannbot-projects/.json` | -| `context.py` | 解析上游 JSON;2 MiB 上限、重复键拒绝、非有限数拒绝(复用 MindIE 的 `_object` / `_invalid_constant`) | -| `worker.py` | 用 grant 里的 python 起子进程,调 `knowledge_query.py search/read` 或 MCP `search`/`read_card` | -| `mcp.py` | stdio MCP 代理三工具,供 agent 直接挂载 | -| `receipt.py` | 落 receipt(§5) | - -授权文件: - -```json -{ - "schema": "inferrsi-cannbot-project/1", - "checkout": "", - "checkout_identity": {"device": 0, "inode": 0, "created": 0}, - "python": "/bin/python", - "backend": "ascend", - "default_platforms": ["a2"], - "enabled_at": "" -} -``` - -`default_platforms` 让常用平台不必每次传;未声明时 `search` 直接报错(与上游 fail-closed 一致)。 - -### 4.2 MindIE Agent - -不重写,增加一层 adapter 让现有的 `call()` 满足 `Channel` 协议: - -| 现状 | 改造 | -| --- | --- | -| `call("query", args)` 返回 `repository.query(navigation_only=True)` | adapter 中映射为 `ChannelResult` | -| ref 形如 `mindie:///units/@` | 作为 `locator` | -| backend 未声明 | grant 增可选的 `backend` 字段,默认 `ascend` | -| `state` 由 `ReadFailure` 区分 | 映射到 `not_enabled` / `version_mismatch` / `rejected` | - -`mindie-agent` 的 `describe` 已有:`repository.available()` + `component_version`。 - -## 5. 证据与许可 - -### 5.1 receipt 只存定位符 - -```json -{ - "schema": "inferrsi-external-receipt/1", - "channel": "cannbot-knowledge", - "revision": "92dc260756e91e72e6dcd60f3996a4d54f43c8f5", - "index": {"schema": "cannbot-knowledge-index.v1", "version": 3, - "content_fingerprint": "0047983f..."}, - "query": "CeilDivision 向上取整", - "platforms": ["a2"], - "decision": "确认 a3 上的取整语义", - "hits": [{ - "locator": "ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division", - "card_sha256": "b97cbe82492d25cc0363d1852eddcf87d991c9d75a57891a9aed3e75188ffc86", - "status": "stable", "platforms": ["a2", "a3"] - }], - "delivered": "locators_only", - "locally_verified": false -} -``` - -**不含卡片正文、不含 frontmatter 全文、不含 local_path。** 原因:前两者会把 CANN OSL 内容带进 `evidence/`,把许可问题从 `data/` 原样搬到 `evidence/`;`local_path` 是个人绝对路径,违反隐私规则。 - -`card_sha256` 的作用是"我引用的确实是这一版这张卡",且摘要不构成再分发。 - -### 5.2 引用话术 - -引用外部结论时必须写成:`cannbot-knowledge @ ` 或 `mindie-knowledge 0.13.13 / @`,并显式标注 `locally_verified=false`。外部命中**不参与** `PASS` 判定(沿用 `AGENTS.md` 的证据纪律)。 - -## 6. 代码改动清单 - -| # | 文件 | 改动 | 风险 | -| --- | --- | --- | --- | -| 1 | `knowledge/channels.py` | 新增 `Channel` / `ChannelStatus` / `ChannelHit` / `ChannelResult` / `ChannelRegistry` | 无 | -| 2 | `knowledge/knowledge_find.py:732` `find()` | 增 `channels`、`backend`、`external_limit`、`external_timeout` 参数;并行调度;输出 `external[]` | 需保证 `channels=None` 时行为与现在**逐字节一致** | -| 3 | `knowledge/knowledge_find.py:620` `load_corpus` | 不改 | — | -| 4 | `knowledge/knowledge_operations.py:25` `AUTHORITIES` | **不改**(外部内容不作为 bundle 读取) | — | -| 5 | `apps/cannbot/*` | 新增 five files | 新代码 | -| 6 | `apps/mindie/` | 增 `channel.py` adapter;grant 增 `backend` | 小 | -| 7 | `apps/cli/arguments.py` | `find` 增 `--backend` / `--no-external` / `--external-limit` / `--external-timeout`;新增 `cannbot` 子命令组 | 小 | -| 8 | `apps/cli/__init__.py` | 分发 `cannbot`;`campaign import-cannbot` / `import-mindie` 统一 | 中 | -| 9 | `tests/` | fake channel(本地 stdio 假进程 + 固定 JSON),覆盖七种 `state` | 必需 | - -第 2 条的不变量:**`channels` 为空或 `--no-external` 时,`find` 的输出必须与 #466 之后的行为完全一致。** 这条要写成测试,否则外部通道会悄悄改变现有 143 篇 skill 的排序基线。 - -## 7. 使用方式 - -### 7.1 首次配置(每台机器、每个 checkout 一次) - -```bash -# cannbot-knowledge -inferrsi cannbot enable \ - --checkout ~/src/cannbot-knowledge \ - --python ~/venvs/cannbot/bin/python \ - --backend ascend \ - --platform a2 - -# MindIE(沿用现有命令,仅补 backend) -inferrsi mindie enable --domain --backend ascend -``` - -自查: - -```bash -inferrsi cannbot describe -# {"state":"ok","revision":"92dc2607...","cards":10550, -# "index":{"version":3,"content_fingerprint":"0047983f..."}} - -inferrsi mindie describe -# {"available":true,"component_version":"0.13.13","domain":""} -``` - -失败时直接给修复命令: - -```text -state: index_stale -detail: index built at differs from checkout -fix: cd ~/src/cannbot-knowledge && python3 .agents/skills/knowledge-query/scripts/knowledge_index.py build -``` - -### 7.2 日常检索(Ascend 任务,常驻双查) - -```bash -$ inferrsi find "AscendC tiling 参数怎么算" --backend ascend -``` - -输出(示意): - -```text -本仓 (backend=ascend) - 1. ascendc-tiling-design score 42.1 - 2. ascend-performance-analysis score 31.7 - -外部通道 - cannbot-knowledge @ 92dc2607 ok 1.5s 3 hits - 1. Matmul Tiling类使用说明 18.4 ops/ascendc/apis/.../cube_compute/matmul_tiling - 2. 使用高阶API时配套的Tiling实现 17.9 ops/ascendc/guides/.../tiling - 3. Matmul tiling 形状与核数推导链 17.2 ops/ascendc/guides/.../derivation_chain - → inferrsi cannbot read - - mindie-agent not_enabled (fix: inferrsi mindie enable --domain ) -``` - -要点:本地有命中时**照样**查外部;两个通道各自一段;分数不跨栏比较。 - -### 7.3 非 Ascend 任务 - -```bash -$ inferrsi find "CUDA graph capture" --backend cuda -# 只查本仓;external: [] -``` - -不传 `--backend` 且 grant 未声明时: - -```text -external: [{"channel":"cannbot-knowledge","state":"backend_unknown", - "fix":"inferrsi find --backend ascend"}] -``` - -### 7.4 读外部卡正文 - -```bash -inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division -inferrsi cannbot read --page 2 # 分页续读,默认 200 行 -inferrsi mindie read # MindIE 侧沿用现有 read 动作 -``` - -正文只在终端呈现,不落盘、不进 receipt。 - -### 7.5 Agent 挂载(Claude Code / Codex) - -```bash -inferrsi cannbot serve # stdio MCP:discover / search / read_card -``` - -**透传,不收窄。** 三个工具的名称、参数与语义与上游保持一致,只是多一层授权检查、receipt 与错误归一化。理由:自造一层收窄契约会在上游演进(例如新增过滤参数、`search` schema 从 v5 升级)时变成必须同步维护的负债,而透传可以靠 `schema_version` 字段直接暴露差异。在 agent 配置中注册一条命令即可。 - -### 7.6 进 campaign evidence - -```bash -inferrsi cannbot search "CeilDivision 向上取整" --platform a2 \ - --decision "确认 a3 上的取整语义" \ - --receipt .inferrsi/receipts/ceil.json - -inferrsi campaign import-cannbot .inferrsi/receipts/ceil.json -``` - -## 8. 切片 - -| 片 | 内容 | 依赖 | 可独立合入 | -| --- | --- | --- | --- | -| C0 | `channels.py` 协议 + fake channel + 七种 state 的测试 | — | ✅ | -| C1 | `find()` 融合 + `--backend` 门控(只接 fake channel,不接真源) | C0 | ✅ | -| C2 | `apps/cannbot/` 的 `enable/disable/describe` | C0 | ✅ | -| C3 | cannbot `search` / `read`(worker + 子进程) | C2 | ✅ | -| C4 | MindIE channel adapter + grant `backend` | C0 | ✅ | -| C5 | `cannbot serve`(stdio MCP 代理) | C3 | ✅ | -| C6 | `receipt.py` + `campaign import-cannbot` 统一 | C3、C4 | ✅ | - -C1 先行的理由:它把"平台门控 + 分栏 + 超时语义"这套契约固定下来,之后两个真源接入都只是实现 `Channel`,不会反过来改 `find` 的输出结构。 - -## 9. 风险与未决 - -| # | 风险 | 现状 | 缓解 | -| --- | --- | --- | --- | -| R1 | 热路径增加 1.5–3.7s | 实测 | 并行 + `describe` 缓存 300s + `--no-external` | -| R2 | 破坏 `find` 的确定性与离线可测 | 真实风险 | 不传 `--backend` 时不查外部;`--no-external` 关闭全部通道;fake channel 覆盖全部 state;两种路径下的输出都逐字节固定 | -| R3 | 上游索引过期导致"昨天能查今天不能查" | 上游 fail-closed | `state: index_stale` + `fix`;不做静默重试 | -| R4 | 平台枚举不可映射(`a2/a3/950/310p` vs `ascend`) | 未决 | grant 声明 `backend` + `platform` 两级;未知值拒绝,不猜 | -| R5 | 许可内容意外落盘 | 已设计 | receipt 只存 locator + sha256;模板里剔除 `local_path` | -| R6 | 两套评分被下游误混 | 已设计 | `score_scale` 字段 + 分栏结构 + `notes` | -| R7 | `uv` / 完整 checkout 在 CI 不可用 | 未验证 | 单测只用 fake channel;真上游走人工验收 | -| R8 | MindIE 0.13.13 版本钉导致升级即断 | 现有行为 | `state: version_mismatch` + `fix`,与现状一致 | - -### 已定决策 - -以下四项已确认,不再是待决项;实现按此执行,若需变更须另行评审。 - -| # | 问题 | 决定 | 落实位置 | -| --- | --- | --- | --- | -| D1 | `--backend` 是否允许一次声明多个 | **只允许一个。** 多平台任务由调用方分两次查询,各自保留结果;不接受多值,避免输出无法回答"这条外部结果是为哪个平台召回的" | §2.2 平台门控、§6 改动 7 | -| D2 | `find` 默认是否带 `--no-external` | **不带。** Ascend 任务开箱即双查;CI 与单测必须显式传 `--no-external`,以隔离外部通道可用性对基线的影响 | §3.2 预算与降级、§6 改动 9 | -| D3 | `external[]` 每通道默认条数 | **`cannbot-knowledge` 10 条 / `mindie-agent` 5 条。** 上游定位符精确到 `path` 且平台过滤已生效,多给几条有助于一次选中正确粒度;MindIE 返回粗粒度 ref,过多会淹没本地结果 | §3.2 每通道条数默认值 | -| D4 | `cannbot serve` 的 MCP 面 | **透传上游 `discover` / `search` / `read_card` 三工具**,名称、参数、语义保持一致;只增加授权检查、receipt 与错误归一化。不收窄,以免在上游 `search` schema 升级时产生必须同步维护的自造契约 | §7.5 Agent 挂载 | - -## 10. 与既有文档的关系 - -| 文档 | 关系 | -| --- | --- | -| #476 | 本文把方案 D(MCP 旁路)升格为主接法,并补充"常驻查询"语义;#476 §4 的方案 D 需据此修订 | -| #388 | 本文的 receipt 设计、六类拒绝状态、许可门禁是 #388 契约的具体落地 | -| #387 | `find` 新增的 `external[]` 属于 Knowledge 公开接口的一部分,需在 #387 的 schema 中登记 | -| `AGENTS.md` | 外部命中不参与 `PASS`;引用须带 revision 与 `locally_verified=false` | From 7789dcb3409ef93880f75f23d23d81b280784d88 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Tue, 17 Nov 2026 22:56:52 +0800 Subject: [PATCH 3/5] fix(knowledge): bound channel work and bind reads to card bytes Use one cancellable deadline for discovery and search, cap both output streams while reading, and validate original upstream JSON. Preserve the upstream semantic fingerprint separately from the raw-byte digest carried by generated read commands. Cover timeout process cleanup, malformed responses, digest drift and pagination, platform overrides, and non-project lookup. Document deferred discovery caching and platform validation limits. Agent-assisted-by: Claude Code (author) Agent-assisted-by: Codex (author) Signed-off-by: jiangkuaixue123 --- docs/external-channels.md | 51 +++-- docs/operations.md | 8 +- src/inferrsi/apps/cannbot/__init__.py | 21 +- src/inferrsi/apps/cannbot/project.py | 52 +++-- src/inferrsi/apps/cannbot/read_worker.py | 260 +++++++++++++++++++--- src/inferrsi/apps/cli/__init__.py | 8 +- src/inferrsi/apps/cli/arguments.py | 8 +- src/inferrsi/knowledge/channels.py | 173 ++++++++++----- src/inferrsi/knowledge/knowledge_find.py | 5 +- tests/test_cannbot_boundaries.py | 268 +++++++++++++++++++++++ tests/test_cannbot_channel_acceptance.py | 12 +- tests/test_knowledge_channels.py | 33 ++- 12 files changed, 762 insertions(+), 137 deletions(-) create mode 100644 tests/test_cannbot_boundaries.py diff --git a/docs/external-channels.md b/docs/external-channels.md index a99c2b23..734df4b7 100644 --- a/docs/external-channels.md +++ b/docs/external-channels.md @@ -29,15 +29,12 @@ to prevent. ## Why it is not downloaded for you -cannbot-knowledge is licensed under **CANN Open Software License Agreement v2.0**, -which is not an OSI licence: the grant is limited, royalty-free but -**non-transferable, non-sublicensable and revocable**, and its use is restricted -to Huawei AI processor systems. This repository is Apache-2.0. - -Cloning that corpus on your behalf would mean accepting its terms for you and -placing a copy on your machine — a decision this project has no standing to -make. So `enable` is a person's explicit action. It is not an unimplemented -convenience; it is the compliance boundary. +cannbot-knowledge has its own CANN Open Software License Agreement v2.0. +InferRSI requires an explicitly approved local checkout and interpreter; it +does not clone, install or deploy that source automatically. Review the upstream +terms for your intended use before enabling it. Keeping the content outside the +package does not establish permission to use, transmit or redistribute it; +that organisational decision remains open in #476 Q1. Nothing from an external checkout is ever copied into `src/inferrsi/knowledge/data/`, a wheel, or `evidence/`. @@ -71,7 +68,7 @@ already could have fixed. | --- | --- | | `--checkout` | Git checkout containing `knowledge/` | | `--python` | Interpreter with `pyyaml`, able to run the checkout's scripts | -| `--backend` | Backend this source serves; `find` compares it to the task backend | +| `--backend` | Default task backend; explicit `find --backend` takes precedence. Cannbot serves Ascend only. | | `--platform` | Default upstream platform (`a2`, `a3`, `950`, `310p`, `910`, …) | Check it: @@ -100,11 +97,37 @@ inferrsi find "AscendC tiling 参数怎么算" Or declare it per call: ```bash -inferrsi find "AscendC tiling 参数怎么算" --backend ascend +inferrsi find "AscendC tiling 参数怎么算" --backend ascend --platform a2 inferrsi cannbot search "CeilDivision 向上取整" --platform a2 --limit 100 inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division ``` +The generated `open` command includes `--sha256 `. Use that command +and retain the same digest with `--page 2` (and later pages): a changed card is +rejected. Every successful read returns the digest of the full card bytes. +The upstream CLI uses `card_sha256` for a parsed metadata/body fingerprint; +this adapter preserves it as `source_card_sha256`. Before emitting a selected +hit, it verifies the source fingerprint using the approved checkout's parser +and computes `card_sha256` over those same raw bytes. It returns neither the +bytes nor private filesystem paths in search results. +A locator-only read returns the current card and its digest, without promising +continuity with an earlier search or page. A card digest does not validate its +cited resources or make it eligible as campaign evidence. + +`find --platform` overrides the grant's upstream platform. If neither supplies +one, the channel reports `rejected` and gives the missing-platform guidance. +The default external result cap is 100 for cannbot; `--external-limit` overrides +it (1–200). These are returned hits, not the total number of upstream matches. + +`--external-timeout` covers discovery and search together. Channels run +concurrently under that deadline. On timeout the caller cancels owned worker +processes and discards late results; subprocess teardown adds a short cleanup +interval. The cannbot adapter terminates its process group on POSIX; Windows +uses `taskkill /T /F` while the worker is alive. The new process-tree regression +runs on POSIX; Windows cleanup has not been qualified by that test. Generic +Python channel adapters must cooperate with the request deadline and register +owned resources; Python cannot forcibly stop arbitrary in-process threads. + ### The backend gate | Task backend | Channels searched | @@ -126,8 +149,9 @@ silent. "score_scale": "knowledge-search.v5", "count": 3, "items": [{"locator": "ops/ascendc/apis/.../ceil_division", "score": 14.0, "status": "stable", "platforms": ["a2", "a3"], - "card_sha256": "b97cbe82...", - "open": "inferrsi cannbot read ops/..."}] + "card_sha256": "", + "source_card_sha256": "", + "open": "inferrsi cannbot read ops/... --sha256 "}] }] ``` @@ -187,6 +211,7 @@ conclusion. | `cannbot serve` (stdio MCP proxying `discover` / `search` / `read_card`) | not yet | | `campaign import-cannbot` (receipts into evidence) | not yet | | Automatic clone, index build or deployment | deliberately absent | +| In-memory discovery cache | deferred; discovery and search share the current call’s budget | ## Related diff --git a/docs/operations.md b/docs/operations.md index 3bf214c6..d1a88810 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -279,14 +279,16 @@ external (not locally verified; scores are not comparable across channels) Enabling a channel is a deliberate, per-checkout action by a person; see [external knowledge channels](external-channels.md). This repository does not -clone, download or bundle a third-party corpus, because doing so would accept -and redistribute a licence on the user's behalf. +clone, download or bundle the corpus automatically. Review the upstream terms +for the intended use before granting access; source permissions remain separate +from this integration mechanism. - **Inputs:** `--backend ` declares the task backend; **without it and without an approved grant, no channel is searched** and each one reports `backend_unknown`. `--no-external` skips all channels for CI and offline runs. `--external-limit ` overrides the per-channel cap (`cannbot-knowledge` 100, - `mindie-agent` 5). `--external-timeout ` bounds each channel (8s default). + `mindie-agent` 5). `--external-timeout ` bounds discovery and search + together (8s default); `--platform` overrides the upstream platform in the grant. - **Output:** `external[]` sits beside `items`, never inside it. Each entry carries `state`, `revision`, `elapsed_ms`, `score_scale` and `count`. A state other than `ok` (`not_enabled`, `unreachable`, `index_stale`, `timeout`, diff --git a/src/inferrsi/apps/cannbot/__init__.py b/src/inferrsi/apps/cannbot/__init__.py index 6b7fae20..ee6b6a5e 100644 --- a/src/inferrsi/apps/cannbot/__init__.py +++ b/src/inferrsi/apps/cannbot/__init__.py @@ -7,6 +7,7 @@ from ...knowledge.channels import ( ChannelHit, + ChannelRequest, ChannelResult, ChannelStatus, ) @@ -38,12 +39,13 @@ def __init__(self, *, root: Path | None = None, timeout: float = 8.0): except (OSError, ValueError) as exc: grant = {} self.hint = ChannelStatus("rejected", detail=str(exc)) - self.backend = grant.get("backend") + self.backend = "ascend" + self.declared_backend = grant.get("backend") self.name = "cannbot-knowledge" - def describe(self) -> ChannelStatus: + def describe(self, *, request: ChannelRequest) -> ChannelStatus: try: - info = self._project.describe(self.root, timeout=max(self.timeout, 15.0)) + info = self._project.describe(self.root, timeout=request.remaining(), request=request) except self._project.ChannelFailure as exc: return ChannelStatus(exc.state, detail=str(exc), fix=exc.fix) state = info.get("state", "rejected") @@ -55,19 +57,20 @@ def describe(self) -> ChannelStatus: return ChannelStatus("ok", revision=info.get("revision")) def search( - self, query: str, *, platform: str | None, limit: int + self, query: str, *, platform: str | None, limit: int, request: ChannelRequest ) -> ChannelResult: started = time.monotonic() try: payload = self._project.search( query, platform=platform, limit=limit, root=self.root, - timeout=self.timeout, + timeout=request.remaining(), request=request, ) except self._project.ChannelFailure as exc: return ChannelResult( name=self.name, status=ChannelStatus( exc.state, detail=str(exc), fix=exc.fix, ), + elapsed_ms=max(1, int((time.monotonic() - started) * 1000)), ) elapsed = int((time.monotonic() - started) * 1000) items = tuple( @@ -80,6 +83,7 @@ def search( status=str(row.get("status") or ""), platforms=tuple(row.get("platforms") or ()), card_sha256=str(row.get("card_sha256") or ""), + source_card_sha256=str(row.get("source_card_sha256") or ""), ) for row in payload.get("items") or [] if row.get("locator") @@ -93,6 +97,9 @@ def search( read_prefix="inferrsi cannbot read", ) - def read(self, locator: str) -> str: - payload = self._project.read(locator, root=self.root, timeout=max(self.timeout, 15.0)) + def read(self, locator: str, *, expected_sha256: str | None = None) -> str: + payload = self._project.read( + locator, expected_sha256=expected_sha256, root=self.root, + timeout=max(self.timeout, 15.0), + ) return str(payload.get("text") or "") diff --git a/src/inferrsi/apps/cannbot/project.py b/src/inferrsi/apps/cannbot/project.py index 0238023e..3d0b572b 100644 --- a/src/inferrsi/apps/cannbot/project.py +++ b/src/inferrsi/apps/cannbot/project.py @@ -15,6 +15,8 @@ import tempfile from pathlib import Path +from .read_worker import OutputLimitError, _finite, capture + _SCHEMA = "inferrsi-cannbot-project/1" _WORKER = Path(__file__).with_name("read_worker.py") MAX_BYTES = 2 * 1024 * 1024 @@ -40,7 +42,10 @@ def project_root() -> Path: for folder in (cwd, *cwd.parents): if (folder / ".git").exists(): return folder - raise ValueError("Open a Git project to use project-scoped cannbot knowledge") + raise ChannelFailure( + "Open a Git project to use project-scoped cannbot knowledge", + state="not_enabled", fix="Open a Git project, then enable cannbot knowledge", + ) def _grant_path(root: Path) -> Path: @@ -74,7 +79,8 @@ def _invalid(_): raise ValueError("cannbot response contains a non-finite JSON value") try: - value = json.loads(raw, object_pairs_hook=_object, parse_constant=_invalid) + value = json.loads(raw, object_pairs_hook=_object, parse_constant=_invalid, + parse_float=_finite) except (ValueError, UnicodeError, RecursionError) as exc: raise ValueError("cannbot did not return a JSON object") from exc if not isinstance(value, dict): @@ -119,7 +125,7 @@ def load_grant(root: Path | None = None) -> dict: return grant -def _run(grant: dict, action: str, arguments: dict, *, timeout: float) -> dict: +def _run(grant: dict, action: str, arguments: dict, *, timeout: float, request=None) -> dict: environment = os.environ.copy() for key in ("CANNBOT_MCP_TOKENS", "CANNBOT_KNOWLEDGE_PLATFORM"): environment.pop(key, None) @@ -127,23 +133,28 @@ def _run(grant: dict, action: str, arguments: dict, *, timeout: float) -> dict: with tempfile.TemporaryDirectory(prefix="inferrsi-cannbot-read-") as temporary: cwd = Path(temporary) / "reader" cwd.mkdir(mode=0o700) - result = subprocess.run( + result = capture( [grant["python"], "-I", "-B", str(_WORKER), - grant["upstream"], action, json.dumps(arguments)], - input=b"", stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - cwd=cwd, env=environment, timeout=timeout, check=False, + grant["upstream"], action, json.dumps(arguments), str(timeout)], + cwd=cwd, environment=environment, timeout=timeout, request=request, + own_group=True, ) except subprocess.TimeoutExpired: raise ChannelFailure( f"cannbot did not answer within {timeout:g}s", state="timeout" ) from None + except OutputLimitError: + raise ChannelFailure("cannbot output exceeded 2 MiB", state="rejected") from None except OSError: raise ChannelFailure( "The approved cannbot reader could not be started", state="unreachable" ) from None if len(result.stdout) > MAX_BYTES: raise ChannelFailure("cannbot returned more than 2 MiB", state="rejected") - payload = _parse(result.stdout) + try: + payload = _parse(result.stdout) + except ValueError: + raise ChannelFailure("cannbot returned invalid JSON", state="rejected") from None if result.returncode or payload.get("isError"): raise ChannelFailure( str(payload.get("error") or "The approved cannbot reader failed"), @@ -154,13 +165,13 @@ def _run(grant: dict, action: str, arguments: dict, *, timeout: float) -> dict: return payload -def describe(root: Path | None = None, *, timeout: float = 15.0) -> dict: +def describe(root: Path | None = None, *, timeout: float = 15.0, request=None) -> dict: try: grant = load_grant(root) except ChannelFailure as exc: return {"state": exc.state, "detail": str(exc), "fix": exc.fix} try: - payload = _run(grant, "discover", {}, timeout=timeout) + payload = _run(grant, "discover", {}, timeout=timeout, request=request) except ChannelFailure as exc: return {"state": exc.state, "detail": str(exc), "fix": exc.fix} return { @@ -173,8 +184,8 @@ def describe(root: Path | None = None, *, timeout: float = 15.0) -> dict: def search( - query: str, *, platform: str | None = None, limit: int = 10, - root: Path | None = None, timeout: float = 8.0, + query: str, *, platform: str | None = None, limit: int = 100, + root: Path | None = None, timeout: float = 8.0, request=None, ) -> dict: if not isinstance(query, str) or not query.strip(): raise ChannelFailure("cannbot search needs a non-empty query") @@ -186,23 +197,30 @@ def search( if not platforms: raise ChannelFailure( "cannbot search needs a platform", state="rejected", - fix="inferrsi cannbot search ... --platform a2", + fix="inferrsi find --backend ascend --platform ", ) platform = platforms[0] if platform not in PLATFORMS: raise ChannelFailure(f"cannbot does not know platform {platform!r}", state="rejected") return _run(grant, "search", { "query": query, "platform": platform, "limit": limit, - }, timeout=timeout) + }, timeout=timeout, request=request) -def read(locator: str, *, page: int = 1, root: Path | None = None, - timeout: float = 15.0) -> dict: +def read(locator: str, *, page: int = 1, expected_sha256: str | None = None, + root: Path | None = None, timeout: float = 15.0) -> dict: if not isinstance(locator, str) or not locator.strip(): raise ChannelFailure("cannbot read needs a locator from a search result") if type(page) is not int or page < 1: raise ChannelFailure("cannbot page must be a positive integer") - return _run(load_grant(root), "read", {"locator": locator, "page": page}, timeout=timeout) + if expected_sha256 is not None and ( + not isinstance(expected_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) + ): + raise ChannelFailure("cannbot expected_sha256 must be a lowercase 64-hex digest") + arguments = {"locator": locator, "page": page} + if expected_sha256: + arguments["expected_sha256"] = expected_sha256 + return _run(load_grant(root), "read", arguments, timeout=timeout) def enable(*, upstream: Path, python: Path, backend: str = "ascend", diff --git a/src/inferrsi/apps/cannbot/read_worker.py b/src/inferrsi/apps/cannbot/read_worker.py index d5047fbf..081359c2 100644 --- a/src/inferrsi/apps/cannbot/read_worker.py +++ b/src/inferrsi/apps/cannbot/read_worker.py @@ -7,9 +7,17 @@ optional receipt, which is not requested. """ +import hashlib import json +import math +import os +import queue +import re +import signal import subprocess import sys +import threading +import time from pathlib import Path _SCRIPT = ".agents/skills/knowledge-query/scripts/knowledge_query.py" @@ -18,6 +26,9 @@ _FIX_BUILD = ( "cd && python3 " + _INDEX + " build" ) +# Upstream output is untrusted input. Bound it while reading, not after the +# whole stream is already in memory. +_MAX_UPSTREAM = 2 * 1024 * 1024 def _fail(message, *, state="rejected", fix=""): @@ -25,24 +36,152 @@ def _fail(message, *, state="rejected", fix=""): raise SystemExit(1) +def _object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("cannbot source JSON has duplicate keys") + value[key] = item + return value + + +def _invalid(_): + raise ValueError("cannbot source JSON has a non-finite number") + + +def _finite(value): + number = float(value) + if not math.isfinite(number): + _invalid(value) + return number + + +def _strict(raw): + """Reject duplicate keys and non-finite numbers at the source boundary. + + `json.loads` collapses a duplicate key before any caller can see it, so a + later re-serialisation cannot repair a document that was ambiguous on + arrival. The check has to run on the upstream bytes themselves. + """ + try: + value = json.loads( + raw, object_pairs_hook=_object, parse_constant=_invalid, parse_float=_finite, + ) + except (ValueError, UnicodeError, RecursionError): + _fail("cannbot returned malformed JSON", state="rejected") + if not isinstance(value, dict): + _fail("cannbot did not return a JSON object", state="rejected") + return value + + +class OutputLimitError(ValueError): + """A pipe exceeded the bounded capture size.""" + + +def _stop(process, *, own_group): + if own_group and os.name == "posix": + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + elif own_group and os.name == "nt" and process.poll() is None: + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2, check=False, + ) + if process.poll() is None: + process.kill() + process.wait(timeout=2) + + +def capture(argv, *, cwd, timeout, environment=None, request=None, own_group=False): + """Capture at most 2 MiB + 1 per pipe, with no unbounded communicate(). + + The outer worker owns a process group; upstream queries inherit it. Both + the caller's cancellation and this function's finally block reap that group. + Each pipe is drained concurrently so a full stderr cannot deadlock stdout. + """ + deadline = time.monotonic() + timeout + + def start(): + return subprocess.Popen( + argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd, env=environment, start_new_session=own_group and os.name == "posix", + creationflags=(subprocess.CREATE_NEW_PROCESS_GROUP + if own_group and os.name == "nt" else 0), + ) + + def stop(process): + _stop(process, own_group=own_group) + + process = request.start(start, stop) if request else start() + received = queue.Queue(maxsize=2) + + def drain(name, stream): + try: + with stream: + received.put((name, stream.read(_MAX_UPSTREAM + 1))) + except OSError: + received.put((name, b"")) + + for name, stream in (("stdout", process.stdout), ("stderr", process.stderr)): + threading.Thread(target=drain, args=(name, stream), daemon=True).start() + try: + output = {} + for _ in range(2): + try: + name, raw = received.get(timeout=max(0, deadline - time.monotonic())) + except queue.Empty: + raise subprocess.TimeoutExpired(argv, timeout) from None + if len(raw) > _MAX_UPSTREAM: + raise OutputLimitError("cannbot output exceeded 2 MiB") + output[name] = raw + code = process.wait(timeout=max(0, deadline - time.monotonic())) + return subprocess.CompletedProcess(argv, code, output["stdout"], output["stderr"]) + finally: + if request: + request.finish(process) + else: + stop(process) + + def _run(upstream, script, argv, *, timeout): - result = subprocess.run( - [sys.executable, "-I", "-B", str(upstream / script), *argv], - capture_output=True, cwd=str(upstream), timeout=timeout, check=False, - ) + try: + result = capture( + [sys.executable, "-I", "-B", str(upstream / script), *argv], + cwd=str(upstream), timeout=timeout, + ) + except OutputLimitError: + _fail("cannbot output exceeded 2 MiB") if result.returncode: - detail = result.stderr.decode("utf-8", "replace").strip() - lowered = detail.casefold() + lowered = result.stderr.decode("utf-8", "replace").casefold() state = "index_stale" if any(m in lowered for m in _STALE_MARKERS) else "rejected" - _fail(detail[-400:] or "cannbot query failed", state=state, + # Never relay arbitrary upstream stderr: it may contain private paths. + _fail("cannbot index needs rebuilding" if state == "index_stale" + else "cannbot query failed", state=state, fix=_FIX_BUILD if state == "index_stale" else "") - return json.loads(result.stdout) + return _strict(result.stdout) + +def _index(payload): + index = payload.get("index") + if not isinstance(index, dict): + _fail("cannbot response needs an index object") + state = index.get("built_repository_state") + if not isinstance(state, dict) or not isinstance(state.get("commit"), str): + _fail("cannbot response needs an index revision") + if not re.fullmatch(r"[0-9a-f]{40}", state["commit"]): + _fail("cannbot response has an invalid index revision") + return index, state -def discover(upstream): - payload = _run(upstream, _SCRIPT, ["discover", "--json"], timeout=30) - index = payload.get("index") or {} - state = index.get("built_repository_state") or {} + +def discover(upstream, *, timeout=30): + payload = _run(upstream, _SCRIPT, ["discover", "--json"], timeout=timeout) + if payload.get("schema_version") != "knowledge-discovery.v1": + _fail("Unsupported cannbot discovery schema", state="version_mismatch") + if type(payload.get("cards")) is not int or payload["cards"] < 0: + _fail("cannbot discovery needs a non-negative card count") + index, state = _index(payload) return { "state": "ok", "revision": state.get("commit"), @@ -56,13 +195,69 @@ def discover(upstream): } -def search(upstream, arguments): +def _card_bytes(upstream, locator): + root = (upstream / "knowledge").resolve() + card = (root / (locator + ".md")).resolve() + if not card.is_relative_to(root) or not card.is_file(): + _fail("No such card in the approved checkout") + with card.open("rb") as source: + raw = source.read(_MAX_UPSTREAM + 1) + if len(raw) > _MAX_UPSTREAM: + _fail("cannbot card exceeded 2 MiB") + return raw + + +def _source_fingerprint(upstream): + # The query index hashes parsed metadata + body, unlike MCP read_card's + # raw-byte digest. Use the approved source's own implementation; never copy + # its parser or fingerprint algorithm into this package. + sys.path.insert(0, str((upstream / _SCRIPT).parent)) + try: + from retrieval.index import card_fingerprint, load_contract + + _, split_frontmatter = load_contract(upstream) + except (ImportError, AttributeError): + _fail("Unsupported cannbot card identity API", state="version_mismatch") + return card_fingerprint, split_frontmatter + + +def search(upstream, arguments, *, timeout=60): payload = _run(upstream, _SCRIPT, [ "search", "--json", "--query", arguments["query"], "--platform", arguments["platform"], "--limit", str(arguments["limit"]), - ], timeout=60) + ], timeout=timeout) + if payload.get("schema_version") != "knowledge-search.v5": + _fail("Unsupported cannbot search schema", state="version_mismatch") + rows = payload.get("results") + if not isinstance(rows, list): + _fail("cannbot search needs a results list") hits = [] - for row in payload.get("results") or []: + if rows: + fingerprint, split_frontmatter = _source_fingerprint(upstream) + for row in rows: + if not isinstance(row, dict): + _fail("cannbot search result must be an object") + locator, digest = row.get("path"), row.get("card_sha256") + if (not isinstance(locator, str) or not locator or locator.startswith("/") + or "\\" in locator or any(p in ("", ".", "..") for p in locator.split("/"))): + _fail("cannbot search result has an invalid locator") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + _fail("cannbot search result needs a card digest") + score = row.get("score") + if type(score) not in (int, float) or not math.isfinite(score): + _fail("cannbot search result needs a finite score") + if any(row.get(key) is not None and not isinstance(row[key], str) + for key in ("title", "description", "status")): + _fail("cannbot search result has invalid text fields") + platforms = row.get("platforms") + if platforms is not None and (not isinstance(platforms, list) + or any(not isinstance(p, str) for p in platforms)): + _fail("cannbot search result has an invalid platform list") + raw = _card_bytes(upstream, locator) + metadata, body = split_frontmatter(raw.decode("utf-8")) + if fingerprint(metadata, body) != digest: + _fail("cannbot card differs from its search index", state="index_stale", + fix=_FIX_BUILD) hits.append({ "locator": row.get("path"), "title": row.get("title") or (row.get("path") or "").rsplit("/", 1)[-1], @@ -70,12 +265,12 @@ def search(upstream, arguments): "score": row.get("score") or 0, "status": row.get("status") or "", "platforms": row.get("platforms") or [], - "card_sha256": row.get("card_sha256") or "", + "source_card_sha256": digest, + "card_sha256": hashlib.sha256(raw).hexdigest(), }) - index = payload.get("index") or {} + index, state = _index(payload) # The pin lives on the index provenance, not at the top level: that is the # revision the index was built from, which is what a citation must name. - state = index.get("built_repository_state") or {} return { "state": "ok", "revision": state.get("commit"), @@ -91,13 +286,20 @@ def search(upstream, arguments): def read(upstream, arguments): locator = arguments["locator"] - card = (upstream / "knowledge" / (locator + ".md")).resolve() - root = (upstream / "knowledge").resolve() - # A locator is a Concept ID, not a path. Refuse anything that leaves the - # bundle rather than trusting the caller to have passed a search result. - if not card.is_relative_to(root) or not card.is_file(): - _fail("No such card in the approved checkout", state="rejected") - lines = card.read_text(encoding="utf-8").splitlines() + body = _card_bytes(upstream, locator) + digest = hashlib.sha256(body).hexdigest() + # Search reports what it matched; read reports what it actually returned. + # Without both, a card edited between the two is served under the old + # locator and a citation names bytes nobody verified. A caller that passed + # an expected digest gets a refusal instead of silent drift. + expected = arguments.get("expected_sha256") + if expected and expected != digest: + _fail( + "The card changed since the search that found it " + f"(expected {expected[:12]}…, read {digest[:12]}…); search again", + state="rejected", + ) + lines = body.decode("utf-8").splitlines() size, page = 200, arguments.get("page", 1) start = (page - 1) * size chunk = lines[start:start + size] @@ -107,6 +309,7 @@ def read(upstream, arguments): "page": page, "pages": max(1, (len(lines) + size - 1) // size), "line_count": len(lines), + "card_sha256": digest, "text": "\n".join(chunk), "has_more": start + size < len(lines), } @@ -116,11 +319,12 @@ def read(upstream, arguments): try: upstream = Path(sys.argv[1]).resolve() action, raw = sys.argv[2], sys.argv[3] if len(sys.argv) > 3 else "{}" - arguments = json.loads(raw or "{}") + arguments = _strict(raw or "{}") + timeout = float(sys.argv[4]) if len(sys.argv) > 4 else 30.0 if action == "discover": - result = discover(upstream) + result = discover(upstream, timeout=timeout) elif action == "search": - result = search(upstream, arguments) + result = search(upstream, arguments, timeout=timeout) elif action == "read": result = read(upstream, arguments) else: diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index f9e9d7fb..28e6ed48 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -152,7 +152,7 @@ def _channels(): from ..cannbot import CannbotChannel channel = CannbotChannel() - return ChannelRegistry([channel], declared_backend=channel.backend) + return ChannelRegistry([channel], declared_backend=channel.declared_backend) def _print_external(result: dict) -> None: @@ -315,7 +315,9 @@ def main(argv: list[str] | None = None) -> int: args.query, platform=args.platform, limit=args.limit, ) else: - result = cannbot.project.read(args.locator, page=args.page) + result = cannbot.project.read( + args.locator, page=args.page, expected_sha256=args.sha256, + ) _print(result) return 0 if args.command == "agents": @@ -447,7 +449,7 @@ def main(argv: list[str] | None = None) -> int: args.query, load_corpus(args.skill_root, kinds=kinds), limit=args.limit, explain=args.explain, registry=None if args.no_external else _channels(), - backend=args.backend, + backend=args.backend, platform=args.platform, external_limit=args.external_limit, external_timeout=args.external_timeout, ) diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index 27b0ef70..fba62d18 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -148,6 +148,11 @@ def parser() -> argparse.ArgumentParser: cannbot_search.add_argument("--limit", type=int, default=100) cannbot_read = cannbot_actions.add_parser("read", help="Read one card by locator") cannbot_read.add_argument("locator") + cannbot_read.add_argument( + "--sha256", + help="Expected card digest from a search result; refuse the read if the " + "card changed since then", + ) cannbot_read.add_argument("--page", type=int, default=1) agents = commands.add_parser("agents", help="Record native host fresh-session handoffs") agent_actions = agents.add_subparsers(dest="agent_action", required=True) @@ -464,13 +469,14 @@ def parser() -> argparse.ArgumentParser: help="Declare the task backend once, e.g. ascend; external channels run only " "for a declared backend and are never inferred from the query", ) + find.add_argument("--platform", help="Override the grant’s upstream platform, e.g. a2") find.add_argument( "--no-external", action="store_true", help="Skip every external channel; use in CI and offline runs", ) find.add_argument( "--external-limit", type=int, - help="Override the per-channel result cap (cannbot 10, MindIE 5 by default)", + help="Override the per-channel result cap (cannbot 100, MindIE 5 by default)", ) find.add_argument( "--external-timeout", type=float, default=8.0, diff --git a/src/inferrsi/knowledge/channels.py b/src/inferrsi/knowledge/channels.py index fbdc4555..66aad923 100644 --- a/src/inferrsi/knowledge/channels.py +++ b/src/inferrsi/knowledge/channels.py @@ -14,8 +14,8 @@ * A failure is a state, not an empty list. "Searched and found nothing" and "could not search" are different facts, and conflating them lets a caller conclude the source has no such knowledge. -* Nothing here reads a channel body on the ranking path. Reading is a separate - call with its own budget. +* No card text is returned on the ranking path. An adapter may verify selected + card bytes to bind locators to a digest; material reads are separate calls. Channels are opt-in. Resolving a registry never imports or probes one until the platform gate has selected it, and an absent registry leaves `find` behaving @@ -25,7 +25,11 @@ from __future__ import annotations import concurrent.futures -from dataclasses import dataclass, field +import math +import shlex +import threading +import time +from dataclasses import dataclass, field, replace from typing import Any, Protocol, runtime_checkable # States a channel can report. `ok` is the only one that carries items; every @@ -95,6 +99,7 @@ class ChannelHit: status: str = "" platforms: tuple[str, ...] = () card_sha256: str = "" + source_card_sha256: str = "" def view(self, *, read_prefix: str) -> dict: record: dict[str, Any] = { @@ -111,8 +116,13 @@ def view(self, *, read_prefix: str) -> dict: record["platforms"] = list(self.platforms) if self.card_sha256: record["card_sha256"] = self.card_sha256 + if self.source_card_sha256: + record["source_card_sha256"] = self.source_card_sha256 if read_prefix: - record["open"] = f"{read_prefix} {self.locator}" + argv = [*shlex.split(read_prefix), self.locator] + if self.card_sha256: + argv.extend(["--sha256", self.card_sha256]) + record["open"] = shlex.join(argv) return record @@ -150,13 +160,13 @@ class Channel(Protocol): name: str backend: str | None - def describe(self) -> ChannelStatus: ... + def describe(self, *, request: ChannelRequest) -> ChannelStatus: ... def search( - self, query: str, *, platform: str | None, limit: int + self, query: str, *, platform: str | None, limit: int, request: ChannelRequest ) -> ChannelResult: ... - def read(self, locator: str) -> str: ... + def read(self, locator: str, *, expected_sha256: str | None = None) -> str: ... @dataclass @@ -202,32 +212,77 @@ def _empty(name: str, state: str, *, detail: str = "", fix: str = "", ) +class ChannelRequest: + """One deadline and owned resources for describe plus search. + + Adapters must start subprocesses through this request, so cancellation and + process creation cannot race. Cleanup is idempotent and synchronous; once + cancelled, a late worker cannot start more work. Pure Python adapters must + cooperate with the deadline. Their daemon threads cannot delay CLI exit. + """ + + def __init__(self, deadline: float): + self.deadline = deadline + self._lock = threading.Lock() + self._cancelled = False + self._resources: dict = {} + + def remaining(self) -> float: + remaining = self.deadline - time.monotonic() + if self._cancelled or remaining <= 0: + raise TimeoutError("external channel deadline expired") + return remaining + + def start(self, factory, cleanup): + with self._lock: + self.remaining() + resource = factory() + self._resources[resource] = cleanup + return resource + + def finish(self, resource): + with self._lock: + cleanup = self._resources.get(resource) + if cleanup: + cleanup(resource) + self._resources.pop(resource) + + def cancel(self): + with self._lock: + self._cancelled = True + failed = False + for resource, cleanup in self._resources.items(): + try: + cleanup(resource) + except Exception: + failed = True + self._resources.clear() + return not failed + + def _run_one( - channel: Channel, query: str, *, platform: str | None, limit: int, timeout: float + channel: Channel, query: str, *, platform: str | None, limit: int, + request: ChannelRequest, started: float, ) -> ChannelResult: + phase = "describe" try: - described = channel.describe() - except Exception as exc: # a broken channel must not break find - return _empty(channel.name, UNREACHABLE, detail=_reason(exc)) - if described.state != OK: - return ChannelResult(name=channel.name, status=described) - try: - result = channel.search(query, platform=platform, limit=limit) + request.remaining() + described = channel.describe(request=request) + request.remaining() + if described.state != OK: + result = ChannelResult(name=channel.name, status=described) + else: + phase = "search" + result = channel.search(query, platform=platform, limit=limit, request=request) + request.remaining() except TimeoutError: - return _empty(channel.name, TIMEOUT, detail=f"no answer within {timeout:g}s") + result = _empty(channel.name, TIMEOUT, detail="external channel deadline expired") except Exception as exc: - return _empty(channel.name, REJECTED, detail=_reason(exc)) - if result.elapsed_ms > timeout * 1000: - # The work finished, so report what it found and the budget it broke. - return ChannelResult( - name=channel.name, status=ChannelStatus( - OK, revision=result.status.revision, - detail=f"answered in {result.elapsed_ms} ms, over the {timeout:g}s budget", - ), - items=result.items, elapsed_ms=result.elapsed_ms, - score_scale=result.score_scale, read_prefix=result.read_prefix, + result = _empty( + channel.name, UNREACHABLE if phase == "describe" else REJECTED, + detail=_reason(exc), ) - return result + return replace(result, elapsed_ms=max(1, int((time.monotonic() - started) * 1000))) def _reason(exc: BaseException) -> str: @@ -244,39 +299,53 @@ def search_channels( limit: int | None = None, timeout: float = 8.0, ) -> list[ChannelResult]: - """Query every eligible channel concurrently; one failure never hides another. + """Query eligible channels concurrently, cancel owned work at one deadline. - Channels are independent I/O, so the wall clock is the slowest one rather - than the sum. An empty registry is not an error: it means this checkout has - no external source, which is the default state. + There is no executor shutdown/atexit join and no fresh timeout per future. + Adapters receive the same request for both phases, including the remaining + budget and resource cleanup. A late answer is discarded, never labelled OK. """ - if timeout <= 0: - raise ValueError("external timeout must be positive") + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("external timeout must be positive and finite") if limit is not None and (type(limit) is not int or limit < 1): raise ValueError("external limit must be a positive integer") eligible = registry.for_backend(backend) - if not eligible: - return [] + started = time.monotonic() + pending = [] - def work(channel: Channel) -> ChannelResult: - return _run_one( + def work(channel, request, future): + future.set_result(_run_one( channel, query, platform=platform, limit=limit or DEFAULT_LIMITS.get(channel.name, FALLBACK_LIMIT), - timeout=timeout, - ) - - if len(eligible) == 1: - return [work(eligible[0])] - with concurrent.futures.ThreadPoolExecutor(max_workers=len(eligible)) as pool: - futures = [pool.submit(work, channel) for channel in eligible] - results = [] - for channel, future in zip(eligible, futures): - try: - results.append(future.result(timeout=timeout)) - except concurrent.futures.TimeoutError: - results.append(_empty( - channel.name, TIMEOUT, detail=f"no answer within {timeout:g}s", - )) + request=request, started=started, + )) + + for channel in eligible: + request = ChannelRequest(started + timeout) + future = concurrent.futures.Future() + threading.Thread( + target=work, args=(channel, request, future), daemon=True, + name="inferrsi-channel", + ).start() + pending.append((channel, request, future)) + + results = [] + for channel, request, future in pending: + try: + result = future.result(timeout=max(0, request.deadline - time.monotonic())) + except concurrent.futures.TimeoutError: + result = replace( + _empty(channel.name, TIMEOUT, detail=f"no answer within {timeout:g}s"), + elapsed_ms=max(1, int((time.monotonic() - started) * 1000)), + ) + finally: + cleaned = request.cancel() + if not cleaned: + result = replace( + _empty(channel.name, REJECTED, detail="channel resource cleanup failed"), + elapsed_ms=max(1, int((time.monotonic() - started) * 1000)), + ) + results.append(result) return results diff --git a/src/inferrsi/knowledge/knowledge_find.py b/src/inferrsi/knowledge/knowledge_find.py index a6910908..f73fdfde 100644 --- a/src/inferrsi/knowledge/knowledge_find.py +++ b/src/inferrsi/knowledge/knowledge_find.py @@ -33,7 +33,10 @@ from io import BytesIO from pathlib import Path from statistics import median -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .channels import ChannelRegistry from ._skill_selection import TASK_STOP_WORDS, named_backends from .catalog import BACKENDS diff --git a/tests/test_cannbot_boundaries.py b/tests/test_cannbot_boundaries.py new file mode 100644 index 00000000..d363fb49 --- /dev/null +++ b/tests/test_cannbot_boundaries.py @@ -0,0 +1,268 @@ +"""Synthetic upstream processes exercise the actual CLI and raw-byte boundary.""" + +import hashlib +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from inferrsi.apps.cannbot import project +from inferrsi.apps.cannbot.read_worker import _MAX_UPSTREAM + +REPO = Path(__file__).resolve().parents[1] +REVISION = "ab" * 20 + + +def cli(*args, cwd=None, timeout=12): + return subprocess.run( + [sys.executable, "-m", "inferrsi", *args], + cwd=cwd, capture_output=True, text=True, timeout=timeout, check=False, + ) + + +def discovery(): + return {"schema_version": "knowledge-discovery.v1", "cards": 1, + "index": {"built_repository_state": {"commit": REVISION}}} + + +@pytest.fixture +def upstream(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("PYTHONPATH", str(REPO / "src")) + root = tmp_path / "project" + root.mkdir() + (root / ".git").mkdir() + monkeypatch.chdir(root) + source = tmp_path / "upstream" + (source / ".git").mkdir(parents=True) + (source / "knowledge/ops").mkdir(parents=True) + card = source / "knowledge/ops/card.md" + card.write_text("\n".join(f"synthetic line {i}" for i in range(401))) + script = source / ".agents/skills/knowledge-query/scripts/knowledge_query.py" + script.parent.mkdir(parents=True) + native = script.parent / "retrieval" + native.mkdir() + (native / "__init__.py").touch() + (native / "index.py").write_text( + "import hashlib\n" + "def load_contract(root):\n" + " return None, lambda text: ({}, text)\n" + "def card_fingerprint(metadata, body):\n" + " return hashlib.sha256(body.encode()).hexdigest()\n" + ) + script.write_text(f"import json\nprint(json.dumps({discovery()!r}))\n") + project.enable(upstream=source, python=Path(sys.executable), platforms=["a2"]) + return source, script, card + + +def search_payload(card): + return {"schema_version": "knowledge-search.v5", "index": discovery()["index"], + "results": [{"path": "ops/card", "title": "synthetic", "score": 1.0, + "platforms": ["a2"], + "card_sha256": hashlib.sha256(card.read_bytes()).hexdigest()}]} + + +def set_source(script, raw): + script.write_text(f"import sys\nsys.stdout.buffer.write({raw!r})\n") + + +@pytest.mark.parametrize("raw", [ + b'{"cards": 1, "cards": 2}', b'{"x": NaN}', b'{"x": Infinity}', + b'{"x": -Infinity}', b'{"x": 1e999}', b'[]', b'{"x": {"y": 1, "y": 2}}', + b'\xff', b'{"schema_version": "knowledge-discovery.v1", "cards": 1, "index": []}', +]) +def test_rejects_original_source_bytes(upstream, raw): + _, script, _ = upstream + set_source(script, raw) + assert project.describe()["state"] == "rejected" + + +@pytest.mark.parametrize("mutation", [ + {"results": {}}, {"results": [42]}, + {"results": [{"path": "ops/card", "score": "bad"}]}, + {"index": []}, +]) +def test_search_shape_is_rejected(upstream, mutation): + _, script, card = upstream + set_source(script, json.dumps({**search_payload(card), **mutation}).encode()) + with pytest.raises(project.ChannelFailure) as error: + project.search("query") + assert error.value.state == "rejected" + + +def test_unknown_schema_is_version_mismatch(upstream): + _, script, _ = upstream + set_source(script, json.dumps({**discovery(), "schema_version": "future"}).encode()) + assert project.describe()["state"] == "version_mismatch" + + +@pytest.mark.parametrize("pipe", ["stdout", "stderr"]) +def test_output_cap_stops_a_producer_before_it_finishes(upstream, pipe): + source, script, _ = upstream + marker = source / "finished" + script.write_text( + "import sys, time\nfrom pathlib import Path\n" + f"sys.{pipe}.buffer.write(b'x' * {_MAX_UPSTREAM + 1})\n" + f"sys.{pipe}.flush()\ntime.sleep(3)\n" + "Path('finished').touch()\n" + ) + started = time.monotonic() + assert project.describe(timeout=6)["state"] == "rejected" + assert time.monotonic() - started < 2 + assert not marker.exists() + + +def test_read_binds_all_pages_to_search_digest(upstream): + _, script, card = upstream + script.write_text( + "import json, sys\n" + f"print(json.dumps({discovery()!r} if sys.argv[1] == 'discover' " + f"else {search_payload(card)!r}))\n" + ) + result = cli("find", "synthetic", "--backend", "ascend", "--json") + assert result.returncode == 0, result.stderr + hit = json.loads(result.stdout)["external"][0]["items"][0] + opened = cli(*shlex.split(hit["open"])[1:]) + body = json.loads(opened.stdout) + assert body["card_sha256"] == hit["card_sha256"] + assert body["has_more"] + page = cli(*shlex.split(hit["open"])[1:], "--page", "2") + assert json.loads(page.stdout)["card_sha256"] == hit["card_sha256"] + card.write_text("changed") + changed = cli(*shlex.split(hit["open"])[1:], "--page", "2") + assert changed.returncode != 0 + assert "changed since the search" in changed.stderr + assert not changed.stdout + current = json.loads(cli("cannbot", "read", "ops/card").stdout) + assert current["card_sha256"] == hashlib.sha256(card.read_bytes()).hexdigest() + + +def test_read_refuses_path_escape(upstream): + source, _, _ = upstream + (source / "private.md").write_text("not knowledge") + result = cli("cannbot", "read", "../private") + assert result.returncode != 0 + assert "not knowledge" not in result.stdout + result.stderr + + +def test_native_fingerprint_and_raw_byte_digest_are_distinct(upstream): + _, script, card = upstream + native = script.parent / "retrieval/index.py" + native.write_text(native.read_text().replace( + "hashlib.sha256(body.encode())", "hashlib.sha256(b'parsed:' + body.encode())" + )) + raw_digest = hashlib.sha256(card.read_bytes()).hexdigest() + native_digest = hashlib.sha256(b"parsed:" + card.read_bytes()).hexdigest() + payload = search_payload(card) + payload["results"][0]["card_sha256"] = native_digest + set_source(script, json.dumps(payload).encode()) + hit = project.search("query")["items"][0] + assert hit["source_card_sha256"] == native_digest + assert hit["card_sha256"] == raw_digest != native_digest + assert project.read(hit["locator"], expected_sha256=raw_digest)["card_sha256"] == raw_digest + card.write_text("different indexed content") + with pytest.raises(project.ChannelFailure) as error: + project.search("query") + assert error.value.state == "index_stale" + + +@pytest.mark.parametrize("backend", [[], ["--backend", "ascend"]]) +def test_outside_git_project_reports_not_enabled(tmp_path, monkeypatch, backend): + monkeypatch.setenv("PYTHONPATH", str(REPO / "src")) + result = cli("find", "tiling", *backend, "--json", cwd=tmp_path) + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["items"] + assert payload["external"][0]["state"] == "not_enabled" + assert "Git project" in payload["external"][0]["fix"] + + +def test_platform_override_and_cuda_gate(upstream): + _, script, card = upstream + script.write_text( + "import json, sys\n" + "if sys.argv[1] == 'search':\n" + " assert sys.argv[sys.argv.index('--platform') + 1] == 'a3'\n" + f"print(json.dumps({discovery()!r} if sys.argv[1] == 'discover' " + f"else {search_payload(card)!r}))\n" + ) + result = cli("find", "synthetic", "--platform", "a3", "--json") + assert json.loads(result.stdout)["external"][0]["state"] == "ok" + script.write_text("raise RuntimeError('must not be called')\n") + for args in (("--backend", "cuda"), ("--no-external",)): + result = cli("find", "tiling", *args, "--json") + assert "external" not in json.loads(result.stdout) + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX process-group cleanup") +@pytest.mark.parametrize("phase", ["discover", "search"]) +def test_cli_deadline_kills_upstream_descendants(upstream, phase): + source, script, _ = upstream + script.write_text( + "import json, os, subprocess, sys, time\nfrom pathlib import Path\n" + f"if sys.argv[1] == {phase!r}:\n" + " child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])\n" + " Path('pids.json').write_text(json.dumps([os.getpid(), child.pid]))\n" + " time.sleep(30)\n" + f"print(json.dumps({discovery()!r}))\n" + ) + started = time.monotonic() + result = cli("find", "tiling", "--external-timeout", "0.8", "--json", timeout=5) + assert result.returncode == 0, result.stderr + assert time.monotonic() - started < 4 + entry = json.loads(result.stdout)["external"][0] + assert entry["state"] == "timeout" + assert entry["elapsed_ms"] >= 700 + assert "items" not in entry + pids = json.loads((source / "pids.json").read_text()) + for pid in pids: + # Linux may briefly retain a reparented zombie; it cannot execute work. + state = subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], + capture_output=True, text=True, check=False).stdout.strip() + assert not state or state.startswith("Z"), (pid, state) + + +def test_cli_larger_budget_reaches_subprocess(upstream): + _, script, card = upstream + script.write_text( + "import json, sys, time\ntime.sleep(0.4)\n" + f"print(json.dumps({discovery()!r} if sys.argv[1] == 'discover' " + f"else {search_payload(card)!r}))\n" + ) + short = cli("find", "tiling", "--external-timeout", "0.6", "--json") + assert json.loads(short.stdout)["external"][0]["state"] == "timeout" + long = cli("find", "tiling", "--external-timeout", "2", "--json") + assert json.loads(long.stdout)["external"][0]["state"] == "ok" + + +@pytest.mark.parametrize("count", [1, 3]) +def test_nonreturning_adapter_cannot_hold_cli_exit(count): + code = f''' +import json, threading, time +from inferrsi.knowledge.channels import ChannelRegistry, search_channels +class Hung: + name = "hung" + backend = "ascend" + def describe(self, *, request): + threading.Event().wait() +started = time.monotonic() +results = search_channels(ChannelRegistry([Hung() for _ in range({count})]), + "q", backend="ascend", timeout=0.1) +print(json.dumps({{"elapsed": time.monotonic() - started, + "states": [r.status.state for r in results], + "times": [r.elapsed_ms for r in results]}})) +''' + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, + timeout=3, check=True) + data = json.loads(result.stdout) + assert data["states"] == ["timeout"] * count + assert data["elapsed"] < 0.25 + assert all(t >= 80 for t in data["times"]) diff --git a/tests/test_cannbot_channel_acceptance.py b/tests/test_cannbot_channel_acceptance.py index 42eab44c..6020c3e5 100644 --- a/tests/test_cannbot_channel_acceptance.py +++ b/tests/test_cannbot_channel_acceptance.py @@ -7,11 +7,10 @@ would be worse than no number. """ +import functools import json -import shutil import subprocess import sys -from importlib.resources import files from pathlib import Path import pytest @@ -37,6 +36,7 @@ def _run(*args): return json.loads(result.stdout) +@functools.lru_cache(maxsize=1) def _approved(): info = _run("cannbot", "describe") return bool(info) and info.get("state") == "ok" @@ -48,8 +48,8 @@ def test_gate_off_matches_the_pre_channel_behaviour(): An unflagged run is *supposed* to differ: it adds the gate explanation. The invariant is only about `--no-external`, and it must be exact. """ - from inferrsi.knowledge.knowledge_find import find, load_corpus from inferrsi.apps.cli import _channels + from inferrsi.knowledge.knowledge_find import find, load_corpus corpus = load_corpus() pre_channel = find("AscendC tiling", corpus, limit=5) @@ -145,8 +145,10 @@ def test_every_query_reaches_more_than_the_local_corpus(): def test_read_returns_one_pinned_card(): result = _run("find", "CeilDivision", "--backend", "ascend") entry = next(item for item in result["external"] if item["channel"] == "cannbot-knowledge") - locator = entry["items"][0]["locator"] - card = _run("cannbot", "read", locator) + hit = entry["items"][0] + locator = hit["locator"] + card = _run("cannbot", "read", locator, "--sha256", hit["card_sha256"]) + assert card["card_sha256"] == hit["card_sha256"] assert card["state"] == "ok" assert card["locator"] == locator assert card["text"].strip() diff --git a/tests/test_knowledge_channels.py b/tests/test_knowledge_channels.py index eb2b4271..7b6190e9 100644 --- a/tests/test_knowledge_channels.py +++ b/tests/test_knowledge_channels.py @@ -1,9 +1,5 @@ """External channels: gating, per-channel states, and no mixing into `items`.""" -import json -import sys -from pathlib import Path - import pytest from inferrsi.knowledge.channels import ( @@ -38,12 +34,12 @@ def __init__(self, *, name="fake", backend="ascend", status=None, items=(), erro self.platform_seen = None self.limit_seen = None - def describe(self): + def describe(self, *, request): if self._error: raise RuntimeError(self._error) return self._status - def search(self, query, *, platform, limit): + def search(self, query, *, platform, limit, request): self.searched += 1 self.platform_seen = platform self.limit_seen = limit @@ -123,7 +119,7 @@ def test_hit_carries_score_scale_and_sha(corpus): hit = result["external"][0]["items"][0] assert hit["score_scale"] == "fake/v1" assert hit["card_sha256"] == "ab" * 32 - assert hit["open"] == "fake read ops/x/card" + assert hit["open"] == "fake read ops/x/card --sha256 " + "ab" * 32 def test_failure_is_a_state_not_empty_list(corpus): @@ -211,3 +207,26 @@ def test_registry_declared_backend_is_what_the_task_asked_for(): assert undeclared.for_backend(undeclared.declared_backend) == [] declared = ChannelRegistry([channel], declared_backend="ascend") assert declared.for_backend(declared.declared_backend) == [channel] + + +def test_cleanup_failure_is_visible_and_does_not_hide_other_channels(): + cleaned = [] + + class FailedCleanup(FakeChannel): + def describe(self, *, request): + def fail(_): + raise OSError("synthetic cleanup failure") + + request.start(object, fail) + request.start(object, lambda _: cleaned.append(True)) + return self._status + + results = search_channels( + ChannelRegistry([FailedCleanup(name="bad"), FakeChannel(name="good", items=[_hit()])]), + "q", backend="ascend", + ) + assert results[0].status.state == "rejected" + assert "cleanup failed" in results[0].status.detail + assert cleaned == [True] + assert results[1].status.state == "ok" + assert results[1].items From 3ce2137d6e2e0bacd96bdace6950594e34a9d214 Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Wed, 18 Nov 2026 09:46:04 +0800 Subject: [PATCH 4/5] fix(knowledge): enforce external hit caps before card reads Clamp adapter responses at the public boundary and limit upstream rows before fingerprint verification. Cover ignored explicit and default limits, and check the outside-project fixture precondition. Agent-assisted-by: Codex (author) Signed-off-by: jiangkuaixue123 --- src/inferrsi/apps/cannbot/read_worker.py | 2 ++ src/inferrsi/knowledge/channels.py | 2 ++ tests/test_cannbot_boundaries.py | 31 ++++++++++++++++++++++++ tests/test_knowledge_channels.py | 20 +++++++++++++++ 4 files changed, 55 insertions(+) diff --git a/src/inferrsi/apps/cannbot/read_worker.py b/src/inferrsi/apps/cannbot/read_worker.py index 081359c2..9c45b0e5 100644 --- a/src/inferrsi/apps/cannbot/read_worker.py +++ b/src/inferrsi/apps/cannbot/read_worker.py @@ -231,6 +231,8 @@ def search(upstream, arguments, *, timeout=60): rows = payload.get("results") if not isinstance(rows, list): _fail("cannbot search needs a results list") + # Bound card verification too, rather than trimming only the final output. + rows = rows[:arguments["limit"]] hits = [] if rows: fingerprint, split_frontmatter = _source_fingerprint(upstream) diff --git a/src/inferrsi/knowledge/channels.py b/src/inferrsi/knowledge/channels.py index 66aad923..e93b55a0 100644 --- a/src/inferrsi/knowledge/channels.py +++ b/src/inferrsi/knowledge/channels.py @@ -275,6 +275,8 @@ def _run_one( phase = "search" result = channel.search(query, platform=platform, limit=limit, request=request) request.remaining() + # The public cap must hold even when an adapter ignores its limit. + result = replace(result, items=result.items[:limit]) except TimeoutError: result = _empty(channel.name, TIMEOUT, detail="external channel deadline expired") except Exception as exc: diff --git a/tests/test_cannbot_boundaries.py b/tests/test_cannbot_boundaries.py index d363fb49..4028db66 100644 --- a/tests/test_cannbot_boundaries.py +++ b/tests/test_cannbot_boundaries.py @@ -98,6 +98,34 @@ def test_search_shape_is_rejected(upstream, mutation): assert error.value.state == "rejected" +@pytest.mark.parametrize("args", [ + ("cannbot", "search", "tiling", "--limit", "1"), + ("find", "tiling", "--external-limit", "1", "--json"), +]) +def test_source_cap_precedes_card_reads(upstream, args): + _, script, card = upstream + payload = search_payload(card) + first = payload["results"][0] + # These extra rows are well-shaped, but their files do not exist. Reading + # either of them would reject the query instead of returning the first hit. + payload["results"].extend([ + {**first, "path": "ops/not-requested-a"}, + {**first, "path": "ops/not-requested-b"}, + ]) + script.write_text( + "import json, sys\n" + f"print(json.dumps({discovery()!r} if sys.argv[1] == 'discover' " + f"else {payload!r}))\n" + ) + result = cli(*args) + assert result.returncode == 0, result.stderr + response = json.loads(result.stdout) + if args[0] == "find": + response = response["external"][0] + assert response["state"] == "ok" + assert [hit["locator"] for hit in response["items"]] == ["ops/card"] + + def test_unknown_schema_is_version_mismatch(upstream): _, script, _ = upstream set_source(script, json.dumps({**discovery(), "schema_version": "future"}).encode()) @@ -176,6 +204,9 @@ def test_native_fingerprint_and_raw_byte_digest_are_distinct(upstream): @pytest.mark.parametrize("backend", [[], ["--backend", "ascend"]]) def test_outside_git_project_reports_not_enabled(tmp_path, monkeypatch, backend): + root = tmp_path.resolve() + if any((parent / ".git").exists() for parent in (root, *root.parents)): + pytest.skip("temporary directory is inside a Git project") monkeypatch.setenv("PYTHONPATH", str(REPO / "src")) result = cli("find", "tiling", *backend, "--json", cwd=tmp_path) assert result.returncode == 0, result.stderr diff --git a/tests/test_knowledge_channels.py b/tests/test_knowledge_channels.py index 7b6190e9..5055843b 100644 --- a/tests/test_knowledge_channels.py +++ b/tests/test_knowledge_channels.py @@ -168,6 +168,26 @@ def test_explicit_limit_overrides_per_channel_default(): assert channel.limit_seen == 3 +@pytest.mark.parametrize("name,limit,expected", [ + ("fake", 1, 1), ("cannbot-knowledge", None, 100), ("mindie-agent", None, 5), +]) +def test_public_cap_holds_when_adapter_ignores_it(corpus, name, limit, expected): + hits = [_hit(f"ops/x/card-{i}") for i in range(expected + 2)] + channel = FakeChannel(name=name, items=hits) + local = find("tiling", corpus) + result = find( + "tiling", corpus, backend="ascend", registry=ChannelRegistry([channel]), + external_limit=limit, + ) + assert result["items"] == local["items"] + entry = result["external"][0] + assert entry["state"] == OK + assert entry["count"] == expected + assert [hit["locator"] for hit in entry["items"]] == [ + hit.locator for hit in hits[:expected] + ] + + def test_each_channel_gets_its_own_default_limit(): cannbot = FakeChannel(name="cannbot-knowledge") mindie = FakeChannel(name="mindie-agent") From edeb3869cf79463348848eec90806e994aa4ff1e Mon Sep 17 00:00:00 2001 From: jiangkuaixue123 Date: Wed, 18 Nov 2026 10:19:36 +0800 Subject: [PATCH 5/5] fix(knowledge): bind card reads to checkout revisions Carry revision and raw-byte digest through generated reads and pagination. Reject observed checkout drift before delivering content while keeping direct current reads explicit. Agent-assisted-by: Codex (implementation) Agent-assisted-by: Codex (independent internal review) Signed-off-by: jiangkuaixue123 --- docs/external-channels.md | 20 ++++--- src/inferrsi/apps/cannbot/__init__.py | 4 +- src/inferrsi/apps/cannbot/project.py | 8 +++ src/inferrsi/apps/cannbot/read_worker.py | 32 ++++++++++- src/inferrsi/apps/cli/__init__.py | 1 + src/inferrsi/apps/cli/arguments.py | 3 + src/inferrsi/knowledge/channels.py | 12 +++- tests/test_cannbot_boundaries.py | 73 +++++++++++++++++++++++- tests/test_cannbot_channel_acceptance.py | 4 +- tests/test_knowledge_channels.py | 4 +- 10 files changed, 145 insertions(+), 16 deletions(-) diff --git a/docs/external-channels.md b/docs/external-channels.md index 734df4b7..6db248f5 100644 --- a/docs/external-channels.md +++ b/docs/external-channels.md @@ -41,7 +41,8 @@ a wheel, or `evidence/`. ## Setting one up -You supply a checkout, an interpreter, and the backend it serves: +You supply a checkout, an interpreter, and the backend it serves. The `git` +executable must be available for read-time revision checks: ```bash git clone https://gitcode.com/cann/cannbot-knowledge.git ~/src/cannbot-knowledge @@ -102,16 +103,21 @@ inferrsi cannbot search "CeilDivision 向上取整" --platform a2 --limit 100 inferrsi cannbot read ops/ascendc/apis/utils_api/cpp_stdlib/math_functions/ceil_division ``` -The generated `open` command includes `--sha256 `. Use that command -and retain the same digest with `--page 2` (and later pages): a changed card is -rejected. Every successful read returns the digest of the full card bytes. +The generated `open` command includes `--revision ` and +`--sha256 `. Retain both with `--page 2` and later pages. A changed +checkout revision returns `version_mismatch`, even if the card bytes did not +change; changed bytes are rejected even if HEAD did not change. The worker +checks HEAD before and after reading, within the read timeout, and refuses an +observed revision change. This is a drift check on a mutable checkout, not an +atomic snapshot or a service for retrieving old versions. Every successful read +returns the observed checkout revision and digest of the full card bytes. The upstream CLI uses `card_sha256` for a parsed metadata/body fingerprint; this adapter preserves it as `source_card_sha256`. Before emitting a selected hit, it verifies the source fingerprint using the approved checkout's parser and computes `card_sha256` over those same raw bytes. It returns neither the bytes nor private filesystem paths in search results. -A locator-only read returns the current card and its digest, without promising -continuity with an earlier search or page. A card digest does not validate its +A locator-only read returns the current card, revision and digest, without +promising continuity with an earlier search or page. A card digest does not validate its cited resources or make it eligible as campaign evidence. `find --platform` overrides the grant's upstream platform. If neither supplies @@ -151,7 +157,7 @@ silent. "status": "stable", "platforms": ["a2", "a3"], "card_sha256": "", "source_card_sha256": "", - "open": "inferrsi cannbot read ops/... --sha256 "}] + "open": "inferrsi cannbot read ops/... --sha256 --revision "}] }] ``` diff --git a/src/inferrsi/apps/cannbot/__init__.py b/src/inferrsi/apps/cannbot/__init__.py index ee6b6a5e..28f61ace 100644 --- a/src/inferrsi/apps/cannbot/__init__.py +++ b/src/inferrsi/apps/cannbot/__init__.py @@ -97,9 +97,11 @@ def search( read_prefix="inferrsi cannbot read", ) - def read(self, locator: str, *, expected_sha256: str | None = None) -> str: + def read(self, locator: str, *, expected_sha256: str | None = None, + expected_revision: str | None = None) -> str: payload = self._project.read( locator, expected_sha256=expected_sha256, root=self.root, + expected_revision=expected_revision, timeout=max(self.timeout, 15.0), ) return str(payload.get("text") or "") diff --git a/src/inferrsi/apps/cannbot/project.py b/src/inferrsi/apps/cannbot/project.py index 3d0b572b..7386f049 100644 --- a/src/inferrsi/apps/cannbot/project.py +++ b/src/inferrsi/apps/cannbot/project.py @@ -208,6 +208,7 @@ def search( def read(locator: str, *, page: int = 1, expected_sha256: str | None = None, + expected_revision: str | None = None, root: Path | None = None, timeout: float = 15.0) -> dict: if not isinstance(locator, str) or not locator.strip(): raise ChannelFailure("cannbot read needs a locator from a search result") @@ -217,9 +218,16 @@ def read(locator: str, *, page: int = 1, expected_sha256: str | None = None, not isinstance(expected_sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256) ): raise ChannelFailure("cannbot expected_sha256 must be a lowercase 64-hex digest") + if expected_revision is not None and ( + not isinstance(expected_revision, str) + or not re.fullmatch(r"[0-9a-f]{40}", expected_revision) + ): + raise ChannelFailure("cannbot expected_revision must be a full lowercase Git revision") arguments = {"locator": locator, "page": page} if expected_sha256: arguments["expected_sha256"] = expected_sha256 + if expected_revision: + arguments["expected_revision"] = expected_revision return _run(load_grant(root), "read", arguments, timeout=timeout) diff --git a/src/inferrsi/apps/cannbot/read_worker.py b/src/inferrsi/apps/cannbot/read_worker.py index 9c45b0e5..0ff4055d 100644 --- a/src/inferrsi/apps/cannbot/read_worker.py +++ b/src/inferrsi/apps/cannbot/read_worker.py @@ -286,9 +286,36 @@ def search(upstream, arguments, *, timeout=60): } -def read(upstream, arguments): +def _revision(upstream, *, timeout): + # Inherited repository-selection variables must not redirect this check. + environment = {key: value for key, value in os.environ.items() + if not key.startswith("GIT_")} + try: + result = capture( + ["git", "-C", str(upstream), "rev-parse", "--verify", "HEAD"], + cwd=str(upstream), timeout=timeout, environment=environment, + ) + except OutputLimitError: + _fail("cannbot revision output exceeded 2 MiB") + revision = result.stdout.decode("ascii", "replace").strip() + if result.returncode or not re.fullmatch(r"[0-9a-f]{40}", revision): + _fail("Cannot determine the approved cannbot checkout revision", + state="version_mismatch") + return revision + + +def read(upstream, arguments, *, timeout=15): + deadline = time.monotonic() + timeout + revision = _revision(upstream, timeout=max(0, deadline - time.monotonic())) + expected_revision = arguments.get("expected_revision") + if expected_revision and revision != expected_revision: + _fail("The cannbot revision changed since the search; search again", + state="version_mismatch") locator = arguments["locator"] body = _card_bytes(upstream, locator) + if _revision(upstream, timeout=max(0, deadline - time.monotonic())) != revision: + _fail("The cannbot revision changed during the read; search again", + state="version_mismatch") digest = hashlib.sha256(body).hexdigest() # Search reports what it matched; read reports what it actually returned. # Without both, a card edited between the two is served under the old @@ -308,6 +335,7 @@ def read(upstream, arguments): return { "state": "ok", "locator": locator, + "revision": revision, "page": page, "pages": max(1, (len(lines) + size - 1) // size), "line_count": len(lines), @@ -328,7 +356,7 @@ def read(upstream, arguments): elif action == "search": result = search(upstream, arguments, timeout=timeout) elif action == "read": - result = read(upstream, arguments) + result = read(upstream, arguments, timeout=timeout) else: _fail("Unknown cannbot read action") except SystemExit: diff --git a/src/inferrsi/apps/cli/__init__.py b/src/inferrsi/apps/cli/__init__.py index 28e6ed48..b8df0fc3 100644 --- a/src/inferrsi/apps/cli/__init__.py +++ b/src/inferrsi/apps/cli/__init__.py @@ -317,6 +317,7 @@ def main(argv: list[str] | None = None) -> int: else: result = cannbot.project.read( args.locator, page=args.page, expected_sha256=args.sha256, + expected_revision=args.revision, ) _print(result) return 0 diff --git a/src/inferrsi/apps/cli/arguments.py b/src/inferrsi/apps/cli/arguments.py index fba62d18..5d24e528 100644 --- a/src/inferrsi/apps/cli/arguments.py +++ b/src/inferrsi/apps/cli/arguments.py @@ -154,6 +154,9 @@ def parser() -> argparse.ArgumentParser: "card changed since then", ) cannbot_read.add_argument("--page", type=int, default=1) + cannbot_read.add_argument( + "--revision", help="Expected full checkout revision from a search result", + ) agents = commands.add_parser("agents", help="Record native host fresh-session handoffs") agent_actions = agents.add_subparsers(dest="agent_action", required=True) prepare = agent_actions.add_parser("prepare", help="Freeze a task for the current host") diff --git a/src/inferrsi/knowledge/channels.py b/src/inferrsi/knowledge/channels.py index e93b55a0..8489b060 100644 --- a/src/inferrsi/knowledge/channels.py +++ b/src/inferrsi/knowledge/channels.py @@ -101,7 +101,7 @@ class ChannelHit: card_sha256: str = "" source_card_sha256: str = "" - def view(self, *, read_prefix: str) -> dict: + def view(self, *, read_prefix: str, revision: str | None = None) -> dict: record: dict[str, Any] = { "locator": self.locator, "title": self.title, @@ -122,6 +122,8 @@ def view(self, *, read_prefix: str) -> dict: argv = [*shlex.split(read_prefix), self.locator] if self.card_sha256: argv.extend(["--sha256", self.card_sha256]) + if revision: + argv.extend(["--revision", revision]) record["open"] = shlex.join(argv) return record @@ -144,7 +146,10 @@ def view(self) -> dict: record["score_scale"] = self.score_scale record["count"] = len(self.items) if self.items: - record["items"] = [item.view(read_prefix=self.read_prefix) for item in self.items] + record["items"] = [ + item.view(read_prefix=self.read_prefix, revision=self.status.revision) + for item in self.items + ] return record @@ -166,7 +171,8 @@ def search( self, query: str, *, platform: str | None, limit: int, request: ChannelRequest ) -> ChannelResult: ... - def read(self, locator: str, *, expected_sha256: str | None = None) -> str: ... + def read(self, locator: str, *, expected_sha256: str | None = None, + expected_revision: str | None = None) -> str: ... @dataclass diff --git a/tests/test_cannbot_boundaries.py b/tests/test_cannbot_boundaries.py index 4028db66..bf0648ee 100644 --- a/tests/test_cannbot_boundaries.py +++ b/tests/test_cannbot_boundaries.py @@ -30,6 +30,17 @@ def discovery(): "index": {"built_repository_state": {"commit": REVISION}}} +def commit_revision(source): + subprocess.run( + ["git", "-C", str(source), "-c", "user.name=Synthetic Fixture", + "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", + "commit", "--allow-empty", "-qm", "synthetic revision"], check=True, + ) + return subprocess.check_output( + ["git", "-C", str(source), "rev-parse", "HEAD"], text=True, + ).strip() + + @pytest.fixture def upstream(tmp_path, monkeypatch): home = tmp_path / "home" @@ -42,7 +53,8 @@ def upstream(tmp_path, monkeypatch): (root / ".git").mkdir() monkeypatch.chdir(root) source = tmp_path / "upstream" - (source / ".git").mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(source)], check=True) + monkeypatch.setattr(sys.modules[__name__], "REVISION", commit_revision(source)) (source / "knowledge/ops").mkdir(parents=True) card = source / "knowledge/ops/card.md" card.write_text("\n".join(f"synthetic line {i}" for i in range(401))) @@ -160,10 +172,12 @@ def test_read_binds_all_pages_to_search_digest(upstream): hit = json.loads(result.stdout)["external"][0]["items"][0] opened = cli(*shlex.split(hit["open"])[1:]) body = json.loads(opened.stdout) + assert body["revision"] == REVISION assert body["card_sha256"] == hit["card_sha256"] assert body["has_more"] page = cli(*shlex.split(hit["open"])[1:], "--page", "2") assert json.loads(page.stdout)["card_sha256"] == hit["card_sha256"] + assert json.loads(page.stdout)["revision"] == REVISION card.write_text("changed") changed = cli(*shlex.split(hit["open"])[1:], "--page", "2") assert changed.returncode != 0 @@ -181,6 +195,63 @@ def test_read_refuses_path_escape(upstream): assert "not knowledge" not in result.stdout + result.stderr +@pytest.mark.parametrize("page", [1, 2]) +def test_read_rejects_revision_drift_with_unchanged_card(upstream, page): + source, script, card = upstream + script.write_text( + "import json, sys\n" + f"print(json.dumps({discovery()!r} if sys.argv[1] == 'discover' " + f"else {search_payload(card)!r}))\n" + ) + found = json.loads(cli("find", "synthetic", "--json").stdout)["external"][0] + hit = found["items"][0] + assert f"--revision {found['revision']}" in hit["open"] + changed_revision = commit_revision(source) + assert changed_revision != found["revision"] + result = cli(*shlex.split(hit["open"])[1:], "--page", str(page)) + assert result.returncode != 0 + assert "revision changed since the search" in result.stderr + assert "synthetic line" not in result.stdout + result.stderr + current = project.read(hit["locator"], page=page) + assert current["revision"] == changed_revision + assert current["card_sha256"] == hit["card_sha256"] + + +def test_read_rejects_revision_change_during_read(upstream, monkeypatch, capsys): + from inferrsi.apps.cannbot import read_worker + + source, _, _ = upstream + original = read_worker._card_bytes + + def changing_card(root, locator): + raw = original(root, locator) + commit_revision(source) + return raw + + monkeypatch.setattr(read_worker, "_card_bytes", changing_card) + with pytest.raises(SystemExit): + read_worker.read(source, {"locator": "ops/card", "expected_revision": REVISION}) + result = json.loads(capsys.readouterr().out) + assert result["state"] == "version_mismatch" + assert "text" not in result + + +def test_read_uses_approved_checkout_not_inherited_git_directory(upstream, monkeypatch): + source, _, _ = upstream + monkeypatch.setenv("GIT_DIR", str(source / "nonexistent-git-directory")) + assert project.read("ops/card", expected_revision=REVISION)["revision"] == REVISION + + +@pytest.mark.parametrize("revision", ["", "HEAD", "abc", "A" * 40, "a" * 39, 42]) +def test_read_rejects_invalid_expected_revision_before_worker(monkeypatch, revision): + def unexpected(*args, **kwargs): + pytest.fail("invalid revision started a worker") + + monkeypatch.setattr(project, "_run", unexpected) + with pytest.raises(project.ChannelFailure, match="expected_revision"): + project.read("ops/card", expected_revision=revision) + + def test_native_fingerprint_and_raw_byte_digest_are_distinct(upstream): _, script, card = upstream native = script.parent / "retrieval/index.py" diff --git a/tests/test_cannbot_channel_acceptance.py b/tests/test_cannbot_channel_acceptance.py index 6020c3e5..9f46e6d5 100644 --- a/tests/test_cannbot_channel_acceptance.py +++ b/tests/test_cannbot_channel_acceptance.py @@ -147,8 +147,10 @@ def test_read_returns_one_pinned_card(): entry = next(item for item in result["external"] if item["channel"] == "cannbot-knowledge") hit = entry["items"][0] locator = hit["locator"] - card = _run("cannbot", "read", locator, "--sha256", hit["card_sha256"]) + card = _run("cannbot", "read", locator, "--sha256", hit["card_sha256"], + "--revision", entry["revision"]) assert card["card_sha256"] == hit["card_sha256"] + assert card["revision"] == entry["revision"] assert card["state"] == "ok" assert card["locator"] == locator assert card["text"].strip() diff --git a/tests/test_knowledge_channels.py b/tests/test_knowledge_channels.py index 5055843b..f039b85f 100644 --- a/tests/test_knowledge_channels.py +++ b/tests/test_knowledge_channels.py @@ -119,7 +119,9 @@ def test_hit_carries_score_scale_and_sha(corpus): hit = result["external"][0]["items"][0] assert hit["score_scale"] == "fake/v1" assert hit["card_sha256"] == "ab" * 32 - assert hit["open"] == "fake read ops/x/card --sha256 " + "ab" * 32 + assert hit["open"] == ( + "fake read ops/x/card --sha256 " + "ab" * 32 + " --revision " + "deadbeef" * 5 + ) def test_failure_is_a_state_not_empty_list(corpus):