feat(twitter): 支持 x-monkey 归档后端与 JWT 续签

.changes/+twitter-archive.added.md

@@ -0,0 +1 @@
+新增可配置的 Peer JWT 签发默认期限并移除 admission 的强制最大寿命；Host SDK 0.3.1 支持扩展在既有 Graph 事务中批量定位原生身份并串行合并。

.env.example

@@ -6,6 +6,9 @@ PEER_NAME=core-py-development
 
 # JWT Authentication (Required)
 JWT_SECRET=replace-with-at-least-32-random-bytes
+# Signing defaults in seconds; neither value limits admission of a valid JWT.
+JWT_DEFAULT_LIFETIME_SECONDS=60
+TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS=604800
 
 # Database lifecycle authority. Compose uses the PostgreSQL owner only in its
 # one-shot init service and gives the application a separate runtime URL.

app/database_contract/constants.py

@@ -4,7 +4,7 @@
 
 
 CONTRACT_FORMAT = 1
-CONTRACT_REVISION = "extension-peer-participation-v2"
+CONTRACT_REVISION = "peer-jwt-signing-defaults-v1"
 
 PROTOCOL_SCHEMA = "inkcre"
 INTERNAL_SCHEMA = "inkcre_internal"
@@ -18,6 +18,7 @@
 JWT_ROLE = AUTHENTICATED_ROLE
 JWT_ISSUER = "inkcre-peer"
 JWT_AUDIENCE = "inkcre-api"
+# Frozen input for admitted historical migrations; live admission has no lifetime cap.
 JWT_MAX_LIFETIME_SECONDS = 24 * 60 * 60
 JWT_MINIMUM_SECRET_BYTES = 32
 

app/database_contract/lifecycle.py

@@ -14,7 +14,6 @@
   JWT_ALGORITHM,
   JWT_AUDIENCE,
   JWT_ISSUER,
-  JWT_MAX_LIFETIME_SECONDS,
   JWT_ROLE,
   RESET_CONFIRMATION,
 )
@@ -79,7 +78,6 @@ def contract_document() -> dict[str, object]:
       "issuer": JWT_ISSUER,
       "audience": JWT_AUDIENCE,
       "required_claims": ["role", "iss", "aud", "iat", "exp"],
-      "maximum_lifetime_seconds": JWT_MAX_LIFETIME_SECONDS,
     },
     "profiles": ["runtime", "development"],
     "protocol": protocol_document(),

app/middleware.py

@@ -1,5 +1,6 @@
 """Middleware for logging and request tracking."""
 
+import math
 import time
 from typing import Callable
 import uuid
@@ -13,7 +14,6 @@
   JWT_ALGORITHM,
   JWT_AUDIENCE,
   JWT_ISSUER,
-  JWT_MAX_LIFETIME_SECONDS,
   JWT_ROLE,
 )
 from app.settings import settings
@@ -63,15 +63,19 @@ def decode_peer_jwt(
   )
   if not valid_numeric_dates:
     raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch")
-  issued_at_number = float(issued_at)
-  expires_at_number = float(expires_at)
+  try:
+    issued_at_number = float(issued_at)
+    expires_at_number = float(expires_at)
+  except OverflowError as error:
+    raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") from error
   current_epoch = time.time() if now is None else now
   if (
     claims["role"] != JWT_ROLE
+    or not math.isfinite(issued_at_number)
+    or not math.isfinite(expires_at_number)
     or issued_at_number > current_epoch + 60
     or expires_at_number <= current_epoch
     or expires_at_number <= issued_at_number
-    or expires_at_number - issued_at_number > JWT_MAX_LIFETIME_SECONDS
   ):
     raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch")
   return claims
@@ -81,11 +85,13 @@ def create_peer_jwt(
   secret: str,
   *,
   now: float | None = None,
-  lifetime_seconds: int = 60,
+  lifetime_seconds: int | None = None,
 ) -> str:
-  """Create one short-lived token under the canonical Peer HTTP contract."""
-  if lifetime_seconds <= 0 or lifetime_seconds > JWT_MAX_LIFETIME_SECONDS:
-    raise ValueError("Peer JWT lifetime is outside the contract bound")
+  """Sign a Peer JWT using the configured default when no lifetime is supplied."""
+  if lifetime_seconds is None:
+    lifetime_seconds = settings.jwt_default_lifetime_seconds
+  if lifetime_seconds <= 0:
+    raise ValueError("Peer JWT lifetime must be positive")
   issued_at = int(time.time() if now is None else now)
   return jwt.encode(
     {

app/persistence/info_base/repository.py

@@ -172,6 +172,41 @@ async def find_json_field(
       statement = statement.where(sqlmodel.col(BlockModel.id).in_(tuple(block_ids)))
     return tuple(await self._session.scalars(statement))
 
+  async def find_json_field_many(
+    self,
+    resolvers: typing.Collection[str],
+    field: str,
+    values: typing.Collection[str],
+  ) -> tuple[BlockModel, ...]:
+    """Find inline JSON records in a bounded native-identity set."""
+    from sqlalchemy.dialects.postgresql import JSONB
+
+    if not resolvers or not values:
+      return ()
+    return tuple(
+      await self._session.scalars(
+        sqlmodel.select(BlockModel)
+        .where(
+          sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)),
+          sqlalchemy.case(
+            (
+              sqlalchemy.func.pg_input_is_valid(BlockModel.content, "jsonb"),
+              sqlalchemy.cast(BlockModel.content, JSONB)[field].astext,
+            )
+          ).in_(tuple(values)),
+          sqlmodel.col(BlockModel.storage).is_(None),
+        )
+        .order_by(sqlmodel.col(BlockModel.id))
+      )
+    )
+
+  async def acquire_transaction_lock(self, namespace: str) -> None:
+    """Serialize an owning use case until its surrounding transaction ends."""
+    await self._session.execute(
+      sqlalchemy.text("SELECT pg_advisory_xact_lock(hashtextextended(:name, 0))"),
+      {"name": namespace},
+    )
+
   async def get_related(
     self, block_id: BlockID, *, content: str, outgoing: bool = True
   ) -> BlockModel | None:

app/settings.py

@@ -52,6 +52,16 @@ class Settings(BaseSettings):
     min_length=JWT_MINIMUM_SECRET_BYTES,
     description="Secret key for JWT token signing and verification",
   )
+  jwt_default_lifetime_seconds: int = Field(
+    default=60,
+    gt=0,
+    description="Default Peer JWT signing lifetime; not an admission limit",
+  )
+  twitter_x_monkey_jwt_lifetime_seconds: int = Field(
+    default=7 * 24 * 60 * 60,
+    gt=0,
+    description="Signing lifetime for the Twitter x-monkey session and its renewal",
+  )
 
   # Observability settings
   obsrv: ObsrvSetting = Field(default_factory=ObsrvSetting)

app/version.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import tomllib
 
-CORE_VERSION = "0.3.0"
+CORE_VERSION = "0.3.1"
 APPLICATION_VERSION = tomllib.loads(
   (Path(__file__).parents[1] / "pyproject.toml").read_text(encoding="utf-8")
 )["project"]["version"]

docker-compose.yml

@@ -56,6 +56,8 @@ services:
       DATABASE_URL: "postgresql+psycopg://inkcre_core:${CORE_DATABASE_PASSWORD:-local-core-database-password-at-least-32-bytes}@postgres:5432/${POSTGRES_DB:-inkcre}"
       INKCRE_ENV_FILE: ""
       JWT_SECRET: ${JWT_SECRET:-local-development-jwt-secret-at-least-32-bytes}
+      JWT_DEFAULT_LIFETIME_SECONDS: ${JWT_DEFAULT_LIFETIME_SECONDS:-60}
+      TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS: ${TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS:-604800}
       OBSRV__LOGGING_BACKEND: ${OBSRV__LOGGING_BACKEND:-postgresql}
       OBSRV__TELEMETRY_ENABLED: ${OBSRV__TELEMETRY_ENABLED:-false}
       OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}

docs/30-unit-tdd/README.md

@@ -13,6 +13,7 @@ This directory owns expensive internal design truth for logical units delivered
 | [mail-extension.md](mail-extension.md) | Mail identity, MIME materialization, collection, graph, and failure boundaries |
 | [memos-extension.md](memos-extension.md) | Memos adapter identity, graph grammar, persistence, and failure boundaries |
 | [rss-extension.md](rss-extension.md) | RSS adapter identity, collection lifecycle, reconciliation, and materialization |
+| [twitter-extension.md](twitter-extension.md) | Twitter 共同身份、Bookmark 过渡、事务合并和按需媒体读取 |
 | [security-model.md](security-model.md) | Core-py realization of the shared security boundary model |
 
 Do not add a document for a directory inventory, a fact cheaply recovered from code, a one-off decision, or a cross-unit contract already owned by `../_shared/20-product-tdd/`. Put repeated physical-subtree hazards in the nearest `AGENTS.md`; put runtime, rollout, recovery, and environment truth in `../40-deployment/`.

docs/30-unit-tdd/twitter-extension.md

@@ -0,0 +1,37 @@
+# Python Twitter 采集与归档
+
+Python Twitter distribution 把 x-monkey 提交和 Bookmark 采集写入同一组普通 Blocks。共同字段与跨分发关系由 `inkcre/twitter` 的 [Twitter 归档契约](../_shared/20-product-tdd/twitter-archive-contract.md) 拥有，`model.py` 实现独立的 User/Post/Media、身份根与捕获集合。现有 JSON 字段、Resolver ID 和 Block ID 保持不变，数据库没有另一套归档表。
+
+`archive_schema.py` 只保存 x-monkey 0.2.0 的固定协议 DTO；`x_monkey.py` 把已校验的提交转成共同模型，并按固定 DTO 字段集递归投影 query 数据、构造协议状态与汇总。自由的 Article JSON 按值保留。共同模型未来新增的字段不会自动进入旧协议，无法用该协议表达的已知结构返回错误，不改写持久事实。Official/Twikit 适配器在 `api.py` 中直接产出共同模型，`bookmark.py` 与 `resolver.py` 不依赖协议 DTO。当前适配器保留它们明确映射的事实，不宣称保存完整 provider 原始响应或所有未知字段。
+
+`reconcile.py` 拥有本实现的身份选择、事务合并和关系维护，只返回变化集合、Block IDs 及共同读取结果，不包含 HTTP 状态或 wire 响应类型。`TwitterBatch` 没有继承远端请求的条数限制；每类 1000 条的准入边界留在 x-monkey DTO。
+
+一次 submit 使用一个 `graph_uow()`。Bookmark 先完成 provider 请求，再进入 `source_uow()`，锁定 Source，使用其 `graph` 合并新主帖及实际捕获的上下文，并在同一事务内推进 cursor。主帖顺序与展开上下文分开，因此引用帖不会成为下一次书签采集的 watermark；旧的数值 cursor 在比较边界转换为十进制字符串，不重置历史状态。
+
+两条写入路径都先取得 `extensions.twitter.reconcile` 全局事务 advisory 锁，再批量重读原生身份。根内容和受管关系一起提交；异常让整批回滚，调用方可安全重放。该锁仅约束正常 Twitter 写入入口，不能替代所有 info-base 原始写入的约束。provider 和媒体网络访问都不能放进此事务。
+
+原生身份、Blocks 和引用边使用批量查询与创建。不要在每个媒体项内单独查询、创建和 flush：Core 与数据库跨网络部署时，这会把一次正常批次放大为数百次串行往返，并在持锁期间耗尽客户端的同步请求期限。
+
+匹配先选新版 resolver，再按 Block ID 稳定选择旧 `tweet.v1`。旧根重新采集后原地更新，保留现有 Block ID、引用和旧附件关系；未重新采集的旧根仍由 v1 decoder 读取。旧加工正文不足以重建 native RichText，不能用它伪造新记录。共同合并提供 `replace` 与 `supplement` 正文策略：正常捕获合并已提交的正文结构，补充捕获在已有正文时保留它并增加其他事实。Bookmark 使用补充策略，避免较少详情的正文覆盖完整捕获；该选择不作为字段写进持久记录。
+
+身份引用严格保存为 `{id}`，不补作者、时间或正文。query 对完整 native 记录返回 `found`，身份引用返回 `missing`，能按 ID 定位但结构损坏的记录返回 `failed`。批量身份查询使用有效 JSON 的条件表达式隔离普通文本和损坏 JSON；无法解析且无法定位 ID 的内容不会命中某个原生 ID，不会导致其他正常记录的查询失败。
+
+共同合并选择缺省保留、对象递归合并和数组整体替换，保留有序媒体引用。原生观察时间保存输入字符串，不在序列化时改写精度。Twitter 以毫秒分辨率逐计数比较观察时间；较旧观察不覆盖新值，相同值的新时间也更新持久记录，无时间的新值删除该计数的旧时间标签。这些是共同语义的规则，x-monkey 兼容基线用于验证其适配结果。协议 Adapter 将无持久变化映射为 `unchanged`，仅观察时间更新仍为 `accepted`。
+
+新版引用边只维护约定的 reference schema，旧 `attachment:*`、`entities:url` 和用户关系保留。媒体引用顺序是 `post.mediaIds`、文章封面、`article.mediaIds` 依次合并去重，首次出现保留。Media 根只保存原生 JSON；写入不创建 `core.image.v1`、`core.video.v1` 或 Media→`content` 关系。较早版本产生的内容块和边保留，须先核对其他引用及来源，再由明确的数据清理处理，正常重放不会删除它们。
+
+媒体原生 URL、variants 和 alt 按捕获值保留，Python 不筛选或规范化这些字段。保存、query、文本投影和索引只消费元数据；用户显式展开媒体时，消费端 Twitter Distribution 才选择图片地址或最高 bitrate MP4，并交给现有 HTTP Storage。HTTP(S) 地址、重定向、超时与体积限制由通用 Storage 合同处理，不增加 Twitter 专属 CDN、HTTPS/443 或禁用重定向策略。HLS 继续保留为元数据。本版本不保存媒体本体，也不创建下载 Job，外部来源失效后仍可读取文字与元数据。
+
+此边界通过减少持久块实现。Core 的普通图片和视频 resolver 在 lexical 索引中仍可读取字节并生成 OCR/STT；如果 Twitter 提前建立这类内容块，独立的索引扫描会触发读取，即使 Post/Media 自身的投影不展开媒体。Twitter 原生记录因此不预建内容块，其他来源的通用 lexical/OCR/STT 行为保持原有合同。
+
+Python 的 v2 Post 文本投影包含原帖 ID、链接、已捕获作者、正文和长文；User/Media 投影使用自身已经捕获的文字。label 保持 Block-local，不为投影或标签下载媒体。检索沿用现有独立索引维护流程，不在提交中隐式维护索引。
+
+Python Twitter 在 `/twitter/x-monkey/account` 自托管连接页面。用户只在自己信任的 HTTPS 部署输入已有 `JWT_SECRET`，页面使用 WebCrypto HMAC SHA-256 本地签发 60 秒普通 Peer JWT，再以同 Origin Bearer 和 JSON `{}` 请求 `POST /twitter/x-monkey/session`。该路由复用 `require_peer_jwt`，拒绝非 Core Origin、非 JSON 和带 `session_use` 的凭据，返回 `SessionResponse` JSON 并建立归档 Cookie。页面随后 GET `api/v1/session`，确认 Cookie 的 `authenticated=true` 后才显示 `Connected`、Peer 和到期时间。
+
+账户页以随机 nonce CSP 限定脚本，`connect-src 'self'` 限定连接目标。JWT 不进入 URL，密钥不持久化或发送给服务器。页面不加载第三方脚本，不接受任意回跳；浏览器内签名不使恶意或被篡改页面安全。
+
+连接不要求 Web 或 MF 分发。`setup.x_monkey_base_url` 继续供用户复制，Web Setup 只提供通向该 Core 账户页的普通链接，不持有此连接流程的密钥、JWT 签发或凭据传输逻辑。账户页显示 Peer 与到期时间，脚本通过会话检测确认连接。七天 Cookie 默认期限与协议续签仍由 Python Twitter 拥有。
+
+候选验证脚本与 fixture 放在 `scripts/tests/twitter`，默认 pytest 与 CI 不收集。离线合同与 HTTP 边界验证入口为 `pdm run python -m pytest -q scripts/tests/twitter/test_archive_contract.py scripts/tests/twitter/test_archive_http.py`；显式运行整个目录可用 `pdm run python -m pytest -q scripts/tests/twitter`。
+
+真实数据库场景要求显式提供 `INKCRE_TEST_DATABASE_URL`，使用已迁移的可丢弃 PostgreSQL runtime；测试只清理自己创建的精确 IDs，不重置开发库。原生 roundtrip 与真实 Twikit Bookmark 验证执行 `LexicalRetrievalManager.maintain` 的实际扫描、投影和检索链，在 HTTP Storage 字节边界断言零调用；同时检查归档不额外创建内容块、blob 或 Job。固定 fixture 同时用于原生 roundtrip 和独立上游合同验证，真实 userscript、HTTP 会话及 Web 消费仍需要各自的纵向验收。

docs/40-deployment/database-contract.md

@@ -122,13 +122,17 @@ role: authenticated
 issuer: inkcre-peer
 audience: inkcre-api
 required: role, iss, aud, iat, exp
-maximum lifetime: 24 hours
 future iat tolerance: 60 seconds
 ```
 
 PostgREST exposes only `inkcre`，connects as `authenticator`，uses denied `anonymous` fallback，and invokes
 `inkcre_internal.check_jwt` before every admitted request。FastAPI peer middleware validates the same vectors。
 
+JWT 没有强制最大寿命。`JWT_DEFAULT_LIFETIME_SECONDS` 是 Core 签发器的默认值（60 秒），
+`TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS` 是 Twitter x-monkey 会话初次签发及续签的默认值（604800 秒，7 天）；
+二者必须为正整数，不参与 token admission。调整默认值影响后续签发，不会撤销已签发的有效 token。
+普通 Web Peer 仍默认签发 24 小时 token。有效期、签名、必需 claims 和未来 iat 容差仍参与验证。
+
 ## Canonical Production Profile
 
 [`deploy/profiles/production.json`](../../deploy/profiles/production.json) 是 checked-in non-secret discovery surface。
@@ -140,7 +144,7 @@ implementation 或 migration verification 不授权修改 production profile 或
 
 ## Versioning
 
-`db contract --json` reports `peer-database-runtime-v3` plus the image source revision.
+`db contract --json` reports the executable contract revision plus the image source revision.
 Every production candidate also contains `/app/database-contract/`:
 
 - `database-roles.sql` recreates the password-free global principals required by policies;

docs/40-deployment/first-party-extension-distribution.md

@@ -147,7 +147,10 @@ Core Host SDK 0.3 adds `app.http.get_public_http_base_url()` without changing th
 or persistence API. Its authority is `app/version.py`, independently of the Core service package
 version in `pyproject.toml`; preparing a service Release PR does not change the Host SDK version.
 
-GitHub, Learn English, Mail, RSS, Telegram, and Twitter retain their SDK 0.2 lower bound and support
+Core Host SDK 0.3.1 adds batched JSON identity lookup and transaction advisory locking to the
+existing Graph repository. Twitter archive reconciliation requires `>=0.3.1 <0.4.0`.
+
+GitHub, Learn English, Mail, RSS, and Telegram retain their SDK 0.2 lower bound and support
 SDK 0.3 through `>=0.2.0 <0.4.0`. Memos uses the new HTTP address API and requires
 `>=0.3.0 <0.4.0`. Compatibility metadata changes require new immutable Extension
 releases; an old wheel's range must not be widened. Before upgrading a deployment, disable affected

extensions/twitter/.changes/+x-monkey.added.md

@@ -0,0 +1 @@
+支持以现有 JWT 连接 x-monkey，保存与合并用户、帖子和媒体元数据，与 Bookmark 共用原生身份；媒体保持按需读取，会话默认七天并自动续签。

extensions/twitter/__init__.py

@@ -33,9 +33,17 @@ class Extension(
 ):
   @classmethod
   def _init_resolvers(cls):
-    from .resolver import TweetResolver
+    from .resolver import (
+      ArchiveMediaResolver,
+      ArchivePostResolver,
+      ArchiveUserResolver,
+      TweetResolver,
+    )
 
     ResolverManager.register_resolver(TweetResolver)
+    ResolverManager.register_resolver(ArchivePostResolver)
+    ResolverManager.register_resolver(ArchiveUserResolver)
+    ResolverManager.register_resolver(ArchiveMediaResolver)
 
   @classmethod
   def _init_sources(cls):
@@ -71,8 +79,10 @@ async def on_start_async(cls, app):
   @classmethod
   def _register_apis(cls, router: APIRouter):
     from .setup_flow import register_setup_routes
+    from .archive_http import router as archive_http_router
 
     register_setup_routes(router)
+    router.include_router(archive_http_router, prefix="/x-monkey")
 
   @classmethod
   def api_dependencies(cls) -> list[typing.Any]:
@@ -87,7 +97,15 @@ def peer_inbounds(cls) -> tuple[typing.Any, ...]:
 
   @classmethod
   def public_http_routes(cls) -> tuple[PublicHTTPRoute, ...]:
-    return (PublicHTTPRoute(method="GET", path="/auth/callback"),)
+    return (
+      PublicHTTPRoute(method="GET", path="/auth/callback"),
+      PublicHTTPRoute(method="POST", path="/x-monkey/session"),
+      PublicHTTPRoute(method="GET", path="/x-monkey/account"),
+      PublicHTTPRoute(method="POST", path="/x-monkey/disconnect"),
+      PublicHTTPRoute(method="GET", path="/x-monkey/api/v1/session"),
+      PublicHTTPRoute(method="POST", path="/x-monkey/api/v1/archive/query"),
+      PublicHTTPRoute(method="POST", path="/x-monkey/api/v1/archive/submit"),
+    )
 
   @classmethod
   async def update_config_async(