diff --git a/.changes/+twitter-archive.added.md b/.changes/+twitter-archive.added.md new file mode 100644 index 00000000..92c08f8d --- /dev/null +++ b/.changes/+twitter-archive.added.md @@ -0,0 +1 @@ +新增可配置的 Peer JWT 签发默认期限并移除 admission 的强制最大寿命；Host SDK 0.3.1 支持扩展在既有 Graph 事务中批量定位原生身份并串行合并。 diff --git a/.env.example b/.env.example index 3180f295..cc8c8c63 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,9 @@ PEER_NAME=core-py-development # JWT Authentication (Required) JWT_SECRET=replace-with-at-least-32-random-bytes +# Signing defaults in seconds; neither value limits admission of a valid JWT. +JWT_DEFAULT_LIFETIME_SECONDS=60 +TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS=604800 # Database lifecycle authority. Compose uses the PostgreSQL owner only in its # one-shot init service and gives the application a separate runtime URL. diff --git a/app/database_contract/constants.py b/app/database_contract/constants.py index 3514854d..984ed84b 100644 --- a/app/database_contract/constants.py +++ b/app/database_contract/constants.py @@ -4,7 +4,7 @@ CONTRACT_FORMAT = 1 -CONTRACT_REVISION = "extension-peer-participation-v2" +CONTRACT_REVISION = "peer-jwt-signing-defaults-v1" PROTOCOL_SCHEMA = "inkcre" INTERNAL_SCHEMA = "inkcre_internal" @@ -18,6 +18,7 @@ JWT_ROLE = AUTHENTICATED_ROLE JWT_ISSUER = "inkcre-peer" JWT_AUDIENCE = "inkcre-api" +# Frozen input for admitted historical migrations; live admission has no lifetime cap. JWT_MAX_LIFETIME_SECONDS = 24 * 60 * 60 JWT_MINIMUM_SECRET_BYTES = 32 diff --git a/app/database_contract/lifecycle.py b/app/database_contract/lifecycle.py index f4e4e88b..d0fc976f 100644 --- a/app/database_contract/lifecycle.py +++ b/app/database_contract/lifecycle.py @@ -14,7 +14,6 @@ JWT_ALGORITHM, JWT_AUDIENCE, JWT_ISSUER, - JWT_MAX_LIFETIME_SECONDS, JWT_ROLE, RESET_CONFIRMATION, ) @@ -79,7 +78,6 @@ def contract_document() -> dict[str, object]: "issuer": JWT_ISSUER, "audience": JWT_AUDIENCE, "required_claims": ["role", "iss", "aud", "iat", "exp"], - "maximum_lifetime_seconds": JWT_MAX_LIFETIME_SECONDS, }, "profiles": ["runtime", "development"], "protocol": protocol_document(), diff --git a/app/middleware.py b/app/middleware.py index c9405e96..7a16662e 100644 --- a/app/middleware.py +++ b/app/middleware.py @@ -1,5 +1,6 @@ """Middleware for logging and request tracking.""" +import math import time from typing import Callable import uuid @@ -13,7 +14,6 @@ JWT_ALGORITHM, JWT_AUDIENCE, JWT_ISSUER, - JWT_MAX_LIFETIME_SECONDS, JWT_ROLE, ) from app.settings import settings @@ -63,15 +63,19 @@ def decode_peer_jwt( ) if not valid_numeric_dates: raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") - issued_at_number = float(issued_at) - expires_at_number = float(expires_at) + try: + issued_at_number = float(issued_at) + expires_at_number = float(expires_at) + except OverflowError as error: + raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") from error current_epoch = time.time() if now is None else now if ( claims["role"] != JWT_ROLE + or not math.isfinite(issued_at_number) + or not math.isfinite(expires_at_number) or issued_at_number > current_epoch + 60 or expires_at_number <= current_epoch or expires_at_number <= issued_at_number - or expires_at_number - issued_at_number > JWT_MAX_LIFETIME_SECONDS ): raise jwt.exceptions.InvalidTokenError("JWT claim contract mismatch") return claims @@ -81,11 +85,13 @@ def create_peer_jwt( secret: str, *, now: float | None = None, - lifetime_seconds: int = 60, + lifetime_seconds: int | None = None, ) -> str: - """Create one short-lived token under the canonical Peer HTTP contract.""" - if lifetime_seconds <= 0 or lifetime_seconds > JWT_MAX_LIFETIME_SECONDS: - raise ValueError("Peer JWT lifetime is outside the contract bound") + """Sign a Peer JWT using the configured default when no lifetime is supplied.""" + if lifetime_seconds is None: + lifetime_seconds = settings.jwt_default_lifetime_seconds + if lifetime_seconds <= 0: + raise ValueError("Peer JWT lifetime must be positive") issued_at = int(time.time() if now is None else now) return jwt.encode( { diff --git a/app/persistence/info_base/repository.py b/app/persistence/info_base/repository.py index cefa2366..f87b6c03 100644 --- a/app/persistence/info_base/repository.py +++ b/app/persistence/info_base/repository.py @@ -172,6 +172,41 @@ async def find_json_field( statement = statement.where(sqlmodel.col(BlockModel.id).in_(tuple(block_ids))) return tuple(await self._session.scalars(statement)) + async def find_json_field_many( + self, + resolvers: typing.Collection[str], + field: str, + values: typing.Collection[str], + ) -> tuple[BlockModel, ...]: + """Find inline JSON records in a bounded native-identity set.""" + from sqlalchemy.dialects.postgresql import JSONB + + if not resolvers or not values: + return () + return tuple( + await self._session.scalars( + sqlmodel.select(BlockModel) + .where( + sqlmodel.col(BlockModel.resolver).in_(tuple(resolvers)), + sqlalchemy.case( + ( + sqlalchemy.func.pg_input_is_valid(BlockModel.content, "jsonb"), + sqlalchemy.cast(BlockModel.content, JSONB)[field].astext, + ) + ).in_(tuple(values)), + sqlmodel.col(BlockModel.storage).is_(None), + ) + .order_by(sqlmodel.col(BlockModel.id)) + ) + ) + + async def acquire_transaction_lock(self, namespace: str) -> None: + """Serialize an owning use case until its surrounding transaction ends.""" + await self._session.execute( + sqlalchemy.text("SELECT pg_advisory_xact_lock(hashtextextended(:name, 0))"), + {"name": namespace}, + ) + async def get_related( self, block_id: BlockID, *, content: str, outgoing: bool = True ) -> BlockModel | None: diff --git a/app/settings.py b/app/settings.py index 8c392cf2..88d3ff2a 100644 --- a/app/settings.py +++ b/app/settings.py @@ -52,6 +52,16 @@ class Settings(BaseSettings): min_length=JWT_MINIMUM_SECRET_BYTES, description="Secret key for JWT token signing and verification", ) + jwt_default_lifetime_seconds: int = Field( + default=60, + gt=0, + description="Default Peer JWT signing lifetime; not an admission limit", + ) + twitter_x_monkey_jwt_lifetime_seconds: int = Field( + default=7 * 24 * 60 * 60, + gt=0, + description="Signing lifetime for the Twitter x-monkey session and its renewal", + ) # Observability settings obsrv: ObsrvSetting = Field(default_factory=ObsrvSetting) diff --git a/app/version.py b/app/version.py index 1ca57f6a..15b5af52 100644 --- a/app/version.py +++ b/app/version.py @@ -3,7 +3,7 @@ from pathlib import Path import tomllib -CORE_VERSION = "0.3.0" +CORE_VERSION = "0.3.1" APPLICATION_VERSION = tomllib.loads( (Path(__file__).parents[1] / "pyproject.toml").read_text(encoding="utf-8") )["project"]["version"] diff --git a/docker-compose.yml b/docker-compose.yml index cd949363..165953d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,8 @@ services: DATABASE_URL: "postgresql+psycopg://inkcre_core:${CORE_DATABASE_PASSWORD:-local-core-database-password-at-least-32-bytes}@postgres:5432/${POSTGRES_DB:-inkcre}" INKCRE_ENV_FILE: "" JWT_SECRET: ${JWT_SECRET:-local-development-jwt-secret-at-least-32-bytes} + JWT_DEFAULT_LIFETIME_SECONDS: ${JWT_DEFAULT_LIFETIME_SECONDS:-60} + TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS: ${TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS:-604800} OBSRV__LOGGING_BACKEND: ${OBSRV__LOGGING_BACKEND:-postgresql} OBSRV__TELEMETRY_ENABLED: ${OBSRV__TELEMETRY_ENABLED:-false} OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-} diff --git a/docs/30-unit-tdd/README.md b/docs/30-unit-tdd/README.md index 0dcb2f42..8a5ef9c5 100644 --- a/docs/30-unit-tdd/README.md +++ b/docs/30-unit-tdd/README.md @@ -13,6 +13,7 @@ This directory owns expensive internal design truth for logical units delivered | [mail-extension.md](mail-extension.md) | Mail identity, MIME materialization, collection, graph, and failure boundaries | | [memos-extension.md](memos-extension.md) | Memos adapter identity, graph grammar, persistence, and failure boundaries | | [rss-extension.md](rss-extension.md) | RSS adapter identity, collection lifecycle, reconciliation, and materialization | +| [twitter-extension.md](twitter-extension.md) | Twitter 共同身份、Bookmark 过渡、事务合并和按需媒体读取 | | [security-model.md](security-model.md) | Core-py realization of the shared security boundary model | Do not add a document for a directory inventory, a fact cheaply recovered from code, a one-off decision, or a cross-unit contract already owned by `../_shared/20-product-tdd/`. Put repeated physical-subtree hazards in the nearest `AGENTS.md`; put runtime, rollout, recovery, and environment truth in `../40-deployment/`. diff --git a/docs/30-unit-tdd/twitter-extension.md b/docs/30-unit-tdd/twitter-extension.md new file mode 100644 index 00000000..358ec2fe --- /dev/null +++ b/docs/30-unit-tdd/twitter-extension.md @@ -0,0 +1,37 @@ +# Python Twitter 采集与归档 + +Python Twitter distribution 把 x-monkey 提交和 Bookmark 采集写入同一组普通 Blocks。共同字段与跨分发关系由 `inkcre/twitter` 的 [Twitter 归档契约](../_shared/20-product-tdd/twitter-archive-contract.md) 拥有，`model.py` 实现独立的 User/Post/Media、身份根与捕获集合。现有 JSON 字段、Resolver ID 和 Block ID 保持不变，数据库没有另一套归档表。 + +`archive_schema.py` 只保存 x-monkey 0.2.0 的固定协议 DTO；`x_monkey.py` 把已校验的提交转成共同模型，并按固定 DTO 字段集递归投影 query 数据、构造协议状态与汇总。自由的 Article JSON 按值保留。共同模型未来新增的字段不会自动进入旧协议，无法用该协议表达的已知结构返回错误，不改写持久事实。Official/Twikit 适配器在 `api.py` 中直接产出共同模型，`bookmark.py` 与 `resolver.py` 不依赖协议 DTO。当前适配器保留它们明确映射的事实，不宣称保存完整 provider 原始响应或所有未知字段。 + +`reconcile.py` 拥有本实现的身份选择、事务合并和关系维护，只返回变化集合、Block IDs 及共同读取结果，不包含 HTTP 状态或 wire 响应类型。`TwitterBatch` 没有继承远端请求的条数限制；每类 1000 条的准入边界留在 x-monkey DTO。 + +一次 submit 使用一个 `graph_uow()`。Bookmark 先完成 provider 请求，再进入 `source_uow()`，锁定 Source，使用其 `graph` 合并新主帖及实际捕获的上下文，并在同一事务内推进 cursor。主帖顺序与展开上下文分开，因此引用帖不会成为下一次书签采集的 watermark；旧的数值 cursor 在比较边界转换为十进制字符串，不重置历史状态。 + +两条写入路径都先取得 `extensions.twitter.reconcile` 全局事务 advisory 锁，再批量重读原生身份。根内容和受管关系一起提交；异常让整批回滚，调用方可安全重放。该锁仅约束正常 Twitter 写入入口，不能替代所有 info-base 原始写入的约束。provider 和媒体网络访问都不能放进此事务。 + +原生身份、Blocks 和引用边使用批量查询与创建。不要在每个媒体项内单独查询、创建和 flush：Core 与数据库跨网络部署时，这会把一次正常批次放大为数百次串行往返，并在持锁期间耗尽客户端的同步请求期限。 + +匹配先选新版 resolver，再按 Block ID 稳定选择旧 `tweet.v1`。旧根重新采集后原地更新，保留现有 Block ID、引用和旧附件关系；未重新采集的旧根仍由 v1 decoder 读取。旧加工正文不足以重建 native RichText，不能用它伪造新记录。共同合并提供 `replace` 与 `supplement` 正文策略：正常捕获合并已提交的正文结构，补充捕获在已有正文时保留它并增加其他事实。Bookmark 使用补充策略，避免较少详情的正文覆盖完整捕获；该选择不作为字段写进持久记录。 + +身份引用严格保存为 `{id}`，不补作者、时间或正文。query 对完整 native 记录返回 `found`，身份引用返回 `missing`，能按 ID 定位但结构损坏的记录返回 `failed`。批量身份查询使用有效 JSON 的条件表达式隔离普通文本和损坏 JSON；无法解析且无法定位 ID 的内容不会命中某个原生 ID，不会导致其他正常记录的查询失败。 + +共同合并选择缺省保留、对象递归合并和数组整体替换，保留有序媒体引用。原生观察时间保存输入字符串，不在序列化时改写精度。Twitter 以毫秒分辨率逐计数比较观察时间；较旧观察不覆盖新值，相同值的新时间也更新持久记录，无时间的新值删除该计数的旧时间标签。这些是共同语义的规则，x-monkey 兼容基线用于验证其适配结果。协议 Adapter 将无持久变化映射为 `unchanged`，仅观察时间更新仍为 `accepted`。 + +新版引用边只维护约定的 reference schema，旧 `attachment:*`、`entities:url` 和用户关系保留。媒体引用顺序是 `post.mediaIds`、文章封面、`article.mediaIds` 依次合并去重，首次出现保留。Media 根只保存原生 JSON；写入不创建 `core.image.v1`、`core.video.v1` 或 Media→`content` 关系。较早版本产生的内容块和边保留，须先核对其他引用及来源，再由明确的数据清理处理，正常重放不会删除它们。 + +媒体原生 URL、variants 和 alt 按捕获值保留，Python 不筛选或规范化这些字段。保存、query、文本投影和索引只消费元数据；用户显式展开媒体时，消费端 Twitter Distribution 才选择图片地址或最高 bitrate MP4，并交给现有 HTTP Storage。HTTP(S) 地址、重定向、超时与体积限制由通用 Storage 合同处理，不增加 Twitter 专属 CDN、HTTPS/443 或禁用重定向策略。HLS 继续保留为元数据。本版本不保存媒体本体，也不创建下载 Job，外部来源失效后仍可读取文字与元数据。 + +此边界通过减少持久块实现。Core 的普通图片和视频 resolver 在 lexical 索引中仍可读取字节并生成 OCR/STT；如果 Twitter 提前建立这类内容块，独立的索引扫描会触发读取，即使 Post/Media 自身的投影不展开媒体。Twitter 原生记录因此不预建内容块，其他来源的通用 lexical/OCR/STT 行为保持原有合同。 + +Python 的 v2 Post 文本投影包含原帖 ID、链接、已捕获作者、正文和长文；User/Media 投影使用自身已经捕获的文字。label 保持 Block-local，不为投影或标签下载媒体。检索沿用现有独立索引维护流程，不在提交中隐式维护索引。 + +Python Twitter 在 `/twitter/x-monkey/account` 自托管连接页面。用户只在自己信任的 HTTPS 部署输入已有 `JWT_SECRET`，页面使用 WebCrypto HMAC SHA-256 本地签发 60 秒普通 Peer JWT，再以同 Origin Bearer 和 JSON `{}` 请求 `POST /twitter/x-monkey/session`。该路由复用 `require_peer_jwt`，拒绝非 Core Origin、非 JSON 和带 `session_use` 的凭据，返回 `SessionResponse` JSON 并建立归档 Cookie。页面随后 GET `api/v1/session`，确认 Cookie 的 `authenticated=true` 后才显示 `Connected`、Peer 和到期时间。 + +账户页以随机 nonce CSP 限定脚本，`connect-src 'self'` 限定连接目标。JWT 不进入 URL，密钥不持久化或发送给服务器。页面不加载第三方脚本，不接受任意回跳；浏览器内签名不使恶意或被篡改页面安全。 + +连接不要求 Web 或 MF 分发。`setup.x_monkey_base_url` 继续供用户复制，Web Setup 只提供通向该 Core 账户页的普通链接，不持有此连接流程的密钥、JWT 签发或凭据传输逻辑。账户页显示 Peer 与到期时间，脚本通过会话检测确认连接。七天 Cookie 默认期限与协议续签仍由 Python Twitter 拥有。 + +候选验证脚本与 fixture 放在 `scripts/tests/twitter`，默认 pytest 与 CI 不收集。离线合同与 HTTP 边界验证入口为 `pdm run python -m pytest -q scripts/tests/twitter/test_archive_contract.py scripts/tests/twitter/test_archive_http.py`；显式运行整个目录可用 `pdm run python -m pytest -q scripts/tests/twitter`。 + +真实数据库场景要求显式提供 `INKCRE_TEST_DATABASE_URL`，使用已迁移的可丢弃 PostgreSQL runtime；测试只清理自己创建的精确 IDs，不重置开发库。原生 roundtrip 与真实 Twikit Bookmark 验证执行 `LexicalRetrievalManager.maintain` 的实际扫描、投影和检索链，在 HTTP Storage 字节边界断言零调用；同时检查归档不额外创建内容块、blob 或 Job。固定 fixture 同时用于原生 roundtrip 和独立上游合同验证，真实 userscript、HTTP 会话及 Web 消费仍需要各自的纵向验收。 diff --git a/docs/40-deployment/database-contract.md b/docs/40-deployment/database-contract.md index b46e032a..7d52dc4e 100644 --- a/docs/40-deployment/database-contract.md +++ b/docs/40-deployment/database-contract.md @@ -122,13 +122,17 @@ role: authenticated issuer: inkcre-peer audience: inkcre-api required: role, iss, aud, iat, exp -maximum lifetime: 24 hours future iat tolerance: 60 seconds ``` PostgREST exposes only `inkcre`，connects as `authenticator`，uses denied `anonymous` fallback，and invokes `inkcre_internal.check_jwt` before every admitted request。FastAPI peer middleware validates the same vectors。 +JWT 没有强制最大寿命。`JWT_DEFAULT_LIFETIME_SECONDS` 是 Core 签发器的默认值（60 秒）， +`TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS` 是 Twitter x-monkey 会话初次签发及续签的默认值（604800 秒，7 天）； +二者必须为正整数，不参与 token admission。调整默认值影响后续签发，不会撤销已签发的有效 token。 +普通 Web Peer 仍默认签发 24 小时 token。有效期、签名、必需 claims 和未来 iat 容差仍参与验证。 + ## Canonical Production Profile [`deploy/profiles/production.json`](../../deploy/profiles/production.json) 是 checked-in non-secret discovery surface。 @@ -140,7 +144,7 @@ implementation 或 migration verification 不授权修改 production profile 或 ## Versioning -`db contract --json` reports `peer-database-runtime-v3` plus the image source revision. +`db contract --json` reports the executable contract revision plus the image source revision. Every production candidate also contains `/app/database-contract/`: - `database-roles.sql` recreates the password-free global principals required by policies; diff --git a/docs/40-deployment/first-party-extension-distribution.md b/docs/40-deployment/first-party-extension-distribution.md index fdd69928..3f707496 100644 --- a/docs/40-deployment/first-party-extension-distribution.md +++ b/docs/40-deployment/first-party-extension-distribution.md @@ -147,7 +147,10 @@ Core Host SDK 0.3 adds `app.http.get_public_http_base_url()` without changing th or persistence API. Its authority is `app/version.py`, independently of the Core service package version in `pyproject.toml`; preparing a service Release PR does not change the Host SDK version. -GitHub, Learn English, Mail, RSS, Telegram, and Twitter retain their SDK 0.2 lower bound and support +Core Host SDK 0.3.1 adds batched JSON identity lookup and transaction advisory locking to the +existing Graph repository. Twitter archive reconciliation requires `>=0.3.1 <0.4.0`. + +GitHub, Learn English, Mail, RSS, and Telegram retain their SDK 0.2 lower bound and support SDK 0.3 through `>=0.2.0 <0.4.0`. Memos uses the new HTTP address API and requires `>=0.3.0 <0.4.0`. Compatibility metadata changes require new immutable Extension releases; an old wheel's range must not be widened. Before upgrading a deployment, disable affected diff --git a/docs/_shared b/docs/_shared index 8fc1ec69..7449135f 160000 --- a/docs/_shared +++ b/docs/_shared @@ -1 +1 @@ -Subproject commit 8fc1ec69124e5b4ea86e7343a64c38a2e148e95d +Subproject commit 7449135fefdd6056fecba387e59e3005aed5eafd diff --git a/extensions/twitter/.changes/+x-monkey.added.md b/extensions/twitter/.changes/+x-monkey.added.md new file mode 100644 index 00000000..16287f1d --- /dev/null +++ b/extensions/twitter/.changes/+x-monkey.added.md @@ -0,0 +1 @@ +支持以现有 JWT 连接 x-monkey，保存与合并用户、帖子和媒体元数据，与 Bookmark 共用原生身份；媒体保持按需读取，会话默认七天并自动续签。 diff --git a/extensions/twitter/__init__.py b/extensions/twitter/__init__.py index 68fd37e1..d0764ef5 100644 --- a/extensions/twitter/__init__.py +++ b/extensions/twitter/__init__.py @@ -33,9 +33,17 @@ class Extension( ): @classmethod def _init_resolvers(cls): - from .resolver import TweetResolver + from .resolver import ( + ArchiveMediaResolver, + ArchivePostResolver, + ArchiveUserResolver, + TweetResolver, + ) ResolverManager.register_resolver(TweetResolver) + ResolverManager.register_resolver(ArchivePostResolver) + ResolverManager.register_resolver(ArchiveUserResolver) + ResolverManager.register_resolver(ArchiveMediaResolver) @classmethod def _init_sources(cls): @@ -71,8 +79,10 @@ async def on_start_async(cls, app): @classmethod def _register_apis(cls, router: APIRouter): from .setup_flow import register_setup_routes + from .archive_http import router as archive_http_router register_setup_routes(router) + router.include_router(archive_http_router, prefix="/x-monkey") @classmethod def api_dependencies(cls) -> list[typing.Any]: @@ -87,7 +97,15 @@ def peer_inbounds(cls) -> tuple[typing.Any, ...]: @classmethod def public_http_routes(cls) -> tuple[PublicHTTPRoute, ...]: - return (PublicHTTPRoute(method="GET", path="/auth/callback"),) + return ( + PublicHTTPRoute(method="GET", path="/auth/callback"), + PublicHTTPRoute(method="POST", path="/x-monkey/session"), + PublicHTTPRoute(method="GET", path="/x-monkey/account"), + PublicHTTPRoute(method="POST", path="/x-monkey/disconnect"), + PublicHTTPRoute(method="GET", path="/x-monkey/api/v1/session"), + PublicHTTPRoute(method="POST", path="/x-monkey/api/v1/archive/query"), + PublicHTTPRoute(method="POST", path="/x-monkey/api/v1/archive/submit"), + ) @classmethod async def update_config_async( diff --git a/extensions/twitter/api.py b/extensions/twitter/api.py index 90ca3978..324f3fc3 100644 --- a/extensions/twitter/api.py +++ b/extensions/twitter/api.py @@ -3,6 +3,8 @@ import abc import asyncio import datetime +from email.utils import parsedate_to_datetime +import html from pathlib import Path import re import typing @@ -14,34 +16,274 @@ import twikit import twikit.media from typing import Optional as Opt -from dd import dd from utils.datetime_ import get_timestamp -from .schema import TweetPhoto, TweetVideo, VideoVariant +from .model import TwitterBatch from . import Extension -class Tweet(sqlmodel.SQLModel): - id: int - user_id: str | None = None - """推文用户ID""" - conversation_id: Opt[int] = None - text: str - """推文文本内容 - - - 移除回复提及 - - 移除媒体、网页链接（用 `[photo]`、`[video]`、`[link]` 占位） - """ - quote: Opt[int] = None - """引用推文ID""" - photos: tuple[TweetPhoto, ...] = () - videos: tuple[TweetVideo, ...] = () - urls: tuple[str, ...] = () - - class TwitterAPIResult(sqlmodel.SQLModel): next_page: Opt[str] = None previous_page: Opt[str] = None - tweets: tuple[Tweet, ...] = () + batch: TwitterBatch = pydantic.Field(default_factory=TwitterBatch) + primary_post_ids: tuple[str, ...] = () + + +def _instant(value: str | datetime.datetime) -> str: + if isinstance(value, str): + try: + value = datetime.datetime.fromisoformat(value) + except ValueError: + value = parsedate_to_datetime(value) + if value.tzinfo is None: + raise ValueError("Twitter provider returned a timestamp without a timezone") + return ( + value.astimezone(datetime.timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z") + ) + + +def _observed_at() -> str: + return ( + datetime.datetime.now(datetime.timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z") + ) + + +def _media_id(media: dict) -> str | None: + if media.get("id"): + return str(media["id"]) + match = re.fullmatch(r"(?:3|7|13)_([1-9][0-9]*)", media.get("media_key", "")) + return match[1] if match else None + + +def _provider_counts(raw: dict, names: dict[str, str], observed_at: str) -> dict: + values = { + name: raw[source] for source, name in names.items() if raw.get(source) is not None + } + return ( + {"counts": values, "countsObservedAt": {name: observed_at for name in values}} + if values + else {} + ) + + +def _official_archive( + raw_tweets: list[dict], includes: dict, observed_at: str +) -> TwitterBatch: + media = [] + media_ids = {} + for raw in includes.get("media", ()): + media_id = _media_id(raw) + if media_id is None: + continue + media_ids[raw["media_key"]] = media_id + kind = {"photo": "photo", "video": "video", "animated_gif": "gif"}.get(raw["type"]) + url = raw.get("url") if kind == "photo" else raw.get("preview_image_url") + if kind is None or not url: + continue + item = {"id": media_id, "type": kind, "url": url} + item.update( + { + target: raw[source] + for source, target in { + "width": "width", + "height": "height", + "alt_text": "alt", + }.items() + if raw.get(source) is not None + } + ) + if raw.get("variants"): + item["video"] = { + "variants": [ + { + "url": variant["url"], + "type": variant["content_type"], + **({"bitrate": variant["bit_rate"]} if "bit_rate" in variant else {}), + } + for variant in raw["variants"] + ] + } + if raw.get("duration_ms") is not None: + item["video"]["durationMs"] = raw["duration_ms"] + media.append(item) + users = [] + for raw in includes.get("users", ()): + user = {"id": raw["id"], "handle": raw["username"], "name": raw["name"]} + user.update( + { + target: raw[source] + for source, target in { + "profile_image_url": "avatarUrl", + "location": "location", + "url": "website", + "protected": "protected", + }.items() + if raw.get(source) is not None + } + ) + if "description" in raw: + user["bio"] = {"text": html.unescape(raw["description"])} + if "created_at" in raw: + user["createdAt"] = _instant(raw["created_at"]) + user.update( + _provider_counts( + raw.get("public_metrics", {}), + { + "followers_count": "followers", + "following_count": "following", + "tweet_count": "posts", + }, + observed_at, + ) + ) + users.append(user) + posts = [] + for raw in [*raw_tweets, *includes.get("tweets", ())]: + post = { + "id": raw["id"], + "authorId": raw["author_id"], + "createdAt": _instant(raw["created_at"]), + } + references = { + reference["type"]: reference["id"] for reference in raw.get("referenced_tweets", ()) + } + if "retweeted" in references: + post["repostOf"] = references["retweeted"] + else: + post["text"] = { + "text": html.unescape(raw.get("note_tweet", {}).get("text", raw["text"])) + } + if "replied_to" in references: + post["replyTo"] = {"postId": references["replied_to"]} + if "quoted" in references: + post["quote"] = {"postId": references["quoted"]} + if raw.get("conversation_id") and raw["conversation_id"] != raw["id"]: + post["conversationId"] = raw["conversation_id"] + post.update( + { + target: raw[source] + for source, target in {"lang": "lang", "possibly_sensitive": "sensitive"}.items() + if raw.get(source) is not None + } + ) + attached = [ + media_ids[key] + for key in raw.get("attachments", {}).get("media_keys", ()) + if key in media_ids + ] + if attached: + post["mediaIds"] = attached + post.update( + _provider_counts( + raw.get("public_metrics", {}), + { + "reply_count": "replies", + "retweet_count": "reposts", + "quote_count": "quotes", + "like_count": "likes", + "bookmark_count": "bookmarks", + "impression_count": "views", + }, + observed_at, + ) + ) + posts.append(post) + return TwitterBatch.model_validate({"users": users, "posts": posts, "media": media}) + + +def _twikit_archive( + tweets: typing.Iterable[twikit.Tweet], observed_at: str +) -> TwitterBatch: + posts: dict[str, dict] = {} + users: dict[str, dict] = {} + media: dict[str, dict] = {} + + def collect(tweet: twikit.Tweet) -> None: + if tweet.id in posts: + return + user = tweet.user + post: dict[str, typing.Any] = { + "id": tweet.id, + "authorId": user.id, + "createdAt": _instant(tweet.created_at_datetime), + } + posts[tweet.id] = post + users[user.id] = {"id": user.id, "handle": user.screen_name, "name": user.name} + if tweet.retweeted_tweet is not None: + post["repostOf"] = tweet.retweeted_tweet.id + collect(tweet.retweeted_tweet) + return + post["text"] = {"text": html.unescape(tweet.full_text)} + if tweet.in_reply_to: + post["replyTo"] = {"postId": tweet.in_reply_to} + conversation_id = tweet._legacy.get("conversation_id_str") + if conversation_id and conversation_id != tweet.id: + post["conversationId"] = conversation_id + if tweet.quote is not None: + post["quote"] = {"postId": tweet.quote.id} + collect(tweet.quote) + if tweet.lang: + post["lang"] = tweet.lang + post.update( + _provider_counts( + tweet._legacy, + { + "reply_count": "replies", + "retweet_count": "reposts", + "quote_count": "quotes", + "favorite_count": "likes", + "bookmark_count": "bookmarks", + }, + observed_at, + ) + ) + attached = [] + for item in tweet.media: + kind = {"photo": "photo", "video": "video", "animated_gif": "gif"}.get(item.type) + if kind is None or not item.id or not item.media_url: + continue + value: dict[str, typing.Any] = {"id": item.id, "type": kind, "url": item.media_url} + original = item._data.get("original_info") or {} + value.update( + { + name: original[name] + for name in ("width", "height") + if original.get(name) is not None + } + ) + if item._data.get("ext_alt_text") is not None: + value["alt"] = item._data["ext_alt_text"] + info = item._data.get("video_info") or {} + if info.get("variants"): + value["video"] = { + "variants": [ + { + "url": variant["url"], + "type": variant["content_type"], + **({"bitrate": variant["bitrate"]} if "bitrate" in variant else {}), + } + for variant in info["variants"] + ] + } + if info.get("duration_millis") is not None: + value["video"]["durationMs"] = info["duration_millis"] + media[item.id] = value + attached.append(item.id) + if attached: + post["mediaIds"] = attached + + for tweet in tweets: + collect(tweet) + return TwitterBatch.model_validate( + { + "users": list(users.values()), + "posts": list(posts.values()), + "media": list(media.values()), + } + ) class TwitterAPI(abc.ABC): @@ -359,74 +601,8 @@ async def get_user(self) -> tuple[str, str]: def _resolve_tweets( self, raw_tweets: list[dict], includes: dict[str, list[dict]] - ) -> list[Tweet]: - include_medias = includes.get("media", ()) - - tweets: list[Tweet] = [] - for tweet in raw_tweets: - tweet = dd(tweet) - tweet_id = tweet.id() - - # resolve medias - media_keys = tweet._.attachments.media_keys() or () - photos: list[TweetPhoto] = [] - videos: list[TweetVideo] = [] - for media_key in media_keys: - for include_media in include_medias: - include_media = dd(include_media) - if include_media._.media_key() == media_key: - media_type = include_media._.type() - if media_type == "video": - videos.append( - TweetVideo( - id=media_key, - variants=tuple( - VideoVariant(**variant) - for variant in (include_media._.variants() or ()) - ), - ) - ) - elif media_type == "photo": - photos.append( - TweetPhoto( - id=media_key, - url=include_media.url(lambda x: x or ""), - ) - ) - else: - # TODO log warning for unsupported media type - pass - break - - # resolve conversation ID - conversation_id = tweet._.conversation_id() - if conversation_id == tweet_id: - conversation_id = None - - # resolve url entities - urls: list[str] = [] - for entity in tweet._.entities.urls() or []: - url = entity.get("expanded_url") - if url: - urls.append(url) - - # resolve text - tweet_text = tweet.text() - tweet_text = re.sub(r"^(?:@\w+\s*)+", "", tweet_text) - - tweets.append( - Tweet( - id=tweet_id, - user_id=tweet._.author_id(), - text=tweet_text, - conversation_id=conversation_id, - photos=tuple(photos), - videos=tuple(videos), - urls=tuple(urls), - ) - ) - - return tweets + ) -> TwitterBatch: + return _official_archive(raw_tweets, includes, _observed_at()) async def get_bookmarks( self, max_results: int = 20, page: str | None = None @@ -440,9 +616,21 @@ async def get_bookmarks( """ bookmarks_query = { "max_results": max_results, - "tweet.fields": "attachments,entities,lang,conversation_id", - "media.fields": "alt_text,media_key,url,type", - "expansions": "attachments.media_keys,attachments.media_source_tweet", + "tweet.fields": ( + "author_id,created_at,attachments,entities,lang,conversation_id," + "referenced_tweets,public_metrics,possibly_sensitive,note_tweet" + ), + "media.fields": ( + "alt_text,media_key,url,type,preview_image_url,width,height,duration_ms,variants" + ), + "user.fields": ( + "id,name,username,description,profile_image_url,url," + "protected,location,public_metrics,created_at" + ), + "expansions": ( + "author_id,attachments.media_keys,referenced_tweets.id," + "referenced_tweets.id.author_id" + ), } if page: bookmarks_query["pagination_token"] = page @@ -452,11 +640,12 @@ async def get_bookmarks( path_params={"id": self.__user_id}, query=bookmarks_query, ) - tweets = self._resolve_tweets(res.get("data", []), res.get("includes", {})) + batch = self._resolve_tweets(res.get("data", []), res.get("includes", {})) return TwitterAPIResult( next_page=res.get("meta", {}).get("next_token"), previous_page=res.get("meta", {}).get("previous_token"), - tweets=tuple(tweets), + batch=batch, + primary_post_ids=tuple(post["id"] for post in res.get("data", ())), ) async def get_tweets( @@ -468,16 +657,29 @@ async def get_tweets( query={ "query": query, "max_results": max_results, - "tweet.fields": "attachments,entities,lang,conversation_id", - "media.fields": "alt_text,media_key,url,type", - "expansions": "attachments.media_keys,attachments.media_source_tweet", + "tweet.fields": ( + "author_id,created_at,attachments,entities,lang,conversation_id," + "referenced_tweets,public_metrics,possibly_sensitive,note_tweet" + ), + "media.fields": ( + "alt_text,media_key,url,type,preview_image_url,width,height,duration_ms,variants" + ), + "user.fields": ( + "id,name,username,description,profile_image_url,url," + "protected,location,public_metrics,created_at" + ), + "expansions": ( + "author_id,attachments.media_keys,referenced_tweets.id," + "referenced_tweets.id.author_id" + ), }, ) - tweets = self._resolve_tweets(res.get("data", []), res.get("includes", {})) + batch = self._resolve_tweets(res.get("data", []), res.get("includes", {})) return TwitterAPIResult( next_page=res.get("meta", {}).get("next_token"), previous_page=res.get("meta", {}).get("previous_token"), - tweets=tuple(tweets), + batch=batch, + primary_post_ids=tuple(post["id"] for post in res.get("data", ())), ) async def get_replies( @@ -550,59 +752,8 @@ async def _login(self): ) @staticmethod - def _resolve_tweet(tweet: twikit.Tweet) -> Tweet: - # resolve text - # remove reply mentions - tweet_text = re.sub(r"^(?:@\w+\s*)+", "", tweet.text) - - # resolve medias - photos: list[TweetPhoto] = [] - videos: list[TweetVideo] = [] - for media in tweet.media: - media_url = media.url - if isinstance(media, twikit.media.Photo): - photos.append(TweetPhoto(id=media.id, url=media.media_url)) - tweet_text = tweet_text.replace(media_url, "[photo]") - elif isinstance(media, twikit.media.Video): - videos.append( - TweetVideo( - id=media.id, - variants=tuple( - VideoVariant( - bitrate=variant.bitrate, - content_type=variant.content_type, - url=variant.url or "", - ) - for variant in media.streams - ), - ) - ) - tweet_text = tweet_text.replace(media_url, "[video]") - - # resolve urls - urls: list[str] = [] - for i in typing.cast(dict, tweet.urls): - i = dd(i) - ex_url = i._.expanded_url() - url = i._.url() - if ex_url: - urls.append(ex_url) - if url: - tweet_text = tweet_text.replace(url, "[link]") - - return Tweet( - id=int(tweet.id), - conversation_id=int(tweet.in_reply_to) if tweet.in_reply_to else None, - user_id=tweet.user.screen_name, - text=tweet_text, - photos=tuple(photos), - videos=tuple(videos), - urls=tuple(urls), - ) - - @classmethod - def _resolve_tweets(cls, result: twikit.utils.Result[twikit.Tweet]) -> tuple[Tweet, ...]: - return tuple(cls._resolve_tweet(tweet) for tweet in result) + def _resolve_tweets(result: typing.Iterable[twikit.Tweet]) -> TwitterBatch: + return _twikit_archive(result, _observed_at()) async def get_bookmarks( self, max_results: int = 20, page: str | None = None @@ -613,7 +764,8 @@ async def get_bookmarks( return TwitterAPIResult( next_page=res.next_cursor, previous_page=res.previous_cursor, - tweets=tuple(self._resolve_tweets(res)), + batch=self._resolve_tweets(res), + primary_post_ids=tuple(tweet.id for tweet in res), ) async def get_tweets( @@ -635,17 +787,18 @@ async def get_tweets( return TwitterAPIResult( next_page=res.next_cursor, previous_page=res.previous_cursor, - tweets=tuple(self._resolve_tweets(res)), + batch=self._resolve_tweets(res), + primary_post_ids=tuple(tweet.id for tweet in res), ) async def _get_a_reply_of( self, from_: str, replies: twikit.utils.Result[twikit.Tweet] - ) -> Tweet | None: + ) -> twikit.Tweet | None: if len(replies) == 0: return None for reply in replies: if reply.user.screen_name == from_: - return self._resolve_tweet(reply) + return reply else: await asyncio.sleep(5) # avoid rate limit replies = await replies.next() @@ -654,7 +807,7 @@ async def _get_a_reply_of( async def get_replies( self, *conversation_ids: str, from_: str | None = None, max_results: int = 20 ) -> TwitterAPIResult: - res_tweets: list[Tweet] = [] + res_tweets: list[twikit.Tweet] = [] for cid in conversation_ids: await asyncio.sleep(3) # avoid rate limit try: @@ -672,6 +825,9 @@ async def get_replies( if the_reply: res_tweets.append(the_reply) else: - res_tweets.extend(self._resolve_tweets(replies)) + res_tweets.extend(replies) - return TwitterAPIResult(tweets=tuple(res_tweets)) + return TwitterAPIResult( + batch=self._resolve_tweets(res_tweets), + primary_post_ids=tuple(tweet.id for tweet in res_tweets), + ) diff --git a/extensions/twitter/archive_http.py b/extensions/twitter/archive_http.py new file mode 100644 index 00000000..34a67998 --- /dev/null +++ b/extensions/twitter/archive_http.py @@ -0,0 +1,79 @@ +"""HTTP endpoints implementing the x-monkey remote archive contract.""" + +from __future__ import annotations + +import fastapi +from fastapi import Request, Response + +from .archive_schema import ArchiveBatch, ArchiveQuery, QueryResponse, SubmitResponse +from .archive_session import ( + SESSION_COOKIE_NAME, + SESSION_USE_CLAIM, + SESSION_USE_TWITTER_ARCHIVE, + _decode_peer_token, + renew_session_if_needed, + validate_archive_request_source, + x_monkey_base_url, +) +from .x_monkey import query_archive, submit_archive + +router = fastapi.APIRouter() +CLIENT_HEADER = "x-tweet-db-client" +CLIENT_NAME = "x-monkey" + + +def _protocol_error(status_code: int, message: str) -> fastapi.responses.JSONResponse: + return fastapi.responses.JSONResponse( + {"error": message}, + status_code=status_code, + headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"}, + ) + + +async def _authorize_archive_request(request: Request, response: Response) -> int | None: + if not await validate_archive_request_source(request): + return 403 + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + return 415 + if request.headers.get(CLIENT_HEADER) != CLIENT_NAME: + return 400 + token = request.cookies.get(SESSION_COOKIE_NAME) + claims = _decode_peer_token(token) if token else None + if claims is None or claims.get(SESSION_USE_CLAIM) != SESSION_USE_TWITTER_ARCHIVE: + return 401 + base_url = await x_monkey_base_url() + if base_url: + renew_session_if_needed(response, claims, base_url=base_url) + return None + + +@router.post( + "/api/v1/archive/query", + response_model=QueryResponse, + response_model_exclude_unset=True, +) +async def archive_query(body: ArchiveQuery, request: Request, response: Response): + failure = await _authorize_archive_request(request, response) + if failure is not None: + return _protocol_error(failure, "archive session or request headers are invalid") + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + return await query_archive(body) + + +@router.post( + "/api/v1/archive/submit", + response_model=SubmitResponse, + response_model_exclude_unset=True, +) +async def archive_submit(body: ArchiveBatch, request: Request, response: Response): + failure = await _authorize_archive_request(request, response) + if failure is not None: + return _protocol_error(failure, "archive session or request headers are invalid") + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + return await submit_archive(body) + + +__all__ = ["CLIENT_HEADER", "CLIENT_NAME", "router"] diff --git a/extensions/twitter/archive_merge.py b/extensions/twitter/archive_merge.py new file mode 100644 index 00000000..ae6fddfb --- /dev/null +++ b/extensions/twitter/archive_merge.py @@ -0,0 +1,79 @@ +"""Twitter-owned sparse fact merge, including per-counter observation times.""" + +from __future__ import annotations + +import copy +import datetime + +from pydantic import JsonValue + + +def _merge_content(target: dict[str, JsonValue], incoming: dict[str, JsonValue]) -> None: + for key, value in incoming.items(): + current = target.get(key) + if isinstance(value, dict) and isinstance(current, dict): + _merge_content(current, value) + else: + target[key] = copy.deepcopy(value) + + +def _observation_time(value: str) -> datetime.datetime: + parsed = datetime.datetime.fromisoformat(value) + # Twitter compares observations at millisecond resolution; retain finer source precision. + return parsed.replace(microsecond=parsed.microsecond // 1000 * 1000) + + +def merge_archive_record( + existing: dict[str, JsonValue], incoming: dict[str, JsonValue] +) -> dict[str, JsonValue]: + """Return a detached merge; delayed counter observations cannot replace newer ones.""" + result = copy.deepcopy(existing) + _merge_content( + result, + { + key: value + for key, value in incoming.items() + if key not in {"counts", "countsObservedAt"} + }, + ) + counts = incoming.get("counts") + if not isinstance(counts, dict): + return result + + current_counts = result.get("counts") + if not isinstance(current_counts, dict): + current_counts = {} + current_times = result.get("countsObservedAt") + if not isinstance(current_times, dict): + current_times = {} + incoming_times = incoming.get("countsObservedAt") + if not isinstance(incoming_times, dict): + incoming_times = {} + + for key, value in counts.items(): + observed_at = incoming_times.get(key) + current_time = current_times.get(key) + if ( + isinstance(observed_at, str) + and isinstance(current_time, str) + and _observation_time(observed_at) < _observation_time(current_time) + ): + continue + count_changed = key not in current_counts or current_counts[key] != value + if count_changed: + current_counts[key] = copy.deepcopy(value) + if isinstance(observed_at, str): + current_times[key] = observed_at + elif count_changed: + current_times.pop(key, None) + + if current_counts: + result["counts"] = current_counts + if current_times: + result["countsObservedAt"] = current_times + else: + result.pop("countsObservedAt", None) + return result + + +__all__ = ["merge_archive_record"] diff --git a/extensions/twitter/archive_schema.py b/extensions/twitter/archive_schema.py new file mode 100644 index 00000000..5fd88d65 --- /dev/null +++ b/extensions/twitter/archive_schema.py @@ -0,0 +1,436 @@ +"""Fixed x-monkey 0.2.0 DTOs; these do not define Twitter persisted semantics.""" + +from __future__ import annotations + +import datetime +import re +from typing import Annotated, Generic, Literal, TypeVar + +import pydantic + + +ArchiveID = Annotated[str, pydantic.Field(strict=True, pattern=r"^[1-9][0-9]*$")] +Count = Annotated[int, pydantic.Field(strict=True, ge=0)] +Number = Annotated[int | float, pydantic.Field(allow_inf_nan=False)] + + +def _validate_instant(value: str) -> str: + if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", value): + raise ValueError("must be a UTC timestamp with second precision") + datetime.datetime.fromisoformat(value) + return value + + +def _validate_observed_at(value: str) -> str: + if not re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", value + ): + raise ValueError("must be a UTC ISO-8601 timestamp including seconds") + datetime.datetime.fromisoformat(value) + return value + + +Instant = Annotated[str, pydantic.AfterValidator(_validate_instant)] +ObservedAt = Annotated[str, pydantic.AfterValidator(_validate_observed_at)] + + +def _validate_range(value: tuple[int, int]) -> tuple[int, int]: + if value[1] < value[0]: + raise ValueError("range end before start") + return value + + +Range = Annotated[ + tuple[Count, Count], + pydantic.Field(strict=False), + pydantic.AfterValidator(_validate_range), +] +Rect = Annotated[tuple[Count, Count, Count, Count], pydantic.Field(strict=False)] + + +class _WireModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", strict=True, allow_inf_nan=False) + + @pydantic.field_validator("*") + @classmethod + def reject_explicit_null(cls, value: object) -> object: + if value is None: + raise ValueError("omit unknown fields instead of sending null") + return value + + +class UrlEntity(_WireModel): + type: Literal["url"] + range: Range + expanded: str + + +class MentionEntity(_WireModel): + type: Literal["mention"] + range: Range + userId: ArchiveID + + +class HashtagEntity(_WireModel): + type: Literal["hashtag"] + range: Range + + +class CashtagEntity(_WireModel): + type: Literal["cashtag"] + range: Range + name: str | None = None + ticker: str | None = None + + +class TimestampEntity(_WireModel): + type: Literal["timestamp"] + range: Range + + +Entity = Annotated[ + UrlEntity | MentionEntity | HashtagEntity | CashtagEntity | TimestampEntity, + pydantic.Field(discriminator="type"), +] + + +class TextStyle(_WireModel): + range: Range + styles: list[str] = pydantic.Field(min_length=1) + + +class RichText(_WireModel): + text: str + visible: Range | None = None + entities: list[Entity] | None = pydantic.Field(default=None, min_length=1) + styles: list[TextStyle] | None = pydantic.Field(default=None, min_length=1) + + +class UserCounts(_WireModel): + followers: Count | None = None + following: Count | None = None + posts: Count | None = None + mediaPosts: Count | None = None + likes: Count | None = None + + +class UserCountsObservedAt(_WireModel): + followers: ObservedAt | None = None + following: ObservedAt | None = None + posts: ObservedAt | None = None + mediaPosts: ObservedAt | None = None + likes: ObservedAt | None = None + + +class Affiliation(_WireModel): + name: str + url: str | None = None + badgeUrl: str | None = None + kind: str | None = None + ownerId: ArchiveID | None = None + + +class Verification(_WireModel): + blue: bool | None = None + type: str | None = None + + +class Professional(_WireModel): + type: str | None = None + categories: list[str] | None = pydantic.Field(default=None, min_length=1) + + +class ArchiveUser(_WireModel): + id: ArchiveID + handle: str + name: str + createdAt: Instant | None = None + avatarUrl: str | None = None + avatarShape: str | None = None + bannerUrl: str | None = None + bio: RichText | None = None + location: str | None = None + website: str | None = None + pinnedPostIds: list[ArchiveID] | None = pydantic.Field(default=None, min_length=1) + protected: bool | None = None + sensitive: bool | None = None + interstitial: str | None = None + withheldIn: list[str] | None = pydantic.Field(default=None, min_length=1) + verified: Verification | None = None + affiliation: Affiliation | None = None + parody: str | None = None + professional: Professional | None = None + counts: UserCounts | None = None + countsObservedAt: UserCountsObservedAt | None = None + + +class VideoVariant(_WireModel): + url: str + type: str + bitrate: Count | None = None + + +class MediaTag(_WireModel): + userId: ArchiveID + handle: str | None = None + + +class Video(_WireModel): + durationMs: Count | None = None + variants: list[VideoVariant] = pydantic.Field(min_length=1) + + +class ArchiveMedia(_WireModel): + id: ArchiveID + type: Literal["photo", "video", "gif"] + url: str + width: Count | None = None + height: Count | None = None + alt: str | None = None + crops: list[Rect] | None = pydantic.Field(default=None, min_length=1) + faces: list[Rect] | None = pydantic.Field(default=None, min_length=1) + tags: list[MediaTag] | None = pydantic.Field(default=None, min_length=1) + video: Video | None = None + sourcePostId: ArchiveID | None = None + sourceUserId: ArchiveID | None = None + sensitive: list[str] | None = pydantic.Field(default=None, min_length=1) + unavailable: str | None = None + title: str | None = None + description: str | None = None + link: str | None = None + + +class PollChoice(_WireModel): + label: str + count: Count | None = None + + +class Poll(_WireModel): + choices: list[PollChoice] = pydantic.Field(min_length=2) + endsAt: Instant | None = None + final: bool | None = None + updatedAt: Instant | None = None + + +class CardImage(_WireModel): + url: str + width: Count | None = None + height: Count | None = None + alt: str | None = None + + +class CardPlayer(_WireModel): + url: str + width: Count | None = None + height: Count | None = None + + +class Card(_WireModel): + type: str + url: str + title: str | None = None + description: str | None = None + domain: str | None = None + image: CardImage | None = None + siteUserId: ArchiveID | None = None + creatorUserId: ArchiveID | None = None + player: CardPlayer | None = None + poll: Poll | None = None + extra: dict[str, str | bool] | None = None + + +class ArticleBlock(_WireModel): + type: str + text: str + styles: list[Annotated[tuple[Count, Count, str], pydantic.Field(strict=False)]] | None = ( + pydantic.Field(default=None, min_length=1) + ) + entities: ( + list[Annotated[tuple[Count, Count, Count], pydantic.Field(strict=False)]] | None + ) = pydantic.Field(default=None, min_length=1) + data: dict[str, pydantic.JsonValue] | None = None + + +class ArticleEntity(_WireModel): + type: str + data: dict[str, pydantic.JsonValue] + + +class Article(_WireModel): + id: ArchiveID + title: str + preview: str | None = None + coverMediaId: ArchiveID | None = None + publishedAt: Instant | None = None + modifiedAt: Instant | None = None + blocks: list[ArticleBlock] | None = pydantic.Field(default=None, min_length=1) + entities: list[ArticleEntity] | None = pydantic.Field(default=None, min_length=1) + mediaIds: list[ArchiveID] | None = pydantic.Field(default=None, min_length=1) + + +class Place(_WireModel): + id: str + name: str | None = None + fullName: str | None = None + country: str | None = None + countryCode: str | None = None + type: str | None = None + bbox: ( + Annotated[tuple[Number, Number, Number, Number], pydantic.Field(strict=False)] | None + ) = None + + +class PostCounts(_WireModel): + replies: Count | None = None + reposts: Count | None = None + quotes: Count | None = None + likes: Count | None = None + bookmarks: Count | None = None + views: Count | None = None + + +class PostCountsObservedAt(_WireModel): + replies: ObservedAt | None = None + reposts: ObservedAt | None = None + quotes: ObservedAt | None = None + likes: ObservedAt | None = None + bookmarks: ObservedAt | None = None + views: ObservedAt | None = None + + +class ReplyTo(_WireModel): + postId: ArchiveID + userId: ArchiveID | None = None + handle: str | None = None + tombstone: str | None = pydantic.Field(default=None, min_length=1) + + +class Quote(_WireModel): + postId: ArchiveID + url: str | None = None + tombstone: str | None = pydantic.Field(default=None, min_length=1) + + +class CommunityNote(_WireModel): + id: ArchiveID | None = None + title: str | None = None + text: RichText + lang: str | None = None + url: str | None = None + + +class ContentWarning(_WireModel): + title: str + text: str | None = None + + +class Community(_WireModel): + id: ArchiveID + name: str | None = None + + +class AIGenerated(_WireModel): + media: bool + source: str + + +class ArchivePost(_WireModel): + id: ArchiveID + authorId: ArchiveID + createdAt: Instant + repostOf: ArchiveID | None = None + conversationId: ArchiveID | None = None + replyTo: ReplyTo | None = None + quote: Quote | None = None + text: RichText | None = None + lang: str | None = None + source: str | None = None + mediaIds: list[ArchiveID] | None = pydantic.Field(default=None, min_length=1) + card: Card | None = None + article: Article | None = None + place: Place | None = None + counts: PostCounts | None = None + countsObservedAt: PostCountsObservedAt | None = None + editIds: list[ArchiveID] | None = pydantic.Field(default=None, min_length=2) + communityNote: CommunityNote | None = None + sensitive: bool | None = None + contentWarning: ContentWarning | None = None + community: Community | None = None + replyPolicy: str | None = None + paidPromotion: bool | None = None + aiGenerated: AIGenerated | None = None + + +class ArchiveBatch(_WireModel): + users: list[ArchiveUser] = pydantic.Field(default_factory=list, max_length=1000) + posts: list[ArchivePost] = pydantic.Field(default_factory=list, max_length=1000) + media: list[ArchiveMedia] = pydantic.Field(default_factory=list, max_length=1000) + + +class ArchiveQuery(_WireModel): + users: list[ArchiveID] = pydantic.Field(default_factory=list, max_length=1000) + posts: list[ArchiveID] = pydantic.Field(default_factory=list, max_length=1000) + media: list[ArchiveID] = pydantic.Field(default_factory=list, max_length=1000) + + +T = TypeVar("T") + + +class QueryResult(_WireModel, Generic[T]): + id: ArchiveID + status: Literal["found", "missing", "failed"] + data: T | None = None + error: str | None = None + + +class QueryResponse(_WireModel): + users: list[QueryResult[ArchiveUser]] + posts: list[QueryResult[ArchivePost]] + media: list[QueryResult[ArchiveMedia]] + + +class SubmitResult(_WireModel): + id: ArchiveID + status: Literal["accepted", "unchanged", "partial", "failed"] + reason: str | None = None + + +class SubmitSummary(_WireModel): + accepted: Count + unchanged: Count + partial: Count + failed: Count + total: Count + + +class SubmitResponse(_WireModel): + summary: SubmitSummary + users: list[SubmitResult] + posts: list[SubmitResult] + media: list[SubmitResult] + + +class SessionResponse(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", strict=True) + + authenticated: bool + registered: bool + username: str | None + expires_at: str | None = None + account_url: str | None = None + + +__all__ = [ + "ArchiveBatch", + "ArchiveID", + "ArchiveMedia", + "ArchivePost", + "ArchiveQuery", + "ArchiveUser", + "QueryResponse", + "QueryResult", + "SessionResponse", + "SubmitResponse", + "SubmitResult", + "SubmitSummary", +] diff --git a/extensions/twitter/archive_session.py b/extensions/twitter/archive_session.py new file mode 100644 index 00000000..0c24c0d8 --- /dev/null +++ b/extensions/twitter/archive_session.py @@ -0,0 +1,450 @@ +"""Browser session boundary for the x-monkey archive protocol.""" + +from __future__ import annotations + +from datetime import UTC, datetime +import html +import secrets +import time +import re +from urllib.parse import urlsplit + +import fastapi +import jwt +from fastapi import Depends, Request, Response +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse + +from app.business.peer import PeerManager +from app.database_contract.constants import ( + JWT_ALGORITHM, + JWT_AUDIENCE, + JWT_ISSUER, + JWT_ROLE, +) +from app.middleware import decode_peer_jwt, require_peer_jwt +from app.settings import settings +from .archive_schema import SessionResponse + +SESSION_USE_CLAIM = "session_use" +SESSION_USE_TWITTER_ARCHIVE = "twitter_archive" +SESSION_COOKIE_NAME = "inkcre_twitter_archive" +SESSION_COOKIE_PATH_SUFFIX = "/twitter/x-monkey/" +SESSION_RENEWAL_REMAINING_FRACTION = 0.5 +CONNECT_PEER_TOKEN_LIFETIME_SECONDS = 60 + +router = fastapi.APIRouter() + + +async def x_monkey_base_url() -> str | None: + """Return the Core's configured public protocol base, preserving its path prefix.""" + try: + base_url = (await PeerManager.get_current_config_async()).http_public_base_url + except (RuntimeError, ValueError): + return None + return f"{base_url.rstrip('/')}/twitter/x-monkey" if base_url else None + + +def _session_cookie_path(base_url: str) -> str: + return f"{urlsplit(base_url).path.rstrip('/')}/" + + +def _format_expiry(epoch_seconds: int) -> str: + return datetime.fromtimestamp(epoch_seconds, UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _decode_peer_token(token: str) -> dict | None: + try: + return decode_peer_jwt(token, settings.jwt_secret) + except jwt.InvalidTokenError: + return None + + +def _decode_session_cookie(request: Request) -> dict | None: + token = request.cookies.get(SESSION_COOKIE_NAME) + if not token: + return None + claims = _decode_peer_token(token) + if claims is None or claims.get(SESSION_USE_CLAIM) != SESSION_USE_TWITTER_ARCHIVE: + return None + return claims + + +def _issue_session_token() -> tuple[str, int]: + issued_at = int(time.time()) + lifetime = settings.twitter_x_monkey_jwt_lifetime_seconds + expires_at = issued_at + lifetime + token = jwt.encode( + { + "role": JWT_ROLE, + "iss": JWT_ISSUER, + "aud": JWT_AUDIENCE, + "iat": issued_at, + "exp": expires_at, + SESSION_USE_CLAIM: SESSION_USE_TWITTER_ARCHIVE, + }, + settings.jwt_secret, + algorithm=JWT_ALGORITHM, + ) + return token, expires_at + + +def _set_session_cookie( + response: Response, + *, + token: str, + expires_at: int, + base_url: str, +) -> None: + response.set_cookie( + SESSION_COOKIE_NAME, + token, + max_age=max(0, expires_at - int(time.time())), + path=_session_cookie_path(base_url), + secure=True, + httponly=True, + samesite="none", + ) + + +def _clear_session_cookie(response: Response, base_url: str | None) -> None: + path = _session_cookie_path(base_url) if base_url else SESSION_COOKIE_PATH_SUFFIX + response.delete_cookie( + SESSION_COOKIE_NAME, + path=path, + secure=True, + httponly=True, + samesite="none", + ) + + +def renew_session_if_needed( + response: Response, + claims: dict, + *, + base_url: str, +) -> int: + """Renew an eligible session JWT at half of its configured lifetime.""" + expires_at = int(claims["exp"]) + lifetime = settings.twitter_x_monkey_jwt_lifetime_seconds + if expires_at - int(time.time()) <= lifetime * SESSION_RENEWAL_REMAINING_FRACTION: + token, expires_at = _issue_session_token() + _set_session_cookie(response, token=token, expires_at=expires_at, base_url=base_url) + return expires_at + + +def _html_page( + body: str, *, status_code: int = 200, script_nonce: str | None = None +) -> HTMLResponse: + script_policy = f"script-src 'nonce-{script_nonce}'; " if script_nonce else "" + return HTMLResponse( + f""" + + + +x-monkey 连接 +

InKCre x-monkey 连接

{body}
""", + status_code=status_code, + headers={ + "Cache-Control": "no-store", + "Content-Security-Policy": ( + f"default-src 'none'; {script_policy}connect-src 'self'; form-action 'self'; " + "base-uri 'none'; frame-ancestors 'none'" + ), + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + }, + ) + + +def _request_origin(request: Request) -> str | None: + value = request.headers.get("origin") + if value is None: + return None + try: + parts = urlsplit(value) + # Accessing .port also rejects malformed authority syntax. + _ = parts.port + except ValueError: + return None + if ( + parts.scheme not in {"http", "https", "chrome-extension", "moz-extension"} + or not parts.netloc + or parts.username is not None + or parts.password is not None + or parts.path not in {"", "/"} + or parts.query + or parts.fragment + ): + return None + if parts.scheme in {"chrome-extension", "moz-extension"}: + if not re.fullmatch(r"[a-zA-Z0-9_-]+", parts.netloc): + return None + elif parts.scheme == "https" and parts.netloc.lower() == "x.com": + return "https://x.com" + return f"{parts.scheme}://{parts.netloc}" + + +async def _core_origin() -> str | None: + try: + base_url = (await PeerManager.get_current_config_async()).http_public_base_url + except (RuntimeError, ValueError): + return None + if not base_url: + return None + parts = urlsplit(base_url) + return f"{parts.scheme}://{parts.netloc}" + + +async def validate_archive_request_source(request: Request) -> bool: + """Apply the protocol's source policy; Origin is a filter, never a credential.""" + origin = _request_origin(request) + if origin is None: + # A missing Origin is expected for some extension-origin GM requests. + return "origin" not in request.headers + if origin in {"https://x.com"} or origin.startswith( + ("chrome-extension://", "moz-extension://") + ): + return True + return origin == await _core_origin() + + +def _is_top_level_navigation(request: Request) -> bool: + return ( + request.headers.get("sec-fetch-mode") == "navigate" + and request.headers.get("sec-fetch-dest") == "document" + ) + + +@router.post("/session", response_model=SessionResponse) +async def connect_session( + request: Request, + response: Response, + claims: dict = Depends(require_peer_jwt), +): + if _request_origin(request) != await _core_origin(): + return JSONResponse({"error": "same-origin request required"}, status_code=403) + if SESSION_USE_CLAIM in claims: + return JSONResponse({"error": "ordinary Peer credential required"}, status_code=401) + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + return JSONResponse({"error": "application/json required"}, status_code=415) + + base_url = await x_monkey_base_url() + if base_url is None: + return JSONResponse({"error": "public Core URL is not configured"}, status_code=503) + + token, expires_at = _issue_session_token() + _set_session_cookie(response, token=token, expires_at=expires_at, base_url=base_url) + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + return SessionResponse( + authenticated=True, + registered=True, + username=settings.peer_name, + expires_at=_format_expiry(expires_at), + account_url=f"{base_url}/account", + ) + + +@router.get("/account", response_class=HTMLResponse) +async def account_page(request: Request): + claims = _decode_session_cookie(request) + base_url = await x_monkey_base_url() + nonce = secrets.token_urlsafe(18) + connected_state = ( + '

Connected

' + f'

InKCre Peer: {html.escape(settings.peer_name)}

' + if claims is not None + else '

尚未连接。

' + '' + ) + expiry = ( + f'

会话到期时间：

" + if claims is not None + else ( + '' + ) + ) + disconnect = ( + f'
' + '
' + if base_url + else "" + ) + script = r""" +""" + for placeholder, value in ( + ("__NONCE__", nonce), + ("__JWT_ROLE__", JWT_ROLE), + ("__JWT_ISSUER__", JWT_ISSUER), + ("__JWT_AUDIENCE__", JWT_AUDIENCE), + ("__JWT_LIFETIME__", str(CONNECT_PEER_TOKEN_LIFETIME_SECONDS)), + ): + script = script.replace(placeholder, value) + return _html_page( + connected_state + + expiry + + "

连接 x-monkey

" + + '' + + '' + + '' + + "

密钥只在此页面内存中用于签发短时 Peer JWT，不会保存。

" + + disconnect + + "

断开只清除此浏览器中的会话，不会删除已保存的信息。

" + + script, + script_nonce=nonce, + ) + + +@router.post("/disconnect") +async def disconnect_session(request: Request): + if _request_origin(request) != await _core_origin() or not _is_top_level_navigation( + request + ): + return _html_page("

断开请求来源无效。

", status_code=403) + + base_url = await x_monkey_base_url() + if base_url is None: + return _html_page("

Core 尚未配置公共 HTTPS 地址。

", status_code=503) + response = RedirectResponse(f"{base_url}/account", status_code=303) + _clear_session_cookie(response, base_url) + return response + + +@router.get("/api/v1/session") +async def get_session(request: Request, response: Response): + from .archive_schema import SessionResponse + + base_url = await x_monkey_base_url() + response.headers["Cache-Control"] = "no-store" + response.headers["X-Content-Type-Options"] = "nosniff" + account_url = f"{base_url}/account" if base_url else None + fetch_mode = request.headers.get("sec-fetch-mode") + if fetch_mode == "no-cors" or not await validate_archive_request_source(request): + return fastapi.responses.JSONResponse( + {"error": "request source is not allowed"}, + status_code=403, + headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "no-store"}, + ) + claims = _decode_session_cookie(request) + if claims is None: + if request.cookies.get(SESSION_COOKIE_NAME): + _clear_session_cookie(response, base_url) + return SessionResponse( + authenticated=False, + registered=False, + username=None, + account_url=account_url, + ) + + if base_url is None: + return SessionResponse( + authenticated=False, + registered=False, + username=None, + account_url=None, + ) + expires_at = renew_session_if_needed(response, claims, base_url=base_url) + return SessionResponse( + authenticated=True, + registered=True, + username=settings.peer_name, + expires_at=_format_expiry(expires_at), + account_url=account_url, + ) + + +__all__ = [ + "SESSION_COOKIE_NAME", + "SESSION_RENEWAL_REMAINING_FRACTION", + "SESSION_USE_CLAIM", + "SESSION_USE_TWITTER_ARCHIVE", + "connect_session", + "disconnect_session", + "get_session", + "renew_session_if_needed", + "router", + "validate_archive_request_source", + "x_monkey_base_url", +] diff --git a/extensions/twitter/bookmark.py b/extensions/twitter/bookmark.py index eb75248d..bf6aa057 100644 --- a/extensions/twitter/bookmark.py +++ b/extensions/twitter/bookmark.py @@ -3,16 +3,12 @@ import pydantic import sqlmodel import typing -from app.business.info_base.commands import persist_stars -from app.business.info_base.resolver import ImageResolver, VideoResolver, HTMLResolver from app.business.source import SourceBase from app.persistence.source.uow import source_uow -from app.schemas.info_base.main import OutArcForm, StarsGraphForm -from app.schemas.info_base.relation import RelationForm from app.schemas.job import JobModel from .api import TwitterAPI -from .resolver import TweetResolver -from .schema import Tweet +from .model import TwitterBatch +from .reconcile import persist_twitter class SourceConfig(sqlmodel.SQLModel): @@ -28,51 +24,33 @@ class CollectConfig(pydantic.BaseModel): result_limit: int = pydantic.Field(default=40, ge=5, le=100) -def _video_url(video) -> str | None: - supported = tuple( - variant - for variant in video.variants - if variant.url and variant.content_type in {None, "video/mp4"} - ) - selected = max(supported, key=lambda variant: variant.bitrate or -1, default=None) - return selected.url if selected is not None else None - - -def tweet_to_graph(tweet) -> StarsGraphForm: - """Map one Twitter API DTO into the persisted root and relation-owned links.""" - canonical = Tweet( - id=tweet.id, - user_id=tweet.user_id, - conversation_id=tweet.conversation_id, - quote=tweet.quote, - text=tweet.text, - ) - videos = tuple( - (video, url) for video in tweet.videos if (url := _video_url(video)) is not None - ) - return StarsGraphForm( - block=TweetResolver.create_block(canonical), - out_arcs=tuple( - OutArcForm( - relation=RelationForm(content=f"attachment:photo:{photo.id}"), - to_graph=ImageResolver.create_graph(url=photo.url, alt_text=photo.alt_text), - ) - for photo in tweet.photos - ) - + tuple( - OutArcForm( - relation=RelationForm(content=f"attachment:video:{video.id}"), - to_graph=VideoResolver.create_graph(url=url), - ) - for video, url in videos - ) - + tuple( - OutArcForm( - relation=RelationForm(content="entities:url"), - to_graph=HTMLResolver.create_graph(url=url), - ) - for url in tweet.urls - ), +def _selected_batch(batch: TwitterBatch, primary_ids: tuple[str, ...]) -> TwitterBatch: + posts = {post.id: post for post in batch.posts} + selected = set(primary_ids) + pending = list(primary_ids) + while pending: + post = posts[pending.pop()] + references = [post.repostOf] + if post.quote is not None: + references.append(post.quote.postId) + if post.replyTo is not None: + references.append(post.replyTo.postId) + for native_id in references: + if native_id is not None and native_id in posts and native_id not in selected: + selected.add(native_id) + pending.append(native_id) + chosen = [post for post in batch.posts if post.id in selected] + authors = {post.authorId for post in chosen} + media_ids = {native_id for post in chosen for native_id in post.mediaIds or ()} + for post in chosen: + if post.article is not None: + media_ids.update(post.article.mediaIds or ()) + if post.article.coverMediaId is not None: + media_ids.add(post.article.coverMediaId) + return TwitterBatch( + posts=chosen, + users=[user for user in batch.users if user.id in authors], + media=[media for media in batch.media if media.id in media_ids], ) @@ -107,35 +85,26 @@ async def collect(self, job: JobModel, config: pydantic.BaseModel) -> None: api_client = await TwitterAPI.new() bookmarks_res = await api_client.get_bookmarks(page=page, max_results=result_limit) - # find new tweets start point - old_start_at = len(bookmarks_res.tweets) - if not full: - state = await self.get_state() - latest_tweet_id = state.get("latest_tweet_id") - if latest_tweet_id: - old_start_at = next( - ( - i for i, tweet in enumerate(bookmarks_res.tweets) if tweet.id == latest_tweet_id - ), - len(bookmarks_res.tweets), - ) - - collected = [] - for tweet in ( - bookmarks_res.tweets if full else reversed(bookmarks_res.tweets[:old_start_at]) - ): - collected.append(tweet_to_graph(tweet)) - async with source_uow() as uow: source = await uow.sources.get(self._id, lock=True) if source is None: raise ValueError("Twitter Source no longer exists") - for graph in reversed(collected) if full else collected: - await persist_stars(graph, uow.graph) - if not full and bookmarks_res.tweets: + primary_ids = bookmarks_res.primary_post_ids + selected = primary_ids + if not full: + latest_id = str((source.state or {}).get("latest_tweet_id", "")) + if latest_id in primary_ids: + selected = primary_ids[: primary_ids.index(latest_id)] + if selected: + await persist_twitter( + _selected_batch(bookmarks_res.batch, selected), + uow.graph, + text_policy="supplement", + ) + if not full and primary_ids: source.state = { **(source.state or {}), - "latest_tweet_id": bookmarks_res.tweets[0].id, + "latest_tweet_id": primary_ids[0], } await uow.sources.save(source) diff --git a/extensions/twitter/docs/global/.vitepress/config.mts b/extensions/twitter/docs/global/.vitepress/config.mts new file mode 100644 index 00000000..fe74a532 --- /dev/null +++ b/extensions/twitter/docs/global/.vitepress/config.mts @@ -0,0 +1,17 @@ +import { extensionDocs } from '../../../../../docs/_shared/website/extension-docs/preset.mts' + +export default { + ...extensionDocs({ + name: 'inkcre/twitter', + title: 'Twitter / X 扩展', + scope: 'global', + description: '通过 x-monkey 保存帖子，或采集自己的 X 书签，并在 InKCre 中检索和阅读。', + sidebar: [ + { text: '扩展概览', link: '/' }, + { text: '用 x-monkey 保存帖子', link: '/x-monkey' }, + { text: '授权与采集 Bookmark', link: '/bookmarks' }, + { text: '检索、阅读与媒体', link: '/reading' }, + ], + }), + lang: 'zh-CN', +} diff --git a/extensions/twitter/docs/global/.vitepress/theme/index.ts b/extensions/twitter/docs/global/.vitepress/theme/index.ts new file mode 100644 index 00000000..e23b0685 --- /dev/null +++ b/extensions/twitter/docs/global/.vitepress/theme/index.ts @@ -0,0 +1 @@ +export { default } from '../../../../../../docs/_shared/website/.vitepress/theme/index' diff --git a/extensions/twitter/docs/global/bookmarks.md b/extensions/twitter/docs/global/bookmarks.md new file mode 100644 index 00000000..6ed5b3e6 --- /dev/null +++ b/extensions/twitter/docs/global/bookmarks.md @@ -0,0 +1,181 @@ +--- +description: 授权自己的 X 账号，创建 Bookmark Source 并观察一次采集。 +outline: false +--- + +# 授权与采集 Bookmark + +Bookmark 采集使用 X 官方 API 读取自己账号的书签，与 +[x-monkey 连接](./x-monkey.md)独立。你需要可访问相关 API 的 X 开发者应用，以及一条能辨认正文的近期书签。开始前检查应用的访问权限和计费；具体条件以 +[X Bookmarks 文档](https://docs.x.com/x-api/posts/bookmarks/introduction)为准。 + +这条路径使用 OAuth 2.0 用户授权。应用级 app-only bearer token 不能代替账号授权；不要把 InKCre 的 +`JWT_SECRET` 填入 X。 + + + + + + +## 采集范围与停止方式 + +一次普通采集只读取一页，`result_limit` +为 5–100，默认 40；命中已记录的上一条主书签时，只保存它之前的新主书签，未命中则保存返回的本页。书签顺序变化或两次采集之间新增过多内容时，不能据此保证全部覆盖。 + +`full=true` 只取消本页的旧书签截断，并在 Job +state 记录下一页位置；当前实现不会自动提交下一页 Job 或排空历史，也没有独立的 Source +backfill 实现。不要把它当作完整回填、文件夹同步、取消书签同步或全部回复采集。 + +停止自动采集时先禁用对应 Cron，再在向导账号步骤选择 +**Disconnect**。也可以在 X 撤销应用。它们都不删除已经保存的信息；断开 x-monkey 会话则不会代替停止 Bookmark 计划。 + +## 授权或采集失败时 + +- **setup 404**：检查 Core 地址及 Twitter 在该 Core 上的运行状态。 +- **setup 401**：检查当前 InKCre Peer 连接；它与 X 的账号授权不同。 +- **X 授权失效 / 要求重新连接**：重新授权，核对账号和应用是否被替换或撤销。 +- **X 返回 402/403**：检查应用访问权限、额度和账号授权；**429** 表示应等待供应商限流恢复。 +- **Job 完成但没有新信息**：核对连接的 handle，新增一条可辨认的书签后再运行，不重置已有 Source 游标来试错。 + +另有 `twikit` 账号登录后端；它不是上述 OAuth 向导流程。运行人员可从 Python +scope 查看实际配置和本地会话文件要求。 + +返回[扩展概览](./index.md)。 diff --git a/extensions/twitter/docs/global/images/x-monkey-read.jpg b/extensions/twitter/docs/global/images/x-monkey-read.jpg new file mode 100644 index 00000000..9aff8b29 Binary files /dev/null and b/extensions/twitter/docs/global/images/x-monkey-read.jpg differ diff --git a/extensions/twitter/docs/global/images/x-monkey-sync.jpg b/extensions/twitter/docs/global/images/x-monkey-sync.jpg new file mode 100644 index 00000000..be3fb7f4 Binary files /dev/null and b/extensions/twitter/docs/global/images/x-monkey-sync.jpg differ diff --git a/extensions/twitter/docs/global/index.md b/extensions/twitter/docs/global/index.md new file mode 100644 index 00000000..6f07fd33 --- /dev/null +++ b/extensions/twitter/docs/global/index.md @@ -0,0 +1,38 @@ +--- +description: 通过 x-monkey 保存帖子，或采集自己的 X 书签，并在 InKCre 中检索和阅读。 +--- + +# Twitter / X 扩展 + +`inkcre/twitter` +把 X 上的信息保存到共享 info-base，提供两种独立入口。保存后的帖子、作者和媒体元数据属于信息库，不依赖采集脚本或某个 Peer 持续运行。 + +| 你要做什么 | 使用路径 | 需要的授权 | +| -------------------------- | ------------------------------------- | --------------------------------------------------------------------- | +| 保存浏览时已捕获的帖子 | [连接 x-monkey](./x-monkey.md) | 在自己的 Core 账户页使用现有 `JWT_SECRET` 连接，不需要 X OAuth 应用。 | +| 定期采集自己的书签 | [授权与采集 Bookmark](./bookmarks.md) | X OAuth 应用、自己的 X 账号授权，以及 Source 和采集 Job。 | +| 保存后找回、阅读和查看媒体 | [检索、阅读与媒体](./reading.md) | 已连接同一 info-base 的 InKCre 界面，以及兼容的 Twitter 阅读分发。 | + +两种入口按原生帖子 ID 补充同一份信息。已捕获的正文、长文、作者和上下文可以继续使用；缺少的内容不会被编造，也不会在阅读时补抓完整线程。 + +## 先准备扩展 + +在实际执行能力的 Peer 上安装、启用兼容的 Distribution。Core 运行 Python 分发，负责协议接入、x-monkey 账户页和 Bookmark 采集；Web 的 Module +Federation 分发提供 Setup 和内容视图。x-monkey 可以直接从 Core 账户页连接，不要求安装 Web 分发；需要在 Web 中配置或阅读时，先[连接 InKCre](https://inkcre.dev/guide/connect)。各分发独立选择版本，不要求版本数字相同。 + +Python 分发要求 Core Host SDK `>=0.3.1 <0.4.0`，Web 分发要求 `@inkcre/core` Host SDK +`>=0.4.0 <0.5.0`。仅安装扩展并不表示它已经在所选 Peer 上运行。x-monkey 和 OAuth 回调需要浏览器可访问的 Core 公共 HTTPS 地址；服务地址保留部署路径前缀，Web +Setup 也可以提供地址供复制。 + +运行人员可从同一 Release 的 Registry 文档入口选择 **Python** scope，查看运行配置、采集参数和 setup +API；本组 **global** 文档说明用户工作流。不同 scope 是独立文档站，不通过拼接 URL 切换。 + +## 已保存的内容与限制 + +本版本保存原生帖子、作者、媒体元数据和已捕获引用，不自动转存图片或视频文件。只有用户明确加载媒体时才读取源站字节；源站失效后，文字和元数据仍保留。具体行为见[媒体边界](./reading.md#媒体按需读取)。 + +保存内容采用 Twitter 扩展自己的共同模型。各入口只写入当前适配器已支持的源事实，不保存官方 API 或 Twikit 的完整原始响应；未知且未映射的字段不能当作已经归档的信息。 + +x-monkey 接入不提供远端历史浏览、搜索恢复或未捕获线程补抓。Bookmark 的一次 Job 读取一页，不保证全量回填或同步删除。采集完成与检索索引维护是两个动作，新增内容需要按既有流程维护索引后再搜索。 + +操作截图来自隔离预览，连接时使用自己的 Core 服务地址，不照抄截图中的预览域名。 diff --git a/extensions/twitter/docs/global/reading.md b/extensions/twitter/docs/global/reading.md new file mode 100644 index 00000000..a176d638 --- /dev/null +++ b/extensions/twitter/docs/global/reading.md @@ -0,0 +1,40 @@ +--- +description: 找回已保存的帖子，阅读正文和上下文，并显式加载外部媒体。 +--- + +# 检索、阅读与媒体 + +[x-monkey](./x-monkey.md) 与 [Bookmark](./bookmarks.md) +保存的信息都可从 InKCre 独立使用。关闭脚本、断开提交会话或停止采集计划，不会删除已经写入 info-base 的记录。Web 需要能够解释当前 Twitter 记录的兼容分发。 + +## 保存后找回同一帖子 + +1. 先确认脚本的逐项同步结果，或 Bookmark 采集 Job 的实际完成状态。 +2. 按既有[索引与搜索流程](https://inkcre.dev/guide/search)维护词法索引，等待维护完成。提交和采集不会隐式维护索引，不需要为了这条检索路径配置 AI 凭据。 +3. 用原帖 ID、原始链接、作者或正文线索搜索，在 Info Base 打开结果。检查正文、长文和 **Original + post** 出处；已捕获的作者及引用、回复、转帖上下文可以继续导航。 + +只知道原生 ID 的引用会显示为未捕获的身份，不伪造正文或作者资料。纯转帖也不补出一段假正文。旧 v1 帖子仍保留读取能力；重新采集到足够原生事实后可沿用原 Block +ID 升级，既有用户引用继续有效。历史上已经存在的重复不会在本版本全量清理。 + +## 媒体按需读取 + +图片、视频或 GIF 初始只显示已经捕获的元数据。需要查看时点击 **Load +media**；Twitter 阅读分发才从已捕获的媒体 URL 经 HTTP +Storage 读取字节。保存、query、列表、正文投影与索引不会提前下载这些原生媒体。 + +![同一帖子在 InKCre 中打开，显示原帖链接、正文与显式加载的照片](images/x-monkey-read.jpg) + +这是隔离预览中真实保存的同一帖子，照片在用户明确加载后显示。本版本不预建持久媒体内容 Block，不转存媒体文件，也不创建媒体下载 Job；**Load +media** 不会将字节保存在信息库。 + +图片使用已捕获的 URL，视频和 GIF 从已捕获的可用 MP4 variants 中选择最高码率；随后由现有 HTTP +Storage 按通用 HTTP(S)、超时、体积与重定向规则读取。HLS 不作为视频本体加载，URL 和其他元数据仍保留，即使它们不能用于当前读取。 + +## 来源失效或内容不完整时 + +源站 URL 失效、超时、体积超限或没有可用地址时，界面会反馈媒体不可用；文字和元数据仍可读取。重新打开内容不代表媒体已经长期保全，不应把“已同步”理解为文件备份成功。 + +阅读只解释已经保存的内容，不额外抓取未捕获的完整线程、引用正文或历史。x-monkey 与 Bookmark 的源信息密度不同，后来的稀疏信息不会抹掉未提交的已知内容；计数按各项实际观察时间合并，较旧观察不会覆盖较新值。 + +返回[扩展概览](./index.md)。 diff --git a/extensions/twitter/docs/global/x-monkey.md b/extensions/twitter/docs/global/x-monkey.md new file mode 100644 index 00000000..3bd080a1 --- /dev/null +++ b/extensions/twitter/docs/global/x-monkey.md @@ -0,0 +1,61 @@ +--- +description: 将原版 x-monkey 连接到自己的 InKCre，保存已捕获的帖子。 +--- + +# 用 x-monkey 保存帖子 + +此入口接收原版 [x-monkey](https://github.com/steven-vvv/x-monkey) +捕获的帖子、作者和媒体元数据，不需要 X OAuth 应用、Bookmark Source 或采集计划。兼容基线为 x-monkey +`v0.2.0`。 + +## 连接自己的 InKCre + +准备可从浏览器访问的 Core 公共 HTTPS 地址、该部署现有的 +`JWT_SECRET`，以及正在使用的原版 x-monkey 脚本。在 Core 安装并启用兼容的 Twitter +Python 分发；建立此连接不要求安装 InKCre Web 或 Twitter Web 分发。 + +1. 在 x-monkey 打开 **设置 → 远端数据库**，启用远端数据库，将 Core 公共基址加 `/twitter/x-monkey` + 填入 **服务地址**，保存并点击 **检测**。地址必须保留部署路径前缀。 +2. 显示“需要登录”时，点击“打开账户页”，进入同一服务地址下的 + `/account`。先确认地址栏是自己信任的 Core HTTPS 部署。 +3. 在该账户页输入这个部署已有的 `JWT_SECRET`，点击 **Connect**。页面在浏览器内签发短期 Peer + JWT，Core 验证后建立归档会话；不要把密钥填进脚本或 X 页面。 +4. 页面检测到浏览器中的实际会话后，显示 **Connected**、已连接的 Peer 和会话到期时间。返回脚本重新 + **检测**，确认已连接再开始同步。 + +如果使用 InKCre Web，也可以在 Twitter **Setup** 中选择 Core，复制 **x-monkey service address** +并通过普通链接打开账户页，完成相同步骤。该链接不携带凭据。 + +密钥输入只用于当前页面的本地签名，页面不持久化它，也不把它发送给服务器；服务器接收的是签发出的 Peer +JWT。只在自己信任的部署页面输入密钥：浏览器内签名不能保护被篡改或恶意的页面，它们仍可能读取输入。页面不加载第三方脚本，不接受任意回跳地址。 + +## 同步已捕获的帖子 + +在 X 打开帖子，让 x-monkey 捕获它，再在脚本中打开该帖详情。找到 **远端归档**，点击 +**同步此帖**，检查帖子、作者及媒体的逐项结果。**同步全部**也可能包含已经捕获的上下文；它不会让 InKCre 补抓尚未捕获的完整线程或历史。 + +![原版 x-monkey 显示帖子、作者和媒体元数据同步成功](images/x-monkey-sync.jpg) + +图中是预览环境的真实同步结果，展示的是连接后的同步操作；使用自己配置的服务地址，记录数量取决于脚本捕获的内容。“媒体已同步”表示媒体元数据已保存，不表示图片或视频文件已经转存。 + +再次同步同一原生帖子会补充同一个信息根，保留未提交的已知字段，并按计数观察时间合并更新。Bookmark 采集到同一帖子时也补充这份信息。较旧捕获可能继续显示计数差异；不能仅凭差异判断保存失败。 + +保存后，按[检索与阅读](./reading.md)找回同一帖子。关闭脚本或结束提交会话后，已保存的信息仍在。 + +## 续签、过期与断开 + +此会话具有现有 Peer +JWT 的完整信息库访问权限，默认有效期为 7 天。有效的会话检测、查询和提交请求在剩余期限不超过当前配置寿命的一半时自动续签。部署者可以调整签发期限；普通 Core 请求不会续签该会话。 + +过期后需回到同一 Core 账户页重新连接，不需要额外的 refresh +token。活跃会话没有从首次连接开始计算的绝对续签上限。账户页的断开操作只清除此浏览器的会话，不删除已保存信息，也不撤销已经复制的 JWT。 + +## 检测或同步失败时 + +- **需要登录 / + 401**：打开自己 Core 的账户页重新连接，然后返回脚本检测。不要把 Core 的 JWT_SECRET 填入脚本来代替连接流程。 +- **账户页或协议返回 404**：检查服务地址、部署路径前缀，以及 Twitter 是否正在所选 Core 上运行。 +- **结果失败或请求超时**：保留捕获内容，检查实际请求结果再重试。成功保存后重复提交仍补充同一帖子，不必创建第二个存储后端。 +- **媒体读取失败**：按[媒体边界](./reading.md#媒体按需读取)处理；元数据同步成功不保证外部媒体 URL 长期可用。 + +返回[扩展概览](./index.md)，或继续配置独立的 [Bookmark 采集](./bookmarks.md)。 diff --git a/extensions/twitter/docs/python/.vitepress/config.mts b/extensions/twitter/docs/python/.vitepress/config.mts new file mode 100644 index 00000000..5aebf784 --- /dev/null +++ b/extensions/twitter/docs/python/.vitepress/config.mts @@ -0,0 +1,12 @@ +import { extensionDocs } from '../../../../../docs/_shared/website/extension-docs/preset.mts' + +export default { + ...extensionDocs({ + name: 'inkcre/twitter', + title: 'Twitter Core 运行与配置', + scope: 'python', + description: '在 Core 上运行 Twitter 的协议接入和 Bookmark 采集，配置授权及运行参数。', + sidebar: [{ text: '运行与配置', link: '/' }], + }), + lang: 'zh-CN', +} diff --git a/extensions/twitter/docs/python/.vitepress/theme/index.ts b/extensions/twitter/docs/python/.vitepress/theme/index.ts new file mode 100644 index 00000000..e23b0685 --- /dev/null +++ b/extensions/twitter/docs/python/.vitepress/theme/index.ts @@ -0,0 +1 @@ +export { default } from '../../../../../../docs/_shared/website/.vitepress/theme/index' diff --git a/extensions/twitter/docs/python/index.md b/extensions/twitter/docs/python/index.md new file mode 100644 index 00000000..b4e3e866 --- /dev/null +++ b/extensions/twitter/docs/python/index.md @@ -0,0 +1,110 @@ +--- +description: Twitter Python 分发的宿主要求、公共地址、授权配置与采集参数。 +--- + +# Core 运行与配置 + +Python 分发在所选 Core 上提供 x-monkey 归档协议与自托管账户页、Bookmark Source 及 Twitter +Resolver。连接 x-monkey 不要求 Web 分发；需要 Web Setup 或阅读界面时，再选择兼容的 Module +Federation 分发。用户工作流由本扩展的 **global** 文档说明。 + +Python 分发要求 Python `>=3.12,<3.13` 和 Core Host SDK +`>=0.3.1 <0.4.0`。选择精确分发、启用意图和实际运行属于相应 Peer；扩展配置与账号状态由部署共享。 + +## 公共地址与生命周期 + +在执行该能力的 Core Peer 配置 +`http_public_base_url`，使用浏览器或脚本实际可访问的 HTTPS 基址。Setup 由此产生 +`/twitter/auth/callback` 和 `/twitter/x-monkey` 地址，x-monkey 账户页位于后者的 +`/account`；所有地址保留部署路径前缀。不要使用 PostgREST 地址代替 Core 地址，也不要把预览环境地址作为其他部署的默认值。 + +启用 Python 分发后才会注册 Source、Resolver 和 HTTP 路由；停用撤回入口，不删除已保存的图信息。Web 独立启用不会让离线 Core 执行采集，安装状态也不等于实际运行状态。多 Peer 可以具备采集能力，但同一 Source 不应配置重复的采集计划。 + +## 扩展配置 + +下表是 `TwitterExtensionConfig` +的实际字段，通过既有扩展配置管理接口修改。OAuth 向导会同时维护相应账号状态，优先使用向导配置官方后端。 + +| 字段 | 默认值 | 用途 | +| ------------------------------- | ---------- | --------------------------------------------------------------- | +| `backend` | `official` | `official` 使用 X OAuth 用户授权；`twikit` 使用账号登录。 | +| `client_id`、`client_secret` | 空字符串 | 官方 OAuth 应用的 Client ID/Secret，不是 API Key/Secret。 | +| `email`、`username`、`password` | 空字符串 | `twikit` 登录所需账号信息。 | +| `totp_secret` | `null` | `twikit` 可选 TOTP 秘密。 | +| `api_language` | `en-US` | 传给 `twikit` 客户端的请求语言。 | +| `proxy` | `null` | 传给 `twikit` 客户端的代理；当前官方 OAuth 客户端不读取此字段。 | + +保存 OAuth 应用会把 `backend` 设为 `official`。如果应用变更会影响已有账号或待完成授权，setup +API 要求显式 +`confirm_account_reset`；配置指纹改变会清除不再匹配的账号和授权事务。不要盲目确认重置或直接编辑持久 state 来绕过它。 + +官方后端把账号 token、user +ID、handle 和续期状态保存在扩展 state；账号授权属于整个扩展，不属于某个 Source。官方客户端使用当前授权刷新 X +token，供应商拒绝授权时标记需要重新连接。凭据及授权链接不应进入日志或截图。 + +`twikit` 在实际采集时登录，使用 `data/extensions/twitter/twikit_cookies.json` +作为运行实例的 cookie 文件。该路径需要可写并保护其中的会话数据；它不是共享 info-base 的账号状态，也不会自动在多个 Core 之间同步。OAuth 向导不能代替这套账号登录配置。改变后端或账号后应在正常生命周期中停用并重新启用，以结束旧的本地客户端。 + +## setup API 与认证边界 + +运行人员或 Agent 可通过既有 Peer HTTP 方式调用下列 Core 路由。setup API 要求 Peer JWT +Bearer；CLI 当前没有专用 OAuth 设置命令。 + +| 方法与路径 | 作用 | +| ---------------------------------------- | ----------------------------------------------------------------------- | +| `GET /twitter/setup` | 读取 callback URL、可用 x-monkey 基址和账号连接状态。 | +| `PUT /twitter/setup/oauth-app` | 提交 `client_id`、`client_secret`，必要时明确 `confirm_account_reset`。 | +| `POST /twitter/setup/oauth-transactions` | 创建十分钟有效的授权事务，返回供用户打开的私有授权链接。 | +| `POST /twitter/setup/oauth-transaction` | 以 `transaction_id` 读取该事务结果。 | +| `DELETE /twitter/setup/account` | 断开扩展当前连接的 X 账号，不删除已采集信息。 | + +用户仍需在 X 页面批准正确账号，浏览器回调为 +`/twitter/auth/callback`。新建授权会替代此前未完成的尝试；替换应用或账号会影响所有 Bookmark +Sources。 + +x-monkey 的 +`/twitter/x-monkey/api/v1/session`、`/twitter/x-monkey/api/v1/archive/query`、`/twitter/x-monkey/api/v1/archive/submit` +使用协议专用 Cookie，与以上 Bearer 路由分开。首次连接从 Core 自托管的 +`GET /twitter/x-monkey/account` 开始：用户在可信部署页面输入现有 +`JWT_SECRET`，页面用 WebCrypto 在浏览器内签发 60 秒的普通 Peer JWT，并以同 Origin +`Authorization: Bearer` 和 JSON `{}` 请求 `POST /twitter/x-monkey/session`。该路由复用 +`require_peer_jwt`，要求 Origin 精确匹配 Core 且 JWT 不带 +`session_use`。Core 为相同完整 Peer 权限签发带 `session_use=twitter_archive` +的会话 JWT，Cookie 路径不构成降权。Cookie 为 Secure、HttpOnly、SameSite=None，要求正式链路使用 HTTPS；普通 Core 保护路由不读取它。 + +连接路由返回 `200 SessionResponse` JSON 和 Cookie。页面随后请求 +`GET /twitter/x-monkey/api/v1/session`，确认 `authenticated=true` 后才显示 +**Connected**、Peer 与到期时间；不能只凭 POST 成功认为浏览器已接受 Cookie。 + +账户页用随机 nonce 限定脚本，CSP 的 `connect-src 'self'` +只允许同源请求。它不持久化或经网络发送密钥，不加载第三方脚本，不接受任意回跳；密钥输入与签名由该部署提供的页面拥有，只应信任自己确认的 HTTPS 页面。页面被篡改时，本地签名不能保护输入。用户完成后返回脚本再次检测。可选的 Web +Setup 仅展示 `x_monkey_base_url` 和账户页普通链接，不参与签名或凭据传递。Web 分发的 Host SDK 范围为 +`@inkcre/core >=0.4.0 <0.5.0`，Python Host 范围保持不变。 + +## x-monkey 寿命配置 + +`TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS` 是 Core 环境配置，不是扩展 JSON 配置字段。默认 `604800` +秒，即 7 天，只接受正整数。首次连接与续签使用同一配置；有效协议请求在剩余期限不超过当前配置寿命的一半时续签。 + +此值是以后签发 token 的默认期限，不是 JWT 准入上限；降低它不会撤销仍有效的既有 token。它与普通 Core 签发使用的 +`JWT_DEFAULT_LIFETIME_SECONDS` 分开。没有额外 PAT、refresh +token、会话表或从首次授权计算的绝对上限。断开只清理当前浏览器 Cookie，不能撤销复制的 JWT；部署更换原有 JWT_SECRET 后按既有机制失效。 + +## Bookmark Source 与采集参数 + +Source 类型为 `extensions.twitter.bookmark.Source`，Source +config 当前为空对象。普通采集通过既有 Source Job 执行，参数如下： + +| 参数 | 默认值 | 实际行为 | +| -------------- | ------- | -------------------------------------------------------------------------------------------------------------- | +| `result_limit` | `40` | 整数，5–100，作为本次一页请求的结果上限。 | +| `full` | `false` | `false` 时按 `latest_tweet_id` 截断本页旧书签并更新主书签游标；`true` 时不截断，并将下一页位置记到 Job state。 | + +每次 Job 只请求一页，不自动提交后续 Job；`full` +不等于完整回填。此 Source 没有独立 backfill 实现。分页位置和 Source 游标是运行状态，不应当成账号配置手工重置来排查授权故障。外部请求失败与数据库写入失败不能被报告为成功采集。 + +媒体只保存原生元数据，Python 写入不创建持久图片/视频内容 Block、媒体 blob 或下载 Job。只有消费端明确展开时才通过现有 HTTP +Storage 读取源站字节。词法索引维护独立执行，不在 submit 或 collect 中隐式完成。 + +持久内容由 Twitter 自有共同模型拥有，x-monkey 固定 DTO 在适配边界转换。官方 API 和 Twikit 只投影当前支持的字段，不保存全部原始响应。消费端选择图片或最高码率的可用 MP4 后，由通用 HTTP +Storage 负责 HTTP(S)、超时、体积与重定向；Twitter 不另设 CDN 或固定端口限制。 diff --git a/extensions/twitter/model.py b/extensions/twitter/model.py new file mode 100644 index 00000000..22775342 --- /dev/null +++ b/extensions/twitter/model.py @@ -0,0 +1,391 @@ +"""Twitter-owned persisted facts, independent of capture-provider wire contracts.""" + +from __future__ import annotations + +import datetime +import re +from typing import Annotated, Literal + +import pydantic + + +TwitterID = Annotated[str, pydantic.Field(strict=True, pattern=r"^[1-9][0-9]*$")] +Count = Annotated[int, pydantic.Field(strict=True, ge=0)] +Number = Annotated[int | float, pydantic.Field(allow_inf_nan=False)] + + +def _validate_instant(value: str) -> str: + if not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", value): + raise ValueError("must be a UTC timestamp with second precision") + datetime.datetime.fromisoformat(value) + return value + + +def _validate_observed_at(value: str) -> str: + if not re.fullmatch( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z", value + ): + raise ValueError("must be a UTC ISO-8601 timestamp including seconds") + datetime.datetime.fromisoformat(value) + return value + + +Instant = Annotated[str, pydantic.AfterValidator(_validate_instant)] +ObservedAt = Annotated[str, pydantic.AfterValidator(_validate_observed_at)] + + +def _validate_range(value: tuple[int, int]) -> tuple[int, int]: + if value[1] < value[0]: + raise ValueError("range end before start") + return value + + +Range = Annotated[ + tuple[Count, Count], + pydantic.Field(strict=False), + pydantic.AfterValidator(_validate_range), +] +Rect = Annotated[tuple[Count, Count, Count, Count], pydantic.Field(strict=False)] + + +class _TwitterModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid", strict=True, allow_inf_nan=False) + + @pydantic.field_validator("*") + @classmethod + def reject_explicit_null(cls, value: object) -> object: + if value is None: + raise ValueError("omit unknown fields instead of sending null") + return value + + +class UrlEntity(_TwitterModel): + type: Literal["url"] + range: Range + expanded: str + + +class MentionEntity(_TwitterModel): + type: Literal["mention"] + range: Range + userId: TwitterID + + +class HashtagEntity(_TwitterModel): + type: Literal["hashtag"] + range: Range + + +class CashtagEntity(_TwitterModel): + type: Literal["cashtag"] + range: Range + name: str | None = None + ticker: str | None = None + + +class TimestampEntity(_TwitterModel): + type: Literal["timestamp"] + range: Range + + +Entity = Annotated[ + UrlEntity | MentionEntity | HashtagEntity | CashtagEntity | TimestampEntity, + pydantic.Field(discriminator="type"), +] + + +class TextStyle(_TwitterModel): + range: Range + styles: list[str] = pydantic.Field(min_length=1) + + +class RichText(_TwitterModel): + text: str + visible: Range | None = None + entities: list[Entity] | None = pydantic.Field(default=None, min_length=1) + styles: list[TextStyle] | None = pydantic.Field(default=None, min_length=1) + + +class UserCounts(_TwitterModel): + followers: Count | None = None + following: Count | None = None + posts: Count | None = None + mediaPosts: Count | None = None + likes: Count | None = None + + +class UserCountsObservedAt(_TwitterModel): + followers: ObservedAt | None = None + following: ObservedAt | None = None + posts: ObservedAt | None = None + mediaPosts: ObservedAt | None = None + likes: ObservedAt | None = None + + +class Affiliation(_TwitterModel): + name: str + url: str | None = None + badgeUrl: str | None = None + kind: str | None = None + ownerId: TwitterID | None = None + + +class Verification(_TwitterModel): + blue: bool | None = None + type: str | None = None + + +class Professional(_TwitterModel): + type: str | None = None + categories: list[str] | None = pydantic.Field(default=None, min_length=1) + + +class TwitterUser(_TwitterModel): + id: TwitterID + handle: str + name: str + createdAt: Instant | None = None + avatarUrl: str | None = None + avatarShape: str | None = None + bannerUrl: str | None = None + bio: RichText | None = None + location: str | None = None + website: str | None = None + pinnedPostIds: list[TwitterID] | None = pydantic.Field(default=None, min_length=1) + protected: bool | None = None + sensitive: bool | None = None + interstitial: str | None = None + withheldIn: list[str] | None = pydantic.Field(default=None, min_length=1) + verified: Verification | None = None + affiliation: Affiliation | None = None + parody: str | None = None + professional: Professional | None = None + counts: UserCounts | None = None + countsObservedAt: UserCountsObservedAt | None = None + + +class VideoVariant(_TwitterModel): + url: str + type: str + bitrate: Count | None = None + + +class MediaTag(_TwitterModel): + userId: TwitterID + handle: str | None = None + + +class Video(_TwitterModel): + durationMs: Count | None = None + variants: list[VideoVariant] = pydantic.Field(min_length=1) + + +class TwitterMedia(_TwitterModel): + id: TwitterID + type: Literal["photo", "video", "gif"] + url: str + width: Count | None = None + height: Count | None = None + alt: str | None = None + crops: list[Rect] | None = pydantic.Field(default=None, min_length=1) + faces: list[Rect] | None = pydantic.Field(default=None, min_length=1) + tags: list[MediaTag] | None = pydantic.Field(default=None, min_length=1) + video: Video | None = None + sourcePostId: TwitterID | None = None + sourceUserId: TwitterID | None = None + sensitive: list[str] | None = pydantic.Field(default=None, min_length=1) + unavailable: str | None = None + title: str | None = None + description: str | None = None + link: str | None = None + + +class PollChoice(_TwitterModel): + label: str + count: Count | None = None + + +class Poll(_TwitterModel): + choices: list[PollChoice] = pydantic.Field(min_length=2) + endsAt: Instant | None = None + final: bool | None = None + updatedAt: Instant | None = None + + +class CardImage(_TwitterModel): + url: str + width: Count | None = None + height: Count | None = None + alt: str | None = None + + +class CardPlayer(_TwitterModel): + url: str + width: Count | None = None + height: Count | None = None + + +class Card(_TwitterModel): + type: str + url: str + title: str | None = None + description: str | None = None + domain: str | None = None + image: CardImage | None = None + siteUserId: TwitterID | None = None + creatorUserId: TwitterID | None = None + player: CardPlayer | None = None + poll: Poll | None = None + extra: dict[str, str | bool] | None = None + + +class ArticleBlock(_TwitterModel): + type: str + text: str + styles: list[Annotated[tuple[Count, Count, str], pydantic.Field(strict=False)]] | None = ( + pydantic.Field(default=None, min_length=1) + ) + entities: ( + list[Annotated[tuple[Count, Count, Count], pydantic.Field(strict=False)]] | None + ) = pydantic.Field(default=None, min_length=1) + data: dict[str, pydantic.JsonValue] | None = None + + +class ArticleEntity(_TwitterModel): + type: str + data: dict[str, pydantic.JsonValue] + + +class Article(_TwitterModel): + id: TwitterID + title: str + preview: str | None = None + coverMediaId: TwitterID | None = None + publishedAt: Instant | None = None + modifiedAt: Instant | None = None + blocks: list[ArticleBlock] | None = pydantic.Field(default=None, min_length=1) + entities: list[ArticleEntity] | None = pydantic.Field(default=None, min_length=1) + mediaIds: list[TwitterID] | None = pydantic.Field(default=None, min_length=1) + + +class Place(_TwitterModel): + id: str + name: str | None = None + fullName: str | None = None + country: str | None = None + countryCode: str | None = None + type: str | None = None + bbox: ( + Annotated[tuple[Number, Number, Number, Number], pydantic.Field(strict=False)] | None + ) = None + + +class PostCounts(_TwitterModel): + replies: Count | None = None + reposts: Count | None = None + quotes: Count | None = None + likes: Count | None = None + bookmarks: Count | None = None + views: Count | None = None + + +class PostCountsObservedAt(_TwitterModel): + replies: ObservedAt | None = None + reposts: ObservedAt | None = None + quotes: ObservedAt | None = None + likes: ObservedAt | None = None + bookmarks: ObservedAt | None = None + views: ObservedAt | None = None + + +class ReplyTo(_TwitterModel): + postId: TwitterID + userId: TwitterID | None = None + handle: str | None = None + tombstone: str | None = pydantic.Field(default=None, min_length=1) + + +class Quote(_TwitterModel): + postId: TwitterID + url: str | None = None + tombstone: str | None = pydantic.Field(default=None, min_length=1) + + +class CommunityNote(_TwitterModel): + id: TwitterID | None = None + title: str | None = None + text: RichText + lang: str | None = None + url: str | None = None + + +class ContentWarning(_TwitterModel): + title: str + text: str | None = None + + +class Community(_TwitterModel): + id: TwitterID + name: str | None = None + + +class AIGenerated(_TwitterModel): + media: bool + source: str + + +class TwitterPost(_TwitterModel): + id: TwitterID + authorId: TwitterID + createdAt: Instant + repostOf: TwitterID | None = None + conversationId: TwitterID | None = None + replyTo: ReplyTo | None = None + quote: Quote | None = None + text: RichText | None = None + lang: str | None = None + source: str | None = None + mediaIds: list[TwitterID] | None = pydantic.Field(default=None, min_length=1) + card: Card | None = None + article: Article | None = None + place: Place | None = None + counts: PostCounts | None = None + countsObservedAt: PostCountsObservedAt | None = None + editIds: list[TwitterID] | None = pydantic.Field(default=None, min_length=2) + communityNote: CommunityNote | None = None + sensitive: bool | None = None + contentWarning: ContentWarning | None = None + community: Community | None = None + replyPolicy: str | None = None + paidPromotion: bool | None = None + aiGenerated: AIGenerated | None = None + + +class TwitterIdentity(_TwitterModel): + """A known native identity without captured entity content.""" + + id: TwitterID + + +class TwitterBatch(_TwitterModel): + """Facts captured for one common reconcile operation; transport limits stay outside.""" + + users: list[TwitterUser] = pydantic.Field(default_factory=list) + posts: list[TwitterPost] = pydantic.Field(default_factory=list) + media: list[TwitterMedia] = pydantic.Field(default_factory=list) + + +TwitterRecord = TwitterUser | TwitterPost | TwitterMedia +TextPolicy = Literal["replace", "supplement"] + + +__all__ = [ + "TextPolicy", + "TwitterBatch", + "TwitterID", + "TwitterIdentity", + "TwitterMedia", + "TwitterPost", + "TwitterRecord", + "TwitterUser", +] diff --git a/extensions/twitter/pdm.lock b/extensions/twitter/pdm.lock index f3c8dd6f..9b6aa030 100644 --- a/extensions/twitter/pdm.lock +++ b/extensions/twitter/pdm.lock @@ -5,11 +5,99 @@ groups = ["default"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:b28244fe9332fd52d2f6c6a5ad59826451c72e3844da288d912be315ba38ce46" +content_hash = "sha256:8f48b4fabbeb47e095370059f4fcbfa900ec72fd4c677199be1518dba61edb9e" [[metadata.targets]] requires_python = "==3.12.*" +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +requires_python = ">=3.10" +summary = "Happy Eyeballs for asyncio" +groups = ["default"] +files = [ + {file = "aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472"}, + {file = "aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d"}, +] + +[[package]] +name = "aiohttp" +version = "3.14.5" +requires_python = ">=3.10" +summary = "Async http client/server framework (asyncio)" +groups = ["default"] +dependencies = [ + "aiohappyeyeballs>=2.5.0", + "aiosignal>=1.4.0", + "async-timeout<6.0,>=4.0; python_version < \"3.11\"", + "attrs>=17.3.0", + "frozenlist>=1.1.1", + "multidict<8.0,>=4.5", + "propcache>=0.2.0", + "typing-extensions>=4.4; python_version < \"3.13\"", + "yarl<2.0,>=1.25.1", +] +files = [ + {file = "aiohttp-3.14.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450"}, + {file = "aiohttp-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8"}, + {file = "aiohttp-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2"}, + {file = "aiohttp-3.14.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764"}, + {file = "aiohttp-3.14.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820"}, + {file = "aiohttp-3.14.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835"}, + {file = "aiohttp-3.14.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02"}, + {file = "aiohttp-3.14.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584"}, + {file = "aiohttp-3.14.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d"}, + {file = "aiohttp-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30"}, + {file = "aiohttp-3.14.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073"}, + {file = "aiohttp-3.14.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794"}, + {file = "aiohttp-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd"}, + {file = "aiohttp-3.14.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d"}, + {file = "aiohttp-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748"}, + {file = "aiohttp-3.14.5-cp312-cp312-win32.whl", hash = "sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1"}, + {file = "aiohttp-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111"}, + {file = "aiohttp-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac"}, + {file = "aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b"}, + {file = "aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178"}, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +requires_python = ">=3.9" +summary = "aiosignal: a list of registered asynchronous callbacks" +groups = ["default"] +dependencies = [ + "frozenlist>=1.1.0", + "typing-extensions>=4.2; python_version < \"3.13\"", +] +files = [ + {file = "aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e"}, + {file = "aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7"}, +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +requires_python = ">=3.9" +summary = "Document parameters, class attributes, return types, and variables inline, with Annotated." +groups = ["default"] +files = [ + {file = "annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101"}, + {file = "annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb"}, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +requires_python = ">=3.10" +summary = "Reusable constraint types to use with typing.Annotated" +groups = ["default"] +files = [ + {file = "annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0"}, + {file = "annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7"}, +] + [[package]] name = "anyio" version = "4.10.0" @@ -27,6 +115,32 @@ files = [ {file = "anyio-4.10.0.tar.gz", hash = "sha256:3f3fae35c96039744587aa5b8371e7e8e603c0702999535961dd336026973ba6"}, ] +[[package]] +name = "attrs" +version = "26.1.0" +requires_python = ">=3.9" +summary = "Classes Without Boilerplate" +groups = ["default"] +files = [ + {file = "attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309"}, + {file = "attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32"}, +] + +[[package]] +name = "authlib" +version = "1.9.1" +requires_python = ">=3.10" +summary = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." +groups = ["default"] +dependencies = [ + "cryptography>=45.0.1", + "joserfc>=1.6.8", +] +files = [ + {file = "authlib-1.9.1-py2.py3-none-any.whl", hash = "sha256:8b9be8b1e5174dcbf5520e984f1a22dc2edae5b6e40a61d4cedbf86b97e9e69b"}, + {file = "authlib-1.9.1.tar.gz", hash = "sha256:5c7d9848f47cac340f060f76ae0bc09b59f5c8e89cc1bb032d4370a0c55b5974"}, +] + [[package]] name = "beautifulsoup4" version = "4.13.4" @@ -53,6 +167,101 @@ files = [ {file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"}, ] +[[package]] +name = "cffi" +version = "2.1.1" +requires_python = ">=3.10" +summary = "Foreign Function Interface for Python calling C code." +groups = ["default"] +marker = "platform_python_implementation != \"PyPy\"" +dependencies = [ + "pycparser; implementation_name != \"PyPy\"", +] +files = [ + {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"}, + {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"}, + {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"}, + {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"}, + {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"}, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +requires_python = "!=3.9.0,!=3.9.1,>=3.9" +summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +groups = ["default"] +dependencies = [ + "cffi>=2.0.0; platform_python_implementation != \"PyPy\"", + "typing-extensions>=4.13.2; python_full_version < \"3.11\"", +] +files = [ + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, +] + +[[package]] +name = "datadot" +version = "1.0.0" +requires_python = ">=3.8" +summary = "A functional data navigation tool for Python" +groups = ["default"] +files = [ + {file = "datadot-1.0.0-py3-none-any.whl", hash = "sha256:65cda58f57d2b9c4172d193f6c9e7952bfea033b0fb0237985133496a08c980f"}, + {file = "datadot-1.0.0.tar.gz", hash = "sha256:10d44a4277813d1f8ce7fd49dacf05c61bfc61ea8f50066279bfe2977ff0b208"}, +] + +[[package]] +name = "fastapi" +version = "0.139.2" +requires_python = ">=3.10" +summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +groups = ["default"] +dependencies = [ + "annotated-doc>=0.0.2", + "pydantic>=2.9.0", + "starlette>=0.46.0", + "typing-extensions>=4.8.0", + "typing-inspection>=0.4.2", +] +files = [ + {file = "fastapi-0.139.2-py3-none-any.whl", hash = "sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c"}, + {file = "fastapi-0.139.2.tar.gz", hash = "sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e"}, +] + [[package]] name = "filetype" version = "1.2.0" @@ -63,6 +272,54 @@ files = [ {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, ] +[[package]] +name = "frozenlist" +version = "1.8.0" +requires_python = ">=3.9" +summary = "A list-like structure which implements collections.abc.MutableSequence" +groups = ["default"] +files = [ + {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1"}, + {file = "frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b"}, + {file = "frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4"}, + {file = "frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383"}, + {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4"}, + {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8"}, + {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b"}, + {file = "frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52"}, + {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29"}, + {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3"}, + {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143"}, + {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608"}, + {file = "frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa"}, + {file = "frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf"}, + {file = "frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746"}, + {file = "frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd"}, + {file = "frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d"}, + {file = "frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad"}, +] + +[[package]] +name = "greenlet" +version = "3.5.6" +requires_python = ">=3.10" +summary = "Lightweight in-process concurrent programming" +groups = ["default"] +marker = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"" +files = [ + {file = "greenlet-3.5.6-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:a5876d0a60355af98d535c47f6cd6eb0f8a432396dab26845d380b92f8412422"}, + {file = "greenlet-3.5.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e85880b538e59a59f55117b81f208a6660ad5ac328aad9305f812d9b8bc67a0f"}, + {file = "greenlet-3.5.6-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f0ba7c2a329d650628f4c8572fd1db29f0a59dd70a3e3e0710dcf18a35cce9d8"}, + {file = "greenlet-3.5.6-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee7d9da3bf493909cf811a3f038840cb34fab5ae2956b8a263919f6e289ab188"}, + {file = "greenlet-3.5.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:975736b002ed080d124cf81a79cb7e05cb26d6b3f5c7a7b651c0fcce70353aa1"}, + {file = "greenlet-3.5.6-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:71890d5247020c25c21a6b65202782bfc281d4e6e244842419d30e3492bb6dcc"}, + {file = "greenlet-3.5.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0616b8f878098c5681fd8f0dc92d887551717402342a70f0abcbfea5f5ad8a44"}, + {file = "greenlet-3.5.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb4596a6a4e5d47121a33ff20533a81e60f302d9e67b69909a8bc21a43f0a7"}, + {file = "greenlet-3.5.6-cp312-cp312-win_amd64.whl", hash = "sha256:7ac4abb3877c43af320392c664774eef6fa2cc063c79a55fc02d844a3cbe7395"}, + {file = "greenlet-3.5.6-cp312-cp312-win_arm64.whl", hash = "sha256:301102a49120b095e72a7838792b41233975fc1c155daec6d98f81c00c9280e0"}, + {file = "greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575"}, +] + [[package]] name = "h11" version = "0.16.0" @@ -133,6 +390,20 @@ files = [ {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] +[[package]] +name = "joserfc" +version = "1.7.7" +requires_python = ">=3.10" +summary = "The ultimate Python library for JOSE RFCs, including JWS, JWE, JWK, JWA, JWT" +groups = ["default"] +dependencies = [ + "cryptography>=45.0.1", +] +files = [ + {file = "joserfc-1.7.7-py3-none-any.whl", hash = "sha256:06979cfbab78bc802f14c0358f32969826cc26720e13cb38926ffdc86bc0cb2c"}, + {file = "joserfc-1.7.7.tar.gz", hash = "sha256:471d0fd12a5a0d1c0942de280f39efe6f597878315033270b504d4d006d25510"}, +] + [[package]] name = "js2py-3-13" version = "0.74.1" @@ -188,6 +459,125 @@ files = [ {file = "m3u8-6.0.0.tar.gz", hash = "sha256:7ade990a1667d7a653bcaf9413b16c3eb5cd618982ff46aaff57fe6d9fa9c0fd"}, ] +[[package]] +name = "multidict" +version = "7.1.0" +requires_python = ">=3.10" +summary = "multidict implementation" +groups = ["default"] +dependencies = [ + "typing-extensions>=4.1.0; python_version < \"3.11\"", +] +files = [ + {file = "multidict-7.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ccfb950359a80de0fcd2030ad60ac1b1a861462de3e2ef746697c9256659af21"}, + {file = "multidict-7.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:939d8cd2d8c35e3956f6bc858390b6ccb611e6152b4920d64ab5e98f3fcf39e4"}, + {file = "multidict-7.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f8e95c95039eab6a2dad8c83c38ab87fc5431d28849e0c8a7e2a4e70ba38710d"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:05d12b4bac53abe0c65f3163af2b45894e2e1c0cc55493ac784d52a350047d88"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e79ed92b1dece6bb57e9b46effd74d7a5d3d00187c85466d880ed184239a698"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fab380fcff8b3555eb2bd04304fa4330909a771a9a9b0dc07666cfc23148a711"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4b5c41e44da74383c924cc5d75ef0a268f301d69305b3c42bd17af685d55e412"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3dbaa7f7c2f0ca8578895fc61fb8c8e50ebb405dad8982f92f4343285c7a3fda"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed6b7705d49dd07e5e0dd5f5c873fc44047e92d714299b13245b5fecac49d01"}, + {file = "multidict-7.1.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53daa47dd176db64bb35170e3d5d0ae2388c060121201883696278f055a0e70c"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:542429c796430de924d03b68a6173bb6d79d5c4967d4e9a18de3e501cad55593"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c44ca6d3cdf4cfcbcd4f928fdcbe87af5fd7319f6ad4169617b7fd6b4527c33c"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eb0228c809b2e7eb47921876050af0bc4214b351bad8d8112f70b6ed4288763c"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6ad60de1f4c702448fc8f1449f05e810f6b7957c08a5b3950c8a792dfb13b50a"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:f79def86aee67b5ba01b2565f1610f262bf88ae53c379f93e5fa29c50fe793be"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:248dabb89b5aa90b2f7e43e045f048f7e5392ec77b6446d80853ba7117d7bbdf"}, + {file = "multidict-7.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0747a83e7ae617793181a4763ee8b84863cec5c0bbbde70c4394e4c0276c36de"}, + {file = "multidict-7.1.0-cp312-cp312-win32.whl", hash = "sha256:1df055e51fe7491120cc84f3362bd43db186be78d0e4c476acad45e435af9ffb"}, + {file = "multidict-7.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:10202ba98cfb3f7eb60da7ca87a2c458a69b7d0d6e4d4388cd6773ebbce89085"}, + {file = "multidict-7.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:0aa1ba3ff7cdda05a1242490612976b2ae1c90fc6200903ef8f53815dcb35c5d"}, + {file = "multidict-7.1.0-py3-none-any.whl", hash = "sha256:d9ef29cfd98e17085b4f91bba8fa1570bec6787d5c52ce653ed33a58785585d0"}, + {file = "multidict-7.1.0.tar.gz", hash = "sha256:61a4e5d81b8d4e4ad61964b230129e7a2b914793d96289029078fc9009f074ec"}, +] + +[[package]] +name = "propcache" +version = "0.5.4" +requires_python = ">=3.10" +summary = "Accelerated property cache" +groups = ["default"] +files = [ + {file = "propcache-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429"}, + {file = "propcache-0.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8"}, + {file = "propcache-0.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21"}, + {file = "propcache-0.5.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef"}, + {file = "propcache-0.5.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f"}, + {file = "propcache-0.5.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546"}, + {file = "propcache-0.5.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b"}, + {file = "propcache-0.5.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab"}, + {file = "propcache-0.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc"}, + {file = "propcache-0.5.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937"}, + {file = "propcache-0.5.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526"}, + {file = "propcache-0.5.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944"}, + {file = "propcache-0.5.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031"}, + {file = "propcache-0.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161"}, + {file = "propcache-0.5.4-cp312-cp312-win32.whl", hash = "sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8"}, + {file = "propcache-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb"}, + {file = "propcache-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737"}, + {file = "propcache-0.5.4-py3-none-any.whl", hash = "sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468"}, + {file = "propcache-0.5.4.tar.gz", hash = "sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558"}, +] + +[[package]] +name = "pycparser" +version = "3.11" +requires_python = ">=3.10" +summary = "C parser in Python" +groups = ["default"] +marker = "implementation_name != \"PyPy\" and platform_python_implementation != \"PyPy\"" +files = [ + {file = "pycparser-3.11-py3-none-any.whl", hash = "sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80"}, + {file = "pycparser-3.11.tar.gz", hash = "sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc"}, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +requires_python = ">=3.10" +summary = "Data validation using Python type hints" +groups = ["default"] +dependencies = [ + "annotated-types>=0.6.0", + "pydantic-core==2.50.1", + "typing-extensions>=4.16.0", + "typing-inspection>=0.4.4", +] +files = [ + {file = "pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454"}, + {file = "pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26"}, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +requires_python = ">=3.10" +summary = "Core functionality for Pydantic validation and serialization" +groups = ["default"] +dependencies = [ + "typing-extensions>=4.16.0", +] +files = [ + {file = "pydantic_core-2.50.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0"}, + {file = "pydantic_core-2.50.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a"}, + {file = "pydantic_core-2.50.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9"}, + {file = "pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff"}, + {file = "pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4"}, + {file = "pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2"}, + {file = "pydantic_core-2.50.1-cp312-cp312-win32.whl", hash = "sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e"}, + {file = "pydantic_core-2.50.1-cp312-cp312-win_amd64.whl", hash = "sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498"}, + {file = "pydantic_core-2.50.1-cp312-cp312-win_arm64.whl", hash = "sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1"}, + {file = "pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09"}, +] + [[package]] name = "pyjsparser" version = "2.7.1" @@ -197,6 +587,20 @@ files = [ {file = "pyjsparser-2.7.1.tar.gz", hash = "sha256:be60da6b778cc5a5296a69d8e7d614f1f870faf94e1b1b6ac591f2ad5d729579"}, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +requires_python = ">=3.9" +summary = "JSON Web Token implementation in Python" +groups = ["default"] +dependencies = [ + "typing-extensions>=4.0; python_version < \"3.11\"", +] +files = [ + {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"}, + {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"}, +] + [[package]] name = "pyotp" version = "2.9.0" @@ -252,6 +656,59 @@ files = [ {file = "soupsieve-2.7.tar.gz", hash = "sha256:ad282f9b6926286d2ead4750552c8a6142bc4c783fd66b0293547c8fe6ae126a"}, ] +[[package]] +name = "sqlalchemy" +version = "2.0.55" +requires_python = ">=3.7" +summary = "Database Abstraction Library" +groups = ["default"] +dependencies = [ + "greenlet>=1; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", + "importlib-metadata; python_version < \"3.8\"", + "typing-extensions>=4.6.0", +] +files = [ + {file = "sqlalchemy-2.0.55-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2c37c286877ca04b577fb02779ed77dbf884539dcb4712507e25ca4d8d48a6ec"}, + {file = "sqlalchemy-2.0.55-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae6058240996c16cccc19dbd948aade826de3854171f45e045219dc9e9ea75ef"}, + {file = "sqlalchemy-2.0.55-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7694e736d63b476d81a0eae7b304d9b747a3f0a1b7dc10134655c4cccf5eb7"}, + {file = "sqlalchemy-2.0.55-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:11d7e28319efa0a52d4601ce4498df7e9c4014a740d9f0eae1d8dd6432a4ace2"}, + {file = "sqlalchemy-2.0.55-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cfa3afdaccf4ddb91de5dcffdc9298395b4a3c740fbae478b4491e96085051d2"}, + {file = "sqlalchemy-2.0.55-cp312-cp312-win32.whl", hash = "sha256:4ff08c8c2f701d2bcbe731833bfbb96195e3ce7a0e400e8b699f8e4a0020545f"}, + {file = "sqlalchemy-2.0.55-cp312-cp312-win_amd64.whl", hash = "sha256:a13820f47eda53d603f56feaadeab9787d0110b7fcbe6ab2ff2415bd3ea31c79"}, + {file = "sqlalchemy-2.0.55-py3-none-any.whl", hash = "sha256:d2954796165c08e2f70082e4577a4c3ce6f141c77e2e4b378b3b528653b54311"}, + {file = "sqlalchemy-2.0.55.tar.gz", hash = "sha256:ae40c563bc10a8d56f1fd1f39f069aeb5eb85ca26913b2314dd8d190139258fa"}, +] + +[[package]] +name = "sqlmodel" +version = "0.0.24" +requires_python = ">=3.7" +summary = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." +groups = ["default"] +dependencies = [ + "SQLAlchemy<2.1.0,>=2.0.14", + "pydantic<3.0.0,>=1.10.13", +] +files = [ + {file = "sqlmodel-0.0.24-py3-none-any.whl", hash = "sha256:6778852f09370908985b667d6a3ab92910d0d5ec88adcaf23dbc242715ff7193"}, + {file = "sqlmodel-0.0.24.tar.gz", hash = "sha256:cc5c7613c1a5533c9c7867e1aab2fd489a76c9e8a061984da11b4e613c182423"}, +] + +[[package]] +name = "starlette" +version = "1.9.0" +requires_python = ">=3.11" +summary = "The little ASGI library that shines." +groups = ["default"] +dependencies = [ + "anyio<5,>=4.0.0", + "typing-extensions>=4.10.0; python_version < \"3.13\"", +] +files = [ + {file = "starlette-1.9.0-py3-none-any.whl", hash = "sha256:fd3f159936e882d06c1108f84688f7cda2cf85fc6a592a50edbd175e882c2ddc"}, + {file = "starlette-1.9.0.tar.gz", hash = "sha256:3d931de246ed870e35ce605974440816ef895e592f255d1396cfdf191d4c4318"}, +] + [[package]] name = "twikit" version = "2.3.3" @@ -275,13 +732,27 @@ files = [ [[package]] name = "typing-extensions" -version = "4.14.1" +version = "4.16.0" requires_python = ">=3.9" summary = "Backported and Experimental Type Hints for Python 3.9+" groups = ["default"] files = [ - {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, - {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, + {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, + {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +requires_python = ">=3.10" +summary = "Runtime typing introspection tools" +groups = ["default"] +dependencies = [ + "typing-extensions>=4.15.0", +] +files = [ + {file = "typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147"}, + {file = "typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47"}, ] [[package]] @@ -320,3 +791,36 @@ files = [ {file = "webvtt-py-0.5.1.tar.gz", hash = "sha256:2040dd325277ddadc1e0c6cc66cbc4a1d9b6b49b24c57a0c3364374c3e8a3dc1"}, {file = "webvtt_py-0.5.1-py3-none-any.whl", hash = "sha256:9d517d286cfe7fc7825e9d4e2079647ce32f5678eb58e39ef544ffbb932610b7"}, ] + +[[package]] +name = "yarl" +version = "1.25.1" +requires_python = ">=3.10" +summary = "Yet another URL library" +groups = ["default"] +dependencies = [ + "idna>=2.0", + "multidict>=4.0", + "propcache>=0.2.1", +] +files = [ + {file = "yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4"}, + {file = "yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d"}, + {file = "yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6"}, + {file = "yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58"}, + {file = "yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad"}, + {file = "yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40"}, + {file = "yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c"}, + {file = "yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d"}, + {file = "yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54"}, + {file = "yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f"}, + {file = "yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc"}, + {file = "yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d"}, + {file = "yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1"}, + {file = "yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68"}, + {file = "yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707"}, + {file = "yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03"}, + {file = "yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85"}, + {file = "yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3"}, + {file = "yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7"}, +] diff --git a/extensions/twitter/pyproject.toml b/extensions/twitter/pyproject.toml index f26febaa..94777605 100644 --- a/extensions/twitter/pyproject.toml +++ b/extensions/twitter/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "datadot>=1.0.0,<2.0.0", "fastapi>=0.139.2,<0.140.0", "httpx>=0.28.1,<0.29.0", + "pydantic>=2.12.0,<3.0.0", + "PyJWT>=2.10.1,<3.0.0", "sqlmodel>=0.0.24,<0.0.25", "twikit>=2.3.3,<3.0.0", ] @@ -26,7 +28,7 @@ twitter = "extensions.twitter:Extension" name = "inkcre/twitter" nickname = "Twitter" host-sdk = "core-py" -host-sdk-version = ">=0.2.0 <0.4.0" +host-sdk-version = ">=0.3.1 <0.4.0" [tool.setuptools] packages = ["extensions.twitter"] diff --git a/extensions/twitter/reconcile.py b/extensions/twitter/reconcile.py new file mode 100644 index 00000000..0374e3a2 --- /dev/null +++ b/extensions/twitter/reconcile.py @@ -0,0 +1,263 @@ +"""One transactional native Twitter archive shared by HTTP and Bookmark sources.""" + +from __future__ import annotations + +from collections.abc import Collection +from dataclasses import dataclass +import json +import typing + +import pydantic + +from app.persistence.info_base.uow import GraphUnitOfWork, graph_uow +from app.schemas.info_base.block import BlockForm, BlockModel +from app.schemas.info_base.relation import RelationCreateForm + +from .archive_merge import merge_archive_record +from .model import ( + TextPolicy, + TwitterBatch, + TwitterMedia, + TwitterPost, + TwitterRecord, + TwitterUser, +) + + +POST_RESOLVER = "extensions.twitter.tweet.v2" +USER_RESOLVER = "extensions.twitter.user.v1" +MEDIA_RESOLVER = "extensions.twitter.media.v1" +LEGACY_POST_RESOLVER = "extensions.twitter.tweet.v1" +REFERENCE_SCHEMA = "extensions.twitter.reference.v1" +TWITTER_WRITE_LOCK = "extensions.twitter.reconcile" +KINDS = ("users", "posts", "media") +RESOLVERS = {"users": USER_RESOLVER, "posts": POST_RESOLVER, "media": MEDIA_RESOLVER} +MODELS = {"users": TwitterUser, "posts": TwitterPost, "media": TwitterMedia} +Record = dict[str, typing.Any] +Identity = tuple[str, str] + + +@dataclass(frozen=True) +class TwitterWriteResult: + changed: frozenset[Identity] + block_ids: dict[str, dict[str, int]] + + +def _json(value: object) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _block_id(block: BlockModel) -> int: + if block.id is None: + raise RuntimeError("Twitter Block has no persisted identity") + return block.id + + +def _references(post: Record) -> tuple[tuple[str, Identity, int | None], ...]: + refs: list[tuple[str, Identity, int | None]] = [] + if "authorId" in post: + refs.append(("author", ("users", post["authorId"]), None)) + article = post.get("article", {}) + media_ids = list(post.get("mediaIds", ())) + if "coverMediaId" in article: + media_ids.append(article["coverMediaId"]) + media_ids.extend(article.get("mediaIds", ())) + for position, media_id in enumerate(dict.fromkeys(media_ids)): + refs.append(("media", ("media", media_id), position)) + for field, kind in (("replyTo", "reply"), ("quote", "quote")): + if field in post: + refs.append((kind, ("posts", post[field]["postId"]), None)) + if "repostOf" in post: + refs.append(("repost", ("posts", post["repostOf"]), None)) + return tuple(refs) + + +def _reference_content(kind: str, position: int | None) -> str: + content: Record = {"schema": REFERENCE_SCHEMA, "type": kind} + if position is not None: + content["position"] = position + return _json(content) + + +def _is_reference(content: str) -> bool: + try: + value = json.loads(content) + except ValueError: + return False + if not isinstance(value, dict) or value.get("schema") != REFERENCE_SCHEMA: + return False + if value.get("type") == "media": + position = value.get("position") + return ( + set(value) == {"schema", "type", "position"} + and type(position) is int + and position >= 0 + ) + return set(value) == {"schema", "type"} and value.get("type") in { + "author", + "reply", + "quote", + "repost", + } + + +async def _load_roots( + identities: Collection[Identity], + uow: GraphUnitOfWork, + *, + include_legacy: bool = False, +) -> dict[Identity, BlockModel]: + result: dict[Identity, BlockModel] = {} + for kind in KINDS: + ids = {native_id for candidate_kind, native_id in identities if candidate_kind == kind} + resolvers = (RESOLVERS[kind],) + if include_legacy and kind == "posts": + resolvers += (LEGACY_POST_RESOLVER,) + candidates = await uow.blocks.find_json_field_many(resolvers, "id", ids) + for block in sorted( + candidates, + key=lambda block: (block.resolver == LEGACY_POST_RESOLVER, _block_id(block)), + ): + native_id = str(json.loads(block.content)["id"]) + result.setdefault((kind, native_id), block) + return result + + +async def persist_twitter( + batch: TwitterBatch, + uow: GraphUnitOfWork, + *, + text_policy: TextPolicy = "replace", +) -> TwitterWriteResult: + """Merge a complete batch without committing the caller's graph transaction. + + Supplement captures retain an already known body while adding other facts. + Capture adapters choose that policy; persistence does not identify providers. + """ + await uow.blocks.acquire_transaction_lock(TWITTER_WRITE_LOCK) + incoming: dict[Identity, Record] = {} + for kind in KINDS: + for record in getattr(batch, kind): + key = (kind, record.id) + incoming[key] = merge_archive_record( + incoming.get(key, {}), record.model_dump(mode="json", exclude_unset=True) + ) + identities = set(incoming) + for (kind, _), record in incoming.items(): + if kind == "posts": + identities.update(target for _, target, _ in _references(record)) + roots = await _load_roots(identities, uow, include_legacy=True) + changed: set[Identity] = set() + records: dict[Identity, Record] = {} + new: list[tuple[Identity, BlockForm]] = [] + updated: list[BlockModel] = [] + for key in sorted(identities): + root = roots.get(key) + existing: Record = ( + json.loads(root.content) + if root is not None and root.resolver != LEGACY_POST_RESOLVER + else {"id": key[1]} + ) + value: Record = incoming.get(key, {"id": key[1]}) + if text_policy == "supplement" and "text" in existing and "text" in value: + value = {field: item for field, item in value.items() if field != "text"} + merged = merge_archive_record(existing, value) + if key in incoming: + MODELS[key[0]].model_validate(merged) + records[key] = merged + if root is None: + new.append((key, BlockForm(resolver=RESOLVERS[key[0]], content=_json(merged)))) + changed.add(key) + elif key in incoming and ( + _json(merged) != _json(existing) or root.resolver != RESOLVERS[key[0]] + ): + root.content = _json(merged) + root.resolver = RESOLVERS[key[0]] + root.storage = None + updated.append(root) + changed.add(key) + for (key, _), root in zip( + new, await uow.blocks.create_many(form for _, form in new), strict=True + ): + roots[key] = root + await uow.blocks.save_many(updated) + + # Merged, retained references may be absent from this sparse input. + retained = { + target + for (kind, _), record in records.items() + if kind == "posts" + for _, target, _ in _references(record) + } - roots.keys() + if retained: + roots.update(await _load_roots(retained, uow, include_legacy=True)) + absent = sorted(retained - roots.keys()) + placeholders = await uow.blocks.create_many( + BlockForm(resolver=RESOLVERS[key[0]], content=_json({"id": key[1]})) for key in absent + ) + roots.update(zip(absent, placeholders, strict=True)) + + desired: set[tuple[int, int, str]] = set() + post_ids: set[int] = set() + for key in incoming: + root_id = _block_id(roots[key]) + record = records[key] + if key[0] == "posts": + post_ids.add(root_id) + desired.update( + (root_id, _block_id(roots[target]), _reference_content(kind, position)) + for kind, target, position in _references(record) + ) + stale: list[int] = [] + for relation in await uow.relations.get_outgoing_many(post_ids): + managed = _is_reference(relation.content) + if not managed: + continue + identity = (relation.from_, relation.to_, relation.content) + if identity in desired: + desired.remove(identity) + elif relation.id is not None: + stale.append(relation.id) + await uow.relations.delete_many(stale) + await uow.relations.create_many( + RelationCreateForm(from_=source, to_=target, content=content) + for source, target, content in sorted(desired) + ) + return TwitterWriteResult( + changed=frozenset(changed), + block_ids={ + kind: { + native_id: _block_id(root) + for (candidate_kind, native_id), root in roots.items() + if candidate_kind == kind + } + for kind in KINDS + }, + ) + + +async def write_twitter(batch: TwitterBatch) -> TwitterWriteResult: + async with graph_uow() as uow: + return await persist_twitter(batch, uow) + + +@dataclass(frozen=True) +class TwitterReadResult: + records: dict[Identity, TwitterRecord] + invalid: frozenset[Identity] + + +async def read_twitter(identities: Collection[Identity]) -> TwitterReadResult: + async with graph_uow() as uow: + roots = await _load_roots(identities, uow) + records: dict[Identity, TwitterRecord] = {} + invalid: set[Identity] = set() + for key in identities: + root = roots.get(key) + if root is None or json.loads(root.content) == {"id": key[1]}: + continue + try: + records[key] = MODELS[key[0]].model_validate_json(root.content) + except pydantic.ValidationError: + invalid.add(key) + return TwitterReadResult(records=records, invalid=frozenset(invalid)) diff --git a/extensions/twitter/resolver.py b/extensions/twitter/resolver.py index 7c83273c..429e0429 100644 --- a/extensions/twitter/resolver.py +++ b/extensions/twitter/resolver.py @@ -1,12 +1,17 @@ import json +import typing from typing import Optional as Opt +import pydantic + from app.business.info_base.resolver import Resolver, ResolverManager, TextProjectionContext from app.business.info_base.resolver.label import format_label from app.business.info_base.services import BlockService from app.schemas.info_base.block import BlockForm from app.schemas.info_base.main import StarsGraphForm from .schema import Tweet +from .model import TwitterIdentity, TwitterMedia, TwitterPost, TwitterUser +from .reconcile import MEDIA_RESOLVER, POST_RESOLVER, REFERENCE_SCHEMA, USER_RESOLVER class TweetResolver(Resolver[Tweet, str], rso_type="extensions.twitter.tweet.v1"): @@ -101,3 +106,144 @@ def create_graph(cls, content: Tweet) -> StarsGraphForm: block=block, out_arcs=(), ) + + +Native = typing.TypeVar("Native", bound=pydantic.BaseModel) + + +async def _native_content( + resolver: Resolver, model: type[Native], *, refresh: bool +) -> Native | TwitterIdentity: + raw = await resolver.get_raw_content(refresh=refresh) + value = json.loads(raw) + if isinstance(value, dict) and set(value) == {"id"}: + return TwitterIdentity.model_validate(value) + return model.model_validate(value) + + +class ArchivePostResolver( + Resolver[TwitterPost | TwitterIdentity, str], rso_type=POST_RESOLVER +): + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> TwitterPost | TwitterIdentity: + del materialize_missing + return await _native_content(self, TwitterPost, refresh=refresh) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, materialize_missing + post = await self.get_solved_content(refresh=refresh, materialize_missing=False) + parts = [f"https://x.com/i/status/{post.id}", post.id] + if isinstance(post, TwitterIdentity): + return "\n".join(parts) + parts.append(f"author: {post.authorId}") + for relation in await self.get_relations(include_in=False, refresh=refresh): + try: + reference = json.loads(relation.content) + except ValueError: + continue + if reference != {"schema": REFERENCE_SCHEMA, "type": "author"}: + continue + author = await BlockService.get(relation.to_) + if author is not None and author.resolver == USER_RESOLVER: + value = json.loads(author.content) + if set(value) != {"id"}: + user = TwitterUser.model_validate(value) + parts.append(f"{user.name} @{user.handle}") + break + if post.text is not None: + parts.append(post.text.text) + if post.article is not None: + parts.append(post.article.title) + if post.article.blocks is not None: + parts.extend(block.text for block in post.article.blocks) + elif post.article.preview is not None: + parts.append(post.article.preview) + return "\n".join(parts) + + async def get_label(self, *, refresh: bool = False) -> str: + post = await self.get_solved_content(refresh=refresh, materialize_missing=False) + text = ( + post.text.text if isinstance(post, TwitterPost) and post.text is not None else post.id + ) + return format_label("tweet", text or post.id, first_line=True) + + +class ArchiveUserResolver( + Resolver[TwitterUser | TwitterIdentity, str], rso_type=USER_RESOLVER +): + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> TwitterUser | TwitterIdentity: + del materialize_missing + return await _native_content(self, TwitterUser, refresh=refresh) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, materialize_missing + user = await self.get_solved_content(refresh=refresh, materialize_missing=False) + if isinstance(user, TwitterIdentity): + return user.id + return "\n".join( + value + for value in ( + user.id, + f"{user.name} @{user.handle}", + f"https://x.com/{user.handle}", + user.bio.text if user.bio is not None else None, + user.location, + user.website, + ) + if value is not None + ) + + async def get_label(self, *, refresh: bool = False) -> str: + user = await self.get_solved_content(refresh=refresh, materialize_missing=False) + return format_label( + "twitter user", f"@{user.handle}" if isinstance(user, TwitterUser) else user.id + ) + + +class ArchiveMediaResolver( + Resolver[TwitterMedia | TwitterIdentity, str], rso_type=MEDIA_RESOLVER +): + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> TwitterMedia | TwitterIdentity: + del materialize_missing + return await _native_content(self, TwitterMedia, refresh=refresh) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + del context, materialize_missing + media = await self.get_solved_content(refresh=refresh, materialize_missing=False) + if isinstance(media, TwitterIdentity): + return media.id + return "\n".join( + value + for value in (media.id, media.alt, media.title, media.description) + if value is not None + ) + + async def get_label(self, *, refresh: bool = False) -> str: + media = await self.get_solved_content(refresh=refresh, materialize_missing=False) + label = ( + media.title or media.alt or media.id if isinstance(media, TwitterMedia) else media.id + ) + return format_label("twitter media", label, first_line=True) diff --git a/extensions/twitter/setup_flow.py b/extensions/twitter/setup_flow.py index 59bf2193..6d6e340d 100644 --- a/extensions/twitter/setup_flow.py +++ b/extensions/twitter/setup_flow.py @@ -98,6 +98,7 @@ class OAuthTransactionView(pydantic.BaseModel): class TwitterSetupStatus(pydantic.BaseModel): callback_url: str + x_monkey_base_url: str | None = None connected: bool user_id: str | None = None handle: str | None = None @@ -251,6 +252,8 @@ def reconcile(config_model, state_model): async def get_setup_status() -> TwitterSetupStatus: + from .archive_session import x_monkey_base_url + config = _config() state = await _reconcile_oauth_state() configured = bool(config.client_id and config.client_secret) @@ -263,6 +266,7 @@ async def get_setup_status() -> TwitterSetupStatus: ) return TwitterSetupStatus( callback_url=await _redirect_uri(), + x_monkey_base_url=await x_monkey_base_url(), connected=connected, user_id=account.user_id if connected and account is not None else None, handle=account.handle if connected and account is not None else None, @@ -671,3 +675,6 @@ async def delete_account(): methods=["GET"], response_class=HTMLResponse, ) + from .archive_session import router as archive_session_router + + router.include_router(archive_session_router, prefix="/x-monkey") diff --git a/extensions/twitter/x_monkey.py b/extensions/twitter/x_monkey.py new file mode 100644 index 00000000..b10d8d85 --- /dev/null +++ b/extensions/twitter/x_monkey.py @@ -0,0 +1,93 @@ +"""Translate the fixed x-monkey protocol to Twitter-owned capture operations.""" + +from __future__ import annotations + +import pydantic +from pydantic import JsonValue + +from .archive_schema import ( + ArchiveBatch, + ArchiveMedia, + ArchivePost, + ArchiveQuery, + ArchiveUser, + QueryResponse, + SubmitResponse, + SubmitResult, + SubmitSummary, +) +from .model import TwitterBatch, TwitterRecord +from .reconcile import ( + KINDS, + TwitterReadResult, + TwitterWriteResult, + read_twitter, + write_twitter, +) + + +WIRE_MODELS = {"users": ArchiveUser, "posts": ArchivePost, "media": ArchiveMedia} + + +def decode_submission(batch: ArchiveBatch) -> TwitterBatch: + """Map accepted provider facts without manufacturing absent values.""" + return TwitterBatch.model_validate(batch.model_dump(mode="json", exclude_unset=True)) + + +def export_record(kind: str, record: TwitterRecord) -> JsonValue: + """Project native facts into the supported wire version, then validate that DTO.""" + model = WIRE_MODELS[kind] + projected = model.model_validate( + record.model_dump(mode="json", exclude_unset=True), extra="ignore" + ) + return projected.model_dump(mode="json", exclude_unset=True) + + +def encode_submission(batch: ArchiveBatch, result: TwitterWriteResult) -> SubmitResponse: + results = { + kind: [ + SubmitResult( + id=record.id, + status="accepted" if (kind, record.id) in result.changed else "unchanged", + ) + for record in getattr(batch, kind) + ] + for kind in KINDS + } + total = sum(len(items) for items in results.values()) + accepted = sum(item.status == "accepted" for items in results.values() for item in items) + return SubmitResponse( + summary=SubmitSummary( + total=total, accepted=accepted, unchanged=total - accepted, partial=0, failed=0 + ), + **results, + ) + + +def encode_query(query: ArchiveQuery, result: TwitterReadResult) -> QueryResponse: + results: dict[str, list[dict[str, JsonValue]]] = {kind: [] for kind in KINDS} + for kind in KINDS: + for native_id in getattr(query, kind): + key = (kind, native_id) + item: dict[str, JsonValue] = {"id": native_id, "status": "missing"} + record = result.records.get(key) + if key in result.invalid: + item.update(status="failed", error="Stored archive record is invalid") + elif record is not None: + try: + data = export_record(kind, record) + except pydantic.ValidationError: + item.update(status="failed", error="Stored record cannot use this archive format") + else: + item.update(status="found", data=data) + results[kind].append(item) + return QueryResponse.model_validate(results) + + +async def submit_archive(batch: ArchiveBatch) -> SubmitResponse: + return encode_submission(batch, await write_twitter(decode_submission(batch))) + + +async def query_archive(query: ArchiveQuery) -> QueryResponse: + identities = {(kind, native_id) for kind in KINDS for native_id in getattr(query, kind)} + return encode_query(query, await read_twitter(identities)) diff --git a/migrations/revision-integrity.json b/migrations/revision-integrity.json index 9504fa8c..181e3f63 100644 --- a/migrations/revision-integrity.json +++ b/migrations/revision-integrity.json @@ -3,6 +3,7 @@ "baseline": "peer-database-runtime-v1", "previous_manifest_sha256": "077705815948319b03286fa8550478710f8964263be19c35432d2a3462e76ed8", "revisions": { + "09a921a95314_remove_peer_jwt_lifetime_cap.py": "c0b62fc9fbf7b2c6ec9c6c3c27c3e80dbbd6022b55e7ccdf5ba9910169dddec3", "143c4f4adc85_add_sink_catalog_and_instances.py": "dc5151df7c73ec84077f7698cfb1e7fdd5596978270055de2812da539d02043d", "1e4c7a9b2d5f_add_block_lexical_records.py": "20cd7c361b0f822811d52beb692a255078e3842fd64d74287ee0c13d0bc8df26", "3d9593b0c855_add_job_submission_trace_context.py": "60d61b798da850ea24d301db3f97cb07e0d8d9695a6c5a9763b9ef4f990d0845", diff --git a/migrations/versions/09a921a95314_remove_peer_jwt_lifetime_cap.py b/migrations/versions/09a921a95314_remove_peer_jwt_lifetime_cap.py new file mode 100644 index 00000000..0b95200c --- /dev/null +++ b/migrations/versions/09a921a95314_remove_peer_jwt_lifetime_cap.py @@ -0,0 +1,87 @@ +"""Make JWT signing defaults independent of admission. + +Revision ID: 09a921a95314 +Revises: e6a1b2c3d4f5 +Create Date: 2026-10-19 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "09a921a95314" +down_revision: str | Sequence[str] | None = "e6a1b2c3d4f5" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _replace_check_jwt(*, enforce_legacy_maximum: bool) -> None: + legacy_maximum = "OR expires_at - issued_at > 86400" if enforce_legacy_maximum else "" + op.execute( + f""" + CREATE OR REPLACE FUNCTION inkcre_internal.check_jwt() + RETURNS void + LANGUAGE plpgsql + SECURITY INVOKER + SET search_path = pg_catalog + AS $$ + DECLARE + claims jsonb; + issued_at numeric; + expires_at numeric; + current_epoch numeric := extract(epoch FROM statement_timestamp()); + BEGIN + claims := nullif(current_setting('request.jwt.claims', true), '')::jsonb; + IF claims IS NULL + OR jsonb_typeof(claims -> 'role') IS DISTINCT FROM 'string' + OR claims ->> 'role' IS DISTINCT FROM 'authenticated' + OR jsonb_typeof(claims -> 'iss') IS DISTINCT FROM 'string' + OR claims ->> 'iss' IS DISTINCT FROM 'inkcre-peer' + OR jsonb_typeof(claims -> 'aud') IS DISTINCT FROM 'string' + OR claims ->> 'aud' IS DISTINCT FROM 'inkcre-api' + OR jsonb_typeof(claims -> 'iat') IS DISTINCT FROM 'number' + OR jsonb_typeof(claims -> 'exp') IS DISTINCT FROM 'number' + THEN + RAISE insufficient_privilege USING MESSAGE = 'invalid jwt claims'; + END IF; + + issued_at := (claims ->> 'iat')::numeric; + expires_at := (claims ->> 'exp')::numeric; + IF issued_at > current_epoch + 60 + OR expires_at <= current_epoch + OR expires_at <= issued_at + {legacy_maximum} + THEN + RAISE insufficient_privilege USING MESSAGE = 'invalid jwt lifetime'; + END IF; + END + $$ + """ + ) + + +def _set_contract_revision(value: str) -> None: + op.execute( + sa.text( + """ + UPDATE inkcre_internal.contract_state + SET contract_revision = :revision, updated_at = CURRENT_TIMESTAMP + WHERE singleton + """ + ).bindparams(revision=value) + ) + op.execute("NOTIFY pgrst, 'reload schema'") + + +def upgrade() -> None: + """Keep claim and time validation without a maximum token lifetime.""" + _replace_check_jwt(enforce_legacy_maximum=False) + _set_contract_revision("peer-jwt-signing-defaults-v1") + + +def downgrade() -> None: + """Restore the previous admission limit; existing longer tokens are rejected.""" + _replace_check_jwt(enforce_legacy_maximum=True) + _set_contract_revision("extension-peer-participation-v2") diff --git a/scripts/dev_database.py b/scripts/dev_database.py index 6b94826e..61535fd0 100644 --- a/scripts/dev_database.py +++ b/scripts/dev_database.py @@ -161,7 +161,6 @@ def _write_runtime_files( "issuer": "inkcre-peer", "audience": "inkcre-api", "required_claims": ["role", "iss", "aud", "iat", "exp"], - "maximum_lifetime_seconds": 86400, }, } published_ports = ( diff --git a/scripts/tests/twitter/conftest.py b/scripts/tests/twitter/conftest.py new file mode 100644 index 00000000..3015882b --- /dev/null +++ b/scripts/tests/twitter/conftest.py @@ -0,0 +1,3 @@ +"""Reuse the existing hermetic environment for explicitly selected Twitter checks.""" + +from tests.conftest import async_runner as async_runner diff --git a/scripts/tests/twitter/fixtures/native_archive.json b/scripts/tests/twitter/fixtures/native_archive.json new file mode 100644 index 00000000..730ecee6 --- /dev/null +++ b/scripts/tests/twitter/fixtures/native_archive.json @@ -0,0 +1,406 @@ +{ + "users": [ + { + "id": "17", + "handle": "author", + "name": "作者🌏", + "createdAt": "2024-02-29T12:00:00Z", + "avatarUrl": "https://pbs.twimg.com/avatar.jpg", + "avatarShape": "Hexagon", + "bannerUrl": "https://pbs.twimg.com/banner.jpg", + "bio": { + "text": "正文🌏 @author #X $BTC 01:30 https://t.co/x", + "visible": [ + 0, + 9 + ], + "entities": [ + { + "type": "mention", + "range": [ + 4, + 11 + ], + "userId": "17" + }, + { + "type": "hashtag", + "range": [ + 12, + 14 + ] + }, + { + "type": "cashtag", + "range": [ + 15, + 19 + ], + "name": "Bitcoin", + "ticker": "BTC" + }, + { + "type": "timestamp", + "range": [ + 20, + 25 + ] + }, + { + "type": "url", + "range": [ + 26, + 40 + ], + "expanded": "https://example.org/source" + } + ], + "styles": [ + { + "range": [ + 0, + 2 + ], + "styles": [ + "Bold", + "Italic" + ] + } + ] + }, + "location": "上海", + "website": "https://example.org/", + "pinnedPostIds": [ + "31" + ], + "protected": false, + "sensitive": true, + "interstitial": "sensitive_media", + "withheldIn": [ + "SA", + "AE" + ], + "verified": { + "blue": false, + "type": "Business" + }, + "affiliation": { + "name": "示例", + "url": "https://example.org/", + "badgeUrl": "https://pbs.twimg.com/badge.jpg", + "kind": "AutomatedLabel", + "ownerId": "18" + }, + "parody": "Commentary", + "professional": { + "type": "Business", + "categories": [ + "Technology" + ] + }, + "counts": { + "followers": 0, + "following": 1, + "posts": 2, + "mediaPosts": 3, + "likes": 4 + }, + "countsObservedAt": { + "followers": "2026-10-19T01:02:03.1200Z", + "following": "2026-10-19T01:02:03Z", + "posts": "2026-10-19T01:02:00Z", + "mediaPosts": "2026-10-19T01:02:03.000001Z", + "likes": "2026-10-19T01:02:03.0000017Z" + } + } + ], + "posts": [ + { + "id": "31", + "authorId": "17", + "createdAt": "2026-10-18T12:34:56Z", + "repostOf": "32", + "conversationId": "29", + "replyTo": { + "postId": "29", + "userId": "18", + "handle": "prior", + "tombstone": "这个帖子已被其作者删除。" + }, + "quote": { + "postId": "30", + "url": "https://x.com/prior/status/30", + "tombstone": "这个账号的帖子受保护。" + }, + "text": { + "text": "正文🌏 @author #X $BTC 01:30 https://t.co/x", + "visible": [ + 0, + 9 + ], + "entities": [ + { + "type": "mention", + "range": [ + 4, + 11 + ], + "userId": "17" + }, + { + "type": "hashtag", + "range": [ + 12, + 14 + ] + }, + { + "type": "cashtag", + "range": [ + 15, + 19 + ], + "name": "Bitcoin", + "ticker": "BTC" + }, + { + "type": "timestamp", + "range": [ + 20, + 25 + ] + }, + { + "type": "url", + "range": [ + 26, + 40 + ], + "expanded": "https://example.org/source" + } + ], + "styles": [ + { + "range": [ + 0, + 2 + ], + "styles": [ + "Bold", + "Italic" + ] + } + ] + }, + "lang": "zh", + "source": "X Web App", + "mediaIds": [ + "41" + ], + "card": { + "type": "poll2choice_text_only", + "url": "card://31", + "title": "问题", + "description": "详细说明", + "domain": "example.org", + "image": { + "url": "https://example.org/image.png", + "width": 640, + "height": 480, + "alt": "示例图" + }, + "siteUserId": "18", + "creatorUserId": "17", + "player": { + "url": "https://example.org/video", + "width": 640, + "height": 480 + }, + "poll": { + "choices": [ + { + "label": "是", + "count": 0 + }, + { + "label": "否", + "count": 3 + } + ], + "endsAt": "2026-10-20T00:00:00Z", + "final": false, + "updatedAt": "2026-10-19T01:02:03Z" + }, + "extra": { + "app_name": "example", + "is_enabled": false + } + }, + "article": { + "id": "51", + "title": "长文", + "preview": "摘要", + "coverMediaId": "41", + "publishedAt": "2026-10-18T12:34:56Z", + "modifiedAt": "2026-10-18T13:34:56Z", + "blocks": [ + { + "type": "unstyled", + "text": "文章🌏", + "styles": [ + [ + 0, + 2, + "BOLD" + ] + ], + "entities": [ + [ + 2, + 2, + 0 + ] + ], + "data": { + "future": { + "items": [ + null, + true, + 1, + 2.5, + "字" + ] + } + } + } + ], + "entities": [ + { + "type": "MEDIA", + "data": { + "mediaId": "41", + "unknown": null + } + } + ], + "mediaIds": [ + "41" + ] + }, + "place": { + "id": "place:shanghai", + "name": "上海", + "fullName": "上海，中国", + "country": "中国", + "countryCode": "CN", + "type": "city", + "bbox": [ + 120, + 30.5, + 122.5, + 32 + ] + }, + "counts": { + "replies": 0, + "reposts": 1, + "quotes": 2, + "likes": 3, + "bookmarks": 4, + "views": 5 + }, + "countsObservedAt": { + "replies": "2026-10-19T01:02:03.100Z", + "reposts": "2026-10-19T01:02:03.200Z", + "quotes": "2026-10-19T01:02:03.300Z", + "likes": "2026-10-19T01:02:03.400Z", + "bookmarks": "2026-10-19T01:02:03.500Z", + "views": "2026-10-19T01:02:03.600Z" + }, + "editIds": [ + "30", + "31" + ], + "communityNote": { + "id": "61", + "title": "补充背景", + "text": { + "text": "来源已核实" + }, + "lang": "zh", + "url": "https://example.org/note" + }, + "sensitive": false, + "contentWarning": { + "title": "敏感内容", + "text": "作者已标注" + }, + "community": { + "id": "71", + "name": "示例社群" + }, + "replyPolicy": "ByInvitation", + "paidPromotion": true, + "aiGenerated": { + "media": true, + "source": "C2paClient" + } + } + ], + "media": [ + { + "id": "41", + "type": "video", + "url": "https://pbs.twimg.com/cover.jpg", + "width": 1920, + "height": 1080, + "alt": "演示视频", + "crops": [ + [ + 0, + 0, + 640, + 480 + ] + ], + "faces": [ + [ + 1, + 2, + 30, + 40 + ] + ], + "tags": [ + { + "userId": "18", + "handle": "prior" + } + ], + "video": { + "durationMs": 1234, + "variants": [ + { + "url": "https://video.twimg.com/a.mp4", + "type": "video/mp4", + "bitrate": 256000 + }, + { + "url": "https://video.twimg.com/a.m3u8", + "type": "application/x-mpegURL" + } + ] + }, + "sourcePostId": "31", + "sourceUserId": "17", + "sensitive": [ + "adult_content", + "other" + ], + "unavailable": "Geoblocked", + "title": "标题", + "description": "说明", + "link": "https://example.org/video" + } + ] +} diff --git a/scripts/tests/twitter/integration/test_archive_graph.py b/scripts/tests/twitter/integration/test_archive_graph.py new file mode 100644 index 00000000..739ee49c --- /dev/null +++ b/scripts/tests/twitter/integration/test_archive_graph.py @@ -0,0 +1,685 @@ +"""Native Twitter facts survive sparse, concurrent and cross-source writes.""" + +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path +import typing +from types import SimpleNamespace +from unittest import mock +import uuid + +import pytest +import sqlalchemy +from sqlalchemy import event +import sqlmodel +import twikit + +from app.business.info_base.resolver import ResolverManager +from app.business.info_base.storage import HTTPStorage +from app.business.lexical_retrieval import LexicalRetrievalManager +from app.business.source import SourceManager +from app.persistence.info_base.repository import BlockRepository, RelationRepository +from app.persistence.info_base.uow import graph_uow +from app.schemas.info_base.block import BlockForm, BlockModel +from app.schemas.info_base.relation import RelationModel +from app.schemas.info_base.storage import StorageBlobModel +from app.schemas.job import JobModel +from app.schemas.lexical_retrieval import ( + BlockLexicalRecordModel, + LexicalMaintenanceOptions, +) +from app.schemas.source import SourceModel +from extensions.twitter.api import ( + TwitterAPI, + TwitterAPIResult, + _official_archive, + _twikit_archive, +) +from extensions.twitter.archive_schema import ArchiveBatch, ArchiveQuery +from extensions.twitter.model import TwitterBatch +from extensions.twitter.x_monkey import ( + decode_submission, + encode_submission, + query_archive, + submit_archive, +) +from extensions.twitter.bookmark import CollectConfig, Source +from extensions.twitter.reconcile import ( + MEDIA_RESOLVER, + POST_RESOLVER, + REFERENCE_SCHEMA, + USER_RESOLVER, + persist_twitter, +) +from extensions.twitter.resolver import TweetResolver +from extensions.twitter.schema import Tweet +from tests.database import TestSession, seed_block, seed_relation + + +pytestmark = pytest.mark.skipif( + not os.getenv("INKCRE_TEST_DATABASE_URL"), + reason="requires an explicitly selected migrated disposable PostgreSQL runtime", +) + + +@pytest.fixture +def native() -> dict[str, typing.Any]: + value = json.loads( + (Path(__file__).parents[1] / "fixtures" / "native_archive.json").read_text() + ) + offset = int(uuid.uuid4().hex[:15], 16) + 10**20 + + def distinct(item): + if isinstance(item, str): + if item.isascii() and item.isdecimal(): + return str(int(item) + offset) + if item.startswith(("https://pbs.twimg.com/", "https://video.twimg.com/")): + return f"{item}{'&' if '?' in item else '?'}twitter_test={offset}" + if isinstance(item, list): + return [distinct(child) for child in item] + if isinstance(item, dict): + return {key: distinct(child) for key, child in item.items()} + return item + + return distinct(value) + + +@pytest.fixture +def tracked() -> typing.Iterator[set[int]]: + ids: set[int] = set() + yield ids + if ids: + with TestSession() as db: + db.connection().execute( + sqlalchemy.delete(BlockModel).where(sqlmodel.col(BlockModel.id).in_(tuple(ids))) + ) + db.commit() + + +async def _write(batch: ArchiveBatch | TwitterBatch, tracked: set[int]): + async with graph_uow() as uow: + result = await persist_twitter( + decode_submission(batch) if isinstance(batch, ArchiveBatch) else batch, uow + ) + tracked.update(block_id for ids in result.block_ids.values() for block_id in ids.values()) + return SimpleNamespace( + block_ids=result.block_ids, + response=encode_submission( + batch + if isinstance(batch, ArchiveBatch) + else ArchiveBatch.model_validate(batch.model_dump(mode="json", exclude_unset=True)), + result, + ), + ) + + +def _minimal(post: dict) -> dict: + return {field: post[field] for field in ("id", "authorId", "createdAt")} + + +def _record(native_id: str) -> BlockModel: + with TestSession() as db: + result = db.exec( + sqlmodel.select(BlockModel).where( + BlockModel.resolver == POST_RESOLVER, + sqlalchemy.cast(BlockModel.content, sqlalchemy.JSON)["id"].as_string() == native_id, + ) + ).one() + return result + + +def test_native_roundtrip_projection_and_media_are_lazy(async_runner, native, tracked): + # Importing the module registers all exact decoders, including legacy v1. + from extensions.twitter import resolver # noqa: F401 + + batch = ArchiveBatch.model_validate(native) + query = ArchiveQuery( + **{ + kind: [record.id for record in getattr(batch, kind)] + for kind in ("users", "posts", "media") + } + ) + with TestSession() as db: + block_count = db.exec( + sqlmodel.select(sqlmodel.func.count()).select_from(BlockModel) + ).one() + blob_count = db.exec( + sqlmodel.select(sqlmodel.func.count()).select_from(StorageBlobModel) + ).one() + job_count = db.exec(sqlmodel.select(sqlmodel.func.count()).select_from(JobModel)).one() + with mock.patch.object( + HTTPStorage, "get_raw_content", new_callable=mock.AsyncMock + ) as fetch: + fetch.side_effect = AssertionError("metadata read downloaded external media") + result = async_runner.run(_write(batch, tracked)) + assert result.response.summary.accepted == 3 + response = async_runner.run(query_archive(query)) + for kind in ("users", "posts", "media"): + assert [item.status for item in getattr(response, kind)] == ["found"] + assert [ + item.data.model_dump(mode="json", exclude_unset=True) + for item in getattr(response, kind) + ] == native[kind] + post = _record(batch.posts[0].id) + text = async_runner.run(ResolverManager.get(post).get_text(materialize_missing=False)) + assert native["posts"][0]["article"]["title"] in text + assert native["users"][0]["handle"] in text + assert native["posts"][0]["id"] in text + report = async_runner.run( + LexicalRetrievalManager.maintain( + LexicalMaintenanceOptions(max_records=1000, scan_page_size=50) + ) + ) + assert report.failed == 0, report.diagnostics + for query_text in ( + batch.posts[0].id, + native["users"][0]["handle"], + native["posts"][0]["text"]["text"], + ): + matches = async_runner.run(LexicalRetrievalManager.retrieve_local(query_text)).matches + assert post.id in {match.block.id for match in matches} + with TestSession() as db: + indexed = db.exec( + sqlmodel.select(BlockLexicalRecordModel.block).where( + sqlmodel.col(BlockLexicalRecordModel.block).in_(tuple(tracked)) + ) + ).all() + assert set(indexed) == tracked + assert fetch.await_count == 0 + replay = async_runner.run(_write(batch, tracked)) + assert replay.response.summary.unchanged == 3 + assert replay.block_ids == result.block_ids + pure_repost = { + **_minimal(native["posts"][0]), + "id": str(int(batch.posts[0].id) + 1000), + "repostOf": batch.posts[0].id, + } + async_runner.run(_write(ArchiveBatch.model_validate({"posts": [pure_repost]}), tracked)) + repost_response = async_runner.run(query_archive(ArchiveQuery(posts=[pure_repost["id"]]))) + assert repost_response.posts[0].data is not None + assert ( + repost_response.posts[0].data.model_dump(mode="json", exclude_unset=True) == pure_repost + ) + assert batch.posts[0].quote is not None + missing = async_runner.run( + query_archive(ArchiveQuery(posts=[batch.posts[0].quote.postId])) + ) + assert missing.posts[0].status == "missing" + with TestSession() as db: + assert ( + db.exec(sqlmodel.select(sqlmodel.func.count()).select_from(StorageBlobModel)).one() + == blob_count + ) + assert ( + db.exec(sqlmodel.select(sqlmodel.func.count()).select_from(JobModel)).one() + == job_count + ) + media_id = result.block_ids["media"][batch.media[0].id] + relations = db.exec( + sqlmodel.select(RelationModel).where( + RelationModel.from_ == media_id, RelationModel.content == "content" + ) + ).all() + assert relations == [] + stored = db.exec( + sqlmodel.select(BlockModel).where(sqlmodel.col(BlockModel.id).in_(tuple(tracked))) + ).all() + assert len(stored) == len(tracked) + assert all(block.storage is None for block in stored) + assert db.exec( + sqlmodel.select(sqlmodel.func.count()).select_from(BlockModel) + ).one() == block_count + len(tracked) + + +def test_sparse_counter_history_and_managed_edges(async_runner, native, tracked): + result = async_runner.run(_write(ArchiveBatch.model_validate(native), tracked)) + post = native["posts"][0] + baseline = _minimal(post) + updates = [ + ( + {"counts": {"likes": 100}, "countsObservedAt": {"likes": "2026-10-20T01:00:00Z"}}, + 100, + "2026-10-20T01:00:00Z", + "accepted", + ), + ( + {"counts": {"likes": 90}, "countsObservedAt": {"likes": "2026-10-19T01:00:00Z"}}, + 100, + "2026-10-20T01:00:00Z", + "unchanged", + ), + ( + { + "counts": {"likes": 100}, + "countsObservedAt": {"likes": "2026-10-20T02:00:00.1200Z"}, + }, + 100, + "2026-10-20T02:00:00.1200Z", + "accepted", + ), + ({"counts": {"likes": 101}}, 101, None, "accepted"), + ] + for patch, expected, observed, status in updates: + response = async_runner.run( + submit_archive(ArchiveBatch.model_validate({"posts": [{**baseline, **patch}]})) + ) + assert response.posts[0].status == status + stored = json.loads(_record(post["id"]).content) + assert stored["counts"]["likes"] == expected + assert stored.get("countsObservedAt", {}).get("likes") == observed + assert stored["text"] == post["text"] + assert stored["article"] == post["article"] + assert _record(post["id"]).id == result.block_ids["posts"][post["id"]] + + # A changed media array replaces its edges without touching user-authored links. + root_id = result.block_ids["posts"][post["id"]] + user_link = seed_relation( + root_id, result.block_ids["users"][post["authorId"]], "my citation" + ) + old_media_id = post["mediaIds"][0] + replacement = str(int(old_media_id) + 1000) + update = ArchiveBatch.model_validate({"posts": [{**baseline, "mediaIds": [replacement]}]}) + async_runner.run(_write(update, tracked)) + with TestSession() as db: + assert db.get(RelationModel, user_link.id) is not None + media_refs = [] + for relation in db.exec( + sqlmodel.select(RelationModel).where(RelationModel.from_ == root_id) + ).all(): + try: + grammar = json.loads(relation.content) + except ValueError: + continue + if grammar.get("schema") == REFERENCE_SCHEMA and grammar["type"] == "media": + target = db.get(BlockModel, relation.to_) + assert target is not None + media_refs.append((grammar["position"], json.loads(target.content)["id"])) + expected = list( + dict.fromkeys( + [replacement, post["article"]["coverMediaId"], *post["article"]["mediaIds"]] + ) + ) + assert sorted(media_refs) == list(enumerate(expected)) + + +def test_bookmark_joins_legacy_and_native_identity_without_overwriting_text( + async_runner, native, tracked +): + post = native["posts"][0] + legacy = seed_block( + TweetResolver.create_block(Tweet(id=int(post["id"]), text="旧加工 [photo]")) + ) + citation = seed_block(BlockForm(resolver="core.text.v1", content="user citation")) + assert legacy.id is not None and citation.id is not None + tracked.update((legacy.id, citation.id)) + reference = seed_relation(citation.id, legacy.id, "reference") + old_attachment = seed_relation(legacy.id, citation.id, "attachment:photo:legacy") + result = async_runner.run(_write(ArchiveBatch.model_validate(native), tracked)) + assert result.block_ids["posts"][post["id"]] == legacy.id + with TestSession() as db: + persisted_reference = db.get(RelationModel, reference.id) + assert persisted_reference is not None and persisted_reference.to_ == legacy.id + assert db.get(RelationModel, old_attachment.id) is not None + + source_type = f"{Source.__module__}.{Source.__qualname__}" + async_runner.run(SourceManager.sync_source_types_async({source_type: Source})) + sparse = {**_minimal(post), "text": {"text": "provider has less detail"}, "lang": "en"} + provider = mock.Mock() + provider.get_bookmarks = mock.AsyncMock( + return_value=TwitterAPIResult( + batch=TwitterBatch.model_validate({"posts": [sparse]}), primary_post_ids=(post["id"],) + ) + ) + source = async_runner.run( + SourceManager.create(source_type, nickname="test Twitter bookmarks") + ) + try: + with mock.patch.object( + TwitterAPI, "new", new_callable=mock.AsyncMock, return_value=provider + ): + async_runner.run( + Source(source.id).collect( + JobModel(type="core.source.collect.v1", parameters={}), CollectConfig() + ) + ) + stored = json.loads(_record(post["id"]).content) + assert stored["text"] == post["text"] + assert stored["lang"] == "en" + with TestSession() as db: + persisted_source = db.get(SourceModel, source.id) + assert persisted_source is not None and persisted_source.state is not None + assert persisted_source.state["latest_tweet_id"] == post["id"] + finally: + with TestSession() as db: + db.delete(db.get(SourceModel, source.id)) + db.commit() + + +def test_real_transaction_lock_preserves_concurrent_supplements_and_rollback( + async_runner, native, tracked +): + baseline = _minimal(native["posts"][0]) + original_lock = BlockRepository.acquire_transaction_lock + + async def exercise(): + locked, attempted, release = asyncio.Event(), asyncio.Event(), asyncio.Event() + + async def coordinated(repository, namespace): + task = asyncio.current_task() + assert task is not None + if task.get_name() == "second": + attempted.set() + await original_lock(repository, namespace) + if task.get_name() == "first": + locked.set() + await release.wait() + + with mock.patch.object(BlockRepository, "acquire_transaction_lock", coordinated): + first = asyncio.create_task( + submit_archive( + ArchiveBatch.model_validate({"posts": [{**baseline, "source": "client"}]}) + ), + name="first", + ) + await locked.wait() + second = asyncio.create_task( + submit_archive( + ArchiveBatch.model_validate({"posts": [{**baseline, "lang": "zh"}]}) + ), + name="second", + ) + await attempted.wait() + release.set() + await asyncio.gather(first, second) + async with graph_uow() as uow: + roots = await uow.blocks.find_json_field_many( + (POST_RESOLVER, USER_RESOLVER), "id", (baseline["id"], baseline["authorId"]) + ) + tracked.update(block.id for block in roots) + posts = [root for root in roots if root.resolver == POST_RESOLVER] + assert len(posts) == 1 + stored = json.loads(posts[0].content) + assert stored["source"] == "client" and stored["lang"] == "zh" + + failed: dict[str, typing.Any] = { + **baseline, + "id": str(int(baseline["id"]) + 1000), + "quote": {"postId": str(int(baseline["id"]) + 1001)}, + } + batch = ArchiveBatch.model_validate({"posts": [failed]}) + with mock.patch.object( + RelationRepository, + "create_many", + new_callable=mock.AsyncMock, + side_effect=RuntimeError("deterministic persistence failure"), + ): + with pytest.raises(RuntimeError, match="deterministic persistence failure"): + await submit_archive(batch) + absent = await query_archive( + ArchiveQuery(posts=[failed["id"], failed["quote"]["postId"]]) + ) + assert [item.status for item in absent.posts] == ["missing", "missing"] + await _write(batch, tracked) + assert (await query_archive(ArchiveQuery(posts=[failed["id"]]))).posts[ + 0 + ].status == "found" + + async_runner.run(exercise()) + + +def test_batch_lookup_tolerates_non_json_and_reports_locatable_corruption( + async_runner, native, tracked +): + batch = ArchiveBatch.model_validate(native) + async_runner.run(_write(batch, tracked)) + unknown = str(int(batch.posts[0].id) + 1000) + broken = [ + seed_block(BlockForm(resolver="core.text.v1", content="ordinary non-JSON text")), + seed_block(BlockForm(resolver=POST_RESOLVER, content="broken JSON {")), + seed_block( + BlockForm( + resolver=POST_RESOLVER, + content=json.dumps({"id": unknown, "authorId": batch.posts[0].authorId}), + ) + ), + ] + tracked.update(block.id for block in broken) + response = async_runner.run( + query_archive(ArchiveQuery(posts=[batch.posts[0].id, unknown])) + ) + assert [item.status for item in response.posts] == ["found", "failed"] + + +def test_media_metadata_roundtrips_without_content_blocks(async_runner, native, tracked): + base = int(native["media"][0]["id"]) + 1000 + records: list[dict[str, typing.Any]] = [ + {"id": str(base), "type": "photo", "url": "https://example.org/image.jpg"}, + { + "id": str(base + 1), + "type": "video", + "url": "https://pbs.twimg.com/poster.jpg", + "video": { + "variants": [ + {"url": "https://video.twimg.com/list.m3u8", "type": "application/x-mpegURL"} + ] + }, + }, + {"id": str(base + 2), "type": "photo", "url": "https://pbs.twimg.com/\nimage.jpg"}, + { + "id": str(base + 3), + "type": "video", + "url": "https://pbs.twimg.com/poster.jpg", + "video": { + "variants": [ + { + "url": f"https://video.twimg.com/{base}-low.mp4", + "type": "video/mp4", + "bitrate": 1, + }, + {"url": "https://example.org/foreign.mp4", "type": "video/mp4", "bitrate": 99}, + { + "url": f"https://video.twimg.com/{base}-high.mp4", + "type": "video/mp4", + "bitrate": 2, + }, + ] + }, + }, + ] + result = async_runner.run( + _write(ArchiveBatch.model_validate({"media": records}), tracked) + ) + response = async_runner.run( + query_archive(ArchiveQuery(media=[record["id"] for record in records])) + ) + assert [item.status for item in response.media] == ["found"] * 4 + assert [ + item.data.model_dump(mode="json", exclude_unset=True) for item in response.media + ] == records + with TestSession() as db: + content = db.exec( + sqlmodel.select(RelationModel).where( + sqlmodel.col(RelationModel.from_).in_(tuple(result.block_ids["media"].values())), + RelationModel.content == "content", + ) + ).all() + assert content == [] + + +def test_representative_batch_keeps_database_round_trips_bounded( + async_runner, native, tracked +): + from app.engine import ASYNC_DB_ENGINE + + base = int(native["posts"][0]["id"]) + raw: dict[str, list[dict[str, typing.Any]]] = {"users": [], "posts": [], "media": []} + for index in range(200): + user, post, media, quoted = ( + str(base + offset + index) for offset in (0, 1000, 2000, 3000) + ) + raw["users"].append({"id": user, "handle": f"author{index}", "name": f"Author {index}"}) + raw["posts"].append( + { + "id": post, + "authorId": user, + "createdAt": "2026-10-19T01:00:00Z", + "text": {"text": "benchmark " + "正文 " * 40}, + "mediaIds": [media], + "quote": {"postId": quoted}, + } + ) + raw["media"].append( + { + "id": media, + "type": "photo", + "url": f"https://pbs.twimg.com/batch-{base}-{index}.jpg", + } + ) + calls: list[str] = [] + + def count_round_trip(_connection, _cursor, statement, _parameters, _context, _many): + calls.append(statement.split(None, 1)[0]) + + event.listen(ASYNC_DB_ENGINE.sync_engine, "before_cursor_execute", count_round_trip) + try: + result = async_runner.run(_write(ArchiveBatch.model_validate(raw), tracked)) + finally: + event.remove(ASYNC_DB_ENGINE.sync_engine, "before_cursor_execute", count_round_trip) + assert result.response.summary.accepted == 600 + # Keep the synchronous native batch bounded when Core and DB are remote. + assert len(calls) < 20, {kind: calls.count(kind) for kind in set(calls)} + assert len(result.block_ids["posts"]) == 400 + with TestSession() as db: + contents = db.exec( + sqlmodel.select(RelationModel).where( + sqlmodel.col(RelationModel.from_).in_(tuple(result.block_ids["media"].values())), + RelationModel.content == "content", + ) + ).all() + assert contents == [] + + +def test_official_provider_keeps_source_text_and_exact_id_semantics( + async_runner, native, tracked +): + root = native["posts"][0] + batch = _official_archive( + [ + { + "id": root["id"], + "author_id": root["authorId"], + "created_at": root["createdAt"], + "text": "@reply 原始🌏 & https://t.co/media", + "conversation_id": "1", + "referenced_tweets": [{"type": "replied_to", "id": "2"}], + } + ], + {}, + "2026-10-19T01:00:00Z", + ) + assert batch.posts[0].text is not None + assert batch.posts[0].text.text == "@reply 原始🌏 & https://t.co/media" + assert batch.posts[0].conversationId == "1" + assert batch.posts[0].replyTo is not None and batch.posts[0].replyTo.postId == "2" + result = async_runner.run(_write(batch, tracked)) + assert result.response.posts[0].status == "accepted" + + +def test_twikit_bookmark_first_then_native_enrichment_keeps_one_root( + async_runner, native, tracked +): + post = native["posts"][0] + media_id = native["media"][0]["id"] + author = SimpleNamespace(id=post["authorId"], screen_name="source_handle", name="source") + raw = { + "rest_id": post["id"], + "legacy": { + "created_at": "Sun Oct 18 12:34:56 +0000 2026", + "full_text": "truncated legacy…", + "lang": "zh", + "conversation_id_str": str(int(post["id"]) - 1), + "in_reply_to_status_id_str": str(int(post["id"]) - 2), + "entities": { + "media": [ + { + "id_str": media_id, + "type": "photo", + "media_url_https": native["media"][0]["url"], + "url": "https://t.co/source", + "original_info": {"width": 1920, "height": 1080}, + } + ], + "urls": [], + }, + }, + "note_tweet": { + "note_tweet_results": { + "result": { + "text": "@mention 完整🌏 & https://t.co/source", + } + } + }, + } + # Exercise the installed provider's real getters, including note-tweet selection. + tweet = twikit.Tweet(mock.Mock(), raw, typing.cast(twikit.User, author)) + batch = _twikit_archive((tweet,), "2026-10-19T01:00:00Z") + assert batch.posts[0].authorId == author.id + assert batch.posts[0].replyTo is not None and batch.posts[0].text is not None + assert batch.posts[0].conversationId != batch.posts[0].replyTo.postId + assert batch.posts[0].text.text == "@mention 完整🌏 & https://t.co/source" + assert [media.id for media in batch.media] == [media_id] + source_type = f"{Source.__module__}.{Source.__qualname__}" + async_runner.run(SourceManager.sync_source_types_async({source_type: Source})) + source = async_runner.run(SourceManager.create(source_type, nickname="Twikit first")) + provider = mock.Mock() + provider.get_bookmarks = mock.AsyncMock( + return_value=TwitterAPIResult(batch=batch, primary_post_ids=(post["id"],)) + ) + try: + with ( + mock.patch.object( + TwitterAPI, "new", new_callable=mock.AsyncMock, return_value=provider + ), + mock.patch.object( + HTTPStorage, + "get_raw_content", + new_callable=mock.AsyncMock, + side_effect=AssertionError("Bookmark indexing downloaded external media"), + ) as fetch, + ): + async_runner.run( + Source(source.id).collect( + JobModel(type="core.source.collect.v1", parameters={}), CollectConfig() + ) + ) + root = _record(post["id"]) + report = async_runner.run( + LexicalRetrievalManager.maintain( + LexicalMaintenanceOptions(max_records=1000, scan_page_size=50) + ) + ) + assert report.failed == 0, report.diagnostics + matches = async_runner.run(LexicalRetrievalManager.retrieve_local(post["id"])).matches + assert root.id in {match.block.id for match in matches} + assert fetch.await_count == 0 + finally: + # Track native identities as well as the root created inside Source's UoW. + async def remember(): + async with graph_uow() as uow: + rows = await uow.blocks.find_json_field_many( + (POST_RESOLVER, USER_RESOLVER, MEDIA_RESOLVER), + "id", + (post["id"], author.id, raw["legacy"]["in_reply_to_status_id_str"], media_id), + ) + tracked.update(row.id for row in rows) + + async_runner.run(remember()) + with TestSession() as db: + db.delete(db.get(SourceModel, source.id)) + db.commit() + enriched = async_runner.run(_write(ArchiveBatch.model_validate(native), tracked)) + assert enriched.block_ids["posts"][post["id"]] == root.id + assert json.loads(_record(post["id"]).content)["text"] == post["text"] diff --git a/scripts/tests/twitter/integration/test_archive_http_roundtrip.py b/scripts/tests/twitter/integration/test_archive_http_roundtrip.py new file mode 100644 index 00000000..ea6cb89a --- /dev/null +++ b/scripts/tests/twitter/integration/test_archive_http_roundtrip.py @@ -0,0 +1,232 @@ +"""The published x-monkey HTTP surface round-trips native records in PostgreSQL.""" + +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path +from types import SimpleNamespace +import uuid + +import pytest +import sqlalchemy +import sqlmodel + +from app.persistence.info_base.uow import graph_uow +from app.business.peer import PeerManager +from app.business.source import SourceManager +from app.schemas.info_base.block import BlockModel +from app.schemas.info_base.relation import RelationModel +from extensions.twitter.archive_session import SESSION_COOKIE_NAME, _issue_session_token +from extensions.twitter.reconcile import MEDIA_RESOLVER, POST_RESOLVER, USER_RESOLVER +from tests.database import TestSession +from tests.extensions.runtime_support import publish_extension + +pytestmark = pytest.mark.skipif( + not os.getenv("INKCRE_TEST_DATABASE_URL"), + reason="requires an explicitly selected migrated disposable PostgreSQL runtime", +) + + +def _distinct_fixture() -> dict: + path = Path(__file__).parents[1] / "fixtures" / "native_archive.json" + value = json.loads(path.read_text()) + offset = int(uuid.uuid4().hex[:15], 16) + 10**20 + + def distinct(item): + if isinstance(item, str): + if item.isascii() and item.isdecimal(): + return str(int(item) + offset) + if item.startswith(("https://pbs.twimg.com/", "https://video.twimg.com/")): + return f"{item}{'&' if '?' in item else '?'}twitter_http_test={offset}" + if isinstance(item, list): + return [distinct(child) for child in item] + if isinstance(item, dict): + return {key: distinct(child) for key, child in item.items()} + return item + + return distinct(value) + + +def _archive_owned_ids( + native: dict, + *, + preexisting_ids: dict[str, set[str]] | None = None, +) -> dict[str, tuple[str, ...]]: + """Return submitted identities and only the absent references this batch creates.""" + resolvers = {"users": USER_RESOLVER, "posts": POST_RESOLVER, "media": MEDIA_RESOLVER} + defined = { + kind: {record["id"] for record in native[kind]} for kind in ("users", "posts", "media") + } + referenced = {kind: set() for kind in ("users", "posts", "media")} + for post in native["posts"]: + if "authorId" in post: + referenced["users"].add(post["authorId"]) + article = post.get("article") or {} + media_ids = list(post.get("mediaIds", ())) + if "coverMediaId" in article: + media_ids.append(article["coverMediaId"]) + media_ids.extend(article.get("mediaIds", ())) + referenced["media"].update(media_ids) + for field in ("replyTo", "quote"): + if field in post: + referenced["posts"].add(post[field]["postId"]) + if "repostOf" in post: + referenced["posts"].add(post["repostOf"]) + + existing = preexisting_ids or {} + return { + resolvers[kind]: tuple( + sorted((defined[kind] | referenced[kind]) - existing.get(resolvers[kind], set())) + ) + for kind in ("users", "posts", "media") + } + + +async def _cleanup_archive(owned_ids: dict[str, tuple[str, ...]]) -> None: + async with graph_uow() as uow: + roots = [] + for resolver, native_ids in owned_ids.items(): + if native_ids: + roots.extend(await uow.blocks.find_json_field_many((resolver,), "id", native_ids)) + tracked_ids = {block.id for block in roots if block.id is not None} + + if not tracked_ids: + return + with TestSession() as db: + for relation in db.exec( + sqlmodel.select(RelationModel).where( + sqlalchemy.or_( + sqlmodel.col(RelationModel.from_).in_(tracked_ids), + sqlmodel.col(RelationModel.to_).in_(tracked_ids), + ) + ) + ).all(): + db.delete(relation) + for block in db.exec( + sqlmodel.select(BlockModel).where(sqlmodel.col(BlockModel.id).in_(tracked_ids)) + ).all(): + db.delete(block) + db.commit() + async with graph_uow() as uow: + remaining = [] + for resolver, native_ids in owned_ids.items(): + if native_ids: + remaining.extend( + await uow.blocks.find_json_field_many((resolver,), "id", native_ids) + ) + if remaining: + raise AssertionError("cleanup left test-owned archive identities in PostgreSQL") + + +def test_real_http_submit_query_preserves_native_wire_payload(monkeypatch): + from extensions.twitter import Extension + + native = _distinct_fixture() + simple_post_id = str(int(uuid.uuid4().hex[:15], 16) + 10**30) + shared_author_id = str(int(simple_post_id) + 1) + owned_ids = _archive_owned_ids( + native, + preexisting_ids={USER_RESOLVER: {shared_author_id}}, + ) + owned_ids[POST_RESOLVER] = tuple(sorted((*owned_ids[POST_RESOLVER], simple_post_id))) + with TestSession() as db: + shared_author = BlockModel( + resolver=USER_RESOLVER, + content=json.dumps({"id": shared_author_id}, separators=(",", ":")), + ) + db.add(shared_author) + db.commit() + db.refresh(shared_author) + shared_author_block_id = shared_author.id + + async def sync_source_types(): + return None + + async def current_config(): + return SimpleNamespace(http_public_base_url="https://core.example") + + monkeypatch.setattr(SourceManager, "sync_source_types_async", sync_source_types) + monkeypatch.setattr(PeerManager, "get_current_config_async", current_config) + token, _ = _issue_session_token() + published = publish_extension(Extension) + shared_author_survived_cleanup = False + try: + with published.client as client: + client.cookies.set(SESSION_COOKIE_NAME, token) + headers = {"Origin": "https://x.com", "X-Tweet-DB-Client": "x-monkey"} + simple_post = { + "id": simple_post_id, + "authorId": shared_author_id, + "createdAt": "2026-10-19T12:00:00Z", + "text": {"text": "HTTP to PostgreSQL round trip"}, + } + simple_submit = client.post( + "/twitter/x-monkey/api/v1/archive/submit", + json={"posts": [simple_post]}, + headers=headers, + ) + assert simple_submit.status_code == 200, simple_submit.text + with TestSession() as db: + simple_root = next( + block + for block in db.exec( + sqlmodel.select(BlockModel).where(BlockModel.resolver == POST_RESOLVER) + ).all() + if json.loads(block.content).get("id") == simple_post_id + ) + assert simple_root.id is not None + assert ( + db.exec( + sqlmodel.select(RelationModel).where( + sqlmodel.col(RelationModel.from_) == simple_root.id, + sqlmodel.col(RelationModel.to_) == shared_author_block_id, + ) + ).first() + is not None + ) + simple_query = client.post( + "/twitter/x-monkey/api/v1/archive/query", + json={"posts": [simple_post_id]}, + headers=headers, + ) + assert simple_query.status_code == 200, simple_query.text + assert simple_query.json()["posts"] == [ + {"id": simple_post_id, "status": "found", "data": simple_post} + ] + + submitted = client.post( + "/twitter/x-monkey/api/v1/archive/submit", + json=native, + headers=headers, + ) + assert submitted.status_code == 200, submitted.text + assert submitted.json()["summary"]["total"] == sum( + len(native[kind]) for kind in ("users", "posts", "media") + ) + + query = { + kind: [record["id"] for record in native[kind]] + for kind in ("users", "posts", "media") + } + queried = client.post( + "/twitter/x-monkey/api/v1/archive/query", + json=query, + headers=headers, + ) + assert queried.status_code == 200, queried.text + response = queried.json() + for kind in ("users", "posts", "media"): + assert [item["status"] for item in response[kind]] == ["found"] * len(native[kind]) + assert [item["data"] for item in response[kind]] == native[kind] + finally: + published.unpublish() + asyncio.run(_cleanup_archive(owned_ids)) + with TestSession() as db: + shared_author_after_cleanup = db.get(BlockModel, shared_author_block_id) + shared_author_survived_cleanup = shared_author_after_cleanup is not None + if shared_author_after_cleanup is not None: + db.delete(shared_author_after_cleanup) + db.commit() + assert shared_author_survived_cleanup, "cleanup removed a preexisting author identity" diff --git a/scripts/tests/twitter/test_archive_contract.py b/scripts/tests/twitter/test_archive_contract.py new file mode 100644 index 00000000..43e7d20b --- /dev/null +++ b/scripts/tests/twitter/test_archive_contract.py @@ -0,0 +1,309 @@ +"""Observable native archive round trips and x-monkey capture merge semantics.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pydantic +import pytest + +from extensions.twitter.archive_merge import merge_archive_record +from extensions.twitter.archive_schema import ( + ArchiveBatch, + ArchivePost, + ArchiveQuery, + QueryResponse, + SessionResponse, + SubmitResponse, +) +from extensions.twitter.model import RichText, TwitterBatch, TwitterPost +from extensions.twitter.reconcile import TwitterReadResult, TwitterWriteResult +from extensions.twitter.x_monkey import ( + decode_submission, + encode_query, + encode_submission, + export_record, +) + + +@pytest.fixture +def full_archive() -> dict[str, Any]: + fixture_path = Path(__file__).parent / "fixtures" / "native_archive.json" + return json.loads(fixture_path.read_text(encoding="utf-8")) + + +def test_native_batch_preserves_complete_source_and_field_presence(full_archive): + original = copy.deepcopy(full_archive) + batch = ArchiveBatch.model_validate(full_archive) + assert batch.model_dump(mode="json", exclude_unset=True) == original + assert json.loads(batch.model_dump_json(exclude_unset=True)) == original + assert full_archive == original + assert batch.posts[0].communityNote is not None + assert batch.posts[0].communityNote.text.model_dump(exclude_unset=True) == { + "text": "来源已核实" + } + + +def test_missing_and_explicit_empty_batches_remain_distinct(): + batch = ArchiveBatch.model_validate({}) + query = ArchiveQuery.model_validate({}) + assert batch.users == batch.posts == batch.media == [] + assert query.users == query.posts == query.media == [] + assert batch.model_dump(mode="json", exclude_unset=True) == {} + assert query.model_dump(mode="json", exclude_unset=True) == {} + assert ArchiveBatch(posts=[]).model_dump(mode="json", exclude_unset=True) == {"posts": []} + + +def test_fixed_protocol_roundtrips_through_owned_facts_and_neutral_results(full_archive): + wire = ArchiveBatch.model_validate(full_archive) + native = decode_submission(wire) + assert type(native) is TwitterBatch + assert type(native.posts[0]) is TwitterPost + assert native.model_dump(mode="json", exclude_unset=True) == full_archive + records = { + (kind, record.id): record + for kind in ("users", "posts", "media") + for record in getattr(native, kind) + } + query = ArchiveQuery(users=["17"], posts=["31", "32", "33"], media=["41"]) + response = encode_query( + query, + TwitterReadResult(records=records, invalid=frozenset({("posts", "33")})), + ) + for kind in ("users", "posts", "media"): + assert ( + getattr(response, kind)[0].data.model_dump(mode="json", exclude_unset=True) + == (full_archive[kind][0]) + ) + assert [item.status for item in response.posts] == ["found", "missing", "failed"] + submitted = encode_submission( + wire, TwitterWriteResult(changed=frozenset(records), block_ids={}) + ) + assert submitted.summary.accepted == submitted.summary.total == 3 + replayed = encode_submission(wire, TwitterWriteResult(changed=frozenset(), block_ids={})) + assert replayed.summary.unchanged == 3 + + +def test_capture_batch_does_not_inherit_remote_request_limits(full_archive): + records = {"posts": [full_archive["posts"][0]] * 1001} + assert len(TwitterBatch.model_validate(records).posts) == 1001 + with pytest.raises(pydantic.ValidationError): + ArchiveBatch.model_validate(records) + + +def test_fixed_export_projects_additive_native_fields_and_retains_open_json(full_archive): + class ExtendedText(RichText): + readingHint: str + + class ExtendedPost(TwitterPost): + text: pydantic.SerializeAsAny[RichText] | None = None + readingHint: str + + post = copy.deepcopy(full_archive["posts"][0]) + post["readingHint"] = "native-only root information" + post["text"]["readingHint"] = "native-only nested information" + native = ExtendedPost.model_validate( + {**post, "text": ExtendedText.model_validate(post["text"])} + ) + exported = export_record("posts", native) + assert exported == full_archive["posts"][0] + assert native.model_dump(mode="json", exclude_unset=True) == post + with pytest.raises(pydantic.ValidationError): + ArchivePost.model_validate(post) + + +@pytest.mark.parametrize( + "invalid_id", ["0", "01", "-1", "1.0", " 1", "1\n", "1１", 1, True] +) +def test_ids_are_positive_decimal_strings(invalid_id): + with pytest.raises(pydantic.ValidationError): + ArchiveQuery.model_validate({"posts": [invalid_id]}) + + +@pytest.mark.parametrize( + ("path", "invalid"), + [ + (("users", 0, "counts", "likes"), True), + (("users", 0, "counts", "likes"), -1), + (("users", 0, "counts", "likes"), 1.0), + (("users", 0, "counts", "likes"), "1"), + (("posts", 0, "createdAt"), "2026-02-29T00:00:00Z"), + (("posts", 0, "createdAt"), "2026-10-19T00:00:00.123Z"), + (("posts", 0, "createdAt"), "2026-10-19T00:00:00+00:00"), + (("posts", 0, "countsObservedAt", "likes"), "2026-10-19T24:00:00Z"), + (("posts", 0, "countsObservedAt", "likes"), "2026-10-19T01:00:00+08:00"), + (("posts", 0, "countsObservedAt", "likes"), "2026-10-19T01:00:00"), + (("posts", 0, "countsObservedAt", "likes"), "2026-10-19T01:00Z"), + (("users", 0, "countsObservedAt", "posts"), "2026-10-19T01:02Z"), + (("posts", 0, "text", "visible"), [9, 2]), + (("posts", 0, "text", "visible"), [False, 2]), + (("posts", 0, "text", "entities"), []), + (("posts", 0, "mediaIds"), []), + (("posts", 0, "editIds"), ["31"]), + (("posts", 0, "card", "poll", "choices"), [{"label": "only one"}]), + (("posts", 0, "card", "extra"), {"unexpected": 1}), + (("posts", 0, "place", "bbox"), [0, True, 2, 3]), + (("posts", 0, "place", "bbox"), [0, float("inf"), 2, 3]), + (("posts", 0, "quote", "tombstone"), ""), + (("media", 0, "video", "variants"), []), + (("media", 0, "crops"), [[0, 0, -1, 10]]), + (("posts", 0, "text"), None), + (("users", 0, "verified", "blue"), None), + (("posts", 0, "article", "blocks", 0, "unknown"), True), + (("media", 0, "unknown"), "discarding this would lose information"), + ], +) +def test_invalid_native_records_fail_instead_of_coercing_or_discarding( + full_archive, path, invalid +): + current = full_archive + for part in path[:-1]: + current = current[part] + current[path[-1]] = invalid + with pytest.raises(pydantic.ValidationError): + ArchiveBatch.model_validate(full_archive) + + +@pytest.mark.parametrize("kind", ["users", "posts", "media"]) +def test_query_and_submission_have_per_kind_limits(full_archive, kind): + assert len(getattr(ArchiveQuery.model_validate({kind: ["1"] * 1000}), kind)) == 1000 + with pytest.raises(pydantic.ValidationError): + ArchiveQuery.model_validate({kind: ["1"] * 1001}) + record = full_archive[kind][0] + assert len(getattr(ArchiveBatch.model_validate({kind: [record] * 1000}), kind)) == 1000 + with pytest.raises(pydantic.ValidationError): + ArchiveBatch.model_validate({kind: [record] * 1001}) + + +def test_query_submit_and_session_wire_roundtrip(full_archive): + query = { + "users": [{"id": "17", "status": "found", "data": full_archive["users"][0]}], + "posts": [{"id": "31", "status": "missing"}], + "media": [{"id": "41", "status": "failed", "error": "invalid stored archive"}], + } + assert ( + QueryResponse.model_validate(query).model_dump(mode="json", exclude_unset=True) == query + ) + submission = { + "summary": {"total": 4, "accepted": 1, "unchanged": 1, "partial": 1, "failed": 1}, + "users": [{"id": "17", "status": "accepted"}], + "posts": [{"id": "31", "status": "unchanged"}, {"id": "32", "status": "partial"}], + "media": [{"id": "41", "status": "failed", "reason": "invalid media"}], + } + assert ( + SubmitResponse.model_validate(submission).model_dump(mode="json", exclude_unset=True) + == submission + ) + session = { + "authenticated": False, + "registered": False, + "username": None, + "expires_at": None, + "account_url": None, + } + assert SessionResponse.model_validate(session).model_dump(mode="json") == session + with pytest.raises(pydantic.ValidationError): + ArchivePost.model_validate({"id": "31", "authorId": "17", "createdAt": None}) + + +def test_merge_is_recursive_replaces_arrays_and_does_not_mutate_inputs(): + existing: dict[str, Any] = { + "id": "31", + "text": {"text": "正文", "visible": [0, 2], "entities": [{"type": "hashtag"}]}, + "article": {"title": "长文", "blocks": [{"text": "第一版"}]}, + } + incoming: dict[str, Any] = { + "text": {"text": "修订正文"}, + "article": {"blocks": [{"text": "完整新版"}]}, + } + old_existing, old_incoming = copy.deepcopy(existing), copy.deepcopy(incoming) + merged: dict[str, Any] = merge_archive_record(existing, incoming) + assert merged == { + "id": "31", + "text": {"text": "修订正文", "visible": [0, 2], "entities": [{"type": "hashtag"}]}, + "article": {"title": "长文", "blocks": [{"text": "完整新版"}]}, + } + merged["article"]["blocks"][0]["text"] = "调用方修改" + merged["text"]["entities"][0]["type"] = "调用方修改" + assert existing == old_existing + assert incoming == old_incoming + + +def test_counter_observations_merge_independently_and_compare_instants(): + existing: dict[str, Any] = { + "counts": {"likes": 10, "views": 100, "quotes": 3}, + "countsObservedAt": { + "likes": "2026-10-19T01:00:00.900Z", + "views": "2026-10-19T01:00:00Z", + "quotes": "2026-10-19T01:00:00Z", + }, + } + merged = merge_archive_record( + existing, + { + "lang": "zh", + "counts": {"likes": 9, "views": 110, "quotes": 2, "bookmarks": 0}, + "countsObservedAt": { + "likes": "2026-10-19T01:00:00Z", + "views": "2026-10-19T01:00:00.100Z", + "quotes": "2026-10-19T01:00:00Z", + }, + }, + ) + assert merged == { + "lang": "zh", + "counts": {"likes": 10, "views": 110, "quotes": 2, "bookmarks": 0}, + "countsObservedAt": { + "likes": "2026-10-19T01:00:00.900Z", + "views": "2026-10-19T01:00:00.100Z", + "quotes": "2026-10-19T01:00:00Z", + }, + } + + +def test_untimed_changes_remove_only_the_changed_counters_previous_time(): + existing: dict[str, Any] = { + "counts": {"likes": 10, "views": 100}, + "countsObservedAt": {"likes": "2026-10-19T01:00:00Z", "views": "2026-10-19T01:00:00Z"}, + } + assert merge_archive_record(existing, {"counts": {"likes": 11, "views": 100}}) == { + "counts": {"likes": 11, "views": 100}, + "countsObservedAt": {"views": "2026-10-19T01:00:00Z"}, + } + assert merge_archive_record(existing, {"counts": {"likes": 11, "views": 101}}) == { + "counts": {"likes": 11, "views": 101} + } + + +def test_equal_counts_keep_new_observation_metadata_and_replay_is_unchanged(): + existing: dict[str, Any] = { + "counts": {"likes": 10}, + "countsObservedAt": {"likes": "2026-10-19T01:00:00Z"}, + } + incoming: dict[str, Any] = { + "counts": {"likes": 10}, + "countsObservedAt": {"likes": "2026-10-19T01:00:00.001Z"}, + } + merged = merge_archive_record(existing, incoming) + assert merged == incoming + assert merged != existing + assert merge_archive_record(merged, incoming) == merged + assert ( + merge_archive_record(existing, {"countsObservedAt": incoming["countsObservedAt"]}) + == existing + ) + + +def test_capture_time_comparison_matches_javascript_millisecond_precision(): + existing: dict[str, Any] = { + "counts": {"likes": 10}, + "countsObservedAt": {"likes": "2026-10-19T01:00:00.1009Z"}, + } + incoming: dict[str, Any] = { + "counts": {"likes": 11}, + "countsObservedAt": {"likes": "2026-10-19T01:00:00.1001Z"}, + } + assert merge_archive_record(existing, incoming) == incoming diff --git a/scripts/tests/twitter/test_archive_http.py b/scripts/tests/twitter/test_archive_http.py new file mode 100644 index 00000000..71a846ce --- /dev/null +++ b/scripts/tests/twitter/test_archive_http.py @@ -0,0 +1,328 @@ +"""HTTP boundary checks for the x-monkey-compatible remote archive API.""" + +from __future__ import annotations + +import time +from types import SimpleNamespace + +from fastapi import FastAPI +from fastapi.testclient import TestClient +import jwt +import pytest + +from app.business.peer import PeerManager +from app.database_contract.constants import JWT_ALGORITHM +from app.middleware import create_peer_jwt +from app.settings import Settings, settings +from extensions.twitter import archive_http, archive_session +from extensions.twitter.archive_schema import QueryResponse, SubmitResponse, SubmitSummary +from tests.extensions.runtime_support import publish_extension + + +@pytest.fixture +def archive_client(monkeypatch): + async def current_config(): + return SimpleNamespace(http_public_base_url="https://core.example/prefix") + + monkeypatch.setattr(PeerManager, "get_current_config_async", current_config) + app = FastAPI() + app.include_router(archive_session.router, prefix="/twitter/x-monkey") + app.include_router(archive_http.router, prefix="/twitter/x-monkey") + return TestClient(app, base_url="https://core.example") + + +def _session_cookie() -> str: + token, _ = archive_session._issue_session_token() + return token + + +def test_session_lifetime_defaults_to_seven_days_and_supports_environment_override( + monkeypatch, +): + required = { + "database_url": "postgresql+psycopg://unused:unused@127.0.0.1:1/unused", + "jwt_secret": "test-only-jwt-secret-at-least-32-bytes", + } + monkeypatch.delenv("TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS", raising=False) + default = Settings(_env_file=None, **required) + assert default.twitter_x_monkey_jwt_lifetime_seconds == 7 * 24 * 60 * 60 + + monkeypatch.setenv("TWITTER_X_MONKEY_JWT_LIFETIME_SECONDS", "3600") + overridden = Settings(_env_file=None, **required) + assert overridden.twitter_x_monkey_jwt_lifetime_seconds == 3600 + + +def test_connect_exchanges_same_origin_bearer_for_scoped_cookie(archive_client): + response = archive_client.post( + "/twitter/x-monkey/session", + json={}, + headers={ + "Origin": "https://core.example", + "Authorization": ( + f"Bearer {create_peer_jwt(settings.jwt_secret, lifetime_seconds=60)}" + ), + }, + ) + + assert response.status_code == 200 + assert response.json()["authenticated"] is True + cookie = response.headers["set-cookie"] + assert "inkcre_twitter_archive=" in cookie + assert "HttpOnly" in cookie and "Secure" in cookie and "samesite=none" in cookie.lower() + assert "Path=/prefix/twitter/x-monkey/" in cookie + readback = archive_client.get( + "/twitter/x-monkey/api/v1/session", + headers={"Cookie": cookie.split(";", 1)[0]}, + ) + assert readback.status_code == 200 + assert readback.json()["authenticated"] is True + + +def test_session_exchange_rejects_non_peer_or_cross_origin_credentials(archive_client): + session_cookie = _session_cookie() + missing_bearer = archive_client.post( + "/twitter/x-monkey/session", + json={}, + headers={ + "Origin": "https://core.example", + "Cookie": f"{archive_session.SESSION_COOKIE_NAME}={session_cookie}", + }, + ) + assert missing_bearer.status_code == 401 + + session_as_bearer = archive_client.post( + "/twitter/x-monkey/session", + json={}, + headers={ + "Origin": "https://core.example", + "Authorization": f"Bearer {session_cookie}", + }, + ) + assert session_as_bearer.status_code == 401 + + foreign_origin = archive_client.post( + "/twitter/x-monkey/session", + json={}, + headers={ + "Origin": "https://attacker.example", + "Authorization": f"Bearer {create_peer_jwt(settings.jwt_secret)}", + }, + ) + assert foreign_origin.status_code == 403 + + +def test_account_page_uses_nonce_csp_and_only_same_origin_webcrypto(archive_client): + response = archive_client.get("/twitter/x-monkey/account") + assert response.status_code == 200 + assert 'id="jwt-secret" type="password"' in response.text + assert "JWT_SECRET" in response.text + assert '

尚未连接。

' in response.text + assert "status.textContent = 'Connected'" in response.text + assert "crypto.subtle" in response.text + assert "exp: now + 60" in response.text + assert "fetch('./session'" in response.text + assert "fetch('./api/v1/session'" in response.text + assert "localStorage" not in response.text and "sessionStorage" not in response.text + csp = response.headers["content-security-policy"] + nonce = response.text.split('