diff --git a/apps/cli/src/ui/view/transcript/assistant-message.ts b/apps/cli/src/ui/view/transcript/assistant-message.ts index 0cf39aaf2..9525a482c 100644 --- a/apps/cli/src/ui/view/transcript/assistant-message.ts +++ b/apps/cli/src/ui/view/transcript/assistant-message.ts @@ -9,7 +9,7 @@ import { Text, } from "@step-harness/pi-tui"; import type { AssistantMessage } from "@step-harness/providers"; -import { createMarkdownTransform } from "./markdown-transform.ts"; +import { createLiveMarkdownTransform } from "./markdown-transform.ts"; const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; @@ -18,6 +18,9 @@ const OSC133_ZONE_FINAL = "\x1b]133;C\x07"; /** Cached zone-marked output, keyed by the child lines it was derived from. */ type ZoneCache = { width: number; native: string[]; lines: string[] }; +/** A `Markdown` instance kept alive across deltas of the same message. */ +type MarkdownSlot = { kind: "text" | "thinking"; markdown: Markdown }; + /** * Component that renders a complete assistant message */ @@ -32,6 +35,15 @@ export class AssistantMessageComponent extends Container { private hasToolCalls = false; private isStreaming = false; private zoneCache?: ZoneCache; + /** + * Markdown children in content order. + * + * A streamed reply calls `updateContent` for every delta. Rebuilding the + * components would throw away the markdown renderer's certified prefix, so the + * instances are kept and re-pointed at the new text instead. + */ + private markdownSlots: MarkdownSlot[] = []; + private slotCursor = 0; constructor( message?: AssistantMessage, @@ -80,7 +92,14 @@ export class AssistantMessageComponent extends Container { } setOutputPad(padding: number): void { + if (padding === this.outputPad) { + return; + } this.outputPad = padding; + // Padding is fixed when a `Markdown` is constructed, so the slots cannot be + // reused across a padding change. + this.markdownSlots = []; + this.slotCursor = 0; if (this.lastMessage) { this.updateContent(this.lastMessage); } @@ -121,6 +140,7 @@ export class AssistantMessageComponent extends Container { // Clear content container this.contentContainer.clear(); + this.slotCursor = 0; const hasVisibleContent = message.content.some( (c) => (c.type === "text" && c.text.trim()) || (c.type === "thinking" && c.thinking.trim()), @@ -136,11 +156,7 @@ export class AssistantMessageComponent extends Container { if (content.type === "text" && content.text.trim()) { // Assistant text messages with no background - trim the text // Set paddingY=0 to avoid extra spacing before tool executions - this.contentContainer.addChild( - new Markdown(content.text.trim(), this.outputPad, 0, this.markdownTheme, undefined, { - transform: createMarkdownTransform("assistant", this.isStreaming, this.markdownTransformers), - }), - ); + this.contentContainer.addChild(this.textMarkdown(content.text.trim())); } else if (content.type === "thinking") { const thinkingBlocks: string[] = []; for (; i < message.content.length; i++) { @@ -172,31 +188,14 @@ export class AssistantMessageComponent extends Container { ); } else { // Render each run of thinking blocks as one Markdown section. - this.contentContainer.addChild( - new Markdown( - thinkingBlocks.join("\n\n"), - this.outputPad, - 0, - { ...this.markdownTheme, code: (text: string) => theme.fg("muted", text) }, - { - color: (text: string) => theme.fg("thinkingText", text), - italic: true, - }, - { - transform: createMarkdownTransform( - "assistant-thinking", - this.isStreaming, - this.markdownTransformers, - ), - }, - ), - ); + this.contentContainer.addChild(this.thinkingMarkdown(thinkingBlocks.join("\n\n"))); } if (hasVisibleContentAfter) { this.contentContainer.addChild(new Spacer(1)); } } } + this.markdownSlots.length = this.slotCursor; // Check if incomplete/failed - show after partial content. // For aborted/error tool calls, tool execution components show the error. @@ -223,4 +222,66 @@ export class AssistantMessageComponent extends Container { } } } + + /** Markdown for an assistant text block, reused while the block keeps its kind. */ + private textMarkdown(text: string): Markdown { + return this.claimSlot( + "text", + (value) => + new Markdown(value, this.outputPad, 0, this.markdownTheme, undefined, { + transform: createLiveMarkdownTransform( + "assistant", + () => this.isStreaming, + () => this.markdownTransformers, + ), + }), + text, + ); + } + + /** Markdown for one run of thinking blocks. */ + private thinkingMarkdown(text: string): Markdown { + return this.claimSlot( + "thinking", + (value) => + new Markdown( + value, + this.outputPad, + 0, + { ...this.markdownTheme, code: (code: string) => theme.fg("muted", code) }, + { + color: (styled: string) => theme.fg("thinkingText", styled), + italic: true, + }, + { + transform: createLiveMarkdownTransform( + "assistant-thinking", + () => this.isStreaming, + () => this.markdownTransformers, + ), + }, + ), + text, + ); + } + + /** + * Return the slot at the current content position, re-pointing it at `text`. + * + * Slots are matched by kind rather than by content index: a message can gain a + * thinking block in front of a text block, and an instance is only worth keeping + * when it renders the same kind of content as before. + */ + private claimSlot(kind: MarkdownSlot["kind"], create: (text: string) => Markdown, text: string): Markdown { + const slot = this.markdownSlots[this.slotCursor]; + if (slot !== undefined && slot.kind === kind) { + slot.markdown.setText(text); + this.slotCursor++; + return slot.markdown; + } + const markdown = create(text); + this.markdownSlots[this.slotCursor] = { kind, markdown }; + this.slotCursor++; + return markdown; + } } diff --git a/apps/cli/src/ui/view/transcript/markdown-transform.ts b/apps/cli/src/ui/view/transcript/markdown-transform.ts index 1b776fec9..6d4008833 100644 --- a/apps/cli/src/ui/view/transcript/markdown-transform.ts +++ b/apps/cli/src/ui/view/transcript/markdown-transform.ts @@ -9,6 +9,26 @@ export function createMarkdownTransform( applyMarkdownTransformers(markdown, { messageType, isStreaming, availableWidth }, transformers); } +/** + * Transform for a component that outlives a single frame. + * + * A reused `Markdown` instance keeps the transform it was constructed with, so the + * streaming state has to be read at call time: the final frame of a reply renders + * in "not streaming" mode, and Mermaid warnings only appear then. + */ +export function createLiveMarkdownTransform( + messageType: MarkdownTransformContext["messageType"], + readIsStreaming: () => boolean, + readTransformers: () => readonly MarkdownTransformer[], +): (markdown: string, availableWidth: number) => string { + return (markdown, availableWidth) => + applyMarkdownTransformers( + markdown, + { messageType, isStreaming: readIsStreaming(), availableWidth }, + readTransformers(), + ); +} + function applyMarkdownTransformers( markdown: string, context: MarkdownTransformContext, diff --git a/docs/tui-rendering-pipeline.md b/docs/tui-rendering-pipeline.md index b2479fd18..c7ccc0eea 100644 --- a/docs/tui-rendering-pipeline.md +++ b/docs/tui-rendering-pipeline.md @@ -115,6 +115,53 @@ 容器层之上没有脏标记：组件靠"返回同一个数组实例"表达"我没变"，靠 `invalidate()` 丢弃自己的缓存。产品侧大量使用后者（主题切换、语法高亮加载完成时全树 `invalidate`）。 +### 4.1 块级数学 tokenizer 的扫描成本 + +`Markdown` 为块级数学（`$$…$$`、`\[…\]`）注册了一个 Marked block 扩展（`LATEX_MARKDOWN_EXTENSIONS` 的 `latexBlock`）。Marked 只在**即将进入 paragraph / text 之前**才收集 block 扩展的 `start()`：space、code、heading、hr、blockquote、list、html、def、table、lheading 命中时根本不会调用它。调用时传入的是 `src.slice(1)`，即**尚未消费的剩余源文本**。所以一次全文正则的代价是 `O(段落数 × 剩余长度)`，而不是 `O(块数 × 长度)`；标题、围栏、表格、分隔线不触发。 + +多数文档根本不含块级数学，所以该判据先做子串检查（`includes("$$")` / `includes("\\[")`），只有可能匹配时才跑正则。语义不变：正则能匹配的字符串必然包含这两个子串之一，短路不会漏判；子串检查的误判（例如围栏代码里的 `\[`）只是多跑一次正则。落点是 `components/markdown-stream.ts` 的 `findBlockMathStart`，tokenizer 的 `start()` 与流式渲染的 hazard 扫描共用它，因此"两处字面量必须同步"这个静默失败点不存在。 + +注意这**不是**把扫描消掉：短路后每个段落边界仍会扫一遍剩余文本，只是常数更小。本机 146 KB 混合消息、120 列、120 B/delta 的流式模拟里，这项短路把累计耗时从 12 340 ms 降到 9 845 ms（约 1.3 倍）。回归门禁用确定性的正则调用计数（`test/markdown-block-math.test.ts`），不依赖实际耗时。 + +### 4.2 认证前缀：流式增量渲染 + +流式回复每个 delta 都调用一次 `Markdown.setText`。整篇重 lex、重渲染、重换行让累计成本按消息长度的平方增长，所以 `Markdown` 保留"已经确定终结"的块的行，只重算可变尾部（`renderContent` / `renderFrom`）。这不是"留最后一块"的启发式：只有能证明"从这里以全新 Lexer 重启，与渲染全文等价"的位置才会被冻结。 + +保留的状态（`packages/tui/src/components/markdown.ts`）： + +- `stableSourceEnd`：已认证前缀的长度（归一化后的 UTF-16 偏移；tab 展开是逐字符替换，前缀关系保持）。 +- `stablePaddedLines`：前缀的最终行（已加上左右 margin、背景、右侧补白），直接参与拼接。 +- `stableNoWrap`：前缀里围栏代码行的 pre-wrap 文本。换行阶段的 no-wrap 判定按**行文本**而不是来源，所以这些行必须继续参与后续帧的判定。 +- `stableWrappedProseRows`：前缀里被**软换行**过的散文行文本。它是"未来影响过去"的唯一入口，见下。 +- `guard`：前缀留给尾部的解析事实——块级数学是否出现、是否有未闭合的行首 `[`、inline `` 与 `` 的嵌套深度。 +- `previousSource` / `layoutWidth`：判断这一帧是不是同一份文档的追加，以及宽度是否变化。 + +认证条件（`findCertifiedCut`）：候选必须紧跟一个**顶层 `space` token**（Marked 会把单个换行折进前一个 token，所以顶层 `space` 一定覆盖一个真实空行），且它后面的 token 是"见证块"——列 0 开始、首物理行已结束的 paragraph 或 heading。list、blockquote、table、围栏、缩进代码都不能当见证：它们可以跨空行继续吞入，把候选点之前的块重新解释（list 合并与序号重排、table 加行改列宽、` a\n\n b` 仍是同一个缩进代码块）。返回最靠右的候选；没有候选就不推进，尾部仍由 canonical Lexer 完整解释并显示所有不完整行。 + +必须整篇重算的情形分三类，混为一谈会把"有未闭合围栏"实现成"每帧全文渲染"，直接吃掉代码占比高的收益： + +- **粘性（直到文本被替换）**：任何行首块级数学 opener（pending latex token 会重剪它前面的段落）、任意 reference definition（`tokens.links` 非空，定义对所有块可见）。 +- **本帧重算一次**：宽度、padding、主题或 `invalidate()` 变化；文本被替换而不是追加；`message_end` 的流式/终态切换。 +- **只重算尾部**：没有新的证书、未闭合的 label 或 inline 标签、持续的 list/table/缩进代码/quote、open 或 EOF 临时围栏、token raw 无法精确重建（Marked 会规范化部分行尾空白，此时本帧不推进偏移，但不必把整份文档标记为粘性）。 + +`nowrapLines` 的语义没有改：它仍然是文档级的行文本集合，围栏代码行让**同文本**的散文行变成硬裁剪。增量路径用两件事保持等价：尾部渲染前用 `stableNoWrap` 播种，尾部散文行因此照旧被裁剪；而尾部新出现的代码行一旦与 `stableWrappedProseRows` 撞文本，就说明已冻结的散文行本该被裁剪，这一帧整篇重渲（`collidesWithStableRows`）。被裁剪过的行不进 `stableWrappedProseRows`——集合只会增长，裁剪是不可逆的。 + +`renderDirtyStart` 报告"相对上一帧输出可能变化的最早行"，复用前缀时等于 `paddingY + 上一帧的前缀行数`，整篇重算时是 0（上下 padding 行内容相同，但换主题时它们的底色也会变，不能假定可复用），交回 `Container.render` 做前缀复用。返回的数组每帧都是新数组，已发出的数组不会被就地修改。 + +正确性门禁在 `test/markdown-stream-prefix.test.ts`：每个前缀、多种宽度，与"新建实例"的冷渲染逐字节比较，覆盖 35 份针对上述构造的对抗 fixture、仓库文档抽样、以及 200 份固定种子的块结构组合。`test/markdown-stream-work.test.ts` 断言工作量而不是耗时：60 KB 混合文档流式渲染只把约两倍文档长度的源码交给 lexer，文档翻倍时 lex/渲染/换行的计数不超过 2.5 倍；块级数学、reference definition、无空行段落、代码行撞文本这些情形分别断言"从不复用前缀"且输出仍与冷渲染一致。 + +实测（本机，120 列，120 B/delta，累计耗时含每帧完整管线）： + +| 场景 | 现状（每帧新建实例） | 增量渲染 | +| --- | --- | --- | +| 60 KB 散文 | 1 437 ms | 18 ms（约 79 倍） | +| 60 KB 表格 | 2 095 ms | 52 ms（约 40 倍） | +| 60 KB 50% 代码 | 462 ms | 13 ms（约 35 倍） | +| 60 KB 单个超长段落（无可认证点） | 403 ms | 411 ms（无收益，输出正确） | +| 146 KB 混合 | 12 340 ms | 275 ms（约 45 倍） | + +调用方必须复用同一个 `Markdown` 实例，否则实例级状态随组件一起消失；`AssistantMessageComponent` 因此按内容种类保留实例并只调 `setText`（`markdownSlots`）。每帧仍要做一次指针复制拼出扁平数组，以及主屏/全屏自己的差分；这两项是独立成本，不随前缀复用消失。Step 呈现的包装层（`step-message.ts` 的 `renderNativeRun`：`hideCompleteFences`、空行折叠、`trimOuterRows`）同样每帧扫一遍全部 native 行，那部分成本不受这里影响，需要单独处理。 + ## 5. 主屏差分渲染（regular） `TuiMainScreen.doRender`（`tui-main-screen.ts:329-750`）每帧的完整顺序： diff --git a/packages/tui/src/components/markdown-stream.ts b/packages/tui/src/components/markdown-stream.ts new file mode 100644 index 000000000..dab4b6fec --- /dev/null +++ b/packages/tui/src/components/markdown-stream.ts @@ -0,0 +1,296 @@ +import type { Token } from "marked"; + +/** + * Block-level math opener at the start of a physical line. + * + * This is the single definition of "where a block-math token may start": the + * Marked `start()` hint and the streaming hazard scan both use it, so the two + * cannot drift apart and silently drop an opener. + */ +const BLOCK_MATH_START = /(?:^|\n) {0,3}(?:\$\$|\\\[)/; + +/** + * Offset of the first block-math opener in `source`, or `undefined`. + * + * The substring pre-check keeps ordinary documents off the regex: Marked calls + * the block extension's `start()` at every paragraph boundary with the whole + * unconsumed remainder, so running the pattern there unconditionally costs + * `O(paragraphs x remaining length)` per render. + */ +export function findBlockMathStart(source: string): number | undefined { + if (!source.includes("$$") && !source.includes("\\[")) { + return undefined; + } + const match = BLOCK_MATH_START.exec(source); + return match ? match.index + (match[0].startsWith("\n") ? 1 : 0) : undefined; +} + +/** + * Top-level token types whose rendering only depends on the token itself, its + * immediate neighbour type, and the document-wide no-wrap set. Anything else + * makes the incremental path bail out instead of guessing. + */ +const LOCAL_TOKEN_TYPES: ReadonlySet = new Set([ + "space", + "paragraph", + "text", + "heading", + "list", + "code", + "table", + "blockquote", + "hr", + "html", + "latexBlock", +]); + +/** Inline `` and `` openers keep lexer state alive. */ +const INLINE_LINK_OPENER = //gi; +const INLINE_RAW_OPENER = /<(?:pre|code|kbd|script)[\s>]/gi; +const INLINE_RAW_CLOSER = /<\/(?:pre|code|kbd|script)>/gi; + +/** + * Facts about the certified prefix that the tail parse depends on but cannot + * observe from its own source text. + */ +export interface StreamGuard { + /** + * Offset up to which the text has been scanned for block-math openers. Always + * a line start, so a scan continuation cannot miss a `^`-anchored match. + */ + mathScanEnd: number; + /** A block-math opener was seen, so no prefix of this document is reusable. */ + blockMath: boolean; + /** A line-start `[` is still open and could become a reference definition. */ + openReferenceLabel: boolean; + /** Depth of inline `` openers in the certified prefix. */ + openLinkTags: number; + /** Depth of inline `` openers in the certified prefix. */ + openRawTags: number; +} + +export function createStreamGuard(): StreamGuard { + return { + mathScanEnd: 0, + blockMath: false, + openReferenceLabel: false, + openLinkTags: 0, + openRawTags: 0, + }; +} + +export function cloneStreamGuard(guard: StreamGuard): StreamGuard { + return { ...guard }; +} + +/** + * Continue the block-math scan over `text`. + * + * Only text appended since the previous call is scanned, so the cost stays + * proportional to the delta even though the decision is document-wide. The scan + * starts at a physical line start and reuses `findBlockMathStart`, so the hazard + * decision and the tokenizer hint share one predicate. + */ +export function scanBlockMath(text: string, guard: StreamGuard): void { + if (guard.blockMath || guard.mathScanEnd >= text.length) { + return; + } + const lineStart = lineStartAt(text, guard.mathScanEnd); + if (findBlockMathStart(text.slice(lineStart)) !== undefined) { + // `tokenizeBlockLatex` can turn this into a token that runs to the end of the + // document, and its `start()` hint re-clips the paragraph in front of it, so + // blocks certified earlier are no longer final. + guard.blockMath = true; + return; + } + // Leave the last line unscanned: it has no line ending yet, so the next delta can + // still complete an opener there. + const lastNewline = text.lastIndexOf("\n"); + guard.mathScanEnd = lastNewline < 0 ? guard.mathScanEnd : Math.max(guard.mathScanEnd, lastNewline + 1); +} + +/** + * Update the inline-tag depth for the tokens being certified. + * + * Marked keeps `inLink`/`inRawBlock` on the lexer for the whole document, so an + * unbalanced `` in the certified prefix changes how the tail is parsed. + * Fenced code is skipped because its content never reaches the inline tokenizer; + * inline code spans are deliberately not skipped, which can only make this check + * more conservative. + */ +export function applyInlineTagDepth(tokens: readonly Token[], guard: StreamGuard): void { + let link = guard.openLinkTags; + let raw = guard.openRawTags; + for (const token of tokens) { + if (token.type === "code") { + continue; + } + link = applyTagDepth(token.raw, INLINE_LINK_OPENER, INLINE_LINK_CLOSER, link); + raw = applyTagDepth(token.raw, INLINE_RAW_OPENER, INLINE_RAW_CLOSER, raw); + } + guard.openLinkTags = link; + guard.openRawTags = raw; +} + +/** + * Track a line-start `[` that could still become a reference definition. + * + * Marked tries `def` before `paragraph`, and a definition label may span blank + * lines, so an unclosed label in the certified prefix can reinterpret it later. + * Escaped brackets and any `]` clear the candidate. + */ +export function scanReferenceLabels(text: string, guard: StreamGuard): void { + let open = guard.openReferenceLabel; + let lineStart = 0; + let backslashes = 0; + for (let index = 0; index < text.length; index++) { + const char = text[index]; + if (char === "\n") { + lineStart = index + 1; + backslashes = 0; + continue; + } + if (char === "\\") { + backslashes++; + continue; + } + if (char === "]" && backslashes % 2 === 0) { + open = false; + } else if (char === "[" && backslashes % 2 === 0 && index - lineStart <= 3 && isBlank(text, lineStart, index)) { + open = true; + } + backslashes = 0; + } + guard.openReferenceLabel = open; +} + +/** Whether the guard permits certifying a prefix that ends here. */ +export function guardIsClean(guard: StreamGuard): boolean { + return !guard.blockMath && !guard.openReferenceLabel && guard.openLinkTags === 0 && guard.openRawTags === 0; +} + +/** + * Index of the first token that must stay mutable, or `undefined` when no prefix + * can be certified. + * + * A candidate is only accepted directly after a top-level `space` token: Marked + * folds a single line ending into the previous token, so a `space` token always + * covers a real blank line and therefore separates two self-contained blocks. + * The token after it must start a new top-level block instead of continuing the + * previous one: a paragraph or heading that begins at column 0 once its first + * physical line has ended. An unfinished first line is rejected because it can + * still turn into a list item, a setext underline, or a table delimiter, all of + * which reach back across the blank line. Lists, quotes, tables, fences, and + * indented code are rejected because they can absorb what follows a blank line + * and would then rewrite the tokens before the candidate. + */ +export function findCertifiedCut(tokens: readonly Token[]): number | undefined { + for (let cut = tokens.length - 1; cut >= 2; cut--) { + if (tokens[cut - 1]?.type !== "space") { + continue; + } + const witness = tokens[cut]; + if (!witness || !isCertifyingWitness(witness, cut + 1 < tokens.length)) { + continue; + } + if (!isLocalPrefix(tokens, cut)) { + continue; + } + return cut; + } + return undefined; +} + +/** + * Raw length of `tokens[0..count)`. + * + * Callers must have verified that the raws concatenate back to the source slice; + * Marked drops repeated reference definitions and normalises some trailing + * whitespace, so the sum is only meaningful when that check passed. + */ +export function rawLengthOf(tokens: readonly Token[], count: number): number { + let total = 0; + for (let index = 0; index < count; index++) { + total += tokens[index]?.raw.length ?? 0; + } + return total; +} + +/** Whether the token raws reproduce `source` exactly. */ +export function rawsCoverSource(tokens: readonly Token[], source: string): boolean { + let offset = 0; + for (const token of tokens) { + const raw = token.raw; + if (!source.startsWith(raw, offset)) { + return false; + } + offset += raw.length; + } + return offset === source.length; +} + +function isCertifyingWitness(token: Token, hasFollowingToken: boolean): boolean { + if (token.type !== "paragraph" && token.type !== "heading") { + return false; + } + const raw = token.raw; + if (raw.length === 0) { + return false; + } + const first = raw.charCodeAt(0); + if (first === 32 || first === 9) { + return false; + } + // An incomplete first line can still become a list marker, a setext underline, + // or a table delimiter, all of which reach back to earlier blocks. + if (raw.indexOf("\n") > 0) { + return true; + } + // A single-line token has no trailing newline of its own; the line ending is + // folded into the following `space` token, so anything after it proves the line + // is closed. + return hasFollowingToken && !raw.includes("\n"); +} + +function isLocalPrefix(tokens: readonly Token[], count: number): boolean { + for (let index = 0; index < count; index++) { + const token = tokens[index]; + if (!token || !LOCAL_TOKEN_TYPES.has(token.type)) { + return false; + } + } + return true; +} + +function applyTagDepth(text: string, opener: RegExp, closer: RegExp, depth: number): number { + let next = depth; + const events: Array<{ index: number; delta: number }> = []; + opener.lastIndex = 0; + for (let match = opener.exec(text); match !== null; match = opener.exec(text)) { + events.push({ index: match.index, delta: 1 }); + } + closer.lastIndex = 0; + for (let match = closer.exec(text); match !== null; match = closer.exec(text)) { + events.push({ index: match.index, delta: -1 }); + } + events.sort((left, right) => left.index - right.index); + for (const event of events) { + next = Math.max(0, next + event.delta); + } + return next; +} + +function lineStartAt(text: string, offset: number): number { + const newline = text.lastIndexOf("\n", Math.max(0, offset - 1)); + return newline < 0 ? 0 : newline + 1; +} + +function isBlank(text: string, from: number, to: number): boolean { + for (let index = from; index < to; index++) { + if (text.charCodeAt(index) !== 32) { + return false; + } + } + return true; +} diff --git a/packages/tui/src/components/markdown-work.ts b/packages/tui/src/components/markdown-work.ts new file mode 100644 index 000000000..3621f018d --- /dev/null +++ b/packages/tui/src/components/markdown-work.ts @@ -0,0 +1,40 @@ +/** + * Deterministic work counters for the markdown renderer. + * + * Streaming performance is a property of *how much work a frame does*, and wall + * clock time cannot assert that: a loaded machine can fail a real optimization + * and pass a regression. Tests therefore read these counters instead, and they + * count the inputs a frame hands to the expensive stages rather than the time + * spent in them. + * + * This is a diagnostic seam, not telemetry: nothing is reported off-process, and + * the counters are only meaningful after `resetMarkdownWork()`. + */ +export interface MarkdownWork { + /** Normalized source units handed to the block lexer. */ + lexUnits: number; + /** Top-level tokens passed to the token renderer. */ + renderedTokens: number; + /** Rows measured and wrapped for terminal width. */ + wrappedRows: number; + /** Frames that re-rendered the whole document. */ + coldFrames: number; + /** Frames that reused a certified prefix. */ + reusedFrames: number; +} + +export const markdownWork: MarkdownWork = { + lexUnits: 0, + renderedTokens: 0, + wrappedRows: 0, + coldFrames: 0, + reusedFrames: 0, +}; + +export function resetMarkdownWork(): void { + markdownWork.lexUnits = 0; + markdownWork.renderedTokens = 0; + markdownWork.wrappedRows = 0; + markdownWork.coldFrames = 0; + markdownWork.reusedFrames = 0; +} diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index 6b27a4671..cdb0b6a47 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -1,8 +1,22 @@ -import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens } from "marked"; +import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens, type TokensList } from "marked"; import { renderLatex } from "../latex.ts"; import { getCapabilities, hyperlink, isImageLine } from "../terminal-image.ts"; import type { Component } from "../tui.ts"; import { applyBackgroundToLine, truncateToWidth, visibleWidth, wrapTextWithAnsi } from "../utils.ts"; +import { + applyInlineTagDepth, + cloneStreamGuard, + createStreamGuard, + findBlockMathStart, + findCertifiedCut, + guardIsClean, + rawLengthOf, + rawsCoverSource, + type StreamGuard, + scanBlockMath, + scanReferenceLabels, +} from "./markdown-stream.ts"; +import { markdownWork } from "./markdown-work.ts"; const STRICT_STRIKETHROUGH_REGEX = /^(~~)(?=[^\s~])((?:\\.|[^\\])*?(?:\\.|[^\s~\\]))\1(?=[^~]|$)/; @@ -125,8 +139,9 @@ const LATEX_MARKDOWN_EXTENSIONS: readonly TokenizerExtension[] = [ name: "latexBlock", level: "block", start(source) { - const match = /(?:^|\n) {0,3}(?:\$\$|\\\[)/.exec(source); - return match ? match.index + (match[0].startsWith("\n") ? 1 : 0) : undefined; + // Marked calls this at every paragraph boundary with the unconsumed + // remainder, so the shared predicate short-circuits before the pattern. + return findBlockMathStart(source); }, tokenizer: tokenizeBlockLatex, }, @@ -258,6 +273,26 @@ export class Markdown implements Component { private cachedWidth?: number; private cachedLines?: string[]; + /** Length of the certified source prefix; 0 when nothing may be reused. */ + private stableSourceEnd = 0; + /** Padded output rows of the certified prefix. */ + private stablePaddedLines: string[] = []; + /** + * Pre-wrap row text of prose rows in the certified prefix that were softly + * wrapped. A code row that arrives later with the same text would clip them. + */ + private stableWrappedProseRows = new Set(); + /** Pre-wrap row text of fenced-code rows in the certified prefix. */ + private stableNoWrap = new Set(); + /** Content width of the previous frame. */ + private layoutWidth = 0; + /** Normalized source of the previous frame, for append detection. */ + private previousSource = ""; + /** First output row that may differ from the previous frame. */ + renderDirtyStart?: number; + /** Facts the certified prefix leaves behind for the tail parse. */ + private guard: StreamGuard = createStreamGuard(); + constructor( text: string, paddingX: number, @@ -276,13 +311,20 @@ export class Markdown implements Component { setText(text: string): void { this.text = text; - this.invalidate(); + // Only the outer cache is stale here: a caller that appends to a streamed + // reply still has a valid prefix, and `render` decides whether to reuse it. + this.cachedText = undefined; + this.cachedWidth = undefined; + this.cachedLines = undefined; } invalidate(): void { this.cachedText = undefined; this.cachedWidth = undefined; this.cachedLines = undefined; + // Theme, width, and option changes make previously rendered rows wrong, not + // just stale, so the certified prefix cannot survive an invalidate. + this.resetStream(); } render(width: number): string[] { @@ -298,6 +340,7 @@ export class Markdown implements Component { // Don't render anything if there's no actual text if (!text || text.trim() === "") { const result: string[] = []; + this.resetStream(); // Update cache this.cachedText = this.text; this.cachedWidth = width; @@ -306,83 +349,265 @@ export class Markdown implements Component { } // Replace tabs with 3 spaces for consistent rendering - const normalizedText = text.replace(/\t/g, " "); + const normalizedText = text.includes("\t") ? text.replace(/\t/g, " ") : text; + const contentLines = this.renderContent(normalizedText, contentWidth, width); + + // Add top/bottom padding (empty lines) + const emptyLine = " ".repeat(width); + const bgFn = this.defaultTextStyle?.bgColor; + const emptyLines: string[] = []; + for (let i = 0; i < this.paddingY; i++) { + const line = bgFn ? applyBackgroundToLine(emptyLine, width, bgFn) : emptyLine; + emptyLines.push(line); + } + + // Combine top padding, content, and bottom padding + const result = emptyLines.concat(contentLines, emptyLines); + + // Update cache + this.cachedText = this.text; + this.cachedWidth = width; + this.cachedLines = result; + + return result.length > 0 ? result : [""]; + } + + /** + * Render the document, reusing the certified prefix when it is still valid. + * + * A streamed reply appends to the same document on every delta. Re-lexing and + * re-laying out everything that already ended makes the total cost quadratic in + * the message length, so the renderer keeps the rows of certified blocks and + * only re-renders the mutable tail. Certification is deliberately narrow: see + * `findCertifiedCut` for the exact conditions, and `renderFrom` for the cases + * that force a full re-render. + */ + private renderContent(normalizedText: string, contentWidth: number, width: number): string[] { + const appendOnly = + this.previousSource.length > 0 && + normalizedText.length >= this.previousSource.length && + normalizedText.startsWith(this.previousSource); + if (!appendOnly || this.layoutWidth !== contentWidth) { + this.resetStream(); + } + this.previousSource = normalizedText; + this.layoutWidth = contentWidth; + + if (!this.guard.blockMath) { + scanBlockMath(normalizedText, this.guard); + } + if (this.guard.blockMath) { + // A pending block-math token can re-clip the paragraph in front of it, so a + // document that has one never reuses a prefix. The guard survives: it keeps + // the hazard sticky and stops the scan from re-reading the document. + this.dropPrefix(); + return this.renderFrom(normalizedText, contentWidth, width, 0, false); + } + + return this.renderFrom(normalizedText, contentWidth, width, this.stableSourceEnd, true); + } + + /** + * Render `normalizedText[base..]` and certify a new prefix out of it. + * + * `certifiable` is false when the document as a whole cannot be split (an + * uncertified parse, a reference definition, ...); the frame is then rendered + * whole and nothing is retained. + */ + private renderFrom( + normalizedText: string, + contentWidth: number, + width: number, + base: number, + certifiable: boolean, + ): string[] { + const tail = base === 0 ? normalizedText : normalizedText.slice(base); + const tokens = this.parse(tail); + + if (tokens.links && Object.keys(tokens.links).length > 0) { + // A definition is visible to every block, so nothing may be retained. + if (base > 0) { + this.resetStream(); + this.previousSource = normalizedText; + return this.renderFrom(normalizedText, contentWidth, width, 0, false); + } + certifiable = false; + } - // Parse markdown to HTML-like tokens - const tokens = markdownParser.lexer(normalizedText); trimPartialClosingFences(tokens); - // Convert tokens to styled terminal output - const renderedLines: string[] = []; + // Code rows of the certified prefix still decide how identical rows elsewhere + // are laid out, so the no-wrap set has to carry over. this.nowrapLines.clear(); + for (const row of this.stableNoWrap) { + this.nowrapLines.add(row); + } - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i]; - const nextToken = tokens[i + 1]; - const tokenLines = this.renderToken(token, contentWidth, nextToken?.type); - for (const tokenLine of tokenLines) { - renderedLines.push(tokenLine); + const rows = new Array(tokens.length); + const wrapped = new Array(tokens.length); + for (let index = 0; index < tokens.length; index++) { + const token = tokens[index]; + rows[index] = this.renderToken(token, contentWidth, tokens[index + 1]?.type); + markdownWork.renderedTokens++; + } + for (let index = 0; index < rows.length; index++) { + wrapped[index] = this.wrapRows(rows[index], contentWidth); + } + + if (base > 0 && this.collidesWithStableRows(tokens, rows)) { + // A code row that arrived after a prose row was certified changes how that + // prose row is laid out, so the retained rows are wrong. Re-rendering this + // frame re-lays everything out with the complete no-wrap set. + this.resetStream(); + this.previousSource = normalizedText; + return this.renderFrom(normalizedText, contentWidth, width, 0, true); + } + + if (base === 0) { + markdownWork.coldFrames++; + } else { + markdownWork.reusedFrames++; + } + + let cut: number | undefined = certifiable ? findCertifiedCut(tokens) : undefined; + let nextGuard: StreamGuard | undefined; + if (cut !== undefined) { + const chunkSource = tail.slice(0, rawLengthOf(tokens, cut)); + if (!rawsCoverSource(tokens.slice(0, cut), chunkSource)) { + cut = undefined; + } else { + const candidate = cloneStreamGuard(this.guard); + scanReferenceLabels(chunkSource, candidate); + applyInlineTagDepth(tokens.slice(0, cut), candidate); + if (!guardIsClean(candidate)) { + cut = undefined; + } else { + nextGuard = candidate; + } } } - // Wrap lines (NO padding, NO background yet). Code-block lines skip - // soft wrapping and are clipped instead, keeping code copyable. - const wrappedLines: string[] = []; - for (const line of renderedLines) { - if (isImageLine(line)) { - wrappedLines.push(line); - } else if (this.nowrapLines.has(line)) { - wrappedLines.push( - visibleWidth(line) > contentWidth ? truncateToWidth(line, contentWidth, "", false) : line, - ); + const dirtyStart = this.stablePaddedLines.length; + let firstTailToken = 0; + if (cut !== undefined && nextGuard !== undefined) { + for (let index = 0; index < cut; index++) { + for (const row of rows[index]) { + if (tokens[index].type === "code") { + this.stableNoWrap.add(row); + } else if (!this.nowrapLines.has(row)) { + // Only a row that was softly wrapped can be changed by a code row + // that arrives later; a clipped row stays clipped because the + // no-wrap set only grows. + this.stableWrappedProseRows.add(row); + } + } + for (const line of this.padRows(wrapped[index], width)) { + this.stablePaddedLines.push(line); + } + } + this.stableSourceEnd += rawLengthOf(tokens, cut); + this.guard = nextGuard; + firstTailToken = cut; + } + + const lines = this.stablePaddedLines.slice(); + for (let index = firstTailToken; index < tokens.length; index++) { + for (const line of this.padRows(wrapped[index], width)) { + lines.push(line); + } + } + // A whole-document render keeps no comparable prefix: the top and bottom + // padding rows are identical empty rows, but a changed style can still repaint + // them, so nothing above the content may be reused. + this.renderDirtyStart = base === 0 ? 0 : this.paddingY + dirtyStart; + return lines; + } + + /** Drop the certified prefix and everything derived from it. */ + private resetStream(): void { + this.layoutWidth = 0; + this.dropPrefix(); + this.guard = createStreamGuard(); + } + + private dropPrefix(): void { + this.stableSourceEnd = 0; + this.stablePaddedLines = []; + this.stableWrappedProseRows.clear(); + this.stableNoWrap.clear(); + } + + private parse(source: string): TokensList { + markdownWork.lexUnits += source.length; + return markdownParser.lexer(source); + } + + /** + * Whether a fenced-code row added by this frame is identical to a prose row + * that was already laid out. + * + * Code rows are never soft-wrapped, and the rule is applied by row text rather + * than by origin, so such a collision silently changes rows that are already + * on screen. Re-rendering the document is the only correct answer. + */ + private collidesWithStableRows(tokens: readonly Token[], rows: readonly string[][]): boolean { + if (this.stableWrappedProseRows.size === 0) { + return false; + } + for (let index = 0; index < tokens.length; index++) { + if (tokens[index].type !== "code") { + continue; + } + for (const row of rows[index]) { + if (this.stableWrappedProseRows.has(row)) { + return true; + } + } + } + return false; + } + + /** + * Wrap rows for terminal width. Code-block rows skip soft wrapping and are + * clipped instead, keeping code copyable. + */ + private wrapRows(rows: readonly string[], contentWidth: number): string[] { + const wrapped: string[] = []; + for (const row of rows) { + if (isImageLine(row)) { + wrapped.push(row); + } else if (this.nowrapLines.has(row)) { + wrapped.push(visibleWidth(row) > contentWidth ? truncateToWidth(row, contentWidth, "", false) : row); } else { - for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { - wrappedLines.push(wrappedLine); + for (const wrappedRow of wrapTextWithAnsi(row, contentWidth)) { + wrapped.push(wrappedRow); } } } + markdownWork.wrappedRows += wrapped.length; + return wrapped; + } - // Add margins and background to each wrapped line + /** Add margins, background, and trailing padding to each rendered row. */ + private padRows(rows: readonly string[], width: number): string[] { const leftMargin = " ".repeat(this.paddingX); const rightMargin = " ".repeat(this.paddingX); const bgFn = this.defaultTextStyle?.bgColor; - const contentLines: string[] = []; - - for (const line of wrappedLines) { - if (isImageLine(line)) { - contentLines.push(line); + const padded: string[] = []; + for (const row of rows) { + if (isImageLine(row)) { + padded.push(row); continue; } - - const lineWithMargins = leftMargin + line + rightMargin; - + const lineWithMargins = leftMargin + row + rightMargin; if (bgFn) { - contentLines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); + padded.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); } else { - // No background - just pad to width const visibleLen = visibleWidth(lineWithMargins); - const paddingNeeded = Math.max(0, width - visibleLen); - contentLines.push(lineWithMargins + " ".repeat(paddingNeeded)); + padded.push(lineWithMargins + " ".repeat(Math.max(0, width - visibleLen))); } } - - // Add top/bottom padding (empty lines) - const emptyLine = " ".repeat(width); - const emptyLines: string[] = []; - for (let i = 0; i < this.paddingY; i++) { - const line = bgFn ? applyBackgroundToLine(emptyLine, width, bgFn) : emptyLine; - emptyLines.push(line); - } - - // Combine top padding, content, and bottom padding - const result = emptyLines.concat(contentLines, emptyLines); - - // Update cache - this.cachedText = this.text; - this.cachedWidth = width; - this.cachedLines = result; - - return result.length > 0 ? result : [""]; + return padded; } /** diff --git a/packages/tui/test/markdown-block-math.test.ts b/packages/tui/test/markdown-block-math.test.ts new file mode 100644 index 000000000..e0c19b79b --- /dev/null +++ b/packages/tui/test/markdown-block-math.test.ts @@ -0,0 +1,163 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Markdown, type MarkdownTheme } from "../src/components/markdown.ts"; + +/** + * Block-math tokenizer coverage. + * + * `latexBlock` is a Marked block-level extension, so its `start()` runs at every + * block boundary with the *unconsumed remainder* as the haystack. The regex it + * uses can only match when that remainder still contains a block-math opener, so + * `start()` short-circuits on a substring check first. These tests pin the + * behaviour on both sides of that short-circuit: + * + * - documents without block math must render exactly as before, and + * - documents with block math must still tokenize, including the pending forms + * a streamed reply produces before its closing delimiter arrives. + * + * Any change that makes the prefilter skip a real opener shows up as a math + * block rendering as literal text. + */ + +const identity = (text: string) => text; + +const theme: MarkdownTheme = { + heading: identity, + link: identity, + linkUrl: identity, + code: identity, + codeBlock: identity, + codeBlockBorder: identity, + quote: identity, + quoteBorder: identity, + hr: identity, + listBullet: identity, + bold: identity, + italic: identity, + strikethrough: identity, + underline: identity, +}; + +/** Render with padding stripped, so assertions read as visible text. */ +function lines(markdown: string, width = 80): string[] { + const rendered = new Markdown(markdown, 0, 0, theme, undefined, {}).render(width); + return rendered.map((line) => line.replace(/\s+$/u, "")); +} + +describe("Markdown block math", () => { + describe("dollar-delimited blocks", () => { + it("renders a complete block", () => { + // LaTeX is rendered to Unicode, so the assertion pins the rendered form. + assert.deepStrictEqual(lines("$$\nE = mc^2\n$$"), ["E = mc²"]); + }); + + it("renders every prefix of a streamed block without losing content", () => { + // A streamed reply renders every prefix. Each frame must match a cold render + // of the same prefix, whichever side of the delimiter it lands on. + const full = "$$\n\\int_0^1 x^2 dx\n$$\n"; + const streamed = new Markdown("", 0, 0, theme, undefined, {}); + for (let end = 1; end <= full.length; end++) streamed.setText(full.slice(0, end)); + for (let end = 1; end <= full.length; end++) { + const prefix = full.slice(0, end); + streamed.setText(prefix); + assert.deepStrictEqual( + streamed.render(80).map((line) => line.replace(/\s+$/u, "")), + lines(prefix), + `prefix of ${end}/${full.length} chars diverged`, + ); + } + }); + + it("keeps a dollar in prose as prose", () => { + assert.deepStrictEqual(lines("cost is $5 today"), ["cost is $5 today"]); + }); + + it("keeps an escaped dollar as prose", () => { + assert.deepStrictEqual(lines("cost is \\$5 today"), ["cost is $5 today"]); + }); + }); + + describe("bracket-delimited blocks", () => { + it("renders a complete block", () => { + assert.deepStrictEqual(lines("\\[\nE = mc^2\n\\]"), ["E = mc²"]); + }); + + it("keeps the opener as text while the closer is still missing", () => { + // Pending bracket math only becomes a block once its closer arrives; until + // then the opener is an ordinary line. Pinned so the prefilter cannot make + // a pending opener disappear early. + assert.deepStrictEqual(lines("\\[\nE = mc^2"), ["\\[", "E = mc^2"]); + }); + + it("starts a block on a paragraph continuation line", () => { + // `start()`'s hint is load-bearing here: without it the paragraph rule + // swallows the opener, so a prefilter that misses this opener renders the + // math as literal text instead of converting it. + assert.deepStrictEqual(lines("intro text\n\\[\nE = mc^2\n\\]"), ["intro text", "", "E = mc²"]); + assert.deepStrictEqual(lines("intro text\n$$\nE = mc^2\n$$"), ["intro text", "", "E = mc²"]); + }); + }); + + describe("short-circuit does not change ordinary documents", () => { + it("still renders every non-math block shape", () => { + const cases: Array<[string, string, string[]]> = [ + ["heading", "# Title", ["# Title"]], + ["code fence", "```ts\nconst a = 1;\n```", ["```ts", " const a = 1;", "```"]], + ["list", "- one\n- two", ["- one", "- two"]], + ["quote", "> quoted", ["│ quoted"]], + ["table", "| a | b |\n| --- | --- |\n| 1 | 2 |", ["a b", "━ ━", "1 2"]], + ]; + for (const [name, source, expected] of cases) { + assert.deepStrictEqual(lines(source), expected, `${name} changed`); + } + }); + + it("does not treat a bracket inside a fence as a math opener", () => { + // The prefilter looks at the raw remainder, so `\[` inside a fence is a + // false positive: it must fall through to the regex, which rejects it. + const rendered = lines("```\nfoo \\[ bar\n```"); + assert.ok( + rendered.some((line) => line.includes("foo \\[ bar")), + "fenced content was rewritten", + ); + }); + + it("runs the block-math regex only when an opener is present", () => { + // Assert work, not wall-clock: the guard must not depend on machine speed. + // `start()` runs once per block boundary, so a math-free document with many + // blocks would execute the pattern many times without the prefilter. + const blockMathPattern = (markdown: string): boolean => { + const rendered = new Markdown(markdown, 0, 0, theme, undefined, {}).render(80); + return rendered.length > 0; + }; + + const paragraph = "a paragraph of ordinary prose without any math delimiters.\n\n"; + + const originalExec = RegExp.prototype.exec; + let blockMathExecs = 0; + let totalExecs = 0; + RegExp.prototype.exec = function patchedExec(this: RegExp, input: string) { + totalExecs++; + // The tokenizer's pattern is `/ (?:^|\n) {0,3}(?:\$\$|\\\[) /`, so its + // source is the only one carrying an escaped `$$` alternative. + if (this.source.includes("\\$\\$|")) blockMathExecs++; + return originalExec.call(this, input); + }; + try { + assert.ok(blockMathPattern(paragraph.repeat(400))); + assert.ok(totalExecs > 0, "no regex ran, so the counter would be vacuous"); + assert.strictEqual( + blockMathExecs, + 0, + "the block-math regex ran on a document with no opener, so the prefilter did not short-circuit", + ); + + blockMathExecs = 0; + assert.ok(blockMathPattern(`${paragraph.repeat(20)}\n$$\nE = mc^2\n$$\n`)); + assert.ok(blockMathExecs > 0, "the block-math regex did not run for a document that has an opener"); + } finally { + RegExp.prototype.exec = originalExec; + } + }); + }); +}); diff --git a/packages/tui/test/markdown-stream-finality.test.ts b/packages/tui/test/markdown-stream-finality.test.ts new file mode 100644 index 000000000..5b69258e7 --- /dev/null +++ b/packages/tui/test/markdown-stream-finality.test.ts @@ -0,0 +1,150 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Marked } from "marked"; +import { + applyInlineTagDepth, + createStreamGuard, + findCertifiedCut, + guardIsClean, + rawLengthOf, + rawsCoverSource, + scanBlockMath, + scanReferenceLabels, +} from "../src/components/markdown-stream.ts"; + +/** + * Unit coverage for the certification rules. + * + * These pin the two halves of the incremental renderer that cannot be observed + * from rendered output: which cut points are allowed, and which document-wide + * facts make a previously certified prefix unsafe. + */ + +const parser = new Marked(); + +function tokensOf(source: string) { + return parser.lexer(source); +} + +function cutOffsetFor(source: string): number | undefined { + const tokens = tokensOf(source); + const cut = findCertifiedCut(tokens); + return cut === undefined ? undefined : rawLengthOf(tokens, cut); +} + +describe("certified cut", () => { + it("stops in front of a paragraph that follows a blank line", () => { + assert.strictEqual(cutOffsetFor("first paragraph\n\nsecond paragraph\n"), "first paragraph\n\n".length); + }); + + it("stops in front of an ATX heading", () => { + assert.strictEqual(cutOffsetFor("first paragraph\n\n# heading\n"), "first paragraph\n\n".length); + }); + + it("rejects an unterminated final paragraph", () => { + // `1` can still become a list marker, and a list merges across blank lines + // into the list that precedes it. + assert.strictEqual(cutOffsetFor("first paragraph\n\n1"), undefined); + assert.strictEqual(cutOffsetFor("first paragraph\n\nsecond"), undefined); + }); + + it("rejects a list, quote, table, or fence as the witness", () => { + assert.strictEqual(cutOffsetFor("first paragraph\n\n- a\n- b\n"), undefined); + assert.strictEqual(cutOffsetFor("first paragraph\n\n> quoted\n"), undefined); + assert.strictEqual(cutOffsetFor("first paragraph\n\n| a |\n| - |\n| 1 |\n"), undefined); + assert.strictEqual(cutOffsetFor("first paragraph\n\n```\ncode\n```\n"), undefined); + }); + + it("rejects an indented witness", () => { + // A blank line does not end a list, so an indented line can still belong to + // the block in front of the cut. + assert.strictEqual(cutOffsetFor("first paragraph\n\n indented continuation\n"), undefined); + }); + + it("rejects a cut with no blank line in front of it", () => { + assert.strictEqual(cutOffsetFor("first paragraph\nsecond paragraph\n"), undefined); + }); + + it("prefers the rightmost candidate", () => { + const source = "one\n\ntwo\n\nthree\n"; + const tokens = tokensOf(source); + const cut = findCertifiedCut(tokens); + assert.strictEqual(cut, tokens.length - 1); + assert.strictEqual(rawLengthOf(tokens, cut!), source.length - "three\n".length); + }); +}); + +describe("document-wide guards", () => { + it("finds a block-math opener and keeps it sticky", () => { + const guard = createStreamGuard(); + scanBlockMath("intro\n\n$$", guard); + assert.strictEqual(guard.blockMath, true); + // Sticky: later scans cannot clear it, even for an innocent remainder. + scanBlockMath("intro\n\n$$\nplain prose\n\n", guard); + assert.strictEqual(guard.blockMath, true); + assert.strictEqual(guardIsClean(guard), false); + }); + + it("detects an opener split across two deltas", () => { + const guard = createStreamGuard(); + scanBlockMath("intro\n\n$", guard); + assert.strictEqual(guard.blockMath, false); + scanBlockMath("intro\n\n$$", guard); + assert.strictEqual(guard.blockMath, true); + }); + + it("ignores a dollar that is not at the start of a line", () => { + const guard = createStreamGuard(); + scanBlockMath("cost is $$5 today\n\nmore\n", guard); + assert.strictEqual(guard.blockMath, false); + assert.strictEqual(guardIsClean(guard), true); + }); + + it("tracks an unclosed reference label until it closes", () => { + const guard = createStreamGuard(); + scanReferenceLabels("intro\n\n[x", guard); + assert.strictEqual(guard.openReferenceLabel, true); + assert.strictEqual(guardIsClean(guard), false); + scanReferenceLabels("]: https://example.com\n\n", guard); + assert.strictEqual(guard.openReferenceLabel, false); + assert.strictEqual(guardIsClean(guard), true); + }); + + it("ignores an escaped bracket", () => { + const guard = createStreamGuard(); + scanReferenceLabels("intro\n\n\\[not a label\n\n", guard); + assert.strictEqual(guard.openReferenceLabel, false); + }); + + it("tracks an unbalanced inline anchor tag", () => { + const guard = createStreamGuard(); + applyInlineTagDepth(tokensOf('open\n\n'), guard); + assert.strictEqual(guard.openLinkTags, 1); + assert.strictEqual(guardIsClean(guard), false); + + applyInlineTagDepth(tokensOf("\n\n"), guard); + assert.strictEqual(guard.openLinkTags, 0); + assert.strictEqual(guardIsClean(guard), true); + }); + + it("ignores tag-like text inside fenced code", () => { + const guard = createStreamGuard(); + applyInlineTagDepth(tokensOf('```\n\n```\n\n'), guard); + assert.strictEqual(guard.openLinkTags, 0); + assert.strictEqual(guardIsClean(guard), true); + }); +}); + +describe("raw coverage", () => { + it("accepts tokens that reproduce their source", () => { + const source = "first\n\nsecond\n\n```\ncode\n```\n"; + assert.strictEqual(rawsCoverSource(tokensOf(source), source), true); + }); + + it("rejects a repeated reference definition, which Marked drops", () => { + // The second definition is not emitted as a token, so offsets cannot be + // derived from the token raws. The renderer must not advance a prefix. + const source = "[x]: https://example.com\n\n[x]: https://example.com\n\n"; + assert.strictEqual(rawsCoverSource(tokensOf(source), source), false); + }); +}); diff --git a/packages/tui/test/markdown-stream-prefix.test.ts b/packages/tui/test/markdown-stream-prefix.test.ts new file mode 100644 index 000000000..786b542c0 --- /dev/null +++ b/packages/tui/test/markdown-stream-prefix.test.ts @@ -0,0 +1,325 @@ +import assert from "node:assert"; +import { readFileSync } from "node:fs"; +import { describe, it } from "node:test"; +import { Markdown } from "../src/components/markdown.ts"; +import { resetCapabilitiesCache, setCapabilities } from "../src/terminal-image.ts"; +import { defaultMarkdownTheme } from "./test-themes.ts"; + +/** + * Streaming equivalence for the incremental renderer. + * + * A streamed reply calls `setText` with a growing prefix of the same document. + * The renderer keeps the rows of blocks it has certified as final, so the oracle + * has to be a *cold* instance: a freshly constructed `Markdown` has no certified + * prefix and always renders the whole document. + * + * Every assertion compares complete line arrays byte for byte. Nothing is + * trimmed or ANSI-stripped: a cache that changes one trailing space or one colour + * escape is a regression, not a rounding error. + */ + +function cold(text: string, width: number): string[] { + return new Markdown(text, 0, 0, defaultMarkdownTheme, undefined, {}).render(width); +} + +function checkPrefixes(document: string, width: number, label: string): void { + const streamed = new Markdown("", 0, 0, defaultMarkdownTheme, undefined, {}); + for (let end = 1; end <= document.length; end++) { + const prefix = document.slice(0, end); + streamed.setText(prefix); + assert.deepStrictEqual( + streamed.render(width), + cold(prefix, width), + `${label}: prefix ${end}/${document.length} diverged at width ${width}`, + ); + } +} + +function firstChangedLine(previous: readonly string[], next: readonly string[]): number { + const shared = Math.min(previous.length, next.length); + for (let index = 0; index < shared; index++) { + if (previous[index] !== next[index]) { + return index; + } + } + return shared; +} + +/** + * Fixtures that break a naive "freeze everything but the last block" cache. + * + * Each entry names the construct that would be corrupted: setext underlines that + * reach back across blocks, lists that merge across blank lines, tables whose + * column widths change with a later row, reference definitions that reinterpret + * earlier links, and code rows that change how an identical prose row wraps. + */ +const HAZARDS: ReadonlyArray = [ + ["setext-after-blank", "a\n\nb\n\nc\n=\n"], + ["setext-across-thematic-break", "a\n***\nb\n=\n"], + ["table-gains-a-row", "| a | b |\n| - | - |\n| 1 | 2 |\n\nprose\n\n| c | d |\n| - | - |\n| 3 | 4 |\n"], + [ + "table-column-widens", + "| a | b |\n| - | - |\n| 1 | 2 |\n\nprose\n\n| a | b |\n| - | - |\n| a much longer cell | 2 |\n", + ], + ["list-merges-across-blank", "- a\n\n- b\n\nprose\n\n- c\n\n- d\n"], + ["ordered-list-restarts", "1. one\ncontinuation\n\n1. two\n\nprose\n\n1. three\n"], + ["reference-definition-late", "use [x] now\n\nprose\n\n[x]: https://example.com\n\nuse [x] again\n"], + ["reference-definition-spans-blank", "[x\n\nword\n\nnext]: https://example.com\n\nafter\n"], + ["reference-definition-escaped-label", "prose\n\n[x\\]y]: https://example.com\n\ntail\n"], + ["html-block-unclosed", "prose\n\n
\nraw\n\nmore\n
\n\ntail\n"], + ["html-inline-anchor-open", 'intro open\n\nhttps://example.com\n\nmore\n'], + ["html-inline-anchor-closed", 'intro open\n\nhttps://example.com\n\nmore\n'], + ["indented-code-continues", "prose\n\n code line\n\n more code\n\ntail\n"], + [ + "code-row-collides-with-prose", + "a paragraph that duplicates a code row\n\nprose\n\n```\na paragraph that duplicates a code row\n```\n\ntail\n", + ], + [ + "code-row-collides-backwards", + "```\na paragraph that duplicates a code row\n```\n\nprose\n\na paragraph that duplicates a code row\n\ntail\n", + ], + ["code-then-identical-prose", "```\nhello world\n```\n\nhello world\n\ntail\n"], + ["blockquote-lazy-continuation", "> a\nlazy\n\nprose\n\n> b\nlazy2\n"], + ["block-math-closed", "$$\nx\n$$\n\nprose\n\nmore\n"], + ["block-math-pending-flip", "a\n\n$$\n` \\\n$\\\n.$\n\n\n\n\n\\~`a>\n\nnext\n"], + ["inline-dollar-amount", "cost is $5 today\n\nprose\n"], + ["bracket-math-pending", "\\[\nE = mc^2\n\nmore\n"], + ["fence-unclosed", "prose\n\n```ts\nconst a = 1;\n\nmore\n"], + ["fence-partial-closer", "prose\n\n```ts\nconst a = 1;\n``\n\nmore\n"], + ["fence-inside-list", "- item\n ```\n code\n ```\n\nprose\n"], + ["nested-list", "- a\n - b\n\n- c\n\nprose\n"], + ["heading-without-blank", "# h\npara\n\n## h2\npara2\n"], + ["thematic-break-vs-setext", "a\n---\n\nb\n\n---\n\nc\n"], + ["task-list", "- [ ] a\n\n- [x] b\n\nprose\n"], + ["underscore-emphasis", "a_b_c\n\nprose\n\nd_e_f\n"], + ["backslash-hard-break", "a\\\nb\n\nprose\n\nc\\\nd\n"], + ["trailing-spaces", "a \n\nb \n\nc \n"], + ["tab-normalisation", "a\n\tb\n\nc\n"], + ["crlf", "a\r\n\r\nb\r\n\r\nc\r\n"], + ["blank-line-runs", "a\n\n\n\n\nb\n\n\nc\n"], + ["emphasis-arrives-late", "plain text\n\nmore text\n\n**bold arrives later**\n"], + ["paragraph-continuation", "intro\n\nfirst line\nsecond line\n\nthird\n"], + ["image-line", "intro\n\n![alt](https://example.com/x.png)\n\ntail\n"], + ["empty-first-block", "\n\nfirst\n\nsecond\n"], +]; + +describe("Markdown streaming", () => { + describe("prefix equivalence", () => { + for (const [name, document] of HAZARDS) { + it(`matches a cold render for every prefix: ${name}`, () => { + for (const width of [20, 40, 80]) { + checkPrefixes(document, width, name); + } + }); + } + + it("matches a cold render when prefixed by an unrelated document", () => { + const preamble = "# Title\n\nA first paragraph.\n\n- a\n- b\n\n"; + for (const [, document] of HAZARDS) { + checkPrefixes(preamble + document, 60, "with preamble"); + } + }); + }); + + describe("delivery shape", () => { + const document = + "# Heading\n\nFirst paragraph with `code` and **bold**.\n\n- one\n- two\n\n```ts\nconst a = 1;\n```\n\nLast paragraph.\n"; + + for (const delta of [1, 2, 7, 120]) { + it(`matches a cold render when delivered in ${delta}-character deltas`, () => { + const streamed = new Markdown("", 0, 0, defaultMarkdownTheme, undefined, {}); + for (let end = delta; end < document.length; end += delta) { + streamed.setText(document.slice(0, end)); + streamed.render(60); + } + streamed.setText(document); + assert.deepStrictEqual(streamed.render(60), cold(document, 60)); + }); + } + + it("does not mutate arrays it already returned", () => { + const streamed = new Markdown("first paragraph\n\n", 0, 0, defaultMarkdownTheme, undefined, {}); + const first = streamed.render(60); + const snapshot = [...first]; + streamed.setText("first paragraph\n\nsecond paragraph that is long enough to wrap across the width\n\n"); + streamed.render(60); + assert.deepStrictEqual(first, snapshot); + }); + + it("returns the same array instance when nothing changed", () => { + const streamed = new Markdown("a paragraph\n\nanother\n", 0, 0, defaultMarkdownTheme, undefined, {}); + const first = streamed.render(60); + assert.strictEqual(streamed.render(60), first); + }); + + it("reports a dirty start at or before the first changed line", () => { + const streamed = new Markdown("first paragraph\n\n", 0, 0, defaultMarkdownTheme, undefined, {}); + let previous = streamed.render(60); + const prefixes = [ + "first paragraph\n\nsecond", + "first paragraph\n\nsecond paragraph\n\nthird", + "first paragraph\n\nsecond paragraph\n\nthird paragraph\n\nfourth", + ]; + for (const prefix of prefixes) { + streamed.setText(prefix); + const next = streamed.render(60); + const changed = firstChangedLine(previous, next); + assert.ok( + (streamed.renderDirtyStart ?? 0) <= changed, + `dirty start ${streamed.renderDirtyStart} is after the first change at ${changed}`, + ); + previous = next; + } + }); + + it("matches a cold render after a replacement and after a width change", () => { + const streamed = new Markdown("first\n\nsecond\n\n", 0, 0, defaultMarkdownTheme, undefined, {}); + streamed.render(60); + streamed.setText("replaced document\n\nwith two paragraphs\n\n"); + assert.deepStrictEqual(streamed.render(60), cold("replaced document\n\nwith two paragraphs\n\n", 60)); + streamed.setText("replaced document\n\nwith two paragraphs\n\nand a third\n\n"); + assert.deepStrictEqual( + streamed.render(40), + cold("replaced document\n\nwith two paragraphs\n\nand a third\n\n", 40), + ); + assert.deepStrictEqual( + streamed.render(60), + cold("replaced document\n\nwith two paragraphs\n\nand a third\n\n", 60), + ); + }); + + it("matches a cold render after invalidate", () => { + const streamed = new Markdown("first\n\nsecond\n\n", 0, 0, defaultMarkdownTheme, undefined, {}); + streamed.render(60); + streamed.setText("first\n\nsecond\n\nthird\n\n"); + streamed.invalidate(); + assert.deepStrictEqual(streamed.render(60), cold("first\n\nsecond\n\nthird\n\n", 60)); + }); + + it("reports no reusable prefix after invalidate", () => { + // Padding rows are identical empty rows, but a style change repaints them, so + // a whole-document render must not claim they are unchanged. + const streamed = new Markdown("first\n\nsecond\n\n", 0, 1, defaultMarkdownTheme, undefined, {}); + streamed.render(60); + streamed.setText("first\n\nsecond\n\nthird\n\n"); + streamed.invalidate(); + streamed.render(60); + assert.strictEqual(streamed.renderDirtyStart, 0); + }); + }); + + describe("document-wide parse state", () => { + // Marked keeps `inLink`/`inRawBlock` on the lexer for the whole document, so an + // unbalanced anchor in a certified prefix changes how a later URL is parsed. + // The difference is only visible with hyperlink support enabled: without it a + // bare URL renders as plain text either way. + it("does not freeze an unbalanced inline anchor", () => { + setCapabilities({ images: null, trueColor: true, hyperlinks: true }); + try { + for (const doc of [ + 'intro open\n\nhttps://example.com\n\n', + 'intro\n\nopen\n\nfirst tail paragraph\n\nhttps://example.com\n\nsecond tail\n\n', + ]) { + checkPrefixes(doc, 80, "unbalanced anchor"); + } + } finally { + resetCapabilitiesCache(); + } + }); + + it("does not freeze a reference label that is still open", () => { + // The definition tokenizer runs before `paragraph` and its label may span + // blank lines, so an open `[` can turn already rendered paragraphs into one + // definition once `]:` arrives. + for (const doc of [ + "[x\n\nprose\n\ntail\n\nmore\n\n]: https://example.com\n\nafter\n", + "first paragraph\n\n[note\n\ninterlude\n\n]: https://example.com\n\ntail\n", + ]) { + checkPrefixes(doc, 80, "open reference label"); + } + }); + }); + + describe("repository documents", () => { + // Sampling: a repository document can be tens of kilobytes, and a cold render + // per prefix would dominate the suite. Small fixtures above cover every prefix. + const files = [ + new URL("../../../README.md", import.meta.url), + new URL("../../../docs/development.md", import.meta.url), + new URL("../../../docs/tui-rendering-pipeline.md", import.meta.url), + ]; + + for (const file of files) { + it(`matches a cold render at sampled prefixes: ${file.pathname.split("/").pop()}`, () => { + const document = readFileSync(file, "utf8"); + const streamed = new Markdown("", 0, 0, defaultMarkdownTheme, undefined, {}); + const step = Math.max(1, Math.floor(document.length / 60)); + for (let end = 1; end <= document.length; end += step) { + const prefix = document.slice(0, end); + streamed.setText(prefix); + assert.deepStrictEqual(streamed.render(100), cold(prefix, 100), `prefix ${end}/${document.length}`); + } + }); + } + }); + + describe("adversarial combinations", () => { + // Deterministic block-level combination generator. A runtime seed would make a + // failure impossible to reproduce, so the sequence is fixed. + const shards = [ + "para text here\n\n", + "second line\n\n", + "- item\n", + "- item\n\n", + " - nested\n", + "1. one\n", + "> quote\n", + "lazy\n", + "```\ncode\n```\n\n", + "```ts\nconst a=1;\n", + "# h\n\n", + "## h2\n\n", + "| a | b |\n| - | - |\n| 1 | 2 |\n\n", + "| 3 | 4 |\n", + "---\n\n", + " indented\n", + "$$\nx\n$$\n\n", + "$$\n", + "\\[\nx\n\\]\n\n", + "[x]: https://example.com\n\n", + "use [x]\n", + "a\n", + "b\n", + "\n", + "\n\n", + "\\\n", + "word word word\n\n", + '\n\n', + "
\n", + "
\n\n", + "hello world\n\n", + "1.\n", + "1. x\n", + ]; + + function nextRandom(state: { seed: number }): number { + state.seed ^= state.seed << 13; + state.seed ^= state.seed >>> 17; + state.seed ^= state.seed << 5; + return Math.abs(state.seed); + } + + it("matches a cold render for 200 seeded documents", () => { + const state = { seed: 0x51ab37 }; + for (let index = 0; index < 200; index++) { + let document = ""; + const blockCount = 2 + (nextRandom(state) % 8); + for (let block = 0; block < blockCount; block++) { + document += shards[nextRandom(state) % shards.length]; + } + const width = 30 + (nextRandom(state) % 90); + checkPrefixes(document, width, `seeded document ${index} (seed 0x51ab37)`); + } + }); + }); +}); diff --git a/packages/tui/test/markdown-stream-work.test.ts b/packages/tui/test/markdown-stream-work.test.ts new file mode 100644 index 000000000..5e22b84bb --- /dev/null +++ b/packages/tui/test/markdown-stream-work.test.ts @@ -0,0 +1,182 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Markdown } from "../src/components/markdown.ts"; +import { markdownWork, resetMarkdownWork } from "../src/components/markdown-work.ts"; +import { defaultMarkdownTheme } from "./test-themes.ts"; + +/** + * Work-per-frame gates for streaming. + * + * Wall-clock assertions cannot separate "the renderer stopped re-reading finished + * blocks" from "the machine was busy", and on a loaded CI runner they can fail a + * real optimization or silently pass a regression. These tests assert the amount + * of source handed to the lexer, the number of tokens rendered, and the number of + * rows wrapped instead: for a streamed reply those are the stages that used to be + * quadratic in the message length. + */ + +const SHARD = [ + "A paragraph of prose with `code` and **bold** text that is long enough to wrap at 100 columns.", + "", + "- first item", + "- second item", + "", + "```ts", + "export const value = 1;", + "```", + "", + "| a | b |", + "| - | - |", + "| 1 | 2 |", + "", + "", +].join("\n"); + +function build(bytes: number): string { + let document = ""; + while (document.length < bytes) { + document += SHARD; + } + return document.slice(0, bytes); +} + +interface Measurement { + frames: number; + lexUnits: number; + renderedTokens: number; + wrappedRows: number; + coldFrames: number; + reusedFrames: number; + lines: number; +} + +function stream(document: string, width: number, delta: number): Measurement { + const markdown = new Markdown("", 0, 0, defaultMarkdownTheme, undefined, {}); + resetMarkdownWork(); + let frames = 0; + for (let end = delta; end < document.length; end += delta) { + frames++; + markdown.setText(document.slice(0, end)); + markdown.render(width); + } + markdown.setText(document); + const lines = markdown.render(width); + return { ...markdownWork, frames: frames + 1, lines: lines.length }; +} + +describe("streaming work", () => { + it("lexes the document roughly once instead of once per frame", () => { + const document = build(60 * 1024); + const measured = stream(document, 100, 120); + assert.ok( + measured.reusedFrames > measured.frames * 0.9, + `only ${measured.reusedFrames}/${measured.frames} frames reused a prefix`, + ); + // One lexical pass over the document, plus one partial block per frame. A + // renderer that re-lexed the whole document per frame would land near + // frames/2 passes: ~250 here. + assert.ok( + measured.lexUnits <= document.length * 6, + `lexed ${measured.lexUnits} units for a ${document.length}-unit document`, + ); + }); + + it("keeps per-frame work independent of the document length", () => { + const single = stream(build(30 * 1024), 100, 120); + const doubled = stream(build(60 * 1024), 100, 120); + assert.ok( + doubled.lexUnits <= single.lexUnits * 2.5, + `doubling the document multiplied lexing by ${(doubled.lexUnits / single.lexUnits).toFixed(2)}`, + ); + assert.ok( + doubled.renderedTokens <= single.renderedTokens * 2.5, + `doubling the document multiplied token renders by ${(doubled.renderedTokens / single.renderedTokens).toFixed(2)}`, + ); + assert.ok( + doubled.wrappedRows <= single.wrappedRows * 2.5, + `doubling the document multiplied wrapping by ${(doubled.wrappedRows / single.wrappedRows).toFixed(2)}`, + ); + }); + + it("renders each line a bounded number of times", () => { + const document = build(60 * 1024); + const measured = stream(document, 100, 120); + assert.ok( + measured.wrappedRows <= measured.lines * 6, + `wrapped ${measured.wrappedRows} rows for ${measured.lines} final lines`, + ); + }); +}); + +describe("streaming fallbacks", () => { + it("never reuses a prefix when the document contains block math", () => { + // A pending block-math token can re-clip the paragraph in front of it, so the + // certificate cannot be trusted anywhere in the document. + const measured = stream(`$$\nE = mc^2\n$$\n\n${build(20 * 1024)}`, 100, 120); + assert.strictEqual(measured.reusedFrames, 0); + }); + + it("never reuses a prefix when the document has a reference definition", () => { + const measured = stream(`use [label]\n\n[label]: https://example.com\n\n${build(20 * 1024)}`, 100, 120); + assert.strictEqual(measured.reusedFrames, 0); + }); + + it("keeps rendering a document with no certifiable cut", () => { + // A single paragraph has no blank line, so there is nothing to certify and the + // frame cost stays what it was. This is a documented limit, not a silent + // truncation: the output still equals a cold render. + const document = "word ".repeat(4000).trimEnd(); + const markdown = new Markdown("", 0, 0, defaultMarkdownTheme, undefined, {}); + resetMarkdownWork(); + for (let end = 120; end < document.length; end += 120) { + markdown.setText(document.slice(0, end)); + markdown.render(100); + } + markdown.setText(document); + assert.deepStrictEqual( + markdown.render(100), + new Markdown(document, 0, 0, defaultMarkdownTheme, undefined, {}).render(100), + ); + assert.strictEqual(markdownWork.reusedFrames, 0); + }); + + it("re-renders the document when a code row collides with a certified prose row", () => { + // Code rows are never soft-wrapped and the rule is applied by row text, so a + // later fenced block containing an already rendered prose row changes how that + // row is laid out. The prefix must be dropped instead of kept. + // + // Reaching this at all needs a theme that does not restyle code, otherwise the + // code row carries different ANSI escapes and can never collide. + const identity = (text: string) => text; + const plainTheme = { + heading: identity, + link: identity, + linkUrl: identity, + code: identity, + codeBlock: identity, + codeBlockBorder: identity, + quote: identity, + quoteBorder: identity, + hr: identity, + listBullet: identity, + bold: identity, + italic: identity, + strikethrough: identity, + underline: identity, + }; + const long = "a long line that will definitely exceed the terminal width and needs wrapping"; + const document = ` ${long}\n\nsecond paragraph\n\n\`\`\`\n${long}\n\`\`\`\n\n`; + const markdown = new Markdown("", 0, 0, plainTheme, undefined, {}); + resetMarkdownWork(); + for (let end = 1; end <= document.length; end++) { + const prefix = document.slice(0, end); + markdown.setText(prefix); + assert.deepStrictEqual( + markdown.render(40), + new Markdown(prefix, 0, 0, plainTheme, undefined, {}).render(40), + `prefix ${end}`, + ); + } + assert.ok(markdownWork.coldFrames > 0, "the colliding document never re-rendered"); + }); +});