From acb96fa2d48a0d3e404e1442fe3dcb397fb7aaab Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Wed, 11 Nov 2026 16:37:08 +0530 Subject: [PATCH 1/2] fix(shared): preserve Unicode during command output overwrites --- packages/shared/package.json | 1 + packages/shared/src/terminalOutput.test.ts | 71 +++++++-- packages/shared/src/terminalOutput.ts | 158 +++++++++++++++------ pnpm-lock.yaml | 3 + 4 files changed, 175 insertions(+), 58 deletions(-) diff --git a/packages/shared/package.json b/packages/shared/package.json index f5f769236f906..72fb59c56f7ae 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -558,6 +558,7 @@ "@t3tools/contracts": "workspace:*", "effect": "catalog:", "ffi-rs": "1.3.2", + "get-east-asian-width": "^1.6.0", "mdast-util-directive": "catalog:", "mdast-util-to-hast": "^13.2.1", "micromark-extension-directive": "catalog:", diff --git a/packages/shared/src/terminalOutput.test.ts b/packages/shared/src/terminalOutput.test.ts index 4401dd1ac1159..d7b8b1922eace 100644 --- a/packages/shared/src/terminalOutput.test.ts +++ b/packages/shared/src/terminalOutput.test.ts @@ -24,6 +24,31 @@ describe("appendTerminalOutput", () => { expect(feed(["[1/3]", "\r[2/3]", "\r[3/3] done\nnext"]).text).toBe("[3/3] done\nnext"); }); + it.each([ + ["😀\u001b[1Gx", "x "], + ["界a\u001b[3GZ", "界Z"], + ["e\u0301a\u001b[2GZ", "e\u0301Z"], + ["𐐀a\u001b[2GZ", "𐐀Z"], + ["界a\bZ", "界Z"], + ["界a\u001b[2GZ", " Za"], + ["a界b\rXY", "XY b"], + ["e\u0301a\rZ", "Za"], + ["界a\u001b[2G\u001b[K", " "], + ["界a\u001b[1G\u001b[1K", " a"], + ["\u001b[31m界\u001b[32ma\u001b[3GZ", "界Z"], + ["e\u001b[31m\u0301a\u001b[1GZ", "Za"], + ])("overwrites Unicode at terminal columns: %j", (input, expected) => { + expect(terminalOutputPlainText(feed([input]).text)).toBe(expected); + for (let cut = 1; cut < input.length; cut += 1) { + const source = feed([input.slice(0, cut)]); + const chunk = input.slice(cut); + expect(terminalOutputPlainText(appendTerminalOutput(source, chunk).text)).toBe(expected); + expect(terminalOutputPlainText(feed([terminalOutputResumeText(source), chunk]).text)).toBe( + expected, + ); + } + }); + it("redraws several lines in place when the cursor moves up", () => { const frames = [ "layer a: 10%\nlayer b: 10%\n", @@ -60,20 +85,38 @@ describe("appendTerminalOutput", () => { expect(feed(["abcdefgh\nijk\nlmnop"], 8).text).toBe("jk\nlmnop"); }); - it("stays bounded across many appends of huge output, including one endless line", () => { - let state = EMPTY_TERMINAL_OUTPUT; - for (let index = 0; index < 5_000; index += 1) { - state = appendTerminalOutput(state, "y\n".repeat(200), 4_096); - } - expect(state.text.length).toBeLessThanOrEqual(4_096); - expect(state.truncated).toBe(true); - for (let index = 0; index < 1_000; index += 1) { - state = appendTerminalOutput(state, ".".repeat(100), 4_096); - } - expect(state.text.length).toBeLessThanOrEqual(4_096); - expect(state.text.endsWith(".")).toBe(true); + it("keeps Unicode tails bounded without shifting saved cursor columns", () => { + const source = feed(["界界e\u0301\u001b7ab"], 4); + expect(source.text).toBe("e\u0301ab"); + expect(source.screen.col).toBe(3); + expect(source.screen.saved?.col).toBe(1); + expect(appendTerminalOutput(source, "\u001b8Z", 4).text).toBe("e\u0301Zb"); + expect(feed([terminalOutputResumeText(source), "\u001b8Z"], 4).text).toBe("e\u0301Zb"); + expect(feed(["𐐀a"], 2).text).toBe("a"); + expect(feed(["e\u0301ab"], 3).text).toBe("ab"); + const wide = feed(["界".repeat(40_000), "\u001b7"]); + expect(feed([terminalOutputResumeText(wide)]).screen.col).toBe(wide.screen.col); }); + it.each([".", "界", "😀", "\u001b[31m界\u001b[0m"])( + "stays bounded across many appends of huge output, including one endless line of %s", + (character) => { + let state = EMPTY_TERMINAL_OUTPUT; + for (let index = 0; index < 5_000; index += 1) { + state = appendTerminalOutput(state, "y\n".repeat(200), 4_096); + } + expect(state.text.length).toBeLessThanOrEqual(4_096); + expect(state.truncated).toBe(true); + for (let index = 0; index < 1_000; index += 1) { + state = appendTerminalOutput(state, character.repeat(100), 4_096); + } + expect(terminalOutputPlainText(state.text).length).toBeLessThanOrEqual(4_096); + expect(terminalOutputPlainText(state.text).endsWith(terminalOutputPlainText(character))).toBe( + true, + ); + }, + ); + it("hides a long string sequence's payload however it is split into chunks", () => { const input = `a\u001b]0;${"t".repeat(100)}\u0007b`; expect(feed([input]).text).toBe("ab"); @@ -228,12 +271,12 @@ describe("terminalOutputResumeText", () => { describe("terminalOutputResumeText with a saved cursor or trimmed output", () => { it("restores a cursor saved before the resync", () => { - const source = feed(["abc\u001b7def"]); + const source = feed(["界e\u0301\u001b7def"]); const viewer = appendTerminalOutput( appendTerminalOutput(EMPTY_TERMINAL_OUTPUT, terminalOutputResumeText(source)), "\u001b8X", ); - expect(viewer.text).toBe("abcXef"); + expect(viewer.text).toBe("界e\u0301Xef"); }); it("stays on the right line when early output was trimmed away", () => { diff --git a/packages/shared/src/terminalOutput.ts b/packages/shared/src/terminalOutput.ts index c59ce6bfb36c8..88a90399d9f2a 100644 --- a/packages/shared/src/terminalOutput.ts +++ b/packages/shared/src/terminalOutput.ts @@ -1,10 +1,12 @@ +import { eastAsianWidth } from "get-east-asian-width"; + /** Server/client folds must agree: only the last LIVE_LINES are cursor-addressable, with no wrapping. * SGR resets at line boundaries for safe tail cuts; unfinished escapes carry through pending. */ export interface TerminalOutputState { /** Fixed lines followed by the live window; contains only SGR escapes. */ readonly text: string; readonly truncated: boolean; - /** An unfinished escape sequence awaiting the next chunk. */ + /** An unfinished escape sequence or high surrogate awaiting the next chunk. */ readonly pending: string; readonly screen: TerminalScreen; } @@ -60,22 +62,71 @@ function lineLength(line: Line): number { return length; } +const lineWidths = new WeakMap(); +const ZERO_WIDTH = /[\p{Mark}\p{Default_Ignorable_Code_Point}]/u; +const ASCII = /^[\x00-\x7f]*$/u; + +function characterWidth(character: string): number { + return ZERO_WIDTH.test(character) ? 0 : eastAsianWidth(character.codePointAt(0)!); +} + +function textWidth(text: string): number { + if (ASCII.test(text)) return text.length; + let width = 0; + for (const character of text) width += characterWidth(character); + return width; +} + +function lineWidth(line: Line): number { + const cached = lineWidths.get(line); + if (cached !== undefined) return cached; + let width = 0; + for (const run of line) width += textWidth(run.text); + lineWidths.set(line, width); + return width; +} + // Strip oldest styles to bound work on color-per-character output. const MAX_LINE_RUNS = 2_048; function sliceLine(line: Line, start: number, end: number): Array { const out: Array = []; - let offset = 0; + let col = 0; + let base = 0; + let baseEnd = 0; for (const run of line) { - const runEnd = offset + run.text.length; - if (runEnd > start && offset < end) { - out.push({ - pen: run.pen, - text: run.text.slice(Math.max(0, start - offset), Math.min(run.text.length, end - offset)), - }); + if (ASCII.test(run.text)) { + const runEnd = col + run.text.length; + if (runEnd > start && col < end) { + pushRun(out, { + pen: run.pen, + text: run.text.slice(Math.max(0, start - col), Math.min(run.text.length, end - col)), + }); + } + base = runEnd - 1; + baseEnd = runEnd; + col = runEnd; + continue; + } + let text = ""; + for (const character of run.text) { + const width = characterWidth(character); + if (width > 0) { + base = col; + baseEnd = col + width; + } + if (width === 0) { + if (base >= start && base < end && baseEnd <= end) text += character; + } else if (col < end && col + width > start) { + // Overwriting either half of a wide glyph clears that glyph's other cell. + text += + col >= start && col + width <= end + ? character + : " ".repeat(Math.min(end, col + width) - Math.max(start, col)); + } + col += width; } - offset = runEnd; - if (offset >= end) break; + pushRun(out, { pen: run.pen, text }); } return out; } @@ -91,40 +142,58 @@ function pushRun(runs: Array, run: Run): void { } /** Mutates only lines owned by this feed. */ -function trimLineStart(runs: Array, count: number): void { +function trimLineStart(runs: Array, count: number): number { + const width = lineWidth(runs); + const length = lineLength(runs); + let removedWidth = 0; + let removedUnits = 0; let dropped = 0; let remaining = count; - while (remaining > 0 && dropped < runs.length) { + while (dropped < runs.length) { const run = runs[dropped]!; - if (run.text.length <= remaining) { - remaining -= run.text.length; - dropped += 1; - } else { - runs[dropped] = { pen: run.pen, text: run.text.slice(remaining) }; - remaining = 0; + let cut = Math.min(remaining, run.text.length); + // A retained tail must start at a complete character, with no detached marks. + while (cut < run.text.length) { + const code = run.text.charCodeAt(cut); + const character = String.fromCodePoint(run.text.codePointAt(cut)!); + if (!(code >= 0xdc00 && code <= 0xdfff) && characterWidth(character) !== 0) break; + cut += character.length; + } + removedWidth += textWidth(run.text.slice(0, cut)); + removedUnits += cut; + remaining = Math.max(0, remaining - cut); + if (cut < run.text.length) { + runs[dropped] = { pen: run.pen, text: run.text.slice(cut) }; + break; } + dropped += 1; } if (dropped > 0) runs.splice(0, dropped); + lineLengths.set(runs, length - removedUnits); + lineWidths.set(runs, width - removedWidth); + return removedWidth; } function writeLine(line: Line, col: number, text: string, pen: string): Line { - const length = lineLength(line); + const length = lineWidth(line); if (col === length) { const runs = line.slice(); pushRun(runs, { pen, text }); + lineWidths.set(runs, length + textWidth(text)); + lineLengths.set(runs, lineLength(line) + text.length); return runs; } const runs: Array = []; for (const run of sliceLine(line, 0, Math.min(col, length))) pushRun(runs, run); if (col > length) pushRun(runs, { pen: "", text: " ".repeat(col - length) }); pushRun(runs, { pen, text }); - for (const run of sliceLine(line, col + text.length, length)) pushRun(runs, run); + for (const run of sliceLine(line, col + textWidth(text), length)) pushRun(runs, run); return runs; } /** Erase in line: 0 cursor to end, 1 start to cursor, 2 whole line. */ function eraseLine(line: Line, col: number, mode: number): Line { - const length = lineLength(line); + const length = lineWidth(line); if (mode === 0) return sliceLine(line, 0, col); if (mode === 1) { const runs: Array = []; @@ -358,15 +427,18 @@ class Terminal { write(text: string): void { let line = this.lines[this.row]!; - const length = lineLength(line); + const length = lineWidth(line); + const width = textWidth(text); if (this.col === length && this.owned.has(line)) { + const units = lineLength(line); pushRun(line as Array, { pen: this.pen, text }); - lineLengths.set(line, length + text.length); + lineWidths.set(line, length + width); + lineLengths.set(line, units + text.length); } else { line = writeLine(line, this.col, text, this.pen); this.owned.add(line); } - this.col += text.length; + this.col += width; if (line.length > MAX_LINE_RUNS) { // Flatten old runs so later trimming stays cheap. const keep = line.length - MAX_LINE_RUNS / 8; @@ -374,32 +446,23 @@ class Terminal { .slice(0, keep) .map((run) => run.text) .join(""); + const columns = lineWidth(line); + const units = lineLength(line); line = [{ pen: "", text: plain }, ...line.slice(keep)]; + lineWidths.set(line, columns); + lineLengths.set(line, units); this.owned.add(line); } - // Trim each write to maxChars columns so results do not depend on chunk boundaries. - let current = lineLength(line); + // The output budget is UTF-16 units; cursor positions are terminal columns. + const current = lineLength(line); if (current > this.maxChars) { - let removed = current - this.maxChars; - if (!this.owned.has(line)) { - line = line.slice(); - this.owned.add(line); - } - trimLineStart(line as Array, removed); - // Never start on a lone low surrogate. - const first = line[0]?.text.charCodeAt(0) ?? 0; - if (first >= 0xdc00 && first <= 0xdfff) { - trimLineStart(line as Array, 1); - removed += 1; - } - current -= removed; + const removed = trimLineStart(line as Array, current - this.maxChars); this.dropped = true; this.col = Math.max(0, this.col - removed); if (this.saved !== null && this.saved.row === this.row) { this.saved = { ...this.saved, col: Math.max(0, this.saved.col - removed) }; } } - lineLengths.set(line, current); this.lines[this.row] = line; } @@ -432,7 +495,7 @@ class Terminal { moveTo(row: number, col: number): void { this.row = Math.max(0, Math.min(this.lines.length - 1, row)); // A huge column (ESC[999999999C) would otherwise pad a line past any string limit. - this.col = Math.max(0, Math.min(TERMINAL_OUTPUT_TAIL_CHARS, col)); + this.col = Math.max(0, Math.min(2 * TERMINAL_OUTPUT_TAIL_CHARS, col)); } csi(params: string, final: string): void { @@ -547,6 +610,11 @@ class Terminal { printable = index + 1; } } + const last = input.charCodeAt(input.length - 1); + if (last >= 0xd800 && last <= 0xdbff) { + flush(input.length - 1); + return input.slice(-1); + } flush(input.length); return ""; } @@ -639,11 +707,13 @@ export function appendTerminalOutput( const keep = Math.max(0, Math.min(length, budget)); if (keep < length) { truncated = true; - terminal.lines[row] = sliceLine(line, length - keep, length); + const runs = line.slice(); + const removed = trimLineStart(runs, length - keep); + terminal.lines[row] = runs; if (terminal.saved !== null && terminal.saved.row === row) { terminal.saved = { ...terminal.saved, - col: Math.max(0, terminal.saved.col - (length - keep)), + col: Math.max(0, terminal.saved.col - removed), }; } } @@ -693,7 +763,7 @@ export function terminalOutputResumeText(state: TerminalOutputState): string { if ( screen.saved !== null || screen.row < lastRow || - screen.col !== lineLength(screen.lines[screen.row]!) + screen.col !== lineWidth(screen.lines[screen.row]!) ) { suffix += goTo(screen.row, screen.col); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6451f3f97cb29..fa988980e8602 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2041,6 +2041,9 @@ importers: ffi-rs: specifier: 1.3.2 version: 1.3.2 + get-east-asian-width: + specifier: ^1.6.0 + version: 1.6.0 mdast-util-directive: specifier: 'catalog:' version: 3.1.0(supports-color@8.1.1) From 734cbfadb8c087804b3836dd6382e261a045ca6a Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Wed, 11 Nov 2026 17:55:52 +0530 Subject: [PATCH 2/2] fix(shared): retain available tail budget after Unicode cuts --- packages/shared/src/terminalOutput.test.ts | 3 +++ packages/shared/src/terminalOutput.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/terminalOutput.test.ts b/packages/shared/src/terminalOutput.test.ts index d7b8b1922eace..fe2f4b5e48c81 100644 --- a/packages/shared/src/terminalOutput.test.ts +++ b/packages/shared/src/terminalOutput.test.ts @@ -204,6 +204,9 @@ describe("appendTerminalOutput edge cases from review", () => { ); expect(windowChars).toBeLessThanOrEqual(4_096); expect(state.text.length).toBeLessThanOrEqual(4_096); + for (const prefix of ["😀", "e\u0301"]) { + expect(feed([`a\n${prefix}bx\nZ\u001b[2A`], 4).text).toBe("bx\nZ"); + } }); }); diff --git a/packages/shared/src/terminalOutput.ts b/packages/shared/src/terminalOutput.ts index 88a90399d9f2a..c33d837ff2713 100644 --- a/packages/shared/src/terminalOutput.ts +++ b/packages/shared/src/terminalOutput.ts @@ -717,7 +717,7 @@ export function appendTerminalOutput( }; } } - budget -= keep; + budget -= lineLength(terminal.lines[row]!); } } if (terminal.dropped) truncated = true;