From fefdc4f1b15d4b8f5e952f35c6ef9cbfbf5d7b3b Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:04:41 -0800 Subject: [PATCH] ci: split web tests over two runners so every run finishes sooner Test Web gated every CI run (p50 211s on main). Browser tests are bound by the Vite server transforming modules for each file, not by Chromium, so a second runner with its own Vite server cuts the job by about 35-40%. AttachmentFilePreview.browser.test.tsx then became the longest file in its shard, so its Office tests move to their own file and run in parallel. Server shards restore Vitest's content-keyed module cache, saved from main, instead of re-transforming every import in each per-file process. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 30 +- .github/workflows/release.yml | 9 +- .../AttachmentFilePreview.browser.test.tsx | 1043 +--------------- ...achmentFilePreview.office.browser.test.tsx | 1086 +++++++++++++++++ 4 files changed, 1121 insertions(+), 1047 deletions(-) create mode 100644 apps/web/src/components/files/AttachmentFilePreview.office.browser.test.tsx diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5770c1f4516b..ba1e1e612e6d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -355,16 +355,22 @@ jobs: # The web suite is the largest outside the server. In the shared Test job it # started last and ran alone for about 40s after everything else finished. - # release.yml mirrors this test, test_web, and test_server split. + # Browser tests are bound by the Vite server transforming modules for each + # file, not by Chromium, so a second runner (and Vite server) halves the job + # that gated every run. release.yml mirrors this test, test_web, and test_server split. test_web: permissions: contents: read id-token: write - name: Test Web + name: Test Web ${{ matrix.shard }} needs: changes if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@v6 @@ -406,7 +412,7 @@ jobs: fi - name: Test - run: vp run --filter @t3tools/web test + run: vp run --filter @t3tools/web test --shard ${{ matrix.shard }}/${{ strategy.job-total }} # apps/server sets `fileParallelism: false`, so its test files run strictly # one at a time. Sharding spreads them over separate runners instead of @@ -481,6 +487,17 @@ jobs: node apps/server/node_modules/playwright-core/cli.js install-deps chromium fi + # Each server test file runs in its own Vitest process, and a fresh install starts + # Vitest's content-keyed module cache empty, so every file re-transformed its imports. + # Only pushes to main save it, so pull requests never evict each other's entries. + - name: Restore Vitest module cache + if: steps.changes.outputs.test_server != 'false' + uses: actions/cache/restore@v6 + with: + path: node_modules/.vitest-cache + key: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + - name: Test if: steps.changes.outputs.test_server != 'false' env: @@ -491,6 +508,13 @@ jobs: --uncached apps/server/integration/transferBudgetV2.integration.test.ts --verify-hits 1 apps/server + - name: Save Vitest module cache + if: github.event_name == 'push' && steps.changes.outputs.test_server != 'false' + uses: actions/cache/save@v6 + with: + path: node_modules/.vitest-cache + key: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + # integration/transferBudgetV2.integration.test.ts writes the report; exactly one shard # produces these files. Gating the upload on their presence keeps a # single `thread-transfer-results` artifact per run, which is the name diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84acc7c74f865..8827876b3829f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -300,11 +300,15 @@ jobs: # Web browser tests time out when they share a runner with the other package # suites: Chromium gets too little CPU and diff workers miss their 10s start budget. test_web: - name: Release web tests + name: Release web tests ${{ matrix.shard }} needs: [preflight] if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@v6 @@ -344,10 +348,11 @@ jobs: fi - name: Test - run: vp run --filter @t3tools/web test + run: vp run --filter @t3tools/web test --shard ${{ matrix.shard }}/${{ strategy.job-total }} # Browser tests run source modules; this boots the production chunk graph. - name: Boot the built web client + if: matrix.shard == 1 run: | vp run --filter @t3tools/web build node apps/web/scripts/verify-built-boot.ts diff --git a/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx b/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx index 4ecc965f6fe32..285032b5b6a0a 100644 --- a/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx +++ b/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx @@ -78,6 +78,7 @@ import { decodeImagePreview } from "./decodeImagePreview"; import type { BinaryWorkerResult } from "./binaryPreviewProtocol"; import type { TextPreviewResult } from "./largeTextProtocol"; import type { ReactElement } from "react"; +import type { OfficeWorkerResult } from "./officePreviewProtocol"; // Real worker imports and WASM initialization can exceed waitFor's default second // when the full browser project runs concurrently on CI. @@ -481,8 +482,6 @@ describe("bounded binary previews", () => { } }); }); -import { OFFICE_PREVIEW_MAX_COLUMNS, type OfficeWorkerResult } from "./officePreviewProtocol"; -import { WORD_LIMITS } from "./officeArchive"; describe("large text preview recovery", () => { it("retains the workspace viewer across URL renewals and reloads revisions and manual refreshes", async () => { @@ -921,1046 +920,6 @@ describe("large text preview recovery", () => { await vi.waitFor(() => expect(load).toHaveBeenCalledWith(0)); }); }); -it.each(["", ""])( - "opens an empty Word document with %s", - async (body) => { - const { default: JSZip } = await import("jszip"); - const { prepareOffice } = await import("./officeArchive"); - const archive = new JSZip().file( - "word/document.xml", - `${body}`, - ); - const prepared = await prepareOffice( - await archive.generateAsync({ type: "arraybuffer" }), - "docx", - ); - const result = await JSZip.loadAsync(prepared.bytes); - const xml = new DOMParser().parseFromString( - await result.file("word/document.xml")!.async("string"), - "application/xml", - ); - expect(xml.querySelector("parsererror")).toBeNull(); - expect( - xml.getElementsByTagNameNS( - "http://schemas.openxmlformats.org/wordprocessingml/2006/main", - "body", - ), - ).toHaveLength(1); - }, -); - -it.each([false, true])( - "rejects excessive archive entries before JSZip (directories: %s)", - async (directories) => { - const { default: JSZip } = await import("jszip"); - const { prepareOffice } = await import("./officeArchive"); - const archive = new JSZip(); - for (let i = 0; i < 4096; i++) - archive.file(`entry-${i}${directories ? "/" : ".txt"}`, "", { dir: directories }); - const bytes = await archive.generateAsync({ type: "arraybuffer" }); - const load = vi.spyOn(JSZip, "loadAsync"); - try { - await expect(prepareOffice(bytes, "docx")).rejects.toThrow("2,048 entries"); - expect(load).not.toHaveBeenCalled(); - // The actual central-directory records must also be bounded if EOCD lies. - const view = new DataView(bytes); - view.setUint16(bytes.byteLength - 22 + 8, 1, true); - view.setUint16(bytes.byteLength - 22 + 10, 1, true); - await expect(prepareOffice(bytes, "docx")).rejects.toThrow("2,048 entries"); - expect(load).not.toHaveBeenCalled(); - } finally { - load.mockRestore(); - } - }, -); - -it("rejects oversized embedded images before decoding pixels and retains ordinary images", async () => { - const { officeImageBitmap } = await import("./office/image-bitmap"); - const canvas = new OffscreenCanvas(2, 3); - canvas.getContext("2d"); - const blob = await canvas.convertToBlob({ type: "image/png" }); - const bytes = await blob.arrayBuffer(); - new DataView(bytes).setUint32(16, 12000); - new DataView(bytes).setUint32(20, 12000); - const decode = vi.spyOn(globalThis, "createImageBitmap"); - try { - await expect(officeImageBitmap(new Blob([bytes]))).rejects.toThrow("pixel limit"); - expect(decode).not.toHaveBeenCalled(); - const bitmap = await officeImageBitmap(blob); - expect([bitmap.width, bitmap.height]).toEqual([2, 3]); - bitmap.close(); - } finally { - decode.mockRestore(); - } -}); - -it.each(["utf-16le", "utf-16be", "windows-1252"])( - "preserves %s Office XML and safe hyperlink targets", - async (encoding) => { - const { default: JSZip } = await import("jszip"); - const { prepareOffice } = await import("./officeArchive"); - const archive = new JSZip(); - const encode = (text: string) => { - const xml = `${text}`; - if (encoding === "windows-1252") return Uint8Array.from(xml, (char) => char.charCodeAt(0)); - const bytes = new Uint8Array(2 + xml.length * 2); - const view = new DataView(bytes.buffer); - const little = encoding === "utf-16le"; - view.setUint16(0, 0xfeff, little); - for (let i = 0; i < xml.length; i++) view.setUint16(2 + i * 2, xml.charCodeAt(i), little); - return bytes; - }; - archive.file( - "word/document.xml", - encode( - 'café', - ), - ); - archive.file( - "word/_rels/document.xml.rels", - encode( - `${["https://example.com/page", "http://example.com", "mailto:maria@example.com", "javascript:alert(1)", "file:///tmp/private"].map((target, i) => ``).join("")}`, - ), - ); - const prepared = await prepareOffice( - await archive.generateAsync({ type: "arraybuffer" }), - "docx", - ); - const result = await JSZip.loadAsync(prepared.bytes); - expect(await result.file("word/document.xml")!.async("string")).toContain("café"); - const links = await result.file("word/_rels/document.xml.rels")!.async("string"); - expect(links).toContain('encoding="UTF-8"'); - expect(links).toContain("https://example.com/page"); - expect(links).toContain("http://example.com"); - expect(links).toContain("mailto:maria@example.com"); - expect(links).not.toContain("javascript:"); - expect(links).not.toContain("file:"); - }, -); - -it.each([ - ["", "worksheets/./sheet1.xml", false, false], - ["x:", "../xl/worksheets/sheet1.xml", false, false], - ["", "/xl/worksheets/sheet1.xml", true, false], - ["", "worksheets/sheet1.xml", false, true], - ["", "worksheets/sheet1.xml", true, true], -])( - "preserves %sworksheet metadata with relationship target %s", - async (prefix, target, empty, localPrefix) => { - const { default: JSZip } = await import("jszip"); - const archive = new JSZip(); - const ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; - archive.file( - "xl/workbook.xml", - '', - ); - archive.file( - "xl/_rels/workbook.xml.rels", - ``, - ); - archive.file( - "xl/worksheets/sheet1.xml", - `<${prefix}worksheet xmlns${prefix ? ":x" : ""}="${ns}"><${prefix}dimension ref="XFD1048576"/><${prefix}sheetViews><${prefix}sheetView showGridLines="0"/><${prefix}sheetFormatPr defaultColWidth="30"/><${prefix}cols><${prefix}col min="${OFFICE_PREVIEW_MAX_COLUMNS - 1}" max="${OFFICE_PREVIEW_MAX_COLUMNS + 10}" width="20"/><${prefix}col min="${OFFICE_PREVIEW_MAX_COLUMNS + 1}" max="${OFFICE_PREVIEW_MAX_COLUMNS + 30}" hidden="1"/><${prefix}sheetData><${prefix}row ht="42" customHeight="1" hidden="1"><${prefix}c r="A50" t="inlineStr"><${prefix}is><${prefix}t>${"value".repeat(1000)}<${prefix}mergeCells><${prefix}mergeCell ref="XFD1048576"/>` - .replace( - /<((?:x:)?(?:col|dimension|mergeCell))\b([^>]*)\/>/g, - prefix ? "<$1$2>" : "$&", - ) - .replace(/[\s\S]*?<\/sheetData>/, (data) => (empty ? "" : data)) - .replace(/<(\/?)sheetData\b/g, (tag, closing: string) => - localPrefix ? `<${closing}x:sheetData${closing ? "" : ` xmlns:x="${ns}"`}` : tag, - ) - .replace(/="([^"]*)"/g, (attribute, value: string) => - prefix || empty - ? ` = '${value.replace(/\d/g, (digit) => `&#${digit.charCodeAt(0)};`)}'` - : attribute, - ), - ); - const bytes = await archive.generateAsync({ type: "arraybuffer" }); - const worker = new Worker(new URL("./officePreview.worker.ts", import.meta.url), { - type: "module", - }); - try { - const result = await new Promise((resolve) => { - worker.addEventListener( - "message", - (event: MessageEvent) => resolve(event.data), - { once: true }, - ); - worker.postMessage({ type: "open", kind: "xlsx", bytes }, [bytes]); - }); - expect(result.type).toBe("xlsx"); - if (result.type !== "xlsx") throw new Error("Expected bounded workbook"); - // The engine counts the chart sheet before it. - expect(result.view?.hiddenGridlines).toEqual([1]); - const bounded = await JSZip.loadAsync(result.bytes); - const sheet = new DOMParser().parseFromString( - await bounded.file("xl/worksheets/sheet1.xml")!.async("string"), - "application/xml", - ); - expect(sheet.querySelector("parsererror")).toBeNull(); - const row = sheet.getElementsByTagNameNS(ns, "row")[0]; - if (empty) expect(row).toBeUndefined(); - else { - expect(row!.getAttribute("ht")).toBe("42"); - expect(row!.getAttribute("hidden")).toBe("1"); - expect(row!.getElementsByTagNameNS(ns, "c")[0]!.getAttribute("r")).toBe("A50"); - expect(row!.getElementsByTagNameNS(ns, "t")[0]!.textContent).toHaveLength(4096); - } - expect(sheet.getElementsByTagNameNS(ns, "dimension")).toHaveLength(0); - expect(sheet.getElementsByTagNameNS(ns, "mergeCell")).toHaveLength(0); - const columns = [...sheet.getElementsByTagNameNS(ns, "col")].map((column) => - ["min", "max", "width"].map((name) => column.getAttribute(name)), - ); - // The engine ignores `defaultColWidth`, so the columns it sizes are spelled out. - expect(columns).toEqual([ - [String(OFFICE_PREVIEW_MAX_COLUMNS - 1), String(OFFICE_PREVIEW_MAX_COLUMNS), "20"], - // Hidden columns keep their span past the limit, so they stay hidden. - [String(OFFICE_PREVIEW_MAX_COLUMNS + 1), String(OFFICE_PREVIEW_MAX_COLUMNS + 30), null], - ["1", String(OFFICE_PREVIEW_MAX_COLUMNS - 2), "30"], - ]); - } finally { - worker.terminate(); - } - }, -); - -it("reports the final slide at maximum scroll after page visibility changes", async () => { - const { OfficePages } = await import("./office/OfficePages"); - const changed = vi.fn(); - const container = document.createElement("div"); - container.style.cssText = "display:flex;flex-direction:column;width:400px;height:400px"; - await using screen = await render( - ({ width: 400, height: 240 }))} - label="Slide" - paint={async () => undefined} - onPageChange={changed} - onError={(error) => { - throw error; - }} - />, - { container }, - ); - const scroller = screen.container.querySelector("[data-office-pages]")!; - await vi.waitFor(() => expect(scroller.scrollHeight).toBeGreaterThan(scroller.clientHeight)); - await act(() => { - scroller.scrollTop = scroller.scrollHeight; - }); - await vi.waitFor(() => expect(changed).toHaveBeenLastCalledWith(5)); - await act(() => { - scroller.scrollTop = 10; - }); - await vi.waitFor(() => expect(changed).toHaveBeenLastCalledWith(0)); -}); - -it("preserves merged text across tile clears", async () => { - const { paintFrame } = await import("./office/sheet-frame"); - const canvas = new OffscreenCanvas(400, 80); - const context = canvas.getContext("2d")!; - paintFrame( - context, - { - viewport: { x: 0, y: 0, width: 400, height: 80 }, - list: { width: 400, height: 80, commands: [] }, - tiles: [ - { - x: 0, - y: 0, - clipX: 0, - clipY: 0, - list: { - width: 208, - height: 80, - commands: [ - { - op: "text", - x: 2, - y: 40, - text: "MMMMMMMMMMMMMMMMMMMMMMMM", - fontSize: 20, - color: "#000", - clip: { x: 0, y: 0, w: 400, h: 80 }, - }, - ], - }, - }, - { x: 192, y: 0, clipX: 200, clipY: 0, list: { width: 208, height: 80, commands: [] } }, - ], - }, - { ratio: 1, origin: { x: 0, y: 0 } }, - ); - const pixels = context.getImageData(240, 15, 100, 35).data; - expect(pixels.some((value, index) => index % 4 === 3 && value > 0)).toBe(true); -}); - -it.each([ - "paragraphs", - "runs", - "tables", - "auxiliary XML", - "XML bytes", - "renamed XML", - "section after prefix", - "oversized section", -])("bounds excessive Word %s before rendering DOM", async (scenario) => { - const { default: JSZip } = await import("jszip"); - const archive = new JSZip(); - const body = ["paragraphs", "section after prefix", "oversized section"].includes(scenario) - ? "x".repeat(200_000) - : scenario === "runs" - ? `${"x".repeat(50_001)}` - : scenario === "tables" - ? "".repeat(2 * WORD_LIMITS.bodyElements) - : scenario === "XML bytes" - ? `${"x".repeat(2 * 1024 * 1024)}` - : ""; - const section = - ''; - archive.file( - "word/document.xml", - `${body}${scenario === "section after prefix" ? '' : ""}${scenario === "oversized section" ? `${"".repeat(2_001)}` : section}`, - ); - if (scenario === "auxiliary XML" || scenario === "renamed XML") - archive.file( - scenario === "renamed XML" ? "word/footnotes.dat" : "word/footnotes.xml", - `${"".repeat(WORD_LIMITS.auxiliaryMarkup + 1)}`, - ); - const file = await archive.generateAsync({ type: "blob", compression: "DEFLATE" }); - const src = URL.createObjectURL(file); - try { - const worker = new Worker(new URL("./officePreview.worker.ts", import.meta.url), { - type: "module", - }); - try { - const result = await new Promise((resolve) => { - worker.addEventListener( - "message", - (event: MessageEvent) => resolve(event.data), - { once: true }, - ); - void file - .arrayBuffer() - .then((bytes) => worker.postMessage({ type: "open", kind: "docx", bytes }, [bytes])); - }); - if ( - scenario === "auxiliary XML" || - scenario === "renamed XML" || - scenario === "oversized section" - ) { - expect(result).toMatchObject({ - type: "error", - message: expect.stringContaining("Preview limited"), - }); - } else { - expect(result.type).toBe("docx"); - if (result.type !== "docx") throw new Error("Expected a Word prefix"); - expect(result.truncated).toBe(true); - const bounded = await JSZip.loadAsync(result.bytes); - const markup = await bounded.file("word/document.xml")!.async("string"); - { - const xml = markup; - const document = new DOMParser().parseFromString(xml, "application/xml"); - expect(document.querySelector("parsererror")).toBeNull(); - const bodies = document.getElementsByTagNameNS("*", "body"); - expect(bodies).toHaveLength(1); - const section = bodies[0]!.lastElementChild!; - expect(section.localName).toBe("sectPr"); - const size = section.getElementsByTagNameNS("*", "pgSz")[0]!; - expect(size.getAttributeNS(section.namespaceURI, "w")).toBe( - scenario === "section after prefix" ? "11906" : "16838", - ); - expect(size.getAttributeNS(section.namespaceURI, "orient")).toBe( - scenario === "section after prefix" ? null : "landscape", - ); - } - expect(markup.length).toBeLessThan(body.length); - expect(markup.length).toBeLessThanOrEqual(WORD_LIMITS.bodyBytes); - expect(markup.match(/<(?![!?/])/g)!.length).toBeLessThanOrEqual(WORD_LIMITS.bodyTags); - } - } finally { - worker.terminate(); - } - } finally { - URL.revokeObjectURL(src); - } -}); - -it.each([ - "relationship markup", - "relationship bytes", - "content type", - "metadata type", - "deep XML", - "query path", -])("bounds XML hidden in media by %s", async (scenario) => { - const { default: JSZip } = await import("jszip"); - const archive = new JSZip(); - const path = `word/media/payload.png${scenario === "query path" ? "?xml" : ""}`; - archive.file( - "word/document.xml", - '', - ); - archive.file( - "[Content_Types].xml", - `${scenario === "metadata type" ? '' : ""}`, - ); - archive.file( - "word/_rels/document.xml.rels", - `${scenario === "content type" ? "" : ``}${scenario === "metadata type" ? "".repeat(WORD_LIMITS.auxiliaryMarkup + 1) : ""}`, - ); - archive.file( - path, - scenario === "relationship bytes" - ? `${"x".repeat(WORD_LIMITS.auxiliaryPartBytes)}` - : scenario === "deep XML" - ? `${"".repeat(65)}${"".repeat(65)}` - : `${"