diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5770c1f4516b..ba1e1e612e6d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -355,16 +355,22 @@ jobs: # The web suite is the largest outside the server. In the shared Test job it # started last and ran alone for about 40s after everything else finished. - # release.yml mirrors this test, test_web, and test_server split. + # Browser tests are bound by the Vite server transforming modules for each + # file, not by Chromium, so a second runner (and Vite server) halves the job + # that gated every run. release.yml mirrors this test, test_web, and test_server split. test_web: permissions: contents: read id-token: write - name: Test Web + name: Test Web ${{ matrix.shard }} needs: changes if: ${{ !cancelled() && needs.changes.outputs.test_web != 'false' }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@v6 @@ -406,7 +412,7 @@ jobs: fi - name: Test - run: vp run --filter @t3tools/web test + run: vp run --filter @t3tools/web test --shard ${{ matrix.shard }}/${{ strategy.job-total }} # apps/server sets `fileParallelism: false`, so its test files run strictly # one at a time. Sharding spreads them over separate runners instead of @@ -481,6 +487,17 @@ jobs: node apps/server/node_modules/playwright-core/cli.js install-deps chromium fi + # Each server test file runs in its own Vitest process, and a fresh install starts + # Vitest's content-keyed module cache empty, so every file re-transformed its imports. + # Only pushes to main save it, so pull requests never evict each other's entries. + - name: Restore Vitest module cache + if: steps.changes.outputs.test_server != 'false' + uses: actions/cache/restore@v6 + with: + path: node_modules/.vitest-cache + key: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + - name: Test if: steps.changes.outputs.test_server != 'false' env: @@ -491,6 +508,13 @@ jobs: --uncached apps/server/integration/transferBudgetV2.integration.test.ts --verify-hits 1 apps/server + - name: Save Vitest module cache + if: github.event_name == 'push' && steps.changes.outputs.test_server != 'false' + uses: actions/cache/save@v6 + with: + path: node_modules/.vitest-cache + key: vitest-modules-server-${{ matrix.shard }}-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + # integration/transferBudgetV2.integration.test.ts writes the report; exactly one shard # produces these files. Gating the upload on their presence keeps a # single `thread-transfer-results` artifact per run, which is the name diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84acc7c74f865..8827876b3829f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -300,11 +300,15 @@ jobs: # Web browser tests time out when they share a runner with the other package # suites: Chromium gets too little CPU and diff workers miss their 10s start budget. test_web: - name: Release web tests + name: Release web tests ${{ matrix.shard }} needs: [preflight] if: ${{ !failure() && !cancelled() && needs.preflight.result == 'success' }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@v6 @@ -344,10 +348,11 @@ jobs: fi - name: Test - run: vp run --filter @t3tools/web test + run: vp run --filter @t3tools/web test --shard ${{ matrix.shard }}/${{ strategy.job-total }} # Browser tests run source modules; this boots the production chunk graph. - name: Boot the built web client + if: matrix.shard == 1 run: | vp run --filter @t3tools/web build node apps/web/scripts/verify-built-boot.ts diff --git a/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx b/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx index 4ecc965f6fe32..285032b5b6a0a 100644 --- a/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx +++ b/apps/web/src/components/files/AttachmentFilePreview.browser.test.tsx @@ -78,6 +78,7 @@ import { decodeImagePreview } from "./decodeImagePreview"; import type { BinaryWorkerResult } from "./binaryPreviewProtocol"; import type { TextPreviewResult } from "./largeTextProtocol"; import type { ReactElement } from "react"; +import type { OfficeWorkerResult } from "./officePreviewProtocol"; // Real worker imports and WASM initialization can exceed waitFor's default second // when the full browser project runs concurrently on CI. @@ -481,8 +482,6 @@ describe("bounded binary previews", () => { } }); }); -import { OFFICE_PREVIEW_MAX_COLUMNS, type OfficeWorkerResult } from "./officePreviewProtocol"; -import { WORD_LIMITS } from "./officeArchive"; describe("large text preview recovery", () => { it("retains the workspace viewer across URL renewals and reloads revisions and manual refreshes", async () => { @@ -921,1046 +920,6 @@ describe("large text preview recovery", () => { await vi.waitFor(() => expect(load).toHaveBeenCalledWith(0)); }); }); -it.each(["", ""])( - "opens an empty Word document with %s", - async (body) => { - const { default: JSZip } = await import("jszip"); - const { prepareOffice } = await import("./officeArchive"); - const archive = new JSZip().file( - "word/document.xml", - `${body}`, - ); - const prepared = await prepareOffice( - await archive.generateAsync({ type: "arraybuffer" }), - "docx", - ); - const result = await JSZip.loadAsync(prepared.bytes); - const xml = new DOMParser().parseFromString( - await result.file("word/document.xml")!.async("string"), - "application/xml", - ); - expect(xml.querySelector("parsererror")).toBeNull(); - expect( - xml.getElementsByTagNameNS( - "http://schemas.openxmlformats.org/wordprocessingml/2006/main", - "body", - ), - ).toHaveLength(1); - }, -); - -it.each([false, true])( - "rejects excessive archive entries before JSZip (directories: %s)", - async (directories) => { - const { default: JSZip } = await import("jszip"); - const { prepareOffice } = await import("./officeArchive"); - const archive = new JSZip(); - for (let i = 0; i < 4096; i++) - archive.file(`entry-${i}${directories ? "/" : ".txt"}`, "", { dir: directories }); - const bytes = await archive.generateAsync({ type: "arraybuffer" }); - const load = vi.spyOn(JSZip, "loadAsync"); - try { - await expect(prepareOffice(bytes, "docx")).rejects.toThrow("2,048 entries"); - expect(load).not.toHaveBeenCalled(); - // The actual central-directory records must also be bounded if EOCD lies. - const view = new DataView(bytes); - view.setUint16(bytes.byteLength - 22 + 8, 1, true); - view.setUint16(bytes.byteLength - 22 + 10, 1, true); - await expect(prepareOffice(bytes, "docx")).rejects.toThrow("2,048 entries"); - expect(load).not.toHaveBeenCalled(); - } finally { - load.mockRestore(); - } - }, -); - -it("rejects oversized embedded images before decoding pixels and retains ordinary images", async () => { - const { officeImageBitmap } = await import("./office/image-bitmap"); - const canvas = new OffscreenCanvas(2, 3); - canvas.getContext("2d"); - const blob = await canvas.convertToBlob({ type: "image/png" }); - const bytes = await blob.arrayBuffer(); - new DataView(bytes).setUint32(16, 12000); - new DataView(bytes).setUint32(20, 12000); - const decode = vi.spyOn(globalThis, "createImageBitmap"); - try { - await expect(officeImageBitmap(new Blob([bytes]))).rejects.toThrow("pixel limit"); - expect(decode).not.toHaveBeenCalled(); - const bitmap = await officeImageBitmap(blob); - expect([bitmap.width, bitmap.height]).toEqual([2, 3]); - bitmap.close(); - } finally { - decode.mockRestore(); - } -}); - -it.each(["utf-16le", "utf-16be", "windows-1252"])( - "preserves %s Office XML and safe hyperlink targets", - async (encoding) => { - const { default: JSZip } = await import("jszip"); - const { prepareOffice } = await import("./officeArchive"); - const archive = new JSZip(); - const encode = (text: string) => { - const xml = `${text}`; - if (encoding === "windows-1252") return Uint8Array.from(xml, (char) => char.charCodeAt(0)); - const bytes = new Uint8Array(2 + xml.length * 2); - const view = new DataView(bytes.buffer); - const little = encoding === "utf-16le"; - view.setUint16(0, 0xfeff, little); - for (let i = 0; i < xml.length; i++) view.setUint16(2 + i * 2, xml.charCodeAt(i), little); - return bytes; - }; - archive.file( - "word/document.xml", - encode( - 'café', - ), - ); - archive.file( - "word/_rels/document.xml.rels", - encode( - `${["https://example.com/page", "http://example.com", "mailto:maria@example.com", "javascript:alert(1)", "file:///tmp/private"].map((target, i) => ``).join("")}`, - ), - ); - const prepared = await prepareOffice( - await archive.generateAsync({ type: "arraybuffer" }), - "docx", - ); - const result = await JSZip.loadAsync(prepared.bytes); - expect(await result.file("word/document.xml")!.async("string")).toContain("café"); - const links = await result.file("word/_rels/document.xml.rels")!.async("string"); - expect(links).toContain('encoding="UTF-8"'); - expect(links).toContain("https://example.com/page"); - expect(links).toContain("http://example.com"); - expect(links).toContain("mailto:maria@example.com"); - expect(links).not.toContain("javascript:"); - expect(links).not.toContain("file:"); - }, -); - -it.each([ - ["", "worksheets/./sheet1.xml", false, false], - ["x:", "../xl/worksheets/sheet1.xml", false, false], - ["", "/xl/worksheets/sheet1.xml", true, false], - ["", "worksheets/sheet1.xml", false, true], - ["", "worksheets/sheet1.xml", true, true], -])( - "preserves %sworksheet metadata with relationship target %s", - async (prefix, target, empty, localPrefix) => { - const { default: JSZip } = await import("jszip"); - const archive = new JSZip(); - const ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; - archive.file( - "xl/workbook.xml", - '', - ); - archive.file( - "xl/_rels/workbook.xml.rels", - ``, - ); - archive.file( - "xl/worksheets/sheet1.xml", - `<${prefix}worksheet xmlns${prefix ? ":x" : ""}="${ns}"><${prefix}dimension ref="XFD1048576"/><${prefix}sheetViews><${prefix}sheetView showGridLines="0"/><${prefix}sheetFormatPr defaultColWidth="30"/><${prefix}cols><${prefix}col min="${OFFICE_PREVIEW_MAX_COLUMNS - 1}" max="${OFFICE_PREVIEW_MAX_COLUMNS + 10}" width="20"/><${prefix}col min="${OFFICE_PREVIEW_MAX_COLUMNS + 1}" max="${OFFICE_PREVIEW_MAX_COLUMNS + 30}" hidden="1"/><${prefix}sheetData><${prefix}row ht="42" customHeight="1" hidden="1"><${prefix}c r="A50" t="inlineStr"><${prefix}is><${prefix}t>${"value".repeat(1000)}<${prefix}mergeCells><${prefix}mergeCell ref="XFD1048576"/>` - .replace( - /<((?:x:)?(?:col|dimension|mergeCell))\b([^>]*)\/>/g, - prefix ? "<$1$2>" : "$&", - ) - .replace(/[\s\S]*?<\/sheetData>/, (data) => (empty ? "" : data)) - .replace(/<(\/?)sheetData\b/g, (tag, closing: string) => - localPrefix ? `<${closing}x:sheetData${closing ? "" : ` xmlns:x="${ns}"`}` : tag, - ) - .replace(/="([^"]*)"/g, (attribute, value: string) => - prefix || empty - ? ` = '${value.replace(/\d/g, (digit) => `&#${digit.charCodeAt(0)};`)}'` - : attribute, - ), - ); - const bytes = await archive.generateAsync({ type: "arraybuffer" }); - const worker = new Worker(new URL("./officePreview.worker.ts", import.meta.url), { - type: "module", - }); - try { - const result = await new Promise((resolve) => { - worker.addEventListener( - "message", - (event: MessageEvent) => resolve(event.data), - { once: true }, - ); - worker.postMessage({ type: "open", kind: "xlsx", bytes }, [bytes]); - }); - expect(result.type).toBe("xlsx"); - if (result.type !== "xlsx") throw new Error("Expected bounded workbook"); - // The engine counts the chart sheet before it. - expect(result.view?.hiddenGridlines).toEqual([1]); - const bounded = await JSZip.loadAsync(result.bytes); - const sheet = new DOMParser().parseFromString( - await bounded.file("xl/worksheets/sheet1.xml")!.async("string"), - "application/xml", - ); - expect(sheet.querySelector("parsererror")).toBeNull(); - const row = sheet.getElementsByTagNameNS(ns, "row")[0]; - if (empty) expect(row).toBeUndefined(); - else { - expect(row!.getAttribute("ht")).toBe("42"); - expect(row!.getAttribute("hidden")).toBe("1"); - expect(row!.getElementsByTagNameNS(ns, "c")[0]!.getAttribute("r")).toBe("A50"); - expect(row!.getElementsByTagNameNS(ns, "t")[0]!.textContent).toHaveLength(4096); - } - expect(sheet.getElementsByTagNameNS(ns, "dimension")).toHaveLength(0); - expect(sheet.getElementsByTagNameNS(ns, "mergeCell")).toHaveLength(0); - const columns = [...sheet.getElementsByTagNameNS(ns, "col")].map((column) => - ["min", "max", "width"].map((name) => column.getAttribute(name)), - ); - // The engine ignores `defaultColWidth`, so the columns it sizes are spelled out. - expect(columns).toEqual([ - [String(OFFICE_PREVIEW_MAX_COLUMNS - 1), String(OFFICE_PREVIEW_MAX_COLUMNS), "20"], - // Hidden columns keep their span past the limit, so they stay hidden. - [String(OFFICE_PREVIEW_MAX_COLUMNS + 1), String(OFFICE_PREVIEW_MAX_COLUMNS + 30), null], - ["1", String(OFFICE_PREVIEW_MAX_COLUMNS - 2), "30"], - ]); - } finally { - worker.terminate(); - } - }, -); - -it("reports the final slide at maximum scroll after page visibility changes", async () => { - const { OfficePages } = await import("./office/OfficePages"); - const changed = vi.fn(); - const container = document.createElement("div"); - container.style.cssText = "display:flex;flex-direction:column;width:400px;height:400px"; - await using screen = await render( - ({ width: 400, height: 240 }))} - label="Slide" - paint={async () => undefined} - onPageChange={changed} - onError={(error) => { - throw error; - }} - />, - { container }, - ); - const scroller = screen.container.querySelector("[data-office-pages]")!; - await vi.waitFor(() => expect(scroller.scrollHeight).toBeGreaterThan(scroller.clientHeight)); - await act(() => { - scroller.scrollTop = scroller.scrollHeight; - }); - await vi.waitFor(() => expect(changed).toHaveBeenLastCalledWith(5)); - await act(() => { - scroller.scrollTop = 10; - }); - await vi.waitFor(() => expect(changed).toHaveBeenLastCalledWith(0)); -}); - -it("preserves merged text across tile clears", async () => { - const { paintFrame } = await import("./office/sheet-frame"); - const canvas = new OffscreenCanvas(400, 80); - const context = canvas.getContext("2d")!; - paintFrame( - context, - { - viewport: { x: 0, y: 0, width: 400, height: 80 }, - list: { width: 400, height: 80, commands: [] }, - tiles: [ - { - x: 0, - y: 0, - clipX: 0, - clipY: 0, - list: { - width: 208, - height: 80, - commands: [ - { - op: "text", - x: 2, - y: 40, - text: "MMMMMMMMMMMMMMMMMMMMMMMM", - fontSize: 20, - color: "#000", - clip: { x: 0, y: 0, w: 400, h: 80 }, - }, - ], - }, - }, - { x: 192, y: 0, clipX: 200, clipY: 0, list: { width: 208, height: 80, commands: [] } }, - ], - }, - { ratio: 1, origin: { x: 0, y: 0 } }, - ); - const pixels = context.getImageData(240, 15, 100, 35).data; - expect(pixels.some((value, index) => index % 4 === 3 && value > 0)).toBe(true); -}); - -it.each([ - "paragraphs", - "runs", - "tables", - "auxiliary XML", - "XML bytes", - "renamed XML", - "section after prefix", - "oversized section", -])("bounds excessive Word %s before rendering DOM", async (scenario) => { - const { default: JSZip } = await import("jszip"); - const archive = new JSZip(); - const body = ["paragraphs", "section after prefix", "oversized section"].includes(scenario) - ? "x".repeat(200_000) - : scenario === "runs" - ? `${"x".repeat(50_001)}` - : scenario === "tables" - ? "".repeat(2 * WORD_LIMITS.bodyElements) - : scenario === "XML bytes" - ? `${"x".repeat(2 * 1024 * 1024)}` - : ""; - const section = - ''; - archive.file( - "word/document.xml", - `${body}${scenario === "section after prefix" ? '' : ""}${scenario === "oversized section" ? `${"".repeat(2_001)}` : section}`, - ); - if (scenario === "auxiliary XML" || scenario === "renamed XML") - archive.file( - scenario === "renamed XML" ? "word/footnotes.dat" : "word/footnotes.xml", - `${"".repeat(WORD_LIMITS.auxiliaryMarkup + 1)}`, - ); - const file = await archive.generateAsync({ type: "blob", compression: "DEFLATE" }); - const src = URL.createObjectURL(file); - try { - const worker = new Worker(new URL("./officePreview.worker.ts", import.meta.url), { - type: "module", - }); - try { - const result = await new Promise((resolve) => { - worker.addEventListener( - "message", - (event: MessageEvent) => resolve(event.data), - { once: true }, - ); - void file - .arrayBuffer() - .then((bytes) => worker.postMessage({ type: "open", kind: "docx", bytes }, [bytes])); - }); - if ( - scenario === "auxiliary XML" || - scenario === "renamed XML" || - scenario === "oversized section" - ) { - expect(result).toMatchObject({ - type: "error", - message: expect.stringContaining("Preview limited"), - }); - } else { - expect(result.type).toBe("docx"); - if (result.type !== "docx") throw new Error("Expected a Word prefix"); - expect(result.truncated).toBe(true); - const bounded = await JSZip.loadAsync(result.bytes); - const markup = await bounded.file("word/document.xml")!.async("string"); - { - const xml = markup; - const document = new DOMParser().parseFromString(xml, "application/xml"); - expect(document.querySelector("parsererror")).toBeNull(); - const bodies = document.getElementsByTagNameNS("*", "body"); - expect(bodies).toHaveLength(1); - const section = bodies[0]!.lastElementChild!; - expect(section.localName).toBe("sectPr"); - const size = section.getElementsByTagNameNS("*", "pgSz")[0]!; - expect(size.getAttributeNS(section.namespaceURI, "w")).toBe( - scenario === "section after prefix" ? "11906" : "16838", - ); - expect(size.getAttributeNS(section.namespaceURI, "orient")).toBe( - scenario === "section after prefix" ? null : "landscape", - ); - } - expect(markup.length).toBeLessThan(body.length); - expect(markup.length).toBeLessThanOrEqual(WORD_LIMITS.bodyBytes); - expect(markup.match(/<(?![!?/])/g)!.length).toBeLessThanOrEqual(WORD_LIMITS.bodyTags); - } - } finally { - worker.terminate(); - } - } finally { - URL.revokeObjectURL(src); - } -}); - -it.each([ - "relationship markup", - "relationship bytes", - "content type", - "metadata type", - "deep XML", - "query path", -])("bounds XML hidden in media by %s", async (scenario) => { - const { default: JSZip } = await import("jszip"); - const archive = new JSZip(); - const path = `word/media/payload.png${scenario === "query path" ? "?xml" : ""}`; - archive.file( - "word/document.xml", - '', - ); - archive.file( - "[Content_Types].xml", - `${scenario === "metadata type" ? '' : ""}`, - ); - archive.file( - "word/_rels/document.xml.rels", - `${scenario === "content type" ? "" : ``}${scenario === "metadata type" ? "".repeat(WORD_LIMITS.auxiliaryMarkup + 1) : ""}`, - ); - archive.file( - path, - scenario === "relationship bytes" - ? `${"x".repeat(WORD_LIMITS.auxiliaryPartBytes)}` - : scenario === "deep XML" - ? `${"".repeat(65)}${"".repeat(65)}` - : `${"