From 76cdda4751a5d112658290f74f6ee02cd8d32eaf Mon Sep 17 00:00:00 2001 From: 0xPratik Date: Thu, 22 Oct 2026 00:36:23 +0530 Subject: [PATCH 1/6] feat(media): add generated images, sources and MediaViewer GeneratedImages holds each tile at its final size under the dither grain while it generates, with the elapsed time beside the label. An image made while watched is revealed by the grain; one already there loads plainly; a failed one shows a message and a retry. The grain reads its colours from existing tokens and draws one still frame under reduced motion. Sources lists citations, and MediaViewer shows images on the native dialog with arrow keys between items and focus returned to the opener. Media addresses from agent data are checked before use. --- e2e/fixtures.tsx | 81 +++++ e2e/media-viewer.spec.ts | 179 ++++++++++ src/hooks/use-media-viewer.test.ts | 90 +++++ src/hooks/use-media-viewer.ts | 91 +++++ src/index.ts | 4 + src/lib/dither-field.test.ts | 117 +++++++ src/lib/dither-field.ts | 165 +++++++++ src/lib/media-src.test.ts | 40 +++ src/lib/media-src.ts | 22 ++ src/ui/dither-field.tsx | 217 ++++++++++++ src/ui/generated-images.test.tsx | 243 +++++++++++++ src/ui/generated-images.tsx | 546 +++++++++++++++++++++++++++++ src/ui/media-viewer.test.tsx | 99 ++++++ src/ui/media-viewer.tsx | 336 ++++++++++++++++++ src/ui/sources.tsx | 88 +++++ stories/chat/media.stories.tsx | 145 ++++++++ 16 files changed, 2463 insertions(+) create mode 100644 e2e/media-viewer.spec.ts create mode 100644 src/hooks/use-media-viewer.test.ts create mode 100644 src/hooks/use-media-viewer.ts create mode 100644 src/lib/dither-field.test.ts create mode 100644 src/lib/dither-field.ts create mode 100644 src/lib/media-src.test.ts create mode 100644 src/lib/media-src.ts create mode 100644 src/ui/dither-field.tsx create mode 100644 src/ui/generated-images.test.tsx create mode 100644 src/ui/generated-images.tsx create mode 100644 src/ui/media-viewer.test.tsx create mode 100644 src/ui/media-viewer.tsx create mode 100644 src/ui/sources.tsx create mode 100644 stories/chat/media.stories.tsx diff --git a/e2e/fixtures.tsx b/e2e/fixtures.tsx index 1d1cd56e..86d7af7a 100644 --- a/e2e/fixtures.tsx +++ b/e2e/fixtures.tsx @@ -77,6 +77,12 @@ import { type AttachmentItem, type AttachmentRejection, useAttachments, + GeneratedImages, + generatedImageViewerItems, + MediaViewer, + useMediaViewer, + type ConversationImage, + type PartImage, } from "@corbits/react-ui"; import { CommandPalette, @@ -1457,6 +1463,79 @@ function AttachmentsScenario() { ); } +/** A flat swatch as an inline SVG, so nothing is fetched. */ +function swatch(index: number): ConversationImage { + const shade = 40 + index * 20; + const svg = ``; + return { + id: `swatch-${index}`, + mediaType: "image/svg+xml", + width: 300, + height: 200, + alt: `Swatch ${index}`, + url: `data:image/svg+xml,${encodeURIComponent(svg)}`, + }; +} + +const SWATCHES: PartImage = { + kind: "image", + id: "swatches", + prompt: "Three swatches", + state: "done", + images: [swatch(1), swatch(2), swatch(3)], +}; + +/** Starts generating, so landing the images is watched and reveals them. */ +function MediaViewerScenario() { + const [landed, setLanded] = useState(false); + const part: PartImage = landed + ? SWATCHES + : { + ...SWATCHES, + state: "generating", + images: [], + expected: { count: 3, width: 300, height: 200 }, + }; + const viewer = useMediaViewer({ items: generatedImageViewerItems(part) }); + return ( +
+ + + +
+ ); +} + +const REMOTE: PartImage = { + kind: "image", + id: "remote", + prompt: "Remote swatches", + state: "done", + images: [1, 2].map((index) => ({ + id: `remote-${index}`, + mediaType: "image/svg+xml", + width: 300, + height: 200, + alt: `Remote ${index}`, + url: `https://images.example.test/remote-${index}.svg`, + })), +}; + +/** Images from another origin inside a scrolling log, on a page that scrolls. */ +function MediaViewerChatScenario() { + const viewer = useMediaViewer({ items: generatedImageViewerItems(REMOTE) }); + return ( +
+
+
+ +
+
+ +
+ ); +} + export const scenarios: Record ReactNode> = { smoke: () => , "approval-card": () => , @@ -1527,4 +1606,6 @@ export const scenarios: Record ReactNode> = { ), attachments: () => , + "media-viewer": () => , + "media-viewer-chat": () => , }; diff --git a/e2e/media-viewer.spec.ts b/e2e/media-viewer.spec.ts new file mode 100644 index 00000000..ebdf7b62 --- /dev/null +++ b/e2e/media-viewer.spec.ts @@ -0,0 +1,179 @@ +import { expect, test, type Page } from "@playwright/test"; + +const grain = (page: Page) => + page.locator("[data-slot='generated-image-grain']"); + +async function land(page: Page) { + await page.goto("/media-viewer"); + await expect( + page.locator("[data-slot='generated-image-placeholder']"), + ).toHaveCount(3); + await expect(page.getByRole("status")).toHaveText("Generating 3 images"); + await page.getByRole("button", { name: "Land images" }).click(); +} + +test("images made while watched are revealed by the grain", async ({ + page, +}) => { + await land(page); + await expect(grain(page)).toHaveCount(3); + await expect(page.getByRole("status")).toHaveText("Made 3 images"); + // The reveal is one 700ms pass per tile, cascading 90ms apart. + await expect(grain(page)).toHaveCount(0, { timeout: 3_000 }); + const tiles = page.locator("[data-slot='generated-image']"); + await expect(tiles).toHaveCount(3); + for (const tile of await tiles.all()) { + await expect(tile).toHaveAttribute("data-state", "loaded"); + } +}); + +test("with reduced motion, landed images skip the grain", async ({ page }) => { + await page.emulateMedia({ reducedMotion: "reduce" }); + await land(page); + await expect( + page.locator("[data-slot='generated-image'][data-state='loaded']"), + ).toHaveCount(3); + await expect(grain(page)).toHaveCount(0); +}); + +test("arrow keys move between items, Escape closes, focus returns to the tile", async ({ + page, +}) => { + await land(page); + await expect(grain(page)).toHaveCount(0, { timeout: 3_000 }); + const tile = page.getByRole("button", { + name: "Open Swatch 2", + exact: true, + }); + await tile.click(); + + const dialog = page.getByRole("dialog"); + await expect(dialog).toHaveAccessibleName("three-swatches-2.svg"); + await expect(page.getByRole("button", { name: "Close" })).toBeFocused(); + await expect(dialog.getByRole("img")).toHaveAttribute("alt", "Swatch 2"); + + await page.keyboard.press("ArrowRight"); + await expect(dialog).toHaveAccessibleName("three-swatches-3.svg"); + await expect(dialog.getByRole("status")).toHaveText( + "three-swatches-3.svg, 3 of 3", + ); + // Stepping stops at the ends rather than wrapping. + await page.keyboard.press("ArrowRight"); + await expect(dialog).toHaveAccessibleName("three-swatches-3.svg"); + await expect(dialog.getByRole("button", { name: "Next" })).toHaveAttribute( + "aria-disabled", + "true", + ); + + await page.keyboard.press("ArrowLeft"); + await page.keyboard.press("ArrowLeft"); + await expect(dialog).toHaveAccessibleName("three-swatches-1.svg"); + await page.keyboard.press("ArrowLeft"); + await expect(dialog).toHaveAccessibleName("three-swatches-1.svg"); + + await page.keyboard.press("Escape"); + await expect(dialog).toHaveCount(0); + await expect(tile).toBeFocused(); +}); + +test("the viewer opens from the keyboard and closes from its button or the backdrop", async ({ + page, +}) => { + await land(page); + await expect(grain(page)).toHaveCount(0, { timeout: 3_000 }); + const tile = page.getByRole("button", { + name: "Open Swatch 3", + exact: true, + }); + await tile.focus(); + await page.keyboard.press("Enter"); + const dialog = page.getByRole("dialog"); + await expect(dialog).toHaveAccessibleName("three-swatches-3.svg"); + await dialog.getByRole("button", { name: "Previous" }).click(); + await expect(dialog).toHaveAccessibleName("three-swatches-2.svg"); + await dialog.getByRole("button", { name: "Close" }).click(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await expect(tile).toBeFocused(); + + // The action button opens it too, and a click outside the media closes it. + const action = page.getByRole("button", { + name: "Open Swatch 1 full size", + }); + await action.focus(); + await page.keyboard.press("Enter"); + await expect(page.getByRole("dialog")).toHaveAccessibleName( + "three-swatches-1.svg", + ); + await page.mouse.click(4, 4); + await expect(page.getByRole("dialog")).toHaveCount(0); + await expect(action).toBeFocused(); +}); + +const REMOTE_SVG = + ''; + +async function openRemote(page: Page) { + await page + .context() + .route("https://images.example.test/**", (route) => + route.fulfill({ contentType: "image/svg+xml", body: REMOTE_SVG }), + ); + await page.goto("/media-viewer-chat"); + await page + .getByRole("button", { name: "Open Remote 2", exact: true }) + .click(); + const dialog = page.getByRole("dialog"); + await expect(dialog).toHaveAccessibleName("remote-swatches-2.svg"); + return dialog; +} + +const scrolls = (page: Page) => + page.evaluate(() => ({ + page: document.scrollingElement?.scrollTop ?? 0, + log: document.querySelector("[data-testid='log']")?.scrollTop ?? 0, + })); + +test("with the viewer open, the wheel scrolls neither the page nor the log behind it", async ({ + page, +}) => { + await openRemote(page); + const before = await scrolls(page); + expect(before.log).toBeGreaterThan(0); + // Over the backdrop, the picture and the bar, in both directions. + for (const [x, y] of [ + [640, 360], + [20, 20], + [640, 640], + ] as const) { + await page.mouse.move(x, y); + await page.mouse.wheel(0, 800); + await page.mouse.wheel(0, -400); + } + await page.waitForTimeout(300); + expect(await scrolls(page)).toEqual(before); + + // Closing hands scrolling back to the page. + await page.keyboard.press("Escape"); + await expect(page.getByRole("dialog")).toHaveCount(0); + expect( + await page.evaluate(() => document.documentElement.style.overflow), + ).toBe(""); + await page.mouse.move(640, 700); + await page.mouse.wheel(0, 400); + await expect + .poll(async () => (await scrolls(page)).page) + .toBeGreaterThan(before.page); +}); + +test("Download of an image from another origin leaves the chat in place", async ({ + page, +}) => { + const dialog = await openRemote(page); + const popup = page.waitForEvent("popup"); + await dialog + .getByRole("link", { name: "Download remote-swatches-2.svg" }) + .click(); + await popup; + expect(new URL(page.url()).pathname).toBe("/media-viewer-chat"); + await expect(dialog).toBeVisible(); +}); diff --git a/src/hooks/use-media-viewer.test.ts b/src/hooks/use-media-viewer.test.ts new file mode 100644 index 00000000..3dfad00f --- /dev/null +++ b/src/hooks/use-media-viewer.test.ts @@ -0,0 +1,90 @@ +import { act } from "react"; +import { describe, expect, test } from "bun:test"; + +import { renderHook } from "../test/render-hook.js"; +import { + useMediaViewer, + type UseMediaViewerOptions, +} from "./use-media-viewer.js"; + +const ITEMS = ["a", "b", "c"] as const; + +function viewer(options: Partial> = {}) { + return renderHook(() => useMediaViewer({ items: ITEMS, ...options })); +} + +describe("useMediaViewer", () => { + test("starts closed and opens an item", () => { + const hook = viewer(); + expect(hook.result.current.index).toBe(null); + expect(hook.result.current.item).toBe(null); + act(() => hook.result.current.open(1)); + expect(hook.result.current.index).toBe(1); + expect(hook.result.current.item).toBe("b"); + hook.unmount(); + }); + + test("next and previous stop at the ends", () => { + const hook = viewer({ defaultIndex: 1 }); + act(() => hook.result.current.next()); + expect(hook.result.current.index).toBe(2); + expect(hook.result.current.hasNext).toBe(false); + act(() => hook.result.current.next()); + expect(hook.result.current.index).toBe(2); + + act(() => hook.result.current.previous()); + act(() => hook.result.current.previous()); + expect(hook.result.current.index).toBe(0); + expect(hook.result.current.hasPrevious).toBe(false); + act(() => hook.result.current.previous()); + expect(hook.result.current.index).toBe(0); + expect(hook.result.current.hasNext).toBe(true); + hook.unmount(); + }); + + test("closing clears the open item", () => { + const hook = viewer({ defaultIndex: 2 }); + act(() => hook.result.current.close()); + expect(hook.result.current.index).toBe(null); + expect(hook.result.current.item).toBe(null); + expect(hook.result.current.hasNext).toBe(false); + expect(hook.result.current.hasPrevious).toBe(false); + // Stepping does nothing while closed. + act(() => hook.result.current.next()); + expect(hook.result.current.index).toBe(null); + hook.unmount(); + }); + + test("an index outside the items reads as closed, and open ignores one", () => { + const hook = viewer({ defaultIndex: 5 }); + expect(hook.result.current.index).toBe(null); + act(() => hook.result.current.open(3)); + expect(hook.result.current.index).toBe(null); + act(() => hook.result.current.open(-1)); + expect(hook.result.current.index).toBe(null); + hook.unmount(); + }); + + test("controlled, it reports changes and follows the value it is given", () => { + const changes: (number | null)[] = []; + const hook = viewer({ + index: 0, + onIndexChange: (index) => changes.push(index), + }); + act(() => hook.result.current.next()); + act(() => hook.result.current.close()); + expect(changes).toEqual([1, null]); + expect(hook.result.current.index).toBe(0); + hook.unmount(); + }); + + test("keeps the element that opened it", () => { + const hook = viewer(); + const button = document.createElement("button"); + act(() => hook.result.current.open(0, button)); + expect(hook.result.current.opener.current).toBe(button); + act(() => hook.result.current.open(1)); + expect(hook.result.current.opener.current).toBe(null); + hook.unmount(); + }); +}); diff --git a/src/hooks/use-media-viewer.ts b/src/hooks/use-media-viewer.ts new file mode 100644 index 00000000..3cc99e3f --- /dev/null +++ b/src/hooks/use-media-viewer.ts @@ -0,0 +1,91 @@ +import { useCallback, useRef } from "react"; +import type { RefObject } from "react"; + +import { useControllableState } from "./use-controllable-state.js"; + +export type UseMediaViewerOptions = { + /** Everything the viewer can step through, in order. */ + readonly items: readonly T[]; + /** Controlled open item; `null` is closed. Supply with `onIndexChange`. */ + readonly index?: number | null | undefined; + /** Seeds the open item when `index` is not supplied. */ + readonly defaultIndex?: number | null; + readonly onIndexChange?: ((index: number | null) => void) | undefined; +}; + +export type MediaViewerState = { + readonly items: readonly T[]; + /** The open item's position, or `null` when closed. */ + readonly index: number | null; + readonly item: T | null; + readonly hasPrevious: boolean; + readonly hasNext: boolean; + /** + * Opens the item at `index`. Pass the element that opened it, so focus can + * go back there on close even where a click does not focus a button. + */ + open(index: number, opener?: HTMLElement | null): void; + close(): void; + next(): void; + previous(): void; + /** The element focus returns to on close, if `open` was given one. */ + readonly opener: RefObject; +}; + +/** + * Which item of a set is open in a viewer. Stepping stops at either end + * rather than wrapping, and an index outside `items` reads as closed, so a + * list that shrinks under an open viewer closes it rather than showing + * nothing. + */ +export function useMediaViewer({ + items, + index: controlledIndex, + defaultIndex = null, + onIndexChange, +}: UseMediaViewerOptions): MediaViewerState { + const [rawIndex, setIndex] = useControllableState({ + value: controlledIndex, + defaultValue: defaultIndex, + onChange: onIndexChange, + name: "useMediaViewer", + }); + const opener = useRef(null); + const count = items.length; + const index = + rawIndex !== null && + Number.isInteger(rawIndex) && + rawIndex >= 0 && + rawIndex < count + ? rawIndex + : null; + + const open = useCallback( + (next: number, element?: HTMLElement | null) => { + if (!Number.isInteger(next) || next < 0 || next >= count) return; + opener.current = element ?? null; + setIndex(next); + }, + [count, setIndex], + ); + const close = useCallback(() => setIndex(null), [setIndex]); + const next = useCallback(() => { + if (index !== null && index < count - 1) setIndex(index + 1); + }, [index, count, setIndex]); + const previous = useCallback(() => { + if (index !== null && index > 0) setIndex(index - 1); + }, [index, setIndex]); + + return { + items, + index, + item: index === null ? null : (items[index] ?? null), + hasPrevious: index !== null && index > 0, + hasNext: index !== null && index < count - 1, + open, + close, + next, + previous, + opener, + }; +} diff --git a/src/index.ts b/src/index.ts index 2564847d..6d7434b1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,6 +29,7 @@ export * from "./hooks/use-history-window.js"; export * from "./hooks/use-reading-position.js"; export * from "./hooks/use-attachments.js"; export * from "./hooks/use-file-drop.js"; +export * from "./hooks/use-media-viewer.js"; export * from "./lib/artifact.js"; export * from "./lib/chart-palette.js"; export * from "./lib/chat-message.js"; @@ -156,3 +157,6 @@ export * from "./ui/conversation-earlier.js"; export * from "./ui/conversation-map.js"; export * from "./ui/attachments.js"; export * from "./ui/attachment-drop-zone.js"; +export * from "./ui/generated-images.js"; +export * from "./ui/sources.js"; +export * from "./ui/media-viewer.js"; diff --git a/src/lib/dither-field.test.ts b/src/lib/dither-field.test.ts new file mode 100644 index 00000000..0e57e78e --- /dev/null +++ b/src/lib/dither-field.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "bun:test"; + +import { + BAYER_4, + DITHER_REVEAL_MS, + createDitherFieldFrame, + ditherFieldFrame, + type DitherFieldFrame, +} from "./dither-field.js"; + +const SIZE = 32; +const START = 10_000; + +function frameAt(timeMs: number, revealStartMs: number | null) { + return ditherFieldFrame({ + cols: SIZE, + rows: SIZE, + seed: 7, + timeMs, + revealStartMs, + }); +} + +function clearedSet(frame: DitherFieldFrame): Set { + const cells = new Set(); + for (let at = 0; at < frame.cleared.length; at += 2) { + cells.add(`${frame.cleared[at]},${frame.cleared[at + 1]}`); + } + return cells; +} + +function threshold(x: number, y: number): number { + const value = BAYER_4[y % 4]?.[x % 4]; + if (value === undefined) throw new Error("outside the matrix"); + return value; +} + +describe("ditherFieldFrame", () => { + test("before the reveal no cell is cleared", () => { + const forming = frameAt(START, null); + expect(forming.cleared).toEqual([]); + expect(forming.done).toBe(false); + expect(forming.ink.some((cells) => cells.length > 0)).toBe(true); + + const scheduled = frameAt(START - 1, START); + expect(scheduled.cleared).toEqual([]); + expect(scheduled.done).toBe(false); + }); + + test("at the end every cell is cleared and nothing is lit", () => { + const frame = frameAt(START + DITHER_REVEAL_MS, START); + expect(clearedSet(frame).size).toBe(SIZE * SIZE); + expect(frame.ink.every((cells) => cells.length === 0)).toBe(true); + expect(frame.band.every((cells) => cells.length === 0)).toBe(true); + expect(frame.done).toBe(true); + }); + + test("inside the edge, cells clear in Bayer order", () => { + let partialDiagonals = 0; + for (let elapsed = 50; elapsed < DITHER_REVEAL_MS; elapsed += 50) { + const cleared = clearedSet(frameAt(START + elapsed, START)); + // With a square grid, cells on one anti-diagonal sit at the same + // distance from the edge, so only their thresholds decide. + for (let sum = 0; sum <= 2 * (SIZE - 1); sum += 1) { + let highestCleared = -1; + let lowestKept = 2; + for ( + let x = Math.max(0, sum - SIZE + 1); + x <= Math.min(sum, SIZE - 1); + x += 1 + ) { + const y = sum - x; + const value = threshold(x, y); + if (cleared.has(`${x},${y}`)) { + highestCleared = Math.max(highestCleared, value); + } else { + lowestKept = Math.min(lowestKept, value); + } + } + expect(highestCleared).toBeLessThan(lowestKept); + if (highestCleared >= 0 && lowestKept <= 1) partialDiagonals += 1; + } + } + // The edge really is dithered: some diagonals are part cleared. + expect(partialDiagonals).toBeGreaterThan(0); + }); + + test("the reveal only ever clears more", () => { + let previous = new Set(); + for (let elapsed = 0; elapsed <= DITHER_REVEAL_MS; elapsed += 35) { + const cleared = clearedSet(frameAt(START + elapsed, START)); + for (const cell of previous) expect(cleared.has(cell)).toBe(true); + previous = cleared; + } + }); + + test("a reused frame is emptied before it is filled", () => { + const out = createDitherFieldFrame(); + ditherFieldFrame( + { + cols: SIZE, + rows: SIZE, + seed: 1, + timeMs: START + 400, + revealStartMs: START, + }, + out, + ); + expect(out.cleared.length).toBeGreaterThan(0); + ditherFieldFrame( + { cols: SIZE, rows: SIZE, seed: 1, timeMs: START, revealStartMs: null }, + out, + ); + expect(out.cleared).toEqual([]); + expect(out.done).toBe(false); + }); +}); diff --git a/src/lib/dither-field.ts b/src/lib/dither-field.ts new file mode 100644 index 00000000..5533a252 --- /dev/null +++ b/src/lib/dither-field.ts @@ -0,0 +1,165 @@ +/** + * 4×4 Bayer thresholds, 0–1: the brand's ordered dither. A pure function of + * position, so the grain stays locked to its surface while the field under it + * moves. `DitherCanvas` keeps its own private copy of the same matrix. + */ +export const BAYER_4: readonly (readonly number[])[] = [ + [0, 8, 2, 10], + [12, 4, 14, 6], + [3, 11, 1, 9], + [15, 7, 13, 5], +].map((row) => row.map((value) => (value + 0.5) / 16)); + +/** Alpha steps a cell fades through, so cells ease in and out instead of blinking. */ +export const DITHER_GRAIN_STEPS = 6; + +/** How long the reveal's edge takes to cross the field. */ +export const DITHER_REVEAL_MS = 700; + +/** Width of the reveal's edge, as a share of the diagonal. Cells inside it + * clear in Bayer order, so the edge itself is grain. */ +const REVEAL_EDGE = 0.16; + +/** How long the ambient band takes to give way once the reveal starts. */ +const AMBIENT_FADE_MS = 300; + +export type DitherFieldInput = { + readonly cols: number; + readonly rows: number; + /** Offsets the field and band, so neighbouring fields never move in step. */ + readonly seed: number; + /** The page clock, in ms. */ + readonly timeMs: number; + /** When the reveal starts on the same clock; `null` while still forming. */ + readonly revealStartMs: number | null; +}; + +/** + * One frame, as flat `[x, y, x, y, …]` cell coordinates. `ink[step]` and + * `band[step]` hold the lit cells at each alpha step (0 faintest) in the + * field colour and the band colour; `cleared` holds the cells the reveal has + * passed, which show what is underneath. + */ +export type DitherFieldFrame = { + readonly ink: number[][]; + readonly band: number[][]; + readonly cleared: number[]; + /** The reveal has crossed the whole field: every cell is cleared. */ + done: boolean; +}; + +export function createDitherFieldFrame(): DitherFieldFrame { + return { + ink: Array.from({ length: DITHER_GRAIN_STEPS }, (): number[] => []), + band: Array.from({ length: DITHER_GRAIN_STEPS }, (): number[] => []), + cleared: [], + done: false, + }; +} + +function easeInOutCubic(p: number): number { + return p < 0.5 ? 4 * p * p * p : 1 - (-2 * p + 2) ** 3 / 2; +} + +function clamp01(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +/** + * The forming grain: a slow interference field and a band sweeping corner to + * corner, each cell lit by comparing the field against its Bayer threshold. + * With a reveal, the band makes one last pass as the reveal's edge: behind it + * cells clear (in Bayer order, so the boundary is grain too), ahead of it the + * grain thickens. + * + * Writes into `out` (cleared first) and returns it, so a draw loop can reuse + * one frame's arrays instead of allocating a fresh set sixty times a second. + */ +export function ditherFieldFrame( + input: DitherFieldInput, + out: DitherFieldFrame = createDitherFieldFrame(), +): DitherFieldFrame { + const { cols, rows, seed, timeMs, revealStartMs } = input; + for (const cells of out.ink) cells.length = 0; + for (const cells of out.band) cells.length = 0; + out.cleared.length = 0; + + const t = timeMs / 1000; + const phase = (seed * 0.37) % 1; + const ambient = ((t / 5.5 + phase) % 1) * 1.6 - 0.3; + const progress = + revealStartMs === null + ? 0 + : clamp01((timeMs - revealStartMs) / DITHER_REVEAL_MS); + // The edge travels from just off one corner to just past the other. + const front = + revealStartMs === null || timeMs < revealStartMs + ? null + : easeInOutCubic(progress) * (1 + 2 * REVEAL_EDGE) - REVEAL_EDGE; + const ambientWeight = + revealStartMs === null + ? 1 + : 1 - clamp01((timeMs - revealStartMs) / AMBIENT_FADE_MS); + + // The diagonal waves split into per-column and per-row terms + // (sin(a + b) = sin a cos b + cos a sin b), so a cell costs a few + // multiplies rather than three trig calls. + const colWave = new Float32Array(cols); + const colSin = new Float32Array(cols); + const colCos = new Float32Array(cols); + for (let x = 0; x < cols; x += 1) { + const u = x / cols; + colWave[x] = Math.sin(u * 7.1 + t * 0.55 + seed); + colSin[x] = Math.sin(u * 11 - t * 0.9 + seed * 2); + colCos[x] = Math.cos(u * 11 - t * 0.9 + seed * 2); + } + const rowWave = new Float32Array(rows); + const rowSin = new Float32Array(rows); + const rowCos = new Float32Array(rows); + for (let y = 0; y < rows; y += 1) { + const v = y / rows; + rowWave[y] = Math.cos(v * 5.3 - t * 0.4); + rowSin[y] = Math.sin(v * 11); + rowCos[y] = Math.cos(v * 11); + } + + let remaining = 0; + for (let y = 0; y < rows; y += 1) { + const thresholds = BAYER_4[y % 4]; + for (let x = 0; x < cols; x += 1) { + const threshold = thresholds?.[x % 4] ?? 0.5; + const diagonal = (x / cols + y / rows) / 2; + let band = 0; + let charge = 0; + if (front !== null) { + if ((front - diagonal) / REVEAL_EDGE > threshold) { + out.cleared.push(x, y); + continue; + } + // Just ahead of the edge the band rides it and the grain thickens. + const ahead = diagonal - front; + band = Math.exp(-(ahead * ahead) / 0.004); + charge = 0.35 * Math.exp(-(ahead * ahead) / 0.03); + } + remaining += 1; + const drift = diagonal - ambient; + band = Math.max(band, ambientWeight * Math.exp(-(drift * drift) / 0.01)); + const field = + 0.3 + + charge + + 0.22 * (colWave[x] ?? 0) * (rowWave[y] ?? 0) + + 0.12 * + ((colSin[x] ?? 0) * (rowCos[y] ?? 0) + + (colCos[x] ?? 0) * (rowSin[y] ?? 0)); + const level = (field + band * 0.5 - threshold) / 0.14 + 0.5; + if (level <= 0) continue; + const step = Math.min( + DITHER_GRAIN_STEPS - 1, + Math.floor(level * DITHER_GRAIN_STEPS), + ); + (band > 0.3 ? out.band : out.ink)[step]?.push(x, y); + } + } + out.done = front !== null && progress >= 1 && remaining === 0; + return out; +} diff --git a/src/lib/media-src.test.ts b/src/lib/media-src.test.ts new file mode 100644 index 00000000..c0f68d56 --- /dev/null +++ b/src/lib/media-src.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; + +import { safeMediaSrc } from "./media-src.js"; + +describe("safeMediaSrc", () => { + test("network and blob URLs load in every sink", () => { + for (const src of [ + "https://example.com/a.png", + "http://example.com/a.pdf", + "blob:https://example.com/0f6c1b52", + ]) { + expect(safeMediaSrc(src, "media")).toBe(src); + expect(safeMediaSrc(src, "frame")).toBe(src); + } + }); + + test("a media element takes a data: URL of image, video or audio only", () => { + const image = "data:image/svg+xml;base64,PHN2Zy8+"; + expect(safeMediaSrc(image, "media")).toBe(image); + expect(safeMediaSrc("data:video/mp4;base64,AAAA", "media")).toBe( + "data:video/mp4;base64,AAAA", + ); + expect(safeMediaSrc("data:text/html,", "media")).toBe( + undefined, + ); + }); + + test("a frame never takes a data: URL", () => { + expect(safeMediaSrc("data:image/png;base64,AAAA", "frame")).toBe(undefined); + expect(safeMediaSrc("data:application/pdf;base64,AAAA", "frame")).toBe( + undefined, + ); + }); + + test("script and relative URLs are refused", () => { + expect(safeMediaSrc("javascript:alert(1)", "media")).toBe(undefined); + expect(safeMediaSrc("JavaScript:alert(1)", "frame")).toBe(undefined); + expect(safeMediaSrc("/files/a.png", "media")).toBe(undefined); + }); +}); diff --git a/src/lib/media-src.ts b/src/lib/media-src.ts new file mode 100644 index 00000000..82670f9b --- /dev/null +++ b/src/lib/media-src.ts @@ -0,0 +1,22 @@ +import { isSafeUrl } from "./url.js"; + +/** Where a media sink may load from. */ +export type MediaSink = "media" | "frame"; + +const NETWORK_PROTOCOLS: readonly string[] = ["http:", "https:", "blob:"]; + +/** `data:` media types an ``, `