From 3e210acbea6dbef37688ec2920304303ed4cd125 Mon Sep 17 00:00:00 2001 From: Navneet Chaudhary Date: Sat, 7 Nov 2026 01:42:28 +0530 Subject: [PATCH 1/2] feat: add bounded PNG motion quality probes --- README.md | 4 + docs/IMPLEMENTATION_PLAN.md | 3 +- docs/PIXEL_QA.md | 40 ++++++++ docs/QUALITY_AUDIT.md | 2 +- src/character/pixel-qa.ts | 190 ++++++++++++++++++++++++++++++++++++ src/cli/index.ts | 15 ++- tests/pixel-qa.test.ts | 79 +++++++++++++++ 7 files changed, 329 insertions(+), 4 deletions(-) create mode 100644 docs/PIXEL_QA.md create mode 100644 src/character/pixel-qa.ts create mode 100644 tests/pixel-qa.test.ts diff --git a/README.md b/README.md index 5ec9f50..64798ba 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The long-term goal: **generate artwork when necessary, animate and reuse it inde - Assisted single-PNG character onboarding through Rev2D landmarks, local rig build, review sheets, unapproved-by-default manifest, and starter scene. - Agent-guided landmark proposals: source-hash-bound task, optional **tested** Codex image-based suggestion, isolated geometry validation, visual overlay and explicit review/acceptance that revokes old rig approval. - Imported avatar asset provenance: source/spec/model/texture fingerprints checked by scene validation and render; untracked alternate imported pose models are refused. Force rebuild revokes approval before modifying art. Suspicious landmarks generate nonblocking geometry warnings. +- Local PNG pixel-QA comparison (full frame or an explicit region), reporting RGB activity and newly transparent border pixels; these metrics never auto-approve a rig. - Agent-readable CLI output, tests and CI. **Not implemented yet:** fully automatic landmark placement or reliable one-click rigging, AI-generated arbitrary new pose rigs, convincing smooth bridges between unrelated meshes (especially crossed arms), automatic transcription/TTS, captions, B-roll and multiple actors. The arm-raise gesture is genuinely animated, but its artwork is still prototype-quality. Other pose families may require edited hard cuts. The pose manifest is capability metadata; visual poses become usable only after their rig files are generated, validated and inspected. @@ -42,6 +43,8 @@ npm run demo:directed # Apply edited camera framing and gradual punch-in npm run demo:gesture # Generate two-joint gesture rig and export edited 1080p demo npm run demo:gesture-cycle # Render continuous raise/lower cycles without cuts npm run demo:avatar # Import a freshly rendered flat PNG and create an unapproved rig + review sheets +# Compare captured PNG frames: +# node dist/cli/index.js character qa compare outputs/rest.png outputs/closed.png --region 175,220,170,75 ~~~ The sample uses Rev2D's own **Hana** rig from the pinned MIT-licensed upstream checkout; generated character rig JSON is not checked into this repository. The derived example video is separately attributed and licensed under the original Rev2D terms. @@ -94,6 +97,7 @@ Our native animation backend for the spike is **Rev2D**, pinned by the setup hel - [Agent-guided landmark review and acceptance](docs/LANDMARK_AGENT_WORKFLOW.md) - [Optional Codex vision inference: real-image results and limitations](docs/CODEX_VISION_LANDMARKS.md) - [Deep quality and reliability audit with observed pixel measurements](docs/QUALITY_AUDIT.md) +- [Pixel-level visual QA (non-approval diagnostics)](docs/PIXEL_QA.md) - [Architecture decision log](docs/DECISIONS.md) Audio/lip-sync works with precomputed Rhubarb cues and a locally installed executable. The smooth pointing demo reuses the same face and deforms one rig continuously; crossed arms still need a more convincing bridge or intentional cut. Next: polish eyelid and mouth deformation by reusing Rev2D's reviewed drawn variants, expand automated image-quality probes and evaluate diverse source artwork. Do not confuse data validation with visual quality. diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 22fd6c1..6527a1b 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -76,7 +76,8 @@ - [x] Add conservative geometry triage for low torso pivot/fade, missing/outside-eye iris and shearing head pivot; return nonblocking warnings. - [x] Revoke existing approval before any --force build mutates Rev2D outputs, and fail closed on stale source/spec/model/texture fingerprints. - [x] Eliminate reproduced black edge artifacts caused by alpha-zero warped pixels in H.264: composite frames onto scene background, with a 60-frame before/after comparison. -- [ ] Add pixel-level motion QA probes and a reviewable report; automated geometry warnings are not proof of visual quality. +- [x] Add bounded per-pixel PNG rest/activity comparison, optional eye/face ROI and unexpected transparent border metrics; expose through the CLI with deterministic tests and actual Hana frame evidence. +- [ ] Extend image-based QA to multi-frame head/neck discontinuity and visual report/contact-sheet generation; metrics must never auto-approve a rig. - [ ] Wrap Rev2D's existing drawn closed-eye/mouth variants to reduce visible deformation artifacts without writing another mesh engine. ## Acceptance benchmark diff --git a/docs/PIXEL_QA.md b/docs/PIXEL_QA.md new file mode 100644 index 0000000..78bcdd0 --- /dev/null +++ b/docs/PIXEL_QA.md @@ -0,0 +1,40 @@ +# Pixel-level visual QA (triage only) + +**Status:** local PNG comparison implemented with deterministic RGBA decoding through the already-required FFmpeg executable. This is **not** a visual-approval model and not a perceptual similarity metric. + +## CLI + +Capture two frame images of exactly the same canvas and resolution. For a Rev2D avatar, use the engine's `render` command (review the current pinned backend help for flags) or the generated scene preview PNGs. + +~~~sh +node dist/cli/index.js character qa compare outputs/rest.png outputs/eyes-closed.png +node dist/cli/index.js character qa compare outputs/rest.png outputs/eyes-closed.png --region 175,220,170,75 +~~~ + +The optional region is an **integer** `x,y,width,height` rectangle within both images. Always use comparable camera, framing and character pose settings; do not compare resized images or a camera cut to an unchanged rest pose and interpret the result as articulation. + +The output includes: + +| Field | Meaning | +| --- | --- | +| `rgbMeanAbsoluteError` | Average absolute difference per RGB channel, in 0–255 intensity levels | +| `changedPixels` and `changedFraction` | Pixels whose RGB or alpha changed by more than 10 levels; fraction of comparable ROI pixels | +| `ignoredTransparentPixels` | ROI pixels where both samples have negligible alpha and RGB is visually undefined | +| `unexpectedTransparentBorderPixels` | Full-image perimeter pixels that were opaque in the reference and became nonopaque in the candidate | +| `region` | Coordinates actually evaluated for motion and RGB differences | + +Inputs must be PNGs under 30 MB with dimensions 256–4096 pixels per side. Files must be inside the active workspace. PNGs with mismatched dimensions or invalid ROI are rejected rather than silently scaled. FFmpeg is invoked with fixed argument arrays, without a shell, and decoded RGBA buffers are bounded. + +## Real-data probe (Hana, 2026-11-07) + +A fresh local 512×640 Hana import was rendered at rest and again with both eye-open parameters set to 0, using the same native camera and an opaque background. The resulting source-versus-rest full-frame comparison gave `rgbMeanAbsoluteError = 0.0002`, `changedPixels = 0`, and `unexpectedTransparentBorderPixels = 0` for that specific imported test drawing. + +For a 170×75 eye-region crop, rest versus closed-eyes produced `rgbMeanAbsoluteError = 33.5879`, `changedFraction = 0.364392`, and `unexpectedTransparentBorderPixels = 0`. These numbers establish **pixel activity**, not the quality of the drawing. Human visual inspection of the same review sheets still showed flat half-eye artwork and double contour artifacts in fades. + +## Limitations and next checks + +- A different or badly deformed eye can produce a *higher* difference score; there is no universal minimum or maximum that certifies good animation. +- Closed-eyelid anatomy, visible tears, head/neck continuity, seam quality, motion cadence, and voice sync still require inspecting rendered frames/video. +- Frame-to-frame cuts, camera movement and background replacement must be identified semantically before interpreting pixel deltas. +- Thresholds should be calibrated per rig on diverse properly licensed artwork, not tuned to one Hana sample. +- This helper does not transmit artwork to an external model. It decodes PNGs locally using FFmpeg. diff --git a/docs/QUALITY_AUDIT.md b/docs/QUALITY_AUDIT.md index ef4fa9b..07d07f2 100644 --- a/docs/QUALITY_AUDIT.md +++ b/docs/QUALITY_AUDIT.md @@ -74,7 +74,7 @@ Rev2D creates eye-white, iris, upper/lower lid, lash, crease and foreground-hair ## 6. Next milestones 1. Wrap Rev2D drawn-variant requests and imports as guarded PuppetControl CLI operations. Require explicit user/artwork permission and renewed review. No new mesh engine. -2. Add image-based QA probes for rest-pose identity, blink/gaze/mouth activity, and head/neck discontinuities. Quantitative reports are triage, not automatic visual approval. +2. **Basic pixel-QA probe implemented** for same-resolution PNG pairs, with optional eye/face region and opaque-border transparency checks. Fresh local 512×640 Hana rest/source RGB mean error 0.0002/255; eye crop rest/closed RGB mean error 33.5879/255 (36.4392% changed), with 0 new transparent border pixels. Full methodology and caution in [PIXEL_QA.md](PIXEL_QA.md). Multi-frame discontinuity scans and meaningful aesthetic review remain open. 3. Build a diverse rights-cleared benchmark matrix: eye size, hair over face, accessories/glasses, different skin tones and line styles, opaque versus transparent backgrounds and constrained 3/4 views. Track success/failure on each, not just demos. 4. Only after a reliable single actor: improve body-pose transitions, synchronized multi-actor shots and user-facing authoring polish. diff --git a/src/character/pixel-qa.ts b/src/character/pixel-qa.ts new file mode 100644 index 0000000..0b8f248 --- /dev/null +++ b/src/character/pixel-qa.ts @@ -0,0 +1,190 @@ +/** + * Deterministic pixel-level *triage* for PNG render comparisons. + * + * These metrics establish changed pixels and reveal opaque-canvas gaps. + * They DO NOT measure artistic quality or automatically approve a rig. + */ +import { spawn } from "node:child_process"; +import { open, stat } from "node:fs/promises"; +import { AvatarWorkflowError, checkedPath, parsePngSize } from "./avatar.js"; + +const MAX_PNG_BYTES = 30 * 1024 * 1024; +const MAX_RAW_BYTES = 4096 * 4096 * 4; +const PIXEL_DIFFERENCE_THRESHOLD = 10; + +export interface PixelRegion { + x: number; + y: number; + width: number; + height: number; +} + +export interface PixelComparison { + width: number; + height: number; + region: PixelRegion; + comparedPixels: number; + ignoredTransparentPixels: number; + rgbMeanAbsoluteError: number; + changedPixels: number; + changedFraction: number; + /** Reference-opaque edge pixels that became semi/fully transparent. */ + unexpectedTransparentBorderPixels: number; + note: string; +} + +function validRegion(region: PixelRegion, width: number, height: number): void { + if ([region.x, region.y, region.width, region.height].some((value) => !Number.isSafeInteger(value)) || + region.x < 0 || region.y < 0 || region.width < 1 || region.height < 1 || + region.x + region.width > width || region.y + region.height > height) { + throw new AvatarWorkflowError("PIXEL_QA_REGION_INVALID", "Region must be an integer rectangle within both images."); + } +} + +export function parsePixelRegionArgument(raw: string): PixelRegion { + if (!/^\d+,\d+,\d+,\d+$/.test(raw)) { + throw new AvatarWorkflowError("PIXEL_QA_REGION_INVALID", "Use --region x,y,width,height with nonnegative integer values."); + } + const [x, y, width, height] = raw.split(",").map(Number); + if (![x, y, width, height].every((value) => Number.isSafeInteger(value)) || width! < 1 || height! < 1) { + throw new AvatarWorkflowError("PIXEL_QA_REGION_INVALID", "Region size must be positive and coordinates must be safe integers."); + } + return { x: x!, y: y!, width: width!, height: height! }; +} + +export function compareRgbaPixels( + reference: Uint8Array, candidate: Uint8Array, width: number, height: number, + region: PixelRegion = { x: 0, y: 0, width, height }, +): PixelComparison { + if (!Number.isSafeInteger(width) || !Number.isSafeInteger(height) || + width < 1 || height < 1 || width * height * 4 > MAX_RAW_BYTES || + reference.byteLength !== width * height * 4 || candidate.byteLength !== width * height * 4) { + throw new AvatarWorkflowError("PIXEL_QA_BUFFER_INVALID", "Both images need matching RGBA buffers and valid bounded dimensions."); + } + validRegion(region, width, height); + + let compared = 0; + let skipped = 0; + let rgbAbsoluteSum = 0; + let changed = 0; + for (let y = region.y; y < region.y + region.height; y++) { + for (let x = region.x; x < region.x + region.width; x++) { + const i = (y * width + x) * 4; + const refAlpha = reference[i + 3]!; + const nextAlpha = candidate[i + 3]!; + // RGB is undefined/invisible when both samples are completely transparent. + if (refAlpha < 16 && nextAlpha < 16) { + skipped++; + continue; + } + let maxChannelDelta = 0; + for (let channel = 0; channel < 3; channel++) { + const delta = Math.abs(reference[i + channel]! - candidate[i + channel]!); + rgbAbsoluteSum += delta; + maxChannelDelta = Math.max(maxChannelDelta, delta); + } + if (maxChannelDelta > PIXEL_DIFFERENCE_THRESHOLD || + Math.abs(refAlpha - nextAlpha) > PIXEL_DIFFERENCE_THRESHOLD) changed++; + compared++; + } + } + + let unexpectedTransparentBorderPixels = 0; + // Count each perimeter pixel once (avoid counting corners twice). + for (let y = 0; y < height; y++) { + for (const x of (width === 1 ? [0] : [0, width - 1])) { + const i = (y * width + x) * 4 + 3; + if (reference[i]! >= 250 && candidate[i]! < 250) unexpectedTransparentBorderPixels++; + } + } + for (let x = 1; x < width - 1; x++) { + for (const y of (height === 1 ? [0] : [0, height - 1])) { + const i = (y * width + x) * 4 + 3; + if (reference[i]! >= 250 && candidate[i]! < 250) unexpectedTransparentBorderPixels++; + } + } + + return { + width, height, region, comparedPixels: compared, ignoredTransparentPixels: skipped, + rgbMeanAbsoluteError: compared ? Number((rgbAbsoluteSum / (compared * 3)).toFixed(4)) : 0, + changedPixels: changed, changedFraction: compared ? Number((changed / compared).toFixed(6)) : 0, + unexpectedTransparentBorderPixels, + note: "Pixel differences are diagnostics only; visually inspect eyes, face, neck and transitions before approving artwork.", + }; +} + +async function pngInfo(path: string, root: string) { + const file = await checkedPath(root, path); + const details = await stat(file); + if (!details.isFile() || details.size < 24 || details.size > MAX_PNG_BYTES) { + throw new AvatarWorkflowError("PIXEL_QA_PNG_INVALID", "QA images must be PNG files under 30 MB."); + } + const handle = await open(file, "r"); + try { + const header = Buffer.alloc(24); + const result = await handle.read(header, 0, 24, 0); + if (result.bytesRead !== 24) throw new AvatarWorkflowError("PIXEL_QA_PNG_INVALID", "Incomplete PNG header."); + return { file, dimensions: parsePngSize(header) }; + } finally { + await handle.close(); + } +} + +async function decodeRgbaPng(file: string, expectedLength: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const fail = (message: string) => { + if (settled) return; + settled = true; + child.kill(); + reject(new AvatarWorkflowError("PIXEL_QA_DECODE_FAILED", message)); + }; + const child = spawn("ffmpeg", [ + "-nostdin", "-hide_banner", "-loglevel", "error", + "-i", file, "-frames:v", "1", "-an", + "-pix_fmt", "rgba", "-f", "rawvideo", "pipe:1", + ], { stdio: ["ignore", "pipe", "pipe"], windowsHide: true, signal: AbortSignal.timeout(60_000) }); + const chunks: Buffer[] = []; + let total = 0; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + total += chunk.length; + if (total > expectedLength) return fail("Decoded image exceeds header dimensions."); + chunks.push(chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + if (stderr.length < 2000) stderr += chunk.toString("utf8").slice(0, 2000 - stderr.length); + }); + child.on("error", (error) => fail("FFmpeg unavailable or failed: " + error.message)); + child.on("close", (code) => { + if (settled) return; + if (code !== 0 || total !== expectedLength) { + return fail("PNG decode failed or pixel count differs from header: " + stderr.slice(0, 1000)); + } + settled = true; + resolve(Buffer.concat(chunks, total)); + }); + }); +} + +/** PNG files must remain inside the current workspace; FFmpeg never executes via a shell. */ +export async function comparePngFiles( + referencePath: string, candidatePath: string, + region?: PixelRegion, root = process.cwd(), +): Promise { + const [reference, candidate] = await Promise.all([ + pngInfo(referencePath, root), + pngInfo(candidatePath, root), + ]); + const { width, height } = reference.dimensions; + if (candidate.dimensions.width !== width || candidate.dimensions.height !== height) { + throw new AvatarWorkflowError("PIXEL_QA_SIZE_MISMATCH", "Images must have exactly matching dimensions; no implicit resizing."); + } + if (region) validRegion(region, width, height); + const length = width * height * 4; + const [first, second] = await Promise.all([ + decodeRgbaPng(reference.file, length), + decodeRgbaPng(candidate.file, length), + ]); + return compareRgbaPixels(first, second, width, height, region); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 0f127df..bf2bc75 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -14,6 +14,7 @@ import { AvatarWorkflowError, importAvatar, buildAvatar } from "../character/ava import { prepareLandmarkTask, stageLandmarks, acceptLandmarks } from "../character/landmarks.js"; import { proposeCodexLandmarks, codexExecutableForPlatform } from "../character/vision-codex.js"; import { verifyAvatarBuildRecord } from "../character/provenance.js"; +import { comparePngFiles, parsePixelRegionArgument } from "../character/pixel-qa.js"; function print(value: unknown): void { process.stdout.write(JSON.stringify(value, null, 2) + "\n"); @@ -27,6 +28,7 @@ function usage(): never { " puppetctl character inspect \n" + " puppetctl character import --out --id \n" + " puppetctl character build [--force]\n" + + " puppetctl character qa compare [--region x,y,w,h]\n" + " puppetctl character landmarks assist \n" + " puppetctl character landmarks propose --provider codex\n" + " puppetctl character landmarks stage --proposal \n" + @@ -47,7 +49,7 @@ async function run(): Promise { if (first === "--help" || first === "help") { process.stdout.write( "PuppetControl (experimental)\n" + - "Available: doctor, character inspect/import/build, landmark assist/propose/stage/accept, audio align, scene validate/plan, single-rig scene preview/render.\n" + + "Available: doctor, character inspect/import/build, image pixel-QA compare, landmark assist/propose/stage/accept, audio align, scene validate/plan, single-rig scene preview/render.\n" + "Not yet available: reliable one-click landmark inference, animated pose bridges, captions, automatic transcription/TTS.\n", ); return; @@ -72,7 +74,7 @@ async function run(): Promise { ffmpegAvailable: ffmpegFound, rev2dAvailable, codexCliAvailable, - availableFeatures: ["character-inspect", "scene-validate", "transition-policy", "rhubarb-cue-parsing", "rhubarb-align-if-installed", ...(rev2dAvailable ? ["assisted-png-avatar-import", "agent-landmark-proposal-staging", ...(codexCliAvailable ? ["optional-codex-vision-proposals"] : []), "avatar-review-sheets", "single-rig-preview", "single-rig-voice-render", "pose-family-hard-cut-render", "experimental-shot-framing"] : [])], + availableFeatures: ["character-inspect", "scene-validate", "transition-policy", "rhubarb-cue-parsing", "rhubarb-align-if-installed", ...(ffmpegFound ? ["png-pixel-qa"] : []), ...(rev2dAvailable ? ["assisted-png-avatar-import", "agent-landmark-proposal-staging", ...(codexCliAvailable ? ["optional-codex-vision-proposals"] : []), "avatar-review-sheets", "single-rig-preview", "single-rig-voice-render", "pose-family-hard-cut-render", "experimental-shot-framing"] : [])], unavailableFeatures: ["animated-pose-bridges", "automatic-landmark-placement", "automatic-transcription", "text-to-speech", "multi-audio-mixing"], }); return; @@ -121,6 +123,15 @@ async function run(): Promise { return; } + if (first === "character" && second === "qa" && third === "compare") { + const [reference, candidate, ...flags] = process.argv.slice(5); + if (!reference || !candidate || + (flags.length !== 0 && (flags.length !== 2 || flags[0] !== "--region" || !flags[1]))) usage(); + const region = flags.length ? parsePixelRegionArgument(flags[1]!) : undefined; + print({ valid: true, comparison: await comparePngFiles(reference, candidate, region) }); + return; + } + if (first === "character" && second === "landmarks" && third) { const [project, ...flags] = process.argv.slice(5); if (!project) usage(); diff --git a/tests/pixel-qa.test.ts b/tests/pixel-qa.test.ts new file mode 100644 index 0000000..60d0828 --- /dev/null +++ b/tests/pixel-qa.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { compareRgbaPixels, parsePixelRegionArgument } from "../src/character/pixel-qa.js"; + +function solid(width: number, height: number, rgba: [number, number, number, number]): Uint8Array { + const bytes = new Uint8Array(width * height * 4); + for (let i = 0; i < width * height; i++) bytes.set(rgba, i * 4); + return bytes; +} + +describe("pixel-level image comparison is deterministic triage, not automatic approval", () => { + it("compares identical pixels without invented motion or border gaps", () => { + const reference = solid(4, 3, [20, 40, 60, 255]); + const result = compareRgbaPixels(reference, reference, 4, 3); + expect(result).toMatchObject({ + comparedPixels: 12, ignoredTransparentPixels: 0, + rgbMeanAbsoluteError: 0, changedPixels: 0, changedFraction: 0, + unexpectedTransparentBorderPixels: 0, + }); + expect(result.note).toMatch(/visually inspect/i); + }); + + it("measures region-specific RGB change without counting unaffected pixels", () => { + const reference = solid(3, 3, [10, 20, 30, 255]); + const changed = reference.slice(); + const center = (1 * 3 + 1) * 4; + changed[center] = 100; + const full = compareRgbaPixels(reference, changed, 3, 3); + expect(full.changedPixels).toBe(1); + expect(full.rgbMeanAbsoluteError).toBeCloseTo(90 / (9 * 3), 3); + const crop = compareRgbaPixels(reference, changed, 3, 3, + { x: 1, y: 1, width: 1, height: 1 }); + expect(crop.changedFraction).toBe(1); + expect(crop.rgbMeanAbsoluteError).toBe(30); + }); + + it("ignores fully transparent RGB noise and finds new opaque-border holes", () => { + const reference = solid(3, 3, [100, 120, 140, 255]); + const candidate = reference.slice(); + // Reference-opaque top-left corner becomes transparent. + candidate[3] = 0; + // Fully transparent center has arbitrary irrelevant RGB values. + const emptyReference = solid(2, 2, [12, 0, 0, 0]); + const emptyCandidate = solid(2, 2, [250, 180, 160, 0]); + const transparent = compareRgbaPixels(emptyReference, emptyCandidate, 2, 2); + expect(transparent.comparedPixels).toBe(0); + expect(transparent.ignoredTransparentPixels).toBe(4); + expect(transparent.changedPixels).toBe(0); + expect(compareRgbaPixels(reference, candidate, 3, 3).unexpectedTransparentBorderPixels).toBe(1); + }); + + it("does not double-count corners or single-row/single-column borders", () => { + const ref = solid(1, 3, [3, 4, 5, 255]); + const result = compareRgbaPixels(ref, solid(1, 3, [3, 4, 5, 0]), 1, 3); + expect(result.unexpectedTransparentBorderPixels).toBe(3); + const flat = compareRgbaPixels(solid(3, 1, [0, 0, 0, 255]), + solid(3, 1, [0, 0, 0, 0]), 3, 1); + expect(flat.unexpectedTransparentBorderPixels).toBe(3); + }); + + it("parses strict CLI region syntax and rejects malformed values", () => { + expect(parsePixelRegionArgument("175,220,170,75")) + .toEqual({ x: 175, y: 220, width: 170, height: 75 }); + for (const bad of ["0,0,0,1", "-1,0,1,1", "2,3,4", "1.5,0,1,1", "1,2,3,4junk", "1,2,3,9999999999999999999"]) { + expect(() => parsePixelRegionArgument(bad)).toThrowError(/region/i); + } + }); + + it("rejects wrong pixel lengths, unsafe sizes and ROIs", () => { + const img = solid(2, 2, [0, 0, 0, 255]); + expect(() => compareRgbaPixels(img, img.slice(0, 8), 2, 2)) + .toThrowError(/matching RGBA/); + expect(() => compareRgbaPixels(img, img, 2, 2, { x: 1, y: 1, width: 2, height: 2 })) + .toThrowError(/rectangle/); + expect(() => compareRgbaPixels(img, img, 2, 2, { x: 0, y: 0, width: 0, height: 1 })) + .toThrowError(/rectangle/); + expect(() => compareRgbaPixels(img, img, Infinity, 2)) + .toThrowError(/bounded dimensions/); + }); +}); From ed2b8a8056d7dae8b7b9e0882408edec29ab8cfd Mon Sep 17 00:00:00 2001 From: Navneet Chaudhary Date: Sat, 7 Nov 2026 01:54:45 +0530 Subject: [PATCH 2/2] feat: flag motion stalls across ordered PNG frames --- .agents/skills/puppetcontrol/SKILL.md | 3 +- README.md | 3 +- docs/IMPLEMENTATION_PLAN.md | 3 +- docs/PIXEL_QA.md | 29 +++++ docs/QUALITY_AUDIT.md | 2 +- src/character/frame-sequence-qa.ts | 171 ++++++++++++++++++++++++++ src/cli/index.ts | 34 ++++- tests/frame-sequence-qa.test.ts | 117 ++++++++++++++++++ 8 files changed, 356 insertions(+), 6 deletions(-) create mode 100644 src/character/frame-sequence-qa.ts create mode 100644 tests/frame-sequence-qa.test.ts diff --git a/.agents/skills/puppetcontrol/SKILL.md b/.agents/skills/puppetcontrol/SKILL.md index 0b4c8d5..11d3837 100644 --- a/.agents/skills/puppetcontrol/SKILL.md +++ b/.agents/skills/puppetcontrol/SKILL.md @@ -18,6 +18,7 @@ Read repo-root AGENTS.md, docs/TECHNICAL_CONCEPT.md, and docs/CHARACTER_IMPORT.m 9. Within a compatible shared rig, semantic pose events may interpolate through validated parameter presets (the neutral-to-pointing demo uses shoulder and elbow controls). Distinct pose models require explicit scene camera cuts; an authored bridge is NOT implemented. Never silently crossfade meshes or call a body cut a smooth physical transition. 10. For speech, prepare WAV, install Rhubarb separately, and use node dist/cli/index.js audio align --out [--binary ]. Reference audio.path, audio.startMs and audio.cuesPath from the scene; review the actual mouth/audio timing. 11. Do not commit private character illustrations, full generated models, user voice recordings or secrets without checking redistribution rights. Demo:avatar generates pinned MIT-licensed sample artwork into ignored .cache. -12. State what actually rendered and which controls remain unsupported. Do not claim a flat PNG is automatically converted into production-quality animation without landmark review. +12. For animation QA, compare real ordered PNGs with `node dist/cli/index.js character qa sequence ... [--region x,y,w,h] [--control-values v0,v1,...] [--cut-after i,j]`. Supply sampled control values when investigating invisible or stalled motion; label known cuts. Inspect the PNGs/video and report possible ghosting even if the motion statistics look normal. The QA report is not an approval. +13. State what actually rendered and which controls remain unsupported. Do not claim a flat PNG is automatically converted into production-quality animation without landmark review. Keep edits small and testable. Run npm run verify and inspect visual evidence for any animation/asset changes. \ No newline at end of file diff --git a/README.md b/README.md index 64798ba..ad87333 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The long-term goal: **generate artwork when necessary, animate and reuse it inde - Assisted single-PNG character onboarding through Rev2D landmarks, local rig build, review sheets, unapproved-by-default manifest, and starter scene. - Agent-guided landmark proposals: source-hash-bound task, optional **tested** Codex image-based suggestion, isolated geometry validation, visual overlay and explicit review/acceptance that revokes old rig approval. - Imported avatar asset provenance: source/spec/model/texture fingerprints checked by scene validation and render; untracked alternate imported pose models are refused. Force rebuild revokes approval before modifying art. Suspicious landmarks generate nonblocking geometry warnings. -- Local PNG pixel-QA comparison (full frame or an explicit region), reporting RGB activity and newly transparent border pixels; these metrics never auto-approve a rig. +- Local PNG pixel and ordered-frame QA (full frame or explicit region) reports RGB activity, transparent borders, candidate motion jumps and changed-control stalls; metrics never auto-approve a rig. - Agent-readable CLI output, tests and CI. **Not implemented yet:** fully automatic landmark placement or reliable one-click rigging, AI-generated arbitrary new pose rigs, convincing smooth bridges between unrelated meshes (especially crossed arms), automatic transcription/TTS, captions, B-roll and multiple actors. The arm-raise gesture is genuinely animated, but its artwork is still prototype-quality. Other pose families may require edited hard cuts. The pose manifest is capability metadata; visual poses become usable only after their rig files are generated, validated and inspected. @@ -45,6 +45,7 @@ npm run demo:gesture-cycle # Render continuous raise/lower cycles without cuts npm run demo:avatar # Import a freshly rendered flat PNG and create an unapproved rig + review sheets # Compare captured PNG frames: # node dist/cli/index.js character qa compare outputs/rest.png outputs/closed.png --region 175,220,170,75 +# node dist/cli/index.js character qa sequence outputs/f00.png outputs/f01.png outputs/f02.png --control-values 1,0.5,0 ~~~ The sample uses Rev2D's own **Hana** rig from the pinned MIT-licensed upstream checkout; generated character rig JSON is not checked into this repository. The derived example video is separately attributed and licensed under the original Rev2D terms. diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 6527a1b..5466367 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -77,7 +77,8 @@ - [x] Revoke existing approval before any --force build mutates Rev2D outputs, and fail closed on stale source/spec/model/texture fingerprints. - [x] Eliminate reproduced black edge artifacts caused by alpha-zero warped pixels in H.264: composite frames onto scene background, with a 60-frame before/after comparison. - [x] Add bounded per-pixel PNG rest/activity comparison, optional eye/face ROI and unexpected transparent border metrics; expose through the CLI with deterministic tests and actual Hana frame evidence. -- [ ] Extend image-based QA to multi-frame head/neck discontinuity and visual report/contact-sheet generation; metrics must never auto-approve a rig. +- [x] Add 2–60 ordered-frame RGB motion triage with optional ROI, expected control samples and intentional cut indices; flag candidate jumps and stalled intervals, verified against a real 13-frame Hana blink cycle. +- [ ] Generate a first-class visual contact sheet from the CLI and add head/neck seam-specific checks; pixel metrics must never auto-approve a rig. - [ ] Wrap Rev2D's existing drawn closed-eye/mouth variants to reduce visible deformation artifacts without writing another mesh engine. ## Acceptance benchmark diff --git a/docs/PIXEL_QA.md b/docs/PIXEL_QA.md index 78bcdd0..56f4bae 100644 --- a/docs/PIXEL_QA.md +++ b/docs/PIXEL_QA.md @@ -31,6 +31,35 @@ A fresh local 512×640 Hana import was rendered at rest and again with both eye- For a 170×75 eye-region crop, rest versus closed-eyes produced `rgbMeanAbsoluteError = 33.5879`, `changedFraction = 0.364392`, and `unexpectedTransparentBorderPixels = 0`. These numbers establish **pixel activity**, not the quality of the drawing. Human visual inspection of the same review sheets still showed flat half-eye artwork and double contour artifacts in fades. +## Ordered-frame sequence QA (temporal motion) + +Provide **2–60 PNGs in time order**. This does not infer sorting from filenames. The CLI analyzes adjacent frames, excludes declared intentional cuts from its local comparison baseline and reports unusually large changes as candidates for visual inspection. + +~~~sh +node dist/cli/index.js character qa sequence outputs/f00.png outputs/f01.png outputs/f02.png outputs/f03.png --region 175,220,170,75 +node dist/cli/index.js character qa sequence outputs/f00.png outputs/f01.png outputs/f02.png outputs/f03.png --region 175,220,170,75 --control-values 1,0.7,0.3,0 +~~~ + +The optional `--control-values` list needs exactly one finite numeric control sample **per frame** and must describe a specific animated rig parameter (for example, `EyeLOpen`). When the control moves but rendered pixels barely change, the result is a `possibleMotionStalls` candidate. A static region, a clamped value, occlusion or an intentionally paused animation can also cause a flagged pair, so **visual review remains mandatory**. + +For edited scenes, supply zero-based `--cut-after` indices to identify intended jumps: for example, `--cut-after 2` means the change between input frames 2 and 3 is an editorial cut and should not be treated as a surprise jump. The CLI reports the per-pair differences and candidate indices; it **never** returns a binary animation-quality verdict. The `firstToLast` measurement helps verify loop returns when the endpoints should match, but a deliberately non-looping scene need not return to start. + +### Real 13-frame blink diagnosis (Hana, 2026-11-07) + +The previously generated, intentionally unapproved Hana sprite trial was rendered locally at 512×640 with `EyeLOpen` and `EyeROpen` stepped together from 1 to 0 and back to 1 in 13 frames. A silent 30fps MP4 and manually assembled contact sheet were inspected under ignored `outputs/qa/temporal-blink-20261107/`. The new report showed: + +| Metric | Observed | +| --- | --- | +| Adjacent pairs / frames | 12 / 13 | +| ROI | x 175, y 220, width 170, height 75 | +| Median adjacent RGB difference | 10.7044 / 255 | +| Potential abrupt pairs | none under generic outlier heuristic | +| Expected-control motion stalls | pairs 3→4 and 8→9 (0.166667 control delta, 0 RGB difference) | +| First versus last frame difference | exactly 0 | +| Video check | 512×640 H.264, 30 fps, 13 frames, 0.433333 seconds | + +Rev2D's documented `eyes_half` variant intentionally remains fully visible over `EyeOpen` 0.3–0.6, explaining the two flat intervals; the tool cannot decide whether this artistic hold is acceptable. **Actual visual inspection** showed eye-outline ghosting at intermediate fade frames, even though the generic jump detector did not flag them. A smooth-looking statistic therefore does not imply smooth-looking animation. The generated variant art is experimental, locally ignored, not published and **not approved for use**. + ## Limitations and next checks - A different or badly deformed eye can produce a *higher* difference score; there is no universal minimum or maximum that certifies good animation. diff --git a/docs/QUALITY_AUDIT.md b/docs/QUALITY_AUDIT.md index 07d07f2..f9c2f13 100644 --- a/docs/QUALITY_AUDIT.md +++ b/docs/QUALITY_AUDIT.md @@ -74,7 +74,7 @@ Rev2D creates eye-white, iris, upper/lower lid, lash, crease and foreground-hair ## 6. Next milestones 1. Wrap Rev2D drawn-variant requests and imports as guarded PuppetControl CLI operations. Require explicit user/artwork permission and renewed review. No new mesh engine. -2. **Basic pixel-QA probe implemented** for same-resolution PNG pairs, with optional eye/face region and opaque-border transparency checks. Fresh local 512×640 Hana rest/source RGB mean error 0.0002/255; eye crop rest/closed RGB mean error 33.5879/255 (36.4392% changed), with 0 new transparent border pixels. Full methodology and caution in [PIXEL_QA.md](PIXEL_QA.md). Multi-frame discontinuity scans and meaningful aesthetic review remain open. +2. **Pixel QA extended to ordered sequences:** 2–60 matching PNGs, optional eye/face ROI, declared hard-cut exemptions and actual parameter samples. The measured 512×640 Hana 13-frame blink probe at 30fps returned to its exact first image, but two intended-control intervals (3→4 and 8→9) yielded *identical* consecutive images, indicating a plateau. This is consistent with Rev2D's documented full-strength `eyes_half` sprite interval (`EyeOpen` 0.3–0.6), so it is **not** evidence of an unexpected renderer bug. The report flagged both intervals for human review, and no abrupt-change candidates were flagged. Manual inspection of the local contact sheet found overlapping outlines in intermediate blink frames (not captured by generic spike detection). See [PIXEL_QA.md](PIXEL_QA.md). Still **not** visually approved. 3. Build a diverse rights-cleared benchmark matrix: eye size, hair over face, accessories/glasses, different skin tones and line styles, opaque versus transparent backgrounds and constrained 3/4 views. Track success/failure on each, not just demos. 4. Only after a reliable single actor: improve body-pose transitions, synchronized multi-actor shots and user-facing authoring polish. diff --git a/src/character/frame-sequence-qa.ts b/src/character/frame-sequence-qa.ts new file mode 100644 index 0000000..a5f291e --- /dev/null +++ b/src/character/frame-sequence-qa.ts @@ -0,0 +1,171 @@ +/** + * Time-ordered PNG motion triage. + * + * Adjacent-frame pixel deltas reveal candidates for *manual* jump inspection. + * A legitimate fast blink, mouth accent, camera cut or hard pose edit can yield + * a high score; the report never certifies or rejects animation quality. + */ +import { AvatarWorkflowError } from "./avatar.js"; +import { comparePngFiles, type PixelComparison, type PixelRegion } from "./pixel-qa.js"; + +export const MAX_SEQUENCE_FRAMES = 60; + +export interface SequenceOptions { + region?: PixelRegion; + /** Transition i→i+1 is editorially intentional and not a surprise-jump candidate. */ + cutAfter?: readonly number[]; + /** Optional expected control sampled at each frame; used only for motion-stall triage. */ + controlValues?: readonly number[]; +} + +export interface SequencePair { + from: number; + to: number; + intentionalCut: boolean; + rgbMeanAbsoluteError: number; + changedFraction: number; + unexpectedTransparentBorderPixels: number; + controlDelta: number | null; + /** Triage only. Do not treat false as proof that an animation is smooth. */ + abruptChangeCandidate: boolean; + motionStallCandidate: boolean; +} + +export interface SequenceQaReport { + frameCount: number; + pairCount: number; + region: PixelRegion; + medianPairError: number; + medianAbsoluteDeviation: number; + reviewThreshold: number; + pairs: SequencePair[]; + possibleAbruptChanges: number[]; + possibleMotionStalls: number[]; + controlValuesProvided: boolean; + firstToLast: { + rgbMeanAbsoluteError: number; + changedFraction: number; + unexpectedTransparentBorderPixels: number; + }; + note: string; +} + +function median(values: readonly number[]): number { + const ordered = [...values].sort((a, b) => a - b); + const middle = Math.floor(ordered.length / 2); + return ordered.length % 2 === 1 + ? ordered[middle]! + : (ordered[middle - 1]! + ordered[middle]!) / 2; +} + +function finiteMetric(comparison: PixelComparison): void { + if (!Number.isFinite(comparison.rgbMeanAbsoluteError) || + !Number.isFinite(comparison.changedFraction) || + comparison.rgbMeanAbsoluteError < 0 || comparison.rgbMeanAbsoluteError > 255 || + comparison.changedFraction < 0 || comparison.changedFraction > 1) { + throw new AvatarWorkflowError("SEQUENCE_QA_METRIC_INVALID", "Pixel comparisons must return bounded numeric metrics."); + } +} + +function checkSequenceOptions(frameCount: number, options: SequenceOptions): Set { + if (frameCount < 2 || frameCount > MAX_SEQUENCE_FRAMES || !Number.isSafeInteger(frameCount)) { + throw new AvatarWorkflowError("SEQUENCE_QA_LENGTH_INVALID", "Supply 2–60 ordered frames."); + } + const cuts = options.cutAfter ?? []; + if (options.controlValues && (options.controlValues.length !== frameCount || + options.controlValues.some((value) => !Number.isFinite(value) || Math.abs(value) > 1_000_000))) { + throw new AvatarWorkflowError("SEQUENCE_QA_CONTROL_VALUES_INVALID", + "Control samples must contain one finite value for each frame."); + } + if (cuts.some((i) => !Number.isSafeInteger(i) || i < 0 || i >= frameCount - 1) || + new Set(cuts).size !== cuts.length) { + throw new AvatarWorkflowError("SEQUENCE_QA_CUTS_INVALID", + "Each intentional cut must be a unique zero-based frame index before the last frame."); + } + return new Set(cuts); +} + +function compactMetric(comparison: PixelComparison) { + return { + rgbMeanAbsoluteError: comparison.rgbMeanAbsoluteError, + changedFraction: comparison.changedFraction, + unexpectedTransparentBorderPixels: comparison.unexpectedTransparentBorderPixels, + }; +} + +/** + * Summarize fixed-size image comparisons. Exclude *declared* editorial cuts + * from baseline statistics; don't mislabel them as unexpected pops. + * In a sequence without at least 3 comparable non-cut pairs, classify no + * abrupt-change candidates (the evidence is too sparse). + */ +export function analyzeSequenceComparisons( + pairs: readonly PixelComparison[], firstToLast: PixelComparison, + options: SequenceOptions = {}, +): SequenceQaReport { + const cuts = checkSequenceOptions(pairs.length + 1, options); + for (const comparison of [...pairs, firstToLast]) finiteMetric(comparison); + const dims = pairs[0]!; + const width = dims.width; + const height = dims.height; + const region = dims.region; + const sameRegion = (a: PixelRegion, b: PixelRegion) => + a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height; + if ([...pairs, firstToLast].some((comparison) => + comparison.width !== width || comparison.height !== height || !sameRegion(comparison.region, region))) { + throw new AvatarWorkflowError("SEQUENCE_QA_DIMENSIONS_MISMATCH", + "Comparisons must share the same exact frame dimensions and selected region."); + } + + const comparable = pairs + .map((comparison, i) => ({ i, score: comparison.rgbMeanAbsoluteError })) + .filter(({ i }) => !cuts.has(i)).map(({ score }) => score); + const center = comparable.length ? median(comparable) : 0; + const mad = comparable.length ? median(comparable.map((score) => Math.abs(score - center))) : 0; + + // Avoid zero-noise score jumps and render compression artifacts. + // Stronger than typical adjacent variation, not a quality certification. + const threshold = Number(Math.min(255, Math.max(8, center * 2.5, center + 6 * mad)).toFixed(4)); + const pairReports: SequencePair[] = pairs.map((comparison, from) => ({ + from, to: from + 1, intentionalCut: cuts.has(from), + ...compactMetric(comparison), + controlDelta: options.controlValues + ? Number(Math.abs(options.controlValues[from + 1]! - options.controlValues[from]!).toFixed(6)) + : null, + motionStallCandidate: !!options.controlValues && + Math.abs(options.controlValues[from + 1]! - options.controlValues[from]!) > 0.000001 && + comparison.rgbMeanAbsoluteError <= 0.05 && comparison.changedFraction <= 0.001, + abruptChangeCandidate: comparable.length >= 3 && !cuts.has(from) && + comparison.rgbMeanAbsoluteError > threshold, + })); + return { + frameCount: pairs.length + 1, + pairCount: pairs.length, + region, + medianPairError: Number(center.toFixed(4)), + medianAbsoluteDeviation: Number(mad.toFixed(4)), + reviewThreshold: threshold, + pairs: pairReports, + possibleAbruptChanges: pairReports.filter((p) => p.abruptChangeCandidate).map((p) => p.from), + possibleMotionStalls: pairReports.filter((p) => p.motionStallCandidate).map((p) => p.from), + controlValuesProvided: !!options.controlValues, + firstToLast: compactMetric(firstToLast), + note: "Jump and changed-control motion-stall candidates are triage, not proof of defects. Inspect frames, rig clamping, artwork visibility, timing and declared cuts before judging smoothness.", + }; +} + +/** File-by-file scan bounds memory usage; all files must stay inside the current workspace. */ +export async function comparePngSequence( + files: readonly string[], options: SequenceOptions = {}, root = process.cwd(), +): Promise { + checkSequenceOptions(files.length, options); + if (files.some((file) => typeof file !== "string" || !file.toLowerCase().endsWith(".png"))) { + throw new AvatarWorkflowError("SEQUENCE_QA_PNG_REQUIRED", "Every explicitly ordered frame must be a PNG."); + } + const pairs: PixelComparison[] = []; + for (let i = 0; i < files.length - 1; i++) { + pairs.push(await comparePngFiles(files[i]!, files[i + 1]!, options.region, root)); + } + const last = await comparePngFiles(files[0]!, files[files.length - 1]!, options.region, root); + return analyzeSequenceComparisons(pairs, last, options); +} diff --git a/src/cli/index.ts b/src/cli/index.ts index bf2bc75..fa7b3b6 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -15,6 +15,7 @@ import { prepareLandmarkTask, stageLandmarks, acceptLandmarks } from "../charact import { proposeCodexLandmarks, codexExecutableForPlatform } from "../character/vision-codex.js"; import { verifyAvatarBuildRecord } from "../character/provenance.js"; import { comparePngFiles, parsePixelRegionArgument } from "../character/pixel-qa.js"; +import { comparePngSequence } from "../character/frame-sequence-qa.js"; function print(value: unknown): void { process.stdout.write(JSON.stringify(value, null, 2) + "\n"); @@ -29,6 +30,7 @@ function usage(): never { " puppetctl character import --out --id \n" + " puppetctl character build [--force]\n" + " puppetctl character qa compare [--region x,y,w,h]\n" + + " puppetctl character qa sequence ... [--region x,y,w,h] [--cut-after i,j] [--control-values v1,v2,...]\n" + " puppetctl character landmarks assist \n" + " puppetctl character landmarks propose --provider codex\n" + " puppetctl character landmarks stage --proposal \n" + @@ -49,7 +51,7 @@ async function run(): Promise { if (first === "--help" || first === "help") { process.stdout.write( "PuppetControl (experimental)\n" + - "Available: doctor, character inspect/import/build, image pixel-QA compare, landmark assist/propose/stage/accept, audio align, scene validate/plan, single-rig scene preview/render.\n" + + "Available: doctor, character inspect/import/build, image pixel-QA compare/sequence, landmark assist/propose/stage/accept, audio align, scene validate/plan, single-rig scene preview/render.\n" + "Not yet available: reliable one-click landmark inference, animated pose bridges, captions, automatic transcription/TTS.\n", ); return; @@ -74,7 +76,7 @@ async function run(): Promise { ffmpegAvailable: ffmpegFound, rev2dAvailable, codexCliAvailable, - availableFeatures: ["character-inspect", "scene-validate", "transition-policy", "rhubarb-cue-parsing", "rhubarb-align-if-installed", ...(ffmpegFound ? ["png-pixel-qa"] : []), ...(rev2dAvailable ? ["assisted-png-avatar-import", "agent-landmark-proposal-staging", ...(codexCliAvailable ? ["optional-codex-vision-proposals"] : []), "avatar-review-sheets", "single-rig-preview", "single-rig-voice-render", "pose-family-hard-cut-render", "experimental-shot-framing"] : [])], + availableFeatures: ["character-inspect", "scene-validate", "transition-policy", "rhubarb-cue-parsing", "rhubarb-align-if-installed", ...(ffmpegFound ? ["png-pixel-qa", "png-sequence-qa"] : []), ...(rev2dAvailable ? ["assisted-png-avatar-import", "agent-landmark-proposal-staging", ...(codexCliAvailable ? ["optional-codex-vision-proposals"] : []), "avatar-review-sheets", "single-rig-preview", "single-rig-voice-render", "pose-family-hard-cut-render", "experimental-shot-framing"] : [])], unavailableFeatures: ["animated-pose-bridges", "automatic-landmark-placement", "automatic-transcription", "text-to-speech", "multi-audio-mixing"], }); return; @@ -132,6 +134,34 @@ async function run(): Promise { return; } + if (first === "character" && second === "qa" && third === "sequence") { + const args = process.argv.slice(5); + const firstOption = args.findIndex((arg) => arg.startsWith("--")); + const frameFiles = firstOption < 0 ? args : args.slice(0, firstOption); + const flags = firstOption < 0 ? [] : args.slice(firstOption); + if (frameFiles.length < 2 || frameFiles.length > 60) usage(); + let region: ReturnType | undefined; + let cutAfter: number[] = []; + let controlValues: number[] | undefined; + const seen = new Set(); + for (let index = 0; index < flags.length; index += 2) { + const flag = flags[index]; + const value = flags[index + 1]; + if (!flag || !value || seen.has(flag) || !["--region", "--cut-after", "--control-values"].includes(flag)) usage(); + seen.add(flag); + if (flag === "--region") region = parsePixelRegionArgument(value); + else if (flag === "--cut-after") { + if (!/^\d+(,\d+)*$/.test(value)) usage(); + cutAfter = value.split(",").map(Number); + } else { + if (!/^-?(?:\d+(?:\.\d+)?|\.\d+)(,-?(?:\d+(?:\.\d+)?|\.\d+))*$/.test(value)) usage(); + controlValues = value.split(",").map(Number); + } + } + print({ valid: true, report: await comparePngSequence(frameFiles, { ...(region ? { region } : {}), cutAfter, ...(controlValues ? { controlValues } : {}) }) }); + return; + } + if (first === "character" && second === "landmarks" && third) { const [project, ...flags] = process.argv.slice(5); if (!project) usage(); diff --git a/tests/frame-sequence-qa.test.ts b/tests/frame-sequence-qa.test.ts new file mode 100644 index 0000000..1998153 --- /dev/null +++ b/tests/frame-sequence-qa.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; +import { analyzeSequenceComparisons, comparePngSequence, MAX_SEQUENCE_FRAMES } from "../src/character/frame-sequence-qa.js"; +import type { PixelComparison } from "../src/character/pixel-qa.js"; + +const region = { x: 0, y: 0, width: 256, height: 256 }; +function metric(error: number, changedFraction = error / 255): PixelComparison { + return { + width: 256, height: 256, region, comparedPixels: 65_536, + ignoredTransparentPixels: 0, rgbMeanAbsoluteError: error, + changedPixels: Math.round(65_536 * changedFraction), changedFraction, + unexpectedTransparentBorderPixels: 0, note: "manual review only", + }; +} + +describe("time-ordered visual motion QA", () => { + it("reports a single anomalous pair without claiming artistic failure", () => { + const report = analyzeSequenceComparisons( + [metric(3), metric(3), metric(35), metric(3), metric(3)], metric(0), + ); + expect(report.frameCount).toBe(6); + expect(report.medianPairError).toBe(3); + expect(report.possibleAbruptChanges).toEqual([2]); + expect(report.pairs[2]).toMatchObject({ + from: 2, to: 3, abruptChangeCandidate: true, intentionalCut: false, + }); + expect(report.firstToLast.rgbMeanAbsoluteError).toBe(0); + expect(report.possibleMotionStalls).toEqual([]); + expect(report.controlValuesProvided).toBe(false); + expect(report.note).toMatch(/triage/i); + }); + + it("does not flag an explicitly authored editorial cut or poison baseline with its high score", () => { + const report = analyzeSequenceComparisons( + [metric(2), metric(2), metric(175), metric(2), metric(2)], metric(170), { cutAfter: [2] }, + ); + expect(report.medianPairError).toBe(2); + expect(report.possibleAbruptChanges).toEqual([]); + expect(report.pairs[2]).toMatchObject({ + abruptChangeCandidate: false, intentionalCut: true, + }); + }); + + it("refuses to classify a two/three-frame sample as a proven jump", () => { + const small = analyzeSequenceComparisons([metric(210)], metric(210)); + expect(small.possibleAbruptChanges).toEqual([]); + const next = analyzeSequenceComparisons([metric(210), metric(1)], metric(0)); + expect(next.possibleAbruptChanges).toEqual([]); + }); + + it("tolerates smooth regular steps and zero movement", () => { + const regular = analyzeSequenceComparisons( + [metric(11), metric(10), metric(12), metric(11), metric(10)], metric(1), + ); + expect(regular.possibleAbruptChanges).toEqual([]); + const frozen = analyzeSequenceComparisons( + [metric(0), metric(0), metric(0)], metric(0), + ); + expect(frozen.possibleAbruptChanges).toEqual([]); + expect(frozen.firstToLast.changedFraction).toBe(0); + }); + + it("flags unchanged frames when an explicitly sampled control moved", () => { + const report = analyzeSequenceComparisons( + [metric(5), metric(0), metric(5), metric(0)], metric(0), + { controlValues: [1, 0.8, 0.6, 0.4, 0.2] }, + ); + expect(report.controlValuesProvided).toBe(true); + expect(report.possibleMotionStalls).toEqual([1, 3]); + expect(report.pairs[1]).toMatchObject({ + from: 1, to: 2, motionStallCandidate: true, controlDelta: 0.2, + }); + const noChangeExpected = analyzeSequenceComparisons( + [metric(0), metric(0), metric(0)], metric(0), + { controlValues: [1, 1, 1, 1] }, + ); + expect(noChangeExpected.possibleMotionStalls).toEqual([]); + }); + + it("refuses short, non-finite or unrelated control arrays", () => { + expect(() => analyzeSequenceComparisons([metric(0), metric(0)], metric(0), + { controlValues: [1, 0.8] })).toThrow(/one finite value/); + expect(() => analyzeSequenceComparisons([metric(0), metric(0)], metric(0), + { controlValues: [1, NaN, 0] })).toThrow(/one finite value/); + expect(() => analyzeSequenceComparisons([metric(0), metric(0)], metric(0), + { controlValues: [1, Infinity, 0] })).toThrow(/one finite value/); + }); + + it("rejects malformed and duplicate cut indexes, excessive sequences and inconsistent regions", () => { + expect(MAX_SEQUENCE_FRAMES).toBe(60); + for (const cuts of [[2, 2], [-1], [4], [1.5], [NaN]]) { + expect(() => analyzeSequenceComparisons([metric(0),metric(0),metric(0)], metric(0), { cutAfter: cuts })) + .toThrow(/cut/i); + } + expect(() => analyzeSequenceComparisons([], metric(0))).toThrow(/2–60/); + expect(() => analyzeSequenceComparisons(Array.from({ length: 60 }, () => metric(0)), metric(0))) + .toThrow(/2–60/); + const other = { ...metric(4), region: { x: 1, y: 0, width: 255, height: 256 } }; + expect(() => analyzeSequenceComparisons([metric(1), other, metric(1)], metric(0))) + .toThrow(/same exact/); + }); + + it("refuses out-of-bounds and nonfinite comparison metrics", () => { + for (const bad of [NaN, Infinity, -1, 256]) { + expect(() => analyzeSequenceComparisons([metric(0), metric(bad), metric(0)], metric(0))) + .toThrow(/bounded numeric metrics/); + } + expect(() => analyzeSequenceComparisons([metric(0, 1.1)], metric(0))) + .toThrow(/bounded numeric metrics/); + }); + + it("requires explicitly ordered PNG filenames before starting external decoding", async () => { + await expect(comparePngSequence(["a.jpg", "b.png"])) + .rejects.toMatchObject({ code: "SEQUENCE_QA_PNG_REQUIRED" }); + await expect(comparePngSequence(["a.png"])) + .rejects.toMatchObject({ code: "SEQUENCE_QA_LENGTH_INVALID" }); + }); +});