From b61413185ca09f4e78ad7a138f9152c456009608 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 09:38:56 +0100 Subject: [PATCH 01/10] chore: local performance benchmark harness benchmark/: fixtures served with simulated latency, browsers launched by the harness, interleaved variants, CPU and memory of the browser process tree read from /proc. Experiments: wait strategies, browser binaries, launch flags, screenshot and pdf encoders, gif pipeline, concurrency, CDP transport, the real service over HTTP, and the public applications of a data-fair instance. Findings and recommendations in benchmark/FINDINGS.md. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + benchmark/FINDINGS.md | 341 +++++++++++++++++++++++++++ benchmark/README.md | 76 ++++++ benchmark/experiments/args.ts | 45 ++++ benchmark/experiments/browser.ts | 70 ++++++ benchmark/experiments/common.ts | 23 ++ benchmark/experiments/concurrency.ts | 90 +++++++ benchmark/experiments/console.ts | 36 +++ benchmark/experiments/context.ts | 81 +++++++ benchmark/experiments/e2e.ts | 137 +++++++++++ benchmark/experiments/gif.ts | 108 +++++++++ benchmark/experiments/hotspot.ts | 90 +++++++ benchmark/experiments/index.ts | 15 ++ benchmark/experiments/output.ts | 72 ++++++ benchmark/experiments/real.ts | 107 +++++++++ benchmark/experiments/transport.ts | 41 ++++ benchmark/experiments/wait.ts | 60 +++++ benchmark/fixtures/anim.html | 34 +++ benchmark/fixtures/app.html | 54 +++++ benchmark/fixtures/console.html | 13 + benchmark/fixtures/heavy.html | 23 ++ benchmark/fixtures/long.html | 31 +++ benchmark/fixtures/settle.html | 68 ++++++ benchmark/fixtures/static.html | 10 + benchmark/index.ts | 23 ++ benchmark/lib/browsers.ts | 45 ++++ benchmark/lib/capture.ts | 155 ++++++++++++ benchmark/lib/pngjs.d.ts | 4 + benchmark/lib/proc.ts | 72 ++++++ benchmark/lib/runner.ts | 96 ++++++++ benchmark/lib/server.ts | 68 ++++++ benchmark/lib/stats.ts | 24 ++ benchmark/results/.gitkeep | 0 package.json | 3 +- 34 files changed, 2116 insertions(+), 1 deletion(-) create mode 100644 benchmark/FINDINGS.md create mode 100644 benchmark/README.md create mode 100644 benchmark/experiments/args.ts create mode 100644 benchmark/experiments/browser.ts create mode 100644 benchmark/experiments/common.ts create mode 100644 benchmark/experiments/concurrency.ts create mode 100644 benchmark/experiments/console.ts create mode 100644 benchmark/experiments/context.ts create mode 100644 benchmark/experiments/e2e.ts create mode 100644 benchmark/experiments/gif.ts create mode 100644 benchmark/experiments/hotspot.ts create mode 100644 benchmark/experiments/index.ts create mode 100644 benchmark/experiments/output.ts create mode 100644 benchmark/experiments/real.ts create mode 100644 benchmark/experiments/transport.ts create mode 100644 benchmark/experiments/wait.ts create mode 100644 benchmark/fixtures/anim.html create mode 100644 benchmark/fixtures/app.html create mode 100644 benchmark/fixtures/console.html create mode 100644 benchmark/fixtures/heavy.html create mode 100644 benchmark/fixtures/long.html create mode 100644 benchmark/fixtures/settle.html create mode 100644 benchmark/fixtures/static.html create mode 100644 benchmark/index.ts create mode 100644 benchmark/lib/browsers.ts create mode 100644 benchmark/lib/capture.ts create mode 100644 benchmark/lib/pngjs.d.ts create mode 100644 benchmark/lib/proc.ts create mode 100644 benchmark/lib/runner.ts create mode 100644 benchmark/lib/server.ts create mode 100644 benchmark/lib/stats.ts create mode 100644 benchmark/results/.gitkeep diff --git a/.gitignore b/.gitignore index 5ccbe95..842bb3d 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ docs/superpowers/ .superpowers/ .playwright-mcp/ +benchmark/results/* +!benchmark/results/.gitkeep diff --git a/benchmark/FINDINGS.md b/benchmark/FINDINGS.md new file mode 100644 index 0000000..7816cb8 --- /dev/null +++ b/benchmark/FINDINGS.md @@ -0,0 +1,341 @@ +# capture performance investigation, 2026-11 + +Scope: how the service orchestrates captures, how it drives the browser through puppeteer, which +browser it embeds, and the alternatives at each of these levels. Every claim marked +**measured** comes from an experiment of this harness (`npm run benchmark -- `, see +[README.md](./README.md)). Claims marked **code** come from reading the code. Claims marked +**not measured** are reasoned but untested. + +Machine: 16 cores, 31 GB of RAM, a shared desktop under load (other apps running). Browsers: +google-chrome-stable 150 (the one the Docker image installs), Chrome for Testing 156 and +chrome-headless-shell 156. Treat absolute numbers as indicative and trust the ratios. The +tables below are from the 8-run pass. Raw JSON is in `results/` (git-ignored). + +## Status (2026-11-09) + +- **Done on this branch**: the wait strategy (§1, network idle then visual quiet, capped), a + `capture-no-trigger` warning for pages that do not call `window.triggerCapture`, the PNG + fast encoding above `pngFastEncodingPixels` and for full-page captures (§2), and the fixes of + `type=jpg` (PNG bytes labelled `image/jpeg`) and of `type=gif` on a page without + animation (PNG bytes labelled `image/gif`). +- **Approved, to do**: chrome-headless-shell (§3), the GIF encoding in a piscina worker (§5), + the console listener behind `DEBUG` (§6), skipping abandoned requests and bounding the + queue (§7). +- **Declined**: untagged PDFs (§4). Accessibility comes first. + +## TL;DR + +Where a capture's time goes today, for a typical data-fair page that does not call +`triggerCapture`: + +``` +newPage+setup ~35-90 ms (negligible) +networkidle0 1000-2100 ms <- Chrome's idleness detector, not 500 ms +fixed sleep 1000 ms <- "for safety" +screenshot 30-330 ms (PNG encoded on the browser's main thread) +``` + +Ranked by gain for effort: + +| # | change | gain (measured) | cost / risk | +|---|---|---|---| +| 1 | replace `networkidle0` + 1 s with puppeteer's `waitForNetworkIdle` + 2 animation frames (+ a configurable short settle) | static page 2.0 s → 0.52 s, app page 3.1 s → 1.1 s; a context is held ~2 s less, so ~2× throughput at the same concurrency on I/O-bound pages | pages that render >0.5 s after their last response without any network activity are captured too early. Today they are only caught by accident (see §1). A product decision | +| 2 | `optimizeForSpeed: true` on PNG screenshots | encoding ×2.5 faster, browser CPU −60%; under CPU-bound concurrency **+40% throughput, −32% p50** (PNG encoding serializes on the browser's main thread) | files 1.1-1.9× bigger | +| 3 | switch to **chrome-headless-shell** (Debian `chromium-headless-shell`) | idle memory −51% (145 vs 296 MB), launch 39 vs 155 ms, browser CPU −20 to −43% per capture (PDF ×1.6 faster), +30% CPU-bound throughput; multi-arch (arm64), 230 MB installed vs 416 MB, security updates through apt | a different build (Chromium, not Google Chrome). Needs the test suite and a visual check of real pages | +| 4 | ~~PDF `tagged: false` (puppeteer defaults to `true`)~~ declined | PDF ×2 faster, files **÷9** (1.2 MB → 137 KB for 30 sections) | tagged PDF is what makes a PDF accessible (screen readers, RGAA). A product decision: maybe opt-in per request | +| 5 | move the GIF encoding off the main thread (worker thread, or sharp/libvips) | today a 30-frame GIF burns ~4.3 s of main-thread CPU and stalls every other request (probe p90 83 ms, max 131 ms) | small code change. sharp would be a new native dependency | +| 6 | attach the `console` listener only when `DEBUG` is set | node CPU ÷4 on a page logging 500 messages | none | +| 7 | drop queued work whose client left; bound the queue | after 30 abandoned requests, a 0.7 s capture answered in **18 s** (§7) | small | + +Not worth it (measured): launch flags (none helps, the commented-out swiftshader option costs +×3.7 CPU), websocket vs pipe CDP transport (no difference), a fresh context vs the pool (no +difference), the sandbox (no measurable cost). Not worth it (reasoned): Playwright (same engine, +same CDP), Lightpanda (no rendering engine, so no screenshots or PDFs), Firefox through BiDi. + +## 1. The wait strategy: most of the latency + +**code**: `waitForPage` (`api/utils/page.ts`) runs `page.goto(target, { waitUntil: 'networkidle0' })` +raced against `window.triggerCapture`. Then it looks up 2 metas, and when neither is set it +sleeps 1000 ms "for safety". + +**measured** (`wait`): `networkidle0` is not "500 ms without requests". It maps to Chrome's own +`networkIdle` lifecycle event, fired by Blink's `IdlenessDetector`. That detector only checks for +quiet when a main-thread task runs or when its watchdog timer fires. It also pushes the quiet +window forward by the time spent in tasks. So the event lands well after the nominal 500 ms, at a +moment that depends on page activity: + +Median wait in ms (8 runs), and the share of captures where the page had finished rendering: + +| page | current | idle0 (no +1s) | idle0-300 | idle200-raf | **idle500-raf** | idle2 | +|---|---|---|---|---|---|---| +| static (no sub-resource) | 2014, 8/8 | 1012, 8/8 | 1313, 8/8 | 223, 8/8 | **524, 8/8** | 2013, 8/8 | +| app (3 API calls of 150 ms) | 3114, 8/8 | 2113, 8/8 | 2413, 8/8 | 781, 8/8 | **1080, 8/8** | 1669, 8/8 | +| app, render 300 ms after the last response | 3115, 8/8 | 2112, 8/8 | 2413, 8/8 | 780, **0/8** | **1078, 8/8** | 1669, 8/8 | +| app, render 800 ms after the last response | 2381, 8/8 | 1377, 8/8 | 1678, 8/8 | 782, **0/8** | **1075, 0/8** | 1670, 8/8 | +| app, API calls of 400 ms | 3113, 8/8 | 2109, 8/8 | 2414, 8/8 | 1533, 8/8 | **1830, 8/8** | 1914, 8/8 | +| app calling triggerCapture | 572 | 573 | 573 | 571 | 574 | 574 | + +Oddity that shows the detector is not a network timer: the page that renders 800 ms late reaches +`networkidle0` *sooner* (1377 ms) than the one that renders 300 ms late (2112 ms). + +Reading: +- the floor of the service is **2 s on a static page with no sub-resource**. `idle500-raf` + (puppeteer-side `waitForNetworkIdle({ idleTime: 500 })` after `load`, then 2 animation frames) + does the same page in ~0.55 s; +- on the app page (data rendered ~0.55 s after navigation), the service answers in 2.7-3.1 s. + `idle500-raf` answers in ~1.1 s and the render is complete; +- `late300`/`late800` render 300/800 ms after their last response, with no network in between. + The current strategy catches them through Chrome's detector delay plus the 1 s. The fast + strategies miss renders later than their idle window. **This is the real trade-off.** Today's + margin is accidental and unbounded (up to 2+ s). A deliberate margin would be a config value, + e.g. `idleTime` 500 ms plus a configurable settle, with the existing `df:capture-delay` meta + and `triggerCapture` for pages that need more; +- `networkidle2` is not a fix: it fires earlier, but the 1 s sleep remains. + +**measured** (`e2e`, the real service): app page at concurrency 5, the service's own timer steps: +`wait1-network-idle` 2110 ms + `wait2-1s` 1000 ms out of a 3.2 s total; with `triggerCapture` +the total is 0.69 s. + +### Synthetic risks after the last response (`wait`, fixtures/settle.html, 4 runs) + +| page | current | idle500-raf | idle500-500 | **quiet** (idle 500 ms, then 200 ms of visual quiet, cap 2 s) | +|---|---|---|---|---| +| canvas chart animated 1 s by requestAnimationFrame | 1699 ms, 4/4 | 692, 0/4 | 1169, 0/4 | **1421, 4/4** | +| canvas chart animated 2.5 s | 1694, **0/4** | 690, 0/4 | 1170, 0/4 | 2676 (cap), 3/4 | +| CSS transition 1 s | 1695, 4/4 | 691, 0/4 | 1171, 1/4 | **1424, 4/4** | +| endless spinner next to the content | 1676, 4/4 | 688, 4/4 | 1170, 4/4 | **672, 4/4** (endless animations ignored) | +| web font loaded late | 3009, 4/4 | 1289, 4/4 | 1782, 4/4 | **1281, 4/4** | +| content inserted by a 800 ms setTimeout | 1971, 4/4 | 684, 0/4 | 1170, 4/4 | 671, **0/4** | + +`quiet` is faster than today and safer on animations: an animation longer than today's +accidental margin is captured half-drawn today. Its blind spot is a render delayed by a plain +timer after a quiet period. The `capture-no-trigger` warning exists for that: such pages should +call `triggerCapture`. + +### Real pages (`real`: the 66 public applications of opendata.koumoul.com, 1050×450) + +Each capture is compared to a reference, today's strategy + 3 s, and to a second reference that +measures the page's own noise. +- 54 of 66 applications call `triggerCapture` (app-charts, app-dashboards, maps, catalog...): + the strategy barely changes them. +- the 12 without any signal: app-timelines, list-details, periodic-series, data-fair-quizz, + data-fair-locate-game, data-fair-sort-game, app-carousel. `quiet` takes 1.0-1.3 s vs 2.0-2.3 s, + with no difference to the reference, except: + - app-carousel rotates its slides, so every strategy differs from the reference (noise 90%); + - one list-details app has a map whose tiles load and decode in web workers, invisible to + every page-level signal. Today's capture is half-loaded too (5.5% off the reference), + `quiet` captures the map empty (25%). The fix belongs in the application: call + `triggerCapture` on the map's `idle` event; +- a semantic signal was tried and rejected: waiting while a visible `role=progressbar` is present. + The same list-details page shows 19 Vuetify progress bars that are value bars, not loaders. + +**A/B of the real service** (this branch vs master, 2 interleaved captures per application and +side, public applications): + +| | master | this branch | pixel diff | +|---|---|---|---| +| 9 applications without triggerCapture, median | 2093 ms | **1075 ms** | 0% on all | +| 5 applications with triggerCapture, median | 1345 ms | 1334 ms | 0% on all | + +### Throughput + +Since a capture holds its browser context while it waits, the wait also caps throughput: +`concurrency` captures every ~3.2 s = 1.5/s at the default concurrency of 5 (**measured**). + +## 2. Screenshot encoding, and the single-browser ceiling + +**measured** (`output`): + +| page | png (today) | png `optimizeForSpeed` | jpeg q80 | webp q80 | +|---|---|---|---|---| +| app 800×450 | 25 ms, 44 KB | 23 ms, 83 KB | 23 ms, 40 KB | 43 ms, 14 KB | +| heavy 1920×1080 | 302 ms, 2.5 MB | **113 ms**, 2.7 MB | **52 ms**, 675 KB | 232 ms, 570 KB | +| long page, full height at 1280 | 552 ms, 1.8 MB | **267 ms**, 2.7 MB | 306 ms, 2.8 MB | 1110 ms, 1.6 MB | + +On small viewports encoding is negligible. It matters for large viewports and full-page +captures (`height=auto`). + +**measured** (`concurrency`, `hotspot`): on a paint-heavy page, one browser stops scaling at about +2 cores, whatever the concurrency (c10 and c20 give the same throughput). Splitting c10 over 2 +browsers gives +40% throughput and −30% p50. The CPU split by process and by thread shows the +serial bottleneck: the **browser process main thread** (`CrBrowserMain`) at ~0.7-0.85 of a +core. Renderers use ~0.7 cores in total, the GPU process 0.1. `Page.captureScreenshot` encodes +the image in the browser process. Proof: on the same browser at c10, `optimizeForSpeed` gives +5.6/s vs 4.0/s (+40%), and jpeg-80 gives 6.2/s (+55%). + +| load | c1 | c2 | c5 (default) | c10 | c20 | c10 over 2 browsers | +|---|---|---|---|---|---|---| +| app, current wait (I/O-bound): captures/s | 0.3 | 0.6 | 1.5 | 2.7 | 4.6 | 2.8 | +| p50 | 3.2 s | 3.3 s | 3.4 s | 3.5 s | 3.7 s | 3.5 s | +| cores used / peak PSS | 0.1 / 320 MB | 0.2 / 351 | 0.4 / 443 | 0.9 / 623 | 1.3 / 942 | 0.8 / 855 | +| heavy 1280×800 (CPU-bound): captures/s | 0.8 | 1.4 | 2.9 | 3.9 | **3.9** | **5.3** | +| p50 | 1.25 s | 1.42 s | 1.52 s | 2.35 s | 4.98 s | 1.58 s | +| cores used / peak PSS | 0.3 / 423 MB | 0.7 / 500 | 1.4 / 720 | 2.1 / 1097 | 2.1 / 1860 | 2.9 / 1339 | + +`hotspot` (heavy, c10): browser process 0.8-0.9 core (its main thread alone ~0.7), renderers +0.7 cores in total, GPU process 0.1. With chrome-headless-shell: 4.9/s, p50 1.74 s. + +Consequences: +- for light pages, the service is I/O-bound (waiting): throughput scales linearly with + concurrency, at ~25-30 MB of PSS per extra concurrent page. The default concurrency of 5 uses + 0.4 core. With a shorter wait (§1), the same concurrency gives ~2× the throughput; +- for heavy pages, scale with more browsers (more pods, or N browsers per pod with the contexts + pool sharded), not with more contexts in one browser; +- `optimizeForSpeed` raises the ceiling of a browser by ~40%. The bigger files matter for + thumbnails that are stored; jpeg/webp could be an option for callers that accept it. Beware, **code**: + `type=jpeg` does not work today. The router sets the response type but calls + `page.screenshot()` without `type`, so it returns PNG bytes labelled `image/jpeg`. + +## 3. The browser binary + +**measured** (`browser`): + +| | chrome-stable 150 (today) | same, --no-sandbox | Chrome for Testing 156 | **chrome-headless-shell 156** | +|---|---|---|---|---| +| launch | 155 ms | 147 ms | 141 ms | **39 ms** | +| idle memory (PSS) | 296 MB | 299 MB | 269 MB | **145 MB** | +| processes | 11 | 10 | 8 | 6 | +| app png: total / browser CPU | 639 ms / 240 ms | 637 / 230 | 651 / 230 | 637 / **150** | +| heavy 1920×1080 png: total / CPU | 1372 / 660 | 1372 / 670 | 1373 / 700 | 1337 / **535** | +| long PDF: pdf() time / CPU | 267 / 595 | 267 / 600 | 291 / 625 | **169** / **340** | +| memory after the captures | 266 MB | 266 MB | 662 MB | **149 MB** | + +Under CPU-bound concurrency (c10, heavy page, `hotspot`): 5.0-5.1/s with the shell vs 3.8/s, +p50 1.6 s vs 2.2-2.3 s. + +- **chrome-headless-shell** is the "old headless" Chromium, shipped as a separate binary since + Chrome 132 removed old headless from the main binary. It has no browser UI layer. It uses half + the memory at idle, 15-45% less browser CPU per capture, and handles ~30% more throughput when + CPU-bound (**measured**, `hotspot`). Page latency is the same, because the waits dominate; +- the 150 → 156 version change (chrome-stable vs CfT) explains little. The headless mode + (CfT 156 vs shell 156) explains most of the difference; +- the sandbox has no measurable cost (chrome-stable vs chrome-stable-nosandbox); +- packaging: Debian trixie (the base of the image) ships `chromium-headless-shell` 156 for amd64 + and arm64, 230 MB installed vs 416 MB for `/opt/google/chrome`, with fewer dependencies (no + GTK, cups…), and updates through the same `apt-get upgrade` as the rest of the image. The + current Dockerfile pins the Google repo to `arch=amd64` while it handles arm64 for dumb-init, + so an arm64 image has no browser today. Puppeteer drives it with `headless: 'shell'`; +- risks (**not measured**): Chromium is not Google Chrome (no proprietary codecs, which do not + matter for captures), and old headless has small behavioural differences. The test suite and + a visual diff of a sample of real data-fair pages should gate the switch; +- alternatives: + - **Playwright**: same Chromium, same CDP, and its default headless is already the headless + shell. No performance gain to expect over puppeteer + shell. The setup calls measured below + cost < 1 ms each, so the driver is not a cost; + - **Lightpanda**: a fast headless browser without a rendering engine. It cannot produce + screenshots or PDFs (placeholder images per its docs), so it is out of scope; + - **Firefox through WebDriver BiDi** (puppeteer supports it): a different rendering of the + same pages, no known performance advantage. Not measured; + - dedicated PDF engines (WeasyPrint, Gotenberg): Gotenberg is Chromium too. WeasyPrint does not + run JavaScript, and data-fair pages are JS applications. Out of scope. + +## 4. PDF + +**measured** (`output`): + +| document | default (tagged) | `tagged: false` | `outline: true` | footer template | +|---|---|---|---|---| +| 30 sections (~40 pages) | 245 ms, 1222 KB | **119 ms, 137 KB** | 251 ms, 1247 KB | 282 ms, 1233 KB | +| 120 sections | 1026 ms, 4.9 MB | **422 ms, 489 KB** | 1064 ms, 5.0 MB | 1169 ms, 4.9 MB | + +Puppeteer defaults to `tagged: true` and also launches Chrome with `--export-tagged-pdf`. +Tagging builds the accessibility structure of the document. It doubles the PDF time and makes +the file ~9× bigger. `outline` costs nothing. The footer template costs +10-15%. + +## 5. Animated GIF + +**code**: `api/utils/animation.ts` takes a PNG screenshot per frame, decodes it in pure JS +(get-pixels/pngjs), quantizes and encodes it in pure JS (gif-encoder, NeuQuant), then runs +gifsicle. All of it runs on the service's main thread. + +**measured** (`gif`, 800×450, 30 frames): + +| variant | total | screenshots | JS decode | JS encode | gifsicle | node CPU | max event loop stall | size | +|---|---|---|---|---|---|---|---|---| +| today (png `optimizeForSpeed`, get-pixels, gif-encoder) | 4.5 s | 1.43 s | 0.37 s | **2.44 s** | 0.26 s | **4.2 s** | **138 ms** | 3.4 MB | +| jpeg-90 frames | 4.4 s | 1.03 s | 0.38 s | 2.72 s | 0.27 s | 4.0 s | 169 ms | 3.3 MB | +| sharp, default options | 7.1 s | 1.76 s | - | 5.03 s (libvips threads) | 0.28 s | 5.2 s (threads) | **12 ms** | 3.7 MB | + +**measured** (`e2e`): during one GIF capture, a probe requesting `api-docs.json` every 20 ms sees +p50 1 ms but p90 83 ms and max 131 ms. Every request of the service (other captures, metrics, +health checks) stalls for the length of a frame encoding. + +- the browser side is ~1/3 of the time. Node-side decode + quantize + encode is ~2/3, all on + the main thread; +- jpeg frames do not help: a cheaper screenshot, but a slower JS decode; +- sharp (libvips + libimagequant + cgif) with `effort: 1, dither: 0` uses about the same CPU + as today (2.3 s vs 3.0 s for 30 frames, **measured** on saved frames), but in libuv threads. + The event loop stays free (max stall 13 ms vs 139 ms). Its defaults (effort 7, dithering) are + 2× slower than today; +- the cheapest fix with no new dependency is to run the existing decode + encode in a + `worker_thread`. A semaphore like the one of `/inspect` should bound the concurrent GIFs: + each one burns a full core for seconds; +- an animated WebP (sharp, q80) takes ~0.5 s and 359 KB vs ~4.7 s and 3.4 MB for the GIF. That + could be an option for clients that accept it; +- **code**: `maxAnimationFrames` is 1800 ("2 minutes at 15 fps"), but the callback of an + animated capture has a `screenshotTimeout * 2` = 40 s budget. At ~160 ms per 800×450 frame, + that budget allows ~250 frames (~17 s of animation). Larger viewports allow fewer. + +## 6. Plumbing: contexts, pages, setup calls, console, transport + +**measured** (`context`): `newPage` ~30 ms, each setup call (`emulateTimezone`, +`setExtraHTTPHeaders`, `evaluateOnNewDocument`, `exposeFunction`, `setViewport`, request +interception) < 1 ms, the meta lookups ~4 ms, cleanup ~2.5 ms. A fresh incognito context per +capture costs the same as the pool. The pool is fine, and so is the per-capture setup. + +**measured** (`console`): `debugPage` attaches a `console` listener on every page, even when +`DEBUG` is off. The listener does 2 CDP round trips per argument of every message. On a page +logging 500 messages: node CPU 167 ms → 40 ms without it. Latency barely moves, because the work +lands after the response, but the CPU is paid on the service's main thread. Fix: attach it only +when `debug.enabled`. + +**measured** (`transport`): websocket vs `pipe: true`, on a 1920×3000 PNG and an 800 KB PDF: no +difference. + +**measured** (`args`): `--disable-gpu`, no site isolation, `--renderer-process-limit=2`, +disabling background features: within noise for throughput. No site isolation or `--renderer-process-limit=2` save +10-18% of browser CPU (fewer renderer processes). That is not worth giving up process isolation +in a service that renders arbitrary pages. The +commented-out option of `config/default.cjs` (`--use-angle=swiftshader --in-process-gpu`) costs +**×3.7 CPU**: remove it, so nobody enables it. + +## 7. Orchestration under load + +**code**, `api/utils/page.ts`: +- the contexts pool (`generic-pool`) has no `maxWaitingClients` and no `acquireTimeoutMillis`. + Under overload the queue is unbounded. The capture timeouts only start after the acquisition, + so a queued request can wait indefinitely; +- `/screenshot` and `/print` never check whether their client is still there. `/inspect` does + (`req.socket.destroyed` before rendering). A request abandoned by a proxy timeout (60 s) or by a + user still gets rendered for nobody when its turn comes, and delays the requests behind it. + **measured** (`e2e`, overload scenario): 30 clients give up after 2 s, then one client asks for a page +that takes 0.7 s alone. It gets its answer after **18.0 s**, because the service first renders the +abandoned requests for nobody (concurrency 5, ~3.2 s each); +- `promiseTimeout` never clears its timer: each capture leaves 2 pending timers of 20-40 s. Minor; +- **not measured**: one browser process lives as long as the pod. Chrome's memory tends to grow + over long uptimes, and nothing recycles it except a crash. A restart after N captures or above + a memory threshold (drain the pool, launch a new browser, close the old one) would bound it. + +Suggested shape: check `req.socket.destroyed` before and after the acquisition in `withPage`, +and abort the render when the client closes (`res.on('close')`). Bound the queue +(`maxWaitingClients`, then a 503 that the proxy or client can retry), and give the acquisition a +timeout. + +## 8. Applications to fix + +The `capture-no-trigger` warning (`code: capture-no-trigger`, with the target and what settled the +capture) lists them in production. From the public sample: app-timelines, list-details (on the +map's `idle` event), periodic-series, data-fair-quizz, data-fair-locate-game, +data-fair-sort-game, app-carousel, and infos-territoires (it relies on its `df:capture-delay` +meta, 2 s, without calling `triggerCapture`). + +## 9. Suggested order of work (initial) + +1. Low risk, no product decision: console listener behind `debug.enabled`; `optimizeForSpeed` + on PNG screenshots (if bigger files are acceptable); skip abandoned requests and bound the + queue; GIF encoding in a worker thread plus a GIF semaphore; remove the swiftshader comment. +2. Wait strategy: decide the safety margin (§1), then implement it behind config, A/B it with + `e2e` (`BENCH_E2E_DIR` = a master worktree), and check real data-fair applications. +3. Browser: build an image on `chromium-headless-shell`, run the tests, diff screenshots of real + pages, then roll it out. Revisit `concurrency` per pod with the memory figures above. +4. PDF tagging: product decision (accessibility vs size/time), maybe a `tagged` query parameter. diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..9a0f824 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,76 @@ +# capture benchmark + +A local harness to test performance hypotheses about capture: the wait strategy, the browser +binary and its flags, the output encoders, the GIF pipeline, the concurrency. Findings and +recommendations are in [FINDINGS.md](./FINDINGS.md). + +No dev environment needed. The harness starts its own fixtures server (pages served with +artificial latency, no internet) and its own browsers. Only `e2e` starts the service itself, as a +child process. + +```bash +npm run benchmark # list the experiments +npm run benchmark -- wait --runs=8 # one experiment +npm run benchmark -- all --runs=8 # everything, ~1 hour +npm run benchmark -- concurrency --browser=shell +``` + +`--browser` picks the binary for single-browser experiments: +- `chrome-stable`, the default, is what the Docker image installs (`/usr/bin/google-chrome-stable`); +- `cft` and `shell` are the Chrome for Testing and chrome-headless-shell builds in + `~/.cache/puppeteer`. Install them with `npx puppeteer browsers install chrome-headless-shell`. + +On Ubuntu ≥ 23.10, AppArmor blocks the Chrome sandbox for binaries that have no AppArmor profile. +Only the google-chrome-stable deb installs one. So `cft` and `shell` run with `--no-sandbox`, and +`chrome-stable-nosandbox` gives the like-for-like comparison. + +## Experiments + +| name | question | +|---|---| +| `wait` | latency, and share of complete renders, for each wait strategy after navigation | +| `browser` | chrome-stable vs Chrome for Testing vs chrome-headless-shell: launch, latency, CPU, memory | +| `context` | pooled context vs fresh context vs reused page, and the cost of each page setup call | +| `console` | cost of the console listener that `debugPage` always attaches | +| `output` | screenshot formats (png, png `optimizeForSpeed`, jpeg, webp) and PDF options (tagged, outline, footer) | +| `gif` | per-stage cost of the animated GIF pipeline, and how long it blocks the event loop | +| `concurrency` | throughput, latency, CPU and memory for 1 to 20 concurrent captures, with 1 or 2 browsers | +| `args` | chrome launch flags: CPU, memory and process count at concurrency 5 | +| `transport` | websocket vs pipe CDP transport, on large outputs | +| `e2e` | the real service over HTTP: timer steps under load, event loop blocking during a GIF, overload with abandoned requests | +| `hotspot` | CPU by chrome process type under paint-heavy load, to find what caps a single browser | +| `real` | the wait strategies on the public applications of a data-fair instance, pixel-compared to a settled reference | + +`e2e` options, as environment variables: +- `BENCH_E2E_DIR`: run the service from another checkout, for example a worktree of master, to + A/B two versions of the code; +- `BENCH_E2E_CONFIG`: JSON merged into the service config, for example + `{"puppeteerLaunchOptions":{"headless":"shell","executablePath":"..."}}`. + +`real` options: `BENCH_REAL_URL` (default `https://opendata.koumoul.com/data-fair`) and +`BENCH_REAL_APPS` (comma separated ids; default: up to 2 applications per base application). +It captures one page at a time, to stay gentle with the instance. Suspect captures and their +reference are written to `results/real/`. + +`wait` option: `BENCH_WAIT_PAGES` (comma separated page names) for a subset. + +`gif` option: `BENCH_SHARP` is the path of a sharp install +(`npm i sharp` in a scratch directory, then point to `node_modules/sharp/dist/index.cjs`). +It adds a variant that encodes with libvips, cgif and libimagequant. sharp is not a dependency +of the service. + +## Method + +- Variants are interleaved (A B C, B C A, …), so that drift in the machine's load hits all + variants alike. One warmup iteration per variant is discarded. +- Each cell shows the median, then `[min-max]`, then the ratio to the first row. `LOWER` and + `HIGHER` flag a difference only when the min–max ranges do not overlap and the medians differ by + more than 10%. This is the rule of the data-fair harness. +- Browser CPU is the CPU time of the browser process tree, read from `/proc` (Linux only). Child + processes that already exited are counted through their parent's cumulated fields. Memory is the + PSS of the tree, which does not count shared pages twice the way a sum of RSS would. +- `lib/capture.ts` copies the page lifecycle of `api/utils/page.ts`, with each lever turned + into a parameter. It is kept in sync by hand. `e2e` is the check against the real code. +- Results are printed as markdown tables and saved as JSON in `results/` (git-ignored), tagged + with the commit. +- The dev machine is shared and loaded. Trust the ratios more than the absolute values. diff --git a/benchmark/experiments/args.ts b/benchmark/experiments/args.ts new file mode 100644 index 0000000..30e4431 --- /dev/null +++ b/benchmark/experiments/args.ts @@ -0,0 +1,45 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch } from '../lib/browsers.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import { runLoad } from './concurrency.ts' +import type { Experiment } from './common.ts' + +// H: some launch flags reduce the browser's CPU or memory per capture: no GPU process (software +// raster in the renderer), fewer background features, no site isolation (fewer processes), +// a renderer process limit. Measured at concurrency 5 on the CPU-bound page and on the app page. + +const argVariants: Record = { + default: [], + 'no-sandbox': ['--no-sandbox'], + 'disable-gpu': ['--disable-gpu'], + 'no-site-isolation': ['--disable-site-isolation-trials', '--disable-features=IsolateOrigins,site-per-process'], + 'renderer-limit-2': ['--renderer-process-limit=2'], + 'disable-features': ['--disable-features=Translate,OptimizationHints,MediaRouter,DialMediaRouteProvider,InterestFeedContentSuggestions,CertificateTransparencyComponentUpdater,AutofillServerCommunication,PrivacySandboxSettings4,LensOverlay,PaintHolding', '--disable-component-update', '--no-default-browser-check'], + 'swiftshader-in-process': ['--use-gl=angle', '--use-angle=swiftshader', '--in-process-gpu'] +} + +export const args: Experiment = { + description: 'chrome launch flags: CPU, memory and process count at concurrency 5', + async run (opts) { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + const all: Record = {} + for (const [loadName, load] of Object.entries({ + 'heavy 1280x800': { path: '/fixtures/heavy.html?n=15000', viewport: { width: 1280, height: 800 }, strategy: 'idle0-raf' as const }, + 'app-trigger 800x450': { path: '/fixtures/app.html?trigger=1', viewport: { width: 800, height: 450 }, strategy: 'idle0-raf' as const } + })) { + const results = await runInterleaved(Object.keys(argVariants), Math.max(2, Math.round(opts.runs / 3)), async (name) => { + const l = await launch(variant, { args: argVariants[name] }) + try { + return await runLoad([l], server.url, load, 5, 20) + } finally { + await l.browser.close() + } + }, 0) + all[loadName] = results + report(loadName, results, ['perSec', 'p50Ms', 'cpuMsPerCapture', 'peakPssMB', 'peakProcs']) + } + await server.close() + save('args', all) + } +} diff --git a/benchmark/experiments/browser.ts b/benchmark/experiments/browser.ts new file mode 100644 index 0000000..86a5eb2 --- /dev/null +++ b/benchmark/experiments/browser.ts @@ -0,0 +1,70 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch, browserFootprint, type Launched } from '../lib/browsers.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool } from '../lib/capture.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save, step } from '../lib/runner.ts' +import type { Experiment } from './common.ts' + +// H: chrome-headless-shell (the old headless mode, a separate lighter binary) launches faster, +// renders and encodes faster and uses less memory than the full chrome binary in new headless mode +// that the image installs today. Chrome for Testing in new headless mode separates the effect of +// the version (stable 150 vs CfT 156) from the effect of the headless mode (CfT 156 vs shell 156). + +const tasks = { + 'app-png': { path: '/fixtures/app.html?trigger=1', viewport: { width: 800, height: 450 }, out: 'png' }, + 'heavy-png': { path: '/fixtures/heavy.html', viewport: { width: 1920, height: 1080 }, out: 'png' }, + 'long-pdf': { path: '/fixtures/long.html', viewport: undefined, out: 'pdf' } +} as const + +export const browser: Experiment = { + description: 'chrome-stable vs Chrome for Testing vs chrome-headless-shell: launch, capture time, CPU, memory', + async run (opts) { + const server = await startFixturesServer() + const names = Object.keys(browserVariants) + const launched: Record = {} + const launchResults = await runInterleaved(names, Math.min(opts.runs, 5), async (name) => { + const l = await launch(browserVariants[name]) + const sample = { launchMs: l.launchMs, idlePssMB: browserFootprint(l).pssMB, processes: browserFootprint(l).processes } + await l.browser.close() + return sample + }, 0) + report('launch (fresh process, no page)', launchResults, ['launchMs', 'idlePssMB', 'processes']) + for (const name of names) { + launched[name] = await launch(browserVariants[name]) + console.log(`${name}: ${launched[name].version}`) + } + const pools = Object.fromEntries(names.map(n => [n, createContextPool(() => launched[n].browser.createBrowserContext())])) + const all: Record = { launch: launchResults } + for (const [taskName, task] of Object.entries(tasks)) { + const results = await runInterleaved(names, opts.runs, async (name) => { + const l = launched[name] + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + step('newPage') + const context = await pools[name].acquire() + const page = await context.newPage() + if (task.viewport) await page.setViewport(task.viewport) + step('setup') + await setupPage(page) + step('goto') + await gotoAndWait(page, server.url(task.path), 'idle0-raf') + const t1 = performance.now() + step('output') + const buf = task.out === 'png' ? await page.screenshot() : await page.pdf({ format: 'A4', printBackground: true }) + const t2 = performance.now() + step('clean') + await cleanPage(page) + pools[name].release(context) + await new Promise(resolve => setTimeout(resolve, 100)) // let the renderer exit before reading CPU + return { openMs: t1 - t0, outputMs: t2 - t1, totalMs: t2 - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0, KB: buf.length / 1024 } + }) + all[taskName] = results + report(taskName, results, ['openMs', 'outputMs', 'totalMs', 'browserCpuMs', 'KB']) + } + const footprint = Object.fromEntries(names.map(n => [n, [browserFootprint(launched[n])]])) + report('footprint after the captures above (contexts kept in pool)', footprint, ['pssMB', 'processes']) + for (const name of names) await launched[name].browser.close() + await server.close() + save('browser', all) + } +} diff --git a/benchmark/experiments/common.ts b/benchmark/experiments/common.ts new file mode 100644 index 0000000..614fd77 --- /dev/null +++ b/benchmark/experiments/common.ts @@ -0,0 +1,23 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch, type Launched } from '../lib/browsers.ts' +import type { LaunchOptions } from 'puppeteer' + +export type Opts = { runs: number, browser: string } +export type Experiment = { description: string, run: (opts: Opts) => Promise } + +// a fixtures server and a browser for the duration of fn +export const withEnv = async (opts: Opts, fn: (env: { url: (p: string) => string, l: Launched }) => Promise, launchOptions: LaunchOptions = {}) => { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + if (!variant) throw new Error('unknown browser ' + opts.browser) + const l = await launch(variant, launchOptions) + console.log(`browser: ${variant.name} ${l.version}`) + try { + return await fn({ url: server.url, l }) + } finally { + await l.browser.close() + await server.close() + } +} + +export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) diff --git a/benchmark/experiments/concurrency.ts b/benchmark/experiments/concurrency.ts new file mode 100644 index 0000000..77022d8 --- /dev/null +++ b/benchmark/experiments/concurrency.ts @@ -0,0 +1,90 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch, type Launched } from '../lib/browsers.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool, type WaitStrategy } from '../lib/capture.ts' +import { treeCpuMs, treePssMB, treeSize } from '../lib/proc.ts' +import { aggregate } from '../lib/stats.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Experiment } from './common.ts' +import type { ScreenshotOptions } from 'puppeteer' + +// H: with the current wait strategy a capture is mostly idle time, so throughput scales almost +// linearly with the number of contexts, and the default concurrency (5) leaves the machine idle. +// On a paint-heavy page the browser CPU saturates instead and more concurrency only adds latency. +// Also: one browser vs 2 browsers for the same concurrency. + +type Load = { path: string, viewport: { width: number, height: number }, strategy: WaitStrategy, shot?: ScreenshotOptions } +const loads: Record = { + 'app (I/O bound, current wait)': { path: '/fixtures/app.html', viewport: { width: 800, height: 450 }, strategy: 'current' }, + 'heavy 1280x800 (CPU bound)': { path: '/fixtures/heavy.html?n=15000', viewport: { width: 1280, height: 800 }, strategy: 'idle0-raf' } +} + +const runLoad = async (browsers: Launched[], url: (p: string) => string, load: Load, concurrency: number, total: number) => { + const pools = browsers.map(l => createContextPool(() => l.browser.createBrowserContext())) + const latencies: number[] = [] + let next = 0 + let peakPss = 0 + let peakProcs = 0 + const sampler = setInterval(() => { + peakPss = Math.max(peakPss, browsers.reduce((a, l) => a + treePssMB(l.pid), 0)) + peakProcs = Math.max(peakProcs, browsers.reduce((a, l) => a + treeSize(l.pid), 0)) + }, 250) + const cpu0 = browsers.reduce((a, l) => a + treeCpuMs(l.pid), 0) + const t0 = performance.now() + await Promise.all(Array.from({ length: concurrency }, async (_, w) => { + const pool = pools[w % pools.length] + while (next < total) { + next++ + const s = performance.now() + const c = await pool.acquire() + const page = await c.newPage() + await page.setViewport(load.viewport) + await setupPage(page) + await gotoAndWait(page, url(load.path), load.strategy) + await page.screenshot(load.shot) + await cleanPage(page) + pool.release(c) + latencies.push(performance.now() - s) + } + })) + const wallMs = performance.now() - t0 + clearInterval(sampler) + await new Promise(resolve => setTimeout(resolve, 200)) + const cpuMs = browsers.reduce((a, l) => a + treeCpuMs(l.pid), 0) - cpu0 + for (const p of pools) await p.drain() + const lat = aggregate(latencies) + return { perSec: total / wallMs * 1000, p50Ms: lat.median, p90Ms: lat.p90, cpuMsPerCapture: cpuMs / total, cores: cpuMs / wallMs, peakPssMB: peakPss, peakProcs } +} + +export const concurrency: Experiment = { + description: 'throughput, latency, CPU and memory for 1..20 concurrent captures, 1 or 2 browsers', + async run (opts) { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + const b1 = await launch(variant) + const b2 = await launch(variant) + console.log(`browser: ${variant.name} ${b1.version}`) + const all: Record = {} + const configs: Record = { + c1: { c: 1, browsers: [b1] }, + c2: { c: 2, browsers: [b1] }, + c5: { c: 5, browsers: [b1] }, + c10: { c: 10, browsers: [b1] }, + c20: { c: 20, browsers: [b1] }, + 'c10-2browsers': { c: 10, browsers: [b1, b2] } + } + for (const [loadName, load] of Object.entries(loads)) { + const results = await runInterleaved(Object.keys(configs), Math.max(2, Math.round(opts.runs / 4)), async (name) => { + const { c, browsers } = configs[name] + return runLoad(browsers, server.url, load, c, Math.max(10, c * 3)) + }, 0) + all[loadName] = results + report(loadName, results, ['perSec', 'p50Ms', 'p90Ms', 'cpuMsPerCapture', 'cores', 'peakPssMB', 'peakProcs']) + } + await b1.browser.close() + await b2.browser.close() + await server.close() + save('concurrency', all) + } +} + +export { runLoad } diff --git a/benchmark/experiments/console.ts b/benchmark/experiments/console.ts new file mode 100644 index 0000000..1713b66 --- /dev/null +++ b/benchmark/experiments/console.ts @@ -0,0 +1,36 @@ +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool } from '../lib/capture.ts' +import { selfCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' + +// H: the debugPage console listener is attached on every page even when DEBUG is off, and does +// 2 CDP round trips per argument of every console message (getProperty + jsonValue). On a verbose +// page this costs latency and node CPU for nothing. + +export const consoleListener: Experiment = { + description: 'cost of the always-on console listener on a page logging 500 messages', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const pool = createContextPool(() => l.browser.createBrowserContext()) + const results = await runInterleaved(['listener', 'no-listener'], opts.runs, async (variant) => { + const c = await pool.acquire() + const page = await c.newPage() + await setupPage(page, { consoleListener: variant === 'listener' }) + const cpu0 = selfCpuMs() + const t0 = performance.now() + await gotoAndWait(page, url('/fixtures/console.html?logs=500'), 'idle0') + await page.screenshot() + const totalMs = performance.now() - t0 + // let the pending listener round trips land, they are paid even if after the response + await new Promise(resolve => setTimeout(resolve, 300)) + const nodeCpuMs = selfCpuMs() - cpu0 + await cleanPage(page) + pool.release(c) + return { totalMs, nodeCpuMs } + }) + report('console.html?logs=500', results, ['totalMs', 'nodeCpuMs']) + await pool.drain() + save('console', results) + }) + } +} diff --git a/benchmark/experiments/context.ts b/benchmark/experiments/context.ts new file mode 100644 index 0000000..dc925c0 --- /dev/null +++ b/benchmark/experiments/context.ts @@ -0,0 +1,81 @@ +import { withEnv, type Experiment } from './common.ts' +import { cleanPage, createContextPool } from '../lib/capture.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Page } from 'puppeteer' + +// H: the per-capture cost of the browser plumbing (context, page, setup CDP calls) is small next +// to the waits. Variants: the service's pooled context + new page; a fresh context per capture +// (no pool, no cookie cleanup); a reused page (the floor, not safe across users). +// Also: the cost of each setup step taken one by one. + +export const context: Experiment = { + description: 'pooled context vs fresh context vs reused page, and the cost of each page setup call', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const pool = createContextPool(() => l.browser.createBrowserContext()) + const reused = await (await l.browser.createBrowserContext()).newPage() + await reused.setViewport({ width: 800, height: 450 }) + const target = url('/fixtures/static.html') + const results = await runInterleaved(['pool+newPage', 'fresh-context', 'reused-page'], opts.runs * 2, async (variant) => { + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + let page: Page + if (variant === 'reused-page') { + page = reused + } else if (variant === 'fresh-context') { + page = await (await l.browser.createBrowserContext()).newPage() + await page.setViewport({ width: 800, height: 450 }) + } else { + page = await (await pool.acquire()).newPage() + await page.setViewport({ width: 800, height: 450 }) + } + const tPage = performance.now() + await page.goto(target, { waitUntil: 'load' }) + await page.screenshot() + const tShot = performance.now() + if (variant === 'fresh-context') await page.browserContext().close() + else if (variant === 'pool+newPage') { + const c = page.browserContext() + await cleanPage(page) + pool.release(c) + } + const tEnd = performance.now() + await new Promise(resolve => setTimeout(resolve, 50)) + return { pageMs: tPage - t0, gotoShotMs: tShot - tPage, cleanupMs: tEnd - tShot, totalMs: tEnd - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0 } + }) + report('static page, goto load + screenshot', results, ['pageMs', 'gotoShotMs', 'cleanupMs', 'totalMs', 'browserCpuMs']) + + // each setup call of the service, timed alone on a fresh page + const steps: Record Promise> = { + newPage: async () => {}, + setViewport: p => p.setViewport({ width: 800, height: 450 }), + emulateTimezone: p => p.emulateTimezone('Europe/Paris'), + setExtraHTTPHeaders: p => p.setExtraHTTPHeaders({ 'Accept-Language': 'fr-FR' }), + evaluateOnNewDocument: p => p.evaluateOnNewDocument(() => {}), + exposeFunction: p => p.exposeFunction('triggerCapture' + Math.random().toString(36).slice(2), () => {}), + setRequestInterception: p => p.setRequestInterception(true), + metaLookup: p => p.$eval('head > meta[name=\'x-capture\']', el => el.content).catch(() => undefined), + cleanPage: p => cleanPage(p) + } + const stepResults = await runInterleaved(['steps'], opts.runs * 2, async () => { + const sample: Record = {} + const t0 = performance.now() + const c = await pool.acquire() + const p = await c.newPage() + sample.newPage = performance.now() - t0 + for (const [k, fn] of Object.entries(steps)) { + if (k === 'newPage') continue + const t = performance.now() + await fn(p) + sample[k] = performance.now() - t + } + pool.release(c) + return sample + }) + report('setup steps, ms each', stepResults, Object.keys(steps)) + await pool.drain() + save('context', { results, stepResults }) + }) + } +} diff --git a/benchmark/experiments/e2e.ts b/benchmark/experiments/e2e.ts new file mode 100644 index 0000000..77da23f --- /dev/null +++ b/benchmark/experiments/e2e.ts @@ -0,0 +1,137 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import net from 'node:net' +import path from 'node:path' +import { startFixturesServer } from '../lib/server.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { aggregate } from '../lib/stats.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Experiment } from './common.ts' + +// The real service, started as a child process from the working tree (or from BENCH_E2E_DIR, e.g. +// a worktree of master, to compare 2 versions), and loaded over HTTP. Scenarios: +// - png: 20 screenshots of the app page at concurrency 5, the timer breakdown of the service +// - gif+ping: an animated capture while a probe requests api-docs every 20ms, the probe latency +// shows how long the service's event loop is blocked for every other request +// BENCH_E2E_CONFIG (JSON) is merged in the service's config, e.g. a browser or a concurrency. + +const freePort = () => new Promise(resolve => { + const s = net.createServer().listen(0, () => { + const port = (s.address() as net.AddressInfo).port + s.close(() => resolve(port)) + }) +}) + +const startService = async (dir: string, extraConfig: Record) => { + const port = await freePort() + const child: ChildProcess = spawn('node', ['--disable-warning=ExperimentalWarning', 'api/index.ts'], { + cwd: dir, + env: { + ...process.env, + NODE_ENV: 'development', + NODE_CONFIG: JSON.stringify({ port, onlySameHost: false, concurrency: 5, observer: { active: false }, ...extraConfig }) + }, + stdio: ['ignore', 'ignore', 'pipe'] + }) + child.stderr?.on('data', () => {}) + const base = `http://127.0.0.1:${port}/api/v1` + for (let i = 0; i < 100; i++) { + try { + if ((await fetch(base + '/api-docs.json')).ok) return { base, child } + } catch {} + await new Promise(resolve => setTimeout(resolve, 200)) + } + child.kill() + throw new Error('service did not start') +} + +export const e2e: Experiment = { + description: 'the real service over HTTP: png load at concurrency 5, event loop blocking during a gif', + async run (opts) { + const fixtures = await startFixturesServer() + const dir = process.env.BENCH_E2E_DIR ?? path.join(import.meta.dirname, '..', '..') + const extra = JSON.parse(process.env.BENCH_E2E_CONFIG ?? '{}') + const { base, child } = await startService(dir, extra) + console.log(`service from ${dir}, config ${JSON.stringify(extra)}`) + const shot = (target: string, q = '') => fetch(`${base}/screenshot?key=capture&target=${encodeURIComponent(target)}${q}`, { headers: { 'x-secret-key': 'capture' } }) + try { + const all: Record = {} + for (const [name, page] of Object.entries({ app: '/fixtures/app.html', 'app-trigger': '/fixtures/app.html?trigger=1' })) { + const results = await runInterleaved(['c5'], Math.max(2, Math.round(opts.runs / 2)), async () => { + const total = 20 + let next = 0 + const latencies: number[] = [] + const steps: Record = {} + const cpu0 = treeCpuMs(child.pid!) + const t0 = performance.now() + await Promise.all(Array.from({ length: 5 }, async () => { + while (next < total) { + next++ + const s = performance.now() + const res = await shot(fixtures.url(page), '&timer=true') + const times = await res.json() as Record + latencies.push(performance.now() - s) + for (const [k, v] of Object.entries(times)) (steps[k] ??= []).push(v) + } + })) + const wallMs = performance.now() - t0 + const lat = aggregate(latencies) + return { + perSec: total / wallMs * 1000, + p50Ms: lat.median, + p90Ms: lat.p90, + cpuMsPerCapture: (treeCpuMs(child.pid!) - cpu0) / total, + ...Object.fromEntries(Object.entries(steps).map(([k, v]) => [k, aggregate(v).median])) + } + }, 1) + all[name] = results + report(`${name}, 20 screenshots at concurrency 5 (service + browser CPU)`, results, ['perSec', 'p50Ms', 'p90Ms', 'cpuMsPerCapture']) + const stepNames = Object.keys(results.c5[0]).filter(k => !['perSec', 'p50Ms', 'p90Ms', 'cpuMsPerCapture'].includes(k)) + report(`${name}, median of the service timer steps (ms)`, results, stepNames) + } + const gifResults = await runInterleaved(['gif'], Math.max(2, Math.round(opts.runs / 4)), async () => { + const state = { stop: false } + const pings: number[] = [] + const probe = (async () => { + while (!state.stop) { + const s = performance.now() + await fetch(base + '/api-docs.json').then(r => r.arrayBuffer()) + pings.push(performance.now() - s) + await new Promise(resolve => setTimeout(resolve, 20)) + } + })() + const s = performance.now() + const res = await shot(fixtures.url('/fixtures/anim.html?frames=30'), '&type=gif&width=800&height=450') + const buf = await res.arrayBuffer() + const gifMs = performance.now() - s + state.stop = true + await probe + const p = aggregate(pings) + return { gifMs, KB: buf.byteLength / 1024, pingP50Ms: p.median, pingP90Ms: p.p90, pingMaxMs: p.max } + }, 0) + all.gif = gifResults + + // overload: 30 clients that give up after 2s (a proxy timeout, a user leaving), then one + // patient client. If the abandoned requests still wait for a context and get rendered, + // the patient one queues behind work done for nobody. + const abandon = await (async () => { + const controllers = Array.from({ length: 30 }, () => new AbortController()) + const pending = controllers.map(c => fetch(`${base}/screenshot?target=${encodeURIComponent(fixtures.url('/fixtures/app.html'))}`, { headers: { 'x-secret-key': 'capture' }, signal: c.signal }).then(r => r.arrayBuffer()).catch(() => {})) + await new Promise(resolve => setTimeout(resolve, 2000)) + for (const c of controllers) c.abort() + await Promise.all(pending) + const s = performance.now() + const res = await shot(fixtures.url('/fixtures/app.html?trigger=1')) + await res.arrayBuffer() + return { patientMs: performance.now() - s, status: String(res.status) } + })() + all.abandon = abandon + console.log(`\n### overload: 30 requests abandoned after 2s, then 1 request (app?trigger=1, ~0.7s alone)\n\npatient request: ${Math.round(abandon.patientMs)}ms, status ${abandon.status}`) + report('gif 800x450 30 frames, with a probe on api-docs every 20ms', gifResults, ['gifMs', 'KB', 'pingP50Ms', 'pingP90Ms', 'pingMaxMs']) + save('e2e', all) + } finally { + child.kill('SIGINT') + await new Promise(resolve => child.once('exit', resolve)) + await fixtures.close() + } + } +} diff --git a/benchmark/experiments/gif.ts b/benchmark/experiments/gif.ts new file mode 100644 index 0000000..2182c93 --- /dev/null +++ b/benchmark/experiments/gif.ts @@ -0,0 +1,108 @@ +import { promisify } from 'node:util' +import { monitorEventLoopDelay } from 'node:perf_hooks' +import GifEncoder from 'gif-encoder' +import getPixelsCb from 'get-pixels' +import { optimizeGif } from '../../api/utils/gifsicle.ts' +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage } from '../lib/capture.ts' +import { selfCpuMs, treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' + +// H: an animated capture is dominated by node-side work, not by the browser: each frame is a PNG +// that is decoded in pure JS (get-pixels/pngjs) then color-quantized in pure JS (gif-encoder, +// NeuQuant), all on the main thread, which blocks the event loop of the whole service (other +// requests, metrics, health checks) for the duration. The stages are timed separately, with the +// event loop delay measured meanwhile. Variant: jpeg frames instead of png. + +const getPixels = promisify(getPixelsCb) + +// sharp (libvips + cgif + libimagequant, native, off the main thread) is not a dependency of the +// service: point BENCH_SHARP to an install of it to add the variant (npm i sharp in a scratch dir) +const loadSharp = async () => { + if (!process.env.BENCH_SHARP) return undefined + return (await import(process.env.BENCH_SHARP)).default +} + +export const gif: Experiment = { + description: 'animated GIF pipeline: per-stage cost and event loop blocking', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const context = await l.browser.createBrowserContext() + const width = 800 + const height = 450 + const frames = 30 + const sharp = await loadSharp() + const variants = ['png-fast', 'jpeg-90'] + if (sharp) variants.push('sharp') + const results = await runInterleaved(variants, Math.max(2, Math.round(opts.runs / 2)), async (variant) => { + const page = await context.newPage() + await page.setViewport({ width, height }) + await setupPage(page) + await gotoAndWait(page, url(`/fixtures/anim.html?frames=${frames}`), 'idle0') + const h = monitorEventLoopDelay({ resolution: 5 }) + h.enable() + const cpu0 = selfCpuMs() + const bcpu0 = treeCpuMs(l.pid) + const t = { evaluate: 0, screenshot: 0, decode: 0, encode: 0, gifsicle: 0 } + const pngs: Buffer[] = [] + const gifEnc = new GifEncoder(width, height) + gifEnc.setFrameRate(15) + const chunks: Buffer[] = [] + gifEnc.on('data', (c: Buffer) => chunks.push(c)) + gifEnc.writeHeader() + let stopped = false + let n = 0 + const t0 = performance.now() + while (!stopped) { + n++ + let s = performance.now() + // @ts-ignore + stopped = await page.evaluate(() => window.animateCaptureFrame()) + t.evaluate += performance.now() - s + s = performance.now() + const buffer = await page.screenshot(variant === 'jpeg-90' ? { type: 'jpeg', quality: 90 } : { optimizeForSpeed: true }) + t.screenshot += performance.now() - s + if (variant === 'sharp') { + pngs.push(Buffer.from(buffer)) + continue + } + s = performance.now() + const pixels = await getPixels(Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength), variant === 'jpeg-90' ? 'image/jpeg' : 'image/png') + t.decode += performance.now() - s + s = performance.now() + gifEnc.addFrame(pixels.data) + t.encode += performance.now() - s + } + gifEnc.finish() + await new Promise(resolve => setImmediate(resolve)) + let raw = Buffer.concat(chunks) + if (variant === 'sharp') { + // decoding, quantization (libimagequant) and encoding (cgif) in libvips threads + const s = performance.now() + raw = await sharp(pngs, { join: { animated: true } }).gif({ delay: pngs.map(() => 67), loop: 0 }).toBuffer() + t.encode = performance.now() - s + } + const s = performance.now() + const optimized = await optimizeGif(raw, 2) + t.gifsicle = performance.now() - s + const totalMs = performance.now() - t0 + h.disable() + await cleanPage(page) + return { + frames: n, + totalMs, + ...Object.fromEntries(Object.entries(t).map(([k, v]) => [k + 'Ms', v])), + nodeCpuMs: selfCpuMs() - cpu0, + browserCpuMs: treeCpuMs(l.pid) - bcpu0, + loopMaxMs: h.max / 1e6, + loopP99Ms: h.percentile(99) / 1e6, + rawKB: raw.length / 1024, + KB: optimized.length / 1024 + } + }) + report(`anim.html ${width}x${height}, 30 frames`, results, ['totalMs', 'evaluateMs', 'screenshotMs', 'decodeMs', 'encodeMs', 'gifsicleMs', 'nodeCpuMs', 'browserCpuMs', 'loopMaxMs', 'KB']) + await context.close() + save('gif', results) + }) + } +} diff --git a/benchmark/experiments/hotspot.ts b/benchmark/experiments/hotspot.ts new file mode 100644 index 0000000..9fd32c4 --- /dev/null +++ b/benchmark/experiments/hotspot.ts @@ -0,0 +1,90 @@ +import fs from 'node:fs' +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch } from '../lib/browsers.ts' +import { processTree } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import { runLoad } from './concurrency.ts' +import type { Experiment } from './common.ts' + +// H: a single browser saturates at about 2 cores on paint-heavy pages, whatever the +// concurrency, because one of its processes is a serial bottleneck (the GPU process +// composites and rasterizes for all renderers). The CPU time is split by process type, from +// the --type= argument of each chrome process, sampled during the load. A process that sits +// near 100% of one core is the bottleneck. + +const typeOf = (pid: number) => { + try { + // chrome rewrites its title as a single space separated string + const cmd = fs.readFileSync(`/proc/${pid}/cmdline`, 'utf8').split(/[\0 ]/) + const type = cmd.find(a => a.startsWith('--type='))?.slice(7) ?? 'browser' + const sub = cmd.find(a => a.startsWith('--utility-sub-type='))?.slice(19).split('.').pop() + return sub ? `${type}:${sub}` : type + } catch { + return 'gone' + } +} +const ticks = (pid: number) => { + try { + const raw = fs.readFileSync(`/proc/${pid}/stat`, 'utf8') + const f = raw.slice(raw.lastIndexOf(')') + 2).split(' ') + return Number(f[11]) + Number(f[12]) + } catch { + return 0 + } +} + +const launchVariants: Record = { + 'chrome-stable': { browser: 'chrome-stable', args: [] }, + 'chrome-stable --disable-gpu': { browser: 'chrome-stable', args: ['--disable-gpu'] }, + shell: { browser: 'shell', args: [] } +} + +export const hotspot: Experiment = { + description: 'CPU by chrome process type at concurrency 10 on a paint-heavy page: which process caps a browser', + async run (opts) { + const server = await startFixturesServer() + const results = await runInterleaved(Object.keys(launchVariants), Math.max(1, Math.round(opts.runs / 4)), async (name) => { + const v = launchVariants[name] + const l = await launch(browserVariants[v.browser], { args: v.args }) + // per pid: first and last ticks seen, the processes of closed pages included + const seen = new Map() + const sample = () => { + for (const pid of processTree(l.pid)) { + const t = ticks(pid) + const s = seen.get(pid) + if (s) s.last = t + else seen.set(pid, { type: typeOf(pid), first: t, last: t }) + } + } + sample() + const t0 = performance.now() + const sampler = setInterval(sample, 100) + const load = await runLoad([l], server.url, { path: '/fixtures/heavy.html?n=15000', viewport: { width: 1280, height: 800 }, strategy: 'idle0-raf' }, 10, 30) + clearInterval(sampler) + sample() + const wallMs = performance.now() - t0 + await l.browser.close() + // share of one core used by each process type, and by its busiest single process + const byType: Record = {} + const maxProc: Record = {} + for (const s of seen.values()) { + const cores = (s.last - s.first) * 10 / wallMs + const key = s.type.startsWith('renderer') ? 'renderer' : s.type + byType[key] = (byType[key] ?? 0) + cores + maxProc[key] = Math.max(maxProc[key] ?? 0, cores) + } + if (process.env.BENCH_DEBUG) console.log([...seen.entries()].map(([pid, s]) => `${pid} ${s.type} ${s.last - s.first}`).join('\n')) + const out: Record = { perSec: load.perSec, p50Ms: load.p50Ms } + for (const k of ['browser', 'gpu-process', 'renderer', 'utility:mojom']) { + out[k + '.cores'] = byType[k] ?? 0 + out[k + '.maxProc'] = maxProc[k] ?? 0 + } + for (const [k, c] of Object.entries(byType)) if (!(k + '.cores' in out)) out[k + '.cores'] = c + return out + }, 0) + const metrics = [...new Set(Object.values(results).flat().flatMap(s => Object.keys(s)))] + report('heavy 1280x800 at concurrency 10, CPU in cores by process type (maxProc = busiest single process)', results, metrics) + await server.close() + save('hotspot', results) + } +} diff --git a/benchmark/experiments/index.ts b/benchmark/experiments/index.ts new file mode 100644 index 0000000..0092da7 --- /dev/null +++ b/benchmark/experiments/index.ts @@ -0,0 +1,15 @@ +import type { Experiment } from './common.ts' +import { wait } from './wait.ts' +import { browser } from './browser.ts' +import { context } from './context.ts' +import { consoleListener } from './console.ts' +import { output } from './output.ts' +import { gif } from './gif.ts' +import { concurrency } from './concurrency.ts' +import { args } from './args.ts' +import { transport } from './transport.ts' +import { e2e } from './e2e.ts' +import { hotspot } from './hotspot.ts' +import { real } from './real.ts' + +export const experiments: Record = { wait, browser, context, console: consoleListener, output, gif, concurrency, args, transport, e2e, hotspot, real } diff --git a/benchmark/experiments/output.ts b/benchmark/experiments/output.ts new file mode 100644 index 0000000..e3cdf15 --- /dev/null +++ b/benchmark/experiments/output.ts @@ -0,0 +1,72 @@ +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage } from '../lib/capture.ts' +import { treeCpuMs } from '../lib/proc.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Page, ScreenshotOptions, PDFOptions } from 'puppeteer' + +// H: encoding the output is a sizable share of a capture on large viewports. PNG compression in +// the browser is slow, optimizeForSpeed / jpeg / webp are cheaper. For PDF, the tagged structure +// (accessibility tree, on by default) and the document outline cost time on long documents. +// The page is opened once per task and the output is taken repeatedly. + +const shots: Record = { + png: {}, + 'png-fast': { optimizeForSpeed: true }, + 'jpeg-80': { type: 'jpeg', quality: 80 }, + 'webp-80': { type: 'webp', quality: 80 } +} +const shotPages = { + 'app 800x450': { path: '/fixtures/app.html?trigger=1', viewport: { width: 800, height: 450 }, fullPage: false }, + 'heavy 1920x1080': { path: '/fixtures/heavy.html', viewport: { width: 1920, height: 1080 }, fullPage: false }, + 'long fullPage 1280': { path: '/fixtures/long.html?sections=10', viewport: { width: 1280, height: 800 }, fullPage: true } +} + +const pdfs: Record = { + default: {}, + 'tagged-false': { tagged: false }, + outline: { outline: true }, + footer: { displayHeaderFooter: true, headerTemplate: ' ', footerTemplate: '
/
' } +} + +export const output: Experiment = { + description: 'screenshot formats (png, png optimizeForSpeed, jpeg, webp) and pdf options (tagged, outline, footer)', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const all: Record = {} + const context = await l.browser.createBrowserContext() + const open = async (path: string, viewport?: { width: number, height: number }) => { + const page: Page = await context.newPage() + if (viewport) await page.setViewport(viewport) + await setupPage(page) + await gotoAndWait(page, url(path), 'idle0-raf') + return page + } + for (const [name, p] of Object.entries(shotPages)) { + const page = await open(p.path, p.viewport) + const results = await runInterleaved(Object.keys(shots), opts.runs, async (v) => { + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + const buf = await page.screenshot({ ...shots[v], fullPage: p.fullPage }) + return { ms: performance.now() - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0, KB: buf.length / 1024 } + }) + all[name] = results + report(`screenshot: ${name}`, results, ['ms', 'browserCpuMs', 'KB']) + await cleanPage(page) + } + for (const sections of [30, 120]) { + const page = await open(`/fixtures/long.html?sections=${sections}`) + const results = await runInterleaved(Object.keys(pdfs), opts.runs, async (v) => { + const cpu0 = treeCpuMs(l.pid) + const t0 = performance.now() + const buf = await page.pdf({ format: 'A4', printBackground: true, margin: { left: '1.5cm', right: '1.5cm', top: '1.5cm', bottom: '1.5cm' }, ...pdfs[v] }) + return { ms: performance.now() - t0, browserCpuMs: treeCpuMs(l.pid) - cpu0, KB: buf.length / 1024 } + }) + all[`pdf-${sections}`] = results + report(`pdf: long.html, ${sections} sections`, results, ['ms', 'browserCpuMs', 'KB']) + await cleanPage(page) + } + await context.close() + save('output', all) + }) + } +} diff --git a/benchmark/experiments/real.ts b/benchmark/experiments/real.ts new file mode 100644 index 0000000..fb58ebf --- /dev/null +++ b/benchmark/experiments/real.ts @@ -0,0 +1,107 @@ +import fs from 'node:fs' +import path from 'node:path' +import { PNG } from 'pngjs' +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage, type WaitStrategy } from '../lib/capture.ts' +import { report, save } from '../lib/runner.ts' + +// Wait strategies on real pages: the public applications of a data-fair instance (by default +// opendata.koumoul.com), captured at the size of data-fair thumbnails. Each capture is compared +// pixel by pixel to a reference taken on a surely settled page (current strategy + 3s). A second +// reference measures the noise of the page itself (maps, dates, random colors...). +// BENCH_REAL_URL data-fair base url (default https://opendata.koumoul.com/data-fair) +// BENCH_REAL_APPS comma separated application ids (default: up to 2 per base application) +// Captures run one at a time to stay gentle with the instance. Suspect captures are written +// to results/real/ for a visual check. + +const baseUrl = process.env.BENCH_REAL_URL ?? 'https://opendata.koumoul.com/data-fair' +const candidates: WaitStrategy[] = ['current', 'idle500-raf', 'idle500-500', 'quiet'] +const viewport = { width: 1050, height: 450 } + +type App = { id: string, title: string, base: string } + +// https://cdn.jsdelivr.net/npm/@data-fair/app-charts@1.4/dist/ and https://koumoul.com/apps/atelier-carto/1.8/ +// -> app-charts and atelier-carto +const baseName = (url: string) => url.split('/').filter(s => s && s !== 'dist' && !/^v?\d[\d.]*$/.test(s)).pop()?.replace(/@[^@]*$/, '') || '?' + +const listApps = async (): Promise => { + const res = await fetch(`${baseUrl}/api/v1/applications?size=1000&select=id,title,url&visibility=public`) + const { results } = await res.json() as { results: { id: string, title: string, url: string }[] } + const apps = results.map(a => ({ id: a.id, title: a.title, base: baseName(a.url ?? '') })) + const only = process.env.BENCH_REAL_APPS?.split(',') + if (only) return apps.filter(a => only.includes(a.id)) + const perBase: Record = {} + for (const a of apps) { + perBase[a.base] ??= [] + if (perBase[a.base].length < 2) perBase[a.base].push(a) + } + return Object.values(perBase).flat() +} + +// share of pixels (%) whose channels differ by more than 40 out of 255 +const diffPct = (a: Uint8Array, b: Uint8Array) => { + const pa = PNG.sync.read(Buffer.from(a)) + const pb = PNG.sync.read(Buffer.from(b)) + if (pa.width !== pb.width || pa.height !== pb.height) return 100 + let diff = 0 + for (let i = 0; i < pa.data.length; i += 4) { + if (Math.abs(pa.data[i] - pb.data[i]) > 40 || Math.abs(pa.data[i + 1] - pb.data[i + 1]) > 40 || Math.abs(pa.data[i + 2] - pb.data[i + 2]) > 40) diff++ + } + return Math.round(diff / (pa.width * pa.height) * 10000) / 100 +} + +export const real: Experiment = { + description: 'wait strategies on the public applications of a data-fair instance: latency and pixel diff to a settled reference', + async run (opts) { + const apps = await listApps() + console.log(`${apps.length} applications from ${baseUrl}`) + const outDir = path.join(import.meta.dirname, '..', 'results', 'real') + fs.mkdirSync(outDir, { recursive: true }) + await withEnv(opts, async ({ l }) => { + const capture = async (target: string, strategy: WaitStrategy) => { + const context = await l.browser.createBrowserContext() + try { + const page = await context.newPage() + await page.setViewport(viewport) + await setupPage(page) + const t0 = performance.now() + const r = await gotoAndWait(page, target, strategy) + const waitMs = performance.now() - t0 + const png = await page.screenshot() + await cleanPage(page) + return { ...r, waitMs, png } + } finally { + await context.close() + } + } + const rows: Record[] = [] + const perStrategy: Record = Object.fromEntries(candidates.map(c => [c, []])) + for (const app of apps) { + const target = `${baseUrl}/app/${app.id}` + try { + const ref = await capture(target, 'reference') + const ref2 = await capture(target, 'reference') + const noise = diffPct(ref.png, ref2.png) + const row: Record = { app: app.id, base: app.base, signal: ref.ready === 'triggered' ? 'trigger' : (ref.meta ?? 'none'), noise } + for (const strategy of candidates) { + const c = await capture(target, strategy) + const diff = diffPct(ref.png, c.png) + // a capture differs from the reference beyond what the page itself varies + const excess = Math.max(0, diff - noise) + const suspect = excess > 1 + if (suspect) fs.writeFileSync(path.join(outDir, `${app.base}-${app.id}-${strategy}.png`), c.png) + row[strategy] = `${Math.round(c.waitMs)}ms ${diff}%${c.quiet ? ' ' + c.quiet : ''}` + perStrategy[strategy].push({ waitMs: c.waitMs, diff, excess, suspect, ready: c.ready }) + } + if (Object.values(perStrategy).some(s => s.at(-1)?.suspect)) fs.writeFileSync(path.join(outDir, `${app.base}-${app.id}-reference.png`), ref.png) + rows.push(row) + console.log(JSON.stringify(row)) + } catch (err: any) { + console.log(`${app.id} (${app.base}) failed: ${err.message}`) + } + } + report('per strategy, over all applications', Object.fromEntries(Object.entries(perStrategy).map(([k, v]) => [k, v.map(s => ({ waitMs: s.waitMs, excessDiffPct: s.excess, suspect: s.suspect }))])), ['waitMs', 'excessDiffPct', 'suspect']) + save('real', rows) + }) + } +} diff --git a/benchmark/experiments/transport.ts b/benchmark/experiments/transport.ts new file mode 100644 index 0000000..437694a --- /dev/null +++ b/benchmark/experiments/transport.ts @@ -0,0 +1,41 @@ +import { startFixturesServer } from '../lib/server.ts' +import { browserVariants, launch } from '../lib/browsers.ts' +import { setupPage, gotoAndWait } from '../lib/capture.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' +import type { Experiment } from './common.ts' +import type { Page } from 'puppeteer' + +// H: the outputs travel back base64 encoded in CDP messages over a websocket. A pipe transport +// (--remote-debugging-pipe) spares the websocket framing and the TCP loopback on large outputs. + +export const transport: Experiment = { + description: 'websocket vs pipe CDP transport on large screenshots and PDFs', + async run (opts) { + const server = await startFixturesServer() + const variant = browserVariants[opts.browser] + const ws = await launch(variant, { pipe: false }) + const pipe = await launch(variant, { pipe: true }) + const pages: Record = {} + for (const [name, l] of Object.entries({ websocket: ws, pipe })) { + const page = await (await l.browser.createBrowserContext()).newPage() + await page.setViewport({ width: 1920, height: 1080 }) + await setupPage(page) + await gotoAndWait(page, server.url('/fixtures/long.html?sections=20'), 'idle0-raf') + pages[name] = page + } + const results = await runInterleaved(Object.keys(pages), opts.runs, async (name) => { + let t = performance.now() + const shot = await pages[name].screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: 1920, height: 3000 } }) + const shotMs = performance.now() - t + t = performance.now() + const pdf = await pages[name].pdf({ format: 'A4' }) + const pdfMs = performance.now() - t + return { shotMs, shotKB: shot.length / 1024, pdfMs, pdfKB: pdf.length / 1024 } + }) + report('1920x3000 png and long pdf', results, ['shotMs', 'shotKB', 'pdfMs', 'pdfKB']) + await ws.browser.close() + await pipe.browser.close() + await server.close() + save('transport', results) + } +} diff --git a/benchmark/experiments/wait.ts b/benchmark/experiments/wait.ts new file mode 100644 index 0000000..ea2d480 --- /dev/null +++ b/benchmark/experiments/wait.ts @@ -0,0 +1,60 @@ +import { withEnv, type Experiment } from './common.ts' +import { setupPage, gotoAndWait, cleanPage, createContextPool, type WaitStrategy } from '../lib/capture.ts' +import { runInterleaved, report, save } from '../lib/runner.ts' + +// H: on pages that do not call triggerCapture, the latency floor is the networkidle0 window (500ms +// of no request) plus a fixed 1s. The 1s is a safety margin for rendering after the last response; +// the question is how much of it is needed, measured by the share of captures taken before the page +// declared itself done (data-done), with renders that land 0, 300 and 800ms after the last response. + +const strategies: WaitStrategy[] = ['current', 'idle0', 'idle0-raf', 'idle0-300', 'idle200-raf', 'idle500-raf', 'idle500-500', 'quiet', 'idle2'] +// BENCH_WAIT_PAGES: comma separated subset of the pages below +const allPages: Record = { + static: '/fixtures/static.html', + app: '/fixtures/app.html', + 'app-late300': '/fixtures/app.html?late=300', + 'app-late800': '/fixtures/app.html?late=800', + // a slower backend: 400ms per API call + 'app-slow-api': '/fixtures/app.html?apiDelay=400', + 'app-trigger': '/fixtures/app.html?trigger=1', + // what may happen after the last response (fixtures/settle.html) + 'raf chart 1s': '/fixtures/settle.html?mode=raf&ms=1000', + 'raf chart 2.5s': '/fixtures/settle.html?mode=raf&ms=2500', + 'css transition 1s': '/fixtures/settle.html?mode=css&ms=1000', + spinner: '/fixtures/settle.html?mode=spinner', + 'font 600ms': '/fixtures/settle.html?mode=font&ms=600', + 'timer 800ms': '/fixtures/settle.html?mode=timer&ms=800' +} + +const pages = process.env.BENCH_WAIT_PAGES + ? Object.fromEntries(Object.entries(allPages).filter(([k]) => process.env.BENCH_WAIT_PAGES!.split(',').includes(k))) + : allPages + +export const wait: Experiment = { + description: 'latency and completeness of the wait strategies after navigation', + async run (opts) { + await withEnv(opts, async ({ url, l }) => { + const pool = createContextPool(() => l.browser.createBrowserContext()) + const all: Record = {} + for (const [pageName, path] of Object.entries(pages)) { + const results = await runInterleaved(strategies, opts.runs, async (strategy) => { + const context = await pool.acquire() + const page = await context.newPage() + await page.setViewport({ width: 800, height: 450 }) + await setupPage(page) + const t0 = performance.now() + const { ready, done, quiet } = await gotoAndWait(page, url(path), strategy as WaitStrategy) + const waitMs = performance.now() - t0 + await page.screenshot() + await cleanPage(page) + pool.release(context) + return { waitMs, done, ready, quiet: quiet ?? '' } + }) + all[pageName] = results + report(`${pageName}`, results, ['waitMs', 'done', 'ready', 'quiet']) + } + await pool.drain() + save('wait', all) + }) + } +} diff --git a/benchmark/fixtures/anim.html b/benchmark/fixtures/anim.html new file mode 100644 index 0000000..5f91574 --- /dev/null +++ b/benchmark/fixtures/anim.html @@ -0,0 +1,34 @@ + + +Anim + + + + + + + diff --git a/benchmark/fixtures/app.html b/benchmark/fixtures/app.html new file mode 100644 index 0000000..fdcd2b3 --- /dev/null +++ b/benchmark/fixtures/app.html @@ -0,0 +1,54 @@ + + + + + Bench app + + + + + +

Bench app

+
+
+ + + diff --git a/benchmark/fixtures/console.html b/benchmark/fixtures/console.html new file mode 100644 index 0000000..9eb485a --- /dev/null +++ b/benchmark/fixtures/console.html @@ -0,0 +1,13 @@ + + +Console + + +

Console

+ + + diff --git a/benchmark/fixtures/heavy.html b/benchmark/fixtures/heavy.html new file mode 100644 index 0000000..dd047fe --- /dev/null +++ b/benchmark/fixtures/heavy.html @@ -0,0 +1,23 @@ + + +Heavy + + + +
+ + + diff --git a/benchmark/fixtures/long.html b/benchmark/fixtures/long.html new file mode 100644 index 0000000..fae5680 --- /dev/null +++ b/benchmark/fixtures/long.html @@ -0,0 +1,31 @@ + + +Long report + + + +

Rapport

+
+ + + diff --git a/benchmark/fixtures/settle.html b/benchmark/fixtures/settle.html new file mode 100644 index 0000000..edecba4 --- /dev/null +++ b/benchmark/fixtures/settle.html @@ -0,0 +1,68 @@ + + + + + Settle + + + + +

Settle

+
+ + + diff --git a/benchmark/fixtures/static.html b/benchmark/fixtures/static.html new file mode 100644 index 0000000..d727b13 --- /dev/null +++ b/benchmark/fixtures/static.html @@ -0,0 +1,10 @@ + + +Static + + +

Static page

+

No script, no sub-resource: the floor cost of a capture.

+ + + diff --git a/benchmark/index.ts b/benchmark/index.ts new file mode 100644 index 0000000..c6ee78d --- /dev/null +++ b/benchmark/index.ts @@ -0,0 +1,23 @@ +import { parseArgs } from 'node:util' +import { experiments } from './experiments/index.ts' + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + runs: { type: 'string', default: '8' }, + browser: { type: 'string', default: 'chrome-stable' } + } +}) + +const name = positionals[0] +if (!name || !(name in experiments || name === 'all')) { + console.log(`usage: npm run benchmark -- [--runs=8] [--browser=chrome-stable|cft|shell]\n\nexperiments:\n${Object.entries(experiments).map(([k, e]) => ` ${k.padEnd(12)} ${e.description}`).join('\n')}`) + process.exit(name ? 1 : 0) +} + +const opts = { runs: Number(values.runs), browser: values.browser } +for (const [k, e] of Object.entries(experiments)) { + if (name !== 'all' && k !== name) continue + console.log(`\n## ${k}: ${e.description}`) + await e.run(opts) +} diff --git a/benchmark/lib/browsers.ts b/benchmark/lib/browsers.ts new file mode 100644 index 0000000..dd56bdd --- /dev/null +++ b/benchmark/lib/browsers.ts @@ -0,0 +1,45 @@ +import os from 'node:os' +import fs from 'node:fs' +import path from 'node:path' +import puppeteer, { type Browser, type LaunchOptions } from 'puppeteer' +import { treePssMB, treeSize } from './proc.ts' + +// Browser binaries compared. chrome-stable is what the Docker image installs (google-chrome-stable +// deb), the others are Chrome for Testing builds downloaded by @puppeteer/browsers +// (npx puppeteer browsers install chrome-headless-shell@) +const cache = path.join(os.homedir(), '.cache', 'puppeteer') +const latest = (dir: string) => { + if (!fs.existsSync(dir)) return undefined + return fs.readdirSync(dir).sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).pop() +} +const cftVersion = latest(path.join(cache, 'chrome')) +const shellVersion = latest(path.join(cache, 'chrome-headless-shell')) + +export type BrowserVariant = { name: string, launch: LaunchOptions } + +// On Ubuntu >= 23.10 AppArmor forbids the unprivileged user namespaces of the sandbox to any +// binary without a profile: the google-chrome-stable deb installs one, the Chrome for Testing +// builds do not and only run with --no-sandbox. chrome-stable-nosandbox is the fair comparison. +export const browserVariants: Record = { + 'chrome-stable': { name: 'chrome-stable', launch: { executablePath: '/usr/bin/google-chrome-stable', headless: true } }, + 'chrome-stable-nosandbox': { name: 'chrome-stable-nosandbox', launch: { executablePath: '/usr/bin/google-chrome-stable', headless: true, args: ['--no-sandbox'] } }, + cft: { name: `cft-${cftVersion}`, launch: { executablePath: path.join(cache, 'chrome', cftVersion ?? '', 'chrome-linux64', 'chrome'), headless: true, args: ['--no-sandbox'] } }, + shell: { name: `shell-${shellVersion}`, launch: { executablePath: path.join(cache, 'chrome-headless-shell', shellVersion ?? '', 'chrome-headless-shell-linux64', 'chrome-headless-shell'), headless: 'shell', args: ['--no-sandbox'] } } +} + +export type Launched = { browser: Browser, pid: number, launchMs: number, version: string } + +export const launch = async (variant: BrowserVariant, extra: LaunchOptions = {}): Promise => { + const t0 = performance.now() + const browser = await puppeteer.launch({ + ...variant.launch, + ...extra, + args: [...(variant.launch.args ?? []), ...(extra.args ?? [])] + }) + const launchMs = performance.now() - t0 + const pid = browser.process()?.pid + if (!pid) throw new Error('no browser pid') + return { browser, pid, launchMs, version: await browser.version() } +} + +export const browserFootprint = (l: Launched) => ({ pssMB: treePssMB(l.pid), processes: treeSize(l.pid) }) diff --git a/benchmark/lib/capture.ts b/benchmark/lib/capture.ts new file mode 100644 index 0000000..9de831d --- /dev/null +++ b/benchmark/lib/capture.ts @@ -0,0 +1,155 @@ +import type { BrowserContext, Page } from 'puppeteer' + +// A copy of the capture service's page lifecycle (api/utils/page.ts) with each lever made a +// parameter. Kept in sync by hand: the point is to measure the variants, not to import the +// service with its config. + +export type WaitStrategy = + | 'current' // goto networkidle0 raced with triggerCapture, 2 meta lookups, then +1s + | 'idle0' // same without the final 1s + | 'idle0-raf' // same, the 1s replaced by 2 animation frames + | 'idle0-300' // same, the 1s replaced by 300ms + | 'idle200-raf' // goto 'load', then a 200ms network idle, then 2 animation frames + | 'idle500-raf' // goto 'load', then a 500ms network idle (puppeteer side), then 2 animation frames + | 'idle500-500' // goto 'load', then a 500ms network idle, then a fixed 500ms settle + | 'quiet' // goto 'load', a 500ms network idle, then visual quiet (see waitForQuiet), capped + | 'idle2' // goto networkidle2 (<= 2 connections for 500ms), then +1s + | 'reference' // 'current' + 3s: a page surely settled, the reference of the real pages comparison + +// the visual quiet of the 'quiet' strategy: no DOM mutation, no animation frame callback, no +// running finite CSS/Web animation and no font loading for quietMs, or maxMs elapsed +export const quietOptions = { quietMs: 200, maxMs: 2000 } + +// injected before any script of the page: records the last time the page did visible work +const installActivityTracker = () => { + const w = window as any + w.__captureLastActivity = performance.now() + const mark = () => { w.__captureLastActivity = performance.now() } + const raf = window.requestAnimationFrame.bind(window) + window.requestAnimationFrame = (cb) => raf((t) => { mark(); cb(t) }) + new MutationObserver(mark).observe(document, { subtree: true, childList: true, attributes: true, characterData: true }) +} + +export const waitForQuiet = (page: Page, quietMs = quietOptions.quietMs, maxMs = quietOptions.maxMs) => page.evaluate((quietMs, maxMs) => new Promise(resolve => { + const start = performance.now() + const check = () => { + const now = performance.now() + const busy = [] + if (now - ((window as any).__captureLastActivity ?? 0) < quietMs) busy.push('activity') + // endless animations (spinners, pulses) never settle, they are not waited for + if (document.getAnimations().some(a => a.playState === 'running' && a.effect?.getTiming().iterations !== Infinity)) busy.push('animations') + if (document.fonts.status === 'loading') busy.push('fonts') + if (!busy.length) return resolve('quiet') + if (now - start >= maxMs) return resolve('cap:' + busy.join('+')) + setTimeout(check, 50) + } + check() +}), quietMs, maxMs) + +export type SetupOptions = { + lang?: string, + timezone?: string, + consoleListener?: boolean, // the debugPage listener, attached whether DEBUG is set or not + interception?: boolean // setRequestInterception, as /inspect does +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +export const setupPage = async (page: Page, opts: SetupOptions = {}) => { + const lang = opts.lang ?? 'fr-FR' + await page.emulateTimezone(opts.timezone ?? 'Europe/Paris') + await page.setExtraHTTPHeaders({ 'Accept-Language': lang }) + await page.evaluateOnNewDocument((lang) => { + const langs = [lang] + if (lang.includes('-')) langs.push(lang.split('-')[0]) + Object.defineProperty(navigator, 'language', { get: () => lang }) + Object.defineProperty(navigator, 'languages', { get: () => langs }) + }, lang) + if (opts.consoleListener) { + page.on('console', async msg => { + const args = [] + for (const arg of msg.args()) { + try { + args.push((await (await arg.getProperty('message')).jsonValue()) || await arg.jsonValue()) + } catch {} + } + }) + } + if (opts.interception) { + await page.setRequestInterception(true) + page.on('request', request => { request.continue().catch(() => {}) }) + } +} + +export type WaitResult = { ready: 'triggered' | 'idle' | 'timeout', done: boolean, meta?: string, quiet?: string } + +export const gotoAndWait = async (page: Page, url: string, strategy: WaitStrategy, timeout = 20000): Promise => { + let triggered = false + const triggerCapture = new Promise(resolve => { + page.exposeFunction('triggerCapture', (animationSupported?: boolean) => { + triggered = true + resolve() + return !!animationSupported + }).catch(() => {}) + }) + if (strategy === 'quiet') await page.evaluateOnNewDocument(installActivityTracker) + const puppeteerIdle = ['idle200-raf', 'idle500-raf', 'idle500-500', 'quiet'].includes(strategy) + let ready: WaitResult['ready'] = 'idle' + try { + if (puppeteerIdle) { + const idleTime = strategy === 'idle200-raf' ? 200 : 500 + await Promise.race([ + page.goto(url, { waitUntil: 'load', timeout }).then(() => page.waitForNetworkIdle({ idleTime, timeout })), + triggerCapture + ]) + } else { + const waitUntil = strategy === 'idle2' ? 'networkidle2' : 'networkidle0' + await Promise.race([page.goto(url, { waitUntil, timeout }), triggerCapture]) + } + } catch (err: any) { + if (err.name !== 'TimeoutError') throw err + ready = 'timeout' + } + if (triggered) ready = 'triggered' + let meta: string | undefined + let quiet: string | undefined + if (ready === 'idle') { + // the 2 metas of the service: an explicit delay, or a promise to call triggerCapture + const delayMeta = await page.$eval('head > meta[name=\'df:capture-delay\']', el => el.content).catch(() => undefined) + const captureMeta = delayMeta ? undefined : await page.$eval('head > meta[name=\'x-capture\']', el => el.content).catch(() => undefined) + if (delayMeta) { + meta = 'delay=' + delayMeta + await Promise.race([triggerCapture, sleep(Math.min(Number(delayMeta) * 1000, timeout))]) + } else if (captureMeta === 'trigger') { + meta = 'trigger' + await Promise.race([triggerCapture, sleep(timeout)]) + } else if (strategy === 'current' || strategy === 'idle2') await sleep(1000) + else if (strategy === 'reference') await sleep(4000) + else if (strategy === 'idle0-300') await sleep(300) + else if (strategy === 'idle500-500') await sleep(500) + else if (strategy === 'quiet') quiet = await waitForQuiet(page) + else await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))) + if (triggered) ready = 'triggered' + } + const done = await page.evaluate(() => document.body?.dataset.done === '1') + return { ready, done, meta, quiet } +} + +// what the service does after a capture: delete cookies, close the page, give the context back +export const cleanPage = async (page: Page) => { + const context = page.browserContext() + const cookies = await context.cookies() + if (cookies.length) await context.deleteCookie(...cookies) + await page.close() +} + +// a minimal pool, enough to reproduce generic-pool's behaviour of the service (lazy creation, +// reuse of released contexts) +export const createContextPool = (create: () => Promise) => { + const free: BrowserContext[] = [] + return { + acquire: async () => free.pop() ?? await create(), + release: (c: BrowserContext) => { free.push(c) }, + drain: async () => { await Promise.all(free.splice(0).map(c => c.close().catch(() => {}))) } + } +} diff --git a/benchmark/lib/pngjs.d.ts b/benchmark/lib/pngjs.d.ts new file mode 100644 index 0000000..efe140c --- /dev/null +++ b/benchmark/lib/pngjs.d.ts @@ -0,0 +1,4 @@ +// the subset of pngjs (a transitive dependency, via get-pixels) used by the real pages experiment +declare module 'pngjs' { + export const PNG: { sync: { read: (buffer: Buffer) => { width: number, height: number, data: Buffer } } } +} diff --git a/benchmark/lib/proc.ts b/benchmark/lib/proc.ts new file mode 100644 index 0000000..51e62a3 --- /dev/null +++ b/benchmark/lib/proc.ts @@ -0,0 +1,72 @@ +import fs from 'node:fs' + +// CPU and memory of a browser: the launched process and all its descendants (zygotes, renderers, +// gpu, network service...), read from /proc. Linux only. + +const CLK_TCK = 100 + +const readStat = (pid: number) => { + try { + const raw = fs.readFileSync(`/proc/${pid}/stat`, 'utf8') + // the command name is between parentheses and may contain spaces + const fields = raw.slice(raw.lastIndexOf(')') + 2).split(' ') + return { + ppid: Number(fields[1]), + // utime + stime + cutime + cstime: the children reaped by a process (renderers closed with + // their page) are accounted in the cumulated fields of their parent + ticks: Number(fields[11]) + Number(fields[12]) + Number(fields[13]) + Number(fields[14]) + } + } catch { + return null + } +} + +export const processTree = (rootPid: number) => { + const parents = new Map() + for (const entry of fs.readdirSync('/proc')) { + const pid = Number(entry) + if (!pid) continue + const stat = readStat(pid) + if (stat) parents.set(pid, stat.ppid) + } + const tree = new Set([rootPid]) + let grown = true + while (grown) { + grown = false + for (const [pid, ppid] of parents) { + if (!tree.has(pid) && tree.has(ppid)) { + tree.add(pid) + grown = true + } + } + } + return [...tree] +} + +// cumulated CPU milliseconds of the tree +export const treeCpuMs = (rootPid: number) => { + let ticks = 0 + for (const pid of processTree(rootPid)) ticks += readStat(pid)?.ticks ?? 0 + return ticks * 1000 / CLK_TCK +} + +// proportional set size of the tree in MB: shared pages are split between the processes that map +// them, so the sum is not inflated like a sum of RSS +export const treePssMB = (rootPid: number) => { + let kb = 0 + for (const pid of processTree(rootPid)) { + try { + const m = fs.readFileSync(`/proc/${pid}/smaps_rollup`, 'utf8').match(/^Pss:\s+(\d+) kB/m) + if (m) kb += Number(m[1]) + } catch {} + } + return Math.round(kb / 1024) +} + +export const treeSize = (rootPid: number) => processTree(rootPid).length + +// CPU of the current node process +export const selfCpuMs = () => { + const u = process.cpuUsage() + return (u.user + u.system) / 1000 +} diff --git a/benchmark/lib/runner.ts b/benchmark/lib/runner.ts new file mode 100644 index 0000000..6c97fc8 --- /dev/null +++ b/benchmark/lib/runner.ts @@ -0,0 +1,96 @@ +import fs from 'node:fs' +import path from 'node:path' +import { execSync } from 'node:child_process' +import { aggregate, compare } from './stats.ts' + +export type Sample = Record +export type Results = Record + +// Runs every variant `runs` times, interleaved (A B C, B C A, C A B...) so that a drift of the +// machine load hits all the variants alike. Warmup iterations are run and discarded. +// the current step of a sample, named by the experiment with step(), reported if it hangs +let currentStep = '' +export const step = (name: string) => { currentStep = name } + +const watchdog = async (variant: string, i: number, fn: () => Promise, ms: number) => { + let timer: NodeJS.Timeout | undefined + currentStep = '' + try { + return await Promise.race([ + fn(), + new Promise((resolve, reject) => { + timer = setTimeout(() => reject(new Error(`watchdog: variant ${variant}, run ${i}, step "${currentStep}" did not end in ${ms}ms`)), ms) + }) + ]) + } finally { + clearTimeout(timer) + } +} + +export const runInterleaved = async ( + variants: string[], + runs: number, + fn: (variant: string, i: number) => Promise, + warmup = 1, + timeoutMs = 120000 +): Promise => { + const results: Results = Object.fromEntries(variants.map(v => [v, []])) + for (let w = 0; w < warmup; w++) for (const v of variants) await watchdog(v, -1, () => fn(v, -1), timeoutMs) + for (let i = 0; i < runs; i++) { + const order = variants.map((_, k) => variants[(k + i) % variants.length]) + for (const v of order) results[v].push(await watchdog(v, i, () => fn(v, i), timeoutMs)) + process.stdout.write('.') + } + process.stdout.write('\n') + return results +} + +const fmt = (x: number) => Number.isNaN(x) ? '-' : Math.abs(x) >= 100 ? Math.round(x).toString() : x.toFixed(1) + +// one table: a row per variant, a column per metric, median [min-max] and the ratio to the first +// variant. Boolean metrics are shown as a rate. +export const report = (title: string, results: Results, metrics: string[]) => { + const variants = Object.keys(results) + const base = variants[0] + const rows = [['variant', ...metrics]] + for (const v of variants) { + const row = [v] + for (const m of metrics) { + const values = results[v].map(s => s[m]) + if (typeof values[0] === 'boolean') { + row.push(`${values.filter(Boolean).length}/${values.length}`) + continue + } + if (typeof values[0] === 'string') { + row.push([...new Set(values)].join(',')) + continue + } + const agg = aggregate(values.filter(x => typeof x === 'number') as number[]) + let cell = `${fmt(agg.median)} [${fmt(agg.min)}-${fmt(agg.max)}]` + if (v !== base) { + const b = aggregate(results[base].map(s => s[m]).filter(x => typeof x === 'number') as number[]) + cell += ' ' + compare(b, agg) + } + row.push(cell) + } + rows.push(row) + } + const widths = rows[0].map((_, c) => Math.max(...rows.map(r => r[c].length))) + const line = (r: string[]) => '| ' + r.map((cell, c) => cell.padEnd(widths[c])).join(' | ') + ' |' + const out = [`\n### ${title}\n`, line(rows[0]), '|' + widths.map(w => '-'.repeat(w + 2)).join('|') + '|', ...rows.slice(1).map(line)].join('\n') + console.log(out) + return out +} + +const sha = () => { + try { + return execSync('git rev-parse --short HEAD', { encoding: 'utf8' }).trim() + (execSync('git status --porcelain', { encoding: 'utf8' }).trim() ? '-dirty' : '') + } catch { return 'unknown' } +} + +export const save = (experiment: string, data: unknown) => { + const dir = path.join(import.meta.dirname, '..', 'results') + const file = path.join(dir, `${experiment}-${new Date().toISOString().replace(/[:.]/g, '-')}.json`) + fs.writeFileSync(file, JSON.stringify({ experiment, commit: sha(), date: new Date().toISOString(), data }, null, 2)) + console.log(`\nsaved ${path.relative(process.cwd(), file)}`) +} diff --git a/benchmark/lib/server.ts b/benchmark/lib/server.ts new file mode 100644 index 0000000..7887c29 --- /dev/null +++ b/benchmark/lib/server.ts @@ -0,0 +1,68 @@ +import http from 'node:http' +import fs from 'node:fs' +import path from 'node:path' +import type { AddressInfo } from 'node:net' + +// Serves the fixtures pages, with an artificial latency (?delay=ms) on any response so that pages +// behave like data-fair applications that fetch from a remote API. +// /fixtures/.html a page +// /asset.js?kb=300&delay=80 a script of the given size (parsed and executed by the page) +// /data?n=500&delay=150 a JSON array of n random points +// /font.woff2?delay=500 a font (DejaVu Sans from the system, whatever the extension says) + +const fixturesDir = path.join(import.meta.dirname, '..', 'fixtures') + +const assetCache = new Map() +const asset = (kb: number) => { + let src = assetCache.get(kb) + if (!src) { + const lines = ['window.__lib = {}'] + let i = 0 + while (lines.join('\n').length < kb * 1024) { + lines.push(`window.__lib.f${i} = function (a, b) { const r = []; for (let k = 0; k < a; k++) r.push({ k, v: Math.sin(k * ${i}) * b, s: 'item-' + k }); return r.filter(x => x.v > 0).map(x => x.s).join(',') }`) + i++ + } + src = lines.join('\n') + assetCache.set(kb, src) + } + return src +} + +const data = (n: number) => { + const points = [] + for (let i = 0; i < n; i++) points.push({ x: i, y: Math.round(Math.abs(Math.sin(i / 10) * 100 + Math.random() * 20)), label: 'Point ' + i }) + return JSON.stringify(points) +} + +export const startFixturesServer = async () => { + const server = http.createServer((req, res) => { + const url = new URL(req.url ?? '/', 'http://localhost') + const delay = Number(url.searchParams.get('delay') ?? 0) + setTimeout(() => { + if (url.pathname.startsWith('/fixtures/')) { + const file = path.join(fixturesDir, path.basename(url.pathname)) + if (!fs.existsSync(file)) return res.writeHead(404).end() + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }) + res.end(fs.readFileSync(file)) + } else if (url.pathname === '/asset.js') { + res.writeHead(200, { 'content-type': 'application/javascript', 'cache-control': 'no-store' }) + res.end(asset(Number(url.searchParams.get('kb') ?? 100))) + } else if (url.pathname === '/font.woff2') { + res.writeHead(200, { 'content-type': 'font/ttf', 'cache-control': 'no-store' }) + res.end(fs.readFileSync('/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf')) + } else if (url.pathname === '/data') { + res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'no-store' }) + res.end(data(Number(url.searchParams.get('n') ?? 100))) + } else { + res.writeHead(404).end() + } + }, delay) + }) + server.keepAliveTimeout = 60000 + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + const port = (server.address() as AddressInfo).port + return { + url: (p: string) => `http://127.0.0.1:${port}${p}`, + close: () => new Promise(resolve => { server.closeAllConnections(); server.close(() => resolve()) }) + } +} diff --git a/benchmark/lib/stats.ts b/benchmark/lib/stats.ts new file mode 100644 index 0000000..b0b6f46 --- /dev/null +++ b/benchmark/lib/stats.ts @@ -0,0 +1,24 @@ +export type Agg = { n: number, median: number, p90: number, min: number, max: number, mean: number } + +export const aggregate = (samples: number[]): Agg => { + if (!samples.length) return { n: 0, median: NaN, p90: NaN, min: NaN, max: NaN, mean: NaN } + const s = [...samples].sort((a, b) => a - b) + const q = (p: number) => { + const i = (s.length - 1) * p + const lo = Math.floor(i) + const hi = Math.ceil(i) + return s[lo] + (s[hi] - s[lo]) * (i - lo) + } + return { n: s.length, median: q(0.5), p90: q(0.9), min: s[0], max: s[s.length - 1], mean: s.reduce((a, b) => a + b, 0) / s.length } +} + +// a variant is flagged against the baseline only when the min-max ranges do not overlap and the +// medians differ by more than 10%, the rule of the data-fair harness: noise on a loaded dev +// machine is large and a single median is not evidence +export const compare = (base: Agg, other: Agg) => { + if (!base.n || !other.n) return '' + const ratio = other.median / base.median + const disjoint = other.max < base.min || other.min > base.max + if (!disjoint || Math.abs(ratio - 1) < 0.1) return `x${ratio.toFixed(2)}` + return `x${ratio.toFixed(2)} ${ratio < 1 ? 'LOWER' : 'HIGHER'}` +} diff --git a/benchmark/results/.gitkeep b/benchmark/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json index b2e0c0a..29e826b 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "lint-fix": "eslint --fix .", "check-types": "tsc", "quality": "npm run lint && npm run build-types && npm run check-types && npm run test && npm audit --omit=dev --audit-level=critical", - "prepare": "husky || true" + "prepare": "husky || true", + "benchmark": "node --disable-warning=ExperimentalWarning benchmark/index.ts" }, "imports": { "#config": "./api/config.ts" From fc07fb8bfb8e3cdc277db94222aaa0f22c6a9504 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 09:39:04 +0100 Subject: [PATCH 02/10] perf: wait for network idle then visual quiet instead of networkidle0 + 1s Pages that do not call window.triggerCapture paid a 2s floor: Chrome's networkidle0 fires 0.5-1.5s after the last request, then a fixed 1s sleep. Now puppeteer's network idle (500ms) then 200ms without DOM mutation, animation frame, finite animation or font loading, capped at 2s (config.wait). On the public applications of opendata.koumoul.com that do not call triggerCapture: 2093ms -> 1075ms median, identical screenshots. Long chart animations are now waited for instead of captured half drawn. A capture-no-trigger warning names the pages that do not call triggerCapture. Large (config.pngFastEncodingPixels) and full page png screenshots are encoded for speed: the encoding runs on the browser's main thread, shared by all the captures. Fixes: type=jpg returned png bytes labelled image/jpeg, type=gif on a page without animation support returned png bytes labelled image/gif. Co-Authored-By: Claude Opus 5.5 --- api/routers/capture.ts | 19 +- api/utils/page.ts | 74 ++++- config/default.cjs | 10 +- config/type/.type/index.d.ts | 21 ++ config/type/.type/validate.js | 423 ++++++++++++++++++++++------- config/type/schema.json | 26 ++ test-it/capture.ts | 26 ++ test-it/resources/test-settle.html | 28 ++ 8 files changed, 508 insertions(+), 119 deletions(-) create mode 100644 test-it/resources/test-settle.html diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 8a57ba2..5e7a59d 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -113,14 +113,21 @@ router.get('/screenshot', auth, async (req, res, next) => { if (typeof req.query.filename === 'string') res.attachment(req.query.filename.replace('.png', '.gif')) res.send(buffer) } else { - const buffer = await page.screenshot({ fullPage: req.query.height === 'auto' }) - debug(`[${target}] png screenshot is taken`) - timer.type = 'png' + // a gif was asked from a page that does not support animation: a png is returned + const imageType = (type === 'jpg' || type === 'jpeg') ? 'jpeg' : 'png' + const extension = imageType === 'jpeg' ? type : 'png' + const fullPage = req.query.height === 'auto' + // png encoding runs on the browser's main thread, shared by all captures: the large + // captures are encoded for speed (cf benchmark/FINDINGS.md) + const optimizeForSpeed = imageType === 'png' && (fullPage || width * height > config.pngFastEncodingPixels) + const buffer = await page.screenshot({ fullPage, type: imageType, optimizeForSpeed }) + debug(`[${target}] ${imageType} screenshot is taken`) + timer.type = imageType if (req.query.timer === 'true') { res.send(timer.times) } else { - res.type(type) - if (typeof req.query.filename === 'string') res.attachment(req.query.filename.replace('.gif', '.' + type)) + res.type(extension) + if (typeof req.query.filename === 'string') res.attachment(req.query.filename.replace(/\.(gif|png|jpe?g)$/, '.' + extension)) res.send(buffer) } } @@ -263,7 +270,7 @@ router.get('/inspect', async (req, res) => { let buffer if (fullPage) { const height = await page.evaluate(() => document.documentElement.scrollHeight) - buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) + buffer = await page.screenshot({ captureBeyondViewport: true, optimizeForSpeed: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) } else { buffer = await page.screenshot() } diff --git a/api/utils/page.ts b/api/utils/page.ts index 47dab1f..85da7ef 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -257,8 +257,39 @@ const safeCleanContext = async (page: Page, openSuccess: boolean) => { } } -// quite complex strategy to wait for the page to be ready for capture. -// it can either explitly call a triggerCapture function or we wait for idle network + 1s +// injected before any script of the page: records the last time the page did visible work, read +// by waitForQuiet. A copy of benchmark/lib/capture.ts, where the strategy was measured +const installActivityTracker = () => { + const w = window as any + w.__captureLastActivity = performance.now() + const mark = () => { w.__captureLastActivity = performance.now() } + const raf = window.requestAnimationFrame.bind(window) + window.requestAnimationFrame = (cb) => raf((t) => { mark(); cb(t) }) + new MutationObserver(mark).observe(document, { subtree: true, childList: true, attributes: true, characterData: true }) +} + +// resolves when the page has been visually quiet for quietTime (no DOM mutation, no animation frame, +// no running finite animation, no font loading), or after maxSettleTime. Endless animations +// (spinners) are ignored, they never settle +const waitForQuiet = (page: Page) => page.evaluate((quietTime, maxSettleTime) => new Promise(resolve => { + const start = performance.now() + const check = () => { + const now = performance.now() + const busy = [] + if (now - ((window as any).__captureLastActivity ?? 0) < quietTime) busy.push('activity') + if (document.getAnimations().some(a => a.playState === 'running' && a.effect?.getTiming().iterations !== Infinity)) busy.push('animations') + if (document.fonts.status === 'loading') busy.push('fonts') + if (!busy.length) return resolve('quiet') + if (now - start >= maxSettleTime) return resolve('max settle time, busy with ' + busy.join(' and ')) + setTimeout(check, 50) + } + check() +}), config.wait.quietTime, config.wait.maxSettleTime) + +// the strategy to wait for the page to be ready for capture. +// the page can explicitly call a triggerCapture function, this is the fast and reliable way. +// otherwise we wait for the network to be idle, then for the page to be visually quiet (cf +// benchmark/FINDINGS.md for the measures behind this strategy) const waitForPage = async ( page: Page, target: string, @@ -269,6 +300,7 @@ const waitForPage = async ( let captureTriggered = false let timeoutReached = false let animationActivated = false + let settledBy = '' const triggerCapture = new Promise(resolve => page.exposeFunction('triggerCapture', (animationSupported?: boolean) => { captureTriggered = true animationActivated = !!(animate && animationSupported) @@ -276,18 +308,25 @@ const waitForPage = async ( // return animate to the page inside the capture return animate })) + await page.evaluateOnNewDocument(installActivityTracker) try { // wait for network inactivity, but it can be interrupted if triggerCapture is called - const waitOptions = { waitUntil: 'networkidle0' as const, timeout: config.screenshotTimeout } - await Promise.race([page.goto(target, waitOptions), triggerCapture]) + // puppeteer's own network idle: networkidle0 is Chrome's lifecycle event, its idleness + // detector fires 0.5 to 1.5s after the last request depending on the page's activity + const deadline = Date.now() + config.screenshotTimeout + const networkIdle = page.goto(target, { waitUntil: 'load', timeout: config.screenshotTimeout }) + .then(() => page.waitForNetworkIdle({ idleTime: config.wait.networkIdleTime, timeout: Math.max(1, deadline - Date.now()) })) + // the race may be won by triggerCapture, a later rejection (page closed) must not go unhandled + networkIdle.catch(() => {}) + await Promise.race([networkIdle, triggerCapture]) if (captureTriggered) { timer.step('wait1-capture-triggered') debug(`[${target}] capture was expicitly triggered by window.triggerCapture`) } else { timer.step('wait1-network-idle') - debug(`[${target}] network was idle during 500ms`) + debug(`[${target}] network was idle during ${config.wait.networkIdleTime}ms`) } } catch (err: any) { if (err.name !== 'TimeoutError') throw err @@ -302,9 +341,6 @@ const waitForPage = async ( // we are done here, capture was already explicitly triggered or we already waited for a long time } else { // network was found to be idle, adapt the wait strategy based on some meta - // Adapt the wait strategy based on the x-capture meta - - // Adapt the wait strategy based on the x-capture meta let captureDelayMeta try { captureDelayMeta = await page.$eval('head > meta[name=\'df:capture-delay\']', el => el.content) @@ -323,6 +359,7 @@ const waitForPage = async ( debug(`[${target}] capture was expicitly triggered by window.triggerCapture`) } else { timer.step('wait2-delay') + settledBy = `the df:capture-delay meta (${delay}ms)` debug(`[${target}] delay of ${delay / 1000} seconds was reached`) } } else { @@ -349,12 +386,27 @@ const waitForPage = async ( timeoutReached = true } } else { - debug(`[${target}] wait 1000ms more after idle network for safety`) - await new Promise(resolve => setTimeout(resolve, 1000)) - timer.step('wait2-1s') + // the page may still render after its last response (charts animations, vue rendering..) + const quiet = await waitForQuiet(page) + if (captureTriggered) { + timer.step('wait2-capture-triggered') + } else { + timer.step('wait2-settle') + settledBy = quiet === 'quiet' ? 'network idle then visual quiet' : 'network idle then ' + quiet + debug(`[${target}] settled: ${settledBy}`) + } } } } + if (!captureTriggered) { + // the heuristics are slower and less reliable than an explicit signal, this guides the + // applications that should call window.triggerCapture when they are rendered + log.warn('captured page did not call window.triggerCapture', { + code: 'capture-no-trigger', + target, + settledBy: timeoutReached ? `the timeout (${config.screenshotTimeout}ms)` : settledBy + }) + } const ready: Ready = captureTriggered ? 'triggered' : timeoutReached ? 'timeout' : 'idle' return { animationActivated, ready } } diff --git a/config/default.cjs b/config/default.cjs index c43bb0a..19a9d11 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -7,6 +7,15 @@ module.exports = { active: true }, screenshotTimeout: 20000, + // for pages that do not call window.triggerCapture: network idle, then visual quiet, capped + wait: { + networkIdleTime: 500, + quietTime: 200, + maxSettleTime: 2000 + }, + // PNG encoding runs on the main thread of the browser, shared by all the captures: the large + // ones are encoded fast (2-3x quicker, 1.1-1.5x bigger files) + pngFastEncodingPixels: 1500000, concurrency: 5, inspectConcurrency: 1, // concurrent /inspect renders, each one holds a browser page defaultLang: 'fr-FR', @@ -18,7 +27,6 @@ module.exports = { maxAnimationFrames: 1800, // 2 minutes at 15fps puppeteerLaunchOptions: { executablePath: '/usr/bin/google-chrome-stable', - // args: ['--use-gl=egl', '--use-angle=swiftshader', '--in-process-gpu'], args: [], headless: true }, diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index 5023df9..d600a81 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -28,6 +28,27 @@ export type ApiConfig = { onlySameHost?: boolean; useHostHeader?: boolean; screenshotTimeout: number; + /** + * How long a page that does not call window.triggerCapture is waited for + */ + wait: { + /** + * milliseconds without any network request before the page is considered loaded + */ + networkIdleTime: number; + /** + * then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture + */ + quietTime: number; + /** + * the longest wait for that quiet, after the network is idle + */ + maxSettleTime: number; + }; + /** + * above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size + */ + pngFastEncodingPixels: number; maxAnimationFrames: number; util?: unknown; get?: unknown; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index 156d33f..06b07cb 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","wait","pngFastEncodingPixels","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"wait":{"type":"object","description":"How long a page that does not call window.triggerCapture is waited for","additionalProperties":false,"required":["networkIdleTime","quietTime","maxSettleTime"],"properties":{"networkIdleTime":{"type":"number","description":"milliseconds without any network request before the page is considered loaded"},"quietTime":{"type":"number","description":"then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture"},"maxSettleTime":{"type":"number","description":"the longest wait for that quiet, after the network is idle"}}},"pngFastEncodingPixels":{"type":"number","description":"above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -113,8 +113,8 @@ vErrors.push(err9); } errors++; } -if(data.maxAnimationFrames === undefined){ -const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(data.wait === undefined){ +const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "wait"},message:"must have required property '"+"wait"+"'"}; if(vErrors === null){ vErrors = [err10]; } @@ -123,9 +123,8 @@ vErrors.push(err10); } errors++; } -for(const key0 in data){ -if(!(func2.call(schema16.properties, key0))){ -const err11 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(data.pngFastEncodingPixels === undefined){ +const err11 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "pngFastEncodingPixels"},message:"must have required property '"+"pngFastEncodingPixels"+"'"}; if(vErrors === null){ vErrors = [err11]; } @@ -134,6 +133,27 @@ vErrors.push(err11); } errors++; } +if(data.maxAnimationFrames === undefined){ +const err12 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(vErrors === null){ +vErrors = [err12]; +} +else { +vErrors.push(err12); +} +errors++; +} +for(const key0 in data){ +if(!(func2.call(schema16.properties, key0))){ +const err13 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err13]; +} +else { +vErrors.push(err13); +} +errors++; +} } if(data.port !== undefined){ let data0 = data.port; @@ -153,12 +173,12 @@ if(dataType0 == "boolean" || data0 === null coerced0 = +data0; } else { -const err12 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err14 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err12]; +vErrors = [err14]; } else { -vErrors.push(err12); +vErrors.push(err14); } errors++; } @@ -191,12 +211,12 @@ else if(data1 === null){ coerced1 = ""; } else { -const err13 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err15 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err13]; +vErrors = [err15]; } else { -vErrors.push(err13); +vErrors.push(err15); } errors++; } @@ -229,12 +249,12 @@ else if(data2 === null){ coerced2 = ""; } else { -const err14 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err16 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err14]; +vErrors = [err16]; } else { -vErrors.push(err14); +vErrors.push(err16); } errors++; } @@ -248,12 +268,12 @@ data["privateDirectoryUrl"] = coerced2; } if(typeof data2 === "string"){ if(!pattern0.test(data2)){ -const err15 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; +const err17 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; if(vErrors === null){ -vErrors = [err15]; +vErrors = [err17]; } else { -vErrors.push(err15); +vErrors.push(err17); } errors++; } @@ -263,23 +283,23 @@ if(data.helmet !== undefined){ let data3 = data.helmet; if(data3 && typeof data3 == "object" && !Array.isArray(data3)){ if(data3.active === undefined){ -const err16 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; +const err18 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; if(vErrors === null){ -vErrors = [err16]; +vErrors = [err18]; } else { -vErrors.push(err16); +vErrors.push(err18); } errors++; } for(const key1 in data3){ if(!(key1 === "active")){ -const err17 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; +const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err17]; +vErrors = [err19]; } else { -vErrors.push(err17); +vErrors.push(err19); } errors++; } @@ -304,12 +324,12 @@ else if(data4 === "true" || data4 === 1){ coerced3 = true; } else { -const err18 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err20 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err18]; +vErrors = [err20]; } else { -vErrors.push(err18); +vErrors.push(err20); } errors++; } @@ -324,12 +344,12 @@ data3["active"] = coerced3; } } else { -const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err21 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err19]; +vErrors = [err21]; } else { -vErrors.push(err19); +vErrors.push(err21); } errors++; } @@ -338,23 +358,23 @@ if(data.secretKeys !== undefined){ let data5 = data.secretKeys; if(data5 && typeof data5 == "object" && !Array.isArray(data5)){ if(data5.capture === undefined){ -const err20 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; +const err22 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; if(vErrors === null){ -vErrors = [err20]; +vErrors = [err22]; } else { -vErrors.push(err20); +vErrors.push(err22); } errors++; } for(const key2 in data5){ if(!((key2 === "capture") || (key2 === "inspect"))){ -const err21 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; +const err23 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err21]; +vErrors = [err23]; } else { -vErrors.push(err21); +vErrors.push(err23); } errors++; } @@ -379,12 +399,12 @@ else if(data6 === null){ coerced4 = ""; } else { -const err22 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err24 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err22]; +vErrors = [err24]; } else { -vErrors.push(err22); +vErrors.push(err24); } errors++; } @@ -417,12 +437,12 @@ else if(data7 === null){ coerced5 = ""; } else { -const err23 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err25 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err23]; +vErrors = [err25]; } else { -vErrors.push(err23); +vErrors.push(err25); } errors++; } @@ -437,12 +457,12 @@ data5["inspect"] = coerced5; } } else { -const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err26 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err24]; +vErrors = [err26]; } else { -vErrors.push(err24); +vErrors.push(err26); } errors++; } @@ -470,12 +490,12 @@ else if(data9 === "true" || data9 === 1){ coerced6 = true; } else { -const err25 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err27 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err25]; +vErrors = [err27]; } else { -vErrors.push(err25); +vErrors.push(err27); } errors++; } @@ -506,12 +526,12 @@ if(dataType7 == "boolean" || data10 === null coerced7 = +data10; } else { -const err26 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err28 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err26]; +vErrors = [err28]; } else { -vErrors.push(err26); +vErrors.push(err28); } errors++; } @@ -526,12 +546,12 @@ data8["port"] = coerced7; } } else { -const err27 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err29 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err27]; +vErrors = [err29]; } else { -vErrors.push(err27); +vErrors.push(err29); } errors++; } @@ -539,12 +559,12 @@ errors++; if(data.puppeteerLaunchOptions !== undefined){ let data11 = data.puppeteerLaunchOptions; if(!(data11 && typeof data11 == "object" && !Array.isArray(data11))){ -const err28 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err30 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err28]; +vErrors = [err30]; } else { -vErrors.push(err28); +vErrors.push(err30); } errors++; } @@ -567,12 +587,12 @@ if(dataType8 === "boolean" || data12 === null coerced8 = +data12; } else { -const err29 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err31 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err29]; +vErrors = [err31]; } else { -vErrors.push(err29); +vErrors.push(err31); } errors++; } @@ -603,12 +623,12 @@ if(dataType9 === "boolean" || data13 === null coerced9 = +data13; } else { -const err30 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err32 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err30]; +vErrors = [err32]; } else { -vErrors.push(err30); +vErrors.push(err32); } errors++; } @@ -622,12 +642,12 @@ data["inspectConcurrency"] = coerced9; } if(typeof data13 == "number"){ if(data13 < 1 || isNaN(data13)){ -const err31 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +const err33 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err33]; } else { -vErrors.push(err31); +vErrors.push(err33); } errors++; } @@ -653,12 +673,12 @@ else if(data14 === null){ coerced10 = ""; } else { -const err32 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err34 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err32]; +vErrors = [err34]; } else { -vErrors.push(err32); +vErrors.push(err34); } errors++; } @@ -691,12 +711,12 @@ else if(data15 === null){ coerced11 = ""; } else { -const err33 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err35 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err33]; +vErrors = [err35]; } else { -vErrors.push(err33); +vErrors.push(err35); } errors++; } @@ -729,12 +749,12 @@ else if(data16 === "true" || data16 === 1){ coerced12 = true; } else { -const err34 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err36 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err34]; +vErrors = [err36]; } else { -vErrors.push(err34); +vErrors.push(err36); } errors++; } @@ -767,12 +787,12 @@ else if(data17 === "true" || data17 === 1){ coerced13 = true; } else { -const err35 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err37 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err35]; +vErrors = [err37]; } else { -vErrors.push(err35); +vErrors.push(err37); } errors++; } @@ -803,12 +823,12 @@ if(dataType14 == "boolean" || data18 === null coerced14 = +data18; } else { -const err36 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err38 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err36]; +vErrors = [err38]; } else { -vErrors.push(err36); +vErrors.push(err38); } errors++; } @@ -821,50 +841,251 @@ data["screenshotTimeout"] = coerced14; } } } -if(data.maxAnimationFrames !== undefined){ -let data19 = data.maxAnimationFrames; -if(!(typeof data19 == "number")){ -let dataType15 = typeof data19; +if(data.wait !== undefined){ +let data19 = data.wait; +if(data19 && typeof data19 == "object" && !Array.isArray(data19)){ +if(data19.networkIdleTime === undefined){ +const err39 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "networkIdleTime"},message:"must have required property '"+"networkIdleTime"+"'"}; +if(vErrors === null){ +vErrors = [err39]; +} +else { +vErrors.push(err39); +} +errors++; +} +if(data19.quietTime === undefined){ +const err40 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "quietTime"},message:"must have required property '"+"quietTime"+"'"}; +if(vErrors === null){ +vErrors = [err40]; +} +else { +vErrors.push(err40); +} +errors++; +} +if(data19.maxSettleTime === undefined){ +const err41 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "maxSettleTime"},message:"must have required property '"+"maxSettleTime"+"'"}; +if(vErrors === null){ +vErrors = [err41]; +} +else { +vErrors.push(err41); +} +errors++; +} +for(const key3 in data19){ +if(!(((key3 === "networkIdleTime") || (key3 === "quietTime")) || (key3 === "maxSettleTime"))){ +const err42 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key3},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err42]; +} +else { +vErrors.push(err42); +} +errors++; +} +} +if(data19.networkIdleTime !== undefined){ +let data20 = data19.networkIdleTime; +if(!(typeof data20 == "number")){ +let dataType15 = typeof data20; let coerced15 = undefined; -if(dataType15 == 'object' && Array.isArray(data19) && data19.length == 1){ -data19 = data19[0]; -dataType15 = typeof data19; -if(typeof data19 == "number"){ -coerced15 = data19; +if(dataType15 == 'object' && Array.isArray(data20) && data20.length == 1){ +data20 = data20[0]; +dataType15 = typeof data20; +if(typeof data20 == "number"){ +coerced15 = data20; } } if(!(coerced15 !== undefined)){ -if(dataType15 == "boolean" || data19 === null - || (dataType15 == "string" && data19 && data19 == +data19)){ -coerced15 = +data19; +if(dataType15 == "boolean" || data20 === null + || (dataType15 == "string" && data20 && data20 == +data20)){ +coerced15 = +data20; } else { -const err37 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err43 = {instancePath:instancePath+"/wait/networkIdleTime",schemaPath:"#/properties/wait/properties/networkIdleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err37]; +vErrors = [err43]; } else { -vErrors.push(err37); +vErrors.push(err43); } errors++; } } if(coerced15 !== undefined){ -data19 = coerced15; +data20 = coerced15; +if(data19 !== undefined){ +data19["networkIdleTime"] = coerced15; +} +} +} +} +if(data19.quietTime !== undefined){ +let data21 = data19.quietTime; +if(!(typeof data21 == "number")){ +let dataType16 = typeof data21; +let coerced16 = undefined; +if(dataType16 == 'object' && Array.isArray(data21) && data21.length == 1){ +data21 = data21[0]; +dataType16 = typeof data21; +if(typeof data21 == "number"){ +coerced16 = data21; +} +} +if(!(coerced16 !== undefined)){ +if(dataType16 == "boolean" || data21 === null + || (dataType16 == "string" && data21 && data21 == +data21)){ +coerced16 = +data21; +} +else { +const err44 = {instancePath:instancePath+"/wait/quietTime",schemaPath:"#/properties/wait/properties/quietTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err44]; +} +else { +vErrors.push(err44); +} +errors++; +} +} +if(coerced16 !== undefined){ +data21 = coerced16; +if(data19 !== undefined){ +data19["quietTime"] = coerced16; +} +} +} +} +if(data19.maxSettleTime !== undefined){ +let data22 = data19.maxSettleTime; +if(!(typeof data22 == "number")){ +let dataType17 = typeof data22; +let coerced17 = undefined; +if(dataType17 == 'object' && Array.isArray(data22) && data22.length == 1){ +data22 = data22[0]; +dataType17 = typeof data22; +if(typeof data22 == "number"){ +coerced17 = data22; +} +} +if(!(coerced17 !== undefined)){ +if(dataType17 == "boolean" || data22 === null + || (dataType17 == "string" && data22 && data22 == +data22)){ +coerced17 = +data22; +} +else { +const err45 = {instancePath:instancePath+"/wait/maxSettleTime",schemaPath:"#/properties/wait/properties/maxSettleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err45]; +} +else { +vErrors.push(err45); +} +errors++; +} +} +if(coerced17 !== undefined){ +data22 = coerced17; +if(data19 !== undefined){ +data19["maxSettleTime"] = coerced17; +} +} +} +} +} +else { +const err46 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err46]; +} +else { +vErrors.push(err46); +} +errors++; +} +} +if(data.pngFastEncodingPixels !== undefined){ +let data23 = data.pngFastEncodingPixels; +if(!(typeof data23 == "number")){ +let dataType18 = typeof data23; +let coerced18 = undefined; +if(dataType18 == 'object' && Array.isArray(data23) && data23.length == 1){ +data23 = data23[0]; +dataType18 = typeof data23; +if(typeof data23 == "number"){ +coerced18 = data23; +} +} +if(!(coerced18 !== undefined)){ +if(dataType18 == "boolean" || data23 === null + || (dataType18 == "string" && data23 && data23 == +data23)){ +coerced18 = +data23; +} +else { +const err47 = {instancePath:instancePath+"/pngFastEncodingPixels",schemaPath:"#/properties/pngFastEncodingPixels/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err47]; +} +else { +vErrors.push(err47); +} +errors++; +} +} +if(coerced18 !== undefined){ +data23 = coerced18; if(data !== undefined){ -data["maxAnimationFrames"] = coerced15; +data["pngFastEncodingPixels"] = coerced18; } } } } +if(data.maxAnimationFrames !== undefined){ +let data24 = data.maxAnimationFrames; +if(!(typeof data24 == "number")){ +let dataType19 = typeof data24; +let coerced19 = undefined; +if(dataType19 == 'object' && Array.isArray(data24) && data24.length == 1){ +data24 = data24[0]; +dataType19 = typeof data24; +if(typeof data24 == "number"){ +coerced19 = data24; +} +} +if(!(coerced19 !== undefined)){ +if(dataType19 == "boolean" || data24 === null + || (dataType19 == "string" && data24 && data24 == +data24)){ +coerced19 = +data24; } else { -const err38 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err48 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err38]; +vErrors = [err48]; } else { -vErrors.push(err38); +vErrors.push(err48); +} +errors++; +} +} +if(coerced19 !== undefined){ +data24 = coerced19; +if(data !== undefined){ +data["maxAnimationFrames"] = coerced19; +} +} +} +} +} +else { +const err49 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err49]; +} +else { +vErrors.push(err49); } errors++; } diff --git a/config/type/schema.json b/config/type/schema.json index a6e24bc..f9d71a6 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -21,6 +21,8 @@ "defaultLang", "defaultTimezone", "screenshotTimeout", + "wait", + "pngFastEncodingPixels", "maxAnimationFrames" ], "properties": { @@ -93,6 +95,30 @@ "screenshotTimeout": { "type": "number" }, + "wait": { + "type": "object", + "description": "How long a page that does not call window.triggerCapture is waited for", + "additionalProperties": false, + "required": ["networkIdleTime", "quietTime", "maxSettleTime"], + "properties": { + "networkIdleTime": { + "type": "number", + "description": "milliseconds without any network request before the page is considered loaded" + }, + "quietTime": { + "type": "number", + "description": "then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture" + }, + "maxSettleTime": { + "type": "number", + "description": "the longest wait for that quiet, after the network is idle" + } + } + }, + "pngFastEncodingPixels": { + "type": "number", + "description": "above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size" + }, "maxAnimationFrames": { "type": "number" }, diff --git a/test-it/capture.ts b/test-it/capture.ts index fcd2863..8ba65bc 100644 --- a/test-it/capture.ts +++ b/test-it/capture.ts @@ -58,6 +58,32 @@ describe('capture', () => { assert.equal(res.headers['content-disposition'], 'attachment; filename=screenshot.png') }) + it('return jpeg images, and png ones when a gif is asked from a page without animation', async () => { + let res = await ax.get('screenshot', { params: { key, target, type: 'jpg', filename: 'screenshot.jpg' }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.equal(res.headers['content-type'], 'image/jpeg') + assert.equal(res.headers['content-disposition'], 'attachment; filename=screenshot.jpg') + assert.deepEqual([...new Uint8Array(res.data).subarray(0, 3)], [0xff, 0xd8, 0xff]) + + // test1.html calls triggerCapture() without declaring animation support + res = await ax.get('screenshot', { params: { key, target, type: 'gif', filename: 'screenshot.gif' }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.equal(res.headers['content-type'], 'image/png') + assert.equal(res.headers['content-disposition'], 'attachment; filename=screenshot.png') + assert.equal(Buffer.from(res.data).subarray(1, 4).toString(), 'PNG') + }) + + it('wait for a page without triggerCapture to settle', async () => { + const settleTarget = origin + '/test/resources/test-settle.html' + const res = await ax.get('print', { params: { key, target: settleTarget, type: 'html' } }) + assert.equal(res.status, 200) + assert.ok(res.data.includes('animation done'), 'the capture was taken before the end of the animation') + + const times = (await ax.get('screenshot', { params: { key, target: origin + '/test/resources/test-clean.html', timer: 'true' } })).data + assert.ok('wait1-network-idle' in times) + assert.ok('wait2-settle' in times) + }) + it('make a few prints of a page', async () => { let res = await ax.get('print', { params: { key, target }, responseType: 'arraybuffer' }) assert.equal(res.status, 200) diff --git a/test-it/resources/test-settle.html b/test-it/resources/test-settle.html new file mode 100644 index 0000000..507c99c --- /dev/null +++ b/test-it/resources/test-settle.html @@ -0,0 +1,28 @@ + + + + + Settle page + + +

Settle page

+ +

loading

+ + + From b611c2514002fe8b9d312ec1afdbc579d68aafab Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 09:39:22 +0100 Subject: [PATCH 03/10] perf: attach the debug listeners of captured pages only when DEBUG is enabled The console listener did 2 CDP round trips per argument of every console message even with the debug output off: node CPU x4 on a page logging 500 messages (benchmark: console). Co-Authored-By: Claude Opus 5.5 --- api/utils/page.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/utils/page.ts b/api/utils/page.ts index 85da7ef..838039d 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -206,7 +206,9 @@ export const withPage = async ( } } } - debugPage(page) + // the console listener costs 2 CDP round trips per argument of every console message, on the + // main thread of the service: only paid when the debug output is enabled (DEBUG=capture) + if (debug.enabled) debugPage(page) inspect?.onPage?.(page) timer.step('newPage') From 532c1bed347a77bc276279f810a20bad02ad13f5 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 09:42:34 +0100 Subject: [PATCH 04/10] perf: drop the captures of clients that left, bound the contexts queue Captures waiting for a browser context were kept whatever happened to their client: after a proxy timeout or a user leaving, they were still rendered for nobody and delayed the next ones (benchmark e2e: 30 abandoned requests made a 0.7s capture wait 18s). The routes now pass an AbortSignal, aborted when the response closes before being sent: the capture is skipped if it is still queued, stopped otherwise (499, not logged as a failure). The queue is bounded (config.queue, QUEUE_MAX_WAITING and QUEUE_MAX_WAIT_TIME): beyond 100 waiting captures, or after 20s of waiting, a 503 that the caller can retry instead of an answer after the proxy gave up. promiseTimeout now clears its timer. Co-Authored-By: Claude Opus 5.5 --- api/routers/capture.ts | 21 +- api/utils/page.ts | 74 ++- config/custom-environment-variables.cjs | 4 + config/default.cjs | 6 + config/type/.type/index.d.ts | 13 + config/type/.type/validate.js | 763 ++++++++++++++---------- config/type/schema.json | 18 + test-it/capture.ts | 16 + 8 files changed, 590 insertions(+), 325 deletions(-) diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 5e7a59d..8ae8a23 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -18,6 +18,15 @@ const inspectSemaphore = createSemaphore(config.inspectConcurrency) export const router = express.Router() +// aborted when the client closes the request before its response is sent (a proxy timeout, a +// user who left): the capture is dropped instead of being rendered for nobody +const clientSignal = (res: Response) => { + const controller = new AbortController() + if (res.destroyed) controller.abort() + else res.on('close', () => { if (!res.writableFinished) controller.abort() }) + return controller.signal +} + async function auth (req: Request, res: Response, next: NextFunction) { if (config.privateDirectoryUrl) { const user = reqSession(req).user @@ -131,7 +140,9 @@ router.get('/screenshot', auth, async (req, res, next) => { res.send(buffer) } } - } + }, + undefined, + clientSignal(res) ) timer.finish() }) @@ -198,7 +209,9 @@ router.get('/print', auth, async (req, res, next) => { if (typeof req.query.filename === 'string') res.attachment(req.query.filename) res.send(buffer.toString()) } - } + }, + undefined, + clientSignal(res) ) timer.finish() }) @@ -253,6 +266,7 @@ router.get('/inspect', async (req, res) => { const timer = createTimer(req.originalUrl, 'inspect') let collector: ReturnType | undefined + const signal = clientSignal(res) await inspectSemaphore.run(async () => { // the client may have left while waiting for a slot, no need to render for nobody if (req.socket.destroyed || res.destroyed) return @@ -281,7 +295,8 @@ router.get('/inspect', async (req, res) => { onPage: page => { collector = createReportCollector(page, target) }, declaredErrorUrl, onDeclaredError: body => collector?.declareError(body) - } + }, + signal ) }) timer.finish() diff --git a/api/utils/page.ts b/api/utils/page.ts index 838039d..53f794f 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -33,7 +33,11 @@ export const start = async () => { await context.close() } } - _contextPool = genericPool.createPool(contextFactory, { max: config.concurrency }) + _contextPool = genericPool.createPool(contextFactory, { + max: config.concurrency, + maxWaitingClients: config.queue.maxWaiting, + acquireTimeoutMillis: config.queue.maxWaitTime + }) // auto reconnection, cf https://github.com/GoogleChrome/puppeteer/issues/4428 _browser.on('disconnected', async () => { @@ -166,10 +170,50 @@ async function openInPage ( } export const promiseTimeout = async (promise: Promise, ms: number, msg = 'timeout') => { - return Promise.race([ - promise, - new Promise((resolve, reject) => setTimeout(() => reject(new Error(msg)), ms)) - ]) + let timer: NodeJS.Timeout | undefined + try { + return await Promise.race([ + promise, + new Promise((resolve, reject) => { timer = setTimeout(() => reject(new Error(msg)), ms) }) + ]) + } finally { + clearTimeout(timer) + } +} + +// the client left (proxy timeout, user gone): the render is stopped, nobody would receive it. +// 499 is nginx's "client closed request", below 500 it is not logged as a failure +const clientGone = () => httpError(499, 'client closed the request') + +// rejects as soon as the signal aborts, the promise keeps running but its page gets closed +const unlessAborted = async (promise: Promise, signal: AbortSignal | undefined) => { + if (!signal) return promise + if (signal.aborted) throw clientGone() + let onAbort: (() => void) | undefined + try { + return await Promise.race([ + promise, + new Promise((resolve, reject) => { + onAbort = () => reject(clientGone()) + signal.addEventListener('abort', onAbort, { once: true }) + }) + ]) + } finally { + if (onAbort) signal.removeEventListener('abort', onAbort) + } +} + +// a full queue or a long wait for a context mean an overloaded service: better a quick 503 +// that the caller can retry than a response after the proxy has given up +const acquireContext = async () => { + try { + return await _contextPool.acquire() + } catch (err: any) { + if (err.name === 'TimeoutError' || err.message?.includes('max waitingClients')) { + throw httpError(503, 'the capture service is overloaded, retry later') + } + throw err + } } export const withPage = async ( @@ -182,7 +226,9 @@ export const withPage = async ( timer: Timer, callbackTimeoutMsg: string, callback: (result: OpenPageResult) => Promise, - inspect?: InspectOptions) => { + inspect?: InspectOptions, + // aborted when the client closes the request + signal?: AbortSignal) => { if (target.includes('capture-test-error=true')) { await new Promise(resolve => setTimeout(resolve, 1000)) throw new Error('forced error trigger') @@ -193,8 +239,14 @@ export const withPage = async ( let pageAttempts = 0 while (!page) { pageAttempts++ - context = await _contextPool.acquire() + if (signal?.aborted) throw clientGone() + context = await acquireContext() timer.step('acquire-context') + // the client may have left while waiting for a context, no need to render for nobody + if (signal?.aborted) { + await _contextPool.release(context) + throw clientGone() + } try { page = await context.newPage() } catch (err) { @@ -212,18 +264,18 @@ export const withPage = async ( inspect?.onPage?.(page) timer.step('newPage') - const result = await promiseTimeout( + const result = await unlessAborted(promiseTimeout( openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), config.screenshotTimeout * 2, `Failed to open "${target}" in context before timeout` - ) + ), signal) timer.step('open-in-page') openSuccess = true - await promiseTimeout( + await unlessAborted(promiseTimeout( callback(result), result.animationActivated ? config.screenshotTimeout * 2 : config.screenshotTimeout, callbackTimeoutMsg - ) + ), signal) } finally { if (page) await safeCleanContext(page, !!openSuccess) } diff --git a/config/custom-environment-variables.cjs b/config/custom-environment-variables.cjs index 597219c..a292843 100644 --- a/config/custom-environment-variables.cjs +++ b/config/custom-environment-variables.cjs @@ -10,6 +10,10 @@ module.exports = { screenshotTimeout: 'SCREENSHOT_TIMEOUT', concurrency: 'CONCURRENCY', inspectConcurrency: 'INSPECT_CONCURRENCY', + queue: { + maxWaiting: 'QUEUE_MAX_WAITING', + maxWaitTime: 'QUEUE_MAX_WAIT_TIME' + }, defaultLang: 'DEFAULT_LANG', defaultTimezone: 'DEFAULT_TIMEZONE', secretKeys: { diff --git a/config/default.cjs b/config/default.cjs index 19a9d11..d5efb72 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -17,6 +17,12 @@ module.exports = { // ones are encoded fast (2-3x quicker, 1.1-1.5x bigger files) pngFastEncodingPixels: 1500000, concurrency: 5, + // captures waiting for a browser context: beyond maxWaiting, or after maxWaitTime of waiting, + // they are answered by a 503 (below the 60s timeout of the reverse proxy) + queue: { + maxWaiting: 100, + maxWaitTime: 20000 + }, inspectConcurrency: 1, // concurrent /inspect renders, each one holds a browser page defaultLang: 'fr-FR', defaultTimezone: 'Europe/Paris', diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index d600a81..69a74a0 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -22,6 +22,19 @@ export type ApiConfig = { [k: string]: unknown; }; concurrency: number; + /** + * Captures waiting for a browser context + */ + queue: { + /** + * beyond this number of waiting captures, new ones are answered by a 503 + */ + maxWaiting: number; + /** + * milliseconds of waiting for a context before a 503 + */ + maxWaitTime: number; + }; inspectConcurrency: number; defaultLang: string; defaultTimezone: string; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index 06b07cb..d686b00 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","wait","pngFastEncodingPixels","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"wait":{"type":"object","description":"How long a page that does not call window.triggerCapture is waited for","additionalProperties":false,"required":["networkIdleTime","quietTime","maxSettleTime"],"properties":{"networkIdleTime":{"type":"number","description":"milliseconds without any network request before the page is considered loaded"},"quietTime":{"type":"number","description":"then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture"},"maxSettleTime":{"type":"number","description":"the longest wait for that quiet, after the network is idle"}}},"pngFastEncodingPixels":{"type":"number","description":"above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","queue","defaultLang","defaultTimezone","screenshotTimeout","wait","pngFastEncodingPixels","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"queue":{"type":"object","description":"Captures waiting for a browser context","additionalProperties":false,"required":["maxWaiting","maxWaitTime"],"properties":{"maxWaiting":{"type":"integer","minimum":0,"description":"beyond this number of waiting captures, new ones are answered by a 503"},"maxWaitTime":{"type":"number","description":"milliseconds of waiting for a context before a 503"}}},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"wait":{"type":"object","description":"How long a page that does not call window.triggerCapture is waited for","additionalProperties":false,"required":["networkIdleTime","quietTime","maxSettleTime"],"properties":{"networkIdleTime":{"type":"number","description":"milliseconds without any network request before the page is considered loaded"},"quietTime":{"type":"number","description":"then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture"},"maxSettleTime":{"type":"number","description":"the longest wait for that quiet, after the network is idle"}}},"pngFastEncodingPixels":{"type":"number","description":"above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -83,8 +83,8 @@ vErrors.push(err6); } errors++; } -if(data.defaultLang === undefined){ -const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; +if(data.queue === undefined){ +const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "queue"},message:"must have required property '"+"queue"+"'"}; if(vErrors === null){ vErrors = [err7]; } @@ -93,8 +93,8 @@ vErrors.push(err7); } errors++; } -if(data.defaultTimezone === undefined){ -const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; +if(data.defaultLang === undefined){ +const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; if(vErrors === null){ vErrors = [err8]; } @@ -103,8 +103,8 @@ vErrors.push(err8); } errors++; } -if(data.screenshotTimeout === undefined){ -const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; +if(data.defaultTimezone === undefined){ +const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; if(vErrors === null){ vErrors = [err9]; } @@ -113,8 +113,8 @@ vErrors.push(err9); } errors++; } -if(data.wait === undefined){ -const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "wait"},message:"must have required property '"+"wait"+"'"}; +if(data.screenshotTimeout === undefined){ +const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; if(vErrors === null){ vErrors = [err10]; } @@ -123,8 +123,8 @@ vErrors.push(err10); } errors++; } -if(data.pngFastEncodingPixels === undefined){ -const err11 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "pngFastEncodingPixels"},message:"must have required property '"+"pngFastEncodingPixels"+"'"}; +if(data.wait === undefined){ +const err11 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "wait"},message:"must have required property '"+"wait"+"'"}; if(vErrors === null){ vErrors = [err11]; } @@ -133,8 +133,8 @@ vErrors.push(err11); } errors++; } -if(data.maxAnimationFrames === undefined){ -const err12 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(data.pngFastEncodingPixels === undefined){ +const err12 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "pngFastEncodingPixels"},message:"must have required property '"+"pngFastEncodingPixels"+"'"}; if(vErrors === null){ vErrors = [err12]; } @@ -143,9 +143,8 @@ vErrors.push(err12); } errors++; } -for(const key0 in data){ -if(!(func2.call(schema16.properties, key0))){ -const err13 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(data.maxAnimationFrames === undefined){ +const err13 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; if(vErrors === null){ vErrors = [err13]; } @@ -154,6 +153,17 @@ vErrors.push(err13); } errors++; } +for(const key0 in data){ +if(!(func2.call(schema16.properties, key0))){ +const err14 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err14]; +} +else { +vErrors.push(err14); +} +errors++; +} } if(data.port !== undefined){ let data0 = data.port; @@ -173,12 +183,12 @@ if(dataType0 == "boolean" || data0 === null coerced0 = +data0; } else { -const err14 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err15 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err14]; +vErrors = [err15]; } else { -vErrors.push(err14); +vErrors.push(err15); } errors++; } @@ -211,12 +221,12 @@ else if(data1 === null){ coerced1 = ""; } else { -const err15 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err16 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err15]; +vErrors = [err16]; } else { -vErrors.push(err15); +vErrors.push(err16); } errors++; } @@ -249,12 +259,12 @@ else if(data2 === null){ coerced2 = ""; } else { -const err16 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err17 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err16]; +vErrors = [err17]; } else { -vErrors.push(err16); +vErrors.push(err17); } errors++; } @@ -268,12 +278,12 @@ data["privateDirectoryUrl"] = coerced2; } if(typeof data2 === "string"){ if(!pattern0.test(data2)){ -const err17 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; +const err18 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; if(vErrors === null){ -vErrors = [err17]; +vErrors = [err18]; } else { -vErrors.push(err17); +vErrors.push(err18); } errors++; } @@ -283,23 +293,23 @@ if(data.helmet !== undefined){ let data3 = data.helmet; if(data3 && typeof data3 == "object" && !Array.isArray(data3)){ if(data3.active === undefined){ -const err18 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; +const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; if(vErrors === null){ -vErrors = [err18]; +vErrors = [err19]; } else { -vErrors.push(err18); +vErrors.push(err19); } errors++; } for(const key1 in data3){ if(!(key1 === "active")){ -const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; +const err20 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err19]; +vErrors = [err20]; } else { -vErrors.push(err19); +vErrors.push(err20); } errors++; } @@ -324,12 +334,12 @@ else if(data4 === "true" || data4 === 1){ coerced3 = true; } else { -const err20 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err21 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err20]; +vErrors = [err21]; } else { -vErrors.push(err20); +vErrors.push(err21); } errors++; } @@ -344,12 +354,12 @@ data3["active"] = coerced3; } } else { -const err21 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err22 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err21]; +vErrors = [err22]; } else { -vErrors.push(err21); +vErrors.push(err22); } errors++; } @@ -358,23 +368,23 @@ if(data.secretKeys !== undefined){ let data5 = data.secretKeys; if(data5 && typeof data5 == "object" && !Array.isArray(data5)){ if(data5.capture === undefined){ -const err22 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; +const err23 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; if(vErrors === null){ -vErrors = [err22]; +vErrors = [err23]; } else { -vErrors.push(err22); +vErrors.push(err23); } errors++; } for(const key2 in data5){ if(!((key2 === "capture") || (key2 === "inspect"))){ -const err23 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; +const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err23]; +vErrors = [err24]; } else { -vErrors.push(err23); +vErrors.push(err24); } errors++; } @@ -399,12 +409,12 @@ else if(data6 === null){ coerced4 = ""; } else { -const err24 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err25 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err24]; +vErrors = [err25]; } else { -vErrors.push(err24); +vErrors.push(err25); } errors++; } @@ -437,12 +447,12 @@ else if(data7 === null){ coerced5 = ""; } else { -const err25 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err26 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err25]; +vErrors = [err26]; } else { -vErrors.push(err25); +vErrors.push(err26); } errors++; } @@ -457,12 +467,12 @@ data5["inspect"] = coerced5; } } else { -const err26 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err27 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err26]; +vErrors = [err27]; } else { -vErrors.push(err26); +vErrors.push(err27); } errors++; } @@ -490,12 +500,12 @@ else if(data9 === "true" || data9 === 1){ coerced6 = true; } else { -const err27 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err28 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err27]; +vErrors = [err28]; } else { -vErrors.push(err27); +vErrors.push(err28); } errors++; } @@ -526,12 +536,12 @@ if(dataType7 == "boolean" || data10 === null coerced7 = +data10; } else { -const err28 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err29 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err28]; +vErrors = [err29]; } else { -vErrors.push(err28); +vErrors.push(err29); } errors++; } @@ -546,12 +556,12 @@ data8["port"] = coerced7; } } else { -const err29 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err30 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err29]; +vErrors = [err30]; } else { -vErrors.push(err29); +vErrors.push(err30); } errors++; } @@ -559,12 +569,12 @@ errors++; if(data.puppeteerLaunchOptions !== undefined){ let data11 = data.puppeteerLaunchOptions; if(!(data11 && typeof data11 == "object" && !Array.isArray(data11))){ -const err30 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err31 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err30]; +vErrors = [err31]; } else { -vErrors.push(err30); +vErrors.push(err31); } errors++; } @@ -587,12 +597,12 @@ if(dataType8 === "boolean" || data12 === null coerced8 = +data12; } else { -const err31 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err32 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err32]; } else { -vErrors.push(err31); +vErrors.push(err32); } errors++; } @@ -605,325 +615,339 @@ data["concurrency"] = coerced8; } } } -if(data.inspectConcurrency !== undefined){ -let data13 = data.inspectConcurrency; -if(!((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13)))){ -let dataType9 = typeof data13; +if(data.queue !== undefined){ +let data13 = data.queue; +if(data13 && typeof data13 == "object" && !Array.isArray(data13)){ +if(data13.maxWaiting === undefined){ +const err33 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaiting"},message:"must have required property '"+"maxWaiting"+"'"}; +if(vErrors === null){ +vErrors = [err33]; +} +else { +vErrors.push(err33); +} +errors++; +} +if(data13.maxWaitTime === undefined){ +const err34 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaitTime"},message:"must have required property '"+"maxWaitTime"+"'"}; +if(vErrors === null){ +vErrors = [err34]; +} +else { +vErrors.push(err34); +} +errors++; +} +for(const key3 in data13){ +if(!((key3 === "maxWaiting") || (key3 === "maxWaitTime"))){ +const err35 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key3},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err35]; +} +else { +vErrors.push(err35); +} +errors++; +} +} +if(data13.maxWaiting !== undefined){ +let data14 = data13.maxWaiting; +if(!((typeof data14 == "number") && (!(data14 % 1) && !isNaN(data14)))){ +let dataType9 = typeof data14; let coerced9 = undefined; -if(dataType9 == 'object' && Array.isArray(data13) && data13.length == 1){ -data13 = data13[0]; -dataType9 = typeof data13; -if((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13))){ -coerced9 = data13; +if(dataType9 == 'object' && Array.isArray(data14) && data14.length == 1){ +data14 = data14[0]; +dataType9 = typeof data14; +if((typeof data14 == "number") && (!(data14 % 1) && !isNaN(data14))){ +coerced9 = data14; } } if(!(coerced9 !== undefined)){ -if(dataType9 === "boolean" || data13 === null - || (dataType9 === "string" && data13 && data13 == +data13 && !(data13 % 1))){ -coerced9 = +data13; +if(dataType9 === "boolean" || data14 === null + || (dataType9 === "string" && data14 && data14 == +data14 && !(data14 % 1))){ +coerced9 = +data14; } else { -const err32 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err36 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err32]; +vErrors = [err36]; } else { -vErrors.push(err32); +vErrors.push(err36); } errors++; } } if(coerced9 !== undefined){ -data13 = coerced9; -if(data !== undefined){ -data["inspectConcurrency"] = coerced9; +data14 = coerced9; +if(data13 !== undefined){ +data13["maxWaiting"] = coerced9; } } } -if(typeof data13 == "number"){ -if(data13 < 1 || isNaN(data13)){ -const err33 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(typeof data14 == "number"){ +if(data14 < 0 || isNaN(data14)){ +const err37 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/minimum",keyword:"minimum",params:{comparison: ">=", limit: 0},message:"must be >= 0"}; if(vErrors === null){ -vErrors = [err33]; +vErrors = [err37]; } else { -vErrors.push(err33); +vErrors.push(err37); } errors++; } } } -if(data.defaultLang !== undefined){ -let data14 = data.defaultLang; -if(typeof data14 !== "string"){ -let dataType10 = typeof data14; +if(data13.maxWaitTime !== undefined){ +let data15 = data13.maxWaitTime; +if(!(typeof data15 == "number")){ +let dataType10 = typeof data15; let coerced10 = undefined; -if(dataType10 == 'object' && Array.isArray(data14) && data14.length == 1){ -data14 = data14[0]; -dataType10 = typeof data14; -if(typeof data14 === "string"){ -coerced10 = data14; +if(dataType10 == 'object' && Array.isArray(data15) && data15.length == 1){ +data15 = data15[0]; +dataType10 = typeof data15; +if(typeof data15 == "number"){ +coerced10 = data15; } } if(!(coerced10 !== undefined)){ -if(dataType10 == "number" || dataType10 == "boolean"){ -coerced10 = "" + data14; -} -else if(data14 === null){ -coerced10 = ""; +if(dataType10 == "boolean" || data15 === null + || (dataType10 == "string" && data15 && data15 == +data15)){ +coerced10 = +data15; } else { -const err34 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err38 = {instancePath:instancePath+"/queue/maxWaitTime",schemaPath:"#/properties/queue/properties/maxWaitTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err34]; +vErrors = [err38]; } else { -vErrors.push(err34); +vErrors.push(err38); } errors++; } } if(coerced10 !== undefined){ -data14 = coerced10; -if(data !== undefined){ -data["defaultLang"] = coerced10; +data15 = coerced10; +if(data13 !== undefined){ +data13["maxWaitTime"] = coerced10; } } } } -if(data.defaultTimezone !== undefined){ -let data15 = data.defaultTimezone; -if(typeof data15 !== "string"){ -let dataType11 = typeof data15; +} +else { +const err39 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err39]; +} +else { +vErrors.push(err39); +} +errors++; +} +} +if(data.inspectConcurrency !== undefined){ +let data16 = data.inspectConcurrency; +if(!((typeof data16 == "number") && (!(data16 % 1) && !isNaN(data16)))){ +let dataType11 = typeof data16; let coerced11 = undefined; -if(dataType11 == 'object' && Array.isArray(data15) && data15.length == 1){ -data15 = data15[0]; -dataType11 = typeof data15; -if(typeof data15 === "string"){ -coerced11 = data15; +if(dataType11 == 'object' && Array.isArray(data16) && data16.length == 1){ +data16 = data16[0]; +dataType11 = typeof data16; +if((typeof data16 == "number") && (!(data16 % 1) && !isNaN(data16))){ +coerced11 = data16; } } if(!(coerced11 !== undefined)){ -if(dataType11 == "number" || dataType11 == "boolean"){ -coerced11 = "" + data15; -} -else if(data15 === null){ -coerced11 = ""; +if(dataType11 === "boolean" || data16 === null + || (dataType11 === "string" && data16 && data16 == +data16 && !(data16 % 1))){ +coerced11 = +data16; } else { -const err35 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err40 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err35]; +vErrors = [err40]; } else { -vErrors.push(err35); +vErrors.push(err40); } errors++; } } if(coerced11 !== undefined){ -data15 = coerced11; +data16 = coerced11; if(data !== undefined){ -data["defaultTimezone"] = coerced11; +data["inspectConcurrency"] = coerced11; } } } +if(typeof data16 == "number"){ +if(data16 < 1 || isNaN(data16)){ +const err41 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err41]; } -if(data.onlySameHost !== undefined){ -let data16 = data.onlySameHost; -if(typeof data16 !== "boolean"){ -let dataType12 = typeof data16; +else { +vErrors.push(err41); +} +errors++; +} +} +} +if(data.defaultLang !== undefined){ +let data17 = data.defaultLang; +if(typeof data17 !== "string"){ +let dataType12 = typeof data17; let coerced12 = undefined; -if(dataType12 == 'object' && Array.isArray(data16) && data16.length == 1){ -data16 = data16[0]; -dataType12 = typeof data16; -if(typeof data16 === "boolean"){ -coerced12 = data16; +if(dataType12 == 'object' && Array.isArray(data17) && data17.length == 1){ +data17 = data17[0]; +dataType12 = typeof data17; +if(typeof data17 === "string"){ +coerced12 = data17; } } if(!(coerced12 !== undefined)){ -if(data16 === "false" || data16 === 0 || data16 === null){ -coerced12 = false; +if(dataType12 == "number" || dataType12 == "boolean"){ +coerced12 = "" + data17; } -else if(data16 === "true" || data16 === 1){ -coerced12 = true; +else if(data17 === null){ +coerced12 = ""; } else { -const err36 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err42 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err36]; +vErrors = [err42]; } else { -vErrors.push(err36); +vErrors.push(err42); } errors++; } } if(coerced12 !== undefined){ -data16 = coerced12; +data17 = coerced12; if(data !== undefined){ -data["onlySameHost"] = coerced12; +data["defaultLang"] = coerced12; } } } } -if(data.useHostHeader !== undefined){ -let data17 = data.useHostHeader; -if(typeof data17 !== "boolean"){ -let dataType13 = typeof data17; +if(data.defaultTimezone !== undefined){ +let data18 = data.defaultTimezone; +if(typeof data18 !== "string"){ +let dataType13 = typeof data18; let coerced13 = undefined; -if(dataType13 == 'object' && Array.isArray(data17) && data17.length == 1){ -data17 = data17[0]; -dataType13 = typeof data17; -if(typeof data17 === "boolean"){ -coerced13 = data17; +if(dataType13 == 'object' && Array.isArray(data18) && data18.length == 1){ +data18 = data18[0]; +dataType13 = typeof data18; +if(typeof data18 === "string"){ +coerced13 = data18; } } if(!(coerced13 !== undefined)){ -if(data17 === "false" || data17 === 0 || data17 === null){ -coerced13 = false; +if(dataType13 == "number" || dataType13 == "boolean"){ +coerced13 = "" + data18; } -else if(data17 === "true" || data17 === 1){ -coerced13 = true; +else if(data18 === null){ +coerced13 = ""; } else { -const err37 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err43 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err37]; +vErrors = [err43]; } else { -vErrors.push(err37); +vErrors.push(err43); } errors++; } } if(coerced13 !== undefined){ -data17 = coerced13; +data18 = coerced13; if(data !== undefined){ -data["useHostHeader"] = coerced13; +data["defaultTimezone"] = coerced13; } } } } -if(data.screenshotTimeout !== undefined){ -let data18 = data.screenshotTimeout; -if(!(typeof data18 == "number")){ -let dataType14 = typeof data18; +if(data.onlySameHost !== undefined){ +let data19 = data.onlySameHost; +if(typeof data19 !== "boolean"){ +let dataType14 = typeof data19; let coerced14 = undefined; -if(dataType14 == 'object' && Array.isArray(data18) && data18.length == 1){ -data18 = data18[0]; -dataType14 = typeof data18; -if(typeof data18 == "number"){ -coerced14 = data18; +if(dataType14 == 'object' && Array.isArray(data19) && data19.length == 1){ +data19 = data19[0]; +dataType14 = typeof data19; +if(typeof data19 === "boolean"){ +coerced14 = data19; } } if(!(coerced14 !== undefined)){ -if(dataType14 == "boolean" || data18 === null - || (dataType14 == "string" && data18 && data18 == +data18)){ -coerced14 = +data18; +if(data19 === "false" || data19 === 0 || data19 === null){ +coerced14 = false; +} +else if(data19 === "true" || data19 === 1){ +coerced14 = true; } else { -const err38 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err44 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err38]; +vErrors = [err44]; } else { -vErrors.push(err38); +vErrors.push(err44); } errors++; } } if(coerced14 !== undefined){ -data18 = coerced14; +data19 = coerced14; if(data !== undefined){ -data["screenshotTimeout"] = coerced14; -} -} -} -} -if(data.wait !== undefined){ -let data19 = data.wait; -if(data19 && typeof data19 == "object" && !Array.isArray(data19)){ -if(data19.networkIdleTime === undefined){ -const err39 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "networkIdleTime"},message:"must have required property '"+"networkIdleTime"+"'"}; -if(vErrors === null){ -vErrors = [err39]; -} -else { -vErrors.push(err39); -} -errors++; -} -if(data19.quietTime === undefined){ -const err40 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "quietTime"},message:"must have required property '"+"quietTime"+"'"}; -if(vErrors === null){ -vErrors = [err40]; -} -else { -vErrors.push(err40); +data["onlySameHost"] = coerced14; } -errors++; } -if(data19.maxSettleTime === undefined){ -const err41 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "maxSettleTime"},message:"must have required property '"+"maxSettleTime"+"'"}; -if(vErrors === null){ -vErrors = [err41]; } -else { -vErrors.push(err41); -} -errors++; -} -for(const key3 in data19){ -if(!(((key3 === "networkIdleTime") || (key3 === "quietTime")) || (key3 === "maxSettleTime"))){ -const err42 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key3},message:"must NOT have additional properties"}; -if(vErrors === null){ -vErrors = [err42]; } -else { -vErrors.push(err42); -} -errors++; -} -} -if(data19.networkIdleTime !== undefined){ -let data20 = data19.networkIdleTime; -if(!(typeof data20 == "number")){ +if(data.useHostHeader !== undefined){ +let data20 = data.useHostHeader; +if(typeof data20 !== "boolean"){ let dataType15 = typeof data20; let coerced15 = undefined; if(dataType15 == 'object' && Array.isArray(data20) && data20.length == 1){ data20 = data20[0]; dataType15 = typeof data20; -if(typeof data20 == "number"){ +if(typeof data20 === "boolean"){ coerced15 = data20; } } if(!(coerced15 !== undefined)){ -if(dataType15 == "boolean" || data20 === null - || (dataType15 == "string" && data20 && data20 == +data20)){ -coerced15 = +data20; +if(data20 === "false" || data20 === 0 || data20 === null){ +coerced15 = false; +} +else if(data20 === "true" || data20 === 1){ +coerced15 = true; } else { -const err43 = {instancePath:instancePath+"/wait/networkIdleTime",schemaPath:"#/properties/wait/properties/networkIdleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err45 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err43]; +vErrors = [err45]; } else { -vErrors.push(err43); +vErrors.push(err45); } errors++; } } if(coerced15 !== undefined){ data20 = coerced15; -if(data19 !== undefined){ -data19["networkIdleTime"] = coerced15; +if(data !== undefined){ +data["useHostHeader"] = coerced15; } } } } -if(data19.quietTime !== undefined){ -let data21 = data19.quietTime; +if(data.screenshotTimeout !== undefined){ +let data21 = data.screenshotTimeout; if(!(typeof data21 == "number")){ let dataType16 = typeof data21; let coerced16 = undefined; @@ -940,152 +964,269 @@ if(dataType16 == "boolean" || data21 === null coerced16 = +data21; } else { -const err44 = {instancePath:instancePath+"/wait/quietTime",schemaPath:"#/properties/wait/properties/quietTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err46 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err44]; +vErrors = [err46]; } else { -vErrors.push(err44); +vErrors.push(err46); } errors++; } } if(coerced16 !== undefined){ data21 = coerced16; -if(data19 !== undefined){ -data19["quietTime"] = coerced16; +if(data !== undefined){ +data["screenshotTimeout"] = coerced16; } } } } -if(data19.maxSettleTime !== undefined){ -let data22 = data19.maxSettleTime; -if(!(typeof data22 == "number")){ -let dataType17 = typeof data22; -let coerced17 = undefined; -if(dataType17 == 'object' && Array.isArray(data22) && data22.length == 1){ -data22 = data22[0]; -dataType17 = typeof data22; -if(typeof data22 == "number"){ -coerced17 = data22; +if(data.wait !== undefined){ +let data22 = data.wait; +if(data22 && typeof data22 == "object" && !Array.isArray(data22)){ +if(data22.networkIdleTime === undefined){ +const err47 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "networkIdleTime"},message:"must have required property '"+"networkIdleTime"+"'"}; +if(vErrors === null){ +vErrors = [err47]; } +else { +vErrors.push(err47); } -if(!(coerced17 !== undefined)){ -if(dataType17 == "boolean" || data22 === null - || (dataType17 == "string" && data22 && data22 == +data22)){ -coerced17 = +data22; +errors++; +} +if(data22.quietTime === undefined){ +const err48 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "quietTime"},message:"must have required property '"+"quietTime"+"'"}; +if(vErrors === null){ +vErrors = [err48]; } else { -const err45 = {instancePath:instancePath+"/wait/maxSettleTime",schemaPath:"#/properties/wait/properties/maxSettleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +vErrors.push(err48); +} +errors++; +} +if(data22.maxSettleTime === undefined){ +const err49 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "maxSettleTime"},message:"must have required property '"+"maxSettleTime"+"'"}; if(vErrors === null){ -vErrors = [err45]; +vErrors = [err49]; } else { -vErrors.push(err45); +vErrors.push(err49); } errors++; } +for(const key4 in data22){ +if(!(((key4 === "networkIdleTime") || (key4 === "quietTime")) || (key4 === "maxSettleTime"))){ +const err50 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key4},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err50]; } -if(coerced17 !== undefined){ -data22 = coerced17; -if(data19 !== undefined){ -data19["maxSettleTime"] = coerced17; +else { +vErrors.push(err50); } +errors++; } } +if(data22.networkIdleTime !== undefined){ +let data23 = data22.networkIdleTime; +if(!(typeof data23 == "number")){ +let dataType17 = typeof data23; +let coerced17 = undefined; +if(dataType17 == 'object' && Array.isArray(data23) && data23.length == 1){ +data23 = data23[0]; +dataType17 = typeof data23; +if(typeof data23 == "number"){ +coerced17 = data23; +} } +if(!(coerced17 !== undefined)){ +if(dataType17 == "boolean" || data23 === null + || (dataType17 == "string" && data23 && data23 == +data23)){ +coerced17 = +data23; } else { -const err46 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err51 = {instancePath:instancePath+"/wait/networkIdleTime",schemaPath:"#/properties/wait/properties/networkIdleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err46]; +vErrors = [err51]; } else { -vErrors.push(err46); +vErrors.push(err51); } errors++; } } -if(data.pngFastEncodingPixels !== undefined){ -let data23 = data.pngFastEncodingPixels; -if(!(typeof data23 == "number")){ -let dataType18 = typeof data23; +if(coerced17 !== undefined){ +data23 = coerced17; +if(data22 !== undefined){ +data22["networkIdleTime"] = coerced17; +} +} +} +} +if(data22.quietTime !== undefined){ +let data24 = data22.quietTime; +if(!(typeof data24 == "number")){ +let dataType18 = typeof data24; let coerced18 = undefined; -if(dataType18 == 'object' && Array.isArray(data23) && data23.length == 1){ -data23 = data23[0]; -dataType18 = typeof data23; -if(typeof data23 == "number"){ -coerced18 = data23; +if(dataType18 == 'object' && Array.isArray(data24) && data24.length == 1){ +data24 = data24[0]; +dataType18 = typeof data24; +if(typeof data24 == "number"){ +coerced18 = data24; } } if(!(coerced18 !== undefined)){ -if(dataType18 == "boolean" || data23 === null - || (dataType18 == "string" && data23 && data23 == +data23)){ -coerced18 = +data23; +if(dataType18 == "boolean" || data24 === null + || (dataType18 == "string" && data24 && data24 == +data24)){ +coerced18 = +data24; } else { -const err47 = {instancePath:instancePath+"/pngFastEncodingPixels",schemaPath:"#/properties/pngFastEncodingPixels/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err52 = {instancePath:instancePath+"/wait/quietTime",schemaPath:"#/properties/wait/properties/quietTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err47]; +vErrors = [err52]; } else { -vErrors.push(err47); +vErrors.push(err52); } errors++; } } if(coerced18 !== undefined){ -data23 = coerced18; -if(data !== undefined){ -data["pngFastEncodingPixels"] = coerced18; +data24 = coerced18; +if(data22 !== undefined){ +data22["quietTime"] = coerced18; } } } } -if(data.maxAnimationFrames !== undefined){ -let data24 = data.maxAnimationFrames; -if(!(typeof data24 == "number")){ -let dataType19 = typeof data24; +if(data22.maxSettleTime !== undefined){ +let data25 = data22.maxSettleTime; +if(!(typeof data25 == "number")){ +let dataType19 = typeof data25; let coerced19 = undefined; -if(dataType19 == 'object' && Array.isArray(data24) && data24.length == 1){ -data24 = data24[0]; -dataType19 = typeof data24; -if(typeof data24 == "number"){ -coerced19 = data24; +if(dataType19 == 'object' && Array.isArray(data25) && data25.length == 1){ +data25 = data25[0]; +dataType19 = typeof data25; +if(typeof data25 == "number"){ +coerced19 = data25; } } if(!(coerced19 !== undefined)){ -if(dataType19 == "boolean" || data24 === null - || (dataType19 == "string" && data24 && data24 == +data24)){ -coerced19 = +data24; +if(dataType19 == "boolean" || data25 === null + || (dataType19 == "string" && data25 && data25 == +data25)){ +coerced19 = +data25; } else { -const err48 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err53 = {instancePath:instancePath+"/wait/maxSettleTime",schemaPath:"#/properties/wait/properties/maxSettleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err48]; +vErrors = [err53]; } else { -vErrors.push(err48); +vErrors.push(err53); } errors++; } } if(coerced19 !== undefined){ -data24 = coerced19; +data25 = coerced19; +if(data22 !== undefined){ +data22["maxSettleTime"] = coerced19; +} +} +} +} +} +else { +const err54 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err54]; +} +else { +vErrors.push(err54); +} +errors++; +} +} +if(data.pngFastEncodingPixels !== undefined){ +let data26 = data.pngFastEncodingPixels; +if(!(typeof data26 == "number")){ +let dataType20 = typeof data26; +let coerced20 = undefined; +if(dataType20 == 'object' && Array.isArray(data26) && data26.length == 1){ +data26 = data26[0]; +dataType20 = typeof data26; +if(typeof data26 == "number"){ +coerced20 = data26; +} +} +if(!(coerced20 !== undefined)){ +if(dataType20 == "boolean" || data26 === null + || (dataType20 == "string" && data26 && data26 == +data26)){ +coerced20 = +data26; +} +else { +const err55 = {instancePath:instancePath+"/pngFastEncodingPixels",schemaPath:"#/properties/pngFastEncodingPixels/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err55]; +} +else { +vErrors.push(err55); +} +errors++; +} +} +if(coerced20 !== undefined){ +data26 = coerced20; if(data !== undefined){ -data["maxAnimationFrames"] = coerced19; +data["pngFastEncodingPixels"] = coerced20; +} } } } +if(data.maxAnimationFrames !== undefined){ +let data27 = data.maxAnimationFrames; +if(!(typeof data27 == "number")){ +let dataType21 = typeof data27; +let coerced21 = undefined; +if(dataType21 == 'object' && Array.isArray(data27) && data27.length == 1){ +data27 = data27[0]; +dataType21 = typeof data27; +if(typeof data27 == "number"){ +coerced21 = data27; +} } +if(!(coerced21 !== undefined)){ +if(dataType21 == "boolean" || data27 === null + || (dataType21 == "string" && data27 && data27 == +data27)){ +coerced21 = +data27; } else { -const err49 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err56 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err49]; +vErrors = [err56]; } else { -vErrors.push(err49); +vErrors.push(err56); +} +errors++; +} +} +if(coerced21 !== undefined){ +data27 = coerced21; +if(data !== undefined){ +data["maxAnimationFrames"] = coerced21; +} +} +} +} +} +else { +const err57 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err57]; +} +else { +vErrors.push(err57); } errors++; } diff --git a/config/type/schema.json b/config/type/schema.json index f9d71a6..801b661 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -18,6 +18,7 @@ "puppeteerLaunchOptions", "concurrency", "inspectConcurrency", + "queue", "defaultLang", "defaultTimezone", "screenshotTimeout", @@ -76,6 +77,23 @@ "concurrency": { "type": "integer" }, + "queue": { + "type": "object", + "description": "Captures waiting for a browser context", + "additionalProperties": false, + "required": ["maxWaiting", "maxWaitTime"], + "properties": { + "maxWaiting": { + "type": "integer", + "minimum": 0, + "description": "beyond this number of waiting captures, new ones are answered by a 503" + }, + "maxWaitTime": { + "type": "number", + "description": "milliseconds of waiting for a context before a 503" + } + } + }, "inspectConcurrency": { "type": "integer", "minimum": 1 diff --git a/test-it/capture.ts b/test-it/capture.ts index 8ba65bc..8a399b4 100644 --- a/test-it/capture.ts +++ b/test-it/capture.ts @@ -84,6 +84,22 @@ describe('capture', () => { assert.ok('wait2-settle' in times) }) + it('drop the captures of clients that left', async () => { + // more abandoned captures than browser contexts (concurrency 5) of a page that is never idle, + // each would hold a context for the whole screenshot timeout (2s in tests) + const busyTarget = origin + '/test/resources/test-busy.html' + const abandoned = [] + for (let i = 0; i < 15; i++) { + abandoned.push(ax.get('screenshot', { params: { key, target: busyTarget }, timeout: 300 }).catch(err => err.code)) + } + assert.ok((await Promise.all(abandoned)).every(code => code === 'ECONNABORTED')) + // 3 rounds of 5 abandoned captures would delay this one by ~6s if they were rendered + const t0 = Date.now() + const res = await ax.get('screenshot', { params: { key, target }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.ok(Date.now() - t0 < 3500, `the capture waited ${Date.now() - t0}ms behind abandoned ones`) + }) + it('make a few prints of a page', async () => { let res = await ax.get('print', { params: { key, target }, responseType: 'arraybuffer' }) assert.equal(res.status, 200) From 01eb6e298bcd618a4509543e0149a2865b399ccf Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 09:45:47 +0100 Subject: [PATCH 05/10] perf: encode the animated captures in piscina worker threads The png decoding and the gif encoding (pure JS, NeuQuant quantization) of an animated capture ran on the main thread: ~4s of CPU for 30 frames at 800x450, stalling every other request of the service meanwhile. The frames are now collected during the capture and encoded by a worker thread (same libraries, same output). Benchmark e2e, a probe requesting api-docs every 20ms during a gif capture: p90 83ms -> 1.2ms, max 131ms -> 4.6ms. config.gifThreads (GIF_THREADS, default 2) also bounds the number of gifs encoded at the same time. tmp-promise is not needed anymore. Co-Authored-By: Claude Opus 5.5 --- api/server.ts | 2 + api/threads/gif-encode.ts | 31 ++ api/utils/animation.ts | 50 ++-- config/custom-environment-variables.cjs | 1 + config/default.cjs | 1 + config/type/.type/index.d.ts | 4 + config/type/.type/validate.js | 324 ++++++++++++--------- config/type/schema.json | 8 +- package-lock.json | 360 ++++++++++++++++++++++-- package.json | 4 +- 10 files changed, 603 insertions(+), 182 deletions(-) create mode 100644 api/threads/gif-encode.ts diff --git a/api/server.ts b/api/server.ts index a90d464..6208cfa 100644 --- a/api/server.ts +++ b/api/server.ts @@ -7,6 +7,7 @@ import { createHttpTerminator } from 'http-terminator' import app from './app.ts' import * as pageUtils from './utils/page.ts' import * as gifsicle from './utils/gifsicle.ts' +import * as animation from './utils/animation.ts' import config from '#config' const server = createServer(app) @@ -38,4 +39,5 @@ export const stop = async () => { await httpTerminator.terminate() if (config.observer.active) await stopObserver() await pageUtils.stop() + await animation.stop() } diff --git a/api/threads/gif-encode.ts b/api/threads/gif-encode.ts new file mode 100644 index 0000000..7a94840 --- /dev/null +++ b/api/threads/gif-encode.ts @@ -0,0 +1,31 @@ +// Runs in a piscina worker thread: decodes the png frames of an animated capture and encodes them +// as a gif. Both are pure JS and CPU heavy (NeuQuant quantization), on the main thread they +// blocked every other request of the service for seconds (cf benchmark/FINDINGS.md). +import { promisify } from 'node:util' +import GifEncoder from 'gif-encoder' +import getPixelsCb from 'get-pixels' +import { move } from 'piscina' + +const getPixels = promisify(getPixelsCb) + +export type GifEncodeTask = { width: number, height: number, frameRate: number, frames: Uint8Array[] } + +export default async ({ width, height, frameRate, frames }: GifEncodeTask) => { + const gif = new GifEncoder(width, height, { highWaterMark: 64 * 1024 * 1024 }) + const chunks: Buffer[] = [] + gif.on('data', (chunk: Buffer) => chunks.push(chunk)) + // finish() pushes the end of the stream, the last chunks are emitted asynchronously + const ended = new Promise(resolve => gif.once('end', () => resolve())) + gif.setFrameRate(frameRate) + gif.writeHeader() + for (const frame of frames) { + // get-pixels switches on Buffer.isBuffer to tell raw image data from a file path + const pixels = await getPixels(Buffer.from(frame.buffer, frame.byteOffset, frame.byteLength), 'image/png') + gif.addFrame(pixels.data) + } + gif.finish() + await ended + const result = Buffer.concat(chunks) + // transferred to the main thread, not copied + return move(new Uint8Array(result.buffer, result.byteOffset, result.byteLength)) +} diff --git a/api/utils/animation.ts b/api/utils/animation.ts index bc2c7aa..c9eed30 100644 --- a/api/utils/animation.ts +++ b/api/utils/animation.ts @@ -1,29 +1,30 @@ -import { file } from 'tmp-promise' +import path from 'node:path' +import { Piscina } from 'piscina' import config from '#config' -import fs from 'node:fs' -import { promisify } from 'node:util' -import { pipeline } from 'node:stream/promises' -import GifEncoder from 'gif-encoder' -import getPixelsCb from 'get-pixels' import { optimizeGif } from './gifsicle.ts' import debugModule from 'debug' +import log from '@data-fair/lib-node/log.js' import { type Page } from 'puppeteer' +import type { GifEncodeTask } from '../threads/gif-encode.ts' const debug = debugModule('capture') -const getPixels = promisify(getPixelsCb) +// the gif encoding is CPU heavy pure JS, it runs in worker threads so that it does not block the +// service. maxThreads also bounds the number of gifs encoded at the same time +export const gifEncodePiscina = new Piscina({ + filename: path.resolve(import.meta.dirname, '../threads/gif-encode.ts'), + minThreads: 0, + idleTimeout: 10 * 60 * 1000, + maxThreads: config.gifThreads +}) +// a pool emits 'error' when a thread dies with no task in flight; without a listener it crashes the process +gifEncodePiscina.on('error', (err) => log.error('gif encoding worker error', { code: 'piscina-gif-encode', err })) export const capture = async (target: string, page: Page, width: number, height: number) => { let stopped = false - const gif = new GifEncoder(width, height) - gif.setFrameRate(15) // 15fps seams like a good compromise for a gif - gif.writeHeader() - const { path, cleanup } = await file({ postfix: '.gif' }) - const pipelinePromise = pipeline(gif, fs.createWriteStream(path)) - let i = 0 - while (!stopped && i < config.maxAnimationFrames) { - i++ - if (i % 15 === 0) debug(`[${target}] ${i} frames taken`) + const frames: Uint8Array[] = [] + while (!stopped && frames.length < config.maxAnimationFrames) { + if (frames.length % 15 === 0) debug(`[${target}] ${frames.length} frames taken`) stopped = await page.evaluate(() => { // @ts-ignore return window.animateCaptureFrame() @@ -35,16 +36,13 @@ export const capture = async (target: string, page: Page, width: number, height: new Promise(resolve => setTimeout(resolve, config.screenshotTimeout)) ]) if (!buffer) throw new Error(`Failed to capture animation frame of page "${target}" before timeout`) - // puppeteer returns a plain Uint8Array, but get-pixels switches on Buffer.isBuffer to tell - // raw image data from a file path, and would treat the frame as a filename otherwise - const frame = Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength) - const pixels = await getPixels(frame, 'image/png') - gif.addFrame(pixels.data) + frames.push(buffer) } - gif.finish() - await pipelinePromise + debug(`[${target}] ${frames.length} frames taken, encode the gif`) + // 15fps seams like a good compromise for a gif + const raw = await gifEncodePiscina.run({ width, height, frameRate: 15, frames }) debug(`[${target}] gif screenshot is taken`) - const rawBuffer = await fs.promises.readFile(path) - cleanup() - return await optimizeGif(rawBuffer, 2) + return await optimizeGif(Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength), 2) } + +export const stop = () => gifEncodePiscina.destroy() diff --git a/config/custom-environment-variables.cjs b/config/custom-environment-variables.cjs index a292843..4d8dc0a 100644 --- a/config/custom-environment-variables.cjs +++ b/config/custom-environment-variables.cjs @@ -10,6 +10,7 @@ module.exports = { screenshotTimeout: 'SCREENSHOT_TIMEOUT', concurrency: 'CONCURRENCY', inspectConcurrency: 'INSPECT_CONCURRENCY', + gifThreads: 'GIF_THREADS', queue: { maxWaiting: 'QUEUE_MAX_WAITING', maxWaitTime: 'QUEUE_MAX_WAIT_TIME' diff --git a/config/default.cjs b/config/default.cjs index d5efb72..74d9599 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -31,6 +31,7 @@ module.exports = { inspect: null // required on /inspect, which is disabled without it }, maxAnimationFrames: 1800, // 2 minutes at 15fps + gifThreads: 2, // worker threads encoding the animated captures, also the max number of gifs encoded at the same time puppeteerLaunchOptions: { executablePath: '/usr/bin/google-chrome-stable', args: [], diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index 69a74a0..ff3114a 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -63,6 +63,10 @@ export type ApiConfig = { */ pngFastEncodingPixels: number; maxAnimationFrames: number; + /** + * worker threads encoding the animated captures, also the max number of gifs encoded at the same time + */ + gifThreads: number; util?: unknown; get?: unknown; has?: unknown; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index d686b00..8201c3d 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","queue","defaultLang","defaultTimezone","screenshotTimeout","wait","pngFastEncodingPixels","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"queue":{"type":"object","description":"Captures waiting for a browser context","additionalProperties":false,"required":["maxWaiting","maxWaitTime"],"properties":{"maxWaiting":{"type":"integer","minimum":0,"description":"beyond this number of waiting captures, new ones are answered by a 503"},"maxWaitTime":{"type":"number","description":"milliseconds of waiting for a context before a 503"}}},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"wait":{"type":"object","description":"How long a page that does not call window.triggerCapture is waited for","additionalProperties":false,"required":["networkIdleTime","quietTime","maxSettleTime"],"properties":{"networkIdleTime":{"type":"number","description":"milliseconds without any network request before the page is considered loaded"},"quietTime":{"type":"number","description":"then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture"},"maxSettleTime":{"type":"number","description":"the longest wait for that quiet, after the network is idle"}}},"pngFastEncodingPixels":{"type":"number","description":"above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","queue","defaultLang","defaultTimezone","screenshotTimeout","wait","pngFastEncodingPixels","maxAnimationFrames","gifThreads"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"queue":{"type":"object","description":"Captures waiting for a browser context","additionalProperties":false,"required":["maxWaiting","maxWaitTime"],"properties":{"maxWaiting":{"type":"integer","minimum":0,"description":"beyond this number of waiting captures, new ones are answered by a 503"},"maxWaitTime":{"type":"number","description":"milliseconds of waiting for a context before a 503"}}},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"wait":{"type":"object","description":"How long a page that does not call window.triggerCapture is waited for","additionalProperties":false,"required":["networkIdleTime","quietTime","maxSettleTime"],"properties":{"networkIdleTime":{"type":"number","description":"milliseconds without any network request before the page is considered loaded"},"quietTime":{"type":"number","description":"then, milliseconds without DOM mutation, animation frame, finite animation or font loading before the capture"},"maxSettleTime":{"type":"number","description":"the longest wait for that quiet, after the network is idle"}}},"pngFastEncodingPixels":{"type":"number","description":"above this number of pixels, and for full page captures, PNG screenshots are encoded for speed rather than size"},"maxAnimationFrames":{"type":"number"},"gifThreads":{"type":"integer","minimum":1,"description":"worker threads encoding the animated captures, also the max number of gifs encoded at the same time"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -153,9 +153,8 @@ vErrors.push(err13); } errors++; } -for(const key0 in data){ -if(!(func2.call(schema16.properties, key0))){ -const err14 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(data.gifThreads === undefined){ +const err14 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "gifThreads"},message:"must have required property '"+"gifThreads"+"'"}; if(vErrors === null){ vErrors = [err14]; } @@ -164,6 +163,17 @@ vErrors.push(err14); } errors++; } +for(const key0 in data){ +if(!(func2.call(schema16.properties, key0))){ +const err15 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err15]; +} +else { +vErrors.push(err15); +} +errors++; +} } if(data.port !== undefined){ let data0 = data.port; @@ -183,12 +193,12 @@ if(dataType0 == "boolean" || data0 === null coerced0 = +data0; } else { -const err15 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err16 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err15]; +vErrors = [err16]; } else { -vErrors.push(err15); +vErrors.push(err16); } errors++; } @@ -221,12 +231,12 @@ else if(data1 === null){ coerced1 = ""; } else { -const err16 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err17 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err16]; +vErrors = [err17]; } else { -vErrors.push(err16); +vErrors.push(err17); } errors++; } @@ -259,12 +269,12 @@ else if(data2 === null){ coerced2 = ""; } else { -const err17 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err18 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err17]; +vErrors = [err18]; } else { -vErrors.push(err17); +vErrors.push(err18); } errors++; } @@ -278,12 +288,12 @@ data["privateDirectoryUrl"] = coerced2; } if(typeof data2 === "string"){ if(!pattern0.test(data2)){ -const err18 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; +const err19 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; if(vErrors === null){ -vErrors = [err18]; +vErrors = [err19]; } else { -vErrors.push(err18); +vErrors.push(err19); } errors++; } @@ -293,23 +303,23 @@ if(data.helmet !== undefined){ let data3 = data.helmet; if(data3 && typeof data3 == "object" && !Array.isArray(data3)){ if(data3.active === undefined){ -const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; +const err20 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; if(vErrors === null){ -vErrors = [err19]; +vErrors = [err20]; } else { -vErrors.push(err19); +vErrors.push(err20); } errors++; } for(const key1 in data3){ if(!(key1 === "active")){ -const err20 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; +const err21 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err20]; +vErrors = [err21]; } else { -vErrors.push(err20); +vErrors.push(err21); } errors++; } @@ -334,12 +344,12 @@ else if(data4 === "true" || data4 === 1){ coerced3 = true; } else { -const err21 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err22 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err21]; +vErrors = [err22]; } else { -vErrors.push(err21); +vErrors.push(err22); } errors++; } @@ -354,12 +364,12 @@ data3["active"] = coerced3; } } else { -const err22 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err23 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err22]; +vErrors = [err23]; } else { -vErrors.push(err22); +vErrors.push(err23); } errors++; } @@ -368,23 +378,23 @@ if(data.secretKeys !== undefined){ let data5 = data.secretKeys; if(data5 && typeof data5 == "object" && !Array.isArray(data5)){ if(data5.capture === undefined){ -const err23 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; +const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; if(vErrors === null){ -vErrors = [err23]; +vErrors = [err24]; } else { -vErrors.push(err23); +vErrors.push(err24); } errors++; } for(const key2 in data5){ if(!((key2 === "capture") || (key2 === "inspect"))){ -const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; +const err25 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err24]; +vErrors = [err25]; } else { -vErrors.push(err24); +vErrors.push(err25); } errors++; } @@ -409,12 +419,12 @@ else if(data6 === null){ coerced4 = ""; } else { -const err25 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err26 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err25]; +vErrors = [err26]; } else { -vErrors.push(err25); +vErrors.push(err26); } errors++; } @@ -447,12 +457,12 @@ else if(data7 === null){ coerced5 = ""; } else { -const err26 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err27 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err26]; +vErrors = [err27]; } else { -vErrors.push(err26); +vErrors.push(err27); } errors++; } @@ -467,12 +477,12 @@ data5["inspect"] = coerced5; } } else { -const err27 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err28 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err27]; +vErrors = [err28]; } else { -vErrors.push(err27); +vErrors.push(err28); } errors++; } @@ -500,12 +510,12 @@ else if(data9 === "true" || data9 === 1){ coerced6 = true; } else { -const err28 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err29 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err28]; +vErrors = [err29]; } else { -vErrors.push(err28); +vErrors.push(err29); } errors++; } @@ -536,12 +546,12 @@ if(dataType7 == "boolean" || data10 === null coerced7 = +data10; } else { -const err29 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err30 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err29]; +vErrors = [err30]; } else { -vErrors.push(err29); +vErrors.push(err30); } errors++; } @@ -556,12 +566,12 @@ data8["port"] = coerced7; } } else { -const err30 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err31 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err30]; +vErrors = [err31]; } else { -vErrors.push(err30); +vErrors.push(err31); } errors++; } @@ -569,12 +579,12 @@ errors++; if(data.puppeteerLaunchOptions !== undefined){ let data11 = data.puppeteerLaunchOptions; if(!(data11 && typeof data11 == "object" && !Array.isArray(data11))){ -const err31 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err32 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err32]; } else { -vErrors.push(err31); +vErrors.push(err32); } errors++; } @@ -597,12 +607,12 @@ if(dataType8 === "boolean" || data12 === null coerced8 = +data12; } else { -const err32 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err33 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err32]; +vErrors = [err33]; } else { -vErrors.push(err32); +vErrors.push(err33); } errors++; } @@ -619,33 +629,33 @@ if(data.queue !== undefined){ let data13 = data.queue; if(data13 && typeof data13 == "object" && !Array.isArray(data13)){ if(data13.maxWaiting === undefined){ -const err33 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaiting"},message:"must have required property '"+"maxWaiting"+"'"}; +const err34 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaiting"},message:"must have required property '"+"maxWaiting"+"'"}; if(vErrors === null){ -vErrors = [err33]; +vErrors = [err34]; } else { -vErrors.push(err33); +vErrors.push(err34); } errors++; } if(data13.maxWaitTime === undefined){ -const err34 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaitTime"},message:"must have required property '"+"maxWaitTime"+"'"}; +const err35 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/required",keyword:"required",params:{missingProperty: "maxWaitTime"},message:"must have required property '"+"maxWaitTime"+"'"}; if(vErrors === null){ -vErrors = [err34]; +vErrors = [err35]; } else { -vErrors.push(err34); +vErrors.push(err35); } errors++; } for(const key3 in data13){ if(!((key3 === "maxWaiting") || (key3 === "maxWaitTime"))){ -const err35 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key3},message:"must NOT have additional properties"}; +const err36 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key3},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err35]; +vErrors = [err36]; } else { -vErrors.push(err35); +vErrors.push(err36); } errors++; } @@ -668,12 +678,12 @@ if(dataType9 === "boolean" || data14 === null coerced9 = +data14; } else { -const err36 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err37 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err36]; +vErrors = [err37]; } else { -vErrors.push(err36); +vErrors.push(err37); } errors++; } @@ -687,12 +697,12 @@ data13["maxWaiting"] = coerced9; } if(typeof data14 == "number"){ if(data14 < 0 || isNaN(data14)){ -const err37 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/minimum",keyword:"minimum",params:{comparison: ">=", limit: 0},message:"must be >= 0"}; +const err38 = {instancePath:instancePath+"/queue/maxWaiting",schemaPath:"#/properties/queue/properties/maxWaiting/minimum",keyword:"minimum",params:{comparison: ">=", limit: 0},message:"must be >= 0"}; if(vErrors === null){ -vErrors = [err37]; +vErrors = [err38]; } else { -vErrors.push(err37); +vErrors.push(err38); } errors++; } @@ -716,12 +726,12 @@ if(dataType10 == "boolean" || data15 === null coerced10 = +data15; } else { -const err38 = {instancePath:instancePath+"/queue/maxWaitTime",schemaPath:"#/properties/queue/properties/maxWaitTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err39 = {instancePath:instancePath+"/queue/maxWaitTime",schemaPath:"#/properties/queue/properties/maxWaitTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err38]; +vErrors = [err39]; } else { -vErrors.push(err38); +vErrors.push(err39); } errors++; } @@ -736,12 +746,12 @@ data13["maxWaitTime"] = coerced10; } } else { -const err39 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err40 = {instancePath:instancePath+"/queue",schemaPath:"#/properties/queue/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err39]; +vErrors = [err40]; } else { -vErrors.push(err39); +vErrors.push(err40); } errors++; } @@ -764,12 +774,12 @@ if(dataType11 === "boolean" || data16 === null coerced11 = +data16; } else { -const err40 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err41 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err40]; +vErrors = [err41]; } else { -vErrors.push(err40); +vErrors.push(err41); } errors++; } @@ -783,12 +793,12 @@ data["inspectConcurrency"] = coerced11; } if(typeof data16 == "number"){ if(data16 < 1 || isNaN(data16)){ -const err41 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +const err42 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; if(vErrors === null){ -vErrors = [err41]; +vErrors = [err42]; } else { -vErrors.push(err41); +vErrors.push(err42); } errors++; } @@ -814,12 +824,12 @@ else if(data17 === null){ coerced12 = ""; } else { -const err42 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err43 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err42]; +vErrors = [err43]; } else { -vErrors.push(err42); +vErrors.push(err43); } errors++; } @@ -852,12 +862,12 @@ else if(data18 === null){ coerced13 = ""; } else { -const err43 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err44 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err43]; +vErrors = [err44]; } else { -vErrors.push(err43); +vErrors.push(err44); } errors++; } @@ -890,12 +900,12 @@ else if(data19 === "true" || data19 === 1){ coerced14 = true; } else { -const err44 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err45 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err44]; +vErrors = [err45]; } else { -vErrors.push(err44); +vErrors.push(err45); } errors++; } @@ -928,12 +938,12 @@ else if(data20 === "true" || data20 === 1){ coerced15 = true; } else { -const err45 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err46 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err45]; +vErrors = [err46]; } else { -vErrors.push(err45); +vErrors.push(err46); } errors++; } @@ -964,12 +974,12 @@ if(dataType16 == "boolean" || data21 === null coerced16 = +data21; } else { -const err46 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err47 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err46]; +vErrors = [err47]; } else { -vErrors.push(err46); +vErrors.push(err47); } errors++; } @@ -986,43 +996,43 @@ if(data.wait !== undefined){ let data22 = data.wait; if(data22 && typeof data22 == "object" && !Array.isArray(data22)){ if(data22.networkIdleTime === undefined){ -const err47 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "networkIdleTime"},message:"must have required property '"+"networkIdleTime"+"'"}; +const err48 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "networkIdleTime"},message:"must have required property '"+"networkIdleTime"+"'"}; if(vErrors === null){ -vErrors = [err47]; +vErrors = [err48]; } else { -vErrors.push(err47); +vErrors.push(err48); } errors++; } if(data22.quietTime === undefined){ -const err48 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "quietTime"},message:"must have required property '"+"quietTime"+"'"}; +const err49 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "quietTime"},message:"must have required property '"+"quietTime"+"'"}; if(vErrors === null){ -vErrors = [err48]; +vErrors = [err49]; } else { -vErrors.push(err48); +vErrors.push(err49); } errors++; } if(data22.maxSettleTime === undefined){ -const err49 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "maxSettleTime"},message:"must have required property '"+"maxSettleTime"+"'"}; +const err50 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/required",keyword:"required",params:{missingProperty: "maxSettleTime"},message:"must have required property '"+"maxSettleTime"+"'"}; if(vErrors === null){ -vErrors = [err49]; +vErrors = [err50]; } else { -vErrors.push(err49); +vErrors.push(err50); } errors++; } for(const key4 in data22){ if(!(((key4 === "networkIdleTime") || (key4 === "quietTime")) || (key4 === "maxSettleTime"))){ -const err50 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key4},message:"must NOT have additional properties"}; +const err51 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key4},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err50]; +vErrors = [err51]; } else { -vErrors.push(err50); +vErrors.push(err51); } errors++; } @@ -1045,12 +1055,12 @@ if(dataType17 == "boolean" || data23 === null coerced17 = +data23; } else { -const err51 = {instancePath:instancePath+"/wait/networkIdleTime",schemaPath:"#/properties/wait/properties/networkIdleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err52 = {instancePath:instancePath+"/wait/networkIdleTime",schemaPath:"#/properties/wait/properties/networkIdleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err51]; +vErrors = [err52]; } else { -vErrors.push(err51); +vErrors.push(err52); } errors++; } @@ -1081,12 +1091,12 @@ if(dataType18 == "boolean" || data24 === null coerced18 = +data24; } else { -const err52 = {instancePath:instancePath+"/wait/quietTime",schemaPath:"#/properties/wait/properties/quietTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err53 = {instancePath:instancePath+"/wait/quietTime",schemaPath:"#/properties/wait/properties/quietTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err52]; +vErrors = [err53]; } else { -vErrors.push(err52); +vErrors.push(err53); } errors++; } @@ -1117,12 +1127,12 @@ if(dataType19 == "boolean" || data25 === null coerced19 = +data25; } else { -const err53 = {instancePath:instancePath+"/wait/maxSettleTime",schemaPath:"#/properties/wait/properties/maxSettleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err54 = {instancePath:instancePath+"/wait/maxSettleTime",schemaPath:"#/properties/wait/properties/maxSettleTime/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err53]; +vErrors = [err54]; } else { -vErrors.push(err53); +vErrors.push(err54); } errors++; } @@ -1137,12 +1147,12 @@ data22["maxSettleTime"] = coerced19; } } else { -const err54 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err55 = {instancePath:instancePath+"/wait",schemaPath:"#/properties/wait/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err54]; +vErrors = [err55]; } else { -vErrors.push(err54); +vErrors.push(err55); } errors++; } @@ -1165,12 +1175,12 @@ if(dataType20 == "boolean" || data26 === null coerced20 = +data26; } else { -const err55 = {instancePath:instancePath+"/pngFastEncodingPixels",schemaPath:"#/properties/pngFastEncodingPixels/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err56 = {instancePath:instancePath+"/pngFastEncodingPixels",schemaPath:"#/properties/pngFastEncodingPixels/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err55]; +vErrors = [err56]; } else { -vErrors.push(err55); +vErrors.push(err56); } errors++; } @@ -1201,12 +1211,12 @@ if(dataType21 == "boolean" || data27 === null coerced21 = +data27; } else { -const err56 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err57 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err56]; +vErrors = [err57]; } else { -vErrors.push(err56); +vErrors.push(err57); } errors++; } @@ -1219,14 +1229,62 @@ data["maxAnimationFrames"] = coerced21; } } } +if(data.gifThreads !== undefined){ +let data28 = data.gifThreads; +if(!((typeof data28 == "number") && (!(data28 % 1) && !isNaN(data28)))){ +let dataType22 = typeof data28; +let coerced22 = undefined; +if(dataType22 == 'object' && Array.isArray(data28) && data28.length == 1){ +data28 = data28[0]; +dataType22 = typeof data28; +if((typeof data28 == "number") && (!(data28 % 1) && !isNaN(data28))){ +coerced22 = data28; +} +} +if(!(coerced22 !== undefined)){ +if(dataType22 === "boolean" || data28 === null + || (dataType22 === "string" && data28 && data28 == +data28 && !(data28 % 1))){ +coerced22 = +data28; } else { -const err57 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err58 = {instancePath:instancePath+"/gifThreads",schemaPath:"#/properties/gifThreads/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err57]; +vErrors = [err58]; } else { -vErrors.push(err57); +vErrors.push(err58); +} +errors++; +} +} +if(coerced22 !== undefined){ +data28 = coerced22; +if(data !== undefined){ +data["gifThreads"] = coerced22; +} +} +} +if(typeof data28 == "number"){ +if(data28 < 1 || isNaN(data28)){ +const err59 = {instancePath:instancePath+"/gifThreads",schemaPath:"#/properties/gifThreads/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err59]; +} +else { +vErrors.push(err59); +} +errors++; +} +} +} +} +else { +const err60 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err60]; +} +else { +vErrors.push(err60); } errors++; } diff --git a/config/type/schema.json b/config/type/schema.json index 801b661..391195e 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -24,7 +24,8 @@ "screenshotTimeout", "wait", "pngFastEncodingPixels", - "maxAnimationFrames" + "maxAnimationFrames", + "gifThreads" ], "properties": { "port": { @@ -140,6 +141,11 @@ "maxAnimationFrames": { "type": "number" }, + "gifThreads": { + "type": "integer", + "minimum": 1, + "description": "worker threads encoding the animated captures, also the max number of gifs encoded at the same time" + }, "util": {}, "get": {}, "has": {} diff --git a/package-lock.json b/package-lock.json index 245717a..c144e89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,9 +22,9 @@ "gif-encoder": "^0.7.2", "helmet": "^8.1.0", "http-terminator": "^3.2.0", + "piscina": "^5.3.2", "prom-client": "^15.1.3", - "puppeteer": "^25.16.0", - "tmp-promise": "^3.0.3" + "puppeteer": "^25.16.0" }, "devDependencies": { "@commitlint/cli": "^21.2.4", @@ -1009,6 +1009,332 @@ "node": ">= 10" } }, + "node_modules/@napi-rs/nice": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice/-/nice-1.1.1.tgz", + "integrity": "sha512-xJIPs+bYuc9ASBl+cvGsKbGrJmS6fAKaSZCnT0lhahT5rhA2VVy9/EcIgd2JhtEuFOJNx7UHNn/qiTPTY4nrQw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/nice-android-arm-eabi": "1.1.1", + "@napi-rs/nice-android-arm64": "1.1.1", + "@napi-rs/nice-darwin-arm64": "1.1.1", + "@napi-rs/nice-darwin-x64": "1.1.1", + "@napi-rs/nice-freebsd-x64": "1.1.1", + "@napi-rs/nice-linux-arm-gnueabihf": "1.1.1", + "@napi-rs/nice-linux-arm64-gnu": "1.1.1", + "@napi-rs/nice-linux-arm64-musl": "1.1.1", + "@napi-rs/nice-linux-ppc64-gnu": "1.1.1", + "@napi-rs/nice-linux-riscv64-gnu": "1.1.1", + "@napi-rs/nice-linux-s390x-gnu": "1.1.1", + "@napi-rs/nice-linux-x64-gnu": "1.1.1", + "@napi-rs/nice-linux-x64-musl": "1.1.1", + "@napi-rs/nice-openharmony-arm64": "1.1.1", + "@napi-rs/nice-win32-arm64-msvc": "1.1.1", + "@napi-rs/nice-win32-ia32-msvc": "1.1.1", + "@napi-rs/nice-win32-x64-msvc": "1.1.1" + } + }, + "node_modules/@napi-rs/nice-android-arm-eabi": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-android-arm-eabi/-/nice-android-arm-eabi-1.1.1.tgz", + "integrity": "sha512-kjirL3N6TnRPv5iuHw36wnucNqXAO46dzK9oPb0wj076R5Xm8PfUVA9nAFB5ZNMmfJQJVKACAPd/Z2KYMppthw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-android-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-android-arm64/-/nice-android-arm64-1.1.1.tgz", + "integrity": "sha512-blG0i7dXgbInN5urONoUCNf+DUEAavRffrO7fZSeoRMJc5qD+BJeNcpr54msPF6qfDD6kzs9AQJogZvT2KD5nw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-darwin-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-darwin-arm64/-/nice-darwin-arm64-1.1.1.tgz", + "integrity": "sha512-s/E7w45NaLqTGuOjC2p96pct4jRfo61xb9bU1unM/MJ/RFkKlJyJDx7OJI/O0ll/hrfpqKopuAFDV8yo0hfT7A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-darwin-x64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-darwin-x64/-/nice-darwin-x64-1.1.1.tgz", + "integrity": "sha512-dGoEBnVpsdcC+oHHmW1LRK5eiyzLwdgNQq3BmZIav+9/5WTZwBYX7r5ZkQC07Nxd3KHOCkgbHSh4wPkH1N1LiQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-freebsd-x64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-freebsd-x64/-/nice-freebsd-x64-1.1.1.tgz", + "integrity": "sha512-kHv4kEHAylMYmlNwcQcDtXjklYp4FCf0b05E+0h6nDHsZ+F0bDe04U/tXNOqrx5CmIAth4vwfkjjUmp4c4JktQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-arm-gnueabihf": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm-gnueabihf/-/nice-linux-arm-gnueabihf-1.1.1.tgz", + "integrity": "sha512-E1t7K0efyKXZDoZg1LzCOLxgolxV58HCkaEkEvIYQx12ht2pa8hoBo+4OB3qh7e+QiBlp1SRf+voWUZFxyhyqg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-arm64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm64-gnu/-/nice-linux-arm64-gnu-1.1.1.tgz", + "integrity": "sha512-CIKLA12DTIZlmTaaKhQP88R3Xao+gyJxNWEn04wZwC2wmRapNnxCUZkVwggInMJvtVElA+D4ZzOU5sX4jV+SmQ==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-arm64-musl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-arm64-musl/-/nice-linux-arm64-musl-1.1.1.tgz", + "integrity": "sha512-+2Rzdb3nTIYZ0YJF43qf2twhqOCkiSrHx2Pg6DJaCPYhhaxbLcdlV8hCRMHghQ+EtZQWGNcS2xF4KxBhSGeutg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-ppc64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-ppc64-gnu/-/nice-linux-ppc64-gnu-1.1.1.tgz", + "integrity": "sha512-4FS8oc0GeHpwvv4tKciKkw3Y4jKsL7FRhaOeiPei0X9T4Jd619wHNe4xCLmN2EMgZoeGg+Q7GY7BsvwKpL22Tg==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-riscv64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-riscv64-gnu/-/nice-linux-riscv64-gnu-1.1.1.tgz", + "integrity": "sha512-HU0nw9uD4FO/oGCCk409tCi5IzIZpH2agE6nN4fqpwVlCn5BOq0MS1dXGjXaG17JaAvrlpV5ZeyZwSon10XOXw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-s390x-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-s390x-gnu/-/nice-linux-s390x-gnu-1.1.1.tgz", + "integrity": "sha512-2YqKJWWl24EwrX0DzCQgPLKQBxYDdBxOHot1KWEq7aY2uYeX+Uvtv4I8xFVVygJDgf6/92h9N3Y43WPx8+PAgQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-x64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-x64-gnu/-/nice-linux-x64-gnu-1.1.1.tgz", + "integrity": "sha512-/gaNz3R92t+dcrfCw/96pDopcmec7oCcAQ3l/M+Zxr82KT4DljD37CpgrnXV+pJC263JkW572pdbP3hP+KjcIg==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-linux-x64-musl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-linux-x64-musl/-/nice-linux-x64-musl-1.1.1.tgz", + "integrity": "sha512-xScCGnyj/oppsNPMnevsBe3pvNaoK7FGvMjT35riz9YdhB2WtTG47ZlbxtOLpjeO9SqqQ2J2igCmz6IJOD5JYw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-openharmony-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-openharmony-arm64/-/nice-openharmony-arm64-1.1.1.tgz", + "integrity": "sha512-6uJPRVwVCLDeoOaNyeiW0gp2kFIM4r7PL2MczdZQHkFi9gVlgm+Vn+V6nTWRcu856mJ2WjYJiumEajfSm7arPQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-win32-arm64-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-arm64-msvc/-/nice-win32-arm64-msvc-1.1.1.tgz", + "integrity": "sha512-uoTb4eAvM5B2aj/z8j+Nv8OttPf2m+HVx3UjA5jcFxASvNhQriyCQF1OB1lHL43ZhW+VwZlgvjmP5qF3+59atA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-win32-ia32-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-ia32-msvc/-/nice-win32-ia32-msvc-1.1.1.tgz", + "integrity": "sha512-CNQqlQT9MwuCsg1Vd/oKXiuH+TcsSPJmlAFc5frFyX/KkOh0UpBLEj7aoY656d5UKZQMQFP7vJNa1DNUNORvug==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/nice-win32-x64-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/nice-win32-x64-msvc/-/nice-win32-x64-msvc-1.1.1.tgz", + "integrity": "sha512-vB+4G/jBQCAh0jelMTY3+kgFy00Hlx2f2/1zjMoH821IbplbWZOkLiTYXQkygNTzQJTq5cvwBDgn2ppHD+bglQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@opentelemetry/api": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.10.0.tgz", @@ -4663,6 +4989,18 @@ "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", "license": "MIT" }, + "node_modules/piscina": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/piscina/-/piscina-5.3.2.tgz", + "integrity": "sha512-vv7l/mM7B9WgrhDaudIqv1x6v4LOBRYYmpWGdWushhpbCmVHp11sUB2v4hj1CAeTGXVxuZFz6xOU1RalAiPKfQ==", + "license": "MIT", + "engines": { + "node": ">=20.x" + }, + "optionalDependencies": { + "@napi-rs/nice": "^1.0.4" + } + }, "node_modules/pngjs": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", @@ -5398,24 +5736,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tmp": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", - "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", - "license": "MIT", - "engines": { - "node": ">=14.14" - } - }, - "node_modules/tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "license": "MIT", - "dependencies": { - "tmp": "^0.2.0" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", diff --git a/package.json b/package.json index 29e826b..7d27c6d 100644 --- a/package.json +++ b/package.json @@ -51,9 +51,9 @@ "gif-encoder": "^0.7.2", "helmet": "^8.1.0", "http-terminator": "^3.2.0", + "piscina": "^5.3.2", "prom-client": "^15.1.3", - "puppeteer": "^25.16.0", - "tmp-promise": "^3.0.3" + "puppeteer": "^25.16.0" }, "devDependencies": { "@commitlint/cli": "^21.2.4", From 5a282163fc4501bd6ca558cad1386e2bf00ce160 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 09:51:15 +0100 Subject: [PATCH 06/10] perf: chrome-headless-shell instead of google-chrome-stable The image installs Debian's chromium-headless-shell (the "old" headless Chromium, without the browser UI layer) instead of google-chrome-stable from Google's repository. Measured: half the memory at idle, 20-45% less browser CPU per capture, pdf x1.6 faster, +30% throughput on paint heavy pages. On the image: 1.12GB -> 904MB, container memory 324 -> 142MiB idle and ~780 -> ~550MiB under 10 concurrent captures. It is packaged for amd64 and arm64 (the google repository was pinned to amd64), updated with the distribution, and still sandboxed with seccomp.json. wget is now purged from the image, it was only kept because google-chrome-stable depended on it. Development, tests and CI use the chrome-headless-shell downloaded by puppeteer (npx puppeteer browsers install chrome-headless-shell), with --no-sandbox: outside of a container AppArmor forbids its sandbox on Ubuntu. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/reuse-quality.yml | 5 ++++- AGENTS.md | 5 +++-- CONTRIBUTING.md | 10 ++++++---- Dockerfile | 17 ++++++++--------- benchmark/FINDINGS.md | 12 +++++++++--- config/default.cjs | 7 +++++-- config/development.cjs | 6 ++++++ config/test.cjs | 6 ++++++ 8 files changed, 47 insertions(+), 21 deletions(-) diff --git a/.github/workflows/reuse-quality.yml b/.github/workflows/reuse-quality.yml index c6b8217..d72f00d 100644 --- a/.github/workflows/reuse-quality.yml +++ b/.github/workflows/reuse-quality.yml @@ -15,13 +15,16 @@ jobs: node-version-file: .nvmrc cache: 'npm' - # google-chrome-stable is preinstalled on the runner, gifsicle is not - name: Install gifsicle run: sudo apt-get update && sudo apt-get install -y --no-install-recommends gifsicle - name: Install dependencies run: npm ci + # the browser of the tests, the version that matches puppeteer + - name: Install chrome-headless-shell + run: npx puppeteer browsers install chrome-headless-shell + - name: Lint run: npm run lint diff --git a/AGENTS.md b/AGENTS.md index 9e437c9..8bf5319 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project overview -Capture renders Web pages with puppeteer (google-chrome) and returns screenshots (png, jpg, +Capture renders Web pages with puppeteer (chrome-headless-shell) and returns screenshots (png, jpg, animated gif through gifsicle) or prints (pdf, html). API routes in `api/routers/capture.ts`, browser contexts pool in `api/utils/page.ts`, config in `config/` (types generated in `config/type/` by `npm run build-types`). @@ -42,7 +42,8 @@ Branch names: letters, digits, `.`, `_`, `-`, no `/`, and `capture_` at ## Testing `node:test` suites in `test-it/`, they start the API in-process on `TEST_API_PORT` (`.env`) and -need google-chrome-stable and gifsicle on the machine. +need gifsicle on the machine and puppeteer's chrome-headless-shell +(`npx puppeteer browsers install chrome-headless-shell`). ```bash npm test # all tests diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3610ab7..3fc1b6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,8 +4,9 @@ - Node.js, the version of `.nvmrc` (start zellij from a shell on it: the panes inherit its PATH) - zellij >= 0.44 -- google-chrome-stable (`/usr/bin/google-chrome-stable`, the `executablePath` of `config/default.cjs`) - and gifsicle (animated captures), installed on your system +- gifsicle (animated captures), installed on your system. The browser is chrome-headless-shell: + the image installs Debian's package, development and tests use the build downloaded by puppeteer + (`npx puppeteer browsers install chrome-headless-shell`, the version that matches puppeteer) ## Setup @@ -13,10 +14,11 @@ The dev environment is the shared one of the data-fair services (`@data-fair/lib session per checkout, described by `.zellij.kdl` and `dev/dev.json`. No containers: in development capture runs without authentication (no `privateDirectoryUrl`). -1. Install dependencies (puppeteer's own browser download is not needed), then write `.env` (free - ports drawn for this checkout) and build the types: +1. Install dependencies and chrome-headless-shell, then write `.env` (free ports drawn for this + checkout) and build the types: ```bash PUPPETEER_SKIP_DOWNLOAD=true npm ci + npx puppeteer browsers install chrome-headless-shell npx --no df-dev setup ``` diff --git a/Dockerfile b/Dockerfile index 717ba10..b225ca7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,14 +15,12 @@ FROM base AS nativedeps ARG TARGETARCH RUN echo "Building for architecture $TARGETARCH" -# Install chrome and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) +# Install chromium-headless-shell (Debian's build of the "old" headless Chromium, amd64 and arm64, +# security updates with the rest of the distribution) and fonts to support major charsets +# (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) RUN apt-get update && \ apt-get upgrade -y && \ - apt-get install -y wget gnupg ca-certificates && \ - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg && \ - sh -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] https://dl-ssl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \ - apt-get update && \ - apt-get install -y google-chrome-stable gifsicle fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst-one fonts-freefont-ttf libxss1 dbus dbus-x11 --no-install-recommends && \ + apt-get install -y chromium-headless-shell gifsicle wget ca-certificates fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst-one fonts-freefont-ttf dbus dbus-x11 --no-install-recommends && \ service dbus start ENV DBUS_SESSION_BUS_ADDRESS=autolaunch: @@ -37,9 +35,10 @@ RUN case "$TARGETARCH" in \ wget -q -O /usr/local/bin/dumb-init "https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_$DUMB_INIT_ARCH" && \ chmod +x /usr/local/bin/dumb-init -# cleanup (wget stays: google-chrome-stable depends on it, purging it removes chrome) -RUN apt-get clean -RUN apt-get purge -y --auto-remove gnupg apt-transport-https +# cleanup: wget was only needed to download dumb-init +RUN apt-get purge -y --auto-remove wget && \ + test -x /usr/lib/chromium/chromium-headless-shell && \ + apt-get clean && rm -rf /var/lib/apt/lists/* # skip the browser download when installing puppeteer ENV PUPPETEER_SKIP_DOWNLOAD=true diff --git a/benchmark/FINDINGS.md b/benchmark/FINDINGS.md index 7816cb8..86bc7a7 100644 --- a/benchmark/FINDINGS.md +++ b/benchmark/FINDINGS.md @@ -18,9 +18,15 @@ tables below are from the 8-run pass. Raw JSON is in `results/` (git-ignored). fast encoding above `pngFastEncodingPixels` and for full-page captures (§2), and the fixes of `type=jpg` (PNG bytes labelled `image/jpeg`) and of `type=gif` on a page without animation (PNG bytes labelled `image/gif`). -- **Approved, to do**: chrome-headless-shell (§3), the GIF encoding in a piscina worker (§5), - the console listener behind `DEBUG` (§6), skipping abandoned requests and bounding the - queue (§7). +- **Done too**: the debug listeners only when `DEBUG` is set (§6); abandoned captures skipped + and the contexts queue bounded (`config.queue`, §7); the GIF encoding in piscina worker + threads (`config.gifThreads`, §5). Event loop probe during a GIF: p90 83 → 1.2 ms, max + 131 → 4.6 ms. chrome-headless-shell (§3): Debian's `chromium-headless-shell` in the image, + puppeteer's download in development, tests and CI. Image 1.12 GB → 904 MB, container memory + 324 → 142 MiB idle and ~780 → ~550 MiB under 10 concurrent captures. It runs sandboxed with + `seccomp.json` (renderers in their own user namespace). Its version comes from Debian's + security updates, not from puppeteer: puppeteer 25.16 expects 156, trixie ships 156.0.8078.79. + Keep the two close when upgrading puppeteer. - **Declined**: untagged PDFs (§4). Accessibility comes first. ## TL;DR diff --git a/config/default.cjs b/config/default.cjs index 74d9599..f8f90f0 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -32,10 +32,13 @@ module.exports = { }, maxAnimationFrames: 1800, // 2 minutes at 15fps gifThreads: 2, // worker threads encoding the animated captures, also the max number of gifs encoded at the same time + // chrome-headless-shell (Debian's chromium-headless-shell package in the image): the "old" + // headless Chromium, without the browser UI layer. Compared to google-chrome-stable it uses half + // the memory and 20-45% less CPU per capture (cf benchmark/FINDINGS.md) puppeteerLaunchOptions: { - executablePath: '/usr/bin/google-chrome-stable', + executablePath: '/usr/lib/chromium/chromium-headless-shell', args: [], - headless: true + headless: 'shell' }, observer: { active: true, diff --git a/config/development.cjs b/config/development.cjs index f85bfcf..0616888 100644 --- a/config/development.cjs +++ b/config/development.cjs @@ -8,6 +8,12 @@ module.exports = { capture: 'capture', inspect: 'inspect' }, + // the chrome-headless-shell downloaded by puppeteer (npx puppeteer browsers install + // chrome-headless-shell). Outside of a container, AppArmor (Ubuntu >= 23.10) forbids its sandbox + puppeteerLaunchOptions: { + executablePath: null, + args: ['--no-sandbox'] + }, observer: { active: false }, diff --git a/config/test.cjs b/config/test.cjs index 6231010..df5911b 100644 --- a/config/test.cjs +++ b/config/test.cjs @@ -9,6 +9,12 @@ module.exports = { capture: 'capture', inspect: 'inspect' }, + // the chrome-headless-shell downloaded by puppeteer (npx puppeteer browsers install + // chrome-headless-shell). Outside of a container, AppArmor (Ubuntu >= 23.10) forbids its sandbox + puppeteerLaunchOptions: { + executablePath: null, + args: ['--no-sandbox'] + }, observer: { active: false }, From 95aaf71c8e2eb84e2d332efaf68cd80fb09fdd9d Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 10:06:25 +0100 Subject: [PATCH 07/10] fix: transfer the gif frames and result between threads instead of copying, small gifs failed The encoded gif was returned with move() on the output of Buffer.concat, which is a slice of the worker's shared Buffer pool below half of Buffer.poolSize: the transfer threw a DataCloneError and every gif under ~32KB failed. It is now joined in an ArrayBuffer of its own. The frames were cloned (copied) to the worker. They are now transferred (transferList): the large frames puppeteer returns own their ArrayBuffer and move without any copy, the small ones (slices of the Buffer pool) are copied once first. The main thread no longer holds a second copy of all the frames during the encoding. Co-Authored-By: Claude Opus 5.5 --- api/threads/gif-encode.ts | 13 ++++++++++--- api/utils/animation.ts | 13 +++++++++++-- test-it/capture.ts | 6 ++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/api/threads/gif-encode.ts b/api/threads/gif-encode.ts index 7a94840..49dd483 100644 --- a/api/threads/gif-encode.ts +++ b/api/threads/gif-encode.ts @@ -25,7 +25,14 @@ export default async ({ width, height, frameRate, frames }: GifEncodeTask) => { } gif.finish() await ended - const result = Buffer.concat(chunks) - // transferred to the main thread, not copied - return move(new Uint8Array(result.buffer, result.byteOffset, result.byteLength)) + // the chunks are joined in an ArrayBuffer of our own, then transferred to the main thread, not + // copied. Buffer.concat would not do: below half of Buffer.poolSize it returns a slice of the + // shared Buffer pool, which cannot be transferred (DataCloneError) + const result = new Uint8Array(chunks.reduce((size, chunk) => size + chunk.byteLength, 0)) + let offset = 0 + for (const chunk of chunks) { + result.set(chunk, offset) + offset += chunk.byteLength + } + return move(result) } diff --git a/api/utils/animation.ts b/api/utils/animation.ts index c9eed30..f0522f6 100644 --- a/api/utils/animation.ts +++ b/api/utils/animation.ts @@ -20,6 +20,14 @@ export const gifEncodePiscina = new Piscina({ // a pool emits 'error' when a thread dies with no task in flight; without a listener it crashes the process gifEncodePiscina.on('error', (err) => log.error('gif encoding worker error', { code: 'piscina-gif-encode', err })) +// a typed array alone in its ArrayBuffer, so that the ArrayBuffer can be transferred to a worker. +// puppeteer returns Buffers decoded from base64, the small ones are slices of the shared Buffer +// pool (not transferable, and other buffers live in it): those are copied, the others kept +const ownBuffer = (frame: Uint8Array) => { + if (frame.byteOffset === 0 && frame.byteLength === frame.buffer.byteLength && !(frame.buffer instanceof SharedArrayBuffer)) return frame + return new Uint8Array(frame) +} + export const capture = async (target: string, page: Page, width: number, height: number) => { let stopped = false const frames: Uint8Array[] = [] @@ -36,11 +44,12 @@ export const capture = async (target: string, page: Page, width: number, height: new Promise(resolve => setTimeout(resolve, config.screenshotTimeout)) ]) if (!buffer) throw new Error(`Failed to capture animation frame of page "${target}" before timeout`) - frames.push(buffer) + frames.push(ownBuffer(buffer)) } debug(`[${target}] ${frames.length} frames taken, encode the gif`) // 15fps seams like a good compromise for a gif - const raw = await gifEncodePiscina.run({ width, height, frameRate: 15, frames }) + // the frames are transferred to the worker, not copied: they are not usable here afterwards + const raw = await gifEncodePiscina.run({ width, height, frameRate: 15, frames }, { transferList: frames.map(f => f.buffer as ArrayBuffer) }) debug(`[${target}] gif screenshot is taken`) return await optimizeGif(Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength), 2) } diff --git a/test-it/capture.ts b/test-it/capture.ts index 8a399b4..312ca9f 100644 --- a/test-it/capture.ts +++ b/test-it/capture.ts @@ -200,6 +200,12 @@ describe('capture', () => { const size1 = Number(res.headers['content-length']) assert.ok(size1 < 100000) + // a small gif is encoded in a slice of the Buffer pool of the worker, it must be returned anyway + res = await ax.get('screenshot', { params: { key, target: origin + '/test/resources/test-anim.html', type: 'gif', width: 100, height: 50 }, responseType: 'arraybuffer' }) + assert.equal(res.status, 200) + assert.equal(res.headers['content-type'], 'image/gif') + assert.equal(Buffer.from(res.data).subarray(0, 3).toString(), 'GIF') + res = await ax.get('screenshot', { params: { key, target: origin + '/test/resources/test-anim.html', filename: 'test.gif' } }) assert.equal(res.status, 200) assert.equal(res.headers['content-type'], 'image/gif') From a3a6cb65059781f5dae2d410090597e8626c3c38 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 10:25:33 +0100 Subject: [PATCH 08/10] perf: inspect returns the browser's base64 screenshot as is The screenshot was decoded from base64 by puppeteer, copied, then encoded back to base64 for the JSON response: 4-8ms of synchronous work on the main thread for the large full page captures, and 2 extra copies of the image. Co-Authored-By: Claude Opus 5.5 --- api/routers/capture.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 8ae8a23..e648d32 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -281,15 +281,17 @@ router.get('/inspect', async (req, res) => { `Failed to inspect page "${target}" before timeout`, async ({ page, ready }) => { const report = await collector!.finish(ready) - let buffer + // the base64 string of the browser as is: decoding it to bytes to encode it back to base64 + // would only copy megabytes on the main thread + let data if (fullPage) { const height = await page.evaluate(() => document.documentElement.scrollHeight) - buffer = await page.screenshot({ captureBeyondViewport: true, optimizeForSpeed: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) + data = await page.screenshot({ encoding: 'base64', captureBeyondViewport: true, optimizeForSpeed: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) } else { - buffer = await page.screenshot() + data = await page.screenshot({ encoding: 'base64' }) } timer.type = 'inspect' - res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) + res.json({ image: { data, mediaType: 'image/png' }, text: reportText(report), report }) }, { onPage: page => { collector = createReportCollector(page, target) }, From 9f11f35fde624032f1afcb67fad2a80a9fd9ee53 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 10:37:42 +0100 Subject: [PATCH 09/10] docs: steady load results on real applications, webgl maps take 90% of the cpu Co-Authored-By: Claude Opus 5.5 --- benchmark/FINDINGS.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/benchmark/FINDINGS.md b/benchmark/FINDINGS.md index 86bc7a7..e8b36fa 100644 --- a/benchmark/FINDINGS.md +++ b/benchmark/FINDINGS.md @@ -29,6 +29,35 @@ tables below are from the 8-run pass. Raw JSON is in `results/` (git-ignored). Keep the two close when upgrading puppeteer. - **Declined**: untagged PDFs (§4). Accessibility comes first. +## Steady load on real applications (2026-11-09) + +The two images (master vs this branch), `CONCURRENCY=5`, 8 clients looping for 60 s over the 66 +public applications of opendata.koumoul.com (52 call `triggerCapture`), 1050×450, 2 runs per +image in alternating order, a fresh container each run: + +| | master | this branch | +|---|---|---| +| captures per minute | 145, 160 | **163, 170** (+~10%) | +| p50 latency, apps with triggerCapture | 2221, 1965 ms | 1930, 1904 ms | +| p50 latency, apps without | 4154, 3615 ms | **2646, 2627 ms** (−30%) | +| CPU per capture (container) | 2966, 2841 ms | 2900, 3004 ms | +| cores busy | 7.2, 7.6 | 7.9, 8.5 | +| memory peak (cgroup) | 2815, 2927 MB | 2873, 2858 MB | + +**The instance is CPU-bound on real applications, and the CPU goes to WebGL maps.** Measured app +by app (browser process tree, sequential): 3.4 s of CPU per capture on average. 21 of the 66 +applications use WebGL (maplibre maps: admin-divs-levels, data-fair-geo-shapes, carto-stats, +admin-divs-catchment, infos-parcelles, infos-loc, carto-explore, atelier-carto...), and they take +**90% of the CPU**, 5-18 s each. Without a GPU, Chrome renders WebGL in software (SwiftShader). +The other 45 applications average ~0.5 s. Consequences: +- the shorter waits mostly cut latency. Throughput only gains ~10%, because the CPU is the limit; +- the browser savings of chrome-headless-shell (~100 ms of CPU per light capture) are a few + percent of a real mix. Its memory gain shows at idle, not at the peak, which is set by the + renderers of heavy pages; +- **the next lever is the WebGL maps**: a capture mode in the map applications (no + transitions or fly-to, `fadeDuration: 0`, `triggerCapture` on the map's `idle` event), or + measuring Chrome's other software GL backends. + ## TL;DR Where a capture's time goes today, for a typical data-fair page that does not call From 404a087ce9e5f5d0fccae65ab0d33038d40de01b Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Mon, 9 Nov 2026 10:56:28 +0100 Subject: [PATCH 10/10] fix: one fresh incognito context per capture, generic-pool replaced by a bounded semaphore The pool reused its incognito contexts from one capture to the next and only deleted their cookies: localStorage, IndexedDB and Cache Storage written by a capture were visible to the next ones of the same origin, whatever the user they were made for. Each capture now runs in its own context, closed after it. It costs a few milliseconds (benchmark: context). This also removes the pool and its cleanup, destroy and retry paths. The captures wait for a slot of the service's semaphore, which gained what the pool provided and more: a bounded queue, a wait timeout, and the removal of a waiting call when its client leaves. /inspect uses the same mechanism instead of checking its socket. The default wait timeout is 30s: data-fair gives up after 60s without a response and retries once. The metrics keep their names (df_capture_acquire_context_*, df_capture_contexts_cleanups) for the existing dashboards. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- api/routers/capture.ts | 10 +- api/utils/metrics.ts | 14 +-- api/utils/page.ts | 155 +++++++++------------------- api/utils/semaphore.ts | 57 ++++++++-- benchmark/FINDINGS.md | 8 ++ config/default.cjs | 7 +- package-lock.json | 10 -- package.json | 1 - test-it/capture.ts | 10 ++ test-it/resources/test-storage.html | 18 ++++ test-it/semaphore.ts | 45 ++++++++ 12 files changed, 193 insertions(+), 146 deletions(-) create mode 100644 test-it/resources/test-storage.html diff --git a/AGENTS.md b/AGENTS.md index 8bf5319..c240972 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,8 +4,8 @@ Capture renders Web pages with puppeteer (chrome-headless-shell) and returns screenshots (png, jpg, animated gif through gifsicle) or prints (pdf, html). API routes in `api/routers/capture.ts`, -browser contexts pool in `api/utils/page.ts`, config in `config/` (types generated in -`config/type/` by `npm run build-types`). +browser and capture slots in `api/utils/page.ts` (one fresh incognito context per capture), config +in `config/` (types generated in `config/type/` by `npm run build-types`). Node.js v24, TypeScript run directly by Node's type stripping (no build step, erasable syntax only), Express 5. diff --git a/api/routers/capture.ts b/api/routers/capture.ts index e648d32..d7a9638 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -13,8 +13,9 @@ import type { PaperFormat, PDFOptions } from 'puppeteer' const debug = debugModule('capture') -// /inspect renders are expensive, a small number of them run at the same time -const inspectSemaphore = createSemaphore(config.inspectConcurrency) +// /inspect renders are expensive, a small number of them run at the same time (each one then takes +// a capture slot like the other routes) +const inspectSemaphore = createSemaphore(config.inspectConcurrency, config.queue) export const router = express.Router() @@ -267,9 +268,8 @@ router.get('/inspect', async (req, res) => { let collector: ReturnType | undefined const signal = clientSignal(res) + // the client may leave while waiting for a slot: the signal drops the render await inspectSemaphore.run(async () => { - // the client may have left while waiting for a slot, no need to render for nobody - if (req.socket.destroyed || res.destroyed) return await pageUtils.withPage( target, lang, @@ -300,6 +300,6 @@ router.get('/inspect', async (req, res) => { }, signal ) - }) + }, signal).catch(err => { throw pageUtils.slotError(err) }) timer.finish() }) diff --git a/api/utils/metrics.ts b/api/utils/metrics.ts index 8cac84d..0f8db4b 100644 --- a/api/utils/metrics.ts +++ b/api/utils/metrics.ts @@ -14,27 +14,27 @@ export const tasks = new client.Histogram({ }) export const contextsCleanups = new client.Counter({ name: 'df_capture_contexts_cleanups', - help: 'Number of times a browser context was cleaned up for later reuse' + help: 'Number of browser contexts closed after their capture (one fresh context per capture)' }) export const acquireContextPending = new client.Gauge({ name: 'df_capture_acquire_context_pending', - help: 'Number of tasks waiting to acquire a browser context', + help: 'Number of captures waiting for a slot', async collect () { - this.set(pageUtils.contextPool && pageUtils.contextPool().pending) + this.set(pageUtils.captureSlots.pending()) } }) export const acquireContextMax = new client.Gauge({ name: 'df_capture_acquire_context_max', - help: 'Max number of browser contexts that can be borrowed by tasks', + help: 'Max number of captures running at the same time', async collect () { - this.set(pageUtils.contextPool && pageUtils.contextPool().max) + this.set(pageUtils.captureSlots.max) } }) export const acquireContextBorrowed = new client.Gauge({ name: 'df_capture_acquire_context_borrowed', - help: 'Number of browser contexts currently borrowed by a task', + help: 'Number of captures running', async collect () { - this.set(pageUtils.contextPool && pageUtils.contextPool().borrowed) + this.set(pageUtils.captureSlots.active()) } }) diff --git a/api/utils/page.ts b/api/utils/page.ts index 53f794f..14f44b4 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -1,11 +1,11 @@ import config from '#config' import puppeteer, { type Viewport, type Browser, type BrowserContext, type Page, type CookieData } from 'puppeteer' -import genericPool, { type Factory, type Pool } from 'generic-pool' import { contextsCleanups } from './metrics.ts' import debugModule from 'debug' import log from '@data-fair/lib-node/log.js' import { Timer } from './timer.ts' import { httpError } from '@data-fair/lib-express' +import { createSemaphore, SemaphoreError } from './semaphore.ts' import type { Ready } from './report.ts' const debug = debugModule('capture') @@ -13,42 +13,20 @@ const debug = debugModule('capture') let _closed = false let _browser: Browser -let _contextPool: Pool -export function contextPool () { - if (!_contextPool) throw new Error('contextPool not initialized') - return _contextPool -} +// the captures running at the same time, each one in its own incognito context. The others wait +// in a bounded queue (config.queue) +export const captureSlots = createSemaphore(config.concurrency, config.queue) export const start = async () => { log.info('init puppeteer browser', { launchOptions: config.puppeteerLaunchOptions }) _browser = await puppeteer.launch(config.puppeteerLaunchOptions) - const contextFactory: Factory = { - async create () { - // create pages in incognito contexts so that cookies are not shared - // each context is used sequentially only because of cookies or other states conflicts - return _browser.createBrowserContext({}) - }, - async destroy (context) { - await context.close() - } - } - _contextPool = genericPool.createPool(contextFactory, { - max: config.concurrency, - maxWaitingClients: config.queue.maxWaiting, - acquireTimeoutMillis: config.queue.maxWaitTime - }) // auto reconnection, cf https://github.com/GoogleChrome/puppeteer/issues/4428 + // the captures running in the lost browser fail, the next ones use the new browser _browser.on('disconnected', async () => { if (!_closed) { log.error('browser was disconnected, reconnect', { code: 'browser-disconnected' }) - try { - await _contextPool.drain() - _contextPool.clear() - } catch (err) { - log.error('failure while draining the replaced contexts pool', { code: 'contexts-pool-drain', err }) - } start().catch(err => log.error('failure while restarting the browser', { code: 'browser-restart', err })) } }) @@ -57,10 +35,6 @@ export const start = async () => { export const stop = async () => { _closed = true if (_browser) await _browser.close() - if (_contextPool) { - await _contextPool.drain() - _contextPool.clear() - } } const debugPage = (page: Page) => { @@ -203,17 +177,14 @@ const unlessAborted = async (promise: Promise, signal: AbortSignal | undef } } -// a full queue or a long wait for a context mean an overloaded service: better a quick 503 -// that the caller can retry than a response after the proxy has given up -const acquireContext = async () => { - try { - return await _contextPool.acquire() - } catch (err: any) { - if (err.name === 'TimeoutError' || err.message?.includes('max waitingClients')) { - throw httpError(503, 'the capture service is overloaded, retry later') - } - throw err +// a full queue or a long wait for a slot mean an overloaded service: better a quick 503 that the +// caller can retry than a response after it has given up +export const slotError = (err: unknown) => { + if (err instanceof SemaphoreError) { + if (err.code === 'aborted') return clientGone() + return httpError(503, 'the capture service is overloaded, retry later') } + return err } export const withPage = async ( @@ -233,81 +204,49 @@ export const withPage = async ( await new Promise(resolve => setTimeout(resolve, 1000)) throw new Error('forced error trigger') } - let context, page, openSuccess try { - debug(`[${target}] use incognito context from pool`) - let pageAttempts = 0 - while (!page) { - pageAttempts++ - if (signal?.aborted) throw clientGone() - context = await acquireContext() + await captureSlots.run(async () => { timer.step('acquire-context') - // the client may have left while waiting for a context, no need to render for nobody - if (signal?.aborted) { - await _contextPool.release(context) - throw clientGone() - } + // a fresh incognito context per capture, closed after it: nothing of a capture (cookies, + // localStorage, IndexedDB, cache, service workers) can leak into the next one, whatever + // the user it is made for. It costs a few milliseconds + const context = await _browser.createBrowserContext() try { - page = await context.newPage() - } catch (err) { - if (pageAttempts === config.concurrency) { - throw err - } else { - debug(`[${target}] delete incognito context from pool because it couldn't be used to open a page`) - await _contextPool.destroy(context) - } - } - } - // the console listener costs 2 CDP round trips per argument of every console message, on the - // main thread of the service: only paid when the debug output is enabled (DEBUG=capture) - if (debug.enabled) debugPage(page) - inspect?.onPage?.(page) - timer.step('newPage') + const page = await context.newPage() + // the console listener costs 2 CDP round trips per argument of every console message, on + // the main thread of the service: only paid when the debug output is enabled (DEBUG=capture) + if (debug.enabled) debugPage(page) + inspect?.onPage?.(page) + timer.step('newPage') - const result = await unlessAborted(promiseTimeout( - openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), - config.screenshotTimeout * 2, - `Failed to open "${target}" in context before timeout` - ), signal) - timer.step('open-in-page') - openSuccess = true - await unlessAborted(promiseTimeout( - callback(result), - result.animationActivated ? config.screenshotTimeout * 2 : config.screenshotTimeout, - callbackTimeoutMsg - ), signal) - } finally { - if (page) await safeCleanContext(page, !!openSuccess) + const result = await unlessAborted(promiseTimeout( + openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), + config.screenshotTimeout * 2, + `Failed to open "${target}" in context before timeout` + ), signal) + timer.step('open-in-page') + await unlessAborted(promiseTimeout( + callback(result), + result.animationActivated ? config.screenshotTimeout * 2 : config.screenshotTimeout, + callbackTimeoutMsg + ), signal) + } finally { + await closeContext(context) + } + }, signal) + } catch (err) { + throw slotError(err) } timer.step('close-page') } -const cleanContext = async (page: Page) => { - // always empty cookies to prevent inheriting them in next use of the context - const context = page.browserContext() - const cookies = await context.cookies() - // deleting them in a single call spares one CDP round trip per cookie, and this - // runs inside a 2s timeout that a session's worth of cookies could otherwise eat - if (cookies.length) await context.deleteCookie(...cookies) - await page.close() -} - -const safeCleanContext = async (page: Page, openSuccess: boolean) => { - if (openSuccess) { - try { - await promiseTimeout( - cleanContext(page), - 2000, - 'timed out while cleaning page context' - ) - await _contextPool.release(page.browserContext()) - contextsCleanups.inc() - } catch (err) { - log.warn('failed to clean page properly, do not reuse this context', { code: 'page-clean', err }) - await _contextPool.destroy(page.browserContext()) - } - } else { - await _contextPool.destroy(page.browserContext()) +const closeContext = async (context: BrowserContext) => { + try { + await promiseTimeout(context.close(), 5000, 'timed out while closing a browser context') + contextsCleanups.inc() + } catch (err) { + // the browser may be gone (disconnected), or stuck: nothing more to do for this capture + log.warn('failed to close a browser context', { code: 'context-close', err }) } } diff --git a/api/utils/semaphore.ts b/api/utils/semaphore.ts index 49bdbcd..2769395 100644 --- a/api/utils/semaphore.ts +++ b/api/utils/semaphore.ts @@ -1,17 +1,51 @@ // Limits how many async functions run at the same time, the others wait in FIFO order. -export const createSemaphore = (max: number) => { +// The queue can be bounded (maxWaiting), the wait limited (maxWaitTime), and a waiting call is +// dropped when its signal aborts. Rejections carry a code: queue-full, queue-timeout, aborted. + +export type SemaphoreOptions = { maxWaiting?: number, maxWaitTime?: number } + +export class SemaphoreError extends Error { + code: 'queue-full' | 'queue-timeout' | 'aborted' + constructor (code: SemaphoreError['code'], message: string) { + super(message) + this.code = code + } +} + +export const createSemaphore = (max: number, options: SemaphoreOptions = {}) => { let active = 0 const queue: Array<() => void> = [] - const acquire = () => new Promise(resolve => { + const acquire = (signal?: AbortSignal) => new Promise((resolve, reject) => { + if (signal?.aborted) return reject(new SemaphoreError('aborted', 'aborted before acquiring a slot')) if (active < max) { + active++ + return resolve() + } + if (options.maxWaiting !== undefined && queue.length >= options.maxWaiting) { + return reject(new SemaphoreError('queue-full', `more than ${options.maxWaiting} calls are already waiting`)) + } + let timer: NodeJS.Timeout | undefined + const leave = (err: SemaphoreError) => { + const i = queue.indexOf(grant) + if (i !== -1) queue.splice(i, 1) + cleanup() + reject(err) + } + const onAbort = () => leave(new SemaphoreError('aborted', 'aborted while waiting for a slot')) + const cleanup = () => { + clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + } + const grant = () => { + cleanup() active++ resolve() - } else { - queue.push(() => { - active++ - resolve() - }) + } + queue.push(grant) + signal?.addEventListener('abort', onAbort, { once: true }) + if (options.maxWaitTime !== undefined) { + timer = setTimeout(() => leave(new SemaphoreError('queue-timeout', `waited more than ${options.maxWaitTime}ms for a slot`)), options.maxWaitTime) } }) @@ -21,8 +55,9 @@ export const createSemaphore = (max: number) => { } return { - async run (fn: () => Promise): Promise { - await acquire() + max, + async run (fn: () => Promise, signal?: AbortSignal): Promise { + await acquire(signal) try { return await fn() } finally { @@ -30,6 +65,8 @@ export const createSemaphore = (max: number) => { } }, // number of calls waiting for a free slot - pending: () => queue.length + pending: () => queue.length, + // number of calls running + active: () => active } } diff --git a/benchmark/FINDINGS.md b/benchmark/FINDINGS.md index e8b36fa..db4e671 100644 --- a/benchmark/FINDINGS.md +++ b/benchmark/FINDINGS.md @@ -27,6 +27,14 @@ tables below are from the 8-run pass. Raw JSON is in `results/` (git-ignored). `seccomp.json` (renderers in their own user namespace). Its version comes from Debian's security updates, not from puppeteer: puppeteer 25.16 expects 156, trixie ships 156.0.8078.79. Keep the two close when upgrading puppeteer. +- **Concurrency review (2026-11-09)**: the contexts pool reused incognito contexts across + captures and only deleted cookies, so localStorage, IndexedDB and Cache Storage of one capture + were visible to the next one, whatever its user (shown by a test on master). Each capture now + runs in a fresh incognito context, closed after it (+3 ms). generic-pool is replaced by the + service's own semaphore, with a bounded queue, a wait timeout (30 s, below data-fair's 60 s) + and abort while waiting. `/inspect` uses the same mechanism. Infrastructure side: the + production pod runs at its 1.5 GiB limit at concurrency 3 on the real mix and uses >5 cores with + no CPU request. Concurrency 2 is planned. - **Declined**: untagged PDFs (§4). Accessibility comes first. ## Steady load on real applications (2026-11-09) diff --git a/config/default.cjs b/config/default.cjs index f8f90f0..59f4a59 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -17,11 +17,12 @@ module.exports = { // ones are encoded fast (2-3x quicker, 1.1-1.5x bigger files) pngFastEncodingPixels: 1500000, concurrency: 5, - // captures waiting for a browser context: beyond maxWaiting, or after maxWaitTime of waiting, - // they are answered by a 503 (below the 60s timeout of the reverse proxy) + // captures waiting for a slot: beyond maxWaiting, or after maxWaitTime of waiting, they are + // answered by a 503. data-fair gives up after 60s without a response and retries once, the wait + // plus the render should stay below queue: { maxWaiting: 100, - maxWaitTime: 20000 + maxWaitTime: 30000 }, inspectConcurrency: 1, // concurrent /inspect renders, each one holds a browser page defaultLang: 'fr-FR', diff --git a/package-lock.json b/package-lock.json index c144e89..cd484c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,6 @@ "cookie-parser": "^1.4.7", "debug": "^4.4.3", "express": "^5.3.0", - "generic-pool": "^3.9.0", "get-pixels": "github:scijs/get-pixels#90e79d5786b13aadefe5427cdc23cf1f164ec637", "gif-encoder": "^0.7.2", "helmet": "^8.1.0", @@ -3444,15 +3443,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/generic-pool": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", - "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", diff --git a/package.json b/package.json index 7d27c6d..61c0455 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "cookie-parser": "^1.4.7", "debug": "^4.4.3", "express": "^5.3.0", - "generic-pool": "^3.9.0", "get-pixels": "github:scijs/get-pixels#90e79d5786b13aadefe5427cdc23cf1f164ec637", "gif-encoder": "^0.7.2", "helmet": "^8.1.0", diff --git a/test-it/capture.ts b/test-it/capture.ts index 312ca9f..6bdffb0 100644 --- a/test-it/capture.ts +++ b/test-it/capture.ts @@ -84,6 +84,16 @@ describe('capture', () => { assert.ok('wait2-settle' in times) }) + it('isolate captures from each other: nothing is left in the storages of the page', async () => { + const storageTarget = origin + '/test/resources/test-storage.html' + // sequential captures, so that a reused browser context would be reused here + for (const mark of ['user-a', 'user-b', 'user-c']) { + const res = await ax.get('print', { params: { key, target: storageTarget + '?mark=' + mark, type: 'html' } }) + assert.equal(res.status, 200) + assert.ok(res.data.includes('previous=null'), `a previous capture left a mark: ${res.data.match(/previous=[^<]*/)?.[0]}`) + } + }) + it('drop the captures of clients that left', async () => { // more abandoned captures than browser contexts (concurrency 5) of a page that is never idle, // each would hold a context for the whole screenshot timeout (2s in tests) diff --git a/test-it/resources/test-storage.html b/test-it/resources/test-storage.html new file mode 100644 index 0000000..3e1bc13 --- /dev/null +++ b/test-it/resources/test-storage.html @@ -0,0 +1,18 @@ + + + + + Storage page + + +

Storage page

+

+ + + diff --git a/test-it/semaphore.ts b/test-it/semaphore.ts index 2a4e28b..5356fa5 100644 --- a/test-it/semaphore.ts +++ b/test-it/semaphore.ts @@ -49,4 +49,49 @@ describe('semaphore', () => { assert.equal(await p2, 'next') assert.equal(sem.pending(), 0) }) + + it('rejects beyond maxWaiting, without disturbing the queue', async () => { + const sem = createSemaphore(1, { maxWaiting: 1 }) + const first = deferred() + const p1 = sem.run(() => first.promise) + const p2 = sem.run(async () => 'second') + await assert.rejects(sem.run(async () => 'third'), (err: any) => err.code === 'queue-full') + assert.equal(sem.pending(), 1) + first.resolve('first') + assert.equal(await p1, 'first') + assert.equal(await p2, 'second') + }) + + it('rejects a call that waited longer than maxWaitTime, and takes it out of the queue', async () => { + const sem = createSemaphore(1, { maxWaitTime: 50 }) + const first = deferred() + const p1 = sem.run(() => first.promise) + let lateStarted = false + await assert.rejects(sem.run(async () => { lateStarted = true }), (err: any) => err.code === 'queue-timeout') + assert.equal(sem.pending(), 0) + first.resolve('first') + await p1 + await tick() + assert.equal(lateStarted, false) + assert.equal(sem.active(), 0) + }) + + it('rejects an aborted call, before or while it waits', async () => { + const sem = createSemaphore(1) + const first = deferred() + const p1 = sem.run(() => first.promise) + const controller = new AbortController() + let abortedStarted = false + const p2 = sem.run(async () => { abortedStarted = true }, controller.signal) + await tick() + controller.abort() + await assert.rejects(p2, (err: any) => err.code === 'aborted') + assert.equal(sem.pending(), 0) + await assert.rejects(sem.run(async () => {}, controller.signal), (err: any) => err.code === 'aborted') + first.resolve('first') + await p1 + await tick() + assert.equal(abortedStarted, false) + assert.equal(sem.active(), 0) + }) })