From d981e077bba8a95477a8a3f85b64ae22d660d587 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Thu, 5 Nov 2026 09:07:44 +0100 Subject: [PATCH 1/7] feat: inspect endpoint returning a screenshot and a render report Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + api/routers/capture.ts | 72 +++++++- api/utils/aria-snapshot.ts | 193 +++++++++++++++++++++ api/utils/page.ts | 28 ++- api/utils/report-text.ts | 47 +++++ api/utils/report.ts | 151 ++++++++++++++++ test-it/inspect.ts | 151 ++++++++++++++++ test-it/resources/test-big.html | 19 ++ test-it/resources/test-busy.html | 14 ++ test-it/resources/test-errors.html | 17 ++ test-it/resources/test-frame.html | 17 ++ test-it/resources/test-login-redirect.html | 12 ++ test-it/resources/test-snapshot.html | 19 ++ 13 files changed, 731 insertions(+), 10 deletions(-) create mode 100644 api/utils/aria-snapshot.ts create mode 100644 api/utils/report-text.ts create mode 100644 api/utils/report.ts create mode 100644 test-it/inspect.ts create mode 100644 test-it/resources/test-big.html create mode 100644 test-it/resources/test-busy.html create mode 100644 test-it/resources/test-errors.html create mode 100644 test-it/resources/test-frame.html create mode 100644 test-it/resources/test-login-redirect.html create mode 100644 test-it/resources/test-snapshot.html diff --git a/README.md b/README.md index 5ce7fe8..2935c13 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ Check the service with these examples (in the dev environment, replace 5607 with - [animated gif screenshot](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&target=http://localhost:5607/test/resources/test-anim.html) - [animated gif screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test-anim.html) - [fallback to standard screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test1.html) + - [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=capture&target=http://localhost:5607/test/resources/test-errors.html) - [remote url with a webgl based map](http://localhost:5617/api/v1/print?key=capture&target=https://staging-koumoul.com/cadastre/parcel?id=56251000AV0142) ## Security diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 3cb5772..b6a6c6b 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -4,6 +4,8 @@ import * as headerFooter from '../utils/header-footer.ts' import * as pageUtils from '../utils/page.ts' import * as animationUtils from '../utils/animation.ts' import { createTimer } from '../utils/timer.ts' +import { createReportCollector } from '../utils/report.ts' +import { reportText } from '../utils/report-text.ts' import config from '#config' import { assertReqInternalSecret, httpError, reqHost, reqIsInternal, reqSession } from '@data-fair/lib-express' import type { PaperFormat, PDFOptions } from 'puppeteer' @@ -50,7 +52,7 @@ async function auth (req: Request, res: Response, next: NextFunction) { if (securedSameHost && req.cookies && Object.keys(req.cookies).length && req.query.cookies !== 'false') { debug(`[${target}] transmit cookies`) - req.cookies = Object.keys(req.cookies).map(name => ({ name, value: req.cookies[name], url: target })) + req.cookies = Object.keys(req.cookies).map(name => ({ name, value: req.cookies[name], url: target })) as any } else { debug(`[${target}] do not transmit cookies`) req.cookies = [] @@ -189,3 +191,71 @@ router.get('/print', auth, async (req, res, next) => { ) timer.finish() }) + +const desktopViewport = { width: 1280, height: 800 } +const mobileViewport = { width: 390, height: 844, isMobile: true, hasTouch: true } +const maxFullPageHeight = 3000 + +const viewportDimension = (value: unknown, fallback: number) => { + if (value === undefined) return fallback + const parsed = typeof value === 'string' ? parseInt(value) : NaN + if (!(parsed > 0 && parsed <= 3000)) throw httpError(400, 'invalid width or height') + return parsed +} + +// internal only: used by data-fair and portals so that agents can see what they configured +router.get('/inspect', async (req, res) => { + assertReqInternalSecret(req, config.secretKeys.capture) + + const target = req.query.target + if (typeof target !== 'string') throw httpError(400, 'parameter "target" is required') + let targetUrl + try { + targetUrl = new URL(target) + } catch (err: any) { + throw httpError(400, 'Failed to parse target url ' + err.message) + } + if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') throw httpError(400, 'target must be an http or https url') + debug(`[${target}] inspect`) + + const lang = req.query.lang + if (lang && typeof lang !== 'string') throw httpError(400, 'invalid lang') + const timezone = req.query.timezone + if (timezone && typeof timezone !== 'string') throw httpError(400, 'invalid timezone') + + const mobile = req.query.mobile === 'true' + const fullPage = req.query.fullPage === 'true' + const viewport = mobile + ? mobileViewport + : { width: viewportDimension(req.query.width, desktopViewport.width), height: viewportDimension(req.query.height, desktopViewport.height) } + + const cookies = Object.keys(req.cookies ?? {}).map(name => ({ name, value: req.cookies[name], url: target })) as any + const timer = createTimer(req.originalUrl, 'inspect') + + let collector: ReturnType | undefined + await pageUtils.withPage( + target, + lang, + timezone, + cookies, + viewport, + false, + timer, + `Failed to inspect page "${target}" before timeout`, + async ({ page, ready }) => { + const report = await collector!.finish(ready) + let buffer + if (fullPage) { + const height = await page.evaluate(() => document.documentElement.scrollHeight) + buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) + } else { + buffer = await page.screenshot() + } + timer.type = 'inspect' + res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) + }, + true, + page => { collector = createReportCollector(page, target) } + ) + timer.finish() +}) diff --git a/api/utils/aria-snapshot.ts b/api/utils/aria-snapshot.ts new file mode 100644 index 0000000..30e1ac3 --- /dev/null +++ b/api/utils/aria-snapshot.ts @@ -0,0 +1,193 @@ +import type { Page, SerializedAXNode } from 'puppeteer' + +/* eslint-disable no-control-regex */ +// yamlEscapeKeyIfNeeded and yamlEscapeValueIfNeeded are copied from Playwright +// https://github.com/microsoft/playwright/blob/main/packages/isomorphic/yaml.ts +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function yamlEscapeKeyIfNeeded (str: string): string { + if (!yamlStringNeedsQuotes(str)) { return str } + return '\'' + str.replace(/'/g, '\'\'') + '\'' +} + +export function yamlEscapeValueIfNeeded (str: string): string { + if (!yamlStringNeedsQuotes(str)) { return str } + return '"' + str.replace(/[\\"\x00-\x1f\x7f-\x9f]/g, c => { + switch (c) { + case '\\': + return '\\\\' + case '"': + return '\\"' + case '\b': + return '\\b' + case '\f': + return '\\f' + case '\n': + return '\\n' + case '\r': + return '\\r' + case '\t': + return '\\t' + default: { + const code = c.charCodeAt(0) + return '\\x' + code.toString(16).padStart(2, '0') + } + } + }) + '"' +} + +function yamlStringNeedsQuotes (str: string): boolean { + if (str.length === 0) { return true } + + // Strings with leading or trailing whitespace need quotes + if (/^\s|\s$/.test(str)) { return true } + + // Strings containing control characters need quotes + if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/.test(str)) { return true } + + // Strings starting with '-' need quotes + if (/^-/.test(str)) { return true } + + // Strings containing ':' or '\n' followed by a space or at the end need quotes + if (/[\n:](\s|$)/.test(str)) { return true } + + // Strings containing '#' preceded by a space need quotes (comment indicator) + if (/\s#/.test(str)) { return true } + + // Strings that contain line breaks need quotes + if (/[\n\r]/.test(str)) { return true } + + // Strings starting with indicator characters or quotes need quotes + if (/^[&*\],?!>|@"'#%]/.test(str)) { return true } + + // Strings containing special characters that could cause ambiguity + if (/[{}`]/.test(str)) { return true } + + // YAML array starts with [ + if (/^\[/.test(str)) { return true } + + // Non-string types recognized by YAML + if (!isNaN(Number(str)) || ['y', 'n', 'yes', 'no', 'true', 'false', 'on', 'off', 'null', '~'].includes(str.toLowerCase())) { return true } + + // YAML special float values: .inf, -.inf, .nan and their case variants + if (/^[-+]?\.(inf|nan)$/i.test(str)) { return true } + + return false +} + +/* eslint-enable no-control-regex */ + +const indent = (depth: number) => ' '.repeat(depth) + +const createKey = (node: SerializedAXNode) => { + let key = node.role === 'Iframe' ? 'iframe' : node.role + // YAML has a limit of 1024 characters per key, leave room for the role and attributes + const name = text(node) + if (name && name.length <= 900) key += ' ' + JSON.stringify(name) + if (node.checked === 'mixed') key += ' [checked=mixed]' + if (node.checked === true) key += ' [checked]' + if (node.disabled) key += ' [disabled]' + if (node.expanded) key += ' [expanded]' + if (node.role === 'heading' && node.level) key += ` [level=${node.level}]` + if (node.pressed === 'mixed') key += ' [pressed=mixed]' + if (node.pressed === true) key += ' [pressed]' + if (node.selected) key += ' [selected]' + if (node.invalid && node.invalid !== 'false') key += ' [invalid]' + return key +} + +const isText = (node: SerializedAXNode) => node.role === 'StaticText' +const text = (node: SerializedAXNode) => (node.name ?? '').replace(/\s+/g, ' ').trim() + +// interestingOnly would drop useful containers (paragraph, list, listitem), so the whole tree is +// requested and its structural noise is flattened here: children of unnamed none/generic nodes and +// of nested documents (iframes) are hoisted, text boxes and list markers are dropped +const flatten = (nodes: SerializedAXNode[] | undefined): SerializedAXNode[] => { + const result: SerializedAXNode[] = [] + for (const node of nodes ?? []) { + if (node.role === 'InlineTextBox' || node.role === 'ListMarker') continue + if (node.role === 'RootWebArea' || ((node.role === 'none' || node.role === 'generic') && !node.name)) result.push(...flatten(node.children)) + else result.push({ ...node, children: flatten(node.children) }) + } + return result +} + +export const ariaSnapshot = async (page: Page, opts: { maxChars: number, maxLinks: number }): Promise => { + const root = await page.accessibility.snapshot({ interestingOnly: false, includeIframes: true }) + if (!root) return '' + const lines: string[] = [] + let links = 0 + + const visitText = (text: string, depth: number) => { + const escaped = yamlEscapeValueIfNeeded(text) + if (escaped) lines.push(indent(depth) + '- text: ' + escaped) + } + + const visit = async (node: SerializedAXNode, depth: number) => { + if (isText(node)) return visitText(text(node), depth) + const key = indent(depth) + '- ' + yamlEscapeKeyIfNeeded(createKey(node)) + + let children = node.children ?? [] + // children texts that only repeat the name are redundant + if (text(node) && children.length && children.every(isText) && text({ name: children.map(c => c.name ?? '').join('') } as SerializedAXNode) === text(node)) children = [] + + let href: string | undefined + if (node.role === 'link' && links < opts.maxLinks) { + links++ + try { + const handle = await node.elementHandle() + href = (await handle?.evaluate(el => el.getAttribute('href'))) ?? node.url + await handle?.dispose() + } catch (err) { + href = node.url + } + } + + let inlineText: string | undefined + if ((node.role === 'textbox' || node.role === 'combobox') && node.value !== undefined && node.value !== '') inlineText = String(node.value) + else if (!text(node) && children.length === 1 && isText(children[0]) && !href) { + inlineText = text(children[0]) + children = [] + } + + if (inlineText !== undefined && !href && !children.length) { + lines.push(key + ': ' + yamlEscapeValueIfNeeded(inlineText)) + } else if (!href && !children.length && inlineText === undefined) { + lines.push(key) + } else { + lines.push(key + ':') + if (href) lines.push(indent(depth + 1) + '- /url: ' + yamlEscapeValueIfNeeded(href)) + if (inlineText !== undefined) visitText(inlineText, depth + 1) + for (const child of children) await visit(child, depth + 1) + } + } + + for (const child of flatten(root.children)) await visit(child, 0) + + // truncate at a line boundary, keeping room for the last line + const reserve = 50 + let size = 0 + let kept = 0 + for (const line of lines) { + if (size + line.length + 1 > opts.maxChars - reserve) break + size += line.length + 1 + kept++ + } + if (kept < lines.length) { + return lines.slice(0, kept).concat(`# snapshot truncated: ${lines.length - kept} more lines`).join('\n') + } + return lines.join('\n') +} diff --git a/api/utils/page.ts b/api/utils/page.ts index bb03376..6da7cec 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -5,6 +5,7 @@ import { contextsCleanups } from './metrics.ts' import debugModule from 'debug' import { Timer } from './timer.ts' import { httpError } from '@data-fair/lib-express' +import type { Ready } from './report.ts' const debug = debugModule('capture') @@ -83,7 +84,8 @@ const debugPage = (page: Page) => { type OpenPageResult = { page: Page, - animationActivated: boolean + animationActivated: boolean, + ready: Ready } async function openInPage ( @@ -94,30 +96,33 @@ async function openInPage ( cookies: CookieData[], viewport: Viewport | undefined, animate: boolean, - timer: Timer + timer: Timer, + reportFrames: boolean ): Promise { await setPageLocale(page, lang || config.defaultLang, timezone || config.defaultTimezone) if (cookies) await page.browserContext().setCookie(...cookies) if (viewport) await page.setViewport(viewport) timer.step('configure-page') - const animationActivated = await waitForPage(page, target, animate, timer) + const { animationActivated, ready } = await waitForPage(page, target, animate, timer) for (const frame of page.frames()) { const frameUrl = frame.url() if (!frameUrl) continue + // the frames are recorded by the report collector instead of failing the request + if (reportFrames && !/^https?:/.test(frameUrl)) continue let sameHost = false try { sameHost = new URL(frameUrl).host === new URL(target).host } catch (err) { throw httpError(400, 'IFrame with invalid URL :' + frameUrl) } - if (!sameHost && config.onlySameHost) { + if (!sameHost && config.onlySameHost && !reportFrames) { debug(`${frameUrl} from iframe in ${target} is NOT on same host, reject`) throw httpError(400, 'IFrame did not have same host :' + new URL(frameUrl).host) } } - return { page, animationActivated } + return { page, animationActivated, ready } } export const promiseTimeout = async (promise: Promise, ms: number, msg = 'timeout') => { @@ -136,7 +141,9 @@ export const withPage = async ( animate: boolean, timer: Timer, callbackTimeoutMsg: string, - callback: (result: OpenPageResult) => Promise) => { + callback: (result: OpenPageResult) => Promise, + reportFrames = false, + onPage?: (page: Page) => void) => { if (target.includes('capture-test-error=true')) { await new Promise(resolve => setTimeout(resolve, 1000)) throw new Error('forced error trigger') @@ -161,10 +168,11 @@ export const withPage = async ( } } debugPage(page) + onPage?.(page) timer.step('newPage') const result = await promiseTimeout( - openInPage(page, target, lang, timezone, cookies, viewport, animate, timer), + openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, reportFrames), config.screenshotTimeout * 2, `Failed to open "${target}" in context before timeout` ) @@ -217,7 +225,7 @@ const waitForPage = async ( target: string, animate: boolean, timer: Timer -) => { +): Promise<{ animationActivated: boolean, ready: Ready }> => { // Prepare a function that the page can call to signal that it is ready for capture let captureTriggered = false let timeoutReached = false @@ -299,6 +307,7 @@ const waitForPage = async ( } else { timer.step('wait2-timeout') debug(`[${target}] timeout of ${config.screenshotTimeout} was reached`) + timeoutReached = true } } else { debug(`[${target}] wait 1000ms more after idle network for safety`) @@ -307,7 +316,8 @@ const waitForPage = async ( } } } - return animationActivated + const ready: Ready = captureTriggered ? 'triggered' : timeoutReached ? 'timeout' : 'idle' + return { animationActivated, ready } } const setPageLocale = async (page: Page, lang: string, timezone: string) => { diff --git a/api/utils/report-text.ts b/api/utils/report-text.ts new file mode 100644 index 0000000..edad25a --- /dev/null +++ b/api/utils/report-text.ts @@ -0,0 +1,47 @@ +import type { RenderReport } from './report.ts' + +// layout of Playwright MCP responses (### Page, ### Console, ### Network, ### Snapshot), +// plus the Ready, Verdict, [BLOCKED] and body lines that are specific to this tool +export const reportText = (report: RenderReport): string => { + const lines: string[] = ['### Page'] + lines.push(`- Page URL: ${report.finalUrl}`) + lines.push(`- Page Title: ${report.title}`) + if (report.status !== undefined && (report.status < 200 || report.status >= 300)) { + lines.push(`- HTTP status: ${report.status} ${report.statusText ?? ''}`.trimEnd()) + } + const errors = report.consoleErrors.length + report.pageErrors.length + if (errors > 0 || report.consoleWarnings > 0) { + lines.push(`- Console: ${errors} errors, ${report.consoleWarnings} warnings`) + } + lines.push(`- Ready: ${report.ready === 'timeout' ? `timeout after ${report.durationMs} ms` : report.ready}`) + lines.push(`- Verdict: ${report.verdict}`) + + if (report.consoleErrors.length || report.pageErrors.length) { + lines.push('### Console') + for (const entry of report.consoleErrors) { + lines.push(`[ERROR] ${entry.text}` + (entry.url ? ` @ ${entry.url}${entry.line !== undefined ? ':' + entry.line : ''}` : '')) + } + for (const error of report.pageErrors) lines.push(error) + } + + if (report.failedRequests.length || report.blockedFrames.length) { + lines.push('### Network') + let i = 0 + for (const request of report.failedRequests) { + i++ + if (request.status !== undefined) { + lines.push(`${i}. [${request.method}] ${request.url} => [${request.status}] ${request.statusText ?? ''}`.trimEnd()) + if (request.body) lines.push(` ${request.body}`) + } else { + lines.push(`${i}. [${request.method}] ${request.url} => [FAILED] ${request.errorText ?? ''}`.trimEnd()) + } + } + for (const origin of report.blockedFrames) { + i++ + lines.push(`${i}. [GET] ${origin} => [BLOCKED] cross-host frame not rendered`) + } + } + + lines.push('### Snapshot', '```yaml', report.snapshot, '```') + return lines.join('\n') +} diff --git a/api/utils/report.ts b/api/utils/report.ts new file mode 100644 index 0000000..dc84207 --- /dev/null +++ b/api/utils/report.ts @@ -0,0 +1,151 @@ +import type { Page } from 'puppeteer' +import { ariaSnapshot } from './aria-snapshot.ts' + +export type Ready = 'triggered' | 'idle' | 'timeout' +export type Verdict = 'ok' | 'errors' | 'timeout' | 'unauthenticated' +export type ConsoleEntry = { text: string, url?: string, line?: number } +export type FailedRequest = { method: string, url: string, path: string, status?: number, statusText?: string, errorText?: string, body?: string } +export type RenderReport = { + url: string, + finalUrl: string, + status?: number, + statusText?: string, + ready: Ready, + durationMs: number, + title: string, + consoleErrors: ConsoleEntry[], + consoleWarnings: number, + pageErrors: string[], + failedRequests: FailedRequest[], + blockedFrames: string[], + snapshot: string, + verdict: Verdict +} + +const MAX_ENTRIES = 20 +const MAX_STRING = 500 +const MAX_BODY = 300 +const SNAPSHOT_MAX_CHARS = 20000 +const SNAPSHOT_MAX_LINKS = 200 + +const cut = (str: string, max = MAX_STRING) => str.length > max ? str.slice(0, max) + '…' : str + +const withoutQuery = (url: string) => { + const parsed = new URL(url) + parsed.search = '' + parsed.hash = '' + return parsed +} + +// browsers ask for it on their own, a missing one is noise for the agent +const isFavicon = (url: string) => new URL(url).pathname === '/favicon.ico' + +const parseOrigin = (url: string) => { + try { + return new URL(url).origin + } catch (err) { + return undefined + } +} + +export const createReportCollector = (page: Page, target: string) => { + const started = Date.now() + const targetOrigin = parseOrigin(target) + const consoleErrors: ConsoleEntry[] = [] + let consoleWarnings = 0 + const pageErrors: string[] = [] + const failedRequests: FailedRequest[] = [] + const pending: Promise[] = [] + let status: number | undefined + let statusText: string | undefined + + const sameOrigin = (url: string) => parseOrigin(url) === targetOrigin + + page.on('console', msg => { + if (msg.type() === 'error') { + const { url, lineNumber } = msg.location() + consoleErrors.push({ text: msg.text(), ...(url ? { url } : {}), ...(url && lineNumber !== undefined ? { line: lineNumber } : {}) }) + } else if (msg.type() === 'warn') { + consoleWarnings++ + } + }) + page.on('pageerror', err => { + pageErrors.push(err instanceof Error ? (err.stack || err.message) : String(err)) + }) + page.on('requestfailed', request => { + if (!sameOrigin(request.url()) || isFavicon(request.url())) return + const url = withoutQuery(request.url()) + failedRequests.push({ method: request.method(), url: url.href, path: url.pathname, errorText: request.failure()?.errorText }) + }) + page.on('response', response => { + const request = response.request() + if (request.isNavigationRequest() && request.frame() === page.mainFrame()) { + status = response.status() + statusText = response.statusText() + } + if (response.status() < 400 || !sameOrigin(response.url()) || isFavicon(response.url())) return + const url = withoutQuery(response.url()) + const entry: FailedRequest = { + method: request.method(), + url: url.href, + path: url.pathname, + status: response.status(), + statusText: response.statusText() + } + failedRequests.push(entry) + pending.push((async () => { + const contentType = response.headers()['content-type'] ?? '' + if (/^text\/|json|html/i.test(contentType)) { + try { + const body = (await response.text()).trim() + if (body) entry.body = cut(body, MAX_BODY) + } catch (err) { + // the body is not always available (redirects, aborted requests) + } + } else if (contentType) { + entry.body = `[${contentType}]` + } + })()) + }) + + const finish = async (ready: Ready): Promise => { + // events of the last requests can reach us a few ms after the page signaled it is ready + await new Promise(resolve => setTimeout(resolve, 100)) + await Promise.all(pending) + const finalUrl = page.url() + const [title, snapshot] = await Promise.all([ + page.title().catch(() => ''), + ariaSnapshot(page, { maxChars: SNAPSHOT_MAX_CHARS, maxLinks: SNAPSHOT_MAX_LINKS }) + ]) + const blockedFrames: string[] = [] + for (const frame of page.frames()) { + const origin = /^https?:/.test(frame.url()) ? parseOrigin(frame.url()) : undefined + if (origin && origin !== targetOrigin && !blockedFrames.includes(origin)) blockedFrames.push(origin) + } + + let verdict: Verdict + if (new URL(finalUrl, target).pathname.startsWith('/simple-directory/login')) verdict = 'unauthenticated' + else if (ready === 'timeout') verdict = 'timeout' + else if (consoleErrors.length || pageErrors.length || failedRequests.length || blockedFrames.length || (status !== undefined && status >= 400)) verdict = 'errors' + else verdict = 'ok' + + return { + url: target, + finalUrl, + status, + statusText, + ready, + durationMs: Date.now() - started, + title: cut(title), + consoleErrors: consoleErrors.slice(0, MAX_ENTRIES).map(e => ({ ...e, text: cut(e.text), ...(e.url ? { url: cut(e.url) } : {}) })), + consoleWarnings, + pageErrors: pageErrors.slice(0, MAX_ENTRIES).map(e => cut(e)), + failedRequests: failedRequests.slice(0, MAX_ENTRIES).map(f => ({ ...f, url: cut(f.url), path: cut(f.path), ...(f.errorText ? { errorText: cut(f.errorText) } : {}) })), + blockedFrames: blockedFrames.slice(0, MAX_ENTRIES), + snapshot, + verdict + } + } + + return { finish } +} diff --git a/test-it/inspect.ts b/test-it/inspect.ts new file mode 100644 index 0000000..248e992 --- /dev/null +++ b/test-it/inspect.ts @@ -0,0 +1,151 @@ +import assert from 'assert/strict' +import http from 'node:http' +import { promisify } from 'node:util' +import getPixelsCb from 'get-pixels' +import * as server from '../api/server.ts' +import axios from 'axios' +import { describe, before, after, it } from 'node:test' +import config from '#config' + +const getPixels = promisify(getPixelsCb) +const pngSize = async (data: Buffer) => { + const [width, height] = (await getPixels(data, 'image/png')).shape + return { width, height } +} + +const origin = `http://localhost:${config.port}` +const ax = axios.create({ baseURL: `${origin}/api/v1/`, validateStatus: () => true }) +const key = 'capture' +const inspect = (path: string, params: Record = {}, headers: Record = { 'x-secret-key': key }) => + ax.get('inspect', { params: { target: origin + path, ...params }, headers }) + +describe('inspect', () => { + before(async function () { + await server.start() + }) + after(async () => { + // let the last request release its page before the browser is closed + await new Promise(resolve => setTimeout(resolve, 300)) + await server.stop() + }) + + it('inspects a page with errors', async () => { + const res = await inspect('/test/resources/test-errors.html') + assert.equal(res.status, 200) + assert.equal(res.data.image.mediaType, 'image/png') + assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 1280, height: 800 }) + const r = res.data.report + assert.equal(r.ready, 'triggered') + assert.equal(r.status, 200) + assert.ok(r.consoleErrors.some((e: any) => e.text.includes('boom-console'))) + assert.ok(r.pageErrors.some((e: string) => e.includes('boom-page'))) + assert.deepEqual(r.failedRequests.map((f: any) => [f.path, f.status]), [['/test/resources/missing.json', 404]]) + assert.equal(r.verdict, 'errors') + assert.ok(res.data.text.startsWith('### Page\n- Page URL: ')) + assert.match(res.data.text, /\[ERROR\] boom-console @ /) + assert.match(res.data.text, /\d+\. \[GET\] .*\/test\/resources\/missing\.json => \[404\] Not Found/) + assert.match(res.data.text, /- Verdict: errors/) + }) + + it('snapshot in Playwright syntax', async () => { + const res = await inspect('/test/resources/test-snapshot.html') + assert.equal(res.status, 200) + const lines: string[] = res.data.report.snapshot.split('\n').map((l: string) => l.trim()) + for (const expected of ['- heading "Hello" [level=1]', '- link "Next":', '- /url: /next', '- paragraph: Some text', '- checkbox "Agree" [checked]']) { + assert.ok(lines.includes(expected), `missing line ${expected} in\n${res.data.report.snapshot}`) + } + const frameIndex = res.data.report.snapshot.split('\n').findIndex((l: string) => l.trim().startsWith('- iframe')) + assert.ok(frameIndex >= 0, 'missing iframe line') + const all = res.data.report.snapshot.split('\n') + const frameIndent = all[frameIndex].search(/\S/) + assert.ok(all.slice(frameIndex + 1).some((l: string) => l.search(/\S/) > frameIndent && l.includes('Test page 1'))) + assert.ok(!res.data.report.snapshot.includes('[ref=')) + assert.ok(res.data.text.includes('### Snapshot\n```yaml\n')) + }) + + it('truncates a big snapshot', async () => { + const res = await inspect('/test/resources/test-big.html') + assert.equal(res.status, 200) + const snapshot: string = res.data.report.snapshot + assert.ok(snapshot.length <= 20000) + assert.match(snapshot.split('\n').at(-1)!, /^# snapshot truncated: \d+ more lines$/) + }) + + it('mobile viewport', async () => { + const res = await inspect('/test/resources/test-snapshot.html', { mobile: 'true' }) + assert.equal(res.status, 200) + assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 390, height: 844 }) + }) + + it('busy page times out but still answers', async () => { + const res = await inspect('/test/resources/test-busy.html') + assert.equal(res.status, 200) + assert.equal(res.data.report.ready, 'timeout') + assert.equal(res.data.report.verdict, 'timeout') + assert.ok(res.data.image.data.length > 0) + assert.match(res.data.text, /- Ready: timeout after \d+ ms/) + }) + + it('login redirect is unauthenticated', async () => { + const res = await inspect('/test/resources/test-login-redirect.html') + assert.equal(res.status, 200) + assert.equal(res.data.report.verdict, 'unauthenticated') + assert.ok(new URL(res.data.report.finalUrl).pathname.startsWith('/simple-directory/login')) + }) + + it('reports cross-host frames', async () => { + const res = await inspect('/test/resources/test-frame.html') + assert.equal(res.status, 200) + assert.deepEqual(res.data.report.blockedFrames, [`http://127.0.0.1:${config.port}`]) + assert.equal(res.data.report.verdict, 'errors') + assert.match(res.data.text, /\[BLOCKED\] cross-host frame not rendered/) + }) + + it('full page is clipped to 3000px', async () => { + const res = await inspect('/test/resources/test-big.html', { fullPage: 'true' }) + assert.equal(res.status, 200) + assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 1280, height: 3000 }) + }) + + it('refuses without the secret', async () => { + assert.equal((await inspect('/test/resources/test1.html', {}, {})).status, 401) + assert.equal((await inspect('/test/resources/test1.html', {}, { 'x-secret-key': 'wrong' })).status, 401) + }) + + it('refuses a forwarded request', async () => { + const res = await inspect('/test/resources/test1.html', {}, { 'x-secret-key': key, 'x-forwarded-host': 'example.com' }) + assert.equal(res.status, 421) + }) + + it('refuses a non http target', async () => { + const res = await ax.get('inspect', { params: { target: 'file:///etc/passwd' }, headers: { 'x-secret-key': key } }) + assert.equal(res.status, 400) + }) + + it('cuts long bodies', async () => { + // a page served by a separate server so that it can answer a long 404 body, the Host header + // makes capture consider it as its own host + const other = http.createServer((req, res) => { + if (req.url === '/page') { + res.setHeader('content-type', 'text/html') + res.end('') + } else { + res.statusCode = 404 + res.setHeader('content-type', 'text/plain') + res.end('x'.repeat(5000)) + } + }) + await new Promise(resolve => other.listen(0, '127.0.0.1', resolve)) + const port = (other.address() as any).port + try { + const host = `127.0.0.1:${port}` + const res = await ax.get('inspect', { params: { target: `http://${host}/page` }, headers: { 'x-secret-key': key, host } }) + assert.equal(res.status, 200) + const failed = res.data.report.failedRequests.find((f: any) => f.path === '/long') + assert.ok(failed.body.length > 0) + assert.ok(failed.body.length <= 301) + } finally { + other.close() + } + }) +}) diff --git a/test-it/resources/test-big.html b/test-it/resources/test-big.html new file mode 100644 index 0000000..da5b29a --- /dev/null +++ b/test-it/resources/test-big.html @@ -0,0 +1,19 @@ + + + + + Big page + + +
    + + + diff --git a/test-it/resources/test-busy.html b/test-it/resources/test-busy.html new file mode 100644 index 0000000..db2e571 --- /dev/null +++ b/test-it/resources/test-busy.html @@ -0,0 +1,14 @@ + + + + + Busy page + + +

    Busy page

    + + + diff --git a/test-it/resources/test-errors.html b/test-it/resources/test-errors.html new file mode 100644 index 0000000..1a786cc --- /dev/null +++ b/test-it/resources/test-errors.html @@ -0,0 +1,17 @@ + + + + + Errors page + + +

    Errors page

    + + + diff --git a/test-it/resources/test-frame.html b/test-it/resources/test-frame.html new file mode 100644 index 0000000..d948812 --- /dev/null +++ b/test-it/resources/test-frame.html @@ -0,0 +1,17 @@ + + + + + Cross-host frame + + +

    Cross-host frame

    + + + diff --git a/test-it/resources/test-login-redirect.html b/test-it/resources/test-login-redirect.html new file mode 100644 index 0000000..01c0645 --- /dev/null +++ b/test-it/resources/test-login-redirect.html @@ -0,0 +1,12 @@ + + + + + Login redirect + + + + + diff --git a/test-it/resources/test-snapshot.html b/test-it/resources/test-snapshot.html new file mode 100644 index 0000000..b8b6a63 --- /dev/null +++ b/test-it/resources/test-snapshot.html @@ -0,0 +1,19 @@ + + + + + Snapshot page + + +

    Hello

    + Next +

    Some text

    + + + + + From a391674684f49e2a3dd232ed02b02311f4efe94d Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Thu, 5 Nov 2026 09:12:13 +0100 Subject: [PATCH 2/7] fix: keep cross-host frames out of inspect output and honor report limits Co-Authored-By: Claude Opus 5.5 --- api/utils/page.ts | 29 +++++++++++++++++++++++------ api/utils/report.ts | 18 +++++++++++------- test-it/inspect.ts | 3 ++- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/api/utils/page.ts b/api/utils/page.ts index 6da7cec..6e4ae35 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -82,6 +82,15 @@ const debugPage = (page: Page) => { }) } +// the host rule of onlySameHost, shared by the frames check and the report collector +export const isSameHost = (url: string, target: string) => { + try { + return new URL(url).host === new URL(target).host + } catch (err) { + return false + } +} + type OpenPageResult = { page: Page, animationActivated: boolean, @@ -102,6 +111,18 @@ async function openInPage ( await setPageLocale(page, lang || config.defaultLang, timezone || config.defaultTimezone) if (cookies) await page.browserContext().setCookie(...cookies) if (viewport) await page.setViewport(viewport) + if (reportFrames && config.onlySameHost) { + // cross-host frames are reported by the collector, but their content must not be loaded + await page.setRequestInterception(true) + page.on('request', request => { + if (request.isInterceptResolutionHandled()) return + if (request.isNavigationRequest() && request.frame() !== page.mainFrame() && !isSameHost(request.url(), target)) { + request.abort('blockedbyclient').catch(() => {}) + } else { + request.continue().catch(() => {}) + } + }) + } timer.step('configure-page') const { animationActivated, ready } = await waitForPage(page, target, animate, timer) @@ -110,12 +131,8 @@ async function openInPage ( if (!frameUrl) continue // the frames are recorded by the report collector instead of failing the request if (reportFrames && !/^https?:/.test(frameUrl)) continue - let sameHost = false - try { - sameHost = new URL(frameUrl).host === new URL(target).host - } catch (err) { - throw httpError(400, 'IFrame with invalid URL :' + frameUrl) - } + if (!URL.canParse(frameUrl)) throw httpError(400, 'IFrame with invalid URL :' + frameUrl) + const sameHost = isSameHost(frameUrl, target) if (!sameHost && config.onlySameHost && !reportFrames) { debug(`${frameUrl} from iframe in ${target} is NOT on same host, reject`) throw httpError(400, 'IFrame did not have same host :' + new URL(frameUrl).host) diff --git a/api/utils/report.ts b/api/utils/report.ts index dc84207..740a7eb 100644 --- a/api/utils/report.ts +++ b/api/utils/report.ts @@ -1,5 +1,7 @@ import type { Page } from 'puppeteer' +import config from '#config' import { ariaSnapshot } from './aria-snapshot.ts' +import { isSameHost } from './page.ts' export type Ready = 'triggered' | 'idle' | 'timeout' export type Verdict = 'ok' | 'errors' | 'timeout' | 'unauthenticated' @@ -28,7 +30,7 @@ const MAX_BODY = 300 const SNAPSHOT_MAX_CHARS = 20000 const SNAPSHOT_MAX_LINKS = 200 -const cut = (str: string, max = MAX_STRING) => str.length > max ? str.slice(0, max) + '…' : str +const cut = (str: string, max = MAX_STRING) => str.length > max ? str.slice(0, max - 1) + '…' : str const withoutQuery = (url: string) => { const parsed = new URL(url) @@ -69,6 +71,14 @@ export const createReportCollector = (page: Page, target: string) => { consoleWarnings++ } }) + // cross-host frames are aborted by openInPage (reportFrames), only their origin is kept + const blockedFrames: string[] = [] + page.on('request', request => { + if (!config.onlySameHost || !request.isNavigationRequest() || request.frame() === page.mainFrame()) return + if (!/^https?:/.test(request.url()) || isSameHost(request.url(), target)) return + const origin = parseOrigin(request.url()) + if (origin && !blockedFrames.includes(origin)) blockedFrames.push(origin) + }) page.on('pageerror', err => { pageErrors.push(err instanceof Error ? (err.stack || err.message) : String(err)) }) @@ -117,12 +127,6 @@ export const createReportCollector = (page: Page, target: string) => { page.title().catch(() => ''), ariaSnapshot(page, { maxChars: SNAPSHOT_MAX_CHARS, maxLinks: SNAPSHOT_MAX_LINKS }) ]) - const blockedFrames: string[] = [] - for (const frame of page.frames()) { - const origin = /^https?:/.test(frame.url()) ? parseOrigin(frame.url()) : undefined - if (origin && origin !== targetOrigin && !blockedFrames.includes(origin)) blockedFrames.push(origin) - } - let verdict: Verdict if (new URL(finalUrl, target).pathname.startsWith('/simple-directory/login')) verdict = 'unauthenticated' else if (ready === 'timeout') verdict = 'timeout' diff --git a/test-it/inspect.ts b/test-it/inspect.ts index 248e992..8d4ff3b 100644 --- a/test-it/inspect.ts +++ b/test-it/inspect.ts @@ -97,6 +97,7 @@ describe('inspect', () => { const res = await inspect('/test/resources/test-frame.html') assert.equal(res.status, 200) assert.deepEqual(res.data.report.blockedFrames, [`http://127.0.0.1:${config.port}`]) + assert.ok(!res.data.report.snapshot.includes('Test page 1'), 'cross-host frame content must not be in the snapshot') assert.equal(res.data.report.verdict, 'errors') assert.match(res.data.text, /\[BLOCKED\] cross-host frame not rendered/) }) @@ -143,7 +144,7 @@ describe('inspect', () => { assert.equal(res.status, 200) const failed = res.data.report.failedRequests.find((f: any) => f.path === '/long') assert.ok(failed.body.length > 0) - assert.ok(failed.body.length <= 301) + assert.ok(failed.body.length <= 300) } finally { other.close() } From ba712923c62dcb68a217089d8c0b30c82036d10a Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Thu, 5 Nov 2026 09:14:39 +0100 Subject: [PATCH 3/7] feat: cap concurrent inspect requests (inspectConcurrency) Renders behind GET /api/v1/inspect now go through a small semaphore created once from the inspectConcurrency config key (default 1, env INSPECT_CONCURRENCY). Co-Authored-By: Claude Opus 5.5 --- api/routers/capture.ts | 8 +- api/utils/semaphore.ts | 35 +++ config/custom-environment-variables.cjs | 1 + config/default.cjs | 1 + config/type/.type/index.d.ts | 1 + config/type/.type/validate.js | 298 ++++++++++++++---------- config/type/schema.json | 5 + test-it/semaphore.ts | 52 +++++ 8 files changed, 279 insertions(+), 122 deletions(-) create mode 100644 api/utils/semaphore.ts create mode 100644 test-it/semaphore.ts diff --git a/api/routers/capture.ts b/api/routers/capture.ts index b6a6c6b..7e9be41 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -4,6 +4,7 @@ import * as headerFooter from '../utils/header-footer.ts' import * as pageUtils from '../utils/page.ts' import * as animationUtils from '../utils/animation.ts' import { createTimer } from '../utils/timer.ts' +import { createSemaphore } from '../utils/semaphore.ts' import { createReportCollector } from '../utils/report.ts' import { reportText } from '../utils/report-text.ts' import config from '#config' @@ -12,6 +13,9 @@ import type { PaperFormat, PDFOptions } from 'puppeteer' const debug = debugModule('capture') +// /inspect renders are expensive, a small number of them run at the same time +const inspectSemaphore = createSemaphore(config.inspectConcurrency) + export const router = express.Router() async function auth (req: Request, res: Response, next: NextFunction) { @@ -233,7 +237,7 @@ router.get('/inspect', async (req, res) => { const timer = createTimer(req.originalUrl, 'inspect') let collector: ReturnType | undefined - await pageUtils.withPage( + await inspectSemaphore.run(() => pageUtils.withPage( target, lang, timezone, @@ -256,6 +260,6 @@ router.get('/inspect', async (req, res) => { }, true, page => { collector = createReportCollector(page, target) } - ) + )) timer.finish() }) diff --git a/api/utils/semaphore.ts b/api/utils/semaphore.ts new file mode 100644 index 0000000..49bdbcd --- /dev/null +++ b/api/utils/semaphore.ts @@ -0,0 +1,35 @@ +// Limits how many async functions run at the same time, the others wait in FIFO order. +export const createSemaphore = (max: number) => { + let active = 0 + const queue: Array<() => void> = [] + + const acquire = () => new Promise(resolve => { + if (active < max) { + active++ + resolve() + } else { + queue.push(() => { + active++ + resolve() + }) + } + }) + + const release = () => { + active-- + queue.shift()?.() + } + + return { + async run (fn: () => Promise): Promise { + await acquire() + try { + return await fn() + } finally { + release() + } + }, + // number of calls waiting for a free slot + pending: () => queue.length + } +} diff --git a/config/custom-environment-variables.cjs b/config/custom-environment-variables.cjs index 3b4130e..bf82683 100644 --- a/config/custom-environment-variables.cjs +++ b/config/custom-environment-variables.cjs @@ -9,6 +9,7 @@ module.exports = { }, screenshotTimeout: 'SCREENSHOT_TIMEOUT', concurrency: 'CONCURRENCY', + inspectConcurrency: 'INSPECT_CONCURRENCY', defaultLang: 'DEFAULT_LANG', defaultTimezone: 'DEFAULT_TIMEZONE', secretKeys: { diff --git a/config/default.cjs b/config/default.cjs index 858b42b..3de159c 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -8,6 +8,7 @@ module.exports = { }, screenshotTimeout: 20000, concurrency: 5, + inspectConcurrency: 1, // concurrent /inspect renders, each one holds a browser page defaultLang: 'fr-FR', defaultTimezone: 'Europe/Paris', secretKeys: { diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index 40ef82d..990d422 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -21,6 +21,7 @@ export type ApiConfig = { [k: string]: unknown; }; concurrency: number; + inspectConcurrency: number; defaultLang: string; defaultTimezone: string; onlySameHost?: boolean; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index 0737b60..67deb80 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -73,8 +73,8 @@ vErrors.push(err5); } errors++; } -if(data.defaultLang === undefined){ -const err6 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; +if(data.inspectConcurrency === undefined){ +const err6 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "inspectConcurrency"},message:"must have required property '"+"inspectConcurrency"+"'"}; if(vErrors === null){ vErrors = [err6]; } @@ -83,8 +83,8 @@ vErrors.push(err6); } errors++; } -if(data.defaultTimezone === undefined){ -const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; +if(data.defaultLang === undefined){ +const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; if(vErrors === null){ vErrors = [err7]; } @@ -93,8 +93,8 @@ vErrors.push(err7); } errors++; } -if(data.screenshotTimeout === undefined){ -const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; +if(data.defaultTimezone === undefined){ +const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; if(vErrors === null){ vErrors = [err8]; } @@ -103,8 +103,8 @@ vErrors.push(err8); } errors++; } -if(data.maxAnimationFrames === undefined){ -const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(data.screenshotTimeout === undefined){ +const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; if(vErrors === null){ vErrors = [err9]; } @@ -113,9 +113,8 @@ vErrors.push(err9); } errors++; } -for(const key0 in data){ -if(!(func2.call(schema16.properties, key0))){ -const err10 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(data.maxAnimationFrames === undefined){ +const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; if(vErrors === null){ vErrors = [err10]; } @@ -124,6 +123,17 @@ vErrors.push(err10); } errors++; } +for(const key0 in data){ +if(!(func2.call(schema16.properties, key0))){ +const err11 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err11]; +} +else { +vErrors.push(err11); +} +errors++; +} } if(data.port !== undefined){ let data0 = data.port; @@ -143,12 +153,12 @@ if(dataType0 == "boolean" || data0 === null coerced0 = +data0; } else { -const err11 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err12 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err11]; +vErrors = [err12]; } else { -vErrors.push(err11); +vErrors.push(err12); } errors++; } @@ -181,12 +191,12 @@ else if(data1 === null){ coerced1 = ""; } else { -const err12 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err13 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err12]; +vErrors = [err13]; } else { -vErrors.push(err12); +vErrors.push(err13); } errors++; } @@ -219,12 +229,12 @@ else if(data2 === null){ coerced2 = ""; } else { -const err13 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err14 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err13]; +vErrors = [err14]; } else { -vErrors.push(err13); +vErrors.push(err14); } errors++; } @@ -238,12 +248,12 @@ data["privateDirectoryUrl"] = coerced2; } if(typeof data2 === "string"){ if(!pattern0.test(data2)){ -const err14 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; +const err15 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; if(vErrors === null){ -vErrors = [err14]; +vErrors = [err15]; } else { -vErrors.push(err14); +vErrors.push(err15); } errors++; } @@ -253,23 +263,23 @@ if(data.helmet !== undefined){ let data3 = data.helmet; if(data3 && typeof data3 == "object" && !Array.isArray(data3)){ if(data3.active === undefined){ -const err15 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; +const err16 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; if(vErrors === null){ -vErrors = [err15]; +vErrors = [err16]; } else { -vErrors.push(err15); +vErrors.push(err16); } errors++; } for(const key1 in data3){ if(!(key1 === "active")){ -const err16 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; +const err17 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err16]; +vErrors = [err17]; } else { -vErrors.push(err16); +vErrors.push(err17); } errors++; } @@ -294,12 +304,12 @@ else if(data4 === "true" || data4 === 1){ coerced3 = true; } else { -const err17 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err18 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err17]; +vErrors = [err18]; } else { -vErrors.push(err17); +vErrors.push(err18); } errors++; } @@ -314,12 +324,12 @@ data3["active"] = coerced3; } } else { -const err18 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err18]; +vErrors = [err19]; } else { -vErrors.push(err18); +vErrors.push(err19); } errors++; } @@ -328,23 +338,23 @@ if(data.secretKeys !== undefined){ let data5 = data.secretKeys; if(data5 && typeof data5 == "object" && !Array.isArray(data5)){ if(data5.capture === undefined){ -const err19 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; +const err20 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; if(vErrors === null){ -vErrors = [err19]; +vErrors = [err20]; } else { -vErrors.push(err19); +vErrors.push(err20); } errors++; } for(const key2 in data5){ if(!(key2 === "capture")){ -const err20 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; +const err21 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err20]; +vErrors = [err21]; } else { -vErrors.push(err20); +vErrors.push(err21); } errors++; } @@ -369,12 +379,12 @@ else if(data6 === null){ coerced4 = ""; } else { -const err21 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err22 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err21]; +vErrors = [err22]; } else { -vErrors.push(err21); +vErrors.push(err22); } errors++; } @@ -389,12 +399,12 @@ data5["capture"] = coerced4; } } else { -const err22 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err23 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err22]; +vErrors = [err23]; } else { -vErrors.push(err22); +vErrors.push(err23); } errors++; } @@ -422,12 +432,12 @@ else if(data8 === "true" || data8 === 1){ coerced5 = true; } else { -const err23 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err24 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err23]; +vErrors = [err24]; } else { -vErrors.push(err23); +vErrors.push(err24); } errors++; } @@ -458,12 +468,12 @@ if(dataType6 == "boolean" || data9 === null coerced6 = +data9; } else { -const err24 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err25 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err24]; +vErrors = [err25]; } else { -vErrors.push(err24); +vErrors.push(err25); } errors++; } @@ -478,12 +488,12 @@ data7["port"] = coerced6; } } else { -const err25 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err26 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err25]; +vErrors = [err26]; } else { -vErrors.push(err25); +vErrors.push(err26); } errors++; } @@ -491,12 +501,12 @@ errors++; if(data.puppeteerLaunchOptions !== undefined){ let data10 = data.puppeteerLaunchOptions; if(!(data10 && typeof data10 == "object" && !Array.isArray(data10))){ -const err26 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err27 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err26]; +vErrors = [err27]; } else { -vErrors.push(err26); +vErrors.push(err27); } errors++; } @@ -519,12 +529,12 @@ if(dataType7 === "boolean" || data11 === null coerced7 = +data11; } else { -const err27 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err28 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err27]; +vErrors = [err28]; } else { -vErrors.push(err27); +vErrors.push(err28); } errors++; } @@ -537,32 +547,30 @@ data["concurrency"] = coerced7; } } } -if(data.defaultLang !== undefined){ -let data12 = data.defaultLang; -if(typeof data12 !== "string"){ +if(data.inspectConcurrency !== undefined){ +let data12 = data.inspectConcurrency; +if(!((typeof data12 == "number") && (!(data12 % 1) && !isNaN(data12)))){ let dataType8 = typeof data12; let coerced8 = undefined; if(dataType8 == 'object' && Array.isArray(data12) && data12.length == 1){ data12 = data12[0]; dataType8 = typeof data12; -if(typeof data12 === "string"){ +if((typeof data12 == "number") && (!(data12 % 1) && !isNaN(data12))){ coerced8 = data12; } } if(!(coerced8 !== undefined)){ -if(dataType8 == "number" || dataType8 == "boolean"){ -coerced8 = "" + data12; -} -else if(data12 === null){ -coerced8 = ""; +if(dataType8 === "boolean" || data12 === null + || (dataType8 === "string" && data12 && data12 == +data12 && !(data12 % 1))){ +coerced8 = +data12; } else { -const err28 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err29 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err28]; +vErrors = [err29]; } else { -vErrors.push(err28); +vErrors.push(err29); } errors++; } @@ -570,13 +578,25 @@ errors++; if(coerced8 !== undefined){ data12 = coerced8; if(data !== undefined){ -data["defaultLang"] = coerced8; +data["inspectConcurrency"] = coerced8; +} } } +if(typeof data12 == "number"){ +if(data12 < 1 || isNaN(data12)){ +const err30 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err30]; } +else { +vErrors.push(err30); } -if(data.defaultTimezone !== undefined){ -let data13 = data.defaultTimezone; +errors++; +} +} +} +if(data.defaultLang !== undefined){ +let data13 = data.defaultLang; if(typeof data13 !== "string"){ let dataType9 = typeof data13; let coerced9 = undefined; @@ -595,12 +615,12 @@ else if(data13 === null){ coerced9 = ""; } else { -const err29 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err31 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err29]; +vErrors = [err31]; } else { -vErrors.push(err29); +vErrors.push(err31); } errors++; } @@ -608,37 +628,37 @@ errors++; if(coerced9 !== undefined){ data13 = coerced9; if(data !== undefined){ -data["defaultTimezone"] = coerced9; +data["defaultLang"] = coerced9; } } } } -if(data.onlySameHost !== undefined){ -let data14 = data.onlySameHost; -if(typeof data14 !== "boolean"){ +if(data.defaultTimezone !== undefined){ +let data14 = data.defaultTimezone; +if(typeof data14 !== "string"){ let dataType10 = typeof data14; let coerced10 = undefined; if(dataType10 == 'object' && Array.isArray(data14) && data14.length == 1){ data14 = data14[0]; dataType10 = typeof data14; -if(typeof data14 === "boolean"){ +if(typeof data14 === "string"){ coerced10 = data14; } } if(!(coerced10 !== undefined)){ -if(data14 === "false" || data14 === 0 || data14 === null){ -coerced10 = false; +if(dataType10 == "number" || dataType10 == "boolean"){ +coerced10 = "" + data14; } -else if(data14 === "true" || data14 === 1){ -coerced10 = true; +else if(data14 === null){ +coerced10 = ""; } else { -const err30 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err32 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err30]; +vErrors = [err32]; } else { -vErrors.push(err30); +vErrors.push(err32); } errors++; } @@ -646,13 +666,13 @@ errors++; if(coerced10 !== undefined){ data14 = coerced10; if(data !== undefined){ -data["onlySameHost"] = coerced10; +data["defaultTimezone"] = coerced10; } } } } -if(data.useHostHeader !== undefined){ -let data15 = data.useHostHeader; +if(data.onlySameHost !== undefined){ +let data15 = data.onlySameHost; if(typeof data15 !== "boolean"){ let dataType11 = typeof data15; let coerced11 = undefined; @@ -671,12 +691,12 @@ else if(data15 === "true" || data15 === 1){ coerced11 = true; } else { -const err31 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err33 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err33]; } else { -vErrors.push(err31); +vErrors.push(err33); } errors++; } @@ -684,35 +704,37 @@ errors++; if(coerced11 !== undefined){ data15 = coerced11; if(data !== undefined){ -data["useHostHeader"] = coerced11; +data["onlySameHost"] = coerced11; } } } } -if(data.screenshotTimeout !== undefined){ -let data16 = data.screenshotTimeout; -if(!(typeof data16 == "number")){ +if(data.useHostHeader !== undefined){ +let data16 = data.useHostHeader; +if(typeof data16 !== "boolean"){ let dataType12 = typeof data16; let coerced12 = undefined; if(dataType12 == 'object' && Array.isArray(data16) && data16.length == 1){ data16 = data16[0]; dataType12 = typeof data16; -if(typeof data16 == "number"){ +if(typeof data16 === "boolean"){ coerced12 = data16; } } if(!(coerced12 !== undefined)){ -if(dataType12 == "boolean" || data16 === null - || (dataType12 == "string" && data16 && data16 == +data16)){ -coerced12 = +data16; +if(data16 === "false" || data16 === 0 || data16 === null){ +coerced12 = false; +} +else if(data16 === "true" || data16 === 1){ +coerced12 = true; } else { -const err32 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err34 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err32]; +vErrors = [err34]; } else { -vErrors.push(err32); +vErrors.push(err34); } errors++; } @@ -720,13 +742,13 @@ errors++; if(coerced12 !== undefined){ data16 = coerced12; if(data !== undefined){ -data["screenshotTimeout"] = coerced12; +data["useHostHeader"] = coerced12; } } } } -if(data.maxAnimationFrames !== undefined){ -let data17 = data.maxAnimationFrames; +if(data.screenshotTimeout !== undefined){ +let data17 = data.screenshotTimeout; if(!(typeof data17 == "number")){ let dataType13 = typeof data17; let coerced13 = undefined; @@ -743,12 +765,12 @@ if(dataType13 == "boolean" || data17 === null coerced13 = +data17; } else { -const err33 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err35 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err33]; +vErrors = [err35]; } else { -vErrors.push(err33); +vErrors.push(err35); } errors++; } @@ -756,19 +778,55 @@ errors++; if(coerced13 !== undefined){ data17 = coerced13; if(data !== undefined){ -data["maxAnimationFrames"] = coerced13; +data["screenshotTimeout"] = coerced13; } } } } +if(data.maxAnimationFrames !== undefined){ +let data18 = data.maxAnimationFrames; +if(!(typeof data18 == "number")){ +let dataType14 = typeof data18; +let coerced14 = undefined; +if(dataType14 == 'object' && Array.isArray(data18) && data18.length == 1){ +data18 = data18[0]; +dataType14 = typeof data18; +if(typeof data18 == "number"){ +coerced14 = data18; +} +} +if(!(coerced14 !== undefined)){ +if(dataType14 == "boolean" || data18 === null + || (dataType14 == "string" && data18 && data18 == +data18)){ +coerced14 = +data18; } else { -const err34 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err36 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err34]; +vErrors = [err36]; } else { -vErrors.push(err34); +vErrors.push(err36); +} +errors++; +} +} +if(coerced14 !== undefined){ +data18 = coerced14; +if(data !== undefined){ +data["maxAnimationFrames"] = coerced14; +} +} +} +} +} +else { +const err37 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err37]; +} +else { +vErrors.push(err37); } errors++; } diff --git a/config/type/schema.json b/config/type/schema.json index 74d1abd..6bb6c65 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -17,6 +17,7 @@ "observer", "puppeteerLaunchOptions", "concurrency", + "inspectConcurrency", "defaultLang", "defaultTimezone", "screenshotTimeout", @@ -70,6 +71,10 @@ "concurrency": { "type": "integer" }, + "inspectConcurrency": { + "type": "integer", + "minimum": 1 + }, "defaultLang": { "type": "string" }, diff --git a/test-it/semaphore.ts b/test-it/semaphore.ts new file mode 100644 index 0000000..2a4e28b --- /dev/null +++ b/test-it/semaphore.ts @@ -0,0 +1,52 @@ +import assert from 'assert/strict' +import { describe, it } from 'node:test' +import { createSemaphore } from '../api/utils/semaphore.ts' + +const deferred = () => { + let resolve!: (value: T) => void + let reject!: (err: any) => void + const promise = new Promise((_resolve, _reject) => { resolve = _resolve; reject = _reject }) + return { promise, resolve, reject } +} + +// let pending microtasks and already-queued continuations run +const tick = () => new Promise(resolve => setImmediate(resolve)) + +describe('semaphore', () => { + it('does not start the second fn before the first settles when max is 1', async () => { + const sem = createSemaphore(1) + const first = deferred() + const second = deferred() + let secondStarted = false + + const p1 = sem.run(() => first.promise) + const p2 = sem.run(async () => { + secondStarted = true + return second.promise + }) + await tick() + assert.equal(secondStarted, false) + assert.equal(sem.pending(), 1) + + first.resolve('a') + assert.equal(await p1, 'a') + await tick() + assert.equal(secondStarted, true) + assert.equal(sem.pending(), 0) + + second.resolve('b') + assert.equal(await p2, 'b') + }) + + it('releases the slot when the first fn rejects', async () => { + const sem = createSemaphore(1) + const failing = deferred() + const p1 = sem.run(() => failing.promise) + const p2 = sem.run(async () => 'next') + + failing.reject(new Error('boom')) + await assert.rejects(p1, /boom/) + assert.equal(await p2, 'next') + assert.equal(sem.pending(), 0) + }) +}) From eb0be4864afd8591c6834bb03ff08eaae37edf62 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Thu, 5 Nov 2026 17:23:11 +0100 Subject: [PATCH 4/7] fix(inspect): quieter verdicts, abandoned renders, bounded text Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +++++ api/routers/capture.ts | 22 ++++++++------- api/utils/aria-snapshot.ts | 7 +++-- api/utils/report-text.ts | 35 ++++++++++++++++++------ api/utils/report.ts | 7 ++++- test-it/inspect.ts | 36 +++++++++++++++++++++++++ test-it/resources/test-clean.html | 10 +++++++ test-it/resources/test-long-text.html | 15 +++++++++++ test-it/resources/test-many-errors.html | 13 +++++++++ test-it/resources/test-other-host.html | 12 +++++++++ 10 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 test-it/resources/test-clean.html create mode 100644 test-it/resources/test-long-text.html create mode 100644 test-it/resources/test-many-errors.html create mode 100644 test-it/resources/test-other-host.html diff --git a/README.md b/README.md index 2935c13..26f3c14 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,12 @@ Check the service with these examples (in the dev environment, replace 5607 with - [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=capture&target=http://localhost:5607/test/resources/test-errors.html) - [remote url with a webgl based map](http://localhost:5617/api/v1/print?key=capture&target=https://staging-koumoul.com/cadastre/parcel?id=56251000AV0142) +## Configuration + +Configuration is in `config/` and can be overridden with environment variables, see `config/custom-environment-variables.cjs`. Notably: + + - `INSPECT_CONCURRENCY` (`inspectConcurrency`, default 1): how many `/inspect` renders may run at the same time, each one holds a browser page; further requests wait for a free slot. + ## Security Consider reading this article https://github.com/Zenika/alpine-chrome#3-ways-to-securely-use-chrome-headless-with-this-image diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 7e9be41..54eb3b5 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -237,14 +237,17 @@ router.get('/inspect', async (req, res) => { const timer = createTimer(req.originalUrl, 'inspect') let collector: ReturnType | undefined - await inspectSemaphore.run(() => pageUtils.withPage( - target, - lang, - timezone, - cookies, - viewport, - false, - timer, + await inspectSemaphore.run(async () => { + // the client may have left while waiting for a slot, no need to render for nobody + if (req.socket.destroyed || res.destroyed) return + await pageUtils.withPage( + target, + lang, + timezone, + cookies, + viewport, + false, + timer, `Failed to inspect page "${target}" before timeout`, async ({ page, ready }) => { const report = await collector!.finish(ready) @@ -260,6 +263,7 @@ router.get('/inspect', async (req, res) => { }, true, page => { collector = createReportCollector(page, target) } - )) + ) + }) timer.finish() }) diff --git a/api/utils/aria-snapshot.ts b/api/utils/aria-snapshot.ts index 30e1ac3..a3f3c88 100644 --- a/api/utils/aria-snapshot.ts +++ b/api/utils/aria-snapshot.ts @@ -110,7 +110,10 @@ const createKey = (node: SerializedAXNode) => { } const isText = (node: SerializedAXNode) => node.role === 'StaticText' -const text = (node: SerializedAXNode) => (node.name ?? '').replace(/\s+/g, ' ').trim() +const MAX_TEXT = 2000 +// one huge value must not use the whole snapshot budget +const capText = (str: string) => str.length > MAX_TEXT ? str.slice(0, MAX_TEXT - 1) + '…' : str +const text = (node: SerializedAXNode) => capText((node.name ?? '').replace(/\s+/g, ' ').trim()) // interestingOnly would drop useful containers (paragraph, list, listitem), so the whole tree is // requested and its structural noise is flattened here: children of unnamed none/generic nodes and @@ -157,7 +160,7 @@ export const ariaSnapshot = async (page: Page, opts: { maxChars: number, maxLink } let inlineText: string | undefined - if ((node.role === 'textbox' || node.role === 'combobox') && node.value !== undefined && node.value !== '') inlineText = String(node.value) + if ((node.role === 'textbox' || node.role === 'combobox') && node.value !== undefined && node.value !== '') inlineText = capText(String(node.value)) else if (!text(node) && children.length === 1 && isText(children[0]) && !href) { inlineText = text(children[0]) children = [] diff --git a/api/utils/report-text.ts b/api/utils/report-text.ts index edad25a..429e969 100644 --- a/api/utils/report-text.ts +++ b/api/utils/report-text.ts @@ -1,4 +1,7 @@ import type { RenderReport } from './report.ts' +import { isSameHost } from './page.ts' + +const DETAILS_MAX_CHARS = 8000 // layout of Playwright MCP responses (### Page, ### Console, ### Network, ### Snapshot), // plus the Ready, Verdict, [BLOCKED] and body lines that are specific to this tool @@ -14,34 +17,50 @@ export const reportText = (report: RenderReport): string => { lines.push(`- Console: ${errors} errors, ${report.consoleWarnings} warnings`) } lines.push(`- Ready: ${report.ready === 'timeout' ? `timeout after ${report.durationMs} ms` : report.ready}`) + if (!isSameHost(report.finalUrl, report.url)) { + lines.push(`- Note: navigated away from ${new URL(report.url).host} to ${new URL(report.finalUrl).host}`) + } lines.push(`- Verdict: ${report.verdict}`) + // Console and Network together are bounded, the snapshot has its own budget + const details: string[] = [] if (report.consoleErrors.length || report.pageErrors.length) { - lines.push('### Console') + details.push('### Console') for (const entry of report.consoleErrors) { - lines.push(`[ERROR] ${entry.text}` + (entry.url ? ` @ ${entry.url}${entry.line !== undefined ? ':' + entry.line : ''}` : '')) + details.push(`[ERROR] ${entry.text}` + (entry.url ? ` @ ${entry.url}${entry.line !== undefined ? ':' + entry.line : ''}` : '')) } - for (const error of report.pageErrors) lines.push(error) + for (const error of report.pageErrors) details.push(error) } if (report.failedRequests.length || report.blockedFrames.length) { - lines.push('### Network') + details.push('### Network') let i = 0 for (const request of report.failedRequests) { i++ if (request.status !== undefined) { - lines.push(`${i}. [${request.method}] ${request.url} => [${request.status}] ${request.statusText ?? ''}`.trimEnd()) - if (request.body) lines.push(` ${request.body}`) + details.push(`${i}. [${request.method}] ${request.url} => [${request.status}] ${request.statusText ?? ''}`.trimEnd()) + if (request.body) details.push(` ${request.body}`) } else { - lines.push(`${i}. [${request.method}] ${request.url} => [FAILED] ${request.errorText ?? ''}`.trimEnd()) + details.push(`${i}. [${request.method}] ${request.url} => [FAILED] ${request.errorText ?? ''}`.trimEnd()) } } for (const origin of report.blockedFrames) { i++ - lines.push(`${i}. [GET] ${origin} => [BLOCKED] cross-host frame not rendered`) + details.push(`${i}. [GET] ${origin} => [BLOCKED] cross-host frame not rendered`) } } + // truncate at a line boundary, keeping room for the last line + let size = 0 + let kept = 0 + for (const line of details) { + if (size + line.length + 1 > DETAILS_MAX_CHARS - 50) break + size += line.length + 1 + kept++ + } + lines.push(...details.slice(0, kept)) + if (kept < details.length) lines.push(`# console and network truncated: ${details.length - kept} more lines`) + lines.push('### Snapshot', '```yaml', report.snapshot, '```') return lines.join('\n') } diff --git a/api/utils/report.ts b/api/utils/report.ts index 740a7eb..39820f4 100644 --- a/api/utils/report.ts +++ b/api/utils/report.ts @@ -66,6 +66,8 @@ export const createReportCollector = (page: Page, target: string) => { page.on('console', msg => { if (msg.type() === 'error') { const { url, lineNumber } = msg.location() + // Chrome logs failed resources on its own, same-origin ones are in the Network section + if (/^Failed to load resource/.test(msg.text()) && url && (isFavicon(url) || sameOrigin(url))) return consoleErrors.push({ text: msg.text(), ...(url ? { url } : {}), ...(url && lineNumber !== undefined ? { line: lineNumber } : {}) }) } else if (msg.type() === 'warn') { consoleWarnings++ @@ -84,6 +86,8 @@ export const createReportCollector = (page: Page, target: string) => { }) page.on('requestfailed', request => { if (!sameOrigin(request.url()) || isFavicon(request.url())) return + // apps cancel superseded fetches + if (request.failure()?.errorText === 'net::ERR_ABORTED') return const url = withoutQuery(request.url()) failedRequests.push({ method: request.method(), url: url.href, path: url.pathname, errorText: request.failure()?.errorText }) }) @@ -127,10 +131,11 @@ export const createReportCollector = (page: Page, target: string) => { page.title().catch(() => ''), ariaSnapshot(page, { maxChars: SNAPSHOT_MAX_CHARS, maxLinks: SNAPSHOT_MAX_LINKS }) ]) + const movedAway = !isSameHost(finalUrl, target) let verdict: Verdict if (new URL(finalUrl, target).pathname.startsWith('/simple-directory/login')) verdict = 'unauthenticated' else if (ready === 'timeout') verdict = 'timeout' - else if (consoleErrors.length || pageErrors.length || failedRequests.length || blockedFrames.length || (status !== undefined && status >= 400)) verdict = 'errors' + else if (movedAway || consoleErrors.length || pageErrors.length || failedRequests.length || blockedFrames.length || (status !== undefined && status >= 400)) verdict = 'errors' else verdict = 'ok' return { diff --git a/test-it/inspect.ts b/test-it/inspect.ts index 8d4ff3b..8de949c 100644 --- a/test-it/inspect.ts +++ b/test-it/inspect.ts @@ -102,6 +102,42 @@ describe('inspect', () => { assert.match(res.data.text, /\[BLOCKED\] cross-host frame not rendered/) }) + it('a healthy page is ok, without console noise', async () => { + const res = await inspect('/test/resources/test-clean.html') + assert.equal(res.status, 200) + assert.equal(res.data.report.verdict, 'ok', res.data.text) + assert.deepEqual(res.data.report.consoleErrors, []) + }) + + it('the main frame leaving the target host is an error', async () => { + const res = await inspect('/test/resources/test-other-host.html') + assert.equal(res.status, 200) + assert.equal(new URL(res.data.report.finalUrl).hostname, '127.0.0.1') + assert.equal(res.data.report.verdict, 'errors') + assert.match(res.data.text, new RegExp(`- Note: navigated away from localhost:${config.port} to 127\\.0\\.0\\.1:${config.port}\n`)) + }) + + it('one huge text does not empty the snapshot', async () => { + const res = await inspect('/test/resources/test-long-text.html') + assert.equal(res.status, 200) + const snapshot: string = res.data.report.snapshot + assert.ok(snapshot.includes('Head of the page')) + assert.ok(snapshot.includes('After the long text')) + assert.ok(snapshot.length < 5000) + assert.ok(snapshot.includes('…')) + assert.ok(!snapshot.includes('# snapshot truncated')) + }) + + it('bounds the console and network sections', async () => { + const res = await inspect('/test/resources/test-many-errors.html') + assert.equal(res.status, 200) + const text: string = res.data.text + const consoleSection = text.slice(text.indexOf('### Console'), text.indexOf('### Snapshot')) + assert.ok(consoleSection.length <= 8000, `console section is ${consoleSection.length} chars`) + assert.match(consoleSection, /# console and network truncated: \d+ more lines\n$/) + assert.ok(text.length < 28000) + }) + it('full page is clipped to 3000px', async () => { const res = await inspect('/test/resources/test-big.html', { fullPage: 'true' }) assert.equal(res.status, 200) diff --git a/test-it/resources/test-clean.html b/test-it/resources/test-clean.html new file mode 100644 index 0000000..c7b6158 --- /dev/null +++ b/test-it/resources/test-clean.html @@ -0,0 +1,10 @@ + + + + + Clean page + + +

    Clean page

    + + diff --git a/test-it/resources/test-long-text.html b/test-it/resources/test-long-text.html new file mode 100644 index 0000000..4fe35a0 --- /dev/null +++ b/test-it/resources/test-long-text.html @@ -0,0 +1,15 @@ + + + + + Long text + + +

    Head of the page

    +

    +

    After the long text

    + + + diff --git a/test-it/resources/test-many-errors.html b/test-it/resources/test-many-errors.html new file mode 100644 index 0000000..48e6b26 --- /dev/null +++ b/test-it/resources/test-many-errors.html @@ -0,0 +1,13 @@ + + + + + Many errors + + +

    Many errors

    + + + diff --git a/test-it/resources/test-other-host.html b/test-it/resources/test-other-host.html new file mode 100644 index 0000000..3d7164c --- /dev/null +++ b/test-it/resources/test-other-host.html @@ -0,0 +1,12 @@ + + + + + Other host redirect + + + + + From 9bafd47a3642ddb737a8c3fbd84ff564588f2e1d Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Thu, 5 Nov 2026 17:37:49 +0100 Subject: [PATCH 5/7] style(inspect): indent the queued render Co-Authored-By: Claude Opus 5.5 --- api/routers/capture.ts | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 54eb3b5..2275d82 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -248,21 +248,21 @@ router.get('/inspect', async (req, res) => { viewport, false, timer, - `Failed to inspect page "${target}" before timeout`, - async ({ page, ready }) => { - const report = await collector!.finish(ready) - let buffer - if (fullPage) { - const height = await page.evaluate(() => document.documentElement.scrollHeight) - buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) - } else { - buffer = await page.screenshot() - } - timer.type = 'inspect' - res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) - }, - true, - page => { collector = createReportCollector(page, target) } + `Failed to inspect page "${target}" before timeout`, + async ({ page, ready }) => { + const report = await collector!.finish(ready) + let buffer + if (fullPage) { + const height = await page.evaluate(() => document.documentElement.scrollHeight) + buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) + } else { + buffer = await page.screenshot() + } + timer.type = 'inspect' + res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) + }, + true, + page => { collector = createReportCollector(page, target) } ) }) timer.finish() From df3e2fbb3154eaba97d36204969dffdd119b9da0 Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Fri, 6 Nov 2026 09:21:49 +0100 Subject: [PATCH 6/7] feat(inspect): report errors declared by the page (declaredErrorUrl) POST requests to declaredErrorUrl are answered by capture with 204 and never sent, their message is reported under declaredErrors and makes the verdict errors. Data-fair applications declare their configuration errors this way, a render must not write them. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + api/routers/capture.ts | 14 +++++++-- api/utils/page.ts | 35 +++++++++++++++++----- api/utils/report-text.ts | 5 ++++ api/utils/report.ts | 19 ++++++++++-- test-it/inspect.ts | 22 ++++++++++++++ test-it/resources/test-declared-error.html | 15 ++++++++++ 7 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 test-it/resources/test-declared-error.html diff --git a/README.md b/README.md index 26f3c14..2731e13 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Check the service with these examples (in the dev environment, replace 5607 with - [animated gif screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test-anim.html) - [fallback to standard screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test1.html) - [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=capture&target=http://localhost:5607/test/resources/test-errors.html) + - [inspect with an error declared by the page](http://localhost:5607/api/v1/inspect?key=capture&target=http://localhost:5607/test/resources/test-declared-error.html&declaredErrorUrl=http://localhost:5607/test/app/error), `declaredErrorUrl` (same host as `target`, query ignored) receives the page's POST requests: capture answers them with 204 instead of sending them, and reports their `message` under `declaredErrors` - [remote url with a webgl based map](http://localhost:5617/api/v1/print?key=capture&target=https://staging-koumoul.com/cadastre/parcel?id=56251000AV0142) ## Configuration diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 2275d82..c8786d7 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -233,6 +233,13 @@ router.get('/inspect', async (req, res) => { ? mobileViewport : { width: viewportDimension(req.query.width, desktopViewport.width), height: viewportDimension(req.query.height, desktopViewport.height) } + // the page may POST its own error diagnosis there, it is reported instead of being sent + const declaredErrorUrl = req.query.declaredErrorUrl + if (declaredErrorUrl !== undefined) { + if (typeof declaredErrorUrl !== 'string' || !URL.canParse(declaredErrorUrl)) throw httpError(400, 'invalid declaredErrorUrl') + if (!pageUtils.isSameHost(declaredErrorUrl, target)) throw httpError(400, 'declaredErrorUrl must be on the host of target') + } + const cookies = Object.keys(req.cookies ?? {}).map(name => ({ name, value: req.cookies[name], url: target })) as any const timer = createTimer(req.originalUrl, 'inspect') @@ -261,8 +268,11 @@ router.get('/inspect', async (req, res) => { timer.type = 'inspect' res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) }, - true, - page => { collector = createReportCollector(page, target) } + { + onPage: page => { collector = createReportCollector(page, target) }, + declaredErrorUrl, + onDeclaredError: body => collector?.declareError(body) + } ) }) timer.finish() diff --git a/api/utils/page.ts b/api/utils/page.ts index 6e4ae35..c51eca4 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -91,6 +91,22 @@ export const isSameHost = (url: string, target: string) => { } } +// options of the /inspect renders +export type InspectOptions = { + onPage?: (page: Page) => void, + // POST requests to this URL (query ignored) are answered by capture itself and never reach the + // server, their body is handed to onDeclaredError: this is how an application declares an error + declaredErrorUrl?: string, + onDeclaredError?: (body: string | undefined) => void +} + +const withoutQuery = (url: string) => { + const parsed = new URL(url) + parsed.search = '' + parsed.hash = '' + return parsed.href +} + type OpenPageResult = { page: Page, animationActivated: boolean, @@ -106,17 +122,23 @@ async function openInPage ( viewport: Viewport | undefined, animate: boolean, timer: Timer, - reportFrames: boolean + inspect: InspectOptions | undefined ): Promise { + const reportFrames = !!inspect await setPageLocale(page, lang || config.defaultLang, timezone || config.defaultTimezone) if (cookies) await page.browserContext().setCookie(...cookies) if (viewport) await page.setViewport(viewport) - if (reportFrames && config.onlySameHost) { + const declaredErrorUrl = inspect?.declaredErrorUrl && withoutQuery(inspect.declaredErrorUrl) + if (reportFrames && (config.onlySameHost || declaredErrorUrl)) { // cross-host frames are reported by the collector, but their content must not be loaded await page.setRequestInterception(true) page.on('request', request => { if (request.isInterceptResolutionHandled()) return - if (request.isNavigationRequest() && request.frame() !== page.mainFrame() && !isSameHost(request.url(), target)) { + if (declaredErrorUrl && request.method() === 'POST' && URL.canParse(request.url()) && withoutQuery(request.url()) === declaredErrorUrl) { + // a render must not write anything, the declared error is only reported + inspect?.onDeclaredError?.(request.postData()) + request.respond({ status: 204 }).catch(() => {}) + } else if (config.onlySameHost && request.isNavigationRequest() && request.frame() !== page.mainFrame() && !isSameHost(request.url(), target)) { request.abort('blockedbyclient').catch(() => {}) } else { request.continue().catch(() => {}) @@ -159,8 +181,7 @@ export const withPage = async ( timer: Timer, callbackTimeoutMsg: string, callback: (result: OpenPageResult) => Promise, - reportFrames = false, - onPage?: (page: Page) => void) => { + inspect?: InspectOptions) => { if (target.includes('capture-test-error=true')) { await new Promise(resolve => setTimeout(resolve, 1000)) throw new Error('forced error trigger') @@ -185,11 +206,11 @@ export const withPage = async ( } } debugPage(page) - onPage?.(page) + inspect?.onPage?.(page) timer.step('newPage') const result = await promiseTimeout( - openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, reportFrames), + openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), config.screenshotTimeout * 2, `Failed to open "${target}" in context before timeout` ) diff --git a/api/utils/report-text.ts b/api/utils/report-text.ts index 429e969..f0156a7 100644 --- a/api/utils/report-text.ts +++ b/api/utils/report-text.ts @@ -24,6 +24,11 @@ export const reportText = (report: RenderReport): string => { // Console and Network together are bounded, the snapshot has its own budget const details: string[] = [] + // first, the page's own diagnosis is the most useful line + if (report.declaredErrors.length) { + details.push('### Declared errors') + for (const message of report.declaredErrors) details.push(`- ${message}`) + } if (report.consoleErrors.length || report.pageErrors.length) { details.push('### Console') for (const entry of report.consoleErrors) { diff --git a/api/utils/report.ts b/api/utils/report.ts index 39820f4..2fdda52 100644 --- a/api/utils/report.ts +++ b/api/utils/report.ts @@ -20,6 +20,7 @@ export type RenderReport = { pageErrors: string[], failedRequests: FailedRequest[], blockedFrames: string[], + declaredErrors: string[], snapshot: string, verdict: Verdict } @@ -60,6 +61,19 @@ export const createReportCollector = (page: Page, target: string) => { const pending: Promise[] = [] let status: number | undefined let statusText: string | undefined + const declaredErrors: string[] = [] + + // the page itself says it is broken, cf InspectOptions.declaredErrorUrl + const declareError = (body: string | undefined) => { + let message = body ?? '' + try { + const parsed = JSON.parse(message) + if (parsed && typeof parsed.message === 'string') message = parsed.message + } catch (err) { + // not JSON, the raw body is the message + } + declaredErrors.push(message.trim() || '(no message)') + } const sameOrigin = (url: string) => parseOrigin(url) === targetOrigin @@ -135,7 +149,7 @@ export const createReportCollector = (page: Page, target: string) => { let verdict: Verdict if (new URL(finalUrl, target).pathname.startsWith('/simple-directory/login')) verdict = 'unauthenticated' else if (ready === 'timeout') verdict = 'timeout' - else if (movedAway || consoleErrors.length || pageErrors.length || failedRequests.length || blockedFrames.length || (status !== undefined && status >= 400)) verdict = 'errors' + else if (movedAway || declaredErrors.length || consoleErrors.length || pageErrors.length || failedRequests.length || blockedFrames.length || (status !== undefined && status >= 400)) verdict = 'errors' else verdict = 'ok' return { @@ -151,10 +165,11 @@ export const createReportCollector = (page: Page, target: string) => { pageErrors: pageErrors.slice(0, MAX_ENTRIES).map(e => cut(e)), failedRequests: failedRequests.slice(0, MAX_ENTRIES).map(f => ({ ...f, url: cut(f.url), path: cut(f.path), ...(f.errorText ? { errorText: cut(f.errorText) } : {}) })), blockedFrames: blockedFrames.slice(0, MAX_ENTRIES), + declaredErrors: declaredErrors.slice(0, MAX_ENTRIES).map(e => cut(e)), snapshot, verdict } } - return { finish } + return { finish, declareError } } diff --git a/test-it/inspect.ts b/test-it/inspect.ts index 8de949c..62a28ca 100644 --- a/test-it/inspect.ts +++ b/test-it/inspect.ts @@ -144,6 +144,28 @@ describe('inspect', () => { assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 1280, height: 3000 }) }) + it('reports a declared error without sending it', async () => { + const res = await inspect('/test/resources/test-declared-error.html', { declaredErrorUrl: origin + '/test/app/error' }) + assert.equal(res.status, 200) + assert.deepEqual(res.data.report.declaredErrors, ['dataset column "missing" not found']) + // answered by capture: the 404 the server would give is not there + assert.deepEqual(res.data.report.failedRequests, []) + assert.equal(res.data.report.verdict, 'errors') + assert.match(res.data.text, /### Declared errors\n- dataset column "missing" not found\n/) + }) + + it('without declaredErrorUrl the error post is a plain request', async () => { + const res = await inspect('/test/resources/test-declared-error.html') + assert.equal(res.status, 200) + assert.deepEqual(res.data.report.declaredErrors, []) + assert.deepEqual(res.data.report.failedRequests.map((f: any) => [f.method, f.path, f.status]), [['POST', '/test/app/error', 404]]) + }) + + it('refuses a declaredErrorUrl on another host', async () => { + const res = await inspect('/test/resources/test-declared-error.html', { declaredErrorUrl: 'http://example.com/error' }) + assert.equal(res.status, 400) + }) + it('refuses without the secret', async () => { assert.equal((await inspect('/test/resources/test1.html', {}, {})).status, 401) assert.equal((await inspect('/test/resources/test1.html', {}, { 'x-secret-key': 'wrong' })).status, 401) diff --git a/test-it/resources/test-declared-error.html b/test-it/resources/test-declared-error.html new file mode 100644 index 0000000..2251c71 --- /dev/null +++ b/test-it/resources/test-declared-error.html @@ -0,0 +1,15 @@ + + + + + Declared error page + + +

    Declared error page

    + + + From 609c58f0ff90713d3f24398f39a26e28019b669d Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Fri, 6 Nov 2026 10:12:23 +0100 Subject: [PATCH 7/7] feat(inspect): a secret of its own (SECRET_INSPECT) /inspect checked secretKeys.capture, so enabling it locked /screenshot and /print, that browsers and other services call without a secret. It now requires secretKeys.inspect and is disabled without it. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- api/routers/capture.ts | 4 +- config/custom-environment-variables.cjs | 3 +- config/default.cjs | 3 +- config/development.cjs | 3 +- config/test.cjs | 3 +- config/type/.type/index.d.ts | 1 + config/type/.type/validate.js | 304 +++++++++++++----------- config/type/schema.json | 3 + test-it/inspect.ts | 4 +- 10 files changed, 194 insertions(+), 142 deletions(-) diff --git a/README.md b/README.md index 2731e13..40385a5 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the dev environment. Build and run the docker image: - docker buildx build -t capture --progress plain --load . && docker run --rm -it --security-opt seccomp=$(pwd)/seccomp.json -p 5607:5607 -p 9090:9090 -e DEBUG=capture,timer -e ONLY_SAME_HOST=false -e SECRET_CAPTURE=capture -e HELMET_ACTIVE=false -e PORT=5607 --name capture capture + docker buildx build -t capture --progress plain --load . && docker run --rm -it --security-opt seccomp=$(pwd)/seccomp.json -p 5607:5607 -p 9090:9090 -e DEBUG=capture,timer -e ONLY_SAME_HOST=false -e SECRET_CAPTURE=capture -e SECRET_INSPECT=inspect -e HELMET_ACTIVE=false -e PORT=5607 --name capture capture Check the service with these examples (in the dev environment, replace 5607 with `DEV_API_PORT` from `.env`): @@ -36,14 +36,16 @@ Check the service with these examples (in the dev environment, replace 5607 with - [animated gif screenshot](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&target=http://localhost:5607/test/resources/test-anim.html) - [animated gif screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test-anim.html) - [fallback to standard screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test1.html) - - [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=capture&target=http://localhost:5607/test/resources/test-errors.html) - - [inspect with an error declared by the page](http://localhost:5607/api/v1/inspect?key=capture&target=http://localhost:5607/test/resources/test-declared-error.html&declaredErrorUrl=http://localhost:5607/test/app/error), `declaredErrorUrl` (same host as `target`, query ignored) receives the page's POST requests: capture answers them with 204 instead of sending them, and reports their `message` under `declaredErrors` + - [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=inspect&target=http://localhost:5607/test/resources/test-errors.html) + - [inspect with an error declared by the page](http://localhost:5607/api/v1/inspect?key=inspect&target=http://localhost:5607/test/resources/test-declared-error.html&declaredErrorUrl=http://localhost:5607/test/app/error), `declaredErrorUrl` (same host as `target`, query ignored) receives the page's POST requests: capture answers them with 204 instead of sending them, and reports their `message` under `declaredErrors` - [remote url with a webgl based map](http://localhost:5617/api/v1/print?key=capture&target=https://staging-koumoul.com/cadastre/parcel?id=56251000AV0142) ## Configuration Configuration is in `config/` and can be overridden with environment variables, see `config/custom-environment-variables.cjs`. Notably: + - `SECRET_CAPTURE` (`secretKeys.capture`, default none): when set, `/screenshot` and `/print` require it in `x-secret-key` from internal callers without a session. Leave it unset while browsers or other services call them directly. + - `SECRET_INSPECT` (`secretKeys.inspect`, default none): required in `x-secret-key` on `/inspect` (internal requests only), which is disabled without it. A secret of its own, so that enabling `/inspect` does not lock `/screenshot` and `/print`. - `INSPECT_CONCURRENCY` (`inspectConcurrency`, default 1): how many `/inspect` renders may run at the same time, each one holds a browser page; further requests wait for a free slot. ## Security diff --git a/api/routers/capture.ts b/api/routers/capture.ts index c8786d7..8a57ba2 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -209,7 +209,9 @@ const viewportDimension = (value: unknown, fallback: number) => { // internal only: used by data-fair and portals so that agents can see what they configured router.get('/inspect', async (req, res) => { - assertReqInternalSecret(req, config.secretKeys.capture) + // a secret of its own: setting it does not lock /screenshot and /print, that public clients call + if (!config.secretKeys.inspect) throw httpError(501, 'inspect is disabled, it needs SECRET_INSPECT') + assertReqInternalSecret(req, config.secretKeys.inspect) const target = req.query.target if (typeof target !== 'string') throw httpError(400, 'parameter "target" is required') diff --git a/config/custom-environment-variables.cjs b/config/custom-environment-variables.cjs index bf82683..597219c 100644 --- a/config/custom-environment-variables.cjs +++ b/config/custom-environment-variables.cjs @@ -13,7 +13,8 @@ module.exports = { defaultLang: 'DEFAULT_LANG', defaultTimezone: 'DEFAULT_TIMEZONE', secretKeys: { - capture: 'SECRET_CAPTURE' + capture: 'SECRET_CAPTURE', + inspect: 'SECRET_INSPECT' }, observer: { active: { diff --git a/config/default.cjs b/config/default.cjs index 3de159c..c43bb0a 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -12,7 +12,8 @@ module.exports = { defaultLang: 'fr-FR', defaultTimezone: 'Europe/Paris', secretKeys: { - capture: null + capture: null, // when set, required on /screenshot and /print from callers without a session + inspect: null // required on /inspect, which is disabled without it }, maxAnimationFrames: 1800, // 2 minutes at 15fps puppeteerLaunchOptions: { diff --git a/config/development.cjs b/config/development.cjs index a57618e..f85bfcf 100644 --- a/config/development.cjs +++ b/config/development.cjs @@ -5,7 +5,8 @@ module.exports = { onlySameHost: false, concurrency: 2, secretKeys: { - capture: 'capture' + capture: 'capture', + inspect: 'inspect' }, observer: { active: false diff --git a/config/test.cjs b/config/test.cjs index c67c113..6231010 100644 --- a/config/test.cjs +++ b/config/test.cjs @@ -6,7 +6,8 @@ module.exports = { onlySameHost: true, useHostHeader: true, secretKeys: { - capture: 'capture' + capture: 'capture', + inspect: 'inspect' }, observer: { active: false diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index 990d422..5023df9 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -11,6 +11,7 @@ export type ApiConfig = { }; secretKeys: { capture: string; + inspect?: string; }; observer: { active?: boolean; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index 67deb80..156d33f 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -348,7 +348,7 @@ vErrors.push(err20); errors++; } for(const key2 in data5){ -if(!(key2 === "capture")){ +if(!((key2 === "capture") || (key2 === "inspect"))){ const err21 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ vErrors = [err21]; @@ -397,9 +397,27 @@ data5["capture"] = coerced4; } } } +if(data5.inspect !== undefined){ +let data7 = data5.inspect; +if(typeof data7 !== "string"){ +let dataType5 = typeof data7; +let coerced5 = undefined; +if(dataType5 == 'object' && Array.isArray(data7) && data7.length == 1){ +data7 = data7[0]; +dataType5 = typeof data7; +if(typeof data7 === "string"){ +coerced5 = data7; +} +} +if(!(coerced5 !== undefined)){ +if(dataType5 == "number" || dataType5 == "boolean"){ +coerced5 = "" + data7; +} +else if(data7 === null){ +coerced5 = ""; } else { -const err23 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err23 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ vErrors = [err23]; } @@ -409,30 +427,17 @@ vErrors.push(err23); errors++; } } -if(data.observer !== undefined){ -let data7 = data.observer; -if(data7 && typeof data7 == "object" && !Array.isArray(data7)){ -if(data7.active !== undefined){ -let data8 = data7.active; -if(typeof data8 !== "boolean"){ -let dataType5 = typeof data8; -let coerced5 = undefined; -if(dataType5 == 'object' && Array.isArray(data8) && data8.length == 1){ -data8 = data8[0]; -dataType5 = typeof data8; -if(typeof data8 === "boolean"){ -coerced5 = data8; +if(coerced5 !== undefined){ +data7 = coerced5; +if(data5 !== undefined){ +data5["inspect"] = coerced5; +} } } -if(!(coerced5 !== undefined)){ -if(data8 === "false" || data8 === 0 || data8 === null){ -coerced5 = false; } -else if(data8 === "true" || data8 === 1){ -coerced5 = true; } else { -const err24 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ vErrors = [err24]; } @@ -442,33 +447,30 @@ vErrors.push(err24); errors++; } } -if(coerced5 !== undefined){ -data8 = coerced5; -if(data7 !== undefined){ -data7["active"] = coerced5; -} -} -} -} -if(data7.port !== undefined){ -let data9 = data7.port; -if(!(typeof data9 == "number")){ +if(data.observer !== undefined){ +let data8 = data.observer; +if(data8 && typeof data8 == "object" && !Array.isArray(data8)){ +if(data8.active !== undefined){ +let data9 = data8.active; +if(typeof data9 !== "boolean"){ let dataType6 = typeof data9; let coerced6 = undefined; if(dataType6 == 'object' && Array.isArray(data9) && data9.length == 1){ data9 = data9[0]; dataType6 = typeof data9; -if(typeof data9 == "number"){ +if(typeof data9 === "boolean"){ coerced6 = data9; } } if(!(coerced6 !== undefined)){ -if(dataType6 == "boolean" || data9 === null - || (dataType6 == "string" && data9 && data9 == +data9)){ -coerced6 = +data9; +if(data9 === "false" || data9 === 0 || data9 === null){ +coerced6 = false; +} +else if(data9 === "true" || data9 === 1){ +coerced6 = true; } else { -const err25 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err25 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ vErrors = [err25]; } @@ -480,15 +482,31 @@ errors++; } if(coerced6 !== undefined){ data9 = coerced6; -if(data7 !== undefined){ -data7["port"] = coerced6; +if(data8 !== undefined){ +data8["active"] = coerced6; +} } } } +if(data8.port !== undefined){ +let data10 = data8.port; +if(!(typeof data10 == "number")){ +let dataType7 = typeof data10; +let coerced7 = undefined; +if(dataType7 == 'object' && Array.isArray(data10) && data10.length == 1){ +data10 = data10[0]; +dataType7 = typeof data10; +if(typeof data10 == "number"){ +coerced7 = data10; +} } +if(!(coerced7 !== undefined)){ +if(dataType7 == "boolean" || data10 === null + || (dataType7 == "string" && data10 && data10 == +data10)){ +coerced7 = +data10; } else { -const err26 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err26 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ vErrors = [err26]; } @@ -498,10 +516,17 @@ vErrors.push(err26); errors++; } } -if(data.puppeteerLaunchOptions !== undefined){ -let data10 = data.puppeteerLaunchOptions; -if(!(data10 && typeof data10 == "object" && !Array.isArray(data10))){ -const err27 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(coerced7 !== undefined){ +data10 = coerced7; +if(data8 !== undefined){ +data8["port"] = coerced7; +} +} +} +} +} +else { +const err27 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ vErrors = [err27]; } @@ -511,25 +536,10 @@ vErrors.push(err27); errors++; } } -if(data.concurrency !== undefined){ -let data11 = data.concurrency; -if(!((typeof data11 == "number") && (!(data11 % 1) && !isNaN(data11)))){ -let dataType7 = typeof data11; -let coerced7 = undefined; -if(dataType7 == 'object' && Array.isArray(data11) && data11.length == 1){ -data11 = data11[0]; -dataType7 = typeof data11; -if((typeof data11 == "number") && (!(data11 % 1) && !isNaN(data11))){ -coerced7 = data11; -} -} -if(!(coerced7 !== undefined)){ -if(dataType7 === "boolean" || data11 === null - || (dataType7 === "string" && data11 && data11 == +data11 && !(data11 % 1))){ -coerced7 = +data11; -} -else { -const err28 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +if(data.puppeteerLaunchOptions !== undefined){ +let data11 = data.puppeteerLaunchOptions; +if(!(data11 && typeof data11 == "object" && !Array.isArray(data11))){ +const err28 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ vErrors = [err28]; } @@ -539,16 +549,8 @@ vErrors.push(err28); errors++; } } -if(coerced7 !== undefined){ -data11 = coerced7; -if(data !== undefined){ -data["concurrency"] = coerced7; -} -} -} -} -if(data.inspectConcurrency !== undefined){ -let data12 = data.inspectConcurrency; +if(data.concurrency !== undefined){ +let data12 = data.concurrency; if(!((typeof data12 == "number") && (!(data12 % 1) && !isNaN(data12)))){ let dataType8 = typeof data12; let coerced8 = undefined; @@ -565,7 +567,7 @@ if(dataType8 === "boolean" || data12 === null coerced8 = +data12; } else { -const err29 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err29 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ vErrors = [err29]; } @@ -578,49 +580,35 @@ errors++; if(coerced8 !== undefined){ data12 = coerced8; if(data !== undefined){ -data["inspectConcurrency"] = coerced8; +data["concurrency"] = coerced8; } } } -if(typeof data12 == "number"){ -if(data12 < 1 || isNaN(data12)){ -const err30 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; -if(vErrors === null){ -vErrors = [err30]; } -else { -vErrors.push(err30); -} -errors++; -} -} -} -if(data.defaultLang !== undefined){ -let data13 = data.defaultLang; -if(typeof data13 !== "string"){ +if(data.inspectConcurrency !== undefined){ +let data13 = data.inspectConcurrency; +if(!((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13)))){ let dataType9 = typeof data13; let coerced9 = undefined; if(dataType9 == 'object' && Array.isArray(data13) && data13.length == 1){ data13 = data13[0]; dataType9 = typeof data13; -if(typeof data13 === "string"){ +if((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13))){ coerced9 = data13; } } if(!(coerced9 !== undefined)){ -if(dataType9 == "number" || dataType9 == "boolean"){ -coerced9 = "" + data13; -} -else if(data13 === null){ -coerced9 = ""; +if(dataType9 === "boolean" || data13 === null + || (dataType9 === "string" && data13 && data13 == +data13 && !(data13 % 1))){ +coerced9 = +data13; } else { -const err31 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err30 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err30]; } else { -vErrors.push(err31); +vErrors.push(err30); } errors++; } @@ -628,13 +616,25 @@ errors++; if(coerced9 !== undefined){ data13 = coerced9; if(data !== undefined){ -data["defaultLang"] = coerced9; +data["inspectConcurrency"] = coerced9; } } } +if(typeof data13 == "number"){ +if(data13 < 1 || isNaN(data13)){ +const err31 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err31]; } -if(data.defaultTimezone !== undefined){ -let data14 = data.defaultTimezone; +else { +vErrors.push(err31); +} +errors++; +} +} +} +if(data.defaultLang !== undefined){ +let data14 = data.defaultLang; if(typeof data14 !== "string"){ let dataType10 = typeof data14; let coerced10 = undefined; @@ -653,7 +653,7 @@ else if(data14 === null){ coerced10 = ""; } else { -const err32 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err32 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ vErrors = [err32]; } @@ -666,32 +666,32 @@ errors++; if(coerced10 !== undefined){ data14 = coerced10; if(data !== undefined){ -data["defaultTimezone"] = coerced10; +data["defaultLang"] = coerced10; } } } } -if(data.onlySameHost !== undefined){ -let data15 = data.onlySameHost; -if(typeof data15 !== "boolean"){ +if(data.defaultTimezone !== undefined){ +let data15 = data.defaultTimezone; +if(typeof data15 !== "string"){ let dataType11 = typeof data15; let coerced11 = undefined; if(dataType11 == 'object' && Array.isArray(data15) && data15.length == 1){ data15 = data15[0]; dataType11 = typeof data15; -if(typeof data15 === "boolean"){ +if(typeof data15 === "string"){ coerced11 = data15; } } if(!(coerced11 !== undefined)){ -if(data15 === "false" || data15 === 0 || data15 === null){ -coerced11 = false; +if(dataType11 == "number" || dataType11 == "boolean"){ +coerced11 = "" + data15; } -else if(data15 === "true" || data15 === 1){ -coerced11 = true; +else if(data15 === null){ +coerced11 = ""; } else { -const err33 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err33 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ vErrors = [err33]; } @@ -704,13 +704,13 @@ errors++; if(coerced11 !== undefined){ data15 = coerced11; if(data !== undefined){ -data["onlySameHost"] = coerced11; +data["defaultTimezone"] = coerced11; } } } } -if(data.useHostHeader !== undefined){ -let data16 = data.useHostHeader; +if(data.onlySameHost !== undefined){ +let data16 = data.onlySameHost; if(typeof data16 !== "boolean"){ let dataType12 = typeof data16; let coerced12 = undefined; @@ -729,7 +729,7 @@ else if(data16 === "true" || data16 === 1){ coerced12 = true; } else { -const err34 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err34 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ vErrors = [err34]; } @@ -742,30 +742,32 @@ errors++; if(coerced12 !== undefined){ data16 = coerced12; if(data !== undefined){ -data["useHostHeader"] = coerced12; +data["onlySameHost"] = coerced12; } } } } -if(data.screenshotTimeout !== undefined){ -let data17 = data.screenshotTimeout; -if(!(typeof data17 == "number")){ +if(data.useHostHeader !== undefined){ +let data17 = data.useHostHeader; +if(typeof data17 !== "boolean"){ let dataType13 = typeof data17; let coerced13 = undefined; if(dataType13 == 'object' && Array.isArray(data17) && data17.length == 1){ data17 = data17[0]; dataType13 = typeof data17; -if(typeof data17 == "number"){ +if(typeof data17 === "boolean"){ coerced13 = data17; } } if(!(coerced13 !== undefined)){ -if(dataType13 == "boolean" || data17 === null - || (dataType13 == "string" && data17 && data17 == +data17)){ -coerced13 = +data17; +if(data17 === "false" || data17 === 0 || data17 === null){ +coerced13 = false; +} +else if(data17 === "true" || data17 === 1){ +coerced13 = true; } else { -const err35 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err35 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ vErrors = [err35]; } @@ -778,13 +780,13 @@ errors++; if(coerced13 !== undefined){ data17 = coerced13; if(data !== undefined){ -data["screenshotTimeout"] = coerced13; +data["useHostHeader"] = coerced13; } } } } -if(data.maxAnimationFrames !== undefined){ -let data18 = data.maxAnimationFrames; +if(data.screenshotTimeout !== undefined){ +let data18 = data.screenshotTimeout; if(!(typeof data18 == "number")){ let dataType14 = typeof data18; let coerced14 = undefined; @@ -801,7 +803,7 @@ if(dataType14 == "boolean" || data18 === null coerced14 = +data18; } else { -const err36 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err36 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ vErrors = [err36]; } @@ -814,14 +816,30 @@ errors++; if(coerced14 !== undefined){ data18 = coerced14; if(data !== undefined){ -data["maxAnimationFrames"] = coerced14; +data["screenshotTimeout"] = coerced14; +} +} } } +if(data.maxAnimationFrames !== undefined){ +let data19 = data.maxAnimationFrames; +if(!(typeof data19 == "number")){ +let dataType15 = typeof data19; +let coerced15 = undefined; +if(dataType15 == 'object' && Array.isArray(data19) && data19.length == 1){ +data19 = data19[0]; +dataType15 = typeof data19; +if(typeof data19 == "number"){ +coerced15 = data19; } } +if(!(coerced15 !== undefined)){ +if(dataType15 == "boolean" || data19 === null + || (dataType15 == "string" && data19 && data19 == +data19)){ +coerced15 = +data19; } else { -const err37 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err37 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ vErrors = [err37]; } @@ -830,6 +848,26 @@ vErrors.push(err37); } errors++; } +} +if(coerced15 !== undefined){ +data19 = coerced15; +if(data !== undefined){ +data["maxAnimationFrames"] = coerced15; +} +} +} +} +} +else { +const err38 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err38]; +} +else { +vErrors.push(err38); +} +errors++; +} validate14.errors = vErrors; return errors === 0; } diff --git a/config/type/schema.json b/config/type/schema.json index 6bb6c65..a6e24bc 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -51,6 +51,9 @@ "properties": { "capture": { "type": "string" + }, + "inspect": { + "type": "string" } } }, diff --git a/test-it/inspect.ts b/test-it/inspect.ts index 62a28ca..638570e 100644 --- a/test-it/inspect.ts +++ b/test-it/inspect.ts @@ -15,7 +15,7 @@ const pngSize = async (data: Buffer) => { const origin = `http://localhost:${config.port}` const ax = axios.create({ baseURL: `${origin}/api/v1/`, validateStatus: () => true }) -const key = 'capture' +const key = 'inspect' const inspect = (path: string, params: Record = {}, headers: Record = { 'x-secret-key': key }) => ax.get('inspect', { params: { target: origin + path, ...params }, headers }) @@ -169,6 +169,8 @@ describe('inspect', () => { it('refuses without the secret', async () => { assert.equal((await inspect('/test/resources/test1.html', {}, {})).status, 401) assert.equal((await inspect('/test/resources/test1.html', {}, { 'x-secret-key': 'wrong' })).status, 401) + // the secret of /screenshot and /print does not open /inspect + assert.equal((await inspect('/test/resources/test1.html', {}, { 'x-secret-key': 'capture' })).status, 401) }) it('refuses a forwarded request', async () => {