diff --git a/README.md b/README.md index 3fd1408..51276c9 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the dev environment. Build and run the docker image: - docker buildx build -t capture --progress plain --load . && docker run --rm -it --security-opt seccomp=$(pwd)/seccomp.json -p 5607:5607 -p 9090:9090 -e DEBUG=capture,timer -e ONLY_SAME_HOST=false -e SECRET_CAPTURE=capture -e HELMET_ACTIVE=false -e PORT=5607 --name capture capture + docker buildx build -t capture --progress plain --load . && docker run --rm -it --security-opt seccomp=$(pwd)/seccomp.json -p 5607:5607 -p 9090:9090 -e DEBUG=capture,timer -e ONLY_SAME_HOST=false -e SECRET_CAPTURE=capture -e SECRET_INSPECT=inspect -e HELMET_ACTIVE=false -e PORT=5607 --name capture capture Check the service with these examples (in the dev environment, replace 5607 with `DEV_API_PORT` from `.env`): @@ -36,8 +36,18 @@ Check the service with these examples (in the dev environment, replace 5607 with - [animated gif screenshot](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&target=http://localhost:5607/test/resources/test-anim.html) - [animated gif screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test-anim.html) - [fallback to standard screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test1.html) + - [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=inspect&target=http://localhost:5607/test/resources/test-errors.html) + - [inspect with an error declared by the page](http://localhost:5607/api/v1/inspect?key=inspect&target=http://localhost:5607/test/resources/test-declared-error.html&declaredErrorUrl=http://localhost:5607/test/app/error), `declaredErrorUrl` (same host as `target`, query ignored) receives the page's POST requests: capture answers them with 204 instead of sending them, and reports their `message` under `declaredErrors` - [remote url with a webgl based map](http://localhost:5617/api/v1/print?key=capture&target=https://staging-koumoul.com/cadastre/parcel?id=56251000AV0142) +## Configuration + +Configuration is in `config/` and can be overridden with environment variables, see `config/custom-environment-variables.cjs`. Notably: + + - `SECRET_CAPTURE` (`secretKeys.capture`, default none): when set, `/screenshot` and `/print` require it in `x-secret-key` from internal callers without a session. Leave it unset while browsers or other services call them directly. + - `SECRET_INSPECT` (`secretKeys.inspect`, default none): required in `x-secret-key` on `/inspect` (internal requests only), which is disabled without it. A secret of its own, so that enabling `/inspect` does not lock `/screenshot` and `/print`. + - `INSPECT_CONCURRENCY` (`inspectConcurrency`, default 1): how many `/inspect` renders may run at the same time, each one holds a browser page; further requests wait for a free slot. +======= ## Logging Logs go through the logger shared by the data-fair stack ([`@data-fair/lib-node/log.js`](https://github.com/data-fair/lib/tree/main/packages/node#logging)): one JSON object per line on stdout when `NODE_ENV=production` (the docker image), pretty output otherwise, `LOG_FORMAT=json|pretty` overrides. Warn, alert and error lines are counted in the `df_logs{level, code, host}` metric. `DEBUG=capture,timer` adds the debug lines of the capture steps and their durations. diff --git a/api/routers/capture.ts b/api/routers/capture.ts index 3cb5772..8a57ba2 100644 --- a/api/routers/capture.ts +++ b/api/routers/capture.ts @@ -4,12 +4,18 @@ import * as headerFooter from '../utils/header-footer.ts' import * as pageUtils from '../utils/page.ts' import * as animationUtils from '../utils/animation.ts' import { createTimer } from '../utils/timer.ts' +import { createSemaphore } from '../utils/semaphore.ts' +import { createReportCollector } from '../utils/report.ts' +import { reportText } from '../utils/report-text.ts' import config from '#config' import { assertReqInternalSecret, httpError, reqHost, reqIsInternal, reqSession } from '@data-fair/lib-express' import type { PaperFormat, PDFOptions } from 'puppeteer' const debug = debugModule('capture') +// /inspect renders are expensive, a small number of them run at the same time +const inspectSemaphore = createSemaphore(config.inspectConcurrency) + export const router = express.Router() async function auth (req: Request, res: Response, next: NextFunction) { @@ -50,7 +56,7 @@ async function auth (req: Request, res: Response, next: NextFunction) { if (securedSameHost && req.cookies && Object.keys(req.cookies).length && req.query.cookies !== 'false') { debug(`[${target}] transmit cookies`) - req.cookies = Object.keys(req.cookies).map(name => ({ name, value: req.cookies[name], url: target })) + req.cookies = Object.keys(req.cookies).map(name => ({ name, value: req.cookies[name], url: target })) as any } else { debug(`[${target}] do not transmit cookies`) req.cookies = [] @@ -189,3 +195,87 @@ router.get('/print', auth, async (req, res, next) => { ) timer.finish() }) + +const desktopViewport = { width: 1280, height: 800 } +const mobileViewport = { width: 390, height: 844, isMobile: true, hasTouch: true } +const maxFullPageHeight = 3000 + +const viewportDimension = (value: unknown, fallback: number) => { + if (value === undefined) return fallback + const parsed = typeof value === 'string' ? parseInt(value) : NaN + if (!(parsed > 0 && parsed <= 3000)) throw httpError(400, 'invalid width or height') + return parsed +} + +// internal only: used by data-fair and portals so that agents can see what they configured +router.get('/inspect', async (req, res) => { + // a secret of its own: setting it does not lock /screenshot and /print, that public clients call + if (!config.secretKeys.inspect) throw httpError(501, 'inspect is disabled, it needs SECRET_INSPECT') + assertReqInternalSecret(req, config.secretKeys.inspect) + + const target = req.query.target + if (typeof target !== 'string') throw httpError(400, 'parameter "target" is required') + let targetUrl + try { + targetUrl = new URL(target) + } catch (err: any) { + throw httpError(400, 'Failed to parse target url ' + err.message) + } + if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') throw httpError(400, 'target must be an http or https url') + debug(`[${target}] inspect`) + + const lang = req.query.lang + if (lang && typeof lang !== 'string') throw httpError(400, 'invalid lang') + const timezone = req.query.timezone + if (timezone && typeof timezone !== 'string') throw httpError(400, 'invalid timezone') + + const mobile = req.query.mobile === 'true' + const fullPage = req.query.fullPage === 'true' + const viewport = mobile + ? mobileViewport + : { width: viewportDimension(req.query.width, desktopViewport.width), height: viewportDimension(req.query.height, desktopViewport.height) } + + // the page may POST its own error diagnosis there, it is reported instead of being sent + const declaredErrorUrl = req.query.declaredErrorUrl + if (declaredErrorUrl !== undefined) { + if (typeof declaredErrorUrl !== 'string' || !URL.canParse(declaredErrorUrl)) throw httpError(400, 'invalid declaredErrorUrl') + if (!pageUtils.isSameHost(declaredErrorUrl, target)) throw httpError(400, 'declaredErrorUrl must be on the host of target') + } + + const cookies = Object.keys(req.cookies ?? {}).map(name => ({ name, value: req.cookies[name], url: target })) as any + const timer = createTimer(req.originalUrl, 'inspect') + + let collector: ReturnType | undefined + await inspectSemaphore.run(async () => { + // the client may have left while waiting for a slot, no need to render for nobody + if (req.socket.destroyed || res.destroyed) return + await pageUtils.withPage( + target, + lang, + timezone, + cookies, + viewport, + false, + timer, + `Failed to inspect page "${target}" before timeout`, + async ({ page, ready }) => { + const report = await collector!.finish(ready) + let buffer + if (fullPage) { + const height = await page.evaluate(() => document.documentElement.scrollHeight) + buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } }) + } else { + buffer = await page.screenshot() + } + timer.type = 'inspect' + res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report }) + }, + { + onPage: page => { collector = createReportCollector(page, target) }, + declaredErrorUrl, + onDeclaredError: body => collector?.declareError(body) + } + ) + }) + timer.finish() +}) diff --git a/api/utils/aria-snapshot.ts b/api/utils/aria-snapshot.ts new file mode 100644 index 0000000..a3f3c88 --- /dev/null +++ b/api/utils/aria-snapshot.ts @@ -0,0 +1,196 @@ +import type { Page, SerializedAXNode } from 'puppeteer' + +/* eslint-disable no-control-regex */ +// yamlEscapeKeyIfNeeded and yamlEscapeValueIfNeeded are copied from Playwright +// https://github.com/microsoft/playwright/blob/main/packages/isomorphic/yaml.ts +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export function yamlEscapeKeyIfNeeded (str: string): string { + if (!yamlStringNeedsQuotes(str)) { return str } + return '\'' + str.replace(/'/g, '\'\'') + '\'' +} + +export function yamlEscapeValueIfNeeded (str: string): string { + if (!yamlStringNeedsQuotes(str)) { return str } + return '"' + str.replace(/[\\"\x00-\x1f\x7f-\x9f]/g, c => { + switch (c) { + case '\\': + return '\\\\' + case '"': + return '\\"' + case '\b': + return '\\b' + case '\f': + return '\\f' + case '\n': + return '\\n' + case '\r': + return '\\r' + case '\t': + return '\\t' + default: { + const code = c.charCodeAt(0) + return '\\x' + code.toString(16).padStart(2, '0') + } + } + }) + '"' +} + +function yamlStringNeedsQuotes (str: string): boolean { + if (str.length === 0) { return true } + + // Strings with leading or trailing whitespace need quotes + if (/^\s|\s$/.test(str)) { return true } + + // Strings containing control characters need quotes + if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/.test(str)) { return true } + + // Strings starting with '-' need quotes + if (/^-/.test(str)) { return true } + + // Strings containing ':' or '\n' followed by a space or at the end need quotes + if (/[\n:](\s|$)/.test(str)) { return true } + + // Strings containing '#' preceded by a space need quotes (comment indicator) + if (/\s#/.test(str)) { return true } + + // Strings that contain line breaks need quotes + if (/[\n\r]/.test(str)) { return true } + + // Strings starting with indicator characters or quotes need quotes + if (/^[&*\],?!>|@"'#%]/.test(str)) { return true } + + // Strings containing special characters that could cause ambiguity + if (/[{}`]/.test(str)) { return true } + + // YAML array starts with [ + if (/^\[/.test(str)) { return true } + + // Non-string types recognized by YAML + if (!isNaN(Number(str)) || ['y', 'n', 'yes', 'no', 'true', 'false', 'on', 'off', 'null', '~'].includes(str.toLowerCase())) { return true } + + // YAML special float values: .inf, -.inf, .nan and their case variants + if (/^[-+]?\.(inf|nan)$/i.test(str)) { return true } + + return false +} + +/* eslint-enable no-control-regex */ + +const indent = (depth: number) => ' '.repeat(depth) + +const createKey = (node: SerializedAXNode) => { + let key = node.role === 'Iframe' ? 'iframe' : node.role + // YAML has a limit of 1024 characters per key, leave room for the role and attributes + const name = text(node) + if (name && name.length <= 900) key += ' ' + JSON.stringify(name) + if (node.checked === 'mixed') key += ' [checked=mixed]' + if (node.checked === true) key += ' [checked]' + if (node.disabled) key += ' [disabled]' + if (node.expanded) key += ' [expanded]' + if (node.role === 'heading' && node.level) key += ` [level=${node.level}]` + if (node.pressed === 'mixed') key += ' [pressed=mixed]' + if (node.pressed === true) key += ' [pressed]' + if (node.selected) key += ' [selected]' + if (node.invalid && node.invalid !== 'false') key += ' [invalid]' + return key +} + +const isText = (node: SerializedAXNode) => node.role === 'StaticText' +const MAX_TEXT = 2000 +// one huge value must not use the whole snapshot budget +const capText = (str: string) => str.length > MAX_TEXT ? str.slice(0, MAX_TEXT - 1) + '…' : str +const text = (node: SerializedAXNode) => capText((node.name ?? '').replace(/\s+/g, ' ').trim()) + +// interestingOnly would drop useful containers (paragraph, list, listitem), so the whole tree is +// requested and its structural noise is flattened here: children of unnamed none/generic nodes and +// of nested documents (iframes) are hoisted, text boxes and list markers are dropped +const flatten = (nodes: SerializedAXNode[] | undefined): SerializedAXNode[] => { + const result: SerializedAXNode[] = [] + for (const node of nodes ?? []) { + if (node.role === 'InlineTextBox' || node.role === 'ListMarker') continue + if (node.role === 'RootWebArea' || ((node.role === 'none' || node.role === 'generic') && !node.name)) result.push(...flatten(node.children)) + else result.push({ ...node, children: flatten(node.children) }) + } + return result +} + +export const ariaSnapshot = async (page: Page, opts: { maxChars: number, maxLinks: number }): Promise => { + const root = await page.accessibility.snapshot({ interestingOnly: false, includeIframes: true }) + if (!root) return '' + const lines: string[] = [] + let links = 0 + + const visitText = (text: string, depth: number) => { + const escaped = yamlEscapeValueIfNeeded(text) + if (escaped) lines.push(indent(depth) + '- text: ' + escaped) + } + + const visit = async (node: SerializedAXNode, depth: number) => { + if (isText(node)) return visitText(text(node), depth) + const key = indent(depth) + '- ' + yamlEscapeKeyIfNeeded(createKey(node)) + + let children = node.children ?? [] + // children texts that only repeat the name are redundant + if (text(node) && children.length && children.every(isText) && text({ name: children.map(c => c.name ?? '').join('') } as SerializedAXNode) === text(node)) children = [] + + let href: string | undefined + if (node.role === 'link' && links < opts.maxLinks) { + links++ + try { + const handle = await node.elementHandle() + href = (await handle?.evaluate(el => el.getAttribute('href'))) ?? node.url + await handle?.dispose() + } catch (err) { + href = node.url + } + } + + let inlineText: string | undefined + if ((node.role === 'textbox' || node.role === 'combobox') && node.value !== undefined && node.value !== '') inlineText = capText(String(node.value)) + else if (!text(node) && children.length === 1 && isText(children[0]) && !href) { + inlineText = text(children[0]) + children = [] + } + + if (inlineText !== undefined && !href && !children.length) { + lines.push(key + ': ' + yamlEscapeValueIfNeeded(inlineText)) + } else if (!href && !children.length && inlineText === undefined) { + lines.push(key) + } else { + lines.push(key + ':') + if (href) lines.push(indent(depth + 1) + '- /url: ' + yamlEscapeValueIfNeeded(href)) + if (inlineText !== undefined) visitText(inlineText, depth + 1) + for (const child of children) await visit(child, depth + 1) + } + } + + for (const child of flatten(root.children)) await visit(child, 0) + + // truncate at a line boundary, keeping room for the last line + const reserve = 50 + let size = 0 + let kept = 0 + for (const line of lines) { + if (size + line.length + 1 > opts.maxChars - reserve) break + size += line.length + 1 + kept++ + } + if (kept < lines.length) { + return lines.slice(0, kept).concat(`# snapshot truncated: ${lines.length - kept} more lines`).join('\n') + } + return lines.join('\n') +} diff --git a/api/utils/page.ts b/api/utils/page.ts index b13cba8..47dab1f 100644 --- a/api/utils/page.ts +++ b/api/utils/page.ts @@ -6,6 +6,7 @@ import debugModule from 'debug' import log from '@data-fair/lib-node/log.js' import { Timer } from './timer.ts' import { httpError } from '@data-fair/lib-express' +import type { Ready } from './report.ts' const debug = debugModule('capture') @@ -82,9 +83,35 @@ const debugPage = (page: Page) => { }) } +// the host rule of onlySameHost, shared by the frames check and the report collector +export const isSameHost = (url: string, target: string) => { + try { + return new URL(url).host === new URL(target).host + } catch (err) { + return false + } +} + +// options of the /inspect renders +export type InspectOptions = { + onPage?: (page: Page) => void, + // POST requests to this URL (query ignored) are answered by capture itself and never reach the + // server, their body is handed to onDeclaredError: this is how an application declares an error + declaredErrorUrl?: string, + onDeclaredError?: (body: string | undefined) => void +} + +const withoutQuery = (url: string) => { + const parsed = new URL(url) + parsed.search = '' + parsed.hash = '' + return parsed.href +} + type OpenPageResult = { page: Page, - animationActivated: boolean + animationActivated: boolean, + ready: Ready } async function openInPage ( @@ -95,30 +122,47 @@ async function openInPage ( cookies: CookieData[], viewport: Viewport | undefined, animate: boolean, - timer: Timer + timer: Timer, + inspect: InspectOptions | undefined ): Promise { + const reportFrames = !!inspect await setPageLocale(page, lang || config.defaultLang, timezone || config.defaultTimezone) if (cookies) await page.browserContext().setCookie(...cookies) if (viewport) await page.setViewport(viewport) + const declaredErrorUrl = inspect?.declaredErrorUrl && withoutQuery(inspect.declaredErrorUrl) + if (reportFrames && (config.onlySameHost || declaredErrorUrl)) { + // cross-host frames are reported by the collector, but their content must not be loaded + await page.setRequestInterception(true) + page.on('request', request => { + if (request.isInterceptResolutionHandled()) return + if (declaredErrorUrl && request.method() === 'POST' && URL.canParse(request.url()) && withoutQuery(request.url()) === declaredErrorUrl) { + // a render must not write anything, the declared error is only reported + inspect?.onDeclaredError?.(request.postData()) + request.respond({ status: 204 }).catch(() => {}) + } else if (config.onlySameHost && request.isNavigationRequest() && request.frame() !== page.mainFrame() && !isSameHost(request.url(), target)) { + request.abort('blockedbyclient').catch(() => {}) + } else { + request.continue().catch(() => {}) + } + }) + } timer.step('configure-page') - const animationActivated = await waitForPage(page, target, animate, timer) + const { animationActivated, ready } = await waitForPage(page, target, animate, timer) for (const frame of page.frames()) { const frameUrl = frame.url() if (!frameUrl) continue - let sameHost = false - try { - sameHost = new URL(frameUrl).host === new URL(target).host - } catch (err) { - throw httpError(400, 'IFrame with invalid URL :' + frameUrl) - } - if (!sameHost && config.onlySameHost) { + // the frames are recorded by the report collector instead of failing the request + if (reportFrames && !/^https?:/.test(frameUrl)) continue + if (!URL.canParse(frameUrl)) throw httpError(400, 'IFrame with invalid URL :' + frameUrl) + const sameHost = isSameHost(frameUrl, target) + if (!sameHost && config.onlySameHost && !reportFrames) { debug(`${frameUrl} from iframe in ${target} is NOT on same host, reject`) throw httpError(400, 'IFrame did not have same host :' + new URL(frameUrl).host) } } - return { page, animationActivated } + return { page, animationActivated, ready } } export const promiseTimeout = async (promise: Promise, ms: number, msg = 'timeout') => { @@ -137,7 +181,8 @@ export const withPage = async ( animate: boolean, timer: Timer, callbackTimeoutMsg: string, - callback: (result: OpenPageResult) => Promise) => { + callback: (result: OpenPageResult) => Promise, + inspect?: InspectOptions) => { if (target.includes('capture-test-error=true')) { await new Promise(resolve => setTimeout(resolve, 1000)) throw new Error('forced error trigger') @@ -162,10 +207,11 @@ export const withPage = async ( } } debugPage(page) + inspect?.onPage?.(page) timer.step('newPage') const result = await promiseTimeout( - openInPage(page, target, lang, timezone, cookies, viewport, animate, timer), + openInPage(page, target, lang, timezone, cookies, viewport, animate, timer, inspect), config.screenshotTimeout * 2, `Failed to open "${target}" in context before timeout` ) @@ -218,7 +264,7 @@ const waitForPage = async ( target: string, animate: boolean, timer: Timer -) => { +): Promise<{ animationActivated: boolean, ready: Ready }> => { // Prepare a function that the page can call to signal that it is ready for capture let captureTriggered = false let timeoutReached = false @@ -300,6 +346,7 @@ const waitForPage = async ( } else { timer.step('wait2-timeout') debug(`[${target}] timeout of ${config.screenshotTimeout} was reached`) + timeoutReached = true } } else { debug(`[${target}] wait 1000ms more after idle network for safety`) @@ -308,7 +355,8 @@ const waitForPage = async ( } } } - return animationActivated + const ready: Ready = captureTriggered ? 'triggered' : timeoutReached ? 'timeout' : 'idle' + return { animationActivated, ready } } const setPageLocale = async (page: Page, lang: string, timezone: string) => { diff --git a/api/utils/report-text.ts b/api/utils/report-text.ts new file mode 100644 index 0000000..f0156a7 --- /dev/null +++ b/api/utils/report-text.ts @@ -0,0 +1,71 @@ +import type { RenderReport } from './report.ts' +import { isSameHost } from './page.ts' + +const DETAILS_MAX_CHARS = 8000 + +// layout of Playwright MCP responses (### Page, ### Console, ### Network, ### Snapshot), +// plus the Ready, Verdict, [BLOCKED] and body lines that are specific to this tool +export const reportText = (report: RenderReport): string => { + const lines: string[] = ['### Page'] + lines.push(`- Page URL: ${report.finalUrl}`) + lines.push(`- Page Title: ${report.title}`) + if (report.status !== undefined && (report.status < 200 || report.status >= 300)) { + lines.push(`- HTTP status: ${report.status} ${report.statusText ?? ''}`.trimEnd()) + } + const errors = report.consoleErrors.length + report.pageErrors.length + if (errors > 0 || report.consoleWarnings > 0) { + lines.push(`- Console: ${errors} errors, ${report.consoleWarnings} warnings`) + } + lines.push(`- Ready: ${report.ready === 'timeout' ? `timeout after ${report.durationMs} ms` : report.ready}`) + if (!isSameHost(report.finalUrl, report.url)) { + lines.push(`- Note: navigated away from ${new URL(report.url).host} to ${new URL(report.finalUrl).host}`) + } + lines.push(`- Verdict: ${report.verdict}`) + + // Console and Network together are bounded, the snapshot has its own budget + const details: string[] = [] + // first, the page's own diagnosis is the most useful line + if (report.declaredErrors.length) { + details.push('### Declared errors') + for (const message of report.declaredErrors) details.push(`- ${message}`) + } + if (report.consoleErrors.length || report.pageErrors.length) { + details.push('### Console') + for (const entry of report.consoleErrors) { + details.push(`[ERROR] ${entry.text}` + (entry.url ? ` @ ${entry.url}${entry.line !== undefined ? ':' + entry.line : ''}` : '')) + } + for (const error of report.pageErrors) details.push(error) + } + + if (report.failedRequests.length || report.blockedFrames.length) { + details.push('### Network') + let i = 0 + for (const request of report.failedRequests) { + i++ + if (request.status !== undefined) { + details.push(`${i}. [${request.method}] ${request.url} => [${request.status}] ${request.statusText ?? ''}`.trimEnd()) + if (request.body) details.push(` ${request.body}`) + } else { + details.push(`${i}. [${request.method}] ${request.url} => [FAILED] ${request.errorText ?? ''}`.trimEnd()) + } + } + for (const origin of report.blockedFrames) { + i++ + details.push(`${i}. [GET] ${origin} => [BLOCKED] cross-host frame not rendered`) + } + } + + // truncate at a line boundary, keeping room for the last line + let size = 0 + let kept = 0 + for (const line of details) { + if (size + line.length + 1 > DETAILS_MAX_CHARS - 50) break + size += line.length + 1 + kept++ + } + lines.push(...details.slice(0, kept)) + if (kept < details.length) lines.push(`# console and network truncated: ${details.length - kept} more lines`) + + lines.push('### Snapshot', '```yaml', report.snapshot, '```') + return lines.join('\n') +} diff --git a/api/utils/report.ts b/api/utils/report.ts new file mode 100644 index 0000000..2fdda52 --- /dev/null +++ b/api/utils/report.ts @@ -0,0 +1,175 @@ +import type { Page } from 'puppeteer' +import config from '#config' +import { ariaSnapshot } from './aria-snapshot.ts' +import { isSameHost } from './page.ts' + +export type Ready = 'triggered' | 'idle' | 'timeout' +export type Verdict = 'ok' | 'errors' | 'timeout' | 'unauthenticated' +export type ConsoleEntry = { text: string, url?: string, line?: number } +export type FailedRequest = { method: string, url: string, path: string, status?: number, statusText?: string, errorText?: string, body?: string } +export type RenderReport = { + url: string, + finalUrl: string, + status?: number, + statusText?: string, + ready: Ready, + durationMs: number, + title: string, + consoleErrors: ConsoleEntry[], + consoleWarnings: number, + pageErrors: string[], + failedRequests: FailedRequest[], + blockedFrames: string[], + declaredErrors: string[], + snapshot: string, + verdict: Verdict +} + +const MAX_ENTRIES = 20 +const MAX_STRING = 500 +const MAX_BODY = 300 +const SNAPSHOT_MAX_CHARS = 20000 +const SNAPSHOT_MAX_LINKS = 200 + +const cut = (str: string, max = MAX_STRING) => str.length > max ? str.slice(0, max - 1) + '…' : str + +const withoutQuery = (url: string) => { + const parsed = new URL(url) + parsed.search = '' + parsed.hash = '' + return parsed +} + +// browsers ask for it on their own, a missing one is noise for the agent +const isFavicon = (url: string) => new URL(url).pathname === '/favicon.ico' + +const parseOrigin = (url: string) => { + try { + return new URL(url).origin + } catch (err) { + return undefined + } +} + +export const createReportCollector = (page: Page, target: string) => { + const started = Date.now() + const targetOrigin = parseOrigin(target) + const consoleErrors: ConsoleEntry[] = [] + let consoleWarnings = 0 + const pageErrors: string[] = [] + const failedRequests: FailedRequest[] = [] + const pending: Promise[] = [] + let status: number | undefined + let statusText: string | undefined + const declaredErrors: string[] = [] + + // the page itself says it is broken, cf InspectOptions.declaredErrorUrl + const declareError = (body: string | undefined) => { + let message = body ?? '' + try { + const parsed = JSON.parse(message) + if (parsed && typeof parsed.message === 'string') message = parsed.message + } catch (err) { + // not JSON, the raw body is the message + } + declaredErrors.push(message.trim() || '(no message)') + } + + const sameOrigin = (url: string) => parseOrigin(url) === targetOrigin + + page.on('console', msg => { + if (msg.type() === 'error') { + const { url, lineNumber } = msg.location() + // Chrome logs failed resources on its own, same-origin ones are in the Network section + if (/^Failed to load resource/.test(msg.text()) && url && (isFavicon(url) || sameOrigin(url))) return + consoleErrors.push({ text: msg.text(), ...(url ? { url } : {}), ...(url && lineNumber !== undefined ? { line: lineNumber } : {}) }) + } else if (msg.type() === 'warn') { + consoleWarnings++ + } + }) + // cross-host frames are aborted by openInPage (reportFrames), only their origin is kept + const blockedFrames: string[] = [] + page.on('request', request => { + if (!config.onlySameHost || !request.isNavigationRequest() || request.frame() === page.mainFrame()) return + if (!/^https?:/.test(request.url()) || isSameHost(request.url(), target)) return + const origin = parseOrigin(request.url()) + if (origin && !blockedFrames.includes(origin)) blockedFrames.push(origin) + }) + page.on('pageerror', err => { + pageErrors.push(err instanceof Error ? (err.stack || err.message) : String(err)) + }) + page.on('requestfailed', request => { + if (!sameOrigin(request.url()) || isFavicon(request.url())) return + // apps cancel superseded fetches + if (request.failure()?.errorText === 'net::ERR_ABORTED') return + const url = withoutQuery(request.url()) + failedRequests.push({ method: request.method(), url: url.href, path: url.pathname, errorText: request.failure()?.errorText }) + }) + page.on('response', response => { + const request = response.request() + if (request.isNavigationRequest() && request.frame() === page.mainFrame()) { + status = response.status() + statusText = response.statusText() + } + if (response.status() < 400 || !sameOrigin(response.url()) || isFavicon(response.url())) return + const url = withoutQuery(response.url()) + const entry: FailedRequest = { + method: request.method(), + url: url.href, + path: url.pathname, + status: response.status(), + statusText: response.statusText() + } + failedRequests.push(entry) + pending.push((async () => { + const contentType = response.headers()['content-type'] ?? '' + if (/^text\/|json|html/i.test(contentType)) { + try { + const body = (await response.text()).trim() + if (body) entry.body = cut(body, MAX_BODY) + } catch (err) { + // the body is not always available (redirects, aborted requests) + } + } else if (contentType) { + entry.body = `[${contentType}]` + } + })()) + }) + + const finish = async (ready: Ready): Promise => { + // events of the last requests can reach us a few ms after the page signaled it is ready + await new Promise(resolve => setTimeout(resolve, 100)) + await Promise.all(pending) + const finalUrl = page.url() + const [title, snapshot] = await Promise.all([ + page.title().catch(() => ''), + ariaSnapshot(page, { maxChars: SNAPSHOT_MAX_CHARS, maxLinks: SNAPSHOT_MAX_LINKS }) + ]) + const movedAway = !isSameHost(finalUrl, target) + let verdict: Verdict + if (new URL(finalUrl, target).pathname.startsWith('/simple-directory/login')) verdict = 'unauthenticated' + else if (ready === 'timeout') verdict = 'timeout' + else if (movedAway || declaredErrors.length || consoleErrors.length || pageErrors.length || failedRequests.length || blockedFrames.length || (status !== undefined && status >= 400)) verdict = 'errors' + else verdict = 'ok' + + return { + url: target, + finalUrl, + status, + statusText, + ready, + durationMs: Date.now() - started, + title: cut(title), + consoleErrors: consoleErrors.slice(0, MAX_ENTRIES).map(e => ({ ...e, text: cut(e.text), ...(e.url ? { url: cut(e.url) } : {}) })), + consoleWarnings, + pageErrors: pageErrors.slice(0, MAX_ENTRIES).map(e => cut(e)), + failedRequests: failedRequests.slice(0, MAX_ENTRIES).map(f => ({ ...f, url: cut(f.url), path: cut(f.path), ...(f.errorText ? { errorText: cut(f.errorText) } : {}) })), + blockedFrames: blockedFrames.slice(0, MAX_ENTRIES), + declaredErrors: declaredErrors.slice(0, MAX_ENTRIES).map(e => cut(e)), + snapshot, + verdict + } + } + + return { finish, declareError } +} diff --git a/api/utils/semaphore.ts b/api/utils/semaphore.ts new file mode 100644 index 0000000..49bdbcd --- /dev/null +++ b/api/utils/semaphore.ts @@ -0,0 +1,35 @@ +// Limits how many async functions run at the same time, the others wait in FIFO order. +export const createSemaphore = (max: number) => { + let active = 0 + const queue: Array<() => void> = [] + + const acquire = () => new Promise(resolve => { + if (active < max) { + active++ + resolve() + } else { + queue.push(() => { + active++ + resolve() + }) + } + }) + + const release = () => { + active-- + queue.shift()?.() + } + + return { + async run (fn: () => Promise): Promise { + await acquire() + try { + return await fn() + } finally { + release() + } + }, + // number of calls waiting for a free slot + pending: () => queue.length + } +} diff --git a/config/custom-environment-variables.cjs b/config/custom-environment-variables.cjs index 3b4130e..597219c 100644 --- a/config/custom-environment-variables.cjs +++ b/config/custom-environment-variables.cjs @@ -9,10 +9,12 @@ module.exports = { }, screenshotTimeout: 'SCREENSHOT_TIMEOUT', concurrency: 'CONCURRENCY', + inspectConcurrency: 'INSPECT_CONCURRENCY', defaultLang: 'DEFAULT_LANG', defaultTimezone: 'DEFAULT_TIMEZONE', secretKeys: { - capture: 'SECRET_CAPTURE' + capture: 'SECRET_CAPTURE', + inspect: 'SECRET_INSPECT' }, observer: { active: { diff --git a/config/default.cjs b/config/default.cjs index 858b42b..c43bb0a 100644 --- a/config/default.cjs +++ b/config/default.cjs @@ -8,10 +8,12 @@ module.exports = { }, screenshotTimeout: 20000, concurrency: 5, + inspectConcurrency: 1, // concurrent /inspect renders, each one holds a browser page defaultLang: 'fr-FR', defaultTimezone: 'Europe/Paris', secretKeys: { - capture: null + capture: null, // when set, required on /screenshot and /print from callers without a session + inspect: null // required on /inspect, which is disabled without it }, maxAnimationFrames: 1800, // 2 minutes at 15fps puppeteerLaunchOptions: { diff --git a/config/development.cjs b/config/development.cjs index a57618e..f85bfcf 100644 --- a/config/development.cjs +++ b/config/development.cjs @@ -5,7 +5,8 @@ module.exports = { onlySameHost: false, concurrency: 2, secretKeys: { - capture: 'capture' + capture: 'capture', + inspect: 'inspect' }, observer: { active: false diff --git a/config/test.cjs b/config/test.cjs index c67c113..6231010 100644 --- a/config/test.cjs +++ b/config/test.cjs @@ -6,7 +6,8 @@ module.exports = { onlySameHost: true, useHostHeader: true, secretKeys: { - capture: 'capture' + capture: 'capture', + inspect: 'inspect' }, observer: { active: false diff --git a/config/type/.type/index.d.ts b/config/type/.type/index.d.ts index 40ef82d..5023df9 100644 --- a/config/type/.type/index.d.ts +++ b/config/type/.type/index.d.ts @@ -11,6 +11,7 @@ export type ApiConfig = { }; secretKeys: { capture: string; + inspect?: string; }; observer: { active?: boolean; @@ -21,6 +22,7 @@ export type ApiConfig = { [k: string]: unknown; }; concurrency: number; + inspectConcurrency: number; defaultLang: string; defaultTimezone: string; onlySameHost?: boolean; diff --git a/config/type/.type/validate.js b/config/type/.type/validate.js index 0737b60..156d33f 100644 --- a/config/type/.type/validate.js +++ b/config/type/.type/validate.js @@ -4,7 +4,7 @@ "use strict"; export const validate = validate14; export default validate14; -const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; +const schema16 = {"$id":"https://github.com/data-fair/events/api/config","x-exports":["types","validate"],"x-ajv":{"coerceTypes":"array"},"type":"object","title":"Api config","additionalProperties":false,"required":["port","secretKeys","helmet","observer","puppeteerLaunchOptions","concurrency","inspectConcurrency","defaultLang","defaultTimezone","screenshotTimeout","maxAnimationFrames"],"properties":{"port":{"type":"number"},"publicUrl":{"type":"string"},"privateDirectoryUrl":{"type":"string","pattern":"^https?://"},"helmet":{"type":"object","additionalProperties":false,"required":["active"],"properties":{"active":{"type":"boolean"}}},"secretKeys":{"type":"object","additionalProperties":false,"required":["capture"],"properties":{"capture":{"type":"string"},"inspect":{"type":"string"}}},"observer":{"type":"object","properties":{"active":{"type":"boolean"},"port":{"type":"number"}}},"puppeteerLaunchOptions":{"type":"object"},"concurrency":{"type":"integer"},"inspectConcurrency":{"type":"integer","minimum":1},"defaultLang":{"type":"string"},"defaultTimezone":{"type":"string"},"onlySameHost":{"type":"boolean"},"useHostHeader":{"type":"boolean"},"screenshotTimeout":{"type":"number"},"maxAnimationFrames":{"type":"number"},"util":{},"get":{},"has":{}}}; const func2 = Object.prototype.hasOwnProperty; const pattern0 = new RegExp("^https?://", "u"); @@ -73,8 +73,8 @@ vErrors.push(err5); } errors++; } -if(data.defaultLang === undefined){ -const err6 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; +if(data.inspectConcurrency === undefined){ +const err6 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "inspectConcurrency"},message:"must have required property '"+"inspectConcurrency"+"'"}; if(vErrors === null){ vErrors = [err6]; } @@ -83,8 +83,8 @@ vErrors.push(err6); } errors++; } -if(data.defaultTimezone === undefined){ -const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; +if(data.defaultLang === undefined){ +const err7 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultLang"},message:"must have required property '"+"defaultLang"+"'"}; if(vErrors === null){ vErrors = [err7]; } @@ -93,8 +93,8 @@ vErrors.push(err7); } errors++; } -if(data.screenshotTimeout === undefined){ -const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; +if(data.defaultTimezone === undefined){ +const err8 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "defaultTimezone"},message:"must have required property '"+"defaultTimezone"+"'"}; if(vErrors === null){ vErrors = [err8]; } @@ -103,8 +103,8 @@ vErrors.push(err8); } errors++; } -if(data.maxAnimationFrames === undefined){ -const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; +if(data.screenshotTimeout === undefined){ +const err9 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "screenshotTimeout"},message:"must have required property '"+"screenshotTimeout"+"'"}; if(vErrors === null){ vErrors = [err9]; } @@ -113,9 +113,8 @@ vErrors.push(err9); } errors++; } -for(const key0 in data){ -if(!(func2.call(schema16.properties, key0))){ -const err10 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(data.maxAnimationFrames === undefined){ +const err10 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "maxAnimationFrames"},message:"must have required property '"+"maxAnimationFrames"+"'"}; if(vErrors === null){ vErrors = [err10]; } @@ -124,6 +123,17 @@ vErrors.push(err10); } errors++; } +for(const key0 in data){ +if(!(func2.call(schema16.properties, key0))){ +const err11 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"}; +if(vErrors === null){ +vErrors = [err11]; +} +else { +vErrors.push(err11); +} +errors++; +} } if(data.port !== undefined){ let data0 = data.port; @@ -143,12 +153,12 @@ if(dataType0 == "boolean" || data0 === null coerced0 = +data0; } else { -const err11 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err12 = {instancePath:instancePath+"/port",schemaPath:"#/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err11]; +vErrors = [err12]; } else { -vErrors.push(err11); +vErrors.push(err12); } errors++; } @@ -181,12 +191,12 @@ else if(data1 === null){ coerced1 = ""; } else { -const err12 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err13 = {instancePath:instancePath+"/publicUrl",schemaPath:"#/properties/publicUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err12]; +vErrors = [err13]; } else { -vErrors.push(err12); +vErrors.push(err13); } errors++; } @@ -219,12 +229,12 @@ else if(data2 === null){ coerced2 = ""; } else { -const err13 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err14 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err13]; +vErrors = [err14]; } else { -vErrors.push(err13); +vErrors.push(err14); } errors++; } @@ -238,12 +248,12 @@ data["privateDirectoryUrl"] = coerced2; } if(typeof data2 === "string"){ if(!pattern0.test(data2)){ -const err14 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; +const err15 = {instancePath:instancePath+"/privateDirectoryUrl",schemaPath:"#/properties/privateDirectoryUrl/pattern",keyword:"pattern",params:{pattern: "^https?://"},message:"must match pattern \""+"^https?://"+"\""}; if(vErrors === null){ -vErrors = [err14]; +vErrors = [err15]; } else { -vErrors.push(err14); +vErrors.push(err15); } errors++; } @@ -253,23 +263,23 @@ if(data.helmet !== undefined){ let data3 = data.helmet; if(data3 && typeof data3 == "object" && !Array.isArray(data3)){ if(data3.active === undefined){ -const err15 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; +const err16 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/required",keyword:"required",params:{missingProperty: "active"},message:"must have required property '"+"active"+"'"}; if(vErrors === null){ -vErrors = [err15]; +vErrors = [err16]; } else { -vErrors.push(err15); +vErrors.push(err16); } errors++; } for(const key1 in data3){ if(!(key1 === "active")){ -const err16 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; +const err17 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key1},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err16]; +vErrors = [err17]; } else { -vErrors.push(err16); +vErrors.push(err17); } errors++; } @@ -294,12 +304,12 @@ else if(data4 === "true" || data4 === 1){ coerced3 = true; } else { -const err17 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err18 = {instancePath:instancePath+"/helmet/active",schemaPath:"#/properties/helmet/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err17]; +vErrors = [err18]; } else { -vErrors.push(err17); +vErrors.push(err18); } errors++; } @@ -314,12 +324,12 @@ data3["active"] = coerced3; } } else { -const err18 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err19 = {instancePath:instancePath+"/helmet",schemaPath:"#/properties/helmet/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ -vErrors = [err18]; +vErrors = [err19]; } else { -vErrors.push(err18); +vErrors.push(err19); } errors++; } @@ -328,23 +338,23 @@ if(data.secretKeys !== undefined){ let data5 = data.secretKeys; if(data5 && typeof data5 == "object" && !Array.isArray(data5)){ if(data5.capture === undefined){ -const err19 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; +const err20 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/required",keyword:"required",params:{missingProperty: "capture"},message:"must have required property '"+"capture"+"'"}; if(vErrors === null){ -vErrors = [err19]; +vErrors = [err20]; } else { -vErrors.push(err19); +vErrors.push(err20); } errors++; } for(const key2 in data5){ -if(!(key2 === "capture")){ -const err20 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; +if(!((key2 === "capture") || (key2 === "inspect"))){ +const err21 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key2},message:"must NOT have additional properties"}; if(vErrors === null){ -vErrors = [err20]; +vErrors = [err21]; } else { -vErrors.push(err20); +vErrors.push(err21); } errors++; } @@ -369,12 +379,12 @@ else if(data6 === null){ coerced4 = ""; } else { -const err21 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err22 = {instancePath:instancePath+"/secretKeys/capture",schemaPath:"#/properties/secretKeys/properties/capture/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err21]; +vErrors = [err22]; } else { -vErrors.push(err21); +vErrors.push(err22); } errors++; } @@ -387,42 +397,27 @@ data5["capture"] = coerced4; } } } -} -else { -const err22 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; -if(vErrors === null){ -vErrors = [err22]; -} -else { -vErrors.push(err22); -} -errors++; -} -} -if(data.observer !== undefined){ -let data7 = data.observer; -if(data7 && typeof data7 == "object" && !Array.isArray(data7)){ -if(data7.active !== undefined){ -let data8 = data7.active; -if(typeof data8 !== "boolean"){ -let dataType5 = typeof data8; +if(data5.inspect !== undefined){ +let data7 = data5.inspect; +if(typeof data7 !== "string"){ +let dataType5 = typeof data7; let coerced5 = undefined; -if(dataType5 == 'object' && Array.isArray(data8) && data8.length == 1){ -data8 = data8[0]; -dataType5 = typeof data8; -if(typeof data8 === "boolean"){ -coerced5 = data8; +if(dataType5 == 'object' && Array.isArray(data7) && data7.length == 1){ +data7 = data7[0]; +dataType5 = typeof data7; +if(typeof data7 === "string"){ +coerced5 = data7; } } if(!(coerced5 !== undefined)){ -if(data8 === "false" || data8 === 0 || data8 === null){ -coerced5 = false; +if(dataType5 == "number" || dataType5 == "boolean"){ +coerced5 = "" + data7; } -else if(data8 === "true" || data8 === 1){ -coerced5 = true; +else if(data7 === null){ +coerced5 = ""; } else { -const err23 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err23 = {instancePath:instancePath+"/secretKeys/inspect",schemaPath:"#/properties/secretKeys/properties/inspect/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ vErrors = [err23]; } @@ -433,65 +428,85 @@ errors++; } } if(coerced5 !== undefined){ -data8 = coerced5; -if(data7 !== undefined){ -data7["active"] = coerced5; +data7 = coerced5; +if(data5 !== undefined){ +data5["inspect"] = coerced5; +} } } } } -if(data7.port !== undefined){ -let data9 = data7.port; -if(!(typeof data9 == "number")){ +else { +const err24 = {instancePath:instancePath+"/secretKeys",schemaPath:"#/properties/secretKeys/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err24]; +} +else { +vErrors.push(err24); +} +errors++; +} +} +if(data.observer !== undefined){ +let data8 = data.observer; +if(data8 && typeof data8 == "object" && !Array.isArray(data8)){ +if(data8.active !== undefined){ +let data9 = data8.active; +if(typeof data9 !== "boolean"){ let dataType6 = typeof data9; let coerced6 = undefined; if(dataType6 == 'object' && Array.isArray(data9) && data9.length == 1){ data9 = data9[0]; dataType6 = typeof data9; -if(typeof data9 == "number"){ +if(typeof data9 === "boolean"){ coerced6 = data9; } } if(!(coerced6 !== undefined)){ -if(dataType6 == "boolean" || data9 === null - || (dataType6 == "string" && data9 && data9 == +data9)){ -coerced6 = +data9; +if(data9 === "false" || data9 === 0 || data9 === null){ +coerced6 = false; +} +else if(data9 === "true" || data9 === 1){ +coerced6 = true; } else { -const err24 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err25 = {instancePath:instancePath+"/observer/active",schemaPath:"#/properties/observer/properties/active/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err24]; +vErrors = [err25]; } else { -vErrors.push(err24); +vErrors.push(err25); } errors++; } } if(coerced6 !== undefined){ data9 = coerced6; -if(data7 !== undefined){ -data7["port"] = coerced6; -} +if(data8 !== undefined){ +data8["active"] = coerced6; } } } } -else { -const err25 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; -if(vErrors === null){ -vErrors = [err25]; -} -else { -vErrors.push(err25); +if(data8.port !== undefined){ +let data10 = data8.port; +if(!(typeof data10 == "number")){ +let dataType7 = typeof data10; +let coerced7 = undefined; +if(dataType7 == 'object' && Array.isArray(data10) && data10.length == 1){ +data10 = data10[0]; +dataType7 = typeof data10; +if(typeof data10 == "number"){ +coerced7 = data10; } -errors++; } +if(!(coerced7 !== undefined)){ +if(dataType7 == "boolean" || data10 === null + || (dataType7 == "string" && data10 && data10 == +data10)){ +coerced7 = +data10; } -if(data.puppeteerLaunchOptions !== undefined){ -let data10 = data.puppeteerLaunchOptions; -if(!(data10 && typeof data10 == "object" && !Array.isArray(data10))){ -const err26 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +else { +const err26 = {instancePath:instancePath+"/observer/port",schemaPath:"#/properties/observer/properties/port/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ vErrors = [err26]; } @@ -501,25 +516,17 @@ vErrors.push(err26); errors++; } } -if(data.concurrency !== undefined){ -let data11 = data.concurrency; -if(!((typeof data11 == "number") && (!(data11 % 1) && !isNaN(data11)))){ -let dataType7 = typeof data11; -let coerced7 = undefined; -if(dataType7 == 'object' && Array.isArray(data11) && data11.length == 1){ -data11 = data11[0]; -dataType7 = typeof data11; -if((typeof data11 == "number") && (!(data11 % 1) && !isNaN(data11))){ -coerced7 = data11; +if(coerced7 !== undefined){ +data10 = coerced7; +if(data8 !== undefined){ +data8["port"] = coerced7; +} +} } } -if(!(coerced7 !== undefined)){ -if(dataType7 === "boolean" || data11 === null - || (dataType7 === "string" && data11 && data11 == +data11 && !(data11 % 1))){ -coerced7 = +data11; } else { -const err27 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; +const err27 = {instancePath:instancePath+"/observer",schemaPath:"#/properties/observer/type",keyword:"type",params:{type: "object"},message:"must be object"}; if(vErrors === null){ vErrors = [err27]; } @@ -529,40 +536,43 @@ vErrors.push(err27); errors++; } } -if(coerced7 !== undefined){ -data11 = coerced7; -if(data !== undefined){ -data["concurrency"] = coerced7; +if(data.puppeteerLaunchOptions !== undefined){ +let data11 = data.puppeteerLaunchOptions; +if(!(data11 && typeof data11 == "object" && !Array.isArray(data11))){ +const err28 = {instancePath:instancePath+"/puppeteerLaunchOptions",schemaPath:"#/properties/puppeteerLaunchOptions/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err28]; } +else { +vErrors.push(err28); } +errors++; } } -if(data.defaultLang !== undefined){ -let data12 = data.defaultLang; -if(typeof data12 !== "string"){ +if(data.concurrency !== undefined){ +let data12 = data.concurrency; +if(!((typeof data12 == "number") && (!(data12 % 1) && !isNaN(data12)))){ let dataType8 = typeof data12; let coerced8 = undefined; if(dataType8 == 'object' && Array.isArray(data12) && data12.length == 1){ data12 = data12[0]; dataType8 = typeof data12; -if(typeof data12 === "string"){ +if((typeof data12 == "number") && (!(data12 % 1) && !isNaN(data12))){ coerced8 = data12; } } if(!(coerced8 !== undefined)){ -if(dataType8 == "number" || dataType8 == "boolean"){ -coerced8 = "" + data12; -} -else if(data12 === null){ -coerced8 = ""; +if(dataType8 === "boolean" || data12 === null + || (dataType8 === "string" && data12 && data12 == +data12 && !(data12 % 1))){ +coerced8 = +data12; } else { -const err28 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err29 = {instancePath:instancePath+"/concurrency",schemaPath:"#/properties/concurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err28]; +vErrors = [err29]; } else { -vErrors.push(err28); +vErrors.push(err29); } errors++; } @@ -570,37 +580,35 @@ errors++; if(coerced8 !== undefined){ data12 = coerced8; if(data !== undefined){ -data["defaultLang"] = coerced8; +data["concurrency"] = coerced8; } } } } -if(data.defaultTimezone !== undefined){ -let data13 = data.defaultTimezone; -if(typeof data13 !== "string"){ +if(data.inspectConcurrency !== undefined){ +let data13 = data.inspectConcurrency; +if(!((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13)))){ let dataType9 = typeof data13; let coerced9 = undefined; if(dataType9 == 'object' && Array.isArray(data13) && data13.length == 1){ data13 = data13[0]; dataType9 = typeof data13; -if(typeof data13 === "string"){ +if((typeof data13 == "number") && (!(data13 % 1) && !isNaN(data13))){ coerced9 = data13; } } if(!(coerced9 !== undefined)){ -if(dataType9 == "number" || dataType9 == "boolean"){ -coerced9 = "" + data13; -} -else if(data13 === null){ -coerced9 = ""; +if(dataType9 === "boolean" || data13 === null + || (dataType9 === "string" && data13 && data13 == +data13 && !(data13 % 1))){ +coerced9 = +data13; } else { -const err29 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; +const err30 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/type",keyword:"type",params:{type: "integer"},message:"must be integer"}; if(vErrors === null){ -vErrors = [err29]; +vErrors = [err30]; } else { -vErrors.push(err29); +vErrors.push(err30); } errors++; } @@ -608,37 +616,49 @@ errors++; if(coerced9 !== undefined){ data13 = coerced9; if(data !== undefined){ -data["defaultTimezone"] = coerced9; +data["inspectConcurrency"] = coerced9; } } } +if(typeof data13 == "number"){ +if(data13 < 1 || isNaN(data13)){ +const err31 = {instancePath:instancePath+"/inspectConcurrency",schemaPath:"#/properties/inspectConcurrency/minimum",keyword:"minimum",params:{comparison: ">=", limit: 1},message:"must be >= 1"}; +if(vErrors === null){ +vErrors = [err31]; } -if(data.onlySameHost !== undefined){ -let data14 = data.onlySameHost; -if(typeof data14 !== "boolean"){ +else { +vErrors.push(err31); +} +errors++; +} +} +} +if(data.defaultLang !== undefined){ +let data14 = data.defaultLang; +if(typeof data14 !== "string"){ let dataType10 = typeof data14; let coerced10 = undefined; if(dataType10 == 'object' && Array.isArray(data14) && data14.length == 1){ data14 = data14[0]; dataType10 = typeof data14; -if(typeof data14 === "boolean"){ +if(typeof data14 === "string"){ coerced10 = data14; } } if(!(coerced10 !== undefined)){ -if(data14 === "false" || data14 === 0 || data14 === null){ -coerced10 = false; +if(dataType10 == "number" || dataType10 == "boolean"){ +coerced10 = "" + data14; } -else if(data14 === "true" || data14 === 1){ -coerced10 = true; +else if(data14 === null){ +coerced10 = ""; } else { -const err30 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err32 = {instancePath:instancePath+"/defaultLang",schemaPath:"#/properties/defaultLang/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err30]; +vErrors = [err32]; } else { -vErrors.push(err30); +vErrors.push(err32); } errors++; } @@ -646,37 +666,37 @@ errors++; if(coerced10 !== undefined){ data14 = coerced10; if(data !== undefined){ -data["onlySameHost"] = coerced10; +data["defaultLang"] = coerced10; } } } } -if(data.useHostHeader !== undefined){ -let data15 = data.useHostHeader; -if(typeof data15 !== "boolean"){ +if(data.defaultTimezone !== undefined){ +let data15 = data.defaultTimezone; +if(typeof data15 !== "string"){ let dataType11 = typeof data15; let coerced11 = undefined; if(dataType11 == 'object' && Array.isArray(data15) && data15.length == 1){ data15 = data15[0]; dataType11 = typeof data15; -if(typeof data15 === "boolean"){ +if(typeof data15 === "string"){ coerced11 = data15; } } if(!(coerced11 !== undefined)){ -if(data15 === "false" || data15 === 0 || data15 === null){ -coerced11 = false; +if(dataType11 == "number" || dataType11 == "boolean"){ +coerced11 = "" + data15; } -else if(data15 === "true" || data15 === 1){ -coerced11 = true; +else if(data15 === null){ +coerced11 = ""; } else { -const err31 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; +const err33 = {instancePath:instancePath+"/defaultTimezone",schemaPath:"#/properties/defaultTimezone/type",keyword:"type",params:{type: "string"},message:"must be string"}; if(vErrors === null){ -vErrors = [err31]; +vErrors = [err33]; } else { -vErrors.push(err31); +vErrors.push(err33); } errors++; } @@ -684,35 +704,37 @@ errors++; if(coerced11 !== undefined){ data15 = coerced11; if(data !== undefined){ -data["useHostHeader"] = coerced11; +data["defaultTimezone"] = coerced11; } } } } -if(data.screenshotTimeout !== undefined){ -let data16 = data.screenshotTimeout; -if(!(typeof data16 == "number")){ +if(data.onlySameHost !== undefined){ +let data16 = data.onlySameHost; +if(typeof data16 !== "boolean"){ let dataType12 = typeof data16; let coerced12 = undefined; if(dataType12 == 'object' && Array.isArray(data16) && data16.length == 1){ data16 = data16[0]; dataType12 = typeof data16; -if(typeof data16 == "number"){ +if(typeof data16 === "boolean"){ coerced12 = data16; } } if(!(coerced12 !== undefined)){ -if(dataType12 == "boolean" || data16 === null - || (dataType12 == "string" && data16 && data16 == +data16)){ -coerced12 = +data16; +if(data16 === "false" || data16 === 0 || data16 === null){ +coerced12 = false; +} +else if(data16 === "true" || data16 === 1){ +coerced12 = true; } else { -const err32 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err34 = {instancePath:instancePath+"/onlySameHost",schemaPath:"#/properties/onlySameHost/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err32]; +vErrors = [err34]; } else { -vErrors.push(err32); +vErrors.push(err34); } errors++; } @@ -720,35 +742,37 @@ errors++; if(coerced12 !== undefined){ data16 = coerced12; if(data !== undefined){ -data["screenshotTimeout"] = coerced12; +data["onlySameHost"] = coerced12; } } } } -if(data.maxAnimationFrames !== undefined){ -let data17 = data.maxAnimationFrames; -if(!(typeof data17 == "number")){ +if(data.useHostHeader !== undefined){ +let data17 = data.useHostHeader; +if(typeof data17 !== "boolean"){ let dataType13 = typeof data17; let coerced13 = undefined; if(dataType13 == 'object' && Array.isArray(data17) && data17.length == 1){ data17 = data17[0]; dataType13 = typeof data17; -if(typeof data17 == "number"){ +if(typeof data17 === "boolean"){ coerced13 = data17; } } if(!(coerced13 !== undefined)){ -if(dataType13 == "boolean" || data17 === null - || (dataType13 == "string" && data17 && data17 == +data17)){ -coerced13 = +data17; +if(data17 === "false" || data17 === 0 || data17 === null){ +coerced13 = false; +} +else if(data17 === "true" || data17 === 1){ +coerced13 = true; } else { -const err33 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +const err35 = {instancePath:instancePath+"/useHostHeader",schemaPath:"#/properties/useHostHeader/type",keyword:"type",params:{type: "boolean"},message:"must be boolean"}; if(vErrors === null){ -vErrors = [err33]; +vErrors = [err35]; } else { -vErrors.push(err33); +vErrors.push(err35); } errors++; } @@ -756,19 +780,91 @@ errors++; if(coerced13 !== undefined){ data17 = coerced13; if(data !== undefined){ -data["maxAnimationFrames"] = coerced13; +data["useHostHeader"] = coerced13; +} +} } } +if(data.screenshotTimeout !== undefined){ +let data18 = data.screenshotTimeout; +if(!(typeof data18 == "number")){ +let dataType14 = typeof data18; +let coerced14 = undefined; +if(dataType14 == 'object' && Array.isArray(data18) && data18.length == 1){ +data18 = data18[0]; +dataType14 = typeof data18; +if(typeof data18 == "number"){ +coerced14 = data18; } } +if(!(coerced14 !== undefined)){ +if(dataType14 == "boolean" || data18 === null + || (dataType14 == "string" && data18 && data18 == +data18)){ +coerced14 = +data18; } else { -const err34 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +const err36 = {instancePath:instancePath+"/screenshotTimeout",schemaPath:"#/properties/screenshotTimeout/type",keyword:"type",params:{type: "number"},message:"must be number"}; if(vErrors === null){ -vErrors = [err34]; +vErrors = [err36]; } else { -vErrors.push(err34); +vErrors.push(err36); +} +errors++; +} +} +if(coerced14 !== undefined){ +data18 = coerced14; +if(data !== undefined){ +data["screenshotTimeout"] = coerced14; +} +} +} +} +if(data.maxAnimationFrames !== undefined){ +let data19 = data.maxAnimationFrames; +if(!(typeof data19 == "number")){ +let dataType15 = typeof data19; +let coerced15 = undefined; +if(dataType15 == 'object' && Array.isArray(data19) && data19.length == 1){ +data19 = data19[0]; +dataType15 = typeof data19; +if(typeof data19 == "number"){ +coerced15 = data19; +} +} +if(!(coerced15 !== undefined)){ +if(dataType15 == "boolean" || data19 === null + || (dataType15 == "string" && data19 && data19 == +data19)){ +coerced15 = +data19; +} +else { +const err37 = {instancePath:instancePath+"/maxAnimationFrames",schemaPath:"#/properties/maxAnimationFrames/type",keyword:"type",params:{type: "number"},message:"must be number"}; +if(vErrors === null){ +vErrors = [err37]; +} +else { +vErrors.push(err37); +} +errors++; +} +} +if(coerced15 !== undefined){ +data19 = coerced15; +if(data !== undefined){ +data["maxAnimationFrames"] = coerced15; +} +} +} +} +} +else { +const err38 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"}; +if(vErrors === null){ +vErrors = [err38]; +} +else { +vErrors.push(err38); } errors++; } diff --git a/config/type/schema.json b/config/type/schema.json index 74d1abd..a6e24bc 100644 --- a/config/type/schema.json +++ b/config/type/schema.json @@ -17,6 +17,7 @@ "observer", "puppeteerLaunchOptions", "concurrency", + "inspectConcurrency", "defaultLang", "defaultTimezone", "screenshotTimeout", @@ -50,6 +51,9 @@ "properties": { "capture": { "type": "string" + }, + "inspect": { + "type": "string" } } }, @@ -70,6 +74,10 @@ "concurrency": { "type": "integer" }, + "inspectConcurrency": { + "type": "integer", + "minimum": 1 + }, "defaultLang": { "type": "string" }, diff --git a/test-it/inspect.ts b/test-it/inspect.ts new file mode 100644 index 0000000..638570e --- /dev/null +++ b/test-it/inspect.ts @@ -0,0 +1,212 @@ +import assert from 'assert/strict' +import http from 'node:http' +import { promisify } from 'node:util' +import getPixelsCb from 'get-pixels' +import * as server from '../api/server.ts' +import axios from 'axios' +import { describe, before, after, it } from 'node:test' +import config from '#config' + +const getPixels = promisify(getPixelsCb) +const pngSize = async (data: Buffer) => { + const [width, height] = (await getPixels(data, 'image/png')).shape + return { width, height } +} + +const origin = `http://localhost:${config.port}` +const ax = axios.create({ baseURL: `${origin}/api/v1/`, validateStatus: () => true }) +const key = 'inspect' +const inspect = (path: string, params: Record = {}, headers: Record = { 'x-secret-key': key }) => + ax.get('inspect', { params: { target: origin + path, ...params }, headers }) + +describe('inspect', () => { + before(async function () { + await server.start() + }) + after(async () => { + // let the last request release its page before the browser is closed + await new Promise(resolve => setTimeout(resolve, 300)) + await server.stop() + }) + + it('inspects a page with errors', async () => { + const res = await inspect('/test/resources/test-errors.html') + assert.equal(res.status, 200) + assert.equal(res.data.image.mediaType, 'image/png') + assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 1280, height: 800 }) + const r = res.data.report + assert.equal(r.ready, 'triggered') + assert.equal(r.status, 200) + assert.ok(r.consoleErrors.some((e: any) => e.text.includes('boom-console'))) + assert.ok(r.pageErrors.some((e: string) => e.includes('boom-page'))) + assert.deepEqual(r.failedRequests.map((f: any) => [f.path, f.status]), [['/test/resources/missing.json', 404]]) + assert.equal(r.verdict, 'errors') + assert.ok(res.data.text.startsWith('### Page\n- Page URL: ')) + assert.match(res.data.text, /\[ERROR\] boom-console @ /) + assert.match(res.data.text, /\d+\. \[GET\] .*\/test\/resources\/missing\.json => \[404\] Not Found/) + assert.match(res.data.text, /- Verdict: errors/) + }) + + it('snapshot in Playwright syntax', async () => { + const res = await inspect('/test/resources/test-snapshot.html') + assert.equal(res.status, 200) + const lines: string[] = res.data.report.snapshot.split('\n').map((l: string) => l.trim()) + for (const expected of ['- heading "Hello" [level=1]', '- link "Next":', '- /url: /next', '- paragraph: Some text', '- checkbox "Agree" [checked]']) { + assert.ok(lines.includes(expected), `missing line ${expected} in\n${res.data.report.snapshot}`) + } + const frameIndex = res.data.report.snapshot.split('\n').findIndex((l: string) => l.trim().startsWith('- iframe')) + assert.ok(frameIndex >= 0, 'missing iframe line') + const all = res.data.report.snapshot.split('\n') + const frameIndent = all[frameIndex].search(/\S/) + assert.ok(all.slice(frameIndex + 1).some((l: string) => l.search(/\S/) > frameIndent && l.includes('Test page 1'))) + assert.ok(!res.data.report.snapshot.includes('[ref=')) + assert.ok(res.data.text.includes('### Snapshot\n```yaml\n')) + }) + + it('truncates a big snapshot', async () => { + const res = await inspect('/test/resources/test-big.html') + assert.equal(res.status, 200) + const snapshot: string = res.data.report.snapshot + assert.ok(snapshot.length <= 20000) + assert.match(snapshot.split('\n').at(-1)!, /^# snapshot truncated: \d+ more lines$/) + }) + + it('mobile viewport', async () => { + const res = await inspect('/test/resources/test-snapshot.html', { mobile: 'true' }) + assert.equal(res.status, 200) + assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 390, height: 844 }) + }) + + it('busy page times out but still answers', async () => { + const res = await inspect('/test/resources/test-busy.html') + assert.equal(res.status, 200) + assert.equal(res.data.report.ready, 'timeout') + assert.equal(res.data.report.verdict, 'timeout') + assert.ok(res.data.image.data.length > 0) + assert.match(res.data.text, /- Ready: timeout after \d+ ms/) + }) + + it('login redirect is unauthenticated', async () => { + const res = await inspect('/test/resources/test-login-redirect.html') + assert.equal(res.status, 200) + assert.equal(res.data.report.verdict, 'unauthenticated') + assert.ok(new URL(res.data.report.finalUrl).pathname.startsWith('/simple-directory/login')) + }) + + it('reports cross-host frames', async () => { + const res = await inspect('/test/resources/test-frame.html') + assert.equal(res.status, 200) + assert.deepEqual(res.data.report.blockedFrames, [`http://127.0.0.1:${config.port}`]) + assert.ok(!res.data.report.snapshot.includes('Test page 1'), 'cross-host frame content must not be in the snapshot') + assert.equal(res.data.report.verdict, 'errors') + assert.match(res.data.text, /\[BLOCKED\] cross-host frame not rendered/) + }) + + it('a healthy page is ok, without console noise', async () => { + const res = await inspect('/test/resources/test-clean.html') + assert.equal(res.status, 200) + assert.equal(res.data.report.verdict, 'ok', res.data.text) + assert.deepEqual(res.data.report.consoleErrors, []) + }) + + it('the main frame leaving the target host is an error', async () => { + const res = await inspect('/test/resources/test-other-host.html') + assert.equal(res.status, 200) + assert.equal(new URL(res.data.report.finalUrl).hostname, '127.0.0.1') + assert.equal(res.data.report.verdict, 'errors') + assert.match(res.data.text, new RegExp(`- Note: navigated away from localhost:${config.port} to 127\\.0\\.0\\.1:${config.port}\n`)) + }) + + it('one huge text does not empty the snapshot', async () => { + const res = await inspect('/test/resources/test-long-text.html') + assert.equal(res.status, 200) + const snapshot: string = res.data.report.snapshot + assert.ok(snapshot.includes('Head of the page')) + assert.ok(snapshot.includes('After the long text')) + assert.ok(snapshot.length < 5000) + assert.ok(snapshot.includes('…')) + assert.ok(!snapshot.includes('# snapshot truncated')) + }) + + it('bounds the console and network sections', async () => { + const res = await inspect('/test/resources/test-many-errors.html') + assert.equal(res.status, 200) + const text: string = res.data.text + const consoleSection = text.slice(text.indexOf('### Console'), text.indexOf('### Snapshot')) + assert.ok(consoleSection.length <= 8000, `console section is ${consoleSection.length} chars`) + assert.match(consoleSection, /# console and network truncated: \d+ more lines\n$/) + assert.ok(text.length < 28000) + }) + + it('full page is clipped to 3000px', async () => { + const res = await inspect('/test/resources/test-big.html', { fullPage: 'true' }) + assert.equal(res.status, 200) + assert.deepEqual(await pngSize(Buffer.from(res.data.image.data, 'base64')), { width: 1280, height: 3000 }) + }) + + it('reports a declared error without sending it', async () => { + const res = await inspect('/test/resources/test-declared-error.html', { declaredErrorUrl: origin + '/test/app/error' }) + assert.equal(res.status, 200) + assert.deepEqual(res.data.report.declaredErrors, ['dataset column "missing" not found']) + // answered by capture: the 404 the server would give is not there + assert.deepEqual(res.data.report.failedRequests, []) + assert.equal(res.data.report.verdict, 'errors') + assert.match(res.data.text, /### Declared errors\n- dataset column "missing" not found\n/) + }) + + it('without declaredErrorUrl the error post is a plain request', async () => { + const res = await inspect('/test/resources/test-declared-error.html') + assert.equal(res.status, 200) + assert.deepEqual(res.data.report.declaredErrors, []) + assert.deepEqual(res.data.report.failedRequests.map((f: any) => [f.method, f.path, f.status]), [['POST', '/test/app/error', 404]]) + }) + + it('refuses a declaredErrorUrl on another host', async () => { + const res = await inspect('/test/resources/test-declared-error.html', { declaredErrorUrl: 'http://example.com/error' }) + assert.equal(res.status, 400) + }) + + it('refuses without the secret', async () => { + assert.equal((await inspect('/test/resources/test1.html', {}, {})).status, 401) + assert.equal((await inspect('/test/resources/test1.html', {}, { 'x-secret-key': 'wrong' })).status, 401) + // the secret of /screenshot and /print does not open /inspect + assert.equal((await inspect('/test/resources/test1.html', {}, { 'x-secret-key': 'capture' })).status, 401) + }) + + it('refuses a forwarded request', async () => { + const res = await inspect('/test/resources/test1.html', {}, { 'x-secret-key': key, 'x-forwarded-host': 'example.com' }) + assert.equal(res.status, 421) + }) + + it('refuses a non http target', async () => { + const res = await ax.get('inspect', { params: { target: 'file:///etc/passwd' }, headers: { 'x-secret-key': key } }) + assert.equal(res.status, 400) + }) + + it('cuts long bodies', async () => { + // a page served by a separate server so that it can answer a long 404 body, the Host header + // makes capture consider it as its own host + const other = http.createServer((req, res) => { + if (req.url === '/page') { + res.setHeader('content-type', 'text/html') + res.end('') + } else { + res.statusCode = 404 + res.setHeader('content-type', 'text/plain') + res.end('x'.repeat(5000)) + } + }) + await new Promise(resolve => other.listen(0, '127.0.0.1', resolve)) + const port = (other.address() as any).port + try { + const host = `127.0.0.1:${port}` + const res = await ax.get('inspect', { params: { target: `http://${host}/page` }, headers: { 'x-secret-key': key, host } }) + assert.equal(res.status, 200) + const failed = res.data.report.failedRequests.find((f: any) => f.path === '/long') + assert.ok(failed.body.length > 0) + assert.ok(failed.body.length <= 300) + } finally { + other.close() + } + }) +}) diff --git a/test-it/resources/test-big.html b/test-it/resources/test-big.html new file mode 100644 index 0000000..da5b29a --- /dev/null +++ b/test-it/resources/test-big.html @@ -0,0 +1,19 @@ + + + + + Big page + + +
    + + + diff --git a/test-it/resources/test-busy.html b/test-it/resources/test-busy.html new file mode 100644 index 0000000..db2e571 --- /dev/null +++ b/test-it/resources/test-busy.html @@ -0,0 +1,14 @@ + + + + + Busy page + + +

    Busy page

    + + + diff --git a/test-it/resources/test-clean.html b/test-it/resources/test-clean.html new file mode 100644 index 0000000..c7b6158 --- /dev/null +++ b/test-it/resources/test-clean.html @@ -0,0 +1,10 @@ + + + + + Clean page + + +

    Clean page

    + + diff --git a/test-it/resources/test-declared-error.html b/test-it/resources/test-declared-error.html new file mode 100644 index 0000000..2251c71 --- /dev/null +++ b/test-it/resources/test-declared-error.html @@ -0,0 +1,15 @@ + + + + + Declared error page + + +

    Declared error page

    + + + diff --git a/test-it/resources/test-errors.html b/test-it/resources/test-errors.html new file mode 100644 index 0000000..1a786cc --- /dev/null +++ b/test-it/resources/test-errors.html @@ -0,0 +1,17 @@ + + + + + Errors page + + +

    Errors page

    + + + diff --git a/test-it/resources/test-frame.html b/test-it/resources/test-frame.html new file mode 100644 index 0000000..d948812 --- /dev/null +++ b/test-it/resources/test-frame.html @@ -0,0 +1,17 @@ + + + + + Cross-host frame + + +

    Cross-host frame

    + + + diff --git a/test-it/resources/test-login-redirect.html b/test-it/resources/test-login-redirect.html new file mode 100644 index 0000000..01c0645 --- /dev/null +++ b/test-it/resources/test-login-redirect.html @@ -0,0 +1,12 @@ + + + + + Login redirect + + + + + diff --git a/test-it/resources/test-long-text.html b/test-it/resources/test-long-text.html new file mode 100644 index 0000000..4fe35a0 --- /dev/null +++ b/test-it/resources/test-long-text.html @@ -0,0 +1,15 @@ + + + + + Long text + + +

    Head of the page

    +

    +

    After the long text

    + + + diff --git a/test-it/resources/test-many-errors.html b/test-it/resources/test-many-errors.html new file mode 100644 index 0000000..48e6b26 --- /dev/null +++ b/test-it/resources/test-many-errors.html @@ -0,0 +1,13 @@ + + + + + Many errors + + +

    Many errors

    + + + diff --git a/test-it/resources/test-other-host.html b/test-it/resources/test-other-host.html new file mode 100644 index 0000000..3d7164c --- /dev/null +++ b/test-it/resources/test-other-host.html @@ -0,0 +1,12 @@ + + + + + Other host redirect + + + + + diff --git a/test-it/resources/test-snapshot.html b/test-it/resources/test-snapshot.html new file mode 100644 index 0000000..b8b6a63 --- /dev/null +++ b/test-it/resources/test-snapshot.html @@ -0,0 +1,19 @@ + + + + + Snapshot page + + +

    Hello

    + Next +

    Some text

    + + + + + diff --git a/test-it/semaphore.ts b/test-it/semaphore.ts new file mode 100644 index 0000000..2a4e28b --- /dev/null +++ b/test-it/semaphore.ts @@ -0,0 +1,52 @@ +import assert from 'assert/strict' +import { describe, it } from 'node:test' +import { createSemaphore } from '../api/utils/semaphore.ts' + +const deferred = () => { + let resolve!: (value: T) => void + let reject!: (err: any) => void + const promise = new Promise((_resolve, _reject) => { resolve = _resolve; reject = _reject }) + return { promise, resolve, reject } +} + +// let pending microtasks and already-queued continuations run +const tick = () => new Promise(resolve => setImmediate(resolve)) + +describe('semaphore', () => { + it('does not start the second fn before the first settles when max is 1', async () => { + const sem = createSemaphore(1) + const first = deferred() + const second = deferred() + let secondStarted = false + + const p1 = sem.run(() => first.promise) + const p2 = sem.run(async () => { + secondStarted = true + return second.promise + }) + await tick() + assert.equal(secondStarted, false) + assert.equal(sem.pending(), 1) + + first.resolve('a') + assert.equal(await p1, 'a') + await tick() + assert.equal(secondStarted, true) + assert.equal(sem.pending(), 0) + + second.resolve('b') + assert.equal(await p2, 'b') + }) + + it('releases the slot when the first fn rejects', async () => { + const sem = createSemaphore(1) + const failing = deferred() + const p1 = sem.run(() => failing.promise) + const p2 = sem.run(async () => 'next') + + failing.reject(new Error('boom')) + await assert.rejects(p1, /boom/) + assert.equal(await p2, 'next') + assert.equal(sem.pending(), 0) + }) +})