Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the dev environment.

Build and run the docker image:

docker buildx build -t capture --progress plain --load . && docker run --rm -it --security-opt seccomp=$(pwd)/seccomp.json -p 5607:5607 -p 9090:9090 -e DEBUG=capture,timer -e ONLY_SAME_HOST=false -e SECRET_CAPTURE=capture -e HELMET_ACTIVE=false -e PORT=5607 --name capture capture
docker buildx build -t capture --progress plain --load . && docker run --rm -it --security-opt seccomp=$(pwd)/seccomp.json -p 5607:5607 -p 9090:9090 -e DEBUG=capture,timer -e ONLY_SAME_HOST=false -e SECRET_CAPTURE=capture -e SECRET_INSPECT=inspect -e HELMET_ACTIVE=false -e PORT=5607 --name capture capture

Check the service with these examples (in the dev environment, replace 5607 with `DEV_API_PORT` from `.env`):

Expand All @@ -36,8 +36,18 @@ Check the service with these examples (in the dev environment, replace 5607 with
- [animated gif screenshot](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&target=http://localhost:5607/test/resources/test-anim.html)
- [animated gif screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test-anim.html)
- [fallback to standard screenshot with custom filename](http://localhost:5607/api/v1/screenshot?key=capture&type=gif&filename=test.gif&target=http://localhost:5607/test/resources/test1.html)
- [inspect: screenshot and render report as JSON (internal use)](http://localhost:5607/api/v1/inspect?key=inspect&target=http://localhost:5607/test/resources/test-errors.html)
- [inspect with an error declared by the page](http://localhost:5607/api/v1/inspect?key=inspect&target=http://localhost:5607/test/resources/test-declared-error.html&declaredErrorUrl=http://localhost:5607/test/app/error), `declaredErrorUrl` (same host as `target`, query ignored) receives the page's POST requests: capture answers them with 204 instead of sending them, and reports their `message` under `declaredErrors`
- [remote url with a webgl based map](http://localhost:5617/api/v1/print?key=capture&target=https://staging-koumoul.com/cadastre/parcel?id=56251000AV0142)

## Configuration

Configuration is in `config/` and can be overridden with environment variables, see `config/custom-environment-variables.cjs`. Notably:

- `SECRET_CAPTURE` (`secretKeys.capture`, default none): when set, `/screenshot` and `/print` require it in `x-secret-key` from internal callers without a session. Leave it unset while browsers or other services call them directly.
- `SECRET_INSPECT` (`secretKeys.inspect`, default none): required in `x-secret-key` on `/inspect` (internal requests only), which is disabled without it. A secret of its own, so that enabling `/inspect` does not lock `/screenshot` and `/print`.
- `INSPECT_CONCURRENCY` (`inspectConcurrency`, default 1): how many `/inspect` renders may run at the same time, each one holds a browser page; further requests wait for a free slot.
=======
## Logging

Logs go through the logger shared by the data-fair stack ([`@data-fair/lib-node/log.js`](https://github.com/data-fair/lib/tree/main/packages/node#logging)): one JSON object per line on stdout when `NODE_ENV=production` (the docker image), pretty output otherwise, `LOG_FORMAT=json|pretty` overrides. Warn, alert and error lines are counted in the `df_logs{level, code, host}` metric. `DEBUG=capture,timer` adds the debug lines of the capture steps and their durations.
Expand Down
92 changes: 91 additions & 1 deletion api/routers/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,18 @@ import * as headerFooter from '../utils/header-footer.ts'
import * as pageUtils from '../utils/page.ts'
import * as animationUtils from '../utils/animation.ts'
import { createTimer } from '../utils/timer.ts'
import { createSemaphore } from '../utils/semaphore.ts'
import { createReportCollector } from '../utils/report.ts'
import { reportText } from '../utils/report-text.ts'
import config from '#config'
import { assertReqInternalSecret, httpError, reqHost, reqIsInternal, reqSession } from '@data-fair/lib-express'
import type { PaperFormat, PDFOptions } from 'puppeteer'

const debug = debugModule('capture')

// /inspect renders are expensive, a small number of them run at the same time
const inspectSemaphore = createSemaphore(config.inspectConcurrency)

export const router = express.Router()

async function auth (req: Request, res: Response, next: NextFunction) {
Expand Down Expand Up @@ -50,7 +56,7 @@ async function auth (req: Request, res: Response, next: NextFunction) {

if (securedSameHost && req.cookies && Object.keys(req.cookies).length && req.query.cookies !== 'false') {
debug(`[${target}] transmit cookies`)
req.cookies = Object.keys(req.cookies).map(name => ({ name, value: req.cookies[name], url: target }))
req.cookies = Object.keys(req.cookies).map(name => ({ name, value: req.cookies[name], url: target })) as any
} else {
debug(`[${target}] do not transmit cookies`)
req.cookies = []
Expand Down Expand Up @@ -189,3 +195,87 @@ router.get('/print', auth, async (req, res, next) => {
)
timer.finish()
})

const desktopViewport = { width: 1280, height: 800 }
const mobileViewport = { width: 390, height: 844, isMobile: true, hasTouch: true }
const maxFullPageHeight = 3000

const viewportDimension = (value: unknown, fallback: number) => {
if (value === undefined) return fallback
const parsed = typeof value === 'string' ? parseInt(value) : NaN
if (!(parsed > 0 && parsed <= 3000)) throw httpError(400, 'invalid width or height')
return parsed
}

// internal only: used by data-fair and portals so that agents can see what they configured
router.get('/inspect', async (req, res) => {
// a secret of its own: setting it does not lock /screenshot and /print, that public clients call
if (!config.secretKeys.inspect) throw httpError(501, 'inspect is disabled, it needs SECRET_INSPECT')
assertReqInternalSecret(req, config.secretKeys.inspect)

const target = req.query.target
if (typeof target !== 'string') throw httpError(400, 'parameter "target" is required')
let targetUrl
try {
targetUrl = new URL(target)
} catch (err: any) {
throw httpError(400, 'Failed to parse target url ' + err.message)
}
if (targetUrl.protocol !== 'http:' && targetUrl.protocol !== 'https:') throw httpError(400, 'target must be an http or https url')
debug(`[${target}] inspect`)

const lang = req.query.lang
if (lang && typeof lang !== 'string') throw httpError(400, 'invalid lang')
const timezone = req.query.timezone
if (timezone && typeof timezone !== 'string') throw httpError(400, 'invalid timezone')

const mobile = req.query.mobile === 'true'
const fullPage = req.query.fullPage === 'true'
const viewport = mobile
? mobileViewport
: { width: viewportDimension(req.query.width, desktopViewport.width), height: viewportDimension(req.query.height, desktopViewport.height) }

// the page may POST its own error diagnosis there, it is reported instead of being sent
const declaredErrorUrl = req.query.declaredErrorUrl
if (declaredErrorUrl !== undefined) {
if (typeof declaredErrorUrl !== 'string' || !URL.canParse(declaredErrorUrl)) throw httpError(400, 'invalid declaredErrorUrl')
if (!pageUtils.isSameHost(declaredErrorUrl, target)) throw httpError(400, 'declaredErrorUrl must be on the host of target')
}

const cookies = Object.keys(req.cookies ?? {}).map(name => ({ name, value: req.cookies[name], url: target })) as any
const timer = createTimer(req.originalUrl, 'inspect')

let collector: ReturnType<typeof createReportCollector> | undefined
await inspectSemaphore.run(async () => {
// the client may have left while waiting for a slot, no need to render for nobody
if (req.socket.destroyed || res.destroyed) return
await pageUtils.withPage(
target,
lang,
timezone,
cookies,
viewport,
false,
timer,
`Failed to inspect page "${target}" before timeout`,
async ({ page, ready }) => {
const report = await collector!.finish(ready)
let buffer
if (fullPage) {
const height = await page.evaluate(() => document.documentElement.scrollHeight)
buffer = await page.screenshot({ captureBeyondViewport: true, clip: { x: 0, y: 0, width: viewport.width, height: Math.min(height, maxFullPageHeight) } })
} else {
buffer = await page.screenshot()
}
timer.type = 'inspect'
res.json({ image: { data: Buffer.from(buffer).toString('base64'), mediaType: 'image/png' }, text: reportText(report), report })
},
{
onPage: page => { collector = createReportCollector(page, target) },
declaredErrorUrl,
onDeclaredError: body => collector?.declareError(body)
}
)
})
timer.finish()
})
196 changes: 196 additions & 0 deletions api/utils/aria-snapshot.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
import type { Page, SerializedAXNode } from 'puppeteer'

/* eslint-disable no-control-regex */
// yamlEscapeKeyIfNeeded and yamlEscapeValueIfNeeded are copied from Playwright
// https://github.com/microsoft/playwright/blob/main/packages/isomorphic/yaml.ts
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export function yamlEscapeKeyIfNeeded (str: string): string {
if (!yamlStringNeedsQuotes(str)) { return str }
return '\'' + str.replace(/'/g, '\'\'') + '\''
}

export function yamlEscapeValueIfNeeded (str: string): string {
if (!yamlStringNeedsQuotes(str)) { return str }
return '"' + str.replace(/[\\"\x00-\x1f\x7f-\x9f]/g, c => {
switch (c) {
case '\\':
return '\\\\'
case '"':
return '\\"'
case '\b':
return '\\b'
case '\f':
return '\\f'
case '\n':
return '\\n'
case '\r':
return '\\r'
case '\t':
return '\\t'
default: {
const code = c.charCodeAt(0)
return '\\x' + code.toString(16).padStart(2, '0')
}
}
}) + '"'
}

function yamlStringNeedsQuotes (str: string): boolean {
if (str.length === 0) { return true }

// Strings with leading or trailing whitespace need quotes
if (/^\s|\s$/.test(str)) { return true }

// Strings containing control characters need quotes
if (/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]/.test(str)) { return true }

// Strings starting with '-' need quotes
if (/^-/.test(str)) { return true }

// Strings containing ':' or '\n' followed by a space or at the end need quotes
if (/[\n:](\s|$)/.test(str)) { return true }

// Strings containing '#' preceded by a space need quotes (comment indicator)
if (/\s#/.test(str)) { return true }

// Strings that contain line breaks need quotes
if (/[\n\r]/.test(str)) { return true }

// Strings starting with indicator characters or quotes need quotes
if (/^[&*\],?!>|@"'#%]/.test(str)) { return true }

// Strings containing special characters that could cause ambiguity
if (/[{}`]/.test(str)) { return true }

// YAML array starts with [
if (/^\[/.test(str)) { return true }

// Non-string types recognized by YAML
if (!isNaN(Number(str)) || ['y', 'n', 'yes', 'no', 'true', 'false', 'on', 'off', 'null', '~'].includes(str.toLowerCase())) { return true }

// YAML special float values: .inf, -.inf, .nan and their case variants
if (/^[-+]?\.(inf|nan)$/i.test(str)) { return true }

return false
}

/* eslint-enable no-control-regex */

const indent = (depth: number) => ' '.repeat(depth)

const createKey = (node: SerializedAXNode) => {
let key = node.role === 'Iframe' ? 'iframe' : node.role
// YAML has a limit of 1024 characters per key, leave room for the role and attributes
const name = text(node)
if (name && name.length <= 900) key += ' ' + JSON.stringify(name)
if (node.checked === 'mixed') key += ' [checked=mixed]'
if (node.checked === true) key += ' [checked]'
if (node.disabled) key += ' [disabled]'
if (node.expanded) key += ' [expanded]'
if (node.role === 'heading' && node.level) key += ` [level=${node.level}]`
if (node.pressed === 'mixed') key += ' [pressed=mixed]'
if (node.pressed === true) key += ' [pressed]'
if (node.selected) key += ' [selected]'
if (node.invalid && node.invalid !== 'false') key += ' [invalid]'
return key
}

const isText = (node: SerializedAXNode) => node.role === 'StaticText'
const MAX_TEXT = 2000
// one huge value must not use the whole snapshot budget
const capText = (str: string) => str.length > MAX_TEXT ? str.slice(0, MAX_TEXT - 1) + '…' : str
const text = (node: SerializedAXNode) => capText((node.name ?? '').replace(/\s+/g, ' ').trim())

// interestingOnly would drop useful containers (paragraph, list, listitem), so the whole tree is
// requested and its structural noise is flattened here: children of unnamed none/generic nodes and
// of nested documents (iframes) are hoisted, text boxes and list markers are dropped
const flatten = (nodes: SerializedAXNode[] | undefined): SerializedAXNode[] => {
const result: SerializedAXNode[] = []
for (const node of nodes ?? []) {
if (node.role === 'InlineTextBox' || node.role === 'ListMarker') continue
if (node.role === 'RootWebArea' || ((node.role === 'none' || node.role === 'generic') && !node.name)) result.push(...flatten(node.children))
else result.push({ ...node, children: flatten(node.children) })
}
return result
}

export const ariaSnapshot = async (page: Page, opts: { maxChars: number, maxLinks: number }): Promise<string> => {
const root = await page.accessibility.snapshot({ interestingOnly: false, includeIframes: true })
if (!root) return ''
const lines: string[] = []
let links = 0

const visitText = (text: string, depth: number) => {
const escaped = yamlEscapeValueIfNeeded(text)
if (escaped) lines.push(indent(depth) + '- text: ' + escaped)
}

const visit = async (node: SerializedAXNode, depth: number) => {
if (isText(node)) return visitText(text(node), depth)
const key = indent(depth) + '- ' + yamlEscapeKeyIfNeeded(createKey(node))

let children = node.children ?? []
// children texts that only repeat the name are redundant
if (text(node) && children.length && children.every(isText) && text({ name: children.map(c => c.name ?? '').join('') } as SerializedAXNode) === text(node)) children = []

let href: string | undefined
if (node.role === 'link' && links < opts.maxLinks) {
links++
try {
const handle = await node.elementHandle()
href = (await handle?.evaluate(el => el.getAttribute('href'))) ?? node.url
await handle?.dispose()
} catch (err) {
href = node.url
}
}

let inlineText: string | undefined
if ((node.role === 'textbox' || node.role === 'combobox') && node.value !== undefined && node.value !== '') inlineText = capText(String(node.value))
else if (!text(node) && children.length === 1 && isText(children[0]) && !href) {
inlineText = text(children[0])
children = []
}

if (inlineText !== undefined && !href && !children.length) {
lines.push(key + ': ' + yamlEscapeValueIfNeeded(inlineText))
} else if (!href && !children.length && inlineText === undefined) {
lines.push(key)
} else {
lines.push(key + ':')
if (href) lines.push(indent(depth + 1) + '- /url: ' + yamlEscapeValueIfNeeded(href))
if (inlineText !== undefined) visitText(inlineText, depth + 1)
for (const child of children) await visit(child, depth + 1)
}
}

for (const child of flatten(root.children)) await visit(child, 0)

// truncate at a line boundary, keeping room for the last line
const reserve = 50
let size = 0
let kept = 0
for (const line of lines) {
if (size + line.length + 1 > opts.maxChars - reserve) break
size += line.length + 1
kept++
}
if (kept < lines.length) {
return lines.slice(0, kept).concat(`# snapshot truncated: ${lines.length - kept} more lines`).join('\n')
}
return lines.join('\n')
}
Loading
Loading