diff --git a/.github/workflows/live-tests.yml b/.github/workflows/live-tests.yml new file mode 100644 index 0000000..635d12a --- /dev/null +++ b/.github/workflows/live-tests.yml @@ -0,0 +1,35 @@ +name: Live tests + +# Hits the real Diffbot APIs with a repository secret. Deliberately NOT triggered by +# pull_request / pull_request_target (including forks); never add those triggers. +on: + push: + branches: [main] + workflow_dispatch: + schedule: + - cron: "17 14 1,15 * *" + +permissions: + contents: read + +jobs: + live: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 20 + concurrency: + group: live-tests + cancel-in-progress: false + env: + DIFFBOT_TOKEN: ${{ secrets.DIFFBOT_TOKEN }} + # The library's resolver reads DIFFBOT_API_TOKEN; same secret, never echoed. + DIFFBOT_API_TOKEN: ${{ secrets.DIFFBOT_TOKEN }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm test:live diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..dee31ac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,220 @@ +name: Release + +# Publishes @diffbot/typescript to npm (with provenance) and creates the matching +# GitHub Release. Triggered by pushing a version tag (e.g. v0.6.0); also runnable +# manually, which runs every gate plus `npm publish --dry-run` and publishes nothing. +# +# npm auth: trusted publishing (OIDC) first. If no trusted publisher is configured on +# npmjs.com, npm falls back to the NPM_TOKEN repository secret (if set). +# Live tests are deliberately NOT run here; see live-tests.yml. +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: read + +# Re-pushing a tag (or re-dispatching) queues behind an in-flight run for the same +# ref. Unlike a binaries build, a publish must never be cancelled halfway. +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + verify: + name: Verify version + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + # Fail fast if package.json and src/constants.ts disagree, or the tag doesn't + # match them, so npm, the User-Agent (VERSION) and the GitHub Release can never + # report different versions. + - name: Check tag matches package version + id: version + run: | + set -euo pipefail + pkg=$(node -p 'require("./package.json").version') + const=$(sed -n 's/^export const VERSION = "\([^"]*\)";.*/\1/p' src/constants.ts) + if [ -z "$const" ]; then + echo "::error::Could not read VERSION from src/constants.ts." + exit 1 + fi + if [ "$pkg" != "$const" ]; then + echo "::error::package.json version ${pkg} does not match src/constants.ts VERSION ${const}." + exit 1 + fi + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + tag=${GITHUB_REF_NAME#v} + if [ "$pkg" != "$tag" ]; then + echo "::error::Tag v${tag} does not match package.json version ${pkg}. Bump package.json and src/constants.ts, then re-tag." + exit 1 + fi + fi + echo "version=${pkg}" >> "$GITHUB_OUTPUT" + echo "Versions agree: ${pkg}" + + # Checked before publishing so a tag without release notes fails before + # anything reaches npm. + - name: Check CHANGELOG.md has a section for this version + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + if ! grep -qF "## [${VERSION}]" CHANGELOG.md; then + echo "::error::CHANGELOG.md has no '## [${VERSION}]' section." + exit 1 + fi + + test: + name: Gates (Node ${{ matrix.node-version }}) + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + matrix: + node-version: [18, 20, 22] + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm typecheck + - run: pnpm lint + - run: pnpm test + - run: pnpm test:types + # `build` runs the `postbuild` check (scripts/assert-no-node-builtins.mjs). + - run: pnpm build + + workers: + name: cloudflare worker fixture + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm build + - run: pnpm install --frozen-lockfile + working-directory: fixtures/worker + - run: pnpm exec wrangler deploy --dry-run --outdir .wrangler-build + working-directory: fixtures/worker + - run: pnpm exec vitest run + working-directory: fixtures/worker + + publish: + name: Publish to npm + needs: [verify, test, workers] + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write # npm trusted publishing (OIDC) and provenance + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + registry-url: https://registry.npmjs.org + # Trusted publishing needs npm >= 11.5.1; Node 22 bundles npm 10. + - run: npm install -g npm@^11.5.1 + - run: pnpm install --frozen-lockfile + - run: pnpm build + + - name: Publish + env: + VERSION: ${{ needs.verify.outputs.version }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + set -euo pipefail + name=$(node -p 'require("./package.json").name') + # Prereleases (v1.2.0-rc.1) go to the "next" dist-tag, never "latest". + dist_tag=latest + case "$VERSION" in *-*) dist_tag=next ;; esac + + # Only a tag push publishes. A manual dispatch (even on a tag ref) is a dry run. + if [ "${GITHUB_EVENT_NAME}" != "push" ] || [ "${GITHUB_REF_TYPE}" != "tag" ]; then + echo "Not a tag push: dry run only." + npm publish --dry-run --access public --tag "$dist_tag" + exit 0 + fi + + # Lets a failed run be re-run after the npm step already succeeded. + if npm view "${name}@${VERSION}" version >/dev/null 2>&1; then + echo "${name}@${VERSION} is already on npm; skipping publish." + exit 0 + fi + + # npm tries OIDC trusted publishing first and only falls back to this + # token when no trusted publisher matches. Never echo it. + if [ -n "${NPM_TOKEN}" ]; then + export NODE_AUTH_TOKEN="${NPM_TOKEN}" + echo "NPM_TOKEN secret present (used only if trusted publishing is not configured)." + else + echo "No NPM_TOKEN secret; relying on npm trusted publishing." + fi + npm publish --access public --provenance --tag "$dist_tag" + + release: + name: Publish GitHub Release + needs: [verify, publish] + runs-on: ubuntu-latest + timeout-minutes: 5 + # Only for real tag pushes; workflow_dispatch runs just the gates + dry run. + if: github.event_name == 'push' && github.ref_type == 'tag' + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Extract release notes from CHANGELOG.md + env: + VERSION: ${{ needs.verify.outputs.version }} + run: | + set -euo pipefail + # Everything after "## [VERSION]" up to the next "## " heading. + awk -v hdr="## [${VERSION}]" ' + index($0, hdr) == 1 { found = 1; next } + found && /^## / { exit } + found { print } + ' CHANGELOG.md > "${RUNNER_TEMP}/notes.md" + if ! grep -q '[^[:space:]]' "${RUNNER_TEMP}/notes.md"; then + echo "::error::CHANGELOG.md section for ${VERSION} is empty." + exit 1 + fi + wc -l "${RUNNER_TEMP}/notes.md" + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + VERSION: ${{ needs.verify.outputs.version }} + run: | + set -euo pipefail + tag="${GITHUB_REF_NAME}" + repo="${GITHUB_REPOSITORY}" + if gh release view "$tag" --repo "$repo" >/dev/null 2>&1; then + echo "Release $tag already exists; leaving its notes as they are." + exit 0 + fi + prerelease=() + case "$VERSION" in *-*) prerelease=(--prerelease) ;; esac + gh release create "$tag" \ + --repo "$repo" \ + --title "$tag" \ + --verify-tag \ + --notes-file "${RUNNER_TEMP}/notes.md" \ + "${prerelease[@]}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3dfb481..1f27712 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,7 +21,9 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - run: pnpm typecheck + - run: pnpm lint - run: pnpm test + - run: pnpm test:types - run: pnpm build workers: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e6c4c5e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,153 @@ +# Changelog + +All notable changes to `@diffbot/typescript` are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses +[Semantic Versioning](https://semver.org/) (pre-1.0: a minor bump may break). + +## [0.6.0] - 2026-11-17 + +First release since 0.3.0. It rolls up three development lines that were never +published on their own: the shared transport, full typing and DQL over POST (planned +as 0.4.0), typed completions plus `account`/`verifyToken`/`crawlDownload`/extract +options (0.5.0), and parameter completion on existing endpoints (0.6.0). No new +Diffbot API areas are added. + +### ⚠ Breaking changes + +- **DQL is sent as `POST` with a JSON body.** `dql()` / `dqlParallel()` keep the same + signature and results, but mocks, proxies, or mirrors keyed on `GET` + `?query=` stop + matching. See [Migrating DQL](#migrating-dql-mocks-proxies-mirrors) below. +- **`crawl()` throws `APIError` when the create call answers HTTP 200 with an + `errorCode` body** (for example `505`, job name over 30 characters). It used to yield + `JobCreated` for a job that was never created. +- **A `Retry-After` above `maxDelayMs` (default 5000 ms) is no longer waited out or + retried.** The call fails at once with `RateLimitError` (`retryAfter` = raw header), + or `APIError` with `headers["retry-after"]` for a 5xx. Raise `retry.maxDelayMs` to + keep waiting. +- **`crawlGetJob()` returns `undefined` instead of `{}` for a missing job** (runtime + change, not only a type change). +- **`dql()` throws `ValidationError` when `extra` repeats a named option** (including + the new `type`, `col`, `filterExclude`, ...). Code that passed `extra: { type: "text" }` + must use the named `type` option. +- **Requests are now retried by default** (`429`, `5xx`, connection errors; 3 attempts + with backoff and jitter). `crawlDeleteJob` is never retried. Pass `retry: false` for + the old single-attempt behavior. +- **Type-level:** handlers return specific response types instead of `JsonObject`; + `ChatMessage.role` narrows to `"system" | "user" | "assistant"`; a built-in extract + API name with an option that API lacks is now a type error. Data shapes at runtime + are unchanged. + +### Added + +- **Transport:** per-host base URLs (`apiBase`, `kgBase`, `llmBase`, `nlpBase`; the + older `*Url` options still win when both are set); repeated query params (array + values); `HttpClient.delete()`; raw `body` / `contentType` on every request + (`URLSearchParams` is form-encoded); a per-call `timeout`; the `retry` policy with + per-call `retryable: false`. +- **Errors:** `NotFoundError` (404) and `JobError`, both subclasses of `APIError`. Error + response headers are exposed on `APIError.headers`. +- **Typing:** vendored OpenAPI specs under `openapi/`, `pnpm gen:types` generating + `src/generated/*`, hand-written facade types in `src/types/*`, real `dql()` + overloads, and type tests. `typescript-eslint` strict-type-checked runs on `src/`. +- `account({ days })` and `verifyToken()` over `GET /v4/account` (`AccountResponse`). + `verifyToken()` returns `false` only on `AuthError`. The account body contains the + token and `childTokens`; don't log it. +- `crawlDownload(client, name, { format?, type?, num? })` over `GET /v3/crawl/data`: + `format: "csv"` and the CSV-only `type: "urls"` return text, otherwise + `CrawlDataRecord[]`. The `crawl()` watcher uses it. For tools that can't hold a watch + loop: `crawl(..., { watch: false })`, poll `crawlGetJob`, then `crawlDownload`. +- `extract()` optional fifth argument `ExtractOptions`: `timeout` (server-side, default + 30000), `mode`, per-API typed `fields`, `fallback`, `discussion`, `paging`, + `maxPages`, `naturalLanguage`, `summaryNumSentences`, `maxTags`, `tagConfidence`, + `categoryConfidence`, `useProxy`; `proxy`/`proxyAuth` via typed `extra`. +- `dql()` / `dqlParallel()` named options `type` (`"query" | "text" | + "queryTextFallback" | "crawl"`), `col`, `filterExclude`, `exportseparator`, + `exportfile`, `jsonmode`, `nonCanonicalFacts`, `noDedupArticles`, `cluster` (sent in + the POST body). `report` and `view` stay reachable through `extra`. +- `crawl()` create options `urlCrawlRegEx`, `urlProcessRegEx`, `pageProcessPattern`, + `maxToCrawlPerSubdomain`, `maxToProcessPerSubdomain`, `useCanonical`, + `notifyWebhook`, `notifyEmail`; `customHeaders` accepts `string | readonly string[]` + (repeated params). +- `nlp(client, text, { fields, lang?, format?, documentType?, customSummary? })` on the + existing `POST nl.diffbot.com/v1/` endpoint (single document; result typed as + `Pick`). `entities()` is unchanged. Exports `NlpOptions`, + `NlpResponse`. +- `webSearch()` `text` accepts a string or 1-5 strings (repeated `text=`); 0 or more + than 5 throws `ValidationError` before any request. +- `askJson()` optional `parse` hook for caller-side validation. +- `dql(..., { format: "jsonl" })` streams entities, and `format: "csv"` returns text + (both used to fail in `JSON.parse`). +- Live test workflow (push to `main`, manual dispatch, twice-monthly cron; never on PRs). + +### Changed + +- `dql()` request shape: `POST {dqlUrl}?token=…` with `{ query, size, from?, format?, + filter?, exportspec?, …extra }`. Same defaults (`size` 10, `from` only if non-zero, + `format` only if not `json`). The token stays in the query string; the query text no + longer appears in URLs, and the URL length limit is gone. +- `extract()` keeps its request timeout at least 10 s above the wire `timeout` + (including the 30 s default), so the client no longer aborts just before the server + answers. A larger client-level `timeout` still wins. +- The user agent is now `diffbot-typescript/0.6.0`. + +### Fixed + +- `dql(..., { raw: true })` no longer corrupts binary exports (`xls`/`xlsx`, non-UTF-8 + CSV); the body is read as bytes. +- `crawl()`'s URL-report CSV parser no longer splits a quoted field on an inner comma. +- `crawl()` no longer reports `JobCreated` for a create that failed with HTTP 200 + + `errorCode` (the watch loop then ended silently). +- `APIError.apiMessage` falls back to the body's `error` field, so `{ errorCode, error }` + bodies give a readable message instead of raw JSON. +- A retried attempt's response body is discarded before the retry, so its connection + is released instead of staying pinned until garbage collection. + +### Known limitations + +- `nlp()` requires `fields` (no default); `entities()` remains the shortcut. The + spec's `errors` key is not in the `Pick` type. +- `cluster` is typed `string` (the vendored spec declares it `never`). +- `crawl()` has no `extra`; recurring-crawl admin (`repeat`, `maxRounds`, + `seedRecrawlFrequency`, `onlyProcessIfNew`) is deliberately not exposed. +- `extract()` `extra` collisions with `token`/`url`/`timeout` are caught only by the + compiler, not at runtime. +- `crawlDownload` with `format: "csv"` or `type: "urls"` returns a JSON error body as + text rather than throwing. +- No generic success-path response headers; use `client.http.get/post` for the raw + `Response`. + +### Migrating DQL: mocks, proxies, mirrors + +Anything keyed on `GET` + `?query=` stops matching. Before: + +```ts +fetch: async (input) => { + const url = new URL(String(input)); + if (url.pathname === "/kg/v3/dql" && url.searchParams.get("query")?.includes("Acme")) { + return Response.json({ data: [] }); + } + /* ... */ +}; +``` + +After: + +```ts +fetch: async (input, init) => { + const url = new URL(String(input)); + if (url.pathname === "/kg/v3/dql" && init?.method === "POST") { + const { query } = JSON.parse(String(init.body)) as { query: string }; + if (query.includes("Acme")) return Response.json({ data: [] }); + } + /* ... */ +}; +``` + +GET-only proxies/caches or a custom `dqlUrl` mirror that rejects POST: pin 0.3.x, or +call the endpoint with GET yourself via the existing low-level client: + +```ts +const res = await client.http.get(client.dqlUrl, { + params: { token: client.token, query: "type:Organization", size: 10 }, +}); +``` diff --git a/README.md b/README.md index ae7022a..e6509c8 100644 --- a/README.md +++ b/README.md @@ -46,15 +46,20 @@ settings from the client you hand it. | Option | Default | Used by | |--------|---------|---------| | `token` | — (required) | all | -| `timeout` | `30000` (ms) | all | +| `timeout` | `30000` (ms) | all; overridable per call (see below) | +| `retry` | on, 3 attempts | all; `false` disables, or pass a `RetryPolicy` to tune it | | `fetch` | global `fetch` | all | -| `analyzeUrl` | `https://api.diffbot.com/v3` | `extract` | -| `crawlerUrl` | `https://api.diffbot.com/v3/crawl` | `crawl`, `crawlListJobs`, `crawlGetJob`, `crawlDeleteJob` | -| `llmUrl` | `https://llm.diffbot.com/rag/v1/chat/completions` | `ask`, `askJson` | -| `webSearchUrl` | `https://llm.diffbot.com/api/v1/web_search` | `webSearch` | -| `nlpUrl` | `https://nl.diffbot.com/v1/` | `entities` | -| `dqlUrl` | `https://kg.diffbot.com/kg/v3/dql` | `dql`, `dqlParallel` | -| `ontologyUrl` | `https://kg.diffbot.com/kg/ontology` | `dqlFetchOntology`, `dqlFetchOntologyText`, `OntologyStore` | +| `apiBase` | `https://api.diffbot.com` | `extract`, `account`, `verifyToken`, `crawl`, `crawlListJobs`, `crawlGetJob`, `crawlDownload`, `crawlDeleteJob` | +| `kgBase` | `https://kg.diffbot.com` | `dql`, `dqlParallel`, `dqlFetchOntology`, `dqlFetchOntologyText`, `OntologyStore` | +| `llmBase` | `https://llm.diffbot.com` | `ask`, `askJson`, `webSearch` | +| `nlpBase` | `https://nl.diffbot.com` | `entities` | +| `analyzeUrl` | derived from `apiBase` | `extract` | +| `crawlerUrl` | derived from `apiBase` | `crawl`, `crawlListJobs`, `crawlGetJob`, `crawlDeleteJob` | +| `llmUrl` | derived from `llmBase` | `ask`, `askJson` | +| `webSearchUrl` | derived from `llmBase` | `webSearch` | +| `nlpUrl` | derived from `nlpBase` | `entities` | +| `dqlUrl` | derived from `kgBase` | `dql`, `dqlParallel` | +| `ontologyUrl` | derived from `kgBase` | `dqlFetchOntology`, `dqlFetchOntologyText`, `OntologyStore` | ```typescript import { DiffbotClient } from "@diffbot/typescript"; @@ -66,8 +71,35 @@ const db = new DiffbotClient({ }); ``` -`analyzeUrl` and `crawlerUrl` are bases that the SDK appends to (`/${api}` and `/data` -respectively); the rest are complete endpoints, used as given. +Point the whole SDK at a staging environment with one option instead of one per endpoint: + +```typescript +const staging = new DiffbotClient({ token: "YOUR_TOKEN", apiBase: "https://staging.example.com" }); +``` + +`apiBase`/`kgBase`/`llmBase`/`nlpBase` set the host for every endpoint on that host at +once. The older per-endpoint options (`analyzeUrl`, `crawlerUrl`, `llmUrl`, +`webSearchUrl`, `nlpUrl`, `dqlUrl`, `ontologyUrl`) still work and win over the matching +base when both are set; `analyzeUrl` and `crawlerUrl` are bases the SDK appends to +(`/${api}` and `/data` respectively), the rest are complete endpoints used as given. + +Requests retry `429` and `5xx` responses (and connection errors) with backoff and +jitter, honoring a `Retry-After` header, by default. Disable retries or tune the policy: + +```typescript +const db = new DiffbotClient({ token: "YOUR_TOKEN", retry: false }); +// or: retry: { maxAttempts: 5 } +``` + +`maxDelayMs` (default 5000) caps the backoff and is also the ceiling for `Retry-After`. +If the server asks for a longer wait (in seconds or as an HTTP date), the SDK does not +sleep or retry: the call fails at once with `RateLimitError` (`retryAfter` holds the raw +header) or, for a 5xx, `APIError` (see `headers["retry-after"]`), so you decide when to +try again. + +The transport also accepts a per-call `timeout` (ms) override and a `retryable: false` +escape hatch; the SDK uses the latter internally for `crawlDeleteJob`, a GET-shaped +mutation that must not be retried blindly. Passing `fetch` replaces the transport, which is the hook for retries, logging, extra headers, or mocking in tests: @@ -91,6 +123,32 @@ const db = new DiffbotClient({ token: "YOUR_TOKEN" }); const data = await extract(db, "https://www.example.com"); ``` +Pass an optional options object as the fifth argument (use `undefined` to keep the +defaults for `api` and `fmt`). `timeout` is the server-side limit in ms (default 30000); +the SDK keeps its own request timeout at least 10 s above it. + +```typescript +const article = await extract(db, "https://www.example.com", "article", "json", { + fields: ["links", "meta"], + timeout: 60_000, + naturalLanguage: ["entities", "sentiment"], +}); +``` + +### Check a token and read account usage + +`verifyToken()` is `true` for an accepted token and `false` only when the API rejects it +(`AuthError`); network errors, rate limits and 5xx responses are thrown. The account +response contains your token and any child tokens, so don't log it. + +```typescript +import { account, verifyToken } from "@diffbot/typescript"; + +if (await verifyToken(db)) { + const { plan, usage } = await account(db, { days: 7 }); +} +``` + ### Ask Diffbot LLM ```typescript @@ -154,6 +212,32 @@ for await (const event of crawl(db, "https://www.example.com", { hops: 1 })) { } ``` +To start a crawl without holding a connection open (for example inside a tool call), +start it without `watch`, poll the job, then download the results: + +```typescript +import { crawl, crawlDownload, crawlGetJob } from "@diffbot/typescript"; + +for await (const _ of crawl(db, "https://www.example.com", { jobName: "my-job" })) { + // yields a single job_created event when `watch` is off +} +const job = await crawlGetJob(db, "my-job"); // poll until job?.jobStatus?.status is terminal +const records = await crawlDownload(db, "my-job"); // CrawlDataRecord[] +const csv = await crawlDownload(db, "my-job", { format: "csv" }); // string +``` + +Crawl-create options beyond the basics: `urlCrawlRegEx`, `urlProcessRegEx`, +`pageProcessPattern`, `maxToCrawlPerSubdomain`, `maxToProcessPerSubdomain`, +`useCanonical`, `notifyWebhook`, `notifyEmail`. `customHeaders` takes one +`"Name:value"` string or an array of them. + +```typescript +crawl(db, "https://www.example.com", { + urlProcessRegEx: "/products/", + customHeaders: ["Accept-Language:en-us", "X-Test:1"], +}); +``` + ### Query the Knowledge Graph ```typescript @@ -162,6 +246,17 @@ import { dql } from "@diffbot/typescript"; const results = await dql(db, 'type:Organization name:"Diffbot"'); ``` +Named options beyond `size`/`from`/`format`/`filter`/`exportspec`: `type` +(`"query" | "text" | "queryTextFallback" | "crawl"`), `col`, `filterExclude`, +`exportseparator`, `exportfile`, `jsonmode`, `nonCanonicalFacts`, `noDedupArticles`, +`cluster`. They go in the POST body. A named key repeated in `extra` throws +`ValidationError`. + +```typescript +// Search a crawl collection +const pages = await dql(db, "title:Pricing", { type: "crawl", col: "my-job", size: 5 }); +``` + ### Web Search ```typescript @@ -180,6 +275,13 @@ const brief = await webSearch(db, "diffbot knowledge graph", { numResults: 5, sn const page = await webSearch(db, "url:https://en.wikipedia.org/wiki/Knowledge_graph", { snippetLength: "full" }); ``` +`text` also accepts 1-5 queries, sent as repeated `text=` params (more or fewer throws +`ValidationError`): + +```typescript +const multi = await webSearch(db, ["diffbot knowledge graph", "diffbot extract api"], { numResults: 3 }); +``` + ### Caching the Knowledge Graph ontology `dqlFetchOntology` fetches on every call. For anything that calls it more than once, @@ -212,6 +314,21 @@ import { entities } from "@diffbot/typescript"; const result = await entities(db, "Apple CEO Tim Cook announced record quarterly earnings."); ``` +For other analyses on the same endpoint (single document), use `nlp()`; the result type is +narrowed to the fields you ask for. `entities()` stays the fixed `entities,sentiment` shortcut. + +```typescript +import { nlp } from "@diffbot/typescript"; + +const result = await nlp(db, "Headline\nApple CEO Tim Cook announced record quarterly earnings.", { + fields: ["summary", "language"], + format: "plain text with title", + documentType: "news article", + customSummary: { maxNumberOfSentences: 1 }, +}); +console.log(result.summary, result.language); +``` + ### Automatic cleanup with `await using` `DiffbotClient` supports explicit resource management for automatic cleanup: diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..1f8a2ae --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,59 @@ +// @ts-check +import js from "@eslint/js"; +import tseslint from "typescript-eslint"; + +// Scope is src/ only (per the Phase 0 plan): tests, scripts, and generated/vendored +// files are excluded rather than relaxed, so this stays strict-type-checked as shipped +// rather than drifting rule-by-rule. +export default tseslint.config( + { + ignores: [ + "dist/**", + "coverage/**", + "src/generated/**", + "fixtures/**", + "tests/**", + "scripts/**", + "*.config.*", + ], + }, + js.configs.recommended, + ...tseslint.configs.strictTypeChecked, + { + languageOptions: { + parserOptions: { + projectService: true, + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + "@typescript-eslint/no-unused-vars": [ + "error", + { argsIgnorePattern: "^_", varsIgnorePattern: "^_" }, + ], + }, + }, + { + // `as` is confined to the decode module; everywhere else in src/ must go through + // its readJson/readText/readBytes/readNdjson/parseJsonText instead of casting. + files: ["src/**/*.ts"], + rules: { + "@typescript-eslint/consistent-type-assertions": ["error", { assertionStyle: "never" }], + }, + }, + { + files: ["src/decode.ts"], + rules: { + "@typescript-eslint/consistent-type-assertions": "off", + }, + }, + { + // src/node/** is filesystem-only glue code outside the core fetch/JSON path the + // decode-module rule is about (see src/decode.ts's header); Node error narrowing + // (`error as NodeJS.ErrnoException`) is idiomatic here. + files: ["src/node/**/*.ts"], + rules: { + "@typescript-eslint/consistent-type-assertions": "off", + }, + }, +); diff --git a/openapi/README.md b/openapi/README.md new file mode 100644 index 0000000..b098520 --- /dev/null +++ b/openapi/README.md @@ -0,0 +1,8 @@ +Vendored from `/workspace/cc-runs/sdk-coverage/docs/openapi/*.json` (the Diffbot SDK +coverage audit's local snapshot of Diffbot's published OpenAPI specs). Regenerate +`src/generated/*.ts` from these with `pnpm gen:types`; never hand-edit the generated +output. + +`natural-language.json` was originally Swagger 2.0 (openapi-typescript only accepts +3.x). It was converted once with `swagger2openapi --patch` and is committed here as +OpenAPI 3.0 — there is no ongoing conversion step. diff --git a/openapi/account.json b/openapi/account.json new file mode 100644 index 0000000..5b088f1 --- /dev/null +++ b/openapi/account.json @@ -0,0 +1,212 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Account API", + "description": "Returns account plan, usage, child tokens, and other account details.", + "termsOfService": "https://www.diffbot.com/terms/", + "contact": { + "email": "support@diffbot.com" + }, + "version": "1.1" + }, + "servers": [ + { + "url": "https://api.diffbot.com/v4" + } + ], + "paths": { + "/account": { + "get": { + "summary": "Get Account Details", + "description": "Returns account plan, usage, child tokens, and other account details.", + "operationId": "account", + "parameters": [ + { + "name": "days", + "in": "query", + "description": "Pass the number of days (`&days=365`) for which you would like to retrieve API call volumes (default = 31).", + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "example": { + "created": "2020-04-22", + "usage": [ + { + "date": "2022-05-04", + "subtitles": 0, + "nlp": 0, + "entities": 10, + "credits": 258, + "refresh": 0, + "proxies": 0, + "videos": 0, + "extractions": 8, + "facets": 0 + }, + { + "date": "2022-05-03", + "subtitles": 0, + "nlp": 0, + "entities": 75, + "credits": 1875, + "refresh": 0, + "proxies": 0, + "videos": 0, + "extractions": 0, + "facets": 0 + } + ], + "name": "Diffy", + "childTokens": [ + "xxxx", + "xxxx" + ], + "plan": "diffy-plans-to-hangout-all-day", + "email": "sales@diffbot.com", + "token": "xxxx", + "status": "active" + }, + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name associated with the token." + }, + "email": { + "type": "string", + "description": "Email address associated with the token." + }, + "plan": { + "type": "string", + "description": "Current plan for the token." + }, + "planCredits": { + "type": "integer", + "description": "Amount of monthly calls included." + }, + "status": { + "type": "string", + "description": "Status of the token." + }, + "created": { + "type": "string", + "format": "date" + }, + "token": { + "type": "string" + }, + "childTokens": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of child or sub-tokens, if there are any associated with your account." + }, + "usage": { + "type": "array", + "description": "An array of days and call volume amounts for each day. By default this will return data from the most recent 31 days. Use the `days` argument to adjust the response window. Dates prior to token becoming active will not be returned.", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string", + "format": "date", + "description": "Date, e.g. `2019-12-01`." + }, + "nlp": { + "type": "integer", + "description": "Total number of Natural Language API calls made." + }, + "credits": { + "type": "integer", + "description": "Total number of credits consumed." + }, + "extractions": { + "type": "integer", + "description": "Total number of Extraction API calls made." + }, + "proxies": { + "type": "integer", + "description": "Total number of calls made using proxy servers. [Read more](/docs/extract/using-proxies)." + }, + "entities": { + "type": "integer", + "description": "Total number of entities downloaded from the Diffbot Knowledge Graph." + }, + "refresh": { + "type": "integer", + "description": "Total number of entity refreshes requested." + }, + "facets": { + "type": "integer", + "description": "Total number of facet values returned." + }, + "videos": { + "type": "integer", + "description": "Total number of videos downloaded." + }, + "subtitles": { + "type": "integer", + "description": "Total number of subtitles requested." + } + } + } + } + } + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "application/json": { + "example": { + "errorCode": 401, + "error": "Not authorized API token." + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer", + "example": 401 + }, + "error": { + "type": "string", + "example": "Not authorized API token." + } + } + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + } + }, + "components": { + "securitySchemes": { + "tokenscheme": { + "type": "apiKey", + "name": "token", + "in": "query", + "description": "Your Diffbot API token." + } + } + } +} diff --git a/openapi/bulk.json b/openapi/bulk.json new file mode 100644 index 0000000..ba3701a --- /dev/null +++ b/openapi/bulk.json @@ -0,0 +1,760 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Bulk Extract API", + "termsOfService": "https://www.diffbot.com/terms/", + "contact": { + "email": "support@diffbot.com" + }, + "version": "1.1" + }, + "servers": [ + { + "url": "https://api.diffbot.com/v3" + } + ], + "paths": { + "/bulk": { + "post": { + "summary": "Create a Bulk Extract Job", + "description": "Extract a list of URLs asynchronously", + "operationId": "create-a-bulk-job", + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "type": "object", + "required": [ + "name", + "urls", + "apiUrl" + ], + "properties": { + "name": { + "type": "string", + "description": "Job name. This should be a unique identifier and will be used to modify your bulk job and retrieve its output." + }, + "urls": { + "type": "string", + "description": "Space-delimited list of URLs to process. If you are on the Startup plan, jobs must contain at least 50 URLs." + }, + "apiUrl": { + "type": "string", + "description": "The full Extract (or Custom) API to be used for each URL. For instance, to process each URL via the Article API, supply `https://api.diffbot.com/v3/article`. You may also include API parameters, e.g. `https://api.diffbot.com/v3/article?fields=meta,tags`." + }, + "customHeaders": { + "type": "string", + "description": "Set custom headers to be used for processing each URL. Send multiple `customHeaders` values in your POST body, with header keys/values delimited by a colon (and URL-encoded)." + }, + "notifyEmail": { + "type": "string", + "description": "Send a message to this email address when the bulk job completes." + }, + "notifyWebhook": { + "type": "string", + "description": "Pass a URL to be notified when the bulk job completes. You will receive a POST with a JSON response." + }, + "obeyRobots": { + "type": "string", + "description": "Pass `obeyRobots=0` to ignore a site's robots.txt instructions." + }, + "repeat": { + "type": "number", + "description": "Specify the number of days as a floating-point (e.g. `repeat=7.0`) to repeat this job. By default bulk jobs will not be repeated.", + "format": "float" + }, + "pageProcessPattern": { + "type": "string", + "description": "Enter ||-separated strings to limit pages processed to those whose HTML contains any of the content strings. If a page does not contain at least one of the strings, it will be ignored." + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Bulk job created", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job is initializing.\",\n \"status\": 0\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "response": { + "type": "string", + "example": "Successfully added urls for spidering." + }, + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobStatus": { + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Job is initializing." + }, + "status": { + "type": "integer", + "example": 0 + } + } + }, + "maxHops": { + "type": "integer", + "example": -1 + }, + "downloadJson": { + "type": "string", + "example": "...json" + }, + "urlProcessPattern": { + "type": "string", + "example": "" + }, + "jobCompletionTimeUTC": { + "type": "integer", + "example": 0 + }, + "maxRounds": { + "type": "integer", + "example": -1 + }, + "type": { + "type": "string", + "example": "bulk" + }, + "pageCrawlSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "urlCrawlRegEx": { + "type": "string", + "example": "" + }, + "pageProcessPattern": { + "type": "string", + "example": "" + }, + "apiUrl": { + "type": "string", + "example": "https://api.diffbot.com/v3/analyze" + }, + "useCanonical": { + "type": "integer", + "example": 1 + }, + "jobCreationTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "repeat": { + "type": "integer", + "example": 0 + }, + "downloadUrls": { + "type": "string", + "example": "...csv" + }, + "obeyRobots": { + "type": "integer", + "example": 1 + }, + "roundsCompleted": { + "type": "integer", + "example": 0 + }, + "pageCrawlAttempts": { + "type": "integer", + "example": 0 + }, + "notifyWebhook": { + "type": "string", + "example": "" + }, + "pageProcessSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "customHeaders": { + "type": "object", + "properties": {} + }, + "objectsFound": { + "type": "integer", + "example": 0 + }, + "roundStartTime": { + "type": "integer", + "example": 0 + }, + "urlCrawlPattern": { + "type": "string", + "example": "" + }, + "seedRecrawlFrequency": { + "type": "integer", + "example": -1 + }, + "urlProcessRegEx": { + "type": "string", + "example": "" + }, + "pageProcessSuccesses": { + "type": "integer", + "example": 0 + }, + "urlsHarvested": { + "type": "integer", + "example": 0 + }, + "crawlDelay": { + "type": "integer", + "example": -1 + }, + "currentTime": { + "type": "integer", + "example": 1649950325 + }, + "useProxies": { + "type": "integer", + "example": 0 + }, + "sentJobDoneNotification": { + "type": "integer", + "example": 0 + }, + "currentTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "name": { + "type": "string", + "example": "bulkTest" + }, + "notifyEmail": { + "type": "string", + "example": "" + }, + "pageCrawlSuccesses": { + "type": "integer", + "example": 0 + }, + "pageProcessAttempts": { + "type": "integer", + "example": 0 + } + } + } + } + } + } + } + } + }, + "401": { + "description": "Token missing or invalid", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "{\n \"errorCode\": 401,\n \"error\": \"Not authorized API token.\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer", + "example": 401 + }, + "error": { + "type": "string", + "example": "Not authorized API token." + } + } + } + } + } + }, + "500": { + "description": "No URLs were found in the request", + "content": { + "application/json": { + "examples": { + "No URLs Found": { + "value": "{\n \"errorCode\": \"500\",\n \"error\": \"no urls found\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "string", + "example": "500" + }, + "error": { + "type": "string", + "example": "no urls found" + } + } + } + } + } + }, + "505": { + "description": "Job name rejected \u2014 names can consist only of letters, numbers, underscores, or hyphens", + "content": { + "application/json": { + "examples": { + "Malformed Crawl Name": { + "value": "{\n \"errorCode\": 505,\n \"error\": \"Crawl names can consist only of letters, numbers, underscores or hyphens.\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer", + "example": 505 + }, + "error": { + "type": "string", + "example": "Crawl names can consist only of letters, numbers, underscores or hyphens." + } + } + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + }, + "get": { + "summary": "Manage a Bulk Extract Job", + "description": "Pause, delete, restart, or view the status of a bulk job.", + "operationId": "pause-delete-or-restart-a-bulk-job", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "Job name as defined when the bulk job was created.", + "schema": { + "type": "string" + } + }, + { + "name": "pause", + "in": "query", + "description": "Pass `pause=1` to pause a bulk job. Pass `pause=0` to resume a paused job.", + "schema": { + "type": "string" + } + }, + { + "name": "restart", + "in": "query", + "description": "Pass `restart=1` to restart a bulk job. This will erase all processed data and re-process all of the submitted URLs.", + "schema": { + "type": "string" + } + }, + { + "name": "delete", + "in": "query", + "description": "Pass `delete=1 `to delete a job, and all associated data, completely.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Job managed", + "content": { + "application/json": { + "examples": { + "Restarted Bulk Job": { + "value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job is initializing.\",\n \"status\": 0\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + }, + "Paused Bulk Job": { + "value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job paused.\",\n \"status\": 6\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + }, + "Deleted Bulk Job": { + "value": "{\n \"response\": \"Successfully deleted job.\"\n}" + }, + "Completed Bulk Job": { + "value": "{\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job has completed and no repeat is scheduled.\",\n \"status\": 9\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"bulk\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"bulkTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "response": { + "type": "string", + "example": "Successfully added urls for spidering." + }, + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobStatus": { + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Job is initializing." + }, + "status": { + "type": "integer", + "example": 0 + } + } + }, + "maxHops": { + "type": "integer", + "example": -1 + }, + "downloadJson": { + "type": "string", + "example": "...json" + }, + "urlProcessPattern": { + "type": "string", + "example": "" + }, + "jobCompletionTimeUTC": { + "type": "integer", + "example": 0 + }, + "maxRounds": { + "type": "integer", + "example": -1 + }, + "type": { + "type": "string", + "example": "bulk" + }, + "pageCrawlSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "urlCrawlRegEx": { + "type": "string", + "example": "" + }, + "pageProcessPattern": { + "type": "string", + "example": "" + }, + "apiUrl": { + "type": "string", + "example": "https://api.diffbot.com/v3/analyze" + }, + "useCanonical": { + "type": "integer", + "example": 1 + }, + "jobCreationTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "repeat": { + "type": "integer", + "example": 0 + }, + "downloadUrls": { + "type": "string", + "example": "...csv" + }, + "obeyRobots": { + "type": "integer", + "example": 1 + }, + "roundsCompleted": { + "type": "integer", + "example": 0 + }, + "pageCrawlAttempts": { + "type": "integer", + "example": 0 + }, + "notifyWebhook": { + "type": "string", + "example": "" + }, + "pageProcessSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "customHeaders": { + "type": "object", + "properties": {} + }, + "objectsFound": { + "type": "integer", + "example": 0 + }, + "roundStartTime": { + "type": "integer", + "example": 0 + }, + "urlCrawlPattern": { + "type": "string", + "example": "" + }, + "seedRecrawlFrequency": { + "type": "integer", + "example": -1 + }, + "urlProcessRegEx": { + "type": "string", + "example": "" + }, + "pageProcessSuccesses": { + "type": "integer", + "example": 0 + }, + "urlsHarvested": { + "type": "integer", + "example": 0 + }, + "crawlDelay": { + "type": "integer", + "example": -1 + }, + "currentTime": { + "type": "integer", + "example": 1649950325 + }, + "useProxies": { + "type": "integer", + "example": 0 + }, + "sentJobDoneNotification": { + "type": "integer", + "example": 0 + }, + "currentTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "name": { + "type": "string", + "example": "bulkTest" + }, + "notifyEmail": { + "type": "string", + "example": "" + }, + "pageCrawlSuccesses": { + "type": "integer", + "example": 0 + }, + "pageProcessAttempts": { + "type": "integer", + "example": 0 + } + } + } + } + } + } + } + } + }, + "500": { + "description": "No job found by that name", + "content": { + "application/json": { + "examples": { + "Bulk Job Not Found": { + "value": "{\n \"errorCode\": \"500\",\n \"error\": \"no collection record.\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "string", + "example": "500" + }, + "error": { + "type": "string", + "example": "no collection record." + } + } + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/bulk/data": { + "get": { + "summary": "Retrieve Bulk Extract Job Data", + "description": "Download the extracted results of a bulk job", + "operationId": "retrieve-bulk-job-data", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "Name of the bulk job whose data you wish to download.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "format", + "in": "query", + "description": "Request `format=csv` to download the extracted data in CSV format (default: `json`). Note that CSV files will only contain top-level fields.", + "schema": { + "type": "string" + } + }, + { + "name": "type", + "in": "query", + "description": "Request `type=urls` to retrieve the [URL Report (CSV)](/docs/crawl/faq/url-report).", + "schema": { + "type": "string" + } + }, + { + "name": "num", + "in": "query", + "description": "Pass an integer value (e.g. `num=100`) to request a subset of URLs, most recently processed first.", + "schema": { + "type": "string" + } + } + ], + "security": [ + { + "tokenscheme": [] + } + ], + "responses": { + "200": { + "description": "The bulk job's extracted data", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "[\n {\n \"col\": \"docs-bulkdata-sample\",\n \"parentUrl\": \"https://example.com\",\n \"pageUrl\": \"https://example.com/\",\n \"id\": 6526070754824580571,\n \"type\": \"other\",\n \"title\": \"Example Domain\",\n \"content\": \"Example Domain \\n\\n# Example Domain \\n\\nThis domain is for use in documentation examples without needing permission. Avoid use in operations. \\n\\n[Learn more](https://iana.org/domains/example)\",\n \"token\": \"\",\n \"timestamp\": \"2026-07-24T23:03:01Z\"\n }\n]" + } + }, + "schema": { + "type": "array", + "items": { + "type": "object", + "required": [ + "col", + "parentUrl", + "pageUrl", + "id", + "type", + "token", + "timestamp" + ], + "properties": { + "col": { + "description": "Name of the Crawl or Bulk job this record was extracted from", + "type": "string", + "example": "docs-bulkdata-sample" + }, + "parentUrl": { + "description": "Immediate parent contained a link to this record", + "type": "string", + "example": "https://example.com" + }, + "pageUrl": { + "description": "The URL of this record", + "type": "string", + "example": "https://example.com/" + }, + "id": { + "description": "A reference", + "type": "integer", + "example": 6526070754824580571 + }, + "type": { + "description": "A standard entity type assigned either on job creation or classified by [Analyze API](/docs/extract/analyze). All other keys on this job record follow the [ontology](/docs/ontology/) set by its type.", + "type": "string", + "example": "other" + }, + "token": { + "description": "Your token", + "type": "string", + "example": "" + }, + "timestamp": { + "description": "Time when record was crawled, in ISO 8601 format.", + "type": "string", + "example": "2026-07-24T23:03:01Z" + } + }, + "additionalProperties": true + } + } + } + } + }, + "400": { + "description": "No job found by that name", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "{\n \"errorCode\": 400,\n \"error\": \"No collection record\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer", + "example": 400 + }, + "error": { + "type": "string", + "example": "No collection record" + } + } + } + } + } + } + } + } + } + }, + "components": { + "securitySchemes": { + "tokenscheme": { + "type": "apiKey", + "name": "token", + "in": "query" + } + } + } +} diff --git a/openapi/crawl.json b/openapi/crawl.json new file mode 100644 index 0000000..2548dc6 --- /dev/null +++ b/openapi/crawl.json @@ -0,0 +1,1196 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Crawl API", + "termsOfService": "https://www.diffbot.com/terms/", + "contact": { + "email": "support@diffbot.com" + }, + "version": "1.1" + }, + "servers": [ + { + "url": "https://api.diffbot.com/v3" + } + ], + "paths": { + "/crawl": { + "post": { + "summary": "Create a Crawl", + "description": "Create and start a job to spider and extract pages through a site.", + "operationId": "create-a-crawl", + "requestBody": { + "content": { + "application/x-www-form-urlencoded": { + "schema": { + "type": "object", + "required": [ + "name", + "seeds", + "apiUrl" + ], + "properties": { + "name": { + "type": "string", + "description": "Job name. This should be a unique identifier and can be used to modify your crawl or retrieve its output." + }, + "seeds": { + "type": "string", + "description": "Seed URL(s). Must be [URL encoded](https://en.wikipedia.org/wiki/Percent-encoding). Separate multiple URLs with whitespace to spider multiple sites within the same crawl. If the seed contains a non-www subdomain (\"https://blog.diffbot.com\" or \"https://docs.diffbot.com\") Crawl will restrict spidering to the specified **subdomain**." + }, + "apiUrl": { + "type": "string", + "description": "Full Extract API URL through which to process pages. E.g., `&apiUrl=https://api.diffbot.com/v3/analyze` to process matching links via the Analyze API, which will automatically determine the page type to extract as. The Extract API URL can include querystring parameters to tailor the output.\n\nFor example, `&apiUrl=https://api.diffbot.com/v3/product?fields=querystring,meta` will process matching links using the Product API, and also return the querystring and meta fields." + }, + "customHeaders": { + "type": "string", + "description": "Set multiple custom headers to be used while crawling and processing pages sent to Diffbot APIs. Each header should be sent in its own `customHeaders` argument, with a colon delimiting the header name and value, and should be URL-encoded. For example, `&customHeaders=Accept-Language%3Aen-us`. [See more on using custom headers](/docs/extract/custom-headers)." + }, + "useCanonical": { + "type": "integer", + "description": "Pass `useCanonical=0` to disable deduplication of pages based on a canonical link definition. [See more](/products/crawl/faq/duplicate-content).", + "default": 1 + }, + "obeyRobots": { + "type": "integer", + "description": "Pass `obeyRobots=0` to ignore a site's robots.txt instructions.", + "default": 1 + }, + "restrictDomain": { + "type": "integer", + "description": "Pass `restrictDomain=0` to allow limited crawling across subdomains/domains. [See more](/products/crawl/faq/restrict-to-domains).", + "default": 1 + }, + "useProxies": { + "type": "integer", + "description": "Set value to `1` to force the use of proxy IPs for the crawl. This will utilize proxy servers for both crawling and processing of pages.", + "default": 0 + }, + "maxHops": { + "type": "integer", + "description": "Specify the depth of your crawl. A `maxHops=0` will limit **processing** to the seed URL(s) only -- no other links will be processed; `maxHops=1` will process all (otherwise matching) pages whose links appear on seed URL(s); `maxHops=2` will process pages whose links appear on those pages; and so on.\n\nBy default (`maxHops=-1`) Crawl will crawl and process links at any depth.", + "default": -1 + }, + "notifyEmail": { + "type": "string", + "description": "Send a message to this email address when the crawl hits the `maxToCrawl` or `maxToProcess` limit, or when the crawl completes." + }, + "notifyWebhook": { + "type": "string", + "description": "Pass a URL to be notified when the crawl hits the `maxToCrawl` or `maxToProcess` limit, or when the crawl completes. You will receive a POST with `X-Crawl-Name` and `X-Crawl-Status` in the headers, and the job's JSON metadata in the POST body. Note that in webhook POSTs the parent `jobs` will not be sent\u2014only the individual job object will be returned." + }, + "repeat": { + "type": "number", + "description": "Specify the number of days as a floating-point (e.g. `repeat=7.0`) to repeat this crawl. By default crawls will not be repeated." + }, + "seedRecrawlFrequency": { + "type": "number", + "description": "Useful for specifying a frequency, in number of days, to recrawl seed urls, which is independent of the overall recrawl frequency given by `repeat`. Defaults to `seedRecrawlFrequency=-1` to use the default frequency.", + "default": -1 + }, + "maxRounds": { + "type": "integer", + "description": "Specify the maximum number of crawl repeats. By default (`maxRounds=0`) repeating crawls will continue indefinitely.", + "default": 0 + }, + "urlCrawlPattern": { + "type": "string", + "description": "Specify ||-separated **strings** to limit pages crawled to those whose URLs contain *any* of the content strings. You can use the exclamation point to specify a negative string, e.g. `!product` to exclude URLs containing the string \"product,\" and the `^` and `$` characters to limit matches to the beginning or end of the URL.\n\nThe use of a `urlCrawlPattern` will allow Crawl to spider outside of the seed domain; it will follow all matching URLs regardless of domain." + }, + "urlCrawlRegEx": { + "type": "string", + "description": "Specify a regular expression to limit pages **crawled** to those URLs that contain a match to your expression. This will override any `urlCrawlPattern` value." + }, + "maxToCrawl": { + "type": "integer", + "description": "Specify max pages to spider. Default: 100,000.", + "default": 100000 + }, + "maxToCrawlPerSubdomain": { + "type": "integer", + "description": "Specify max pages to spider per subdomain. Default: no limit (-1)", + "default": -1 + }, + "crawlDelay": { + "type": "number", + "description": "Wait this many seconds between each URL crawled from a single IP address. Specify the number of seconds as an integer or floating-point number (e.g., `crawlDelay=0.25`)." + }, + "urlProcessPattern": { + "type": "string", + "description": "Specify ||-separated **strings** to limit pages processed to those whose URLs contain *any* of the content strings. You can use the exclamation point to specify a negative string, e.g. `!/category` to exclude URLs containing the string \"/category,\" and the `^` and `$` characters to limit matches to the beginning or end of the URL." + }, + "urlProcessRegEx": { + "type": "string", + "description": "Specify a regular expression to limit pages **processed** to those URLs that contain a match to your expression. This will override any `urlProcessPattern` value." + }, + "pageProcessPattern": { + "type": "string", + "description": "Specify ||-separated strings to limit pages **processed** to those whose HTML contains *any* of the content strings." + }, + "maxToProcess": { + "type": "integer", + "description": "Specify max pages to process through Diffbot APIs. Default: 100,000.", + "default": 100000 + }, + "maxToProcessPerSubdomain": { + "type": "integer", + "description": "Specify max pages to process per subdomain. Default: no limit (-1)", + "default": -1 + }, + "onlyProcessIfNew": { + "type": "integer", + "description": "By default repeat crawls will only process new (previously unprocessed) pages. Set to 0 (`onlyProcessIfNew=0`) to process all content on repeat crawls.", + "default": 1 + } + } + }, + "examples": { + "Request Example": { + "value": { + "name": "test-crawl", + "seeds": "https://example.com", + "apiUrl": "https://api.diffbot.com/v3/analyze" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Crawl job created", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"maxToProcessPerSubdomain\": -1,\n \"jobStatus\": {\n \"message\": \"Job is initializing.\",\n \"status\": 0\n },\n \"maxHops\": -1,\n \"onlyProcessIfNew\": 1,\n \"downloadJson\": \"https://api.diffbot.com/v3/crawl/download/.json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"restrictDomain\": 1,\n \"type\": \"crawl\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"maxToCrawlPerSubdomain\": -1,\n \"jobCreationTimeUTC\": 1726163287,\n \"repeat\": 0,\n \"downloadUrls\": \"https://api.diffbot.com/v3/crawl/download/.csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"maxToProcess\": 100000,\n \"seedRecrawlFrequency\": -1,\n \"seeds\": \"https://example.com\",\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": 0.25,\n \"currentTime\": 1726163288,\n \"useProxies\": 0,\n \"maxToCrawl\": 100000,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1726163288,\n \"restrictSubdomain\": 0,\n \"name\": \"test-crawl\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "response": { + "type": "string", + "example": "Successfully added urls for spidering." + }, + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "maxToProcessPerSubdomain": { + "type": "integer", + "example": -1 + }, + "jobStatus": { + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Job is initializing." + }, + "status": { + "type": "integer", + "example": 0 + } + } + }, + "maxHops": { + "type": "integer", + "example": -1 + }, + "onlyProcessIfNew": { + "type": "integer", + "example": 1 + }, + "downloadJson": { + "type": "string", + "example": "https://api.diffbot.com/v3/crawl/download/.json" + }, + "urlProcessPattern": { + "type": "string", + "example": "" + }, + "jobCompletionTimeUTC": { + "type": "integer", + "example": 0 + }, + "maxRounds": { + "type": "integer", + "example": -1 + }, + "restrictDomain": { + "type": "integer", + "example": 1 + }, + "type": { + "type": "string", + "example": "crawl" + }, + "pageCrawlSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "urlCrawlRegEx": { + "type": "string", + "example": "" + }, + "pageProcessPattern": { + "type": "string", + "example": "" + }, + "apiUrl": { + "type": "string", + "example": "https://api.diffbot.com/v3/analyze" + }, + "useCanonical": { + "type": "integer", + "example": 1 + }, + "maxToCrawlPerSubdomain": { + "type": "integer", + "example": -1 + }, + "jobCreationTimeUTC": { + "type": "integer", + "example": 1726163287 + }, + "repeat": { + "type": "integer", + "example": 0 + }, + "downloadUrls": { + "type": "string", + "example": "https://api.diffbot.com/v3/crawl/download/.csv" + }, + "obeyRobots": { + "type": "integer", + "example": 1 + }, + "roundsCompleted": { + "type": "integer", + "example": 0 + }, + "pageCrawlAttempts": { + "type": "integer", + "example": 0 + }, + "notifyWebhook": { + "type": "string", + "example": "" + }, + "pageProcessSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "customHeaders": { + "type": "object", + "properties": {} + }, + "objectsFound": { + "type": "integer", + "example": 0 + }, + "roundStartTime": { + "type": "integer", + "example": 0 + }, + "urlCrawlPattern": { + "type": "string", + "example": "" + }, + "maxToProcess": { + "type": "integer", + "example": 100000 + }, + "seedRecrawlFrequency": { + "type": "integer", + "example": -1 + }, + "seeds": { + "type": "string", + "example": "https://example.com" + }, + "urlProcessRegEx": { + "type": "string", + "example": "" + }, + "pageProcessSuccesses": { + "type": "integer", + "example": 0 + }, + "urlsHarvested": { + "type": "integer", + "example": 0 + }, + "crawlDelay": { + "type": "number", + "example": 0.25 + }, + "currentTime": { + "type": "integer", + "example": 1726163288 + }, + "useProxies": { + "type": "integer", + "example": 0 + }, + "maxToCrawl": { + "type": "integer", + "example": 100000 + }, + "sentJobDoneNotification": { + "type": "integer", + "example": 0 + }, + "currentTimeUTC": { + "type": "integer", + "example": 1726163288 + }, + "restrictSubdomain": { + "type": "integer", + "example": 0 + }, + "name": { + "type": "string", + "example": "test-crawl" + }, + "notifyEmail": { + "type": "string", + "example": "" + }, + "pageCrawlSuccesses": { + "type": "integer", + "example": 0 + }, + "pageProcessAttempts": { + "type": "integer", + "example": 0 + } + } + } + } + } + } + } + } + }, + "505": { + "description": "Too many collections for token \u2014 the 1,000-crawl limit has been reached", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "{\n \"errorCode\":505,\n \"error\":\"Too many collections for token\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer", + "example": 505 + }, + "error": { + "type": "string", + "example": "Too many collections for token" + } + } + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + }, + "get": { + "summary": "Manage a Crawl Job", + "description": "Pause, delete, restart, or view the status of a crawl job.", + "operationId": "manage-a-crawl-job", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "Job name as defined when the crawl job was created.", + "schema": { + "type": "string" + } + }, + { + "name": "roundStart", + "in": "query", + "description": "Pass `roundStart=1` to force the start of a new crawl \"round\" (manually repeat the crawl). If `onlyProcessIfNew` is set to 1 (default), only newly-created pages will be processed.", + "schema": { + "type": "string" + } + }, + { + "name": "pause", + "in": "query", + "description": "Pass `pause=1` to pause a crawl job. Pass `pause=0` to resume a paused job.", + "schema": { + "type": "string" + } + }, + { + "name": "restart", + "in": "query", + "description": "Pass `restart=1` to restart a crawl job. This will erase all processed data and re-process all of the submitted URLs.", + "schema": { + "type": "string" + } + }, + { + "name": "delete", + "in": "query", + "description": "Pass `delete=1`to delete a job, and all associated data, completely.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Job managed", + "content": { + "application/json": { + "examples": { + "Restarted Crawl Job": { + "value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job is initializing.\",\n \"status\": 0\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"crawl\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"crawlTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + }, + "Paused Crawl Job": { + "value": "{\n \"response\": \"Successfully added urls for spidering.\",\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job paused.\",\n \"status\": 6\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"crawl\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"crawlTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + }, + "Deleted Crawl Job": { + "value": "{\n \"response\": \"Successfully deleted job.\"\n}" + }, + "Completed Crawl Job": { + "value": "{\n \"jobs\": [\n {\n \"jobStatus\": {\n \"message\": \"Job has completed and no repeat is scheduled.\",\n \"status\": 9\n },\n \"maxHops\": -1,\n \"downloadJson\": \"...json\",\n \"urlProcessPattern\": \"\",\n \"jobCompletionTimeUTC\": 0,\n \"maxRounds\": -1,\n \"type\": \"crawl\",\n \"pageCrawlSuccessesThisRound\": 0,\n \"urlCrawlRegEx\": \"\",\n \"pageProcessPattern\": \"\",\n \"apiUrl\": \"https://api.diffbot.com/v3/analyze\",\n \"useCanonical\": 1,\n \"jobCreationTimeUTC\": 1649950325,\n \"repeat\": 0,\n \"downloadUrls\": \"...csv\",\n \"obeyRobots\": 1,\n \"roundsCompleted\": 0,\n \"pageCrawlAttempts\": 0,\n \"notifyWebhook\": \"\",\n \"pageProcessSuccessesThisRound\": 0,\n \"customHeaders\": {},\n \"objectsFound\": 0,\n \"roundStartTime\": 0,\n \"urlCrawlPattern\": \"\",\n \"seedRecrawlFrequency\": -1,\n \"urlProcessRegEx\": \"\",\n \"pageProcessSuccesses\": 0,\n \"urlsHarvested\": 0,\n \"crawlDelay\": -1,\n \"currentTime\": 1649950325,\n \"useProxies\": 0,\n \"sentJobDoneNotification\": 0,\n \"currentTimeUTC\": 1649950325,\n \"name\": \"crawlTest\",\n \"notifyEmail\": \"\",\n \"pageCrawlSuccesses\": 0,\n \"pageProcessAttempts\": 0\n }\n ]\n}" + } + }, + "schema": { + "type": "object", + "oneOf": [ + { + "title": "Restarted Crawl Job", + "type": "object", + "properties": { + "response": { + "type": "string", + "example": "Successfully added urls for spidering." + }, + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobStatus": { + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Job is initializing." + }, + "status": { + "type": "integer", + "example": 0 + } + } + }, + "maxHops": { + "type": "integer", + "example": -1 + }, + "downloadJson": { + "type": "string", + "example": "...json" + }, + "urlProcessPattern": { + "type": "string", + "example": "" + }, + "jobCompletionTimeUTC": { + "type": "integer", + "example": 0 + }, + "maxRounds": { + "type": "integer", + "example": -1 + }, + "type": { + "type": "string", + "example": "crawl" + }, + "pageCrawlSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "urlCrawlRegEx": { + "type": "string", + "example": "" + }, + "pageProcessPattern": { + "type": "string", + "example": "" + }, + "apiUrl": { + "type": "string", + "example": "https://api.diffbot.com/v3/analyze" + }, + "useCanonical": { + "type": "integer", + "example": 1 + }, + "jobCreationTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "repeat": { + "type": "integer", + "example": 0 + }, + "downloadUrls": { + "type": "string", + "example": "...csv" + }, + "obeyRobots": { + "type": "integer", + "example": 1 + }, + "roundsCompleted": { + "type": "integer", + "example": 0 + }, + "pageCrawlAttempts": { + "type": "integer", + "example": 0 + }, + "notifyWebhook": { + "type": "string", + "example": "" + }, + "pageProcessSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "customHeaders": { + "type": "object", + "properties": {} + }, + "objectsFound": { + "type": "integer", + "example": 0 + }, + "roundStartTime": { + "type": "integer", + "example": 0 + }, + "urlCrawlPattern": { + "type": "string", + "example": "" + }, + "seedRecrawlFrequency": { + "type": "integer", + "example": -1 + }, + "urlProcessRegEx": { + "type": "string", + "example": "" + }, + "pageProcessSuccesses": { + "type": "integer", + "example": 0 + }, + "urlsHarvested": { + "type": "integer", + "example": 0 + }, + "crawlDelay": { + "type": "number", + "example": -1 + }, + "currentTime": { + "type": "integer", + "example": 1649950325 + }, + "useProxies": { + "type": "integer", + "example": 0 + }, + "sentJobDoneNotification": { + "type": "integer", + "example": 0 + }, + "currentTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "name": { + "type": "string", + "example": "crawlTest" + }, + "notifyEmail": { + "type": "string", + "example": "" + }, + "pageCrawlSuccesses": { + "type": "integer", + "example": 0 + }, + "pageProcessAttempts": { + "type": "integer", + "example": 0 + } + } + } + } + } + }, + { + "title": "Paused Crawl Job", + "type": "object", + "properties": { + "response": { + "type": "string", + "example": "Successfully added urls for spidering." + }, + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobStatus": { + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Job paused." + }, + "status": { + "type": "integer", + "example": 6 + } + } + }, + "maxHops": { + "type": "integer", + "example": -1 + }, + "downloadJson": { + "type": "string", + "example": "...json" + }, + "urlProcessPattern": { + "type": "string", + "example": "" + }, + "jobCompletionTimeUTC": { + "type": "integer", + "example": 0 + }, + "maxRounds": { + "type": "integer", + "example": -1 + }, + "type": { + "type": "string", + "example": "crawl" + }, + "pageCrawlSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "urlCrawlRegEx": { + "type": "string", + "example": "" + }, + "pageProcessPattern": { + "type": "string", + "example": "" + }, + "apiUrl": { + "type": "string", + "example": "https://api.diffbot.com/v3/analyze" + }, + "useCanonical": { + "type": "integer", + "example": 1 + }, + "jobCreationTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "repeat": { + "type": "integer", + "example": 0 + }, + "downloadUrls": { + "type": "string", + "example": "...csv" + }, + "obeyRobots": { + "type": "integer", + "example": 1 + }, + "roundsCompleted": { + "type": "integer", + "example": 0 + }, + "pageCrawlAttempts": { + "type": "integer", + "example": 0 + }, + "notifyWebhook": { + "type": "string", + "example": "" + }, + "pageProcessSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "customHeaders": { + "type": "object", + "properties": {} + }, + "objectsFound": { + "type": "integer", + "example": 0 + }, + "roundStartTime": { + "type": "integer", + "example": 0 + }, + "urlCrawlPattern": { + "type": "string", + "example": "" + }, + "seedRecrawlFrequency": { + "type": "integer", + "example": -1 + }, + "urlProcessRegEx": { + "type": "string", + "example": "" + }, + "pageProcessSuccesses": { + "type": "integer", + "example": 0 + }, + "urlsHarvested": { + "type": "integer", + "example": 0 + }, + "crawlDelay": { + "type": "number", + "example": -1 + }, + "currentTime": { + "type": "integer", + "example": 1649950325 + }, + "useProxies": { + "type": "integer", + "example": 0 + }, + "sentJobDoneNotification": { + "type": "integer", + "example": 0 + }, + "currentTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "name": { + "type": "string", + "example": "crawlTest" + }, + "notifyEmail": { + "type": "string", + "example": "" + }, + "pageCrawlSuccesses": { + "type": "integer", + "example": 0 + }, + "pageProcessAttempts": { + "type": "integer", + "example": 0 + } + } + } + } + } + }, + { + "title": "Deleted Crawl Job", + "type": "object", + "properties": { + "response": { + "type": "string", + "example": "Successfully deleted job." + } + } + }, + { + "title": "Completed Crawl Job", + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobStatus": { + "type": "object", + "properties": { + "message": { + "type": "string", + "example": "Job has completed and no repeat is scheduled." + }, + "status": { + "type": "integer", + "example": 9 + } + } + }, + "maxHops": { + "type": "integer", + "example": -1 + }, + "downloadJson": { + "type": "string", + "example": "...json" + }, + "urlProcessPattern": { + "type": "string", + "example": "" + }, + "jobCompletionTimeUTC": { + "type": "integer", + "example": 0 + }, + "maxRounds": { + "type": "integer", + "example": -1 + }, + "type": { + "type": "string", + "example": "crawl" + }, + "pageCrawlSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "urlCrawlRegEx": { + "type": "string", + "example": "" + }, + "pageProcessPattern": { + "type": "string", + "example": "" + }, + "apiUrl": { + "type": "string", + "example": "https://api.diffbot.com/v3/analyze" + }, + "useCanonical": { + "type": "integer", + "example": 1 + }, + "jobCreationTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "repeat": { + "type": "integer", + "example": 0 + }, + "downloadUrls": { + "type": "string", + "example": "...csv" + }, + "obeyRobots": { + "type": "integer", + "example": 1 + }, + "roundsCompleted": { + "type": "integer", + "example": 0 + }, + "pageCrawlAttempts": { + "type": "integer", + "example": 0 + }, + "notifyWebhook": { + "type": "string", + "example": "" + }, + "pageProcessSuccessesThisRound": { + "type": "integer", + "example": 0 + }, + "customHeaders": { + "type": "object", + "properties": {} + }, + "objectsFound": { + "type": "integer", + "example": 0 + }, + "roundStartTime": { + "type": "integer", + "example": 0 + }, + "urlCrawlPattern": { + "type": "string", + "example": "" + }, + "seedRecrawlFrequency": { + "type": "integer", + "example": -1 + }, + "urlProcessRegEx": { + "type": "string", + "example": "" + }, + "pageProcessSuccesses": { + "type": "integer", + "example": 0 + }, + "urlsHarvested": { + "type": "integer", + "example": 0 + }, + "crawlDelay": { + "type": "number", + "example": -1 + }, + "currentTime": { + "type": "integer", + "example": 1649950325 + }, + "useProxies": { + "type": "integer", + "example": 0 + }, + "sentJobDoneNotification": { + "type": "integer", + "example": 0 + }, + "currentTimeUTC": { + "type": "integer", + "example": 1649950325 + }, + "name": { + "type": "string", + "example": "crawlTest" + }, + "notifyEmail": { + "type": "string", + "example": "" + }, + "pageCrawlSuccesses": { + "type": "integer", + "example": 0 + }, + "pageProcessAttempts": { + "type": "integer", + "example": 0 + } + } + } + } + } + } + ] + } + } + } + }, + "500": { + "description": "No crawl job by that name", + "content": { + "application/json": { + "examples": { + "Crawl Job Not Found": { + "value": "{\n \"errorCode\": \"500\",\n \"error\": \"no collection record.\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "string", + "example": "500" + }, + "error": { + "type": "string", + "example": "no collection record." + } + } + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/crawl/data": { + "get": { + "summary": "Retrieve Crawl Job Data", + "description": "Download the extracted results of a crawl job.", + "operationId": "retrieve-crawl-job-data", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "Name of the crawl job whose data you wish to download.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "format", + "in": "query", + "description": "Request `format=csv` to download the extracted data in CSV format (default: `json`). Note that CSV files will only contain top-level fields.", + "schema": { + "type": "string" + } + }, + { + "name": "type", + "in": "query", + "description": "Request `type=urls` to retrieve the [URL Report (CSV)](/products/crawl/faq/url-report).", + "schema": { + "type": "string" + } + }, + { + "name": "num", + "in": "query", + "description": "Pass an integer value (e.g. `num=100`) to request a subset of URLs, most recently processed first.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The crawl job's extracted data", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "[\n {\n \"col\": \"example-test\",\n \"parentUrl\": \"https://www.example.com\",\n \"pageUrl\": \"https://www.example.com/\",\n \"id\": 1492074227214006500,\n \"type\": \"other\",\n \"title\": \"Example Domain\",\n \"content\": \"Example Domain \\n\\n# Example Domain \\n\\nThis domain is for use in documentation examples without needing permission. Avoid use in operations. \\n\\n[Learn more](https://iana.org/domains/example)\",\n \"token\": \"\",\n \"timestamp\": \"2026-07-24T23:15:09Z\"\n }\n]" + } + }, + "schema": { + "type": "array", + "items": { + "type": "object", + "required": [ + "col", + "parentUrl", + "pageUrl", + "id", + "type", + "token", + "timestamp" + ], + "properties": { + "col": { + "description": "Name of the Crawl or Bulk job this record was extracted from", + "type": "string", + "example": "example-test" + }, + "parentUrl": { + "description": "Immediate parent contained a link to this record", + "type": "string", + "example": "https://www.example.com" + }, + "pageUrl": { + "description": "The URL of this record", + "type": "string", + "example": "https://www.example.com/" + }, + "id": { + "description": "A reference", + "type": "integer", + "example": 1492074227214006500 + }, + "type": { + "description": "A standard entity type assigned either on job creation or classified by [Analyze API](/docs/extract/analyze). All other keys on this job record follow the [ontology](/docs/ontology/) set by its type.", + "type": "string", + "example": "other" + }, + "token": { + "description": "Your token", + "type": "string", + "example": "" + }, + "timestamp": { + "description": "Time when record was crawled, in ISO 8601 format.", + "type": "string", + "example": "2026-07-24T23:15:09Z" + } + }, + "additionalProperties": true + } + } + } + } + }, + "400": { + "description": "No crawl job by that name", + "content": { + "application/json": { + "examples": { + "Result": { + "value": "{\n \"errorCode\": 400,\n \"error\": \"No collection record\"\n}" + } + }, + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer", + "example": 400 + }, + "error": { + "type": "string", + "example": "No collection record" + } + } + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + } + }, + "components": { + "securitySchemes": { + "tokenscheme": { + "type": "apiKey", + "name": "token", + "in": "query" + } + } + } +} diff --git a/openapi/dql.json b/openapi/dql.json new file mode 100644 index 0000000..60cc67d --- /dev/null +++ b/openapi/dql.json @@ -0,0 +1,924 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "DQL API", + "description": "DQL is an API to query entities in the Knowledge Graph", + "termsOfService": "https://www.diffbot.com/terms/", + "contact": { + "email": "support@diffbot.com" + }, + "version": "1.0.0" + }, + "servers": [ + { + "url": "https://kg.diffbot.com", + "description": "Diffbot KG endpoint" + } + ], + "tags": [ + { + "name": "DQL Search", + "description": "DQL Search" + } + ], + "paths": { + "/kg/v3/dql": { + "get": { + "tags": [ + "DQL Search" + ], + "summary": "Search with DQL", + "description": "Search the Knowledge Graph using DQL", + "operationId": "dqlGet", + "parameters": [ + { + "name": "type", + "in": "query", + "description": "Query type", + "schema": { + "type": "string", + "enum": [ + "query", + "crawl" + ], + "default": "query" + } + }, + { + "name": "query", + "in": "query", + "description": "DQL Query", + "required": true, + "schema": { + "type": "string", + "example": "type:Organization locations.city.name:\"San Francisco\" nbEmployees>5000" + } + }, + { + "name": "col", + "in": "query", + "description": "Comma-delimited list of custom crawl collections to query. Specify `all` to query all collections for your token. This parameter is valid only when type=crawl and defaults to `all`", + "schema": { + "type": "string", + "default": "all" + } + }, + { + "name": "from", + "in": "query", + "description": "Return results starting at `from` index. `from+size <= 10_000` for facet queries.\n\nPlease note that the from parameter is no longer applicable to Article queries and has been deprecated. In the future, using this parameter will result in an error.", + "schema": { + "type": "integer", + "format": "int32", + "default": 0 + } + }, + { + "name": "size", + "in": "query", + "description": "Maximum number of results to return. `-1` to return all results. `from+size <= 10_000` for facet queries.", + "schema": { + "type": "integer", + "format": "int32", + "default": 50 + } + }, + { + "name": "format", + "in": "query", + "description": "Output format", + "schema": { + "type": "string", + "enum": [ + "json", + "jsonl", + "csv", + "xls", + "xlsx" + ], + "default": "json" + } + }, + { + "name": "exportspec", + "in": "query", + "description": "The spec defines the columns to export. This is applicable for csv, xls and xslx formats.\n\nExport spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields).", + "schema": { + "type": "string" + } + }, + { + "name": "exportseparator", + "in": "query", + "description": "The export separator", + "schema": { + "type": "string" + } + }, + { + "name": "exportfile", + "in": "query", + "description": "For export endpoint, The file name of the export file", + "schema": { + "type": "string" + } + }, + { + "name": "filter", + "in": "query", + "description": "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema": { + "type": "string" + } + }, + { + "name": "filterExclude", + "in": "query", + "description": "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema": { + "type": "string" + } + }, + { + "name": "jsonmode", + "in": "query", + "description": "`extended` returns origin information for facts. `id` returns just the diffbotIds and origins.", + "schema": { + "type": "string", + "enum": [ + "extended", + "id" + ] + } + }, + { + "name": "nonCanonicalFacts", + "in": "query", + "description": "`nonCanonicalFacts=true` returns non-canonical facts.", + "schema": { + "type": "boolean", + "default": false + } + }, + { + "name": "noDedupArticles", + "in": "query", + "description": "`noDedupArticles=true` indicates that articles should not be deduplicated but all of them returned", + "schema": { + "type": "boolean", + "default": false + } + }, + { + "name": "cluster", + "in": "query", + "description": "Articles only. Utilize Diffbot clustering algorithms to identify related or similar objects. \n- `cluster=all` to return each article with a clusterId identifying thematically-similar articles. \n- `cluster=best` to remove thematically-similar articles from the result set, leaving a single best representative from each cluster. \n- `cluster=dedupe` to remove exact and near-duplicate articles from the result set. Where applicable, clusterId is returned in the `entity_ctx` element of each response entity.", + "schema": { + "type": "string", + "enum": [ + "all", + "best", + "dedupe" + ] + } + }, + { + "name": "report", + "in": "query", + "description": "Generate coverage report. See [Coverage Reports](https://docs.diffbot.com/docs/dql-coverage-reports) for more details on generating coverage reports for DQL queries.", + "schema": { + "type": "boolean", + "default": false + } + }, + { + "name": "view", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Results" + }, + "400": { + "$ref": "#/components/responses/QueryParseError" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "422": { + "$ref": "#/components/responses/ParameterError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + }, + "security": [ + { + "tokenscheme": [] + } + ], + "x-group-parameters": true + }, + "post": { + "tags": [ + "DQL Search" + ], + "summary": "Search with DQL", + "description": "Search the Knowledge Graph using DQL", + "operationId": "dqlPost", + "requestBody": { + "description": "DQL request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DqlPostRequest" + }, + "examples": { + "Search": { + "summary": "A San Francisco org search", + "value": { + "query": "type:Organization locations.city.name:\"San Francisco\" nbEmployees>5000", + "size": 2, + "filter": "$.name;$.homepageUri;$.nbEmployees" + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Results" + }, + "400": { + "$ref": "#/components/responses/QueryParseError" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "422": { + "$ref": "#/components/responses/ParameterError" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + }, + "security": [ + { + "tokenscheme": [] + } + ], + "x-group-parameters": true + } + }, + "/kg/v3/dql/report": { + "get": { + "tags": [ + "DQL Search" + ], + "summary": "Coverage report by Query", + "description": "Download coverage report by DQL query", + "operationId": "reportFind", + "parameters": [ + { + "name": "query", + "in": "query", + "description": "DQL Query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "description": "Return results starting at `from` index. `from+size <= 10_000` for facet queries.\n\nPlease note that the from parameter is no longer applicable to Article queries and has been deprecated. In the future, using this parameter will result in an error.", + "schema": { + "type": "integer", + "format": "int32", + "default": 0 + } + }, + { + "name": "size", + "in": "query", + "description": "Maximum number of results to return. `-1` to return all results. `from+size <= 10_000` for facet queries.", + "schema": { + "type": "integer", + "format": "int32", + "default": 50 + } + }, + { + "name": "filter", + "in": "query", + "description": "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema": { + "type": "string" + } + }, + { + "name": "filterExclude", + "in": "query", + "description": "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema": { + "type": "string" + } + }, + { + "name": "exportspec", + "in": "query", + "description": "The spec defines the columns to export. This is applicable for csv, xls and xslx formats.\n\nExport spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields).", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Coverage report", + "content": { + "text/csv": { + "schema": { + "type": "string" + }, + "example": "DiffbotID,Name,$.name,$.nbEmployees\n\"\",,1.0,1.0\nEl0wZJw_cOn-MAyp_hZZ5Tw,Yandex,1,1\nEUFq-3WlpNsq0pvfUYWXOEA,Google,1,1\nEc5P22-6EPLOXo1-z6-4NzA,Instagram,1,1\n" + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/kg/v3/dql/report/{id}": { + "get": { + "tags": [ + "DQL Search" + ], + "summary": "Coverage report by ID", + "description": "Download coverage report by report ID", + "operationId": "reportGet", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "The report id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Coverage report", + "content": { + "text/csv": { + "schema": { + "type": "string" + }, + "example": "DiffbotID,Name,$.name,$.nbEmployees\n\"\",,1.0,1.0\nEl0wZJw_cOn-MAyp_hZZ5Tw,Yandex,1,1\nEUFq-3WlpNsq0pvfUYWXOEA,Google,1,1\nEc5P22-6EPLOXo1-z6-4NzA,Instagram,1,1\n" + } + } + }, + "404": { + "description": "Report not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ParameterError" + }, + "examples": { + "Not found": { + "summary": "Report not found", + "value": { + "error": true, + "message": "Report not found: C-4d87a965-b43f" + } + } + } + } + } + }, + "500": { + "$ref": "#/components/responses/ServerError" + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + } + }, + "components": { + "schemas": { + "Data": { + "type": "object", + "properties": { + "score": { + "type": "number", + "description": "score", + "format": "double", + "example": 0.75 + }, + "entity": { + "$ref": "#/components/schemas/JSONObject" + }, + "entity_ctx": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Entity-level context" + }, + "description": "Entity-level context" + }, + "errors": { + "type": "array", + "items": { + "type": "string", + "description": "Error/warning messages." + } + }, + "callbackQuery": { + "type": "string", + "description": "The query for the facet" + }, + "upperBound": { + "type": "number", + "description": "The upper bound of the facet", + "format": "double" + }, + "lowerBound": { + "type": "number", + "description": "The lower bound of the facet", + "format": "double" + }, + "count": { + "type": "integer", + "description": "The count of the facet", + "format": "int64" + }, + "value": { + "type": "string", + "description": "The value of the facet" + }, + "uri": { + "type": "string", + "description": "The uri of the facet value (for uri facets)" + } + }, + "description": "A single result" + }, + "DqlResponse": { + "type": "object", + "properties": { + "version": { + "type": "integer", + "description": "Response version", + "format": "int32", + "example": 3 + }, + "hits": { + "type": "integer", + "format": "int64" + }, + "results": { + "type": "integer", + "description": "Number of results in this response", + "format": "int64", + "example": 50 + }, + "kgversion": { + "type": "string", + "description": "Version of KG that was queried", + "example": "250" + }, + "diffbot_type": { + "type": "string", + "description": "Diffbot type of returned results", + "example": "Organization" + }, + "facet": { + "type": "boolean", + "description": "Was this a facet query?" + }, + "data": { + "type": "array", + "description": "Results of the query", + "items": { + "$ref": "#/components/schemas/Data" + } + }, + "rewrites": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Rewrite" + } + }, + "textFallback": { + "type": "boolean", + "description": "Whether the query fell back to text search.", + "example": false + }, + "reportId": { + "type": "string", + "description": "Identifier for the generated coverage report. Present only when `report=true` was passed.", + "example": "C-4d87a965-b43f" + } + } + }, + "JSONObject": { + "type": "object", + "properties": { + "empty": { + "type": "boolean" + } + }, + "description": "Entity. Can be null if no entity was found.", + "example": { + "id": "EYX1i02YVPsuT7fPLUYgRhQ", + "diffbotUri": "http://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + "name": "Diffbot", + "origins": [ + "angel.co/diffbot", + "bloomberg.com/profiles/companies/0506286D:US-diffbot-corp", + "crunchbase.com/organization/diffbot", + "linkedin.com/company/diffbot", + "en.wikipedia.org/wiki/Diffbot" + ] + } + }, + "Rewrite": { + "type": "object", + "properties": { + "query": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "DqlPostRequest": { + "required": [ + "query" + ], + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Query type", + "enum": [ + "query", + "text", + "queryTextFallback", + "crawl" + ], + "default": "query" + }, + "query": { + "type": "string", + "description": "DQL Query" + }, + "col": { + "type": "string", + "description": "Comma-delimited list of custom crawl collections to query. Specify `all` to query all collections for your token. This parameter is valid only when type=crawl and defaults to `all`" + }, + "from": { + "type": "integer", + "description": "Return results starting at `from` index. `from+size <= 10_000` for facet queries.\n\nPlease note that the from parameter is no longer applicable to Article queries and has been deprecated. In the future, using this parameter will result in an error.", + "format": "int32", + "default": 0 + }, + "size": { + "type": "integer", + "description": "Maximum number of results to return. `-1` to return all results. `from+size <= 10_000` for facet queries.", + "format": "int32", + "default": 50 + }, + "format": { + "type": "string", + "description": "Output format", + "enum": [ + "json", + "jsonl", + "csv", + "xls", + "xlsx" + ], + "default": "json" + }, + "exportspec": { + "type": "string", + "description": "The spec defines the columns to export. This is applicable for csv, xls and xslx formats.\n\nExport spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields)." + }, + "exportseparator": { + "type": "string", + "description": "For export endpoint, define separator for fields which contains multiple values" + }, + "exportfile": { + "type": "string", + "description": "For export endpoint, The file name of the export file" + }, + "filter": { + "type": "string", + "description": "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + }, + "filterExclude": { + "type": "string", + "description": "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + }, + "jsonmode": { + "type": "string", + "description": "`extended` returns origin information for facts. `id` returns just the diffbotIds and origins.", + "enum": [ + "extended", + "id" + ] + }, + "nonCanonicalFacts": { + "type": "boolean", + "description": "`nonCanonicalFacts=true` returns non-canonical facts.", + "default": false + }, + "noDedupArticles": { + "type": "boolean", + "description": "`noDedupArticles=true` indicates that articles should not be deduplicated but all of them returned", + "default": false + }, + "cluster": { + "type": "string", + "description": "Articles only. Utilize Diffbot clustering algorithms to identify related or similar objects. \n- `cluster=all` to return each article with a clusterId identifying thematically-similar articles. \n- `cluster=best` to remove thematically-similar articles from the result set, leaving a single best representative from each cluster. \n- `cluster=dedupe` to remove exact and near-duplicate articles from the result set. Where applicable, clusterId is returned in the `entity_ctx` element of each response entity.", + "enum": [] + }, + "report": { + "type": "boolean", + "description": "Generate coverage report. See [Coverage Reports](https://docs.diffbot.com/docs/dql-coverage-reports) for more details on generating coverage reports for DQL queries.", + "default": false + }, + "view": { + "type": "string" + } + } + }, + "QueryParseError": { + "type": "object", + "description": "Returned when the DQL query itself cannot be parsed.", + "properties": { + "error": { + "type": "boolean", + "example": true + }, + "message": { + "type": "string", + "example": "Encountered \" \":\" \": \"\" at line 1, column 6." + }, + "line": { + "type": "integer", + "example": 1 + }, + "column": { + "type": "integer", + "example": 6 + }, + "htmlMessage": { + "type": "string", + "description": "The message with the offending token wrapped in `` for display.", + "example": "type:::Bogus((" + } + } + }, + "ParameterError": { + "type": "object", + "description": "Returned when a request parameter is missing or invalid.", + "properties": { + "error": { + "type": "boolean", + "example": true + }, + "message": { + "type": "string", + "example": "Parameter 'query' not specified" + } + } + }, + "AuthError": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 401 + }, + "message": { + "type": "string", + "example": "Unauthorized. Incorrect token." + }, + "requestId": { + "type": "string", + "example": "a9cbf96bb0e2084d5c1595ae8f526576" + } + } + }, + "ServerError": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 500 + }, + "message": { + "type": "string", + "example": "HTTP 500 Internal Server Error" + } + } + } + }, + "securitySchemes": { + "tokenscheme": { + "type": "apiKey", + "name": "token", + "in": "query", + "description": "Your Diffbot API token." + } + }, + "responses": { + "Results": { + "description": "Query results", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DqlResponse" + }, + "examples": { + "OK": { + "summary": "OK", + "value": { + "version": 3, + "hits": 642, + "results": 2, + "kgversion": "477", + "diffbot_type": "Organization", + "facet": false, + "textFallback": false, + "data": [ + { + "score": 1584.3050537109375, + "entity": { + "name": "Yandex", + "homepageUri": "yandex.com", + "nbEmployees": 26361 + }, + "entity_ctx": { + "inner_hits": { + "locations": [ + 39 + ] + }, + "matched_clauses": [] + } + }, + { + "score": 1558.7650146484375, + "entity": { + "name": "Google", + "homepageUri": "google.com", + "nbEmployees": 187000 + }, + "entity_ctx": { + "inner_hits": { + "locations": [ + 192, + 161, + 261, + 215 + ] + }, + "matched_clauses": [] + } + } + ] + } + } + } + }, + "application/json-lines": { + "schema": { + "type": "object" + } + }, + "text/csv": { + "schema": { + "type": "object" + } + }, + "application/ms-excel": { + "schema": { + "type": "object" + } + }, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + "schema": { + "type": "object" + } + } + } + }, + "QueryParseError": { + "description": "Error parsing the query", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryParseError" + }, + "examples": { + "Parse error": { + "summary": "Malformed DQL", + "value": { + "line": 1, + "column": 6, + "htmlMessage": "type:::Bogus((", + "error": true, + "message": "Encountered \" \":\" \": \"\" at line 1, column 6." + } + } + } + } + } + }, + "Unauthorized": { + "description": "Unauthorized", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AuthError" + }, + "examples": { + "Incorrect token": { + "summary": "Incorrect token", + "value": { + "code": 401, + "message": "Unauthorized. Incorrect token.", + "requestId": "a9cbf96bb0e2084d5c1595ae8f526576" + } + } + } + } + } + }, + "ParameterError": { + "description": "Invalid or missing parameters", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ParameterError" + }, + "examples": { + "Missing query": { + "summary": "Missing query", + "value": { + "error": true, + "message": "Parameter 'query' not specified" + } + } + } + } + } + }, + "ServerError": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServerError" + }, + "examples": { + "Server error": { + "summary": "Server error", + "value": { + "code": 500, + "message": "HTTP 500 Internal Server Error" + } + } + } + } + } + } + } + } +} diff --git a/openapi/enhance.json b/openapi/enhance.json new file mode 100644 index 0000000..9ec6758 --- /dev/null +++ b/openapi/enhance.json @@ -0,0 +1,2099 @@ +{ + "openapi" : "3.0.1", + "info" : { + "title" : "ENHANCE API", + "description" : "Enhance is an API to find a person or organization in the Knowledge Graph using partial data", + "termsOfService" : "https://www.diffbot.com/terms/", + "contact" : { + "email" : "support@diffbot.com" + }, + "version" : "3.0.0" + }, + "servers" : [ { + "url" : "https://kg.diffbot.com", + "description" : "Diffbot KG endpoint" + } ], + "tags" : [ { + "name" : "Combine", + "description" : "Enhance both a person and the employer data" + }, { + "name" : "Enhance", + "description" : "Enhance single records synchronously" + }, { + "name" : "Bulk Enhance", + "description" : "Enhance multiple records asynchronously" + } ], + "paths" : { + "/kg/v3/enhance" : { + "get" : { + "tags" : [ "Enhance" ], + "summary" : "Enhance", + "description" : "Enrich a person or organization record with partial data input", + "operationId" : "enhance", + "parameters" : [ { + "name" : "type", + "in" : "query", + "description" : "Diffbot entity type", + "required" : true, + "schema" : { + "type" : "string", + "enum" : [ "Person", "Organization" ] + } + }, { + "name" : "id", + "in" : "query", + "description" : "DiffbotId of entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "name", + "in" : "query", + "description" : "Name of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, { + "name" : "url", + "in" : "query", + "description" : "Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, { + "name" : "phone", + "in" : "query", + "description" : "Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "email", + "in" : "query", + "description" : "Email of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, { + "name" : "employer", + "in" : "query", + "description" : "Employer of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "title", + "in" : "query", + "description" : "Title of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "school", + "in" : "query", + "description" : "School of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "location", + "in" : "query", + "description" : "Location of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "description", + "in" : "query", + "description" : "Description of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "ip", + "in" : "query", + "description" : "IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "customId", + "in" : "query", + "description" : "User defined ID for correlation", + "schema" : { + "type" : "string" + } + }, { + "name" : "size", + "in" : "query", + "description" : "Maximum number of results to return", + "schema" : { + "type" : "integer", + "format" : "int32", + "default" : 1 + } + }, { + "name" : "threshold", + "in" : "query", + "description" : "Enhance similarity threshold", + "schema" : { + "type" : "number", + "format" : "double" + } + }, { + "name" : "refresh", + "in" : "query", + "description" : "`refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "search", + "in" : "query", + "description" : "`search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "nonCanonicalFacts", + "in" : "query", + "description" : "`nonCanonicalFacts=true` returns non-canonical facts.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "jsonmode", + "in" : "query", + "description" : "`jsonmode=extended` returns origin information for facts.", + "schema" : { + "type" : "string", + "enum" : [ "extended" ] + } + }, { + "name" : "filter", + "in" : "query", + "description" : "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema" : { + "type" : "string" + } + }, { + "name" : "filterExclude", + "in" : "query", + "description" : "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Enhanced entity", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/EnhanceResponse" + } + } + } + }, + "400" : { + "description" : "Error parsing request", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "429" : { + "description" : "Insufficient credits", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + }, + "post" : { + "tags" : [ "Enhance" ], + "summary" : "Enhance", + "description" : "Enrich a person or organization record with partial data input (POST option)", + "operationId" : "enhancePost", + "requestBody" : { + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/EnhancePostRequest" + } + } + } + }, + "responses" : { + "200" : { + "description" : "Enhanced entity", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/EnhanceResponse" + } + } + } + }, + "400" : { + "description" : "Error parsing request", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "429" : { + "description" : "Insufficient credits", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + } + }, + "/kg/v3/enhance/combine" : { + "get" : { + "tags" : [ "Combine", "Enhance" ], + "summary" : "Combine", + "description" : "Enrich a person record and return both person and current employer data", + "operationId" : "combine", + "parameters" : [ { + "name" : "type", + "in" : "query", + "description" : "Diffbot entity type", + "required" : true, + "schema" : { + "type" : "string", + "enum" : [ "Person" ] + } + }, { + "name" : "name", + "in" : "query", + "description" : "Name of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, { + "name" : "url", + "in" : "query", + "description" : "Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, { + "name" : "phone", + "in" : "query", + "description" : "Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "email", + "in" : "query", + "description" : "Email of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "employer", + "in" : "query", + "description" : "Employer of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "title", + "in" : "query", + "description" : "Title of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "school", + "in" : "query", + "description" : "School of the entity to enhance. Parameter can be used only with type `Person`", + "schema" : { + "type" : "string" + } + }, { + "name" : "location", + "in" : "query", + "description" : "Location of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "ip", + "in" : "query", + "description" : "IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "schema" : { + "type" : "string" + } + }, { + "name" : "customId", + "in" : "query", + "description" : "User defined ID for correlation", + "schema" : { + "type" : "string" + } + }, { + "name" : "threshold", + "in" : "query", + "description" : "Enhance similarity threshold", + "schema" : { + "type" : "number", + "format" : "double" + } + }, { + "name" : "refresh", + "in" : "query", + "description" : "`refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "search", + "in" : "query", + "description" : "`search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "nonCanonicalFacts", + "in" : "query", + "description" : "`nonCanonicalFacts=true` returns non-canonical facts.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "jsonmode", + "in" : "query", + "description" : "`jsonmode=extended` returns origin information for facts.", + "schema" : { + "type" : "string", + "enum" : [ "extended" ] + } + }, { + "name" : "filter", + "in" : "query", + "description" : "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema" : { + "type" : "string" + } + }, { + "name" : "filterExclude", + "in" : "query", + "description" : "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Enhanced entity", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/EnhanceResponse" + } + } + } + }, + "400" : { + "description" : "Error parsing request", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "429" : { + "description" : "Insufficient credits", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + } + }, + "/kg/v3/enhance/bulk" : { + "post" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Create a Bulkjob", + "description" : "Enhance multiple records in bulk asynchronously", + "operationId" : "submitBulkjob", + "parameters" : [ { + "name" : "name", + "in" : "query", + "description" : "Enhance Bulkjob name", + "schema" : { + "type" : "string" + } + }, { + "name" : "size", + "in" : "query", + "description" : "Maximum number of results to return", + "schema" : { + "type" : "integer", + "format" : "int32", + "default" : 1 + } + }, { + "name" : "threshold", + "in" : "query", + "description" : "Enhance similarity threshold", + "schema" : { + "type" : "number", + "format" : "double" + } + }, { + "name" : "refresh", + "in" : "query", + "description" : "`refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "search", + "in" : "query", + "description" : "`search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "nonCanonicalFacts", + "in" : "query", + "description" : "`nonCanonicalFacts=true` returns non-canonical facts.", + "schema" : { + "type" : "boolean", + "default" : false + } + }, { + "name" : "jsonmode", + "in" : "query", + "description" : "`jsonmode=extended` returns origin information for facts.", + "schema" : { + "type" : "string", + "enum" : [ "extended" ] + } + }, { + "name" : "webhookurl", + "in" : "query", + "description" : "Webhook URL", + "schema" : { + "type" : "string" + } + } ], + "requestBody" : { + "description" : "Bulk query payload", + "content" : { + "application/json" : { + "schema" : { + "type" : "array", + "items" : { + "oneOf" : [ { + "$ref" : "#/components/schemas/OrganizationRequest" + }, { + "$ref" : "#/components/schemas/PersonRequest" + }, { + "$ref" : "#/components/schemas/IdRequestEntity" + } ] + } + } + } + } + }, + "responses" : { + "202" : { + "description" : "Acknowledgement with bulkjobId", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/BulkjobAccepted" + } + } + } + }, + "400" : { + "description" : "Error parsing request", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "429" : { + "description" : "Insufficient credits", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ], + "x-api-order" : "0" + } + }, + "/kg/v3/enhance/bulk/status" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "List Bulkjobs for Token", + "description" : "Poll the status of all Enhance Bulkjobs for a token", + "operationId" : "bulkjobStatusForToken", + "responses" : { + "200" : { + "description" : "Success status", + "content" : { + "application/json" : { + "schema" : { + "type" : "array", + "items" : { + "$ref" : "#/components/schemas/BulkjobStatus" + } + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ], + "x-api-order" : "1" + } + }, + "/kg/v3/enhance/bulk/{bulkjobId}/status" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Poll bulkjob status", + "description" : "Poll the status of an Enhance Bulkjob", + "operationId" : "bulkjobStatus", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Bulkjob complete", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/BulkjobStatusResponse" + } + } + } + }, + "201" : { + "description" : "Bulkjob is still executing or is stopped", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/BulkjobStatusResponse" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Bulkjob not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ], + "x-api-order" : "2" + } + }, + "/kg/v3/enhance/bulk/{bulkjobId}" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Download Results of Bulkjob", + "description" : "Download the result of a completed Enhance Bulkjob", + "operationId" : "bulkjobResultGet", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + }, { + "name" : "head", + "in" : "query", + "description" : "Return first n results", + "schema" : { + "type" : "integer", + "format" : "int32", + "default" : 0 + } + }, { + "name" : "from", + "in" : "query", + "description" : "1-based index of the first result to return (1 = first result).", + "schema" : { + "type" : "integer", + "format" : "int32", + "default" : 0 + } + }, { + "name" : "size", + "in" : "query", + "description" : "Maximum number of results to return starting from the 'from' position.", + "schema" : { + "type" : "integer", + "format" : "int32", + "default" : -1 + } + }, { + "name" : "onlyMatches", + "in" : "query", + "description" : "Return only records that have a match", + "schema" : { + "type" : "string", + "default" : "false", + "enum" : [ "true", "false" ] + } + }, { + "name" : "format", + "in" : "query", + "description" : "Export format", + "schema" : { + "type" : "string", + "enum" : [ "json", "jsonl", "csv", "xls", "xlsx" ] + } + }, { + "name" : "exportspec", + "in" : "query", + "description" : "The spec defines the columns to export. This is applicable for csv, xls and xslx formats.\n\nExport spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields).", + "schema" : { + "type" : "string" + } + }, { + "name" : "exportfile", + "in" : "query", + "description" : "File name of the export file", + "schema" : { + "type" : "string" + } + }, { + "name" : "exportseparator", + "in" : "query", + "description" : "Separator for multi-value fields when exporting columnar results", + "schema" : { + "type" : "string" + } + }, { + "name" : "exportquery", + "in" : "query", + "description" : "Prefixes the enhance query parameters to the CSV export result", + "schema" : { + "type" : "string", + "default" : "false", + "enum" : [ "true", "false" ] + } + }, { + "name" : "filter", + "in" : "query", + "description" : "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema" : { + "type" : "string" + } + }, { + "name" : "filterExclude", + "in" : "query", + "description" : "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax.", + "schema" : { + "type" : "string" + } + }, { + "name" : "wait", + "in" : "query", + "description" : "Seconds to wait for bulkjob results to export. Results will continue to export in the background. To only trigger an export, use 0.", + "schema" : { + "type" : "integer", + "format" : "int32", + "default" : 120 + } + } ], + "responses" : { + "200" : { + "description" : "Result data. Depends on format (default is json lines).", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/JsonNode" + } + }, + "application/json-lines" : { + "schema" : { + "type" : "object" + } + }, + "text/csv" : { + "schema" : { + "type" : "object" + } + }, + "application/ms-excel" : { + "schema" : { + "type" : "object" + } + }, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" : { + "schema" : { + "type" : "object" + } + } + } + }, + "201" : { + "description" : "Bulkjob or result export is still executing or is stopped", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "400" : { + "description" : "Invalid bulkjobId", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Bulkjob not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ], + "x-api-order" : "4" + }, + "post" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Download Results of Bulkjob", + "description" : "Download the result of a completed Enhance Bulkjob", + "operationId" : "bulkjobResultPost", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "requestBody" : { + "description" : "Bulk results request", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/BulkjobResultPostRequest" + } + } + } + }, + "responses" : { + "200" : { + "description" : "Result data. Depends on format (default is json lines).", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/JsonNode" + } + }, + "application/json-lines" : { + "schema" : { + "type" : "object" + } + }, + "text/csv" : { + "schema" : { + "type" : "object" + } + }, + "application/ms-excel" : { + "schema" : { + "type" : "object" + } + }, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" : { + "schema" : { + "type" : "object" + } + } + } + }, + "201" : { + "description" : "Bulkjob is still executing or is stopped", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "400" : { + "description" : "Invalid bulkjobId", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Bulkjob not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + } + }, + "/kg/v3/enhance/bulk/report/{bulkjobId}/{reportId}" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Download Bulkjob Coverage Report", + "description" : "Download the coverage report of a completed Bulk Enhance job", + "operationId" : "coverageReportGet", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + }, { + "name" : "reportId", + "in" : "path", + "description" : "Enhance Report ID", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Coverage report (csv)", + "content" : { + "text/csv" : { + "schema" : { + "type" : "object" + } + }, + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/JsonNode" + } + } + } + }, + "400" : { + "description" : "Report generation still in progress", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Report not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ], + "x-api-order" : "4" + } + }, + "/kg/v3/enhance/bulk/{bulkjobId}/delete" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Delete Bulkjob", + "description" : "Delete bulkjob", + "operationId" : "deleteBulkjob", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Bulkjob was successfully deleted" + }, + "400" : { + "description" : "Invalid bulkjobId", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Bulkjob not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + } + }, + "/kg/v3/enhance/bulk/{bulkjobId}/{jobIdx}" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Download Single Result of Bulkjob", + "description" : "Use this API to download the result of a single job within a bulkjob by specifying the index of the job.", + "operationId" : "singleResult", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + }, { + "name" : "jobIdx", + "in" : "path", + "description" : "Job index within the bulkjob", + "required" : true, + "schema" : { + "type" : "integer", + "format" : "int32" + } + } ], + "responses" : { + "200" : { + "description" : "Enhance result", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/EnhanceResponse" + } + } + } + }, + "201" : { + "description" : "Bulkjob is still executing or is stopped", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "400" : { + "description" : "Invalid bulkjobId", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Bulkjob not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + } + }, + "/kg/v3/enhance/bulk/{bulkjobId}/stop" : { + "get" : { + "tags" : [ "Bulk Enhance" ], + "summary" : "Stop Bulkjob", + "description" : "Stop an active Enhance Bulkjob", + "operationId" : "stopBulkjob", + "parameters" : [ { + "name" : "bulkjobId", + "in" : "path", + "description" : "Enhance Bulkjob ID", + "required" : true, + "schema" : { + "type" : "string" + } + } ], + "responses" : { + "200" : { + "description" : "Bulkjob status after the stop request", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/BulkjobRecoveryStatusResponse" + } + } + } + }, + "400" : { + "description" : "Invalid bulkjobId", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "401" : { + "description" : "Token not specified or other client errors", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/AuthError" + } + } + } + }, + "404" : { + "description" : "Bulkjob not found", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + }, + "500" : { + "description" : "Internal Server Error", + "content" : { + "application/json" : { + "schema" : { + "$ref" : "#/components/schemas/StatusResponseError" + } + } + } + } + }, + "security" : [ { + "tokenscheme" : [ ] + } ] + } + } + }, + "components" : { + "schemas" : { + "Data" : { + "type" : "object", + "properties" : { + "score" : { + "type" : "number", + "description" : "Enhance score", + "format" : "double" + }, + "esscore" : { + "type" : "number", + "format" : "double" + }, + "entity" : { + "$ref" : "#/components/schemas/ObjectNode" + }, + "errors" : { + "type" : "array", + "items" : { + "type" : "string", + "description" : "Error/warning messages." + } + }, + "refreshInfo" : { + "type" : "object", + "additionalProperties" : { + "$ref" : "#/components/schemas/RefreshDebugInformation" + }, + "description" : "Refresh info for debug" + }, + "refreshDelta" : { + "$ref" : "#/components/schemas/Delta" + }, + "entity_ctx" : { + "type" : "object", + "additionalProperties" : { + "type" : "object", + "description" : "Entity-level context" + }, + "description" : "Entity-level context" + } + }, + "description" : "Enhanced entity data" + }, + "Delta" : { + "type" : "object" + }, + "EnhanceResponse" : { + "type" : "object", + "properties" : { + "version" : { + "type" : "integer", + "description" : "Response version", + "format" : "int32" + }, + "hits" : { + "type" : "integer", + "description" : "Number of matches returned", + "format" : "int32" + }, + "kgversion" : { + "type" : "string", + "description" : "Version of KG that was queried" + }, + "request_ctx" : { + "$ref" : "#/components/schemas/RequestContext" + }, + "data" : { + "type" : "array", + "items" : { + "$ref" : "#/components/schemas/Data" + } + }, + "errors" : { + "type" : "array", + "items" : { + "type" : "string", + "description" : "Error/warning messages." + } + }, + "searchInfo" : { + "type" : "object", + "additionalProperties" : { + "$ref" : "#/components/schemas/SearchCandidate" + }, + "description" : "Search info for debug" + } + } + }, + "ObjectNode" : { + "type" : "object", + "description" : "Enhanced entity. Can be null if no entity was found.", + "example" : { + "id" : "EYX1i02YVPsuT7fPLUYgRhQ", + "images" : [ { + "naturalHeight" : 0, + "width" : 0, + "url" : "https://kg.diffbot.com/image/api/get?fetch=yes&url=g%3Cj7P0St%7D9u.LnOv%7BI%7C%40g7.pBYINs%3Ff%7CGZ7jr-tGZhWf%3DK%2FOvy%5Bp%3CSKbs8i%7CEyNzCa%3EYr%3Ef%3CRc%3CLr5gx_hOgiFuER5%3Cb%7C%3E.I%5Do", + "isCached" : true, + "naturalWidth" : 0, + "primary" : true, + "height" : 0 + } ], + "diffbotUri" : "http://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + "name" : "Diffbot", + "origins" : [ "angel.co/diffbot", "bloomberg.com/profiles/companies/0506286D:US-diffbot-corp", "crunchbase.com/organization/diffbot", "linkedin.com/company/diffbot", "en.wikipedia.org/wiki/Diffbot" ] + } + }, + "OrganizationRequest" : { + "type" : "object", + "properties" : { + "type" : { + "type" : "string", + "description" : "Diffbot entity type" + }, + "name" : { + "type" : "array", + "description" : "Name of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "items" : { + "type" : "string" + } + }, + "url" : { + "type" : "array", + "description" : "Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization`", + "items" : { + "type" : "string" + } + }, + "location" : { + "type" : "string", + "description" : "Location of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "ip" : { + "type" : "string", + "description" : "IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "phone" : { + "type" : "string", + "description" : "Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "description" : { + "type" : "string", + "description" : "Description of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + } + } + }, + "PersonRequest" : { + "type" : "object", + "properties" : { + "type" : { + "type" : "string", + "description" : "Diffbot entity type" + }, + "name" : { + "type" : "array", + "description" : "Name of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "items" : { + "type" : "string" + } + }, + "url" : { + "type" : "array", + "description" : "Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization`", + "items" : { + "type" : "string" + } + }, + "location" : { + "type" : "string", + "description" : "Location of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "ip" : { + "type" : "string", + "description" : "IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "phone" : { + "type" : "string", + "description" : "Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "description" : { + "type" : "string", + "description" : "Description of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "email" : { + "type" : "array", + "description" : "Email of the entity to enhance. Parameter can be used only with type `Person`", + "items" : { + "type" : "string" + } + }, + "employer" : { + "type" : "string", + "description" : "Employer of the entity to enhance. Parameter can be used only with type `Person`" + }, + "title" : { + "type" : "string", + "description" : "Title of the entity to enhance. Parameter can be used only with type `Person`" + }, + "school" : { + "type" : "string", + "description" : "School of the entity to enhance. Parameter can be used only with type `Person`" + } + } + }, + "RefreshDebugInformation" : { + "type" : "object", + "properties" : { + "url" : { + "type" : "string" + }, + "hasResult" : { + "type" : "boolean" + }, + "lastCrawlTime" : { + "type" : "integer", + "format" : "int64" + }, + "message" : { + "type" : "string" + } + } + }, + "RequestContext" : { + "type" : "object", + "properties" : { + "query" : { + "description" : "Submitted query", + "oneOf" : [ { + "$ref" : "#/components/schemas/OrganizationRequest" + }, { + "$ref" : "#/components/schemas/PersonRequest" + }, { + "type" : "string" + } ] + }, + "query_ctx" : { + "type" : "object", + "additionalProperties" : { + "type" : "object", + "description" : "Query context. Example: submitted query parameters etc." + }, + "description" : "Query context. Example: submitted query parameters etc." + } + }, + "description" : "Request context" + }, + "SearchCandidate" : { + "type" : "object", + "properties" : { + "mlScore" : { + "type" : "number", + "format" : "double" + }, + "baseMlScore" : { + "type" : "number", + "format" : "double" + }, + "debugInfo" : { + "$ref" : "#/components/schemas/RefreshDebugInformation" + } + }, + "description" : "Search info for debug" + }, + "BulkjobStatus" : { + "type" : "object", + "properties" : { + "job_id" : { + "type" : "string", + "description" : "Bulkjob Id" + }, + "name" : { + "type" : "string", + "description" : "Bulkjob name" + }, + "refresh" : { + "type" : "boolean", + "description" : "Is Refresh enabled for this job?" + }, + "search" : { + "type" : "string", + "description" : "Is Search enabled for this job?", + "enum" : [ "ON", "OFF", "SEARCHONLY", "FALLBACK" ] + }, + "message" : { + "type" : "string", + "description" : "Bulkjob status message" + }, + "jobs_total" : { + "type" : "integer", + "description" : "Number of jobs in bulkjob", + "format" : "int32" + }, + "jobs_completed" : { + "type" : "integer", + "description" : "Number of jobs completed in bulkjob.", + "format" : "int32" + }, + "status" : { + "type" : "string", + "description" : "Status of Bulkjob. One of {NOT_STARTED, IN_PROCESS, COMPLETE, COMPLETE_WITH_FAILURES, STOPPED, ERROR_FINALIZING, UNKNOWN}", + "enum" : [ "NOT_STARTED", "IN_PROCESS", "COMPLETE", "COMPLETE_WITH_FAILURES", "STOPPED", "ERROR_FINALIZING", "ERROR_SCHEDULING", "UNKNOWN" ] + }, + "estimate_remaining_time" : { + "type" : "integer", + "description" : "Estimated remaining time for bulkjob to complete (in seconds). `null` if job is completed or not started.", + "format" : "int64" + }, + "submit_time_ms" : { + "type" : "integer", + "description" : "Bulkjob submit time", + "format" : "int64" + }, + "start_time_ms" : { + "type" : "integer", + "description" : "Bulkjob start time", + "format" : "int64" + }, + "completed_time_ms" : { + "type" : "integer", + "description" : "Bulkjob completed time", + "format" : "int64" + }, + "total_results" : { + "type" : "integer", + "description" : "Total results", + "format" : "int32" + }, + "bulkjobId" : { + "type" : "string" + }, + "totalJobs" : { + "type" : "integer", + "format" : "int32" + }, + "finishedJobs" : { + "type" : "integer", + "format" : "int32" + }, + "estimateRemainingTimeSec" : { + "type" : "integer", + "format" : "int64" + }, + "submitTimeMs" : { + "type" : "integer", + "format" : "int64" + }, + "startTimeMs" : { + "type" : "integer", + "format" : "int64" + }, + "completedTimeMs" : { + "type" : "integer", + "format" : "int64" + }, + "totalResults" : { + "type" : "integer", + "format" : "int32" + }, + "fileSizeBytes" : { + "type" : "integer", + "description" : "Size of download file in bytes", + "format" : "int64" + }, + "reports" : { + "type" : "array", + "description" : "Coverage reports generated for this bulkjob", + "items" : { + "$ref" : "#/components/schemas/Report" + } + } + }, + "description" : "Bulkjob status" + }, + "Report" : { + "type" : "object", + "properties" : { + "reportId" : { + "type" : "string", + "description" : "Coverage Report ID" + }, + "status" : { + "type" : "string", + "description" : "Status of report generation. One of {NOT_STARTED, IN_PROCESS, COMPLETE, UNKNOWN}", + "enum" : [ "NOT_STARTED", "IN_PROCESS", "COMPLETE", "COMPLETE_WITH_FAILURES", "STOPPED", "ERROR_FINALIZING", "ERROR_SCHEDULING", "UNKNOWN" ] + }, + "format" : { + "type" : "string", + "description" : "Export format" + }, + "spec" : { + "type" : "string", + "description" : "The spec defines the columns to export. This is applicable for csv, xls and xslx formats.\n\nExport spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields)." + }, + "exportfile" : { + "type" : "string" + }, + "exportSeparator" : { + "type" : "string", + "description" : "Separator for multi-value fields when exporting columnar results" + }, + "filter" : { + "type" : "string", + "description" : "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + }, + "onlyMatches" : { + "type" : "boolean", + "description" : "Return only records that have a match" + } + }, + "description" : "Coverage reports generated for this bulkjob" + }, + "StatusResponseError" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "integer", + "description" : "Status code (same as the HTTP status code)", + "format" : "int32" + }, + "message" : { + "type" : "string", + "description" : "Status message" + }, + "request_ctx" : { + "$ref" : "#/components/schemas/RequestContext" + }, + "bulkjobStatus" : { + "$ref" : "#/components/schemas/BulkjobStatus" + }, + "error" : { + "type" : "boolean", + "description" : "`true` if error" + } + } + }, + "AuthError" : { + "type" : "object", + "properties" : { + "requestId" : { + "type" : "string", + "description" : "Identifier of the rejected request, also returned in the `X-Diffbot-RequestId` header" + }, + "code" : { + "type" : "integer", + "description" : "Status code (same as the HTTP status code)", + "format" : "int32" + }, + "message" : { + "type" : "string", + "description" : "Status message" + } + } + }, + "JsonNode" : { + "type" : "object" + }, + "BulkjobResultPostRequest" : { + "type" : "object", + "properties" : { + "head" : { + "type" : "integer", + "description" : "Return first n results", + "format" : "int32" + }, + "from" : { + "type" : "integer", + "description" : "1-based index of the first result to return (1 = first result).", + "format" : "int32" + }, + "size" : { + "type" : "integer", + "description" : "Maximum number of results to return starting from the 'from' position.", + "format" : "int32" + }, + "onlyMatches" : { + "type" : "boolean", + "description" : "Return only records that have a match" + }, + "format" : { + "type" : "string", + "description" : "Export format", + "enum" : [ "json", "jsonl", "csv", "xls", "xlsx" ] + }, + "exportspec" : { + "type" : "string", + "description" : "The spec defines the columns to export. This is applicable for csv, xls and xslx formats.\n\nExport spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields)." + }, + "exportfile" : { + "type" : "string", + "description" : "File name of the export file" + }, + "exportseparator" : { + "type" : "string", + "description" : "Separator for multi-value fields when exporting columnar results" + }, + "exportquery" : { + "type" : "boolean", + "description" : "Prefixes the enhance query parameters to the CSV export result" + }, + "filter" : { + "type" : "string", + "description" : "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + }, + "filterExclude" : { + "type" : "string", + "description" : "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + }, + "wait" : { + "type" : "integer", + "description" : "Seconds to wait for bulkjob results to export. Results will continue to export in the background. To only trigger an export, use 0.", + "format" : "int32" + } + } + }, + "BulkjobStatusResponse" : { + "type" : "object", + "properties" : { + "code" : { + "type" : "integer", + "description" : "Bulkjob status code (same as the HTTP status code)", + "format" : "int32" + }, + "content" : { + "$ref" : "#/components/schemas/BulkjobStatus" + } + } + }, + "BulkjobRecoveryStatusResponse" : { + "type" : "object", + "properties" : { + "status" : { + "type" : "string", + "description" : "Bulkjob recovery status", + "enum" : [ "NOT_STARTED", "IN_PROCESS", "COMPLETE", "COMPLETE_WITH_FAILURES", "STOPPED", "ERROR_FINALIZING", "ERROR_SCHEDULING", "UNKNOWN" ] + }, + "message" : { + "type" : "string", + "description" : "Bulkjob recovery status message" + } + } + }, + "BulkjobAccepted" : { + "type" : "object", + "properties" : { + "job_id" : { + "type" : "string", + "description" : "Bulkjob Id for checking status and downloading results" + } + } + }, + "IdRequestEntity" : { + "type" : "object", + "properties" : { + "id" : { + "type" : "string", + "description" : "DiffbotId of entity to enhance. Parameter can be used with types `Person` and `Organization`" + } + } + }, + "EnhancePostRequest" : { + "required" : [ "type" ], + "type" : "object", + "properties" : { + "type" : { + "type" : "string", + "description" : "Diffbot entity type", + "enum" : [ "Person", "Organization" ] + }, + "id" : { + "type" : "string", + "description" : "DiffbotId of entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "name" : { + "type" : "array", + "description" : "Name of the entity to enhance. Parameter can be used with types `Person` and `Organization`", + "items" : { + "type" : "string" + } + }, + "url" : { + "type" : "array", + "description" : "Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization`", + "items" : { + "type" : "string" + } + }, + "location" : { + "type" : "string", + "description" : "Location of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "ip" : { + "type" : "string", + "description" : "IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "phone" : { + "type" : "string", + "description" : "Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "description" : { + "type" : "string", + "description" : "Description of the entity to enhance. Parameter can be used with types `Person` and `Organization`" + }, + "email" : { + "type" : "array", + "description" : "Email of the entity to enhance. Parameter can be used only with type `Person`", + "items" : { + "type" : "string" + } + }, + "employer" : { + "type" : "string", + "description" : "Employer of the entity to enhance. Parameter can be used only with type `Person`" + }, + "title" : { + "type" : "string", + "description" : "Title of the entity to enhance. Parameter can be used only with type `Person`" + }, + "school" : { + "type" : "string", + "description" : "School of the entity to enhance. Parameter can be used only with type `Person`" + }, + "customId" : { + "type" : "string", + "description" : "User defined ID for correlation" + }, + "size" : { + "type" : "integer", + "description" : "Maximum number of results to return", + "format" : "int32", + "default" : 1 + }, + "threshold" : { + "type" : "number", + "description" : "Enhance similarity threshold", + "format" : "double" + }, + "refresh" : { + "type" : "boolean", + "description" : "`refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data.", + "default" : false + }, + "search" : { + "type" : "boolean", + "description" : "`search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG.", + "default" : false + }, + "nonCanonicalFacts" : { + "type" : "boolean", + "description" : "`nonCanonicalFacts=true` returns non-canonical facts.", + "default" : false + }, + "jsonmode" : { + "type" : "string", + "description" : "`jsonmode=extended` returns origin information for facts.", + "enum" : [ "extended" ] + }, + "filter" : { + "type" : "string", + "description" : "Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + }, + "filterExclude" : { + "type" : "string", + "description" : "Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax." + } + } + } + }, + "securitySchemes" : { + "tokenscheme" : { + "type" : "apiKey", + "description" : "Diffbot API token, supplied as the `token` query parameter on every request.", + "name" : "token", + "in" : "query" + } + } + } +} diff --git a/openapi/extract.json b/openapi/extract.json new file mode 100644 index 0000000..b9aa0c0 --- /dev/null +++ b/openapi/extract.json @@ -0,0 +1,3952 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Extract APIs", + "description": "Diffbot's Extraction APIs include various endpoints to extract JSON data/fields from different types of web pages. The Analyze endpoint is used when the page type is unknown, and will attempt to identify the page type and send it to the appropriate extraction API.", + "termsOfService": "https://www.diffbot.com/terms/", + "contact": { + "email": "support@diffbot.com" + }, + "version": "1.1.0" + }, + "servers": [ + { + "url": "https://api.diffbot.com/v3" + } + ], + "tags": [ + { + "name": "Extract APIs", + "description": "Extract APIs" + } + ], + "paths": { + "/analyze": { + "get": { + "tags": [ + "Analyze" + ], + "summary": "Analyze", + "description": "Automatically classify a page and extract data according to its type.", + "operationId": "extract-analyze", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Web page URL of the analyze to process (URL encoded)", + "required": true, + "schema": { + "type": "string", + "default": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/" + } + }, + { + "name": "mode", + "in": "query", + "description": "By default the Analyze API will fully extract all pages that match an existing Extract API. Set mode to a specific Extract API (e.g., `mode=\"article\"`) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields.", + "schema": { + "type": "string", + "format": "enum", + "enum": [ + "article", + "product", + "discussion", + "image", + "video", + "list", + "event" + ] + } + }, + { + "name": "fallback", + "in": "query", + "description": "If an appropriate API cannot be determined (pages classified with type \"other\"), fall back to this API.", + "schema": { + "type": "string", + "format": "enum", + "enum": [ + "article", + "product", + "discussion", + "image", + "video", + "list", + "event" + ] + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "discussion", + "in": "query", + "description": "Pass `discussion=false` to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions.", + "schema": { + "type": "boolean" + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "example": { + "request": { + "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + "api": "analyze", + "version": 3 + }, + "humanLanguage": "en", + "objects": [ + { + "date": "Fri, 04 Sep 2020 00:00:00 GMT", + "sentiment": 0.153, + "images": [ + { + "naturalHeight": 869, + "width": 654, + "diffbotUri": "image|3|1663647584", + "url": "https://wp.technologyreview.com/wp-content/uploads/2022/03/Flower-Trip-style.jpeg?resize=1006,640", + "naturalWidth": 1366, + "height": 418 + }, + { + "naturalHeight": 1900, + "width": 460, + "diffbotUri": "image|3|683243517", + "url": "https://wp.technologyreview.com/wp-content/uploads/2022/02/MA22_Demis-Hassabis-99-v1.jpg?resize=1006,1400", + "naturalWidth": 1366, + "height": 294 + } + ], + "author": "Will Douglas Heaven", + "estimatedDate": "Fri, 04 Sep 2020 00:00:00 GMT", + "publisherRegion": "North America", + "icon": "https://www.technologyreview.com/static/media/favicon.1cfcdb44.ico", + "diffbotUri": "article|3|973247980", + "siteName": "MIT Technology Review", + "type": "article", + "title": "This know-it-all AI learns by reading the entire web nonstop", + "tags": [ + { + "score": 0.998680055141449, + "sentiment": 0, + "count": 10, + "label": "artificial intelligence", + "uri": "https://diffbot.com/entity/E_lYDrjmAMlKKwXaDf958zg", + "rdfTypes": [ + "http://dbpedia.org/ontology/Skill", + "http://dbpedia.org/ontology/Activity" + ] + }, + { + "score": 0.9686350226402283, + "sentiment": 0.889, + "count": 7, + "label": "Diffbot", + "uri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + "rdfTypes": [ + "http://dbpedia.org/ontology/Organisation" + ] + }, + { + "score": 0.9306924939155579, + "sentiment": 0, + "count": 2, + "label": "Michigan", + "uri": "https://diffbot.com/entity/E2eIrTt0jPUmGmuV6N2O3KQ", + "rdfTypes": [ + "http://dbpedia.org/ontology/Place", + "http://dbpedia.org/ontology/PopulatedPlace", + "http://dbpedia.org/ontology/State" + ] + }, + { + "score": 0.9025880098342896, + "sentiment": 0, + "count": 1, + "label": "Paul Katsen", + "uri": "https://diffbot.com/entity/EqUim_ci0ObmrK2gZM3UfNA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Person" + ] + }, + { + "score": 0.8933213353157043, + "sentiment": 0.48, + "count": 4, + "label": "Katy Perry", + "uri": "https://diffbot.com/entity/E_6rhi_PEOD6vGencwOxd2A", + "rdfTypes": [ + "http://dbpedia.org/ontology/Person" + ] + }, + { + "score": 0.8848651051521301, + "sentiment": 0, + "count": 4, + "label": "Mike Tung", + "uri": "https://diffbot.com/entity/ESGMaGV9uP0SuTmfPTtNEoA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Person" + ] + }, + { + "score": 0.8562507629394531, + "sentiment": 0, + "count": 4, + "label": "Google", + "uri": "https://diffbot.com/entity/EUFq-3WlpNsq0pvfUYWXOEA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Organisation" + ] + }, + { + "score": 0.7750672101974487, + "sentiment": 0, + "count": 2, + "label": "Alaska", + "uri": "https://diffbot.com/entity/E4odwkG_xMNeZTbHrnNrojA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Place", + "http://dbpedia.org/ontology/PopulatedPlace", + "http://dbpedia.org/ontology/State" + ] + }, + { + "score": 0.7653270959854126, + "sentiment": 0, + "count": 1, + "label": "Zola", + "uri": "https://diffbot.com/entity/E0qGTA2o5NjaeezggjMsoVw", + "rdfTypes": [ + "http://dbpedia.org/ontology/Organisation" + ] + }, + { + "score": 0.7643865942955017, + "sentiment": 0.75, + "count": 3, + "label": "GUID Partition Table", + "uri": "https://diffbot.com/entity/EReKbXuSJMYmoM8lawtgEsA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Skill", + "http://dbpedia.org/ontology/Activity" + ] + } + ], + "publisherCountry": "United States", + "humanLanguage": "en", + "authorUrl": "https://www.technologyreview.com/author/will-douglas-heaven/", + "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + "html": "
\"knowledge
\n

Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.

\n

In one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.

\n

Except that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.

\n

Language models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”

\n

This is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.

\n

Like GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.

\n

Pointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.

\n

Knowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.

\n

A few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.

\n

But Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.

\n

Alongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”

\n

Super surfer

\n

To collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.

\n

Every three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”

\n

Browsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.

\n

Diffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.

\n

Researchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.

\n

Fake shoes

\n

Adidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.

\n

For now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.

\n

Tung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.

\n

Still, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”

\n
\"NLP
\n
\"Demis
", + "categories": [ + { + "score": 0.962, + "name": "Technology & Computing", + "id": "iabv2-596" + }, + { + "score": 0.962, + "name": "Artificial Intelligence", + "id": "iabv2-597" + } + ], + "text": "Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.\nIn one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.\nExcept that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.\nLanguage models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”\nThis is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.\nLike GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.\nPointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.\nKnowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.\nA few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.\nBut Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.\nAlongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”\nSuper surfer\nTo collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.\nEvery three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”\nBrowsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.\nDiffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.\nResearchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.\nFake shoes\nAdidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.\nFor now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.\nTung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.\nStill, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”", + "authors": [ + { + "name": "Will Douglas Heavenarchive page", + "link": "technologyreview.com/author/will-douglas-heaven" + } + ] + } + ], + "type": "article", + "title": "This know-it-all AI learns by reading the entire web nonstop | MIT Technology Review" + }, + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "humanLanguage": { + "type": "string" + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "sentiment": { + "type": "number" + }, + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "naturalHeight": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + }, + "author": { + "type": "string" + }, + "estimatedDate": { + "type": "string" + }, + "publisherRegion": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "siteName": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "score": { + "type": "number" + }, + "sentiment": { + "type": "number" + }, + "count": { + "type": "integer" + }, + "label": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "rdfTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "publisherCountry": { + "type": "string" + }, + "humanLanguage": { + "type": "string" + }, + "authorUrl": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "html": { + "type": "string" + }, + "categories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "score": { + "type": "number" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + } + } + } + }, + "text": { + "type": "string" + }, + "authors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + } + } + } + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/article": { + "get": { + "tags": [ + "Article" + ], + "summary": "Article", + "description": "Automatically extract clean article text and other data from news articles, blog posts and other text-heavy pages.", + "operationId": "article", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb", + "quotes" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + }, + { + "name": "paging", + "in": "query", + "description": "Pass `paging=true` to enable automatic concatenation for multi-page articles. (default: false)", + "schema": { + "type": "boolean" + } + }, + { + "name": "maxTags", + "in": "query", + "description": "Set the maximum number of automatically-generated tags to return. (Default: 10)", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "tagConfidence", + "in": "query", + "description": "Set the minimum relevance `score` of tags to return, between 0.0 and 1.0. By default only tags with a `score` equal to or above 0.5 will be returned.", + "schema": { + "type": "number", + "format": "float" + } + }, + { + "name": "categoryConfidence", + "in": "query", + "description": "Set the minimum relevance `score` of categories to return, between 0.0 and 1.0. By default only categories with a `score` equal to or above 0.5 will be returned.", + "schema": { + "type": "number", + "format": "float" + } + }, + { + "name": "discussion", + "in": "query", + "description": "Pass `discussion=false` to disable automatic extraction of article comments.", + "schema": { + "type": "boolean" + } + }, + { + "name": "naturalLanguage", + "in": "query", + "description": "Run extracted text and title through the Diffbot Natural Language API. Example: `&naturalLanguage=entities,facts,categories,sentiment`.", + "schema": { + "type": "string", + "enum": [ + "entities", + "sentiment", + "summary", + "facts", + "openFacts", + "records", + "categories", + "sentences", + "language" + ] + } + }, + { + "name": "summaryNumSentences", + "in": "query", + "description": "Sets the maximum number of sentences for summary generation when using naturalLanguage=summary (Default: 3).", + "schema": { + "type": "integer", + "format": "int32" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "humanLanguage": { + "type": "string" + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "sentiment": { + "type": "number" + }, + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "naturalHeight": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + }, + "author": { + "type": "string" + }, + "estimatedDate": { + "type": "string" + }, + "publisherRegion": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "siteName": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "score": { + "type": "number" + }, + "sentiment": { + "type": "number" + }, + "count": { + "type": "integer" + }, + "label": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "rdfTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "publisherCountry": { + "type": "string" + }, + "humanLanguage": { + "type": "string" + }, + "authorUrl": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "html": { + "type": "string" + }, + "categories": { + "type": "array", + "items": { + "type": "object", + "properties": { + "score": { + "type": "number" + }, + "name": { + "type": "string" + }, + "id": { + "type": "string" + } + } + } + }, + "text": { + "type": "string" + }, + "authors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + } + } + } + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + } + } + }, + "example": { + "request": { + "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + "api": "article", + "version": 3 + }, + "humanLanguage": "en", + "objects": [ + { + "date": "Fri, 04 Sep 2020 00:00:00 GMT", + "sentiment": 0.153, + "images": [ + { + "naturalHeight": 869, + "width": 654, + "diffbotUri": "image|3|1663647584", + "url": "https://wp.technologyreview.com/wp-content/uploads/2022/03/Flower-Trip-style.jpeg?resize=1006,640", + "naturalWidth": 1366, + "height": 418 + } + ], + "author": "Will Douglas Heaven", + "estimatedDate": "Fri, 04 Sep 2020 00:00:00 GMT", + "publisherRegion": "North America", + "icon": "https://www.technologyreview.com/static/media/favicon.1cfcdb44.ico", + "diffbotUri": "article|3|973247980", + "siteName": "MIT Technology Review", + "type": "article", + "title": "This know-it-all AI learns by reading the entire web nonstop", + "tags": [ + { + "score": 0.998680055141449, + "sentiment": 0, + "count": 10, + "label": "artificial intelligence", + "uri": "https://diffbot.com/entity/E_lYDrjmAMlKKwXaDf958zg", + "rdfTypes": [ + "http://dbpedia.org/ontology/Skill", + "http://dbpedia.org/ontology/Activity" + ] + }, + { + "score": 0.9686350226402283, + "sentiment": 0.889, + "count": 7, + "label": "Diffbot", + "uri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + "rdfTypes": [ + "http://dbpedia.org/ontology/Organisation" + ] + }, + { + "score": 0.9306924939155579, + "sentiment": 0, + "count": 2, + "label": "Michigan", + "uri": "https://diffbot.com/entity/E2eIrTt0jPUmGmuV6N2O3KQ", + "rdfTypes": [ + "http://dbpedia.org/ontology/Place", + "http://dbpedia.org/ontology/PopulatedPlace", + "http://dbpedia.org/ontology/State" + ] + }, + { + "score": 0.9025880098342896, + "sentiment": 0, + "count": 1, + "label": "Paul Katsen", + "uri": "https://diffbot.com/entity/EqUim_ci0ObmrK2gZM3UfNA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Person" + ] + }, + { + "score": 0.8933213353157043, + "sentiment": 0.48, + "count": 4, + "label": "Katy Perry", + "uri": "https://diffbot.com/entity/E_6rhi_PEOD6vGencwOxd2A", + "rdfTypes": [ + "http://dbpedia.org/ontology/Person" + ] + }, + { + "score": 0.8848651051521301, + "sentiment": 0, + "count": 4, + "label": "Mike Tung", + "uri": "https://diffbot.com/entity/ESGMaGV9uP0SuTmfPTtNEoA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Person" + ] + }, + { + "score": 0.8562507629394531, + "sentiment": 0, + "count": 4, + "label": "Google", + "uri": "https://diffbot.com/entity/EUFq-3WlpNsq0pvfUYWXOEA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Organisation" + ] + }, + { + "score": 0.7750672101974487, + "sentiment": 0, + "count": 2, + "label": "Alaska", + "uri": "https://diffbot.com/entity/E4odwkG_xMNeZTbHrnNrojA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Place", + "http://dbpedia.org/ontology/PopulatedPlace", + "http://dbpedia.org/ontology/State" + ] + }, + { + "score": 0.7653270959854126, + "sentiment": 0, + "count": 1, + "label": "Zola", + "uri": "https://diffbot.com/entity/E0qGTA2o5NjaeezggjMsoVw", + "rdfTypes": [ + "http://dbpedia.org/ontology/Organisation" + ] + }, + { + "score": 0.7643865942955017, + "sentiment": 0.75, + "count": 3, + "label": "GUID Partition Table", + "uri": "https://diffbot.com/entity/EReKbXuSJMYmoM8lawtgEsA", + "rdfTypes": [ + "http://dbpedia.org/ontology/Skill", + "http://dbpedia.org/ontology/Activity" + ] + } + ], + "publisherCountry": "United States", + "humanLanguage": "en", + "authorUrl": "https://www.technologyreview.com/author/will-douglas-heaven/", + "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + "html": "
\"knowledge
\n

Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.

\n

In one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.

\n

Except that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.

\n

Language models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”

\n

This is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.

\n

Like GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.

\n

Pointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.

\n

Knowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.

\n

A few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.

\n

But Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.

\n

Alongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”

\n

Super surfer

\n

To collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.

\n

Every three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”

\n

Browsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.

\n

Diffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.

\n

Researchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.

\n

Fake shoes

\n

Adidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.

\n

For now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.

\n

Tung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.

\n

Still, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”

\n
\"NLP
\n
\"Demis
", + "categories": [ + { + "score": 0.962, + "name": "Technology & Computing", + "id": "iabv2-596" + }, + { + "score": 0.962, + "name": "Artificial Intelligence", + "id": "iabv2-597" + } + ], + "text": "Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.\nIn one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.\nExcept that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.\nLanguage models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”\nThis is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.\nLike GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.\nPointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.\nKnowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.\nA few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.\nBut Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.\nAlongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”\nSuper surfer\nTo collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.\nEvery three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”\nBrowsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.\nDiffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.\nResearchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.\nFake shoes\nAdidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.\nFor now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.\nTung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.\nStill, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”", + "authors": [ + { + "name": "Will Douglas Heavenarchive page", + "link": "technologyreview.com/author/will-douglas-heaven" + } + ] + } + ], + "type": "article", + "title": "This know-it-all AI learns by reading the entire web nonstop | MIT Technology Review" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/product": { + "get": { + "tags": [ + "Product" + ], + "summary": "Product", + "description": "Automatically extract pricing, product specs, images, and more from an e-commerce product page.", + "operationId": "product", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://www.microcenter.com/product/628738/evga-nvidia-geforce-rtx-3090-ftw3-ultra-triple-fan-24gb-gddr6x-pcie-40-graphics-card" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + }, + { + "name": "discussion", + "in": "query", + "description": "Pass `discussion=false` to disable automatic extraction of product reviews.", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "humanLanguage": { + "type": "string" + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "xpath": { + "type": "string" + }, + "naturalHeight": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "primary": { + "type": "boolean" + }, + "height": { + "type": "integer" + } + } + } + }, + "offerPrice": { + "type": "string" + }, + "productId": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "upc": { + "type": "string" + }, + "productOrigin": { + "type": "string" + }, + "mpn": { + "type": "string" + }, + "gs1Code": { + "type": "string" + }, + "multipleProducts": { + "type": "boolean" + }, + "availability": { + "type": "boolean" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "offerPriceDetails": { + "type": "object", + "properties": { + "symbol": { + "type": "string" + }, + "amount": { + "type": "number" + }, + "text": { + "type": "string" + } + } + }, + "specs": { + "type": "object", + "properties": { + "mfr_part_": { + "type": "string" + }, + "upc": { + "type": "string" + }, + "sku": { + "type": "string" + } + } + }, + "normalizedSpecs": { + "type": "object", + "properties": { + "sku": { + "type": "array", + "items": { + "type": "object", + "properties": { + "cleanLiteral": { + "type": "string" + } + } + } + } + } + }, + "humanLanguage": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "text": { + "type": "string" + }, + "category": { + "type": "string" + }, + "sku": { + "type": "string" + }, + "brand": { + "type": "string" + } + } + } + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + } + } + }, + "example": { + "request": { + "pageUrl": "https://www.microcenter.com/product/628738/evga-nvidia-geforce-rtx-3090-ftw3-ultra-triple-fan-24gb-gddr6x-pcie-40-graphics-card", + "api": "product", + "version": 3 + }, + "humanLanguage": "en", + "objects": [ + { + "images": [ + { + "xpath": "/html[1]/body[1]/main[1]/article[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/a[1]/img[1]", + "naturalHeight": 200, + "width": 260, + "diffbotUri": "image|3|735477840", + "title": "Product Image View 2", + "url": "https://90a1c75758623581b3f8-5c119c3de181c9857fcb2784776b17ef.ssl.cf2.rackcdn.com/628738_177923_01_front_thumbnail.jpg", + "naturalWidth": 200, + "primary": true, + "height": 260 + } + ], + "offerPrice": "$1,919.99", + "productId": "843368067137", + "diffbotUri": "product|3|-1003574237", + "upc": "843368067137", + "productOrigin": "us", + "mpn": "24G-P5-3987-KR", + "gs1Code": "U.S. and Canada", + "multipleProducts": true, + "availability": true, + "type": "product", + "title": "EVGA NVIDIA GeForce RTX 3090 FTW3 Ultra Triple-Fan 24GB GDDR6X PCIe 4.0 Graphics Card", + "offerPriceDetails": { + "symbol": "$", + "amount": 1919.99, + "text": "$1,919.99" + }, + "specs": { + "mfr_part_": "24G-P5-3987-KR", + "upc": "843368067137", + "sku": "177923" + }, + "normalizedSpecs": { + "sku": [ + { + "cleanLiteral": "843368067137" + } + ] + }, + "humanLanguage": "en", + "pageUrl": "https://www.microcenter.com/product/628738/evga-nvidia-geforce-rtx-3090-ftw3-ultra-triple-fan-24gb-gddr6x-pcie-40-graphics-card", + "text": "The EVGA GeForce RTX 3090 is colossally powerful in every way imaginable, giving you a whole new tier of performance at 8K resolution. It's powered by the NVIDIA Ampere architecture, which doubles down on ray tracing and AI performance with enhanced RT Cores, Tensor Cores, and new streaming multiprocessors. Combined with the next generation of design, cooling, and overclocking with EVGA Precision X1, the EVGA GeForce RTX 3090 Series redefines the definition of ultimate performance.", + "category": "Computers", + "sku": "177923", + "brand": "EVGA" + } + ], + "type": "product", + "title": "EVGA NVIDIA GeForce RTX 3090 FTW3 Ultra Triple-Fan 24GB GDDR6X PCIe 4.0 Graphics Card - Micro Center" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/discussion": { + "get": { + "tags": [ + "Discussion" + ], + "summary": "Discussion", + "description": "Automatically structure and extract entire threads of reviews/comments from articles, product pages, and forum threads.", + "operationId": "discussion", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://forum.gl-inet.com/t/ssh-connection-protocol-does-not-work-well-between-vpn-clients/22779" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb", + "sentiment" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + }, + { + "name": "paging", + "in": "query", + "description": "Pass `paging=true` to enable automatic concatenation for multi-page articles. (default: false)", + "schema": { + "type": "boolean" + } + }, + { + "name": "maxPages", + "in": "query", + "description": "Set the maximum number of pages in a thread to automatically concatenate in a single response. Default = 1 (no concatenation). Set `maxPages=all` to retrieve all pages of a thread regardless of length. Each individual page will count as a separate API call.", + "schema": { + "type": "string", + "default": "20" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "numPages": { + "type": "integer" + }, + "humanLanguage": { + "type": "string" + }, + "confidence": { + "type": "number" + }, + "diffbotUri": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "numPosts": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "posts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "naturalHeight": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "height": { + "type": "integer" + } + } + } + }, + "humanLanguage": { + "type": "string" + }, + "author": { + "type": "string" + }, + "authorUrl": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "html": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "text": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "score": { + "type": "number" + }, + "count": { + "type": "integer" + }, + "label": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "rdfTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, + "participants": { + "type": "integer" + }, + "rssUrl": { + "type": "string" + } + } + } + } + } + }, + "example": { + "request": { + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "api": "discussion", + "version": 3 + }, + "objects": [ + { + "numPages": 1, + "humanLanguage": "en", + "confidence": 0.05500000089407453, + "diffbotUri": "discussion|3|-870809033", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "numPosts": 13, + "type": "discussion", + "title": "[OC] 66% of Top 50 Russian Exposed Companies Have Announced Sanctions", + "posts": [ + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "images": [ + { + "naturalHeight": 767, + "width": 457, + "diffbotUri": "image|3|-804821395", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "url": "https://preview.redd.it/l76k59t8jsm81.png?width=457&auto=webp&s=632efa1f24e607358bbec99c161a6aa579aebfe1", + "naturalWidth": 457, + "height": 767 + } + ], + "humanLanguage": "en", + "author": "hicheoo", + "authorUrl": "https://old.reddit.com/user/hicheoo", + "diffbotUri": "post|3|29462830", + "html": "
\n

Want to add to the discussion?

\n

Post a comment!

\n

Create an account

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 0, + "text": "Want to add to the discussion?\nPost a comment!\n\n \nCreate an account", + "type": "post", + "title": "[OC] 66% of Top 50 Russian Exposed Companies Have Announced Sanctions" + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "not_mig", + "authorUrl": "https://old.reddit.com/user/not_mig", + "diffbotUri": "post|3|-720375378", + "html": "

What's the difference between blue, yellow, and green?

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 1, + "text": "What's the difference between blue, yellow, and green?", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "hicheoo", + "authorUrl": "https://old.reddit.com/user/hicheoo", + "diffbotUri": "post|3|-148816221", + "html": "

They're exemptions. I should've clarified up top, but they're basically all in the description.

\n

Green: Typical Sanctions
\n Yellow: Sanctions, but might be a PR move.
\n Blue: Healthcare

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 2, + "text": "They're exemptions. I should've clarified up top, but they're basically all in the description.\nGreen: Typical Sanctions\nYellow: Sanctions, but might be a PR move.\nBlue: Healthcare", + "type": "post", + "parentId": 1 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "Zealousideal-Lie7255", + "authorUrl": "https://old.reddit.com/user/Zealousideal-Lie7255", + "diffbotUri": "post|3|-683402068", + "html": "

A lot of oil service companies have no reported sanctions. Like Schlumberger, Baker Hughes. Some Chinese companies too.

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 3, + "text": "A lot of oil service companies have no reported sanctions. Like Schlumberger, Baker Hughes. Some Chinese companies too.", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "varnima", + "authorUrl": "https://old.reddit.com/user/varnima", + "diffbotUri": "post|3|-603833918", + "html": "

JetBrains changed and imposed sanctions https://blog.jetbrains.com/blog/2022/03/11/jetbrains-statement-on-ukraine/

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 4, + "text": "JetBrains changed and imposed sanctions https://blog.jetbrains.com/blog/2022/03/11/jetbrains-statement-on-ukraine/", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "hicheoo", + "authorUrl": "https://old.reddit.com/user/hicheoo", + "diffbotUri": "post|3|-296888207", + "html": "

Yeah, they're green in the chart.

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 5, + "text": "Yeah, they're green in the chart.", + "type": "post", + "parentId": 4 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "hicheoo", + "authorUrl": "https://old.reddit.com/user/hicheoo", + "diffbotUri": "post|3|624793084", + "html": "

Sources: - Diffbot Sanctions Tracker (https://www.diffbot.com/insights/every-company-affected-by-sanctions/) - Diffbot Knowledge Graph (more detail on query below)

\n

Data Viz Tool: Infogram

\n

Disclaimer: I work for Diffbot

\n

I started by querying the Knowledge Graph for people who live in Russia but work for a non-Russian company. Faceting this query by their employer provides me with a list of non-Russian companies ranked by # of Russian employees.

\n

\ntype:Person location.country.name:"Russia" employments.{employer.{location.country.name!="Russia" nbLocations>0} isCurrent:true} facet:employments.{employer.name isCurrent:true}\n

\n

This data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 6, + "text": "Sources: - Diffbot Sanctions Tracker (https://www.diffbot.com/insights/every-company-affected-by-sanctions/) - Diffbot Knowledge Graph (more detail on query below)\nData Viz Tool: Infogram\nDisclaimer: I work for Diffbot\nI started by querying the Knowledge Graph for people who live in Russia but work for a non-Russian company. Faceting this query by their employer provides me with a list of non-Russian companies ranked by # of Russian employees.\ntype:Person location.country.name:\"Russia\" employments.{employer.{location.country.name!=\"Russia\" nbLocations>0} isCurrent:true} facet:employments.{employer.name isCurrent:true}\nThis data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "zzzmick", + "authorUrl": "https://old.reddit.com/user/zzzmick", + "diffbotUri": "post|3|-130810969", + "html": "

epam had over 10k employees in Russia

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 7, + "text": "epam had over 10k employees in Russia", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "hicheoo", + "authorUrl": "https://old.reddit.com/user/hicheoo", + "diffbotUri": "post|3|-1458692070", + "html": "

Yup. The data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 8, + "text": "Yup. The data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.", + "type": "post", + "parentId": 7 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "JanitorKarl", + "authorUrl": "https://old.reddit.com/user/JanitorKarl", + "diffbotUri": "post|3|-149138223", + "html": "

Schlumberger and Baker Hughes are both in the oilfield services industry.

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 9, + "text": "Schlumberger and Baker Hughes are both in the oilfield services industry.", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "flumenia", + "authorUrl": "https://old.reddit.com/user/flumenia", + "diffbotUri": "post|3|889762151", + "html": "

What if Microsoft stops to extend licenses of Microsoft Office to Russia? That would make the biggest impact, I guess

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 10, + "text": "What if Microsoft stops to extend licenses of Microsoft Office to Russia? That would make the biggest impact, I guess", + "type": "post", + "parentId": 0 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "Imperial_Empirical", + "authorUrl": "https://old.reddit.com/user/Imperial_Empirical", + "diffbotUri": "post|3|-179317804", + "html": "

Putin ordered the development of Russian alternatives after the Crimean annexation due to dependancy/spying fears. I believe from 2016 onwards Microsoft was largely fased out internally.

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 11, + "text": "Putin ordered the development of Russian alternatives after the Crimean annexation due to dependancy/spying fears. I believe from 2016 onwards Microsoft was largely fased out internally.", + "type": "post", + "parentId": 10 + }, + { + "date": "Fri, 11 Mar 2022 00:00:00 GMT", + "humanLanguage": "en", + "author": "Nightblood83", + "authorUrl": "https://old.reddit.com/user/Nightblood83", + "diffbotUri": "post|3|-901046006", + "html": "

A lot of accountants for commies...

", + "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + "id": 12, + "text": "A lot of accountants for commies...", + "type": "post", + "parentId": 0 + } + ], + "tags": [ + { + "score": 0.8428076505661011, + "count": 5, + "label": "economic sanctions", + "uri": "https://diffbot.com/entity/EWnXSPtH6Osi0pmx8-WPKAg", + "rdfTypes": [ + "http://dbpedia.org/ontology/Miscellaneous" + ] + } + ], + "participants": 9, + "rssUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/.rss" + } + ] + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/job": { + "get": { + "tags": [ + "Job" + ], + "summary": "Job", + "description": "Automatically extracts structured information from job postings.", + "operationId": "job", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://www.pathai.com/careers/DirectorClinicalOperations-6715804002" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "requirements": { + "type": "array", + "items": { + "type": "string" + } + }, + "diffbotUri": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "postedDate": { + "type": "string" + }, + "remote": { + "type": "string" + }, + "skills": { + "type": "array", + "items": { + "type": "object", + "properties": { + "salience": { + "type": "number" + }, + "skill": { + "type": "string" + }, + "confidence": { + "type": "number" + }, + "diffbotUri": { + "type": "string" + } + } + } + }, + "humanLanguage": { + "type": "string" + }, + "employer": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + } + } + }, + "text": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "tasks": { + "type": "array", + "items": { + "type": "string" + } + }, + "locations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "country": { + "type": "object", + "properties": { + "summary": { + "type": "string" + }, + "image": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "targetDiffbotUri": { + "type": "string" + }, + "targetDiffbotId": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "address": { + "type": "string" + }, + "city": { + "type": "object", + "properties": { + "summary": { + "type": "string" + }, + "image": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "targetDiffbotUri": { + "type": "string" + }, + "targetDiffbotId": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "region": { + "type": "object", + "properties": { + "summary": { + "type": "string" + }, + "image": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "targetDiffbotUri": { + "type": "string" + }, + "targetDiffbotId": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "subregion": { + "type": "object", + "properties": { + "summary": { + "type": "string" + }, + "image": { + "type": "string" + }, + "types": { + "type": "array", + "items": { + "type": "string" + } + }, + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "targetDiffbotUri": { + "type": "string" + }, + "targetDiffbotId": { + "type": "string" + }, + "type": { + "type": "string" + } + } + }, + "latitude": { + "type": "number" + }, + "precision": { + "type": "number" + }, + "longitude": { + "type": "number" + } + } + } + } + } + } + } + } + }, + "example": { + "request": { + "pageUrl": "https://www.pathai.com/careers/DirectorClinicalOperations-6715804002", + "api": "job", + "version": 3 + }, + "objects": [ + { + "requirements": [ + "Experience in histologic-based or other medical imaging in a regulated environment is strongly preferred." + ], + "diffbotUri": "job|3|1793551482", + "type": "job", + "title": "Director, Clinical Operations", + "remote": "REMOTE", + "skills": [ + { + "salience": 0.7316089, + "skill": "DDT", + "confidence": 0.6169359, + "diffbotUri": "https://diffbot.com/entity/Ey44naXoPNamQWWt5PK0JEQ" + } + ], + "humanLanguage": "en", + "employer": { + "name": "PathAI", + "diffbotUri": "https://diffbot.com/entity/EIf6kAhRvMq2mBsM6DV2Krw" + }, + "pageUrl": "https://www.pathai.com/careers/DirectorClinicalOperations-6715804002", + "text": "Who We Are\nPathAI is on a mission to improve patient outcomes with AI-powered pathology. We are transforming traditional pathology methods into powerful, new technologies. These innovations in pathology can help accelerate drug development, improve confidence in the accuracy of diagnosis, and get life-saving therapies to patients more quickly. At PathAI, you'll work with a diverse and talented team of people, who are dedicated to solving complex problems and making a huge impact.\nWhere You Fit\nThe Director of Clinical Operations will be responsible for programmatic tactical execution of regulated research and clinical products, including companion diagnostic (CDx), non-CDx-IVD, Drug Development Tools (DDT) and RUO strategies. This is a highly matrixed role that requires outstanding emotional intelligence combined with detailed subject matter expertise in order to help coordinate the program across all functional areas including quality, clinical, business development, scientific programs, engineering and product teams. The Director of Clinical Operations will bring together all stakeholders internally and externally to successfully design, execute on all aspects of our clinical development programs. Outstanding detailed organizational skills with a proactive mindset toward identification of all programmatic risks and mitigations is essential.\nWhat You’ll Do\nLead the execution of research and clinical development strategies for new clinical trial, research, and medical device product development, and operationalize these strategies from development to commercialization.\nAccountable for oversight of all assigned studies within the country in accordance with the overall development plan and clinical operations plan, including adherence to quality, timelines and budget.\nEstablish and continually communicate detailed program and project plans with critical paths and dependencies clearly identified across each area and communicate timeline progress and risks through defined operating mechanisms.\nEducate relevant leads from each functional area on the regulations impacting development programs.\nClearly communicate strategy, progress, and risks cross functionally and proactively drive/facilitate the relationship between internal project teams and external stakeholders.\nEffectively prioritize the projects within our clinical development programs to ensure adequate resourcing and efficient delivery on priority projects for PathAI.\nInform leadership of any identified gaps, risks, solutions to the program as a whole.\nWhat You Bring\nOur employees' skills come in all shapes and sizes, but to be successful in this role with us, you'll at least need:\nPh.D (preferred) or M.S. in a Life Science or equivalent field, plus at least 5-7 years of relevant industry experience, including at least 3 years in a role of primary responsibility for clinical trial execution and management.\nYou have a solid understanding of and experience in drug development (clinical trials, biomarker testing), assay development, and co-development for CDx programs.\nDirect management experience and supervisory responsibility.\nYou have knowledge (training preferred) of the software development life cycle (IEC 62304), design control, and experience in leading in vitro diagnostic development teams.\nYou have supported projects for FDA and other regulatory body submissions from development to validation to approval.\nYou have cross therapeutic area clinical biomarker testing and diagnostic testing experience in global clinical trials.\nYou have experience and a deep understanding of biomarker testing and reporting requirements globally.\nYou excel in leading multidisciplinary teams of scientists and engineers in highly matrixed organization is strongly preferred\nYou are a strategic thinker, pro-active and agile, and you have a strong interest in bringing teams together at PathAI, as well as external partners, to bring better treatments to patients.\nExperience in histologic-based or other medical imaging in a regulated environment is strongly preferred.\nWe Want To Hear From You\nAt PathAI, we are looking for individuals who are team players, are willing to do the work no matter how big or small it may be, and who are passionate about everything they do. If this sounds like you, even if you may not match the job description to a tee, we encourage you to apply. You could be exactly what we're looking for.\nPathAI is an equal opportunity employer, dedicated to creating a workplace that is free of harassment and discrimination. We base our employment decisions on business needs, job requirements, and qualifications — that's all. We do not discriminate based on race, gender, religion, health, personal beliefs, age, family or parental status, or any other status. We don't tolerate any kind of discrimination or bias, and we are looking for teammates who feel the same way.\n#LI-Remote\n", + "datePosted": "Sat, 07 May 2022 00:00:00 GMT", + "tasks": [ + "Lead the execution of research and clinical development strategies for new clinical trial, research, and medical device product development, and operationalize these strategies from development to commercialization.", + "Accountable for oversight of all assigned studies within the country in accordance with the overall development plan and clinical operations plan, including adherence to quality, timelines and budget.", + "Establish and continually communicate detailed program and project plans with critical paths and dependencies clearly identified across each area and communicate timeline progress and risks through defined operating mechanisms.\n", + "Educate relevant leads from each functional area on the regulations impacting development programs.\n", + "Clearly communicate strategy, progress, and risks cross functionally and proactively drive/facilitate the relationship between internal project teams and external stakeholders.", + "Effectively prioritize the projects within our clinical development programs to ensure adequate resourcing and efficient delivery on priority projects for PathAI.\n", + "Inform leadership of any identified gaps, risks, solutions to the program as a whole.\n", + "Ph.D (preferred) or M.S. in a Life Science or equivalent field, plus at least 5-7 years of relevant industry experience, including at least 3 years in a role of primary responsibility for clinical trial execution and management.", + "You have a solid understanding of and experience in drug development (clinical trials, biomarker testing), assay development, and co-development for CDx programs.\n", + "Direct management experience and supervisory responsibility.", + "You have knowledge (training preferred) of the software development life cycle (IEC 62304), design control, and experience in leading in vitro diagnostic development teams.", + "You have supported projects for FDA and other regulatory body submissions from development to validation to approval.", + "You have cross therapeutic area clinical biomarker testing and diagnostic testing experience in global clinical trials.", + "You have experience and a deep understanding of biomarker testing and reporting requirements globally.", + "You excel in leading multidisciplinary teams of scientists and engineers in highly matrixed organization is strongly preferred", + "You are a strategic thinker, pro-active and agile, and you have a strong interest in bringing teams together at PathAI, as well as external partners, to bring better treatments to patients.\n" + ] + } + ] + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/image": { + "get": { + "tags": [ + "Image" + ], + "summary": "Image", + "description": "Automatically identifies the primary image(s) on any web page and returns comprehensive information and metadata for each image.", + "operationId": "image", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://www.diffbot.com/products/extract/" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb", + "displayHeight", + "displayWidth" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "xpath": { + "type": "string" + }, + "humanLanguage": { + "type": "string" + }, + "naturalHeight": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "type": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "tags": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "typeHierarchy": { + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "type": "integer" + }, + "label": { + "type": "string" + }, + "type": { + "type": "string" + }, + "uri": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "label": { + "type": "string" + }, + "uri": { + "type": "string" + } + } + } + ] + } + } + } + } + } + } + }, + "example": { + "request": { + "pageUrl": "https://www.diffbot.com/products/extract/", + "api": "image", + "version": 3 + }, + "objects": [ + { + "xpath": "/HTML/BODY/MAIN/DIV[@id='slice-readslikehumans']/DIV[@class='container px-3 py-5 py-md-6 mx-auto']/DIV[@class='row justify-content-center']/DIV[@class='col-12 col-md d-flex justify-content-center align-items-center order-md-1']/IMG[@class='img-fluid mb-4 align-self-start']", + "humanLanguage": "en", + "naturalHeight": 1023, + "diffbotUri": "image|3|666824882", + "pageUrl": "https://www.diffbot.com/products/extract/", + "type": "image", + "url": "https://www.diffbot.com/assets/img/products/extract_screenshot.png", + "naturalWidth": 897, + "tags": [ + { + "typeHierarchy": [ + "http://www.w3.org/2002/07/owl#Thing", + "http://dbpedia.org/ontology/Work", + "http://dbpedia.org/ontology/Website" + ], + "id": 33898, + "label": "Website", + "type": "http://dbpedia.org/ontology/Website", + "uri": "https://www.diffbot.com/entity/Xd90vp_U4MJOoRHIja3quxg" + }, + { + "id": 1404579, + "label": "Ring binder", + "uri": "http://diffbot.com/entity/XYwPij6UgPdaDEY6fXSrrRA" + } + ] + }, + { + "xpath": "/HTML/BODY/MAIN/DIV[@id='slice-lookslikeahuman']/DIV[@class='container px-3 py-5 py-md-6 mx-auto']/DIV[@class='row justify-content-center']/DIV[@class='col-12 col-md d-flex justify-content-center align-items-center order-md-1']/IMG[@class='img-fluid mb-4']", + "humanLanguage": "en", + "naturalHeight": 270, + "diffbotUri": "image|3|1865428953", + "pageUrl": "https://www.diffbot.com/products/extract/", + "type": "image", + "url": "https://www.diffbot.com/assets/img/products/any_language.png", + "naturalWidth": 554, + "tags": [ + { + "id": 5462349, + "label": "Sachet", + "uri": "http://diffbot.com/entity/X1_7MIKBoPpuRnAPVQXqQxA" + }, + { + "id": 479373, + "label": "Eraser", + "uri": "http://diffbot.com/entity/XGPnWNBUZPyuNyfK9rJr6vQ" + }, + { + "id": 57260, + "label": "Envelope", + "uri": "http://diffbot.com/entity/XK5gEV93iP6SqKMDjDr0YBQ" + } + ] + }, + { + "xpath": "/HTML/BODY/MAIN/DIV[@id='slice-one-click-crawling']/DIV[@class='container px-3 py-5 py-md-6 mx-auto']/DIV[@class='row justify-content-center']/DIV[@class='col-12 offset-md-1 col-md d-flex justify-content-center align-items-center order-md-2']/IMG[@class='img-fluid mb-4']", + "humanLanguage": "en", + "naturalHeight": 372, + "diffbotUri": "image|3|1699007329", + "pageUrl": "https://www.diffbot.com/products/extract/", + "type": "image", + "url": "https://www.diffbot.com/assets/img/products/analyze_two.png", + "naturalWidth": 451, + "tags": [ + { + "id": 268267, + "label": "Tray", + "uri": "http://diffbot.com/entity/XwXgQ3A7sNmypX2VX3oZCgg" + }, + { + "id": 467731, + "label": "Spatula", + "uri": "http://diffbot.com/entity/XZohZtDoDPvq1wusJQCSZZA" + }, + { + "id": 2649730, + "label": "Measuring cup", + "uri": "http://diffbot.com/entity/XhwdQ4HnKOsCVrf9YU3hyWw" + } + ] + } + ] + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/video": { + "get": { + "tags": [ + "Video" + ], + "summary": "Video", + "description": "The Video API automatically extracts detailed video information—including most metadata, thumbnail images, direct video URL and embed code from nearly any video page or video platform on the web.", + "operationId": "video", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://www.youtube.com/watch?v=hFZFjoX2cGg" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "diffbotUri": { + "type": "string" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "primary": { + "type": "boolean" + } + } + } + }, + "author": { + "type": "string" + }, + "mime": { + "type": "string" + }, + "naturalHeight": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "duration": { + "type": "integer" + }, + "provider": { + "type": "string" + }, + "humanLanguage": { + "type": "string" + }, + "html": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "text": { + "type": "string" + }, + "viewCount": { + "type": "integer" + } + } + } + } + } + }, + "example": { + "request": { + "pageUrl": "https://www.youtube.com/watch?v=hFZFjoX2cGg", + "api": "video", + "version": 3 + }, + "objects": [ + { + "date": "Sun, 24 May 2020 07:00:00 GMT", + "images": [ + { + "diffbotUri": "image|3|231854607", + "title": "Backyard Squirrel Maze 1.0- Ninja Warrior Course", + "url": "https://i.ytimg.com/vi/hFZFjoX2cGg/maxresdefault.jpg", + "primary": true + } + ], + "author": "Mark Rober", + "mime": "video/mp4", + "naturalHeight": 720, + "diffbotUri": "video|3|1870173316", + "type": "video", + "title": "Backyard Squirrel Maze 1.0- Ninja Warrior Course", + "url": "https://rr6---sn-5uaeznks.googlevideo.com/videoplayback?expire=1649742262&ei=Vr1UYoLhFpDm8wTUt76wAQ&ip=23.229.39.25&id=o-AOUTPxsk0l8eAnvqC6G9PtYnkFK4S-lYoQ1G-mt8W40U&itag=399&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278%2C394%2C395%2C396%2C397%2C398%2C399&source=youtube&requiressl=yes&mh=s4&mm=31%2C29&mn=sn-5uaeznks%2Csn-5ualdnl7&ms=au%2Crdu&mv=u&mvi=6&pl=23&spc=4ocVC0JfKoZvX0FSPklAcHZ7LBTF&vprv=1&mime=video%2Fmp4&ns=ZSXKPHKFPvaGHi2sm3qErYcG&gir=yes&clen=246339629&dur=1220.093&lmt=1637823031873234&mt=1649720229&fvip=3&keepalive=yes&fexp=24001373%2C24007246&c=WEB&txp=5531432&n=b2VCd9pxXsFyoV_m&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&sig=AOq0QJ8wRgIhAKiKoHk2DfvaKA3Adu0CkNNF_lOavT4Mk0CYkQvbG0wpAiEAgiAc4h_1GqjzjGXgdiKdgdWZXWM0M9ubY-ns1qhGyns%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl&lsig=AG3C_xAwRQIhAM5DsCeq2RzEblNQ3qRq-3xEydeAidszCKAg8PvahEK1AiAzkjRXQQmDItpKWMxLq1TTb_32NWjPxiTDIVeOrBvxLw%3D%3D", + "naturalWidth": 1280, + "duration": 1220, + "provider": "YouTube", + "humanLanguage": "en", + "html": "", + "pageUrl": "https://www.youtube.com/watch?v=hFZFjoX2cGg", + "text": "Squirrels were stealing my bird seed so I solved the problem with mechanical engineering :)\n\nHere is an explanation of the illusion dish thing!- https://demos.smu.ca/index.php/demos/optics/69-mirage-mirror\n\nHere is a link to the illusion dish (not sponsored :) https://www.amazon.com/dp/B0718XCG7F/ref=cm_sw_em_r_mt_dp_U_D9PYEbC85X14F\n\n*MUSIC*\n0:02 - Arrow (Instrumental) - Andrew Applepie http://andrewapplepie.com/\n0:27 - Kalimba Jam - Blue Wednesday https://soundcloud.com/bluewednesday/\n3:21 - Zambo - Devil in Disguise https://danijel-zambo.bandcamp.com/track/devil-in-disguise-2 \n3:47 - Cereal Killa - Blue Wednesday https://soundcloud.com/bluewednesday/\n5:28 - J. Thompson - Real Quick Lovin' https://www.amazon.com/Real-Quick-Lovin/dp/B0010YGE3W\n5:39 - New Shoes - Blue Wednesday https://soundcloud.com/bluewednesday/\n7:51- Chi- Ponder- https://www.prodbyponder.com/5-free-beats32029775\n8:31 - Marimba Idea - Blue Wednesday https://soundcloud.com/bluewednesday/ \n9:25 - Josef Falkenskold - Tiny Tumble https://www.epidemicsound.com/artists/josef-falkenskold\n19:07 - Nik- Ponder- https://www.prodbyponder.com/5-free-beats32029775", + "viewCount": 92199455 + } + ] + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/list": { + "get": { + "tags": [ + "List" + ], + "summary": "List", + "description": "Automatically structures a list of items from news index pages, product listings pages, search engine results pages, and other \"list-like\" pages.", + "operationId": "list", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://slashdot.org/" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "diffbotUri": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "link": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "example": { + "request": { + "pageUrl": "https://slashdot.org/", + "api": "list", + "version": 3 + }, + "objects": [ + { + "diffbotUri": "list|3|-768312769", + "icon": "https://slashdot.org/favicon.ico", + "pageUrl": "https://slashdot.org/", + "type": "list", + "title": "News for nerds, stuff that matters", + "items": [ + { + "date": "Tue, 05 Oct 2021 00:00:00 GMT", + "summary": "According to fluent Korean speaker Youngmi Mayer, Squid Game features \"botched\" subtitles that have changed the show's meaning for English-speaking viewers.\nFor those unaware of Squid Game, it's a Korean-language drama about an alternative world where people in debt compete in deadly games. The plot sees a group of people tempted into a survival game where they have the chance to walk away with 45.6 billion Korean won ($38 million) if they win a series of six games. According to a BBC article, it's currently on track to become Netflix's biggest original series. From the report: \"The dialogue was so well written and zero of it was preserved [in the subtitles],\" Youngmi said in a Twitter post. In a TikTok video that's had almost nine million views, Youngmi gave several examples of mistranslation. In one scene a character tries to convince people to play the game with her, and the closed-caption subtitles read: \"I'm not a genius, but I still got it worked out.\" But what the character actually says, Youngmi explains, is: \"I am very smart, I just never got a chance to study.\" That translation puts more emphasis on the wealth disparity in society -- which is also a theme in the Oscar-winning 2019 Korean film, Parasite. \"Almost everything she says is being botched translation-wise... the writers, all they want you to know about her is that,\" Youngmi said. \"[It] seems so small, but it's the entire character's purpose of being in the show.\" Youngmi later clarifies that her initial comments were about the automatically generated closed-caption subtitles rather than the English language subtitles, which are \"substantially better.\" But she added: \"The misses in the metaphors -- and what the writers were trying to actually say -- are still pretty present.\"", + "link": "https://entertainment.slashdot.org/story/21/10/05/0146245/squid-game-subtitles-change-meaning-of-netflix-show", + "title": "Squid Game Subtitles 'Change Meaning' of Netflix Show" + }, + { + "date": "Mon, 04 Oct 2021 00:00:00 GMT", + "summary": "An anonymous reader quotes a report from The Verge: Sarah was the patient in a proof-of-concept trial of a new approach to treating severe, treatment-resistant depression, published today in the journal Nature Medicine. The findings open up another possible strategy for helping people with the disorder. The study only involved Sarah, and it's still not clear how well it might work in other people. The lessons from the trial, though, helped the researchers understand more about the nature of depression and could apply to other efforts to treat the disease. The trial used a technique called deep brain stimulation, where electrodes implanted within the brain deliver electrical impulses in an attempt to change or regulate abnormal brain activity. It's common for conditions like epilepsy and Parkinson's disease. Research over the past decade has shown that it can sometimes help with depression, but the findings have been inconsistent. Most previous efforts delivered stimulation to individual regions of the brain thought to be involved in depression. This study, though, was targeted at regions that were part of specific brain circuits -- interconnected parts of the brain that are responsible for specific functions.\nIn addition, the circuits involved might be different for each person. So in this trial, the study team personalized the treatment approach to the specific patient's depression. They mapped out the type of brain activity that occurred when Sarah's depression symptoms flared. Then, they surgically implanted a device that could detect that brain activity and send stimulation to the circuit where the activity was happening. For Sarah, the procedure was highly effective. Her scores on depression rating scales dropped the morning after the device was turned on. And perhaps more importantly, she felt dramatic changes in her mood. During her first time getting the stimulation, she laughed out loud in the lab. \"And everyone in the room went, 'Oh my god,' because that's the first time I spontaneously laughed and smiled, where it wasn't faked, in five years,\" she said. Sarah's depression circuit flares up hundreds of times a day, and each time, the implanted device delivers a brief stimulating pulse. In total, she gets around 30 minutes of stimulation each day [...]. Sarah can't feel the pulses, but she said she does have a general idea of when they're happening throughout the day. \"There's a sense of alertness and energy or positivity that I'll feel,\" she said.", + "link": "https://science.slashdot.org/story/21/10/04/2241244/a-surgically-implanted-brain-stimulation-device-could-help-treat-severe-depression", + "title": "A Surgically Implanted Brain Stimulation Device Could Help Treat Severe Depression" + }, + { + "date": "Mon, 04 Oct 2021 00:00:00 GMT", + "summary": "German officials on Monday unveiled what they said is the world's first commercial plant for making synthetic kerosene, touted as a climate-friendly fuel of the future. The Associated Press reports: The facility in Werlte, near Germany's northwestern border with the Netherlands, will use water and electricity from four nearby wind farms to produce hydrogen. In a century-old process, the hydrogen is combined with carbon dioxide to make crude oil, which can then be refined into jet fuel. Burning that synthetic kerosene releases only as much CO2 into the atmosphere as was previously removed to produce the fuel, making it \"carbon neutral.\"\nThe amount of fuel that the plant can produce beginning early next year is modest: just eight barrels a day, or about 336 gallons of jet fuel. That would be enough to fill up one small passenger plane every three weeks. By comparison, total fuel consumption of commercial airlines worldwide reached 95 billion gallons in 2019, before the pandemic hit the travel industry, according to the International Air Transport Association, or IATA. But Atmosfair, a German non-profit group behind the project, says its purpose is to show that the process is technologically feasible and -- once it is scaled up and with sufficient demand -- economically viable.\nInitially the price of synthetic kerosene produced in Werlte will be far higher than that of regular jet fuel, though Atmosfair won't divulge how much it will be charging its first customer, the German airline Lufthansa. However Atmosfair's chief executive, Dietrich Brockhagen, says a price of 5 euros ($5.80) per liter (0.26 gallons) is possible. That's still several times what kerosene currently costs, but Atsmofair is banking on carbon taxes driving up the price of fossil fuels, making his product more competitive. Additionally, authorities at the national and European level are putting in place quotas for the amount of e-fuel that airlines will have to use in future. That will create demand, making it more attractive to invest in bigger and better plants. Ueckerdt said 5 euros per liter is feasible by 2030, when the European Union's executive may require airlines to meet 0.7% of their kerosene needs with e-fuels. Under current plans, that would rise to 28% by 2050.", + "link": "https://news.slashdot.org/story/21/10/04/2236211/germany-unveils-worlds-first-commercial-plant-for-making-synthetic-kerosene", + "title": "Germany Unveils World's First Commercial Plant For Making Synthetic Kerosene" + }, + { + "date": "Mon, 04 Oct 2021 00:00:00 GMT", + "summary": "We thought the carnage was over for popular decentralized finance, or DeFi, staking protocol Compound, but as it turns out, millions more than we thought are at risk. About $162 million is up for grabs after an upgrade gone very wrong, according to Robert Leshner, founder of Compound Labs. CNBC reports: At first, the Compound chief tweeted Friday that there was a cap to how many comp tokens could be accidentally distributed, noting that âoethe impact is bounded, at worst, 280,000 comp tokens,â or about $92.6 million. But on Sunday morning, Leshner revealed that the pool of cash that had already been emptied once had been replenished â\" exposing another 202,472.5 comp tokens to exploit, or roughly $66.9 million at its current price.\nOn Wednesday, Compound rolled out what should have been a pretty standard upgrade. Soon after implementation, however, it was clear that something had gone seriously wrong, once users started to receive millions of dollars in comp tokens. For example, $30 million worth of comp tokens were claimed in one transaction. The saving grace of the entire debacle, however, was the fact that the pool of cash that was open to exploit -- something called the Comptroller contract -- had a finite amount of tokens. The problem is that this leaky pool got a fresh influx of cash, and 0.5 comp tokens are being added roughly every 15 seconds, according to Gupta. \"When the drip() function was called this morning, it sent the backlog (202,472.5, about two months of COMP since the last time the function was called) into the protocol for distribution to users,\" Leshner wrote in a tweet Sunday morning. Leshner noted that this brought the total comp at risk to 490,000 comp tokens, or about $162 million.\nThere are a few proposals to fix the bug, but Compound's governance model is such that any changes to the protocol require a multiday voting window, and Gupta said it takes another week for the successful proposal to be executed. In the meantime, this pool of cash is once again up for grabs for users who know how to exploit the bug. Compound made clear that no supplied or borrowed funds were at risk, which is some consolation. \"No user funds are or were at risk so it's not that big of a deal,\" said Gupta. \"Everyone kinda got diluted but didn't lose anything directly.\"", + "link": "https://news.slashdot.org/story/21/10/04/2228209/bug-puts-162-million-up-for-grabs-says-founder-of-defi-platform-compound", + "title": "Bug Puts $162 Million Up For Grabs, Says Founder of DeFi Platform Compound" + } + ] + } + ] + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/event": { + "get": { + "tags": [ + "Event" + ], + "summary": "Event", + "description": "Automatically extracts dates, location and address information, images and event descriptions from event pages.", + "operationId": "event", + "parameters": [ + { + "name": "url", + "in": "query", + "description": "Target URL to extract", + "required": true, + "schema": { + "type": "string", + "default": "https://www.eventbrite.com/e/graphql-summit-2018-tickets-46601841362" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "venue": { + "type": "string" + }, + "images": { + "type": "array", + "items": { + "type": "object", + "properties": { + "naturalHeight": { + "type": "integer" + }, + "width": { + "type": "integer" + }, + "diffbotUri": { + "type": "string" + }, + "url": { + "type": "string" + }, + "naturalWidth": { + "type": "integer" + }, + "primary": { + "type": "boolean" + }, + "height": { + "type": "integer" + } + } + } + }, + "timezoneOffset": { + "type": "string" + }, + "endDate": { + "type": "string" + }, + "endDateLocalTimezone": { + "type": "string" + }, + "humanLanguage": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "description": { + "type": "string" + }, + "pageUrl": { + "type": "string" + }, + "location": { + "type": "object", + "properties": { + "country": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "websiteUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "targetDiffbotUri": { + "type": "string" + }, + "surfaceForm": { + "type": "string" + } + } + }, + "isCurrent": { + "type": "boolean" + }, + "address": { + "type": "string" + }, + "city": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "websiteUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "targetDiffbotUri": { + "type": "string" + }, + "surfaceForm": { + "type": "string" + } + } + }, + "street": { + "type": "string" + }, + "subregion": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "websiteUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "targetDiffbotUri": { + "type": "string" + }, + "surfaceForm": { + "type": "string" + } + } + }, + "latitude": { + "type": "number" + }, + "precision": { + "type": "number" + }, + "postalCode": { + "type": "string" + }, + "region": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "websiteUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "targetDiffbotUri": { + "type": "string" + }, + "surfaceForm": { + "type": "string" + } + } + }, + "longitude": { + "type": "number" + } + } + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "startDate": { + "type": "string" + }, + "startDateLocalTimezone": { + "type": "string" + } + } + } + } + } + }, + "example": { + "request": { + "pageUrl": "https://www.eventbrite.com/e/graphql-summit-2018-tickets-46601841362", + "api": "event", + "version": 3 + }, + "objects": [ + { + "venue": "The Regency Center", + "images": [ + { + "naturalHeight": 0, + "width": 720, + "diffbotUri": "image|3|2088675194", + "url": "https://img.evbuc.com/https%3A%2F%2Fcdn.evbuc.com%2Fimages%2F45534724%2F182934487936%2F1%2Foriginal.jpg?w=800&auto=compress&rect=0%2C0%2C2160%2C1080&s=e73c0be202c2cd939dcd3e809d96e093", + "naturalWidth": 0, + "primary": true, + "height": 360 + } + ], + "timezoneOffset": "-07:00", + "endDate": "Fri, 09 Nov 2018 01:00:00 GMT", + "endDateLocalTimezone": "Thu, 08 Nov 2018 17:00:00", + "humanLanguage": "en", + "diffbotUri": "event|3|-2069254052", + "description": "​Join over 850 developers at the 3rd annual GraphQL Summit, the world's largest conference dedicated to GraphQL. Whether you're just getting started or a veteran GraphQL practitioner, this is the must-attend developer conference of the year.\nChoose from 40+ sessions across two tracks at one of San Francisco's most historic event venues\n​See how teams at top startups and enterprises are modernizing their platforms by using GraphQL in complex production environments\nHear where the technology is heading next, directly from the leaders of GraphQL's major open source projects\nMeet top solution providers from around the world - all under one roof.\nEnroll in hands-on workshops and classroom training to boost your skills to get the most out of GraphQL.\nGraphQL Summit Day Code of Conduct\nWe believe our community and events should be truly open for everyone. As such, we are committed to providing a friendly, safe, and welcoming environment for all, regardless of gender, sexual orientation, disability, ethnicity, or religion. Our event code of conduct outlines our expectations for participant behavior, as well as the consequences for unacceptable behavior.\n*GraphQL Summit is hosted at the Regency Center which is an accessible space with elevator access*", + "pageUrl": "https://archives.diffbot.com/replay/4F02A5A651C6767544362C27B1D9BAC1/", + "location": { + "country": { + "name": "United States", + "diffbotUri": "http://diffbot.com/entity/A01d4EK33MmCosgI2KXa4-A", + "websiteUris": [ + "wikidata.org/entity/Q30" + ], + "targetDiffbotUri": "http://diffbot.com/entity/A01d4EK33MmCosgI2KXa4-A", + "surfaceForm": "United States" + }, + "isCurrent": true, + "address": "1300 Van Ness Ave, San Francisco, California", + "city": { + "name": "San Francisco", + "diffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + "websiteUris": [ + "wikidata.org/entity/Q62" + ], + "targetDiffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + "surfaceForm": "San Francisco" + }, + "street": "1300 Van Ness Ave", + "subregion": { + "name": "San Francisco County", + "diffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + "websiteUris": [ + "wikidata.org/entity/Q62" + ], + "targetDiffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + "surfaceForm": "San Francisco County" + }, + "latitude": 37.78785705566406, + "precision": 0.10000000149011612, + "postalCode": "94109", + "region": { + "name": "California", + "diffbotUri": "http://diffbot.com/entity/Al0_8ehooNxOPT36Y4RdV9w", + "websiteUris": [ + "wikidata.org/entity/Q99" + ], + "targetDiffbotUri": "http://diffbot.com/entity/Al0_8ehooNxOPT36Y4RdV9w", + "surfaceForm": "California" + }, + "longitude": -122.42138671875 + }, + "type": "event", + "title": "GraphQL Summit 2018", + "startDate": "Wed, 07 Nov 2018 16:30:00 GMT", + "startDateLocalTimezone": "Wed, 07 Nov 2018 08:30:00" + } + ] + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/{api}": { + "get": { + "summary": "Extract with Custom API", + "tags": [ + "Custom" + ], + "description": "Extracts a page using a modified Extract API or a custom ruleset.", + "operationId": "customget", + "parameters": [ + { + "name": "api", + "in": "path", + "description": "Name of your Custom API", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "url", + "in": "query", + "description": "Target URL to extract (url encoded)", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "fields", + "in": "query", + "description": "Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`)", + "schema": { + "type": "string", + "enum": [ + "links", + "extlinks", + "meta", + "querystring", + "breadcrumb" + ] + } + }, + { + "name": "timeout", + "in": "query", + "description": "Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000).", + "schema": { + "type": "integer", + "format": "int32" + } + }, + { + "name": "callback", + "in": "query", + "description": "Use for jsonp requests. Needed for cross-domain ajax.", + "schema": { + "type": "string" + } + }, + { + "name": "proxy", + "in": "query", + "description": "Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`)", + "schema": { + "type": "string", + "default": "" + } + }, + { + "name": "proxyAuth", + "in": "query", + "description": "Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`)", + "schema": { + "type": "string" + } + }, + { + "name": "useProxy", + "in": "query", + "description": "Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally.", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "request": { + "type": "object", + "properties": { + "pageUrl": { + "type": "string" + }, + "api": { + "type": "string" + }, + "version": { + "type": "integer" + } + } + }, + "objects": { + "type": "array", + "items": { + "type": "object" + } + } + } + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + }, + "/custom": { + "get": { + "summary": "Retrieve Custom APIs", + "tags": [ + "Custom" + ], + "description": "Get all the Custom APIs and their rules currently defined on your token", + "operationId": "retrieve-a-custom-api", + "parameters": [], + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "xForwardHeaders": { + "type": "object", + "properties": { + "X-Evaluate": { + "type": "string" + } + }, + "required": [ + "X-Evaluate" + ] + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "selector": { + "type": "string" + } + } + } + }, + "api": { + "type": "string" + }, + "urlPattern": { + "type": "string" + }, + "testUrl": { + "type": "string" + } + } + } + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errorCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "errorCode": 500, + "error": "Internal Server Error" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + }, + "post": { + "summary": "Create or Update a Custom API", + "tags": [ + "Custom" + ], + "description": "Create or update the parameters and ruleset of an existing Custom API", + "operationId": "create-a-custom-api", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "notes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of strings that can be added manually. The API automatically adds a notes specifying when the API was last updated." + }, + "xForwardHeaders": { + "type": "object", + "properties": { + "X-Evaluate": { + "type": "string" + } + }, + "description": "Allows you to pass in X-Forward headers by name, including X-Evaluate (aka. X-Forward-X-Evaluate; omit \"X-Forward\" in the header name)" + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "selector": { + "type": "string" + }, + "filters": { + "type": "array", + "items": { + "type": "object", + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "type": "string" + } + } + } + } + } + }, + "description": "An object that defines a set of rules for a specific urlPattern-api combination" + }, + "api": { + "type": "string", + "description": "The specific API being targeted. Always precede the API name with \"/api/\" as in `\"/api/article\"` except for \"all\")" + }, + "urlPattern": { + "type": "string", + "description": "A regex that defines the URLs for which the ruleset will be applied" + }, + "testUrl": { + "type": "string", + "description": "A URL that can be used to check that the rule still works as intended. This is the page that will load automatically when editing the ruleset in the Dashboard UI" + }, + "renderOptions": { + "type": "string", + "description": "Rendering options" + }, + "prefilters": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of string selectors that should be omitted from the DOM before extraction occurs" + }, + "useProxy": { + "type": "string", + "description": "Used to disable proxies (when they have been set globally), by applying the value `\"none\"`" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hashes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "example": { + "hashes": [ + "abcd1234" + ] + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + }, + "delete": { + "summary": "Delete a Custom API", + "tags": [ + "Custom" + ], + "description": "Delete definitions of existing Custom APIs for a given URL pattern and API on your token", + "operationId": "delete-a-custom-api", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "api": { + "type": "string", + "description": "The specific API being targeted. Always precede the API name with \"/api/\" as in `\"/api/article\"` except for \"all\")" + }, + "urlPattern": { + "type": "string", + "description": "A regex that defines the URLs corresponding to which the Custom API definitions will be deleted. Make sure that is json-escaped before url-encoding it." + }, + "token": { + "type": "string", + "description": "Your token. The Custom API definitions only for this token will be deleted" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Successful API Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "token": { + "type": "string" + }, + "urlPattern": { + "type": "string" + }, + "api": { + "type": "string" + }, + "status": { + "type": "string" + } + } + }, + "example": { + "api": "/api/discussion", + "token": "TOKEN", + "urlPattern": "examplePattern.com", + "status": "success" + } + } + } + }, + "500": { + "description": "Internal Server Error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "statusCode": { + "type": "integer" + }, + "error": { + "type": "string" + } + } + }, + "example": { + "statusCode": 500, + "error": "urlPattern and apiPrefix required to delete rule" + } + } + } + } + }, + "security": [ + { + "tokenscheme": [] + } + ] + } + } + }, + "components": { + "securitySchemes": { + "tokenscheme": { + "type": "apiKey", + "name": "token", + "in": "query" + } + } + }, + "x-readme": { + "explorer-enabled": true, + "proxy-enabled": true, + "samples-enabled": true + } +} \ No newline at end of file diff --git a/openapi/natural-language.json b/openapi/natural-language.json new file mode 100644 index 0000000..6bd5763 --- /dev/null +++ b/openapi/natural-language.json @@ -0,0 +1,719 @@ +{ + "openapi": "3.0.0", + "info": { + "description": "Extracts entities (e.g., people, organizations, locations) and facts (e.g., educated at, founded by) about these entities from natural language text.\n", + "version": "1.0.0", + "title": "Natural Language API", + "termsOfService": "https://www.diffbot.com/terms/", + "contact": { + "email": "support@diffbot.com" + } + }, + "paths": { + "/v1/": { + "post": { + "tags": [ + "Natural Language" + ], + "summary": "Process Text", + "parameters": [ + { + "name": "fields", + "in": "query", + "description": "Attributes to include in response", + "required": false, + "style": "form", + "explode": false, + "schema": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "entities", + "sentiment", + "facts", + "records", + "categories", + "sentences", + "language", + "summary" + ] + }, + "uniqueItems": true, + "default": [ + "entities", + "sentiment", + "facts", + "records", + "sentences" + ] + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DocumentArray" + } + }, + "application/x-www-form-urlencoded": { + "schema": { + "$ref": "#/components/schemas/DocumentArray" + } + } + }, + "description": "documents to be processed", + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NaturalLanguageResponseArray" + } + } + } + }, + "400": { + "description": "Invalid input", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + }, + "security": [ + { + "DiffbotToken": [] + } + ] + } + } + }, + "servers": [ + { + "url": "https://nl.diffbot.com" + } + ], + "components": { + "securitySchemes": { + "DiffbotToken": { + "type": "apiKey", + "name": "token", + "in": "query" + } + }, + "schemas": { + "Document": { + "type": "object", + "required": [ + "content" + ], + "properties": { + "content": { + "type": "string", + "format": "blob", + "description": "The text you'd like Diffbot Natural Language to process. \n", + "minLength": 1, + "maxLength": 100000 + }, + "lang": { + "type": "string", + "description": "Language of the text specified as an [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). Review our supported languages for more information.\n", + "default": "auto" + }, + "format": { + "type": "string", + "description": "`plain text` for free text. `plain text with title` includes a title separated from the body by two line breaks (e.g. \"This is a title\\n\\nThis is the body\").\n", + "default": "plain text", + "enum": [ + "plain text", + "plain text with title" + ] + }, + "documentType": { + "type": "string", + "description": "Optional. Provides context to the model and may produce better results.\n", + "enum": [ + "organization bio", + "person bio", + "news article", + "address", + "discussion" + ] + }, + "customSummary": { + "$ref": "#/components/schemas/CustomSummary" + } + }, + "example": { + "content": "Diffbot is a startup based in Menlo Park." + } + }, + "DocumentArray": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Document" + }, + "example": [ + { + "content": "Diffbot is a startup based in Menlo Park." + } + ] + }, + "EntityWithMentions": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "confidence": { + "type": "number", + "format": "float" + }, + "salience": { + "type": "number", + "format": "float" + }, + "sentiment": { + "type": "number", + "format": "float" + }, + "allUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "allTypes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityType" + } + }, + "mentions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Mention" + } + }, + "location": { + "$ref": "#/components/schemas/Location" + } + } + }, + "Mention": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "beginOffset": { + "type": "integer" + }, + "endOffset": { + "type": "integer" + }, + "isPronoun": { + "type": "boolean" + }, + "confidence": { + "type": "number", + "format": "float" + } + } + }, + "Entity": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "confidence": { + "type": "number", + "format": "float" + }, + "allUris": { + "type": "array", + "items": { + "type": "string" + } + }, + "allTypes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityType" + } + }, + "isCustom": { + "type": "boolean" + }, + "entityIndex": { + "type": "integer", + "description": "Index of this entity in the field \"entities\" (if requested)" + } + } + }, + "Span": { + "type": "object", + "properties": { + "beginOffset": { + "type": "integer" + }, + "endOffset": { + "type": "integer" + } + } + }, + "Fact": { + "type": "object", + "properties": { + "humanReadable": { + "type": "string" + }, + "entity": { + "$ref": "#/components/schemas/Entity" + }, + "property": { + "$ref": "#/components/schemas/Property" + }, + "value": { + "$ref": "#/components/schemas/Entity" + }, + "qualifiers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Qualifier" + } + }, + "confidence": { + "type": "number", + "format": "float" + }, + "evidence": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Evidence" + } + } + } + }, + "Qualifier": { + "type": "object", + "properties": { + "property": { + "$ref": "#/components/schemas/Property" + }, + "value": { + "$ref": "#/components/schemas/Qualifier_value" + }, + "confidence": { + "type": "number", + "format": "float" + } + }, + "description": "https://demo.nl.diffbot.com/schema/#qualifiers" + }, + "Property": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "description": "https://demo.nl.diffbot.com/schema/#properties" + }, + "EntityType": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + }, + "dbpediaUri": { + "type": "string" + } + }, + "description": "https://demo.nl.diffbot.com/schema/#types" + }, + "Evidence": { + "type": "object", + "properties": { + "passage": { + "type": "string", + "description": "One or more sentences where the fact was extracted from" + }, + "entityMentions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Mention" + } + }, + "valueMentions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Mention" + } + } + } + }, + "Location": { + "type": "object", + "properties": { + "latitude": { + "type": "number", + "format": "float" + }, + "longitude": { + "type": "number", + "format": "float" + }, + "precision": { + "type": "number", + "format": "float" + } + } + }, + "Category": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "confidence": { + "type": "number", + "format": "float" + }, + "isPrimary": { + "type": "boolean" + } + } + }, + "Categorization": { + "type": "object", + "properties": { + "iabv1": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Category" + } + }, + "iabv2": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Category" + } + }, + "diffbot": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Category" + } + } + }, + "description": "https://demo.nl.diffbot.com/schema/#categories" + }, + "CustomSummary": { + "type": "object", + "properties": { + "maxNumberOfSentences": { + "type": "integer", + "format": "int32", + "description": "Maximum number of sentences for the generated summary.", + "minimum": 1, + "maximum": 10, + "default": 3 + } + }, + "description": "Define custom parameters for summary generation." + }, + "NaturalLanguageResponse": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "type": "string" + } + }, + "entities": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EntityWithMentions" + } + }, + "sentiment": { + "type": "number", + "format": "float" + }, + "facts": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Fact" + } + }, + "records": { + "type": "array", + "description": "Array of entities with attributes extracted from inputDocument according to the KG schema (https://docs.diffbot.com/docs/ontology)", + "items": { + "type": "object", + "properties": {} + } + }, + "categories": { + "$ref": "#/components/schemas/Categorization" + }, + "sentences": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Span" + } + }, + "language": { + "type": "string", + "description": "Language of the text specified as an [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)." + }, + "summary": { + "type": "string", + "description": "Generated summary." + } + } + }, + "NaturalLanguageResponseArray": { + "type": "array", + "items": { + "$ref": "#/components/schemas/NaturalLanguageResponse" + }, + "example": [ + { + "sentiment": 0, + "entities": [ + { + "name": "Diffbot", + "diffbotUri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + "confidence": 0.99898225, + "salience": 0.95394653, + "sentiment": 0, + "isCustom": false, + "allUris": [ + "http://www.wikidata.org/entity/Q17052069" + ], + "allTypes": [ + { + "name": "organization", + "diffbotUri": "https://diffbot.com/entity/EN1ClYEdMMQCxB6AWTkT3mA", + "dbpediaUri": "http://dbpedia.org/ontology/Organisation" + } + ], + "mentions": [ + { + "text": "Diffbot", + "beginOffset": 0, + "endOffset": 7, + "confidence": 0.99898225 + } + ], + "location": { + "latitude": 37.456104, + "longitude": -122.1763, + "precision": 0.1 + } + }, + { + "name": "Menlo Park", + "diffbotUri": "https://diffbot.com/entity/EQiAUr370NpW1gvx-V4fTrQ", + "confidence": 0.9996269, + "salience": 0.9082091, + "sentiment": 0, + "isCustom": false, + "allUris": [ + "http://www.wikidata.org/entity/Q74195" + ], + "allTypes": [ + { + "name": "location", + "diffbotUri": "https://diffbot.com/entity/EiCyWUm4lNziqgLHx47iAIQ", + "dbpediaUri": "http://dbpedia.org/ontology/Place" + }, + { + "name": "administrative area", + "diffbotUri": "https://diffbot.com/entity/EcTIu1tWKPouIa6qZtSpc4A", + "dbpediaUri": "http://dbpedia.org/ontology/PopulatedPlace" + }, + { + "name": "city", + "diffbotUri": "https://diffbot.com/entity/EzdJrGHiyMWu0XbSn101rFA", + "dbpediaUri": "http://dbpedia.org/ontology/City" + } + ], + "mentions": [ + { + "text": "Menlo Park", + "beginOffset": 30, + "endOffset": 40, + "confidence": 0.9996269 + } + ], + "location": { + "latitude": 37.454166, + "longitude": -122.17861, + "precision": 6.708204 + } + }, + { + "name": "startup company", + "diffbotUri": "https://diffbot.com/entity/EfX0EeXroOmaXrLe-SylIuw", + "confidence": 0.81958973, + "salience": 0.6474858, + "sentiment": 0, + "isCustom": false, + "allUris": [], + "mentions": [ + { + "text": "startup", + "beginOffset": 13, + "endOffset": 20, + "confidence": 0.81958973 + } + ] + } + ], + "facts": [ + { + "humanReadable": "[Diffbot] headquarters [Menlo Park]", + "entity": { + "name": "Diffbot", + "diffbotUri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + "confidence": 0.99898225, + "allUris": [ + "http://www.wikidata.org/entity/Q17052069" + ], + "allTypes": [ + { + "name": "organization", + "diffbotUri": "https://diffbot.com/entity/EN1ClYEdMMQCxB6AWTkT3mA", + "dbpediaUri": "http://dbpedia.org/ontology/Organisation" + } + ], + "isCustom": false, + "entityIndex": 1 + }, + "property": { + "name": "headquarters", + "diffbotUri": "https://docs.diffbot.com/docs/ont-organization#location" + }, + "value": { + "name": "Menlo Park", + "diffbotUri": "https://diffbot.com/entity/EQiAUr370NpW1gvx-V4fTrQ", + "confidence": 0.9996269, + "allUris": [ + "http://www.wikidata.org/entity/Q74195" + ], + "allTypes": [ + { + "name": "location", + "diffbotUri": "https://diffbot.com/entity/EiCyWUm4lNziqgLHx47iAIQ", + "dbpediaUri": "http://dbpedia.org/ontology/Place" + }, + { + "name": "administrative area", + "diffbotUri": "https://diffbot.com/entity/EcTIu1tWKPouIa6qZtSpc4A", + "dbpediaUri": "http://dbpedia.org/ontology/PopulatedPlace" + }, + { + "name": "city", + "diffbotUri": "https://diffbot.com/entity/EzdJrGHiyMWu0XbSn101rFA", + "dbpediaUri": "http://dbpedia.org/ontology/City" + } + ], + "isCustom": false, + "entityIndex": 0 + }, + "qualifiers": [ + { + "property": { + "name": "is current" + }, + "value": { + "name": "true" + }, + "confidence": 0 + } + ], + "confidence": 0.95, + "evidence": [ + { + "passage": "Diffbot is a startup based in Menlo Park.", + "entityMentions": [ + { + "text": "Diffbot", + "beginOffset": 0, + "endOffset": 7, + "confidence": 0.99898225 + } + ], + "valueMentions": [ + { + "text": "Menlo Park", + "beginOffset": 30, + "endOffset": 40, + "confidence": 0.9996269 + } + ] + } + ] + } + ] + } + ] + }, + "ErrorResponse": { + "type": "object", + "properties": { + "messages": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "Qualifier_value": { + "properties": { + "name": { + "type": "string" + }, + "diffbotUri": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/openapi/web-search.json b/openapi/web-search.json new file mode 100644 index 0000000..9c6c829 --- /dev/null +++ b/openapi/web-search.json @@ -0,0 +1,323 @@ +{ + "openapi": "3.0.0", + "info": { + "version": "1.0.0", + "title": "Web Search" + }, + "servers": [ + { + "url": "https://llm.diffbot.com/api/v1/web_search" + } + ], + "tags": [], + "paths": { + "/": { + "get": { + "summary": "Run a Web Search", + "description": "Search Diffbot's Web Index", + "responses": { + "200": { + "$ref": "#/components/responses/Results" + }, + "400": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "type": "string", + "example": "query param text size must be between 1 and 5" + } + } + } + }, + "examples": { + "Bad Request": { + "summary": "Bad Request", + "value": { + "errors": [ + "query param text size must be between 1 and 5" + ] + } + } + } + } + }, + "description": "Bad Request" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + } + }, + "operationId": "get_", + "parameters": [ + { + "in": "query", + "name": "text", + "required": true, + "schema": { + "$ref": "#/components/schemas/Query" + } + }, + { + "in": "query", + "name": "size", + "schema": { + "$ref": "#/components/schemas/Size" + } + }, + { + "in": "query", + "name": "maxTokens", + "schema": { + "$ref": "#/components/schemas/MaxTokens" + } + } + ] + }, + "post": { + "summary": "Run a Web Search", + "operationId": "post_", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "text" + ], + "properties": { + "text": { + "$ref": "#/components/schemas/Query" + }, + "size": { + "$ref": "#/components/schemas/Size" + }, + "maxTokens": { + "$ref": "#/components/schemas/MaxTokens" + } + } + }, + "examples": { + "Search": { + "summary": "A single query", + "value": { + "text": [ + "diffbot" + ], + "size": 10 + } + } + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/Results" + }, + "400": { + "description": "Bad Request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 400 + }, + "message": { + "type": "string", + "example": "Missing text parameter" + } + } + }, + "examples": { + "Missing text": { + "summary": "Missing text", + "value": { + "code": 400, + "message": "Missing text parameter" + } + } + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + } + } + } + } + }, + "components": { + "securitySchemes": { + "Authorization": { + "type": "http", + "scheme": "bearer", + "description": "Diffbot API Token" + } + }, + "schemas": { + "Query": { + "type": "array", + "minItems": 1, + "maxItems": 5, + "items": { + "type": "string", + "example": "diffbot" + }, + "description": "Your search query.\n\nUse the following operators to refine your search. \n\n| Operator | Description | Example |\n| --- | --- | --- |\n| `after` | Documents published after the provided date | `after:2026-06-10` |\n| `before` | Documents published before the provided date | `before:2026-05-10` |\n| `site` | Documents matching the provided domain | `site:diffbot.com` |\n| `url` | Documents matching the exact URL | `url:diffbot.com` |" + }, + "Size": { + "type": "number", + "default": "10", + "description": "Number of results to return in the search response" + }, + "MaxTokens": { + "type": "number", + "description": "Cap the search response to this value. Highlight chunks will be reduced in length equitably." + } + }, + "responses": { + "Results": { + "description": "Successful response", + "content": { + "application/json": { + "examples": { + "OK": { + "summary": "OK", + "value": { + "query": [ + "Diffbot" + ], + "search_results": [ + { + "score": 0.973, + "pageUrl": "https://www.diffbot.com/", + "title": "Web Data for your AI", + "content": "Imagine if your app could access the web like a structured database.\n\n Your browser does not support the video tag. \n- 50+ data fields, including categories, revenue, locations, and investments\n- Over 246M companies and non-profits in the Knowledge Graph\n- Extract and refresh orgs on demand\n\n- More than just text \u2014 entity matching, topic-level sentiment, and more\n- Over 1.6B news articles, blog posts, and press releases in the Knowledge Graph\n\n\n## The Web is Noisy, \nDiffbot Straightens it Out \n\nThe world's largest compendium of human knowledge is buried in the code of 1.2 billion public websites. Diffbot reads it all like a human, then transforms it into usable data." + }, + { + "score": 0.887, + "pageUrl": "https://www.diffbot.com/data/article/", + "title": "Diffbot | News & Articles", + "content": "..." + }, + { + "score": 0.847, + "pageUrl": "https://github.com/diffbot", + "title": "Diffbot", + "date": "Thu, 12 Mar 2026 00:00:00 GMT", + "content": "..." + } + ], + "timeMs": 154 + } + } + }, + "schema": { + "type": "object", + "properties": { + "query": { + "type": "array", + "items": { + "type": "string", + "example": "Diffbot" + }, + "description": "The parsed query terms." + }, + "search_results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "score": { + "type": "number", + "example": 0.973, + "default": 0, + "description": "Represents the confidence of the result from 0 (lowest) to highest (1). 3 decimal places of precision." + }, + "pageUrl": { + "type": "string", + "example": "https://www.diffbot.com/", + "description": "URL of the page." + }, + "title": { + "type": "string", + "example": "Web Data for your AI", + "description": "Title of the page." + }, + "content": { + "type": "string", + "example": "Imagine if your app could access the web like a structured database.\n\n Your browser does not support the video tag. \n- 50+ data fields, including categories, revenue, locations, and investments\n- Over 246M companies and non-profits in the Knowledge Graph\n- Extract and refresh orgs on demand\n\n- More than just text \u2014 entity matching, topic-level sentiment, and more\n- Over 1.6B news articles, blog posts, and press releases in the Knowledge Graph\n\n\n## The Web is Noisy, \nDiffbot Straightens it Out \n\nThe world's largest compendium of human knowledge is buried in the code of 1.2 billion public websites. Diffbot reads it all like a human, then transforms it into usable data.", + "description": "A spliced chunk highlight from the original full page contents." + }, + "date": { + "type": "string", + "example": "Thu, 12 Mar 2026 00:00:00 GMT", + "description": "Publication date of the page, when known. RFC 1123 format. Omitted when the page has no discoverable publication date." + } + } + }, + "description": "The ranked results." + }, + "timeMs": { + "type": "integer", + "example": 154, + "default": 0, + "description": "Time taken for retrieval, reranking, and response generation." + } + } + } + } + } + }, + "Unauthorized": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 401, + "default": 0 + }, + "message": { + "type": "string", + "example": "Missing or invalid Authorization header. Expected: Bearer " + } + } + }, + "examples": { + "Unauthorized": { + "summary": "Unauthorized", + "value": { + "code": 401, + "message": "Missing or invalid Authorization header. Expected: Bearer " + } + } + } + } + }, + "description": "Unauthorized" + } + } + }, + "x-readme": { + "explorer-enabled": true, + "proxy-enabled": false + }, + "security": [ + { + "Authorization": [] + } + ] +} diff --git a/package.json b/package.json index cfd4724..57a1b19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@diffbot/typescript", - "version": "0.3.0", + "version": "0.6.0", "description": "TypeScript client library for Diffbot APIs", "type": "module", "packageManager": "pnpm@10.33.0", @@ -36,8 +36,11 @@ "test:watch": "vitest", "test:live": "vitest run --config vitest.live.config.ts", "test:coverage": "vitest run --coverage --coverage.include='src/**'", + "test:types": "vitest run --typecheck", + "lint": "eslint .", "typecheck": "tsc --noEmit", - "postbuild": "node scripts/assert-no-node-builtins.mjs" + "postbuild": "node scripts/assert-no-node-builtins.mjs", + "gen:types": "node scripts/gen-types.mjs" }, "keywords": [ "diffbot", @@ -63,10 +66,14 @@ "access": "public" }, "devDependencies": { + "@eslint/js": "^10.0.1", "@types/node": "^22.15.21", "@vitest/coverage-v8": "^3.2.6", + "eslint": "^10.14.0", + "openapi-typescript": "^7.13.0", "tsup": "^8.5.0", "typescript": "^5.8.3", + "typescript-eslint": "^8.72.2", "vitest": "^3.2.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6d5be2d..6d058ff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,18 +8,30 @@ importers: .: devDependencies: + '@eslint/js': + specifier: ^10.0.1 + version: 10.0.1(eslint@10.14.0) '@types/node': specifier: ^22.15.21 version: 22.19.21 '@vitest/coverage-v8': specifier: ^3.2.6 version: 3.2.6(vitest@3.2.6(@types/node@22.19.21)) + eslint: + specifier: ^10.14.0 + version: 10.14.0 + openapi-typescript: + specifier: ^7.13.0 + version: 7.13.0(typescript@5.9.3) tsup: specifier: ^8.5.0 version: 8.5.1(postcss@8.5.15)(typescript@5.9.3) typescript: specifier: ^5.8.3 version: 5.9.3 + typescript-eslint: + specifier: ^8.72.2 + version: 8.72.2(eslint@10.14.0)(typescript@5.9.3) vitest: specifier: ^3.2.1 version: 3.2.6(@types/node@22.19.21) @@ -30,6 +42,10 @@ packages: resolution: {integrity: sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==} engines: {node: '>=6.0.0'} + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + '@babel/helper-string-parser@7.29.7': resolution: {integrity: sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==} engines: {node: '>=6.9.0'} @@ -51,6 +67,12 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} + '@cacheable/memory@2.2.0': + resolution: {integrity: sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==} + + '@cacheable/utils@2.5.0': + resolution: {integrity: sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==} + '@esbuild/aix-ppc64@0.27.7': resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} @@ -207,6 +229,65 @@ packages: cpu: [x64] os: [win32] + '@eslint-community/eslint-utils@4.10.1': + resolution: {integrity: sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.3': + resolution: {integrity: sha512-R76UKbvsqI68DJJVTVyqFKqU98CN4+B+acxNfEA2IFarSzP0SjtKf/3+fFiXzTyv5Fj3PxLkGHe2wnVWWMxobQ==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.23.5': + resolution: {integrity: sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/config-helpers@0.7.0': + resolution: {integrity: sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/core@1.2.1': + resolution: {integrity: sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/js@10.0.1': + resolution: {integrity: sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + peerDependencies: + eslint: ^10.0.0 + peerDependenciesMeta: + eslint: + optional: true + + '@eslint/object-schema@3.0.5': + resolution: {integrity: sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@eslint/plugin-kit@0.7.3': + resolution: {integrity: sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + '@humanfs/core@0.19.2': + resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.8': + resolution: {integrity: sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==} + engines: {node: '>=18.18.0'} + + '@humanfs/types@0.15.0': + resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -228,10 +309,29 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@keyv/bigmap@1.3.1': + resolution: {integrity: sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==} + engines: {node: '>= 18'} + peerDependencies: + keyv: ^5.6.0 + + '@keyv/serialize@1.1.1': + resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@redocly/ajv@8.11.2': + resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} + + '@redocly/config@0.22.0': + resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==} + + '@redocly/openapi-core@1.34.24': + resolution: {integrity: sha512-jDFMu4zirFVOF9nEWCT31s7OzzVhkxZkb4lO3yZhkdPXAOspLo10ED9thVgm6gtI0dJBTLa+mNP0IYeZpVSYGA==} + engines: {node: '>=18.17.0', npm: '>=9.5.0'} + '@rollup/rollup-android-arm-eabi@4.62.2': resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} cpu: [arm] @@ -376,12 +476,77 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/esrecurse@4.3.1': + resolution: {integrity: sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==} + '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/node@22.19.21': resolution: {integrity: sha512-VMeFBSCKQKmm2swI2kW51SFusDqekC6q9trBCvJ/JliDchFSuoYYKN7yVNjPthP1HKZcx3U1gI/wTcEBjEFKTA==} + '@typescript-eslint/eslint-plugin@8.72.2': + resolution: {integrity: sha512-222PKFXKihmFpi/pS5njoBgdAqsPhj2/GBmFbrNw8Tq8gEG9Sv+kSlKFy4u2q+mJv9ZzJgM2Gkv/bUzRZwZw5g==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.72.2 + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/parser@8.72.2': + resolution: {integrity: sha512-llP+T1YuJOLlxEciP6OyQGtz5SMxyn+9LA2BvZw5RmToGVa1y4beEDVQL6Gje7T5WFYh19WREjDCN9Ev4gm5AA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/project-service@8.72.2': + resolution: {integrity: sha512-6y5dLTZdHhI8Eh3yIQ/bfQhRKwn5e+TpJj64o18Cngc0iW/BCrVvrr6zGYqXPnYH31u6YXAYiFh0+qxX5mybJw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/scope-manager@8.72.2': + resolution: {integrity: sha512-iIoMJ6/8c1/QpFNyFjU0d1YzPco4MsZjQw0k3V+vfzNWAFmUmvhlyMoYEoYYDQZXc+J6Gi1X5XcxlzW2taIIGQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.72.2': + resolution: {integrity: sha512-+h3iCgZXuTNtoLXcnnjP12ZUHLMqoUcIHPfQrFEXTUqvb9dF8QK9b5nv5VtSo5fLkdHuks8ehbTzZj3rImAm9w==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/type-utils@8.72.2': + resolution: {integrity: sha512-STfkR38qIdhfkLpgtXnUjAzsNy4xouqiBO0lrVc/Wpk9Kem5/aHXUx1J8dGC16zpdYN1+yc2ILQ28er1kXX7ag==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/types@8.72.2': + resolution: {integrity: sha512-Xfn37OLZx03+PCU7qWx9DTv2FCRh1E6fk2+wONeNY15HXIwtcwuveaSwVpGAUptkNmPQegRxW5JyRel9T7wrog==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.72.2': + resolution: {integrity: sha512-pQEGqIK/cqEvKivVWaFLYpe7XYejFxfuAOi4mu1oAWFvIo36aRHI9esRFTUxcIy/z8N482RJyH+LpZB0ZIU7Ow==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/utils@8.72.2': + resolution: {integrity: sha512-hv3eRYbpPbbpezuax1EfGrzpa3C4ySgNZkPUkLOWOAjU8HsOKowEDAr8C7nsNl1C+SBSKDpvno6VKHnlhvp9iA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/visitor-keys@8.72.2': + resolution: {integrity: sha512-NAGrjn0Pitjqzq/GkgMC4kBYm233PQ8GffyQ9+kTA3AlNQYEzOf25EIM02BAHZv5FRJSpHpJk+OGUxVkOB93GQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@vitest/coverage-v8@3.2.6': resolution: {integrity: sha512-LsAdmUapA0qSN306d8+zOyawM0hFm2m2Hg9IwVNIKBm+qJV8cijiq2c+gxKZcB1HCfIWAy+0qEZDCUQA58A1cw==} peerDependencies: @@ -420,11 +585,27 @@ packages: '@vitest/utils@3.2.6': resolution: {integrity: sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==} + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + acorn@8.17.0: resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} engines: {node: '>=0.4.0'} hasBin: true + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + ajv@6.15.0: + resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} + + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -444,6 +625,9 @@ packages: any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -461,6 +645,10 @@ packages: brace-expansion@2.1.1: resolution: {integrity: sha512-WR1cURNjuvBLMZBMbqM0UoE+WAfdUcEV1ccD8PVBVOI+Z3ND4+SZbN8RsfT2bMuG1qwz5RFvPukSZm5fF2D5eA==} + brace-expansion@5.0.13: + resolution: {integrity: sha512-Vh3dmFbc4wZeox7M9I/0L9GfFd83Su4+yCck/OF9Gi/P/dAHn0RT7X4K9R7gMNMhDck3DvhR6QXsnQdjjgwNvQ==} + engines: {node: 20 || >=22} + brace-expansion@5.0.6: resolution: {integrity: sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==} engines: {node: 18 || 20 || >=22} @@ -475,10 +663,16 @@ packages: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} + cacheable@2.5.0: + resolution: {integrity: sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==} + chai@5.3.3: resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} engines: {node: '>=18'} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + check-error@2.1.3: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} @@ -494,6 +688,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@1.4.0: + resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} @@ -522,6 +719,9 @@ packages: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} @@ -539,13 +739,68 @@ packages: engines: {node: '>=18'} hasBin: true + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-scope@9.1.2: + resolution: {integrity: sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@5.0.1: + resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint@10.14.0: + resolution: {integrity: sha512-T0pNsxkb9IKmtQjhpK3d7nWOpFv+YHuMbTJnSVMoQblF0onvXfErrtp4V1jHtsQPan7dKaGVD0q1WCXnzMF31A==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@11.2.0: + resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -555,9 +810,22 @@ packages: picomatch: optional: true + file-entry-cache@11.1.5: + resolution: {integrity: sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + fix-dts-default-cjs-exports@1.0.1: resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + flat-cache@6.1.23: + resolution: {integrity: sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==} + + flatted@3.4.4: + resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} + foreground-child@3.3.1: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} @@ -567,6 +835,10 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + glob@10.5.0: resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me @@ -576,13 +848,51 @@ packages: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} + hashery@1.5.1: + resolution: {integrity: sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==} + engines: {node: '>=20'} + + hookified@1.15.1: + resolution: {integrity: sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==} + + hookified@2.2.0: + resolution: {integrity: sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==} + html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.14: + resolution: {integrity: sha512-wCBiZ77u1tuWde6G/nfOtLaCTYcFDU5yCzARzK10y+rLhzYoL7a7yEemcopcc2Mzow7gZOIjnV2ym1NYmdiJOQ==} + engines: {node: '>= 4'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + index-to-position@1.2.0: + resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} + engines: {node: '>=18'} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -609,12 +919,39 @@ packages: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@10.0.0: resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-tokens@9.0.1: resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + js-yaml@4.3.2: + resolution: {integrity: sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==} + hasBin: true + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + keyv@5.6.0: + resolution: {integrity: sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -626,6 +963,10 @@ packages: resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} @@ -646,6 +987,14 @@ packages: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} + minimatch@10.2.6: + resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} + engines: {node: 18 || 20 || >=22} + + minimatch@5.1.9: + resolution: {integrity: sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==} + engines: {node: '>=10'} + minimatch@9.0.9: resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} engines: {node: '>=16 || 14 >=14.17'} @@ -668,13 +1017,42 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + openapi-typescript@7.13.0: + resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==} + hasBin: true + peerDependencies: + typescript: ^5.x + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + parse-json@8.3.0: + resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==} + engines: {node: '>=18'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -704,6 +1082,10 @@ packages: pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -726,10 +1108,26 @@ packages: resolution: {integrity: sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==} engines: {node: ^10 || ^12 || >=14} + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + qified@0.10.1: + resolution: {integrity: sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==} + engines: {node: '>=20'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + resolve-from@5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} @@ -797,6 +1195,10 @@ packages: engines: {node: '>=16 || 14 >=14.17'} hasBin: true + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -838,6 +1240,12 @@ packages: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true + ts-api-utils@2.5.0: + resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} @@ -860,6 +1268,21 @@ packages: typescript: optional: true + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + + typescript-eslint@8.72.2: + resolution: {integrity: sha512-c/+AKMW1KB81r28Sp0AAdcyXj5dzNmAsZFcsrxKbZTKmcBK2xIwBR31dcSYSDu3bKmzXe8GKrSxO+eDQ401Uzg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -871,6 +1294,12 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + uri-js-replace@1.0.2: + resolution: {integrity: sha512-VeBBt/nMxedczht3KDL2yBBMJqIxpCQLMZp3UJ1otcWwUHdQW4IiyHpJivExzSK+FdxTowRfGL4FmwDqH9+S8Q==} + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + vite-node@3.2.4: resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} @@ -954,6 +1383,10 @@ packages: engines: {node: '>=8'} hasBin: true + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -962,6 +1395,17 @@ packages: resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} engines: {node: '>=12'} + yaml-ast-parser@0.0.43: + resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==} + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + snapshots: '@ampproject/remapping@2.3.0': @@ -969,6 +1413,12 @@ snapshots: '@jridgewell/gen-mapping': 0.3.13 '@jridgewell/trace-mapping': 0.3.31 + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + '@babel/helper-string-parser@7.29.7': {} '@babel/helper-validator-identifier@7.29.7': {} @@ -984,6 +1434,18 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} + '@cacheable/memory@2.2.0': + dependencies: + '@cacheable/utils': 2.5.0 + '@keyv/bigmap': 1.3.1(keyv@5.6.0) + hookified: 1.15.1 + keyv: 5.6.0 + + '@cacheable/utils@2.5.0': + dependencies: + hashery: 1.5.1 + keyv: 5.6.0 + '@esbuild/aix-ppc64@0.27.7': optional: true @@ -1062,6 +1524,56 @@ snapshots: '@esbuild/win32-x64@0.27.7': optional: true + '@eslint-community/eslint-utils@4.10.1(eslint@10.14.0)': + dependencies: + eslint: 10.14.0 + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.3': {} + + '@eslint/config-array@0.23.5': + dependencies: + '@eslint/object-schema': 3.0.5 + debug: 4.4.3(supports-color@10.2.2) + minimatch: 10.2.6 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.7.0': + dependencies: + '@eslint/core': 1.2.1 + + '@eslint/core@1.2.1': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/js@10.0.1(eslint@10.14.0)': + optionalDependencies: + eslint: 10.14.0 + + '@eslint/object-schema@3.0.5': {} + + '@eslint/plugin-kit@0.7.3': + dependencies: + '@eslint/core': 1.2.1 + levn: 0.4.1 + + '@humanfs/core@0.19.2': + dependencies: + '@humanfs/types': 0.15.0 + + '@humanfs/node@0.16.8': + dependencies: + '@humanfs/core': 0.19.2 + '@humanfs/types': 0.15.0 + '@humanwhocodes/retry': 0.4.3 + + '@humanfs/types@0.15.0': {} + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -1087,9 +1599,40 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@keyv/bigmap@1.3.1(keyv@5.6.0)': + dependencies: + hashery: 1.5.1 + hookified: 1.15.1 + keyv: 5.6.0 + + '@keyv/serialize@1.1.1': {} + '@pkgjs/parseargs@0.11.0': optional: true + '@redocly/ajv@8.11.2': + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js-replace: 1.0.2 + + '@redocly/config@0.22.0': {} + + '@redocly/openapi-core@1.34.24(supports-color@10.2.2)': + dependencies: + '@redocly/ajv': 8.11.2 + '@redocly/config': 0.22.0 + colorette: 1.4.0 + https-proxy-agent: 7.0.6(supports-color@10.2.2) + js-levenshtein: 1.1.6 + js-yaml: 4.3.2 + minimatch: 5.1.9 + pluralize: 8.0.0 + yaml-ast-parser: 0.0.43 + transitivePeerDependencies: + - supports-color + '@rollup/rollup-android-arm-eabi@4.62.2': optional: true @@ -1172,18 +1715,113 @@ snapshots: '@types/deep-eql@4.0.2': {} + '@types/esrecurse@4.3.1': {} + '@types/estree@1.0.9': {} + '@types/json-schema@7.0.15': {} + '@types/node@22.19.21': dependencies: undici-types: 6.21.0 + '@typescript-eslint/eslint-plugin@8.72.2(@typescript-eslint/parser@8.72.2(eslint@10.14.0)(typescript@5.9.3))(eslint@10.14.0)(typescript@5.9.3)': + dependencies: + '@eslint-community/regexpp': 4.12.3 + '@typescript-eslint/parser': 8.72.2(eslint@10.14.0)(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.72.2 + '@typescript-eslint/type-utils': 8.72.2(eslint@10.14.0)(typescript@5.9.3) + '@typescript-eslint/utils': 8.72.2(eslint@10.14.0)(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.72.2 + eslint: 10.14.0 + ignore: 7.0.14 + natural-compare: 1.4.0 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.72.2(eslint@10.14.0)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.72.2 + '@typescript-eslint/types': 8.72.2 + '@typescript-eslint/typescript-estree': 8.72.2(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.72.2 + debug: 4.4.3(supports-color@10.2.2) + eslint: 10.14.0 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.72.2(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.72.2(typescript@5.9.3) + '@typescript-eslint/types': 8.72.2 + debug: 4.4.3(supports-color@10.2.2) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.72.2': + dependencies: + '@typescript-eslint/types': 8.72.2 + '@typescript-eslint/visitor-keys': 8.72.2 + + '@typescript-eslint/tsconfig-utils@8.72.2(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + + '@typescript-eslint/type-utils@8.72.2(eslint@10.14.0)(typescript@5.9.3)': + dependencies: + '@typescript-eslint/types': 8.72.2 + '@typescript-eslint/typescript-estree': 8.72.2(typescript@5.9.3) + '@typescript-eslint/utils': 8.72.2(eslint@10.14.0)(typescript@5.9.3) + debug: 4.4.3(supports-color@10.2.2) + eslint: 10.14.0 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.72.2': {} + + '@typescript-eslint/typescript-estree@8.72.2(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.72.2(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.72.2(typescript@5.9.3) + '@typescript-eslint/types': 8.72.2 + '@typescript-eslint/visitor-keys': 8.72.2 + debug: 4.4.3(supports-color@10.2.2) + minimatch: 10.2.5 + semver: 7.8.5 + tinyglobby: 0.2.17 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.72.2(eslint@10.14.0)(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.10.1(eslint@10.14.0) + '@typescript-eslint/scope-manager': 8.72.2 + '@typescript-eslint/types': 8.72.2 + '@typescript-eslint/typescript-estree': 8.72.2(typescript@5.9.3) + eslint: 10.14.0 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.72.2': + dependencies: + '@typescript-eslint/types': 8.72.2 + eslint-visitor-keys: 5.0.1 + '@vitest/coverage-v8@3.2.6(vitest@3.2.6(@types/node@22.19.21))': dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 ast-v8-to-istanbul: 0.3.12 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 istanbul-lib-source-maps: 5.0.6 @@ -1239,8 +1877,23 @@ snapshots: loupe: 3.2.1 tinyrainbow: 2.0.0 + acorn-jsx@5.3.2(acorn@8.17.0): + dependencies: + acorn: 8.17.0 + acorn@8.17.0: {} + agent-base@7.1.4: {} + + ajv@6.15.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ansi-colors@4.1.3: {} + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -1253,6 +1906,8 @@ snapshots: any-promise@1.3.0: {} + argparse@2.0.1: {} + assertion-error@2.0.1: {} ast-v8-to-istanbul@0.3.12: @@ -1269,6 +1924,10 @@ snapshots: dependencies: balanced-match: 1.0.2 + brace-expansion@5.0.13: + dependencies: + balanced-match: 4.0.4 + brace-expansion@5.0.6: dependencies: balanced-match: 4.0.4 @@ -1280,6 +1939,14 @@ snapshots: cac@6.7.14: {} + cacheable@2.5.0: + dependencies: + '@cacheable/memory': 2.2.0 + '@cacheable/utils': 2.5.0 + hookified: 1.15.1 + keyv: 5.6.0 + qified: 0.10.1 + chai@5.3.3: dependencies: assertion-error: 2.0.1 @@ -1288,6 +1955,8 @@ snapshots: loupe: 3.2.1 pathval: 2.0.1 + change-case@5.4.4: {} + check-error@2.1.3: {} chokidar@4.0.3: @@ -1300,6 +1969,8 @@ snapshots: color-name@1.1.4: {} + colorette@1.4.0: {} + commander@4.1.1: {} confbox@0.1.8: {} @@ -1312,12 +1983,16 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 - debug@4.4.3: + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 deep-eql@5.0.2: {} + deep-is@0.1.4: {} + eastasianwidth@0.2.0: {} emoji-regex@8.0.0: {} @@ -1355,22 +2030,111 @@ snapshots: '@esbuild/win32-ia32': 0.27.7 '@esbuild/win32-x64': 0.27.7 + escape-string-regexp@4.0.0: {} + + eslint-scope@9.1.2: + dependencies: + '@types/esrecurse': 4.3.1 + '@types/estree': 1.0.9 + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@5.0.1: {} + + eslint@10.14.0: + dependencies: + '@eslint-community/eslint-utils': 4.10.1(eslint@10.14.0) + '@eslint-community/regexpp': 4.12.3 + '@eslint/config-array': 0.23.5 + '@eslint/config-helpers': 0.7.0 + '@eslint/core': 1.2.1 + '@eslint/plugin-kit': 0.7.3 + '@humanfs/node': 0.16.8 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.9 + ajv: 6.15.0 + cross-spawn: 7.0.6 + debug: 4.4.3(supports-color@10.2.2) + escape-string-regexp: 4.0.0 + eslint-scope: 9.1.2 + eslint-visitor-keys: 5.0.1 + espree: 11.2.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 11.1.5 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + minimatch: 10.2.6 + natural-compare: 1.4.0 + optionator: 0.9.4 + transitivePeerDependencies: + - supports-color + + espree@11.2.0: + dependencies: + acorn: 8.17.0 + acorn-jsx: 5.3.2(acorn@8.17.0) + eslint-visitor-keys: 5.0.1 + + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + estree-walker@3.0.3: dependencies: '@types/estree': 1.0.9 + esutils@2.0.3: {} + expect-type@1.3.0: {} + fast-deep-equal@3.1.3: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 + file-entry-cache@11.1.5: + dependencies: + flat-cache: 6.1.23 + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + fix-dts-default-cjs-exports@1.0.1: dependencies: magic-string: 0.30.21 mlly: 1.8.2 rollup: 4.62.2 + flat-cache@6.1.23: + dependencies: + cacheable: 2.5.0 + flatted: 3.4.4 + hookified: 1.15.1 + + flatted@3.4.4: {} + foreground-child@3.3.1: dependencies: cross-spawn: 7.0.6 @@ -1379,6 +2143,10 @@ snapshots: fsevents@2.3.3: optional: true + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + glob@10.5.0: dependencies: foreground-child: 3.3.1 @@ -1390,10 +2158,39 @@ snapshots: has-flag@4.0.0: {} + hashery@1.5.1: + dependencies: + hookified: 1.15.1 + + hookified@1.15.1: {} + + hookified@2.2.0: {} + html-escaper@2.0.2: {} + https-proxy-agent@7.0.6(supports-color@10.2.2): + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + ignore@5.3.2: {} + + ignore@7.0.14: {} + + imurmurhash@0.1.4: {} + + index-to-position@1.2.0: {} + + is-extglob@2.1.1: {} + is-fullwidth-code-point@3.0.0: {} + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} @@ -1407,7 +2204,7 @@ snapshots: istanbul-lib-source-maps@5.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.31 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -1425,16 +2222,43 @@ snapshots: joycon@3.1.1: {} + js-levenshtein@1.1.6: {} + js-tokens@10.0.0: {} + js-tokens@4.0.0: {} + js-tokens@9.0.1: {} + js-yaml@4.3.2: + dependencies: + argparse: 2.0.1 + + json-schema-traverse@0.4.1: {} + + json-schema-traverse@1.0.0: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + keyv@5.6.0: + dependencies: + '@keyv/serialize': 1.1.1 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} load-tsconfig@0.2.5: {} + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + loupe@3.2.1: {} lru-cache@10.4.3: {} @@ -1457,6 +2281,14 @@ snapshots: dependencies: brace-expansion: 5.0.6 + minimatch@10.2.6: + dependencies: + brace-expansion: 5.0.13 + + minimatch@5.1.9: + dependencies: + brace-expansion: 2.1.1 + minimatch@9.0.9: dependencies: brace-expansion: 2.1.1 @@ -1480,10 +2312,47 @@ snapshots: nanoid@3.3.13: {} + natural-compare@1.4.0: {} + object-assign@4.1.1: {} + openapi-typescript@7.13.0(typescript@5.9.3): + dependencies: + '@redocly/openapi-core': 1.34.24(supports-color@10.2.2) + ansi-colors: 4.1.3 + change-case: 5.4.4 + parse-json: 8.3.0 + supports-color: 10.2.2 + typescript: 5.9.3 + yargs-parser: 21.1.1 + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + package-json-from-dist@1.0.1: {} + parse-json@8.3.0: + dependencies: + '@babel/code-frame': 7.29.7 + index-to-position: 1.2.0 + type-fest: 4.41.0 + + path-exists@4.0.0: {} + path-key@3.1.1: {} path-scurry@1.11.1: @@ -1507,6 +2376,8 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 + pluralize@8.0.0: {} + postcss-load-config@6.0.1(postcss@8.5.15): dependencies: lilconfig: 3.1.3 @@ -1519,8 +2390,18 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + prelude-ls@1.2.1: {} + + punycode@2.3.1: {} + + qified@0.10.1: + dependencies: + hookified: 2.2.0 + readdirp@4.1.2: {} + require-from-string@2.0.2: {} + resolve-from@5.0.0: {} rollup@4.62.2: @@ -1608,6 +2489,8 @@ snapshots: tinyglobby: 0.2.17 ts-interface-checker: 0.1.13 + supports-color@10.2.2: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -1643,6 +2526,10 @@ snapshots: tree-kill@1.2.2: {} + ts-api-utils@2.5.0(typescript@5.9.3): + dependencies: + typescript: 5.9.3 + ts-interface-checker@0.1.13: {} tsup@8.5.1(postcss@8.5.15)(typescript@5.9.3): @@ -1651,7 +2538,7 @@ snapshots: cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) esbuild: 0.27.7 fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 @@ -1673,16 +2560,39 @@ snapshots: - tsx - yaml + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + type-fest@4.41.0: {} + + typescript-eslint@8.72.2(eslint@10.14.0)(typescript@5.9.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.72.2(@typescript-eslint/parser@8.72.2(eslint@10.14.0)(typescript@5.9.3))(eslint@10.14.0)(typescript@5.9.3) + '@typescript-eslint/parser': 8.72.2(eslint@10.14.0)(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.72.2(typescript@5.9.3) + '@typescript-eslint/utils': 8.72.2(eslint@10.14.0)(typescript@5.9.3) + eslint: 10.14.0 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + typescript@5.9.3: {} ufo@1.6.4: {} undici-types@6.21.0: {} + uri-js-replace@1.0.2: {} + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + vite-node@3.2.4(@types/node@22.19.21): dependencies: cac: 6.7.14 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) es-module-lexer: 1.7.0 pathe: 2.0.3 vite: 7.3.5(@types/node@22.19.21) @@ -1723,7 +2633,7 @@ snapshots: '@vitest/spy': 3.2.6 '@vitest/utils': 3.2.6 chai: 5.3.3 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) expect-type: 1.3.0 magic-string: 0.30.21 pathe: 2.0.3 @@ -1762,6 +2672,8 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + word-wrap@1.2.5: {} + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 @@ -1773,3 +2685,9 @@ snapshots: ansi-styles: 6.2.3 string-width: 5.1.2 strip-ansi: 7.2.0 + + yaml-ast-parser@0.0.43: {} + + yargs-parser@21.1.1: {} + + yocto-queue@0.1.0: {} diff --git a/scripts/gen-types.mjs b/scripts/gen-types.mjs new file mode 100644 index 0000000..d37daa8 --- /dev/null +++ b/scripts/gen-types.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +// Regenerates src/generated/*.ts from the vendored specs under openapi/. +// Run via `pnpm gen:types`. Committed output — do not hand-edit src/generated/*. + +import { readdir, readFile, writeFile, mkdir } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import openapiTS, { astToString } from "openapi-typescript"; + +const root = fileURLToPath(new URL("..", import.meta.url)); +const openapiDir = new URL("../openapi/", import.meta.url); +const outDir = new URL("../src/generated/", import.meta.url); + +await mkdir(outDir, { recursive: true }); + +const files = (await readdir(openapiDir)).filter((f) => f.endsWith(".json")).sort(); + +for (const file of files) { + const specPath = new URL(file, openapiDir); + const spec = JSON.parse(await readFile(specPath, "utf8")); + const ast = await openapiTS(spec, { cwd: root }); + const body = astToString(ast); + const name = file.replace(/\.json$/, ""); + const banner = `/**\n * Generated from openapi/${file} by \`pnpm gen:types\`. Do not hand-edit.\n * Source: openapi-typescript v${JSON.parse(await readFile(new URL("../node_modules/openapi-typescript/package.json", import.meta.url), "utf8")).version}\n */\n`; + await writeFile(new URL(`${name}.ts`, outDir), banner + body); + console.log(`generated src/generated/${name}.ts`); +} diff --git a/src/account.ts b/src/account.ts new file mode 100644 index 0000000..d06ba73 --- /dev/null +++ b/src/account.ts @@ -0,0 +1,39 @@ +import type { DiffbotClient } from "./client.js"; +import { parseJsonText, readText } from "./decode.js"; +import { AuthError } from "./errors.js"; +import { ensureOk } from "./http.js"; +import type { AccountResponse } from "./types/account.js"; + +export type { AccountResponse, AccountUsage } from "./types/account.js"; + +export interface AccountOptions { + /** Window, in days, for the `usage` array (API default: 31). */ + days?: number; +} + +/** + * `GET /v4/account`: plan, status and per-day usage for the token. Free, and the only + * documented token probe. The response includes the token itself and any child tokens, + * so never log it. + */ +export async function account(client: DiffbotClient, options: AccountOptions = {}): Promise { + const params: Record = { token: client.token }; + if (options.days !== undefined) params.days = options.days; + const response = await client.http.get(`${client.apiBase}/v4/account`, { params }); + await ensureOk(response); + return parseJsonText(await readText(response)); +} + +/** + * `true` if the token is accepted, `false` if the API rejects it (`AuthError`). Every other + * failure (network, 5xx, rate limit, ...) propagates: it says nothing about the token. + */ +export async function verifyToken(client: DiffbotClient): Promise { + try { + await account(client, { days: 1 }); + return true; + } catch (error) { + if (error instanceof AuthError) return false; + throw error; + } +} diff --git a/src/ask.ts b/src/ask.ts index 7b385ec..32c9b66 100644 --- a/src/ask.ts +++ b/src/ask.ts @@ -1,10 +1,14 @@ import type { DiffbotClient } from "./client.js"; +import { parseJsonText, unsafeCast } from "./decode.js"; import { ValidationError } from "./errors.js"; -import { readBodyText, raiseForStatus } from "./http.js"; +import { ensureOk } from "./http.js"; import type { JsonObject } from "./json.js"; +import type { ChatCompletionChunk, ChatCompletionRequest, ChatRole } from "./types/undocumented.js"; + +export type { ChatRole } from "./types/undocumented.js"; export interface ChatMessage { - role: string; + role: ChatRole; content: string; } @@ -14,7 +18,7 @@ const MODEL = "diffbot-small-xl"; export const RESPONSE_FORMAT_TYPES = ["text", "json_object", "json_schema"] as const; export type ResponseFormatType = (typeof RESPONSE_FORMAT_TYPES)[number]; -export interface ResponseFormat { +export interface ResponseFormat extends JsonObject { type: ResponseFormatType; json_schema?: { name?: string; schema: JsonObject }; } @@ -33,6 +37,9 @@ export interface AskOptions { * without an error, so prefer this helper over hand-building the object. */ export function jsonSchemaFormat(schema: JsonObject, name = "response"): ResponseFormat { + // These checks are unreachable if the caller respects `schema: JsonObject` — they + // defend against a plain-JS caller (or an `any`/`as` escape) passing something else. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (schema === null || typeof schema !== "object" || Array.isArray(schema)) { throw new ValidationError("schema must be a JSON Schema object"); } @@ -42,17 +49,22 @@ export function jsonSchemaFormat(schema: JsonObject, name = "response"): Respons /** Reject shapes the endpoint would accept but silently not enforce. */ function validateResponseFormat(responseFormat: ResponseFormat | undefined): void { if (responseFormat === undefined) return; + // Same rationale as jsonSchemaFormat above: defends against a caller not using the + // declared `ResponseFormat` type, where these fields aren't actually guaranteed. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (responseFormat === null || typeof responseFormat !== "object" || Array.isArray(responseFormat)) { throw new ValidationError("responseFormat must be an object"); } + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition const fmtType = responseFormat.type ?? "text"; - if (!(RESPONSE_FORMAT_TYPES as readonly string[]).includes(fmtType)) { + if (!RESPONSE_FORMAT_TYPES.includes(fmtType)) { throw new ValidationError( `responseFormat type must be one of ${RESPONSE_FORMAT_TYPES.join(", ")}; got ${JSON.stringify(fmtType)}`, ); } if (fmtType === "json_schema") { const jsonSchema = responseFormat.json_schema; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (jsonSchema === null || typeof jsonSchema !== "object" || !("schema" in jsonSchema)) { throw new ValidationError( 'responseFormat { type: "json_schema" } requires the schema nested as ' + @@ -123,15 +135,14 @@ function resolveFormat( return jsonSchemaFormat(schema ?? ANY_OBJECT_SCHEMA); } +function isFunctionCallLeak(value: unknown): value is { name: string; arguments: unknown } { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + return "name" in value && value.name === "functioncall" && "arguments" in value; +} + /** Catch the internal tool call surfacing as the answer (see ANY_OBJECT_SCHEMA). */ function checkToolCallLeak(parsed: T): T { - if ( - parsed !== null && - typeof parsed === "object" && - !Array.isArray(parsed) && - (parsed as JsonObject).name === "functioncall" && - "arguments" in (parsed as JsonObject) - ) { + if (isFunctionCallLeak(parsed)) { throw new ValidationError( "the model returned its internal tool call instead of an answer; this happens with " + 'responseFormat { type: "json_object" } because the server applies no grammar to it. ' + @@ -143,11 +154,8 @@ function checkToolCallLeak(parsed: T): T { function parseChunk(line: string): string | undefined { try { - const chunk = JSON.parse(line.replace(/^data: /, "")) as { - choices?: Array<{ delta?: { content?: string } }>; - }; - const content = chunk.choices?.[0]?.delta?.content; - return content ?? undefined; + const chunk = parseJsonText(line.replace(/^data: /, "")); + return chunk.choices?.[0]?.delta?.content ?? undefined; } catch { return undefined; } @@ -164,7 +172,7 @@ async function* linesFromStream(body: ReadableStream): AsyncGenerato const decoder = new TextDecoder(); let buffer = ""; - while (true) { + for (;;) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); @@ -184,15 +192,11 @@ export async function* ask( ): AsyncGenerator { validateResponseFormat(options.responseFormat); const headers = { Authorization: `Bearer ${client.token}` }; - const payload: JsonObject = { model: MODEL, messages, stream: true }; + const payload: ChatCompletionRequest = { model: MODEL, messages, stream: true }; if (options.responseFormat !== undefined) payload.response_format = options.responseFormat; const response = await client.http.post(client.llmUrl, { headers, json: payload }); - if (!response.ok) { - const body = await readBodyText(response); - raiseForStatus(response, body); - return; - } + await ensureOk(response); if (response.body) { for await (const line of linesFromStream(response.body)) { @@ -209,6 +213,16 @@ export async function* ask( } } +export interface AskJsonOptions extends AskOptions { + /** + * Checks (and narrows to) `T` instead of trusting an unchecked cast — e.g. a + * hand-written type guard, or `(v) => mySchema.parse(v)` from a validation + * library of your choice. Without it, the result is cast to `T` unchecked, + * as before. + */ + parse?: (value: unknown) => T; +} + /** * Ask the Diffbot LLM and return the answer parsed as JSON. With `schema` (a JSON Schema * object) the model is constrained to it by grammar; without one it falls back to @@ -218,10 +232,11 @@ export async function askJson( client: DiffbotClient, messages: ChatMessage[], schema?: JsonObject, - options: AskOptions = {}, + options: AskJsonOptions = {}, ): Promise { const responseFormat = resolveFormat(schema, options.responseFormat); let text = ""; for await (const chunk of ask(client, messages, { responseFormat })) text += chunk; - return checkToolCallLeak(extractJson(text)) as T; + const parsed = checkToolCallLeak(extractJson(text)); + return options.parse ? options.parse(parsed) : unsafeCast(parsed); } diff --git a/src/auth.ts b/src/auth.ts index c50b238..a5c42a0 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -1,4 +1,5 @@ import { TOKEN_ENV_VAR } from "./constants.js"; +import { unsafeCast } from "./decode.js"; /* * `object`, not `Record`: a typed Workers `Env` is normally @@ -11,7 +12,7 @@ import { TOKEN_ENV_VAR } from "./constants.js"; export type TokenSource = object; function envToken(env: TokenSource | undefined): string { - const fromEnv = (env as Record | undefined)?.[TOKEN_ENV_VAR]; + const fromEnv = unsafeCast | undefined>(env)?.[TOKEN_ENV_VAR]; if (typeof fromEnv === "string" && fromEnv.trim()) { return fromEnv.trim(); } @@ -21,7 +22,7 @@ function envToken(env: TokenSource | undefined): string { * in Workers/browsers, and touching it unconditionally would reintroduce a * runtime-specific dependency into an otherwise platform-neutral module. */ - const proc = (globalThis as { process?: { env?: Record } }).process; + const proc = unsafeCast<{ process?: { env?: Record } }>(globalThis).process; const fromProcess = proc?.env?.[TOKEN_ENV_VAR]; return fromProcess?.trim() ?? ""; } diff --git a/src/client.ts b/src/client.ts index d212e52..4775f7b 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,32 +1,50 @@ import { - CRAWL_BASE, + API_BASE, DEFAULT_TIMEOUT, - DIFFBOT_LLM_BASE, - EXTRACT_BASE, - KG_DQL_ENDPOINT, - KG_ONTOLOGY_ENDPOINT, - NLP_BASE, + KG_BASE, + LLM_BASE, + NLP_HOST_BASE, VERSION, - WEB_SEARCH_BASE, } from "./constants.js"; import { ValidationError } from "./errors.js"; -import { HttpClient, type FetchFn } from "./http.js"; +import { HttpClient, type FetchFn, type RetryPolicy } from "./http.js"; export interface DiffbotClientOptions { token: string; timeout?: number; + /** `false` disables retries entirely; an object overrides individual defaults. */ + retry?: RetryPolicy | false; + /** Base for `api.diffbot.com` endpoints (Extract, Crawl, Bulk, Custom API, Account). */ + apiBase?: string; + /** Base for `kg.diffbot.com` endpoints (DQL, DQL reports, Enhance, ontology). */ + kgBase?: string; + /** Base for `llm.diffbot.com` endpoints (Ask, Web Search). */ + llmBase?: string; + /** Base for `nl.diffbot.com` (Natural Language). */ + nlpBase?: string; + /** @deprecated prefer `apiBase`; wins over it when set. */ analyzeUrl?: string; + /** @deprecated prefer `llmBase`; wins over it when set. */ llmUrl?: string; + /** @deprecated prefer `apiBase`; wins over it when set. */ crawlerUrl?: string; + /** @deprecated prefer `llmBase`; wins over it when set. */ webSearchUrl?: string; + /** @deprecated prefer `nlpBase`; wins over it when set. */ nlpUrl?: string; + /** @deprecated prefer `kgBase`; wins over it when set. */ dqlUrl?: string; + /** @deprecated prefer `kgBase`; wins over it when set. */ ontologyUrl?: string; fetch?: FetchFn; } export class DiffbotClient { readonly token: string; + readonly apiBase: string; + readonly kgBase: string; + readonly llmBase: string; + readonly nlpBase: string; readonly analyzeUrl: string; readonly llmUrl: string; readonly crawlerUrl: string; @@ -41,17 +59,29 @@ export class DiffbotClient { throw new ValidationError("token is required"); } this.token = options.token; - this.analyzeUrl = options.analyzeUrl ?? EXTRACT_BASE; - this.llmUrl = options.llmUrl ?? DIFFBOT_LLM_BASE; - this.crawlerUrl = options.crawlerUrl ?? CRAWL_BASE; - this.webSearchUrl = options.webSearchUrl ?? WEB_SEARCH_BASE; - this.nlpUrl = options.nlpUrl ?? NLP_BASE; - this.dqlUrl = options.dqlUrl ?? KG_DQL_ENDPOINT; - this.ontologyUrl = options.ontologyUrl ?? KG_ONTOLOGY_ENDPOINT; + + this.apiBase = options.apiBase ?? API_BASE; + this.kgBase = options.kgBase ?? KG_BASE; + this.llmBase = options.llmBase ?? LLM_BASE; + this.nlpBase = options.nlpBase ?? NLP_HOST_BASE; + + // This constructor is the one sanctioned reader of these deprecated aliases: it's + // what makes them "win over" the matching base when set. + /* eslint-disable @typescript-eslint/no-deprecated */ + this.analyzeUrl = options.analyzeUrl ?? `${this.apiBase}/v3`; + this.crawlerUrl = options.crawlerUrl ?? `${this.apiBase}/v3/crawl`; + this.llmUrl = options.llmUrl ?? `${this.llmBase}/rag/v1/chat/completions`; + this.webSearchUrl = options.webSearchUrl ?? `${this.llmBase}/api/v1/web_search`; + this.nlpUrl = options.nlpUrl ?? `${this.nlpBase}/v1/`; + this.dqlUrl = options.dqlUrl ?? `${this.kgBase}/kg/v3/dql`; + this.ontologyUrl = options.ontologyUrl ?? `${this.kgBase}/kg/ontology`; + /* eslint-enable @typescript-eslint/no-deprecated */ + this.http = new HttpClient( options.fetch ?? fetch, options.timeout ?? DEFAULT_TIMEOUT, `diffbot-typescript/${VERSION}`, + options.retry ?? {}, ); } diff --git a/src/constants.ts b/src/constants.ts index 8d2a108..e46c88e 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,13 +1,20 @@ -export const VERSION = "0.3.0"; +export const VERSION = "0.6.0"; export const DEFAULT_TIMEOUT = 30_000; -export const EXTRACT_BASE = "https://api.diffbot.com/v3"; -export const CRAWL_BASE = "https://api.diffbot.com/v3/crawl"; -export const DIFFBOT_LLM_BASE = "https://llm.diffbot.com/rag/v1/chat/completions"; -export const WEB_SEARCH_BASE = "https://llm.diffbot.com/api/v1/web_search"; -export const NLP_BASE = "https://nl.diffbot.com/v1/"; -export const KG_DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql"; -export const KG_ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology"; +/** Per-host base URLs. Overridable via `apiBase`/`kgBase`/`llmBase`/`nlpBase`; the + * older per-endpoint `*Url` options still win over these when set. */ +export const API_BASE = "https://api.diffbot.com"; +export const KG_BASE = "https://kg.diffbot.com"; +export const LLM_BASE = "https://llm.diffbot.com"; +export const NLP_HOST_BASE = "https://nl.diffbot.com"; + +export const EXTRACT_BASE = `${API_BASE}/v3`; +export const CRAWL_BASE = `${API_BASE}/v3/crawl`; +export const DIFFBOT_LLM_BASE = `${LLM_BASE}/rag/v1/chat/completions`; +export const WEB_SEARCH_BASE = `${LLM_BASE}/api/v1/web_search`; +export const NLP_BASE = `${NLP_HOST_BASE}/v1/`; +export const KG_DQL_ENDPOINT = `${KG_BASE}/kg/v3/dql`; +export const KG_ONTOLOGY_ENDPOINT = `${KG_BASE}/kg/ontology`; export const TOKEN_ENV_VAR = "DIFFBOT_API_TOKEN"; diff --git a/src/crawl.ts b/src/crawl.ts index ad427c6..20a2247 100644 --- a/src/crawl.ts +++ b/src/crawl.ts @@ -1,7 +1,13 @@ import type { DiffbotClient } from "./client.js"; -import { APIError } from "./errors.js"; -import { readBodyText, raiseForStatus } from "./http.js"; +import { headersToRecord, parseJsonText, readText } from "./decode.js"; +import { APIError, JobError } from "./errors.js"; +import { ensureOk } from "./http.js"; import type { JsonObject } from "./json.js"; +import type { CrawlDataRecord, CrawlJob } from "./types/undocumented.js"; + +interface CrawlListResponse extends JsonObject { + jobs?: CrawlJob[]; +} export enum CrawlEventType { JobCreated = "job_created", @@ -26,7 +32,20 @@ export interface CrawlOptions { urlProcessPattern?: string; obeyRobots?: boolean; useProxies?: boolean; - customHeaders?: string; + /** One `Name:value` header, or several; each is sent as its own repeated `customHeaders` param. */ + customHeaders?: string | readonly string[]; + /** Regex counterpart of `urlCrawlPattern`. */ + urlCrawlRegEx?: string; + /** Regex counterpart of `urlProcessPattern`. */ + urlProcessRegEx?: string; + /** Only pages whose HTML contains this string are processed. */ + pageProcessPattern?: string; + maxToCrawlPerSubdomain?: number; + maxToProcessPerSubdomain?: number; + /** `false` sends `useCanonical=0` (disables canonical-link deduplication). Server default is on. */ + useCanonical?: boolean; + notifyWebhook?: string; + notifyEmail?: string; watch?: boolean; pollInterval?: number; } @@ -35,10 +54,15 @@ function buildCrawlParams( client: DiffbotClient, jobName: string, site: string, - options: Required> & - Pick, + options: CrawlOptions & + Required< + Pick< + CrawlOptions, + "maxToCrawl" | "maxToProcess" | "hops" | "restrictDomain" | "apiUrl" | "crawlDelay" | "obeyRobots" | "useProxies" + > + >, ) { - const params: Record = { + const params: Record = { token: client.token, name: jobName, seeds: site, @@ -53,23 +77,71 @@ function buildCrawlParams( if (options.urlProcessPattern) params.urlProcessPattern = options.urlProcessPattern; if (options.obeyRobots) params.obeyRobots = 1; if (options.useProxies) params.useProxies = 1; - if (options.customHeaders) params.customHeaders = options.customHeaders; + if (options.urlCrawlRegEx) params.urlCrawlRegEx = options.urlCrawlRegEx; + if (options.urlProcessRegEx) params.urlProcessRegEx = options.urlProcessRegEx; + if (options.pageProcessPattern) params.pageProcessPattern = options.pageProcessPattern; + if (options.maxToCrawlPerSubdomain !== undefined) params.maxToCrawlPerSubdomain = options.maxToCrawlPerSubdomain; + if (options.maxToProcessPerSubdomain !== undefined) { + params.maxToProcessPerSubdomain = options.maxToProcessPerSubdomain; + } + if (options.useCanonical !== undefined) params.useCanonical = options.useCanonical ? 1 : 0; + if (options.notifyWebhook) params.notifyWebhook = options.notifyWebhook; + if (options.notifyEmail) params.notifyEmail = options.notifyEmail; + if (typeof options.customHeaders === "string") { + if (options.customHeaders) params.customHeaders = options.customHeaders; + } else if (options.customHeaders?.length) { + params.customHeaders = options.customHeaders; + } return params; } +/** Splits one CSV line into fields, honoring double-quoted fields with embedded + * commas and `""`-escaped quotes. Does not span quoted newlines across lines. */ +function parseCsvLine(line: string): string[] { + const fields: string[] = []; + let field = ""; + let inQuotes = false; + for (let i = 0; i < line.length; i++) { + const char = line[i]; + if (char === undefined) break; + if (inQuotes) { + if (char === '"') { + if (line[i + 1] === '"') { + field += '"'; + i++; + } else { + inQuotes = false; + } + } else { + field += char; + } + } else if (char === '"') { + inQuotes = true; + } else if (char === ",") { + fields.push(field); + field = ""; + } else { + field += char; + } + } + fields.push(field); + return fields; +} + function parseUrlCsv(text: string, seenUrls: Set): CrawlEvent[] { const events: CrawlEvent[] = []; - const lines = text.trim().split("\n"); - if (lines.length < 2) return events; + const [headerLine, ...dataLines] = text.trim().split("\n"); + if (headerLine === undefined || dataLines.length === 0) return events; - const headers = lines[0]!.split(",").map((h) => h.trim()); + const headers = parseCsvLine(headerLine).map((h) => h.trim()); const urlIdx = headers.indexOf("Url"); const timeIdx = headers.indexOf("Crawled Time"); const statusIdx = headers.indexOf("Crawl Status"); - for (const line of lines.slice(1)) { - const cols = line.split(","); - const url = cols[urlIdx]?.replace(/^"|"$/g, "").trim() ?? ""; + for (const line of dataLines) { + if (!line) continue; + const cols = parseCsvLine(line); + const url = cols[urlIdx]?.trim() ?? ""; if (url && !seenUrls.has(url)) { seenUrls.add(url); events.push({ @@ -85,6 +157,15 @@ function parseUrlCsv(text: string, seenUrls: Set): CrawlEvent[] { return events; } +/** `errorCode` from a crawl-create body, or `undefined` for success or a non-JSON body. */ +function createErrorCode(body: string): unknown { + try { + return parseJsonText(body).errorCode; + } catch { + return undefined; + } +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -94,26 +175,30 @@ export async function* crawl( site: string, options: CrawlOptions = {}, ): AsyncGenerator { - let normalizedSite = site.startsWith("http") ? site : `https://${site}`; - const jobName = options.jobName ?? `crawl-${Math.floor(Date.now() / 1000)}`; + const normalizedSite = site.startsWith("http") ? site : `https://${site}`; + const jobName = options.jobName ?? `crawl-${Math.floor(Date.now() / 1000).toString()}`; const params = buildCrawlParams(client, jobName, normalizedSite, { + ...options, hops: options.hops ?? 2, maxToCrawl: options.maxToCrawl ?? 100, maxToProcess: options.maxToProcess ?? 100, restrictDomain: options.restrictDomain ?? true, apiUrl: options.apiUrl ?? "", crawlDelay: options.crawlDelay ?? -1, - urlCrawlPattern: options.urlCrawlPattern, - urlProcessPattern: options.urlProcessPattern, obeyRobots: options.obeyRobots ?? false, useProxies: options.useProxies ?? false, - customHeaders: options.customHeaders, }); const response = await client.http.get(client.crawlerUrl, { params }); - const body = await readBodyText(response); - raiseForStatus(response, body); + await ensureOk(response); + // Some create failures (e.g. a job name over 30 characters, errorCode 505) arrive as + // HTTP 200 with `{ errorCode, error }` and create nothing; surface them here instead of + // reporting JobCreated and then watching a job that does not exist. + const createBody = await readText(response); + if (createErrorCode(createBody) !== undefined) { + throw new APIError(response.status, createBody, headersToRecord(response.headers)); + } yield { eventType: CrawlEventType.JobCreated, @@ -128,24 +213,19 @@ export async function* crawl( const seenUrls = new Set(); const pollInterval = options.pollInterval ?? 2000; - while (true) { + for (;;) { const statusResponse = await client.http.get(client.crawlerUrl, { params: { token: client.token, name: jobName }, }); - const statusBody = await readBodyText(statusResponse); - raiseForStatus(statusResponse, statusBody); - const jobs = (JSON.parse(statusBody) as JsonObject).jobs as JsonObject[] | undefined; - if (!jobs?.length) break; + await ensureOk(statusResponse); + const jobs = parseJsonText(await readText(statusResponse)).jobs; + const job = jobs?.[0]; + if (!job) break; - const jobStatus = (jobs[0]!.jobStatus ?? {}) as JsonObject; - const statusCode = Number(jobStatus.status ?? 0); + const jobStatus = job.jobStatus ?? {}; + const statusCode = jobStatus.status ?? 0; - const urlsResponse = await client.http.get(`${client.crawlerUrl}/data`, { - params: { token: client.token, name: jobName, type: "urls" }, - redirect: "follow", - }); - const urlsBody = await readBodyText(urlsResponse); - raiseForStatus(urlsResponse, urlsBody); + const urlsBody = await crawlDownload(client, jobName, { type: "urls" }); if (urlsBody) { for (const event of parseUrlCsv(urlsBody, seenUrls)) { yield event; @@ -153,9 +233,9 @@ export async function* crawl( } if (statusCode !== 0 && statusCode !== 7) { - const message = String(jobStatus.message ?? ""); + const message = jobStatus.message ?? ""; if (/fail|error/i.test(message)) { - throw new APIError(500, `Crawler job failed: ${message}`); + throw new JobError(jobName, `Crawler job failed: ${message}`); } break; } @@ -164,29 +244,77 @@ export async function* crawl( } } -export async function crawlListJobs(client: DiffbotClient): Promise { +export async function crawlListJobs(client: DiffbotClient): Promise { const response = await client.http.get(client.crawlerUrl, { params: { token: client.token }, }); - const body = await readBodyText(response); - raiseForStatus(response, body); - return ((JSON.parse(body) as JsonObject).jobs as JsonObject[] | undefined) ?? []; + await ensureOk(response); + return parseJsonText(await readText(response)).jobs ?? []; +} + +export interface CrawlDownloadOptions { + /** `"csv"` returns the raw CSV text (top-level fields only). Default `"json"`. */ + format?: "json" | "csv"; + /** e.g. `"urls"` for the URL report. That report is always CSV text, whatever `format` says. */ + type?: string; + /** Only the N most recently processed URLs. */ + num?: number; +} + +/** CSV text when `format: "csv"` or `type: "urls"`; otherwise parsed records. */ +export type CrawlDownloadResult = O extends { format: "csv" } + ? string + : O extends { type: "urls" } + ? string + : CrawlDataRecord[]; + +/** + * `GET /v3/crawl/data`: download a crawl job's extracted data. For tools that cannot sit in + * a watch loop, the recommended pattern is `crawl(..., { watch: false })`, poll + * `crawlGetJob` until its `jobStatus` is terminal, then `crawlDownload`. + */ +export async function crawlDownload>( + client: DiffbotClient, + jobName: string, + options?: O, +): Promise>; +export async function crawlDownload( + client: DiffbotClient, + jobName: string, + options: CrawlDownloadOptions = {}, +): Promise { + const params: Record = { token: client.token, name: jobName }; + if (options.format === "csv") params.format = "csv"; + if (options.type !== undefined) params.type = options.type; + if (options.num !== undefined) params.num = options.num; + const response = await client.http.get(`${client.crawlerUrl}/data`, { params, redirect: "follow" }); + await ensureOk(response); + const body = await readText(response); + if (options.format === "csv" || options.type === "urls") return body; + if (!body.trim()) return []; + const data = parseJsonText(body); + if (!Array.isArray(data)) { + throw new JobError(jobName, `Crawl data download failed: ${data.error ?? "unexpected response"}`); + } + return data; } -export async function crawlGetJob(client: DiffbotClient, jobName: string): Promise { +/** `undefined` when no job with this name exists (previously an indistinguishable `{}`). */ +export async function crawlGetJob(client: DiffbotClient, jobName: string): Promise { const response = await client.http.get(client.crawlerUrl, { params: { token: client.token, name: jobName }, }); - const body = await readBodyText(response); - raiseForStatus(response, body); - const jobs = (JSON.parse(body) as JsonObject).jobs as JsonObject[] | undefined; - return jobs?.[0] ?? {}; + await ensureOk(response); + const jobs = parseJsonText(await readText(response)).jobs; + return jobs?.[0]; } export async function crawlDeleteJob(client: DiffbotClient, jobName: string): Promise { + // A GET-shaped mutation: retrying it could issue a second delete against a job that + // no longer exists, so it must not be retried like an ordinary idempotent GET. const response = await client.http.get(client.crawlerUrl, { params: { token: client.token, name: jobName, delete: 1 }, + retryable: false, }); - const body = await readBodyText(response); - raiseForStatus(response, body); + await ensureOk(response); } diff --git a/src/decode.ts b/src/decode.ts new file mode 100644 index 0000000..2c0c66b --- /dev/null +++ b/src/decode.ts @@ -0,0 +1,94 @@ +/** + * The one place in the core package allowed to use `as` / unsafe JSON parsing. + * Every other module reads response bodies through these functions instead of + * calling `JSON.parse` or casting directly, so a lint rule can enforce that split. + */ +import type { JsonObject } from "./json.js"; + +export function headersToRecord(headers: Headers): Record { + const record: Record = {}; + headers.forEach((value, key) => { + record[key] = value; + }); + return record; +} + +/** + * Best-effort extraction of `{message, requestId}` from an API error body. Never throws. + * Falls back to `error` for the `{ errorCode, error }` shape several Diffbot APIs use. + */ +export function tryParseErrorBody(body: string): { message?: string; requestId?: string } { + try { + const data = JSON.parse(body) as { message?: unknown; error?: unknown; requestId?: unknown }; + return { + message: + typeof data.message === "string" ? data.message : typeof data.error === "string" ? data.error : undefined, + requestId: typeof data.requestId === "string" ? data.requestId : undefined, + }; + } catch { + return {}; + } +} + +export async function readText(response: Response): Promise { + return response.text(); +} + +export async function readBytes(response: Response): Promise { + const buffer = await response.arrayBuffer(); + return new Uint8Array(buffer); +} + +/* eslint-disable @typescript-eslint/no-unnecessary-type-parameters -- + T appears only in the return position by design: these are type-assertion + helpers, not narrowing generics, so a caller supplying T is the whole point. */ + +/** Parse already-read text as JSON. The sole sanctioned `as` for response payloads. */ +export function parseJsonText(text: string): T { + return JSON.parse(text) as T; +} + +export async function readJson(response: Response): Promise { + return parseJsonText(await response.text()); +} + +/** + * The one sanctioned `as` for the whole package (lint-enforced): every other module + * that needs to assert a type it cannot prove — a runtime environment's shape, an + * unchecked `askJson()` fallback — calls this instead of writing `as` directly, so + * every unprovable assertion in the codebase is findable from one place. + */ +export function unsafeCast(value: unknown): T { + return value as T; +} +/* eslint-enable @typescript-eslint/no-unnecessary-type-parameters */ + +/** + * Line-delimited JSON (`format=jsonl`), streamed when the runtime exposes a readable + * body and falling back to a single read + split otherwise (e.g. in test mocks). + */ +export async function* readNdjson(response: Response): AsyncGenerator { + if (!response.body) { + const text = await response.text(); + for (const line of text.split("\n")) { + if (line.trim()) yield parseJsonText(line); + } + return; + } + + const reader: ReadableStreamDefaultReader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const parts = buffer.split("\n"); + buffer = parts.pop() ?? ""; + for (const line of parts) { + if (line.trim()) yield parseJsonText(line); + } + } + if (buffer.trim()) yield parseJsonText(buffer); +} diff --git a/src/errors.ts b/src/errors.ts index 424ad01..9adc5ed 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,5 +1,7 @@ /** Exception hierarchy for the Diffbot SDK. */ +import { tryParseErrorBody } from "./decode.js"; + export class DiffbotError extends Error { override name = "DiffbotError"; } @@ -13,26 +15,19 @@ export class APIError extends DiffbotError { body: string; apiMessage: string | undefined; requestId: string | undefined; + headers: Record | undefined; - constructor(statusCode: number, body: string) { - let apiMessage: string | undefined; - let requestId: string | undefined; - try { - const data = JSON.parse(body) as { message?: string; requestId?: string }; - apiMessage = data.message; - requestId = data.requestId; - } catch { - /* ignore parse errors */ - } + constructor(statusCode: number, body: string, headers?: Record) { + const { message: apiMessage, requestId } = tryParseErrorBody(body); const display = apiMessage ?? (body.length > 200 ? `${body.slice(0, 200)}...` : body); - super(`Diffbot API error ${statusCode}: ${display}`); + super(`Diffbot API error ${statusCode.toString()}: ${display}`); this.name = "APIError"; this.statusCode = statusCode; this.body = body; this.apiMessage = apiMessage; this.requestId = requestId; + this.headers = headers; } - } export class AuthError extends APIError { @@ -43,19 +38,44 @@ export class RateLimitError extends APIError { retryAfter: string | undefined; override name = "RateLimitError"; - constructor(statusCode: number, body: string, retryAfter?: string | null) { - super(statusCode, body); + constructor( + statusCode: number, + body: string, + retryAfter?: string | null, + headers?: Record, + ) { + super(statusCode, body, headers); this.retryAfter = retryAfter ?? undefined; } } +/** A 404 from an endpoint with a documented "not found" shape (e.g. a missing job or report). */ +export class NotFoundError extends APIError { + override name = "NotFoundError"; +} + +/** + * A job (crawl or bulk) that completed its HTTP request successfully but reports a + * failed status internally. Subclasses `APIError` so existing `catch (APIError)` sites + * keep working; `instanceof JobError` distinguishes it from a transport-level failure. + */ +export class JobError extends APIError { + override name = "JobError"; + jobName: string; + + constructor(jobName: string, message: string) { + super(500, message); + this.jobName = jobName; + } +} + export class ExtractionError extends DiffbotError { errorCode: number; error: string; override name = "ExtractionError"; constructor(errorCode: number, error: string) { - super(`Diffbot extraction error ${errorCode}: ${error}`); + super(`Diffbot extraction error ${errorCode.toString()}: ${error}`); this.errorCode = errorCode; this.error = error; } diff --git a/src/extract.ts b/src/extract.ts index 9114958..c1b7a6d 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -1,26 +1,124 @@ import type { DiffbotClient } from "./client.js"; +import { parseJsonText, readText, unsafeCast } from "./decode.js"; import { ExtractionError } from "./errors.js"; -import { raiseForStatus, readBodyText } from "./http.js"; -import type { JsonObject } from "./json.js"; +import type { operations } from "./generated/extract.js"; +import { ensureOk } from "./http.js"; +import type { + AnalyzeObject, + ArticleObject, + DiscussionObject, + EventObject, + ExtractResponse, + ImageObject, + JobObject, + ListObject, + ProductObject, + VideoObject, +} from "./types/undocumented.js"; +import type { ExtraParams } from "./types/extra.js"; -function buildParams(client: DiffbotClient, url: string, fmt: string) { +/** Default wire `timeout` (ms) when the caller does not set one. */ +const DEFAULT_EXTRACT_TIMEOUT = 30_000; +/** Headroom the transport timeout keeps over the wire `timeout`, so the server answers first. */ +const TRANSPORT_TIMEOUT_HEADROOM = 10_000; + +type Query = operations[K]["parameters"]["query"]; +type FieldOf = Query extends { fields?: infer F } ? NonNullable : never; +type ApiName = NonNullable["mode"]>; +type NaturalLanguageField = NonNullable["naturalLanguage"]>; + +/** Options every Extract API accepts. Names are the wire names. */ +interface CommonOptions { + /** Server-side extraction timeout in ms (wire `timeout`). Default 30000. The transport timeout is raised to at least this + 10s. */ + timeout?: number; + /** Proxy selection (wire `useProxy`). `proxy`/`proxyAuth` go through `extra` only. */ + useProxy?: string; +} + +type WithExtra = O & { + /** Wire query parameters not exposed as named options (e.g. `proxy`, `proxyAuth`). Named keys are rejected. */ + extra?: ExtraParams<"token" | "url" | (keyof O & string)>; +}; + +export type AnalyzeExtractOptions = WithExtra< + CommonOptions & { + /** Forces a specific API (or `"llm"`, which `fmt: "markdown"` already selects). Overrides `fmt`. */ + mode?: ApiName | "llm"; + fallback?: ApiName; + fields?: readonly FieldOf<"extract-analyze">[]; + discussion?: boolean; + } +>; +export type ArticleExtractOptions = WithExtra< + CommonOptions & { + fields?: readonly FieldOf<"article">[]; + paging?: boolean; + maxTags?: number; + tagConfidence?: number; + categoryConfidence?: number; + discussion?: boolean; + naturalLanguage?: NaturalLanguageField | readonly NaturalLanguageField[]; + summaryNumSentences?: number; + } +>; +export type ProductExtractOptions = WithExtra< + CommonOptions & { fields?: readonly FieldOf<"product">[]; discussion?: boolean } +>; +export type DiscussionExtractOptions = WithExtra< + CommonOptions & { fields?: readonly FieldOf<"discussion">[]; paging?: boolean; maxPages?: number | string } +>; +export type JobExtractOptions = WithExtra[] }>; +export type ImageExtractOptions = WithExtra[] }>; +export type VideoExtractOptions = WithExtra[] }>; +export type ListExtractOptions = WithExtra[] }>; +export type EventExtractOptions = WithExtra[] }>; +/** Custom APIs: `fields` is open, since a Custom API can define its own. */ +export type CustomExtractOptions = WithExtra; + +export type ExtractOptions = + | AnalyzeExtractOptions + | ArticleExtractOptions + | ProductExtractOptions + | DiscussionExtractOptions + | JobExtractOptions + | ImageExtractOptions + | VideoExtractOptions + | ListExtractOptions + | EventExtractOptions + | CustomExtractOptions; + +type BuiltInApi = "analyze" | "article" | "product" | "discussion" | "job" | "image" | "video" | "list" | "event"; +/** Excludes the built-in names, so a wrong option for `"product"` is an error instead of falling through to the open Custom API overload. */ +type CustomApiName = A extends BuiltInApi ? never : A; + +type WireValue = string | number | boolean | readonly string[] | undefined; + +function buildParams( + client: DiffbotClient, + url: string, + fmt: string, + options: object, +): { params: Record; wireTimeout: number } { const params: Record = { token: client.token, url, - timeout: 30_000, + timeout: DEFAULT_EXTRACT_TIMEOUT, }; if (fmt === "markdown") { params.mode = "llm"; } - return params; + const { extra, ...named } = unsafeCast<{ extra?: Record } & Record>(options); + for (const [key, value] of Object.entries(named)) { + if (value === undefined) continue; + params[key] = Array.isArray(value) ? value.join(",") : typeof value === "boolean" ? String(value) : unsafeCast(value); + } + if (extra) Object.assign(params, extra); + return { params, wireTimeout: Number(params.timeout) }; } -function parseResponse(data: JsonObject): JsonObject { +function parseResponse(data: T): T { if ("errorCode" in data) { - throw new ExtractionError( - Number(data.errorCode), - String(data.error ?? ""), - ); + throw new ExtractionError(Number(data.errorCode), data.error ?? ""); } return data; } @@ -29,17 +127,91 @@ function normalizeUrl(url: string): string { return url.startsWith("http") ? url : `https://${url}`; } +export async function extract( + client: DiffbotClient, + url: string, + api?: "analyze", + fmt?: string, + options?: AnalyzeExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "article", + fmt?: string, + options?: ArticleExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "product", + fmt?: string, + options?: ProductExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "discussion", + fmt?: string, + options?: DiscussionExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "job", + fmt?: string, + options?: JobExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "image", + fmt?: string, + options?: ImageExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "video", + fmt?: string, + options?: VideoExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "list", + fmt?: string, + options?: ListExtractOptions, +): Promise>; +export async function extract( + client: DiffbotClient, + url: string, + api: "event", + fmt?: string, + options?: EventExtractOptions, +): Promise>; +/** A Custom API name (anything but the built-in `api` values above) returns an open object shape. */ +export async function extract( + client: DiffbotClient, + url: string, + api: CustomApiName, + fmt?: string, + options?: CustomExtractOptions, +): Promise; export async function extract( client: DiffbotClient, url: string, api = "analyze", fmt = "markdown", -): Promise { + options: ExtractOptions = {}, +): Promise { const normalized = normalizeUrl(url); + const { params, wireTimeout } = buildParams(client, normalized, fmt, options); const response = await client.http.get(`${client.analyzeUrl}/${api}`, { - params: buildParams(client, normalized, fmt), + params, + // Never let the client abort before the server's own `timeout` can answer. + timeout: Math.max(client.http.defaultTimeout, wireTimeout + TRANSPORT_TIMEOUT_HEADROOM), }); - const body = await readBodyText(response); - raiseForStatus(response, body); - return parseResponse(JSON.parse(body) as JsonObject); + await ensureOk(response); + return parseResponse(parseJsonText(await readText(response))); } diff --git a/src/generated/account.ts b/src/generated/account.ts new file mode 100644 index 0000000..f65785e --- /dev/null +++ b/src/generated/account.ts @@ -0,0 +1,163 @@ +/** + * Generated from openapi/account.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/account": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Account Details + * @description Returns account plan, usage, child tokens, and other account details. + */ + get: operations["account"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: never; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + account: { + parameters: { + query?: { + /** @description Pass the number of days (`&days=365`) for which you would like to retrieve API call volumes (default = 31). */ + days?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "created": "2020-04-22", + * "usage": [ + * { + * "date": "2022-05-04", + * "subtitles": 0, + * "nlp": 0, + * "entities": 10, + * "credits": 258, + * "refresh": 0, + * "proxies": 0, + * "videos": 0, + * "extractions": 8, + * "facets": 0 + * }, + * { + * "date": "2022-05-03", + * "subtitles": 0, + * "nlp": 0, + * "entities": 75, + * "credits": 1875, + * "refresh": 0, + * "proxies": 0, + * "videos": 0, + * "extractions": 0, + * "facets": 0 + * } + * ], + * "name": "Diffy", + * "childTokens": [ + * "xxxx", + * "xxxx" + * ], + * "plan": "diffy-plans-to-hangout-all-day", + * "email": "sales@diffbot.com", + * "token": "xxxx", + * "status": "active" + * } + */ + "application/json": { + /** @description Name associated with the token. */ + name?: string; + /** @description Email address associated with the token. */ + email?: string; + /** @description Current plan for the token. */ + plan?: string; + /** @description Amount of monthly calls included. */ + planCredits?: number; + /** @description Status of the token. */ + status?: string; + /** Format: date */ + created?: string; + token?: string; + /** @description List of child or sub-tokens, if there are any associated with your account. */ + childTokens?: string[]; + /** @description An array of days and call volume amounts for each day. By default this will return data from the most recent 31 days. Use the `days` argument to adjust the response window. Dates prior to token becoming active will not be returned. */ + usage?: { + /** + * Format: date + * @description Date, e.g. `2019-12-01`. + */ + date?: string; + /** @description Total number of Natural Language API calls made. */ + nlp?: number; + /** @description Total number of credits consumed. */ + credits?: number; + /** @description Total number of Extraction API calls made. */ + extractions?: number; + /** @description Total number of calls made using proxy servers. [Read more](/docs/extract/using-proxies). */ + proxies?: number; + /** @description Total number of entities downloaded from the Diffbot Knowledge Graph. */ + entities?: number; + /** @description Total number of entity refreshes requested. */ + refresh?: number; + /** @description Total number of facet values returned. */ + facets?: number; + /** @description Total number of videos downloaded. */ + videos?: number; + /** @description Total number of subtitles requested. */ + subtitles?: number; + }[]; + }; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 401, + * "error": "Not authorized API token." + * } + */ + "application/json": { + /** @example 401 */ + errorCode?: number; + /** @example Not authorized API token. */ + error?: string; + }; + }; + }; + }; + }; +} diff --git a/src/generated/bulk.ts b/src/generated/bulk.ts new file mode 100644 index 0000000..4af5f0a --- /dev/null +++ b/src/generated/bulk.ts @@ -0,0 +1,438 @@ +/** + * Generated from openapi/bulk.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/bulk": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Manage a Bulk Extract Job + * @description Pause, delete, restart, or view the status of a bulk job. + */ + get: operations["pause-delete-or-restart-a-bulk-job"]; + put?: never; + /** + * Create a Bulk Extract Job + * @description Extract a list of URLs asynchronously + */ + post: operations["create-a-bulk-job"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/bulk/data": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Retrieve Bulk Extract Job Data + * @description Download the extracted results of a bulk job + */ + get: operations["retrieve-bulk-job-data"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: never; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + "pause-delete-or-restart-a-bulk-job": { + parameters: { + query?: { + /** @description Job name as defined when the bulk job was created. */ + name?: string; + /** @description Pass `pause=1` to pause a bulk job. Pass `pause=0` to resume a paused job. */ + pause?: string; + /** @description Pass `restart=1` to restart a bulk job. This will erase all processed data and re-process all of the submitted URLs. */ + restart?: string; + /** @description Pass `delete=1 `to delete a job, and all associated data, completely. */ + delete?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Job managed */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example Successfully added urls for spidering. */ + response?: string; + jobs?: { + jobStatus?: { + /** @example Job is initializing. */ + message?: string; + /** @example 0 */ + status?: number; + }; + /** @example -1 */ + maxHops?: number; + /** @example ...json */ + downloadJson?: string; + /** @example */ + urlProcessPattern?: string; + /** @example 0 */ + jobCompletionTimeUTC?: number; + /** @example -1 */ + maxRounds?: number; + /** @example bulk */ + type?: string; + /** @example 0 */ + pageCrawlSuccessesThisRound?: number; + /** @example */ + urlCrawlRegEx?: string; + /** @example */ + pageProcessPattern?: string; + /** @example https://api.diffbot.com/v3/analyze */ + apiUrl?: string; + /** @example 1 */ + useCanonical?: number; + /** @example 1649950325 */ + jobCreationTimeUTC?: number; + /** @example 0 */ + repeat?: number; + /** @example ...csv */ + downloadUrls?: string; + /** @example 1 */ + obeyRobots?: number; + /** @example 0 */ + roundsCompleted?: number; + /** @example 0 */ + pageCrawlAttempts?: number; + /** @example */ + notifyWebhook?: string; + /** @example 0 */ + pageProcessSuccessesThisRound?: number; + customHeaders?: Record; + /** @example 0 */ + objectsFound?: number; + /** @example 0 */ + roundStartTime?: number; + /** @example */ + urlCrawlPattern?: string; + /** @example -1 */ + seedRecrawlFrequency?: number; + /** @example */ + urlProcessRegEx?: string; + /** @example 0 */ + pageProcessSuccesses?: number; + /** @example 0 */ + urlsHarvested?: number; + /** @example -1 */ + crawlDelay?: number; + /** @example 1649950325 */ + currentTime?: number; + /** @example 0 */ + useProxies?: number; + /** @example 0 */ + sentJobDoneNotification?: number; + /** @example 1649950325 */ + currentTimeUTC?: number; + /** @example bulkTest */ + name?: string; + /** @example */ + notifyEmail?: string; + /** @example 0 */ + pageCrawlSuccesses?: number; + /** @example 0 */ + pageProcessAttempts?: number; + }[]; + }; + }; + }; + /** @description No job found by that name */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 500 */ + errorCode?: string; + /** @example no collection record. */ + error?: string; + }; + }; + }; + }; + }; + "create-a-bulk-job": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/x-www-form-urlencoded": { + /** @description Job name. This should be a unique identifier and will be used to modify your bulk job and retrieve its output. */ + name: string; + /** @description Space-delimited list of URLs to process. If you are on the Startup plan, jobs must contain at least 50 URLs. */ + urls: string; + /** @description The full Extract (or Custom) API to be used for each URL. For instance, to process each URL via the Article API, supply `https://api.diffbot.com/v3/article`. You may also include API parameters, e.g. `https://api.diffbot.com/v3/article?fields=meta,tags`. */ + apiUrl: string; + /** @description Set custom headers to be used for processing each URL. Send multiple `customHeaders` values in your POST body, with header keys/values delimited by a colon (and URL-encoded). */ + customHeaders?: string; + /** @description Send a message to this email address when the bulk job completes. */ + notifyEmail?: string; + /** @description Pass a URL to be notified when the bulk job completes. You will receive a POST with a JSON response. */ + notifyWebhook?: string; + /** @description Pass `obeyRobots=0` to ignore a site's robots.txt instructions. */ + obeyRobots?: string; + /** + * Format: float + * @description Specify the number of days as a floating-point (e.g. `repeat=7.0`) to repeat this job. By default bulk jobs will not be repeated. + */ + repeat?: number; + /** @description Enter ||-separated strings to limit pages processed to those whose HTML contains any of the content strings. If a page does not contain at least one of the strings, it will be ignored. */ + pageProcessPattern?: string; + }; + }; + }; + responses: { + /** @description Bulk job created */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example Successfully added urls for spidering. */ + response?: string; + jobs?: { + jobStatus?: { + /** @example Job is initializing. */ + message?: string; + /** @example 0 */ + status?: number; + }; + /** @example -1 */ + maxHops?: number; + /** @example ...json */ + downloadJson?: string; + /** @example */ + urlProcessPattern?: string; + /** @example 0 */ + jobCompletionTimeUTC?: number; + /** @example -1 */ + maxRounds?: number; + /** @example bulk */ + type?: string; + /** @example 0 */ + pageCrawlSuccessesThisRound?: number; + /** @example */ + urlCrawlRegEx?: string; + /** @example */ + pageProcessPattern?: string; + /** @example https://api.diffbot.com/v3/analyze */ + apiUrl?: string; + /** @example 1 */ + useCanonical?: number; + /** @example 1649950325 */ + jobCreationTimeUTC?: number; + /** @example 0 */ + repeat?: number; + /** @example ...csv */ + downloadUrls?: string; + /** @example 1 */ + obeyRobots?: number; + /** @example 0 */ + roundsCompleted?: number; + /** @example 0 */ + pageCrawlAttempts?: number; + /** @example */ + notifyWebhook?: string; + /** @example 0 */ + pageProcessSuccessesThisRound?: number; + customHeaders?: Record; + /** @example 0 */ + objectsFound?: number; + /** @example 0 */ + roundStartTime?: number; + /** @example */ + urlCrawlPattern?: string; + /** @example -1 */ + seedRecrawlFrequency?: number; + /** @example */ + urlProcessRegEx?: string; + /** @example 0 */ + pageProcessSuccesses?: number; + /** @example 0 */ + urlsHarvested?: number; + /** @example -1 */ + crawlDelay?: number; + /** @example 1649950325 */ + currentTime?: number; + /** @example 0 */ + useProxies?: number; + /** @example 0 */ + sentJobDoneNotification?: number; + /** @example 1649950325 */ + currentTimeUTC?: number; + /** @example bulkTest */ + name?: string; + /** @example */ + notifyEmail?: string; + /** @example 0 */ + pageCrawlSuccesses?: number; + /** @example 0 */ + pageProcessAttempts?: number; + }[]; + }; + }; + }; + /** @description Token missing or invalid */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 401 */ + errorCode?: number; + /** @example Not authorized API token. */ + error?: string; + }; + }; + }; + /** @description No URLs were found in the request */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 500 */ + errorCode?: string; + /** @example no urls found */ + error?: string; + }; + }; + }; + /** @description Job name rejected — names can consist only of letters, numbers, underscores, or hyphens */ + 505: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 505 */ + errorCode?: number; + /** @example Crawl names can consist only of letters, numbers, underscores or hyphens. */ + error?: string; + }; + }; + }; + }; + }; + "retrieve-bulk-job-data": { + parameters: { + query: { + /** @description Name of the bulk job whose data you wish to download. */ + name: string; + /** @description Request `format=csv` to download the extracted data in CSV format (default: `json`). Note that CSV files will only contain top-level fields. */ + format?: string; + /** @description Request `type=urls` to retrieve the [URL Report (CSV)](/docs/crawl/faq/url-report). */ + type?: string; + /** @description Pass an integer value (e.g. `num=100`) to request a subset of URLs, most recently processed first. */ + num?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The bulk job's extracted data */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": ({ + /** + * @description Name of the Crawl or Bulk job this record was extracted from + * @example docs-bulkdata-sample + */ + col: string; + /** + * @description Immediate parent contained a link to this record + * @example https://example.com + */ + parentUrl: string; + /** + * @description The URL of this record + * @example https://example.com/ + */ + pageUrl: string; + /** + * @description A reference + * @example 6526070754824580000 + */ + id: number; + /** + * @description A standard entity type assigned either on job creation or classified by [Analyze API](/docs/extract/analyze). All other keys on this job record follow the [ontology](/docs/ontology/) set by its type. + * @example other + */ + type: string; + /** + * @description Your token + * @example + */ + token: string; + /** + * @description Time when record was crawled, in ISO 8601 format. + * @example 2026-07-24T23:03:01Z + */ + timestamp: string; + } & { + [key: string]: unknown; + })[]; + }; + }; + /** @description No job found by that name */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 400 */ + errorCode?: number; + /** @example No collection record */ + error?: string; + }; + }; + }; + }; + }; +} diff --git a/src/generated/crawl.ts b/src/generated/crawl.ts new file mode 100644 index 0000000..6cfdcd0 --- /dev/null +++ b/src/generated/crawl.ts @@ -0,0 +1,668 @@ +/** + * Generated from openapi/crawl.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/crawl": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Manage a Crawl Job + * @description Pause, delete, restart, or view the status of a crawl job. + */ + get: operations["manage-a-crawl-job"]; + put?: never; + /** + * Create a Crawl + * @description Create and start a job to spider and extract pages through a site. + */ + post: operations["create-a-crawl"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/crawl/data": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Retrieve Crawl Job Data + * @description Download the extracted results of a crawl job. + */ + get: operations["retrieve-crawl-job-data"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: never; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + "manage-a-crawl-job": { + parameters: { + query?: { + /** @description Job name as defined when the crawl job was created. */ + name?: string; + /** @description Pass `roundStart=1` to force the start of a new crawl "round" (manually repeat the crawl). If `onlyProcessIfNew` is set to 1 (default), only newly-created pages will be processed. */ + roundStart?: string; + /** @description Pass `pause=1` to pause a crawl job. Pass `pause=0` to resume a paused job. */ + pause?: string; + /** @description Pass `restart=1` to restart a crawl job. This will erase all processed data and re-process all of the submitted URLs. */ + restart?: string; + /** @description Pass `delete=1`to delete a job, and all associated data, completely. */ + delete?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Job managed */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example Successfully added urls for spidering. */ + response?: string; + jobs?: { + jobStatus?: { + /** @example Job is initializing. */ + message?: string; + /** @example 0 */ + status?: number; + }; + /** @example -1 */ + maxHops?: number; + /** @example ...json */ + downloadJson?: string; + /** @example */ + urlProcessPattern?: string; + /** @example 0 */ + jobCompletionTimeUTC?: number; + /** @example -1 */ + maxRounds?: number; + /** @example crawl */ + type?: string; + /** @example 0 */ + pageCrawlSuccessesThisRound?: number; + /** @example */ + urlCrawlRegEx?: string; + /** @example */ + pageProcessPattern?: string; + /** @example https://api.diffbot.com/v3/analyze */ + apiUrl?: string; + /** @example 1 */ + useCanonical?: number; + /** @example 1649950325 */ + jobCreationTimeUTC?: number; + /** @example 0 */ + repeat?: number; + /** @example ...csv */ + downloadUrls?: string; + /** @example 1 */ + obeyRobots?: number; + /** @example 0 */ + roundsCompleted?: number; + /** @example 0 */ + pageCrawlAttempts?: number; + /** @example */ + notifyWebhook?: string; + /** @example 0 */ + pageProcessSuccessesThisRound?: number; + customHeaders?: Record; + /** @example 0 */ + objectsFound?: number; + /** @example 0 */ + roundStartTime?: number; + /** @example */ + urlCrawlPattern?: string; + /** @example -1 */ + seedRecrawlFrequency?: number; + /** @example */ + urlProcessRegEx?: string; + /** @example 0 */ + pageProcessSuccesses?: number; + /** @example 0 */ + urlsHarvested?: number; + /** @example -1 */ + crawlDelay?: number; + /** @example 1649950325 */ + currentTime?: number; + /** @example 0 */ + useProxies?: number; + /** @example 0 */ + sentJobDoneNotification?: number; + /** @example 1649950325 */ + currentTimeUTC?: number; + /** @example crawlTest */ + name?: string; + /** @example */ + notifyEmail?: string; + /** @example 0 */ + pageCrawlSuccesses?: number; + /** @example 0 */ + pageProcessAttempts?: number; + }[]; + } | { + /** @example Successfully added urls for spidering. */ + response?: string; + jobs?: { + jobStatus?: { + /** @example Job paused. */ + message?: string; + /** @example 6 */ + status?: number; + }; + /** @example -1 */ + maxHops?: number; + /** @example ...json */ + downloadJson?: string; + /** @example */ + urlProcessPattern?: string; + /** @example 0 */ + jobCompletionTimeUTC?: number; + /** @example -1 */ + maxRounds?: number; + /** @example crawl */ + type?: string; + /** @example 0 */ + pageCrawlSuccessesThisRound?: number; + /** @example */ + urlCrawlRegEx?: string; + /** @example */ + pageProcessPattern?: string; + /** @example https://api.diffbot.com/v3/analyze */ + apiUrl?: string; + /** @example 1 */ + useCanonical?: number; + /** @example 1649950325 */ + jobCreationTimeUTC?: number; + /** @example 0 */ + repeat?: number; + /** @example ...csv */ + downloadUrls?: string; + /** @example 1 */ + obeyRobots?: number; + /** @example 0 */ + roundsCompleted?: number; + /** @example 0 */ + pageCrawlAttempts?: number; + /** @example */ + notifyWebhook?: string; + /** @example 0 */ + pageProcessSuccessesThisRound?: number; + customHeaders?: Record; + /** @example 0 */ + objectsFound?: number; + /** @example 0 */ + roundStartTime?: number; + /** @example */ + urlCrawlPattern?: string; + /** @example -1 */ + seedRecrawlFrequency?: number; + /** @example */ + urlProcessRegEx?: string; + /** @example 0 */ + pageProcessSuccesses?: number; + /** @example 0 */ + urlsHarvested?: number; + /** @example -1 */ + crawlDelay?: number; + /** @example 1649950325 */ + currentTime?: number; + /** @example 0 */ + useProxies?: number; + /** @example 0 */ + sentJobDoneNotification?: number; + /** @example 1649950325 */ + currentTimeUTC?: number; + /** @example crawlTest */ + name?: string; + /** @example */ + notifyEmail?: string; + /** @example 0 */ + pageCrawlSuccesses?: number; + /** @example 0 */ + pageProcessAttempts?: number; + }[]; + } | { + /** @example Successfully deleted job. */ + response?: string; + } | { + jobs?: { + jobStatus?: { + /** @example Job has completed and no repeat is scheduled. */ + message?: string; + /** @example 9 */ + status?: number; + }; + /** @example -1 */ + maxHops?: number; + /** @example ...json */ + downloadJson?: string; + /** @example */ + urlProcessPattern?: string; + /** @example 0 */ + jobCompletionTimeUTC?: number; + /** @example -1 */ + maxRounds?: number; + /** @example crawl */ + type?: string; + /** @example 0 */ + pageCrawlSuccessesThisRound?: number; + /** @example */ + urlCrawlRegEx?: string; + /** @example */ + pageProcessPattern?: string; + /** @example https://api.diffbot.com/v3/analyze */ + apiUrl?: string; + /** @example 1 */ + useCanonical?: number; + /** @example 1649950325 */ + jobCreationTimeUTC?: number; + /** @example 0 */ + repeat?: number; + /** @example ...csv */ + downloadUrls?: string; + /** @example 1 */ + obeyRobots?: number; + /** @example 0 */ + roundsCompleted?: number; + /** @example 0 */ + pageCrawlAttempts?: number; + /** @example */ + notifyWebhook?: string; + /** @example 0 */ + pageProcessSuccessesThisRound?: number; + customHeaders?: Record; + /** @example 0 */ + objectsFound?: number; + /** @example 0 */ + roundStartTime?: number; + /** @example */ + urlCrawlPattern?: string; + /** @example -1 */ + seedRecrawlFrequency?: number; + /** @example */ + urlProcessRegEx?: string; + /** @example 0 */ + pageProcessSuccesses?: number; + /** @example 0 */ + urlsHarvested?: number; + /** @example -1 */ + crawlDelay?: number; + /** @example 1649950325 */ + currentTime?: number; + /** @example 0 */ + useProxies?: number; + /** @example 0 */ + sentJobDoneNotification?: number; + /** @example 1649950325 */ + currentTimeUTC?: number; + /** @example crawlTest */ + name?: string; + /** @example */ + notifyEmail?: string; + /** @example 0 */ + pageCrawlSuccesses?: number; + /** @example 0 */ + pageProcessAttempts?: number; + }[]; + }; + }; + }; + /** @description No crawl job by that name */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 500 */ + errorCode?: string; + /** @example no collection record. */ + error?: string; + }; + }; + }; + }; + }; + "create-a-crawl": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/x-www-form-urlencoded": { + /** @description Job name. This should be a unique identifier and can be used to modify your crawl or retrieve its output. */ + name: string; + /** @description Seed URL(s). Must be [URL encoded](https://en.wikipedia.org/wiki/Percent-encoding). Separate multiple URLs with whitespace to spider multiple sites within the same crawl. If the seed contains a non-www subdomain ("https://blog.diffbot.com" or "https://docs.diffbot.com") Crawl will restrict spidering to the specified **subdomain**. */ + seeds: string; + /** + * @description Full Extract API URL through which to process pages. E.g., `&apiUrl=https://api.diffbot.com/v3/analyze` to process matching links via the Analyze API, which will automatically determine the page type to extract as. The Extract API URL can include querystring parameters to tailor the output. + * + * For example, `&apiUrl=https://api.diffbot.com/v3/product?fields=querystring,meta` will process matching links using the Product API, and also return the querystring and meta fields. + */ + apiUrl: string; + /** @description Set multiple custom headers to be used while crawling and processing pages sent to Diffbot APIs. Each header should be sent in its own `customHeaders` argument, with a colon delimiting the header name and value, and should be URL-encoded. For example, `&customHeaders=Accept-Language%3Aen-us`. [See more on using custom headers](/docs/extract/custom-headers). */ + customHeaders?: string; + /** + * @description Pass `useCanonical=0` to disable deduplication of pages based on a canonical link definition. [See more](/products/crawl/faq/duplicate-content). + * @default 1 + */ + useCanonical?: number; + /** + * @description Pass `obeyRobots=0` to ignore a site's robots.txt instructions. + * @default 1 + */ + obeyRobots?: number; + /** + * @description Pass `restrictDomain=0` to allow limited crawling across subdomains/domains. [See more](/products/crawl/faq/restrict-to-domains). + * @default 1 + */ + restrictDomain?: number; + /** + * @description Set value to `1` to force the use of proxy IPs for the crawl. This will utilize proxy servers for both crawling and processing of pages. + * @default 0 + */ + useProxies?: number; + /** + * @description Specify the depth of your crawl. A `maxHops=0` will limit **processing** to the seed URL(s) only -- no other links will be processed; `maxHops=1` will process all (otherwise matching) pages whose links appear on seed URL(s); `maxHops=2` will process pages whose links appear on those pages; and so on. + * + * By default (`maxHops=-1`) Crawl will crawl and process links at any depth. + * @default -1 + */ + maxHops?: number; + /** @description Send a message to this email address when the crawl hits the `maxToCrawl` or `maxToProcess` limit, or when the crawl completes. */ + notifyEmail?: string; + /** @description Pass a URL to be notified when the crawl hits the `maxToCrawl` or `maxToProcess` limit, or when the crawl completes. You will receive a POST with `X-Crawl-Name` and `X-Crawl-Status` in the headers, and the job's JSON metadata in the POST body. Note that in webhook POSTs the parent `jobs` will not be sent—only the individual job object will be returned. */ + notifyWebhook?: string; + /** @description Specify the number of days as a floating-point (e.g. `repeat=7.0`) to repeat this crawl. By default crawls will not be repeated. */ + repeat?: number; + /** + * @description Useful for specifying a frequency, in number of days, to recrawl seed urls, which is independent of the overall recrawl frequency given by `repeat`. Defaults to `seedRecrawlFrequency=-1` to use the default frequency. + * @default -1 + */ + seedRecrawlFrequency?: number; + /** + * @description Specify the maximum number of crawl repeats. By default (`maxRounds=0`) repeating crawls will continue indefinitely. + * @default 0 + */ + maxRounds?: number; + /** + * @description Specify ||-separated **strings** to limit pages crawled to those whose URLs contain *any* of the content strings. You can use the exclamation point to specify a negative string, e.g. `!product` to exclude URLs containing the string "product," and the `^` and `$` characters to limit matches to the beginning or end of the URL. + * + * The use of a `urlCrawlPattern` will allow Crawl to spider outside of the seed domain; it will follow all matching URLs regardless of domain. + */ + urlCrawlPattern?: string; + /** @description Specify a regular expression to limit pages **crawled** to those URLs that contain a match to your expression. This will override any `urlCrawlPattern` value. */ + urlCrawlRegEx?: string; + /** + * @description Specify max pages to spider. Default: 100,000. + * @default 100000 + */ + maxToCrawl?: number; + /** + * @description Specify max pages to spider per subdomain. Default: no limit (-1) + * @default -1 + */ + maxToCrawlPerSubdomain?: number; + /** @description Wait this many seconds between each URL crawled from a single IP address. Specify the number of seconds as an integer or floating-point number (e.g., `crawlDelay=0.25`). */ + crawlDelay?: number; + /** @description Specify ||-separated **strings** to limit pages processed to those whose URLs contain *any* of the content strings. You can use the exclamation point to specify a negative string, e.g. `!/category` to exclude URLs containing the string "/category," and the `^` and `$` characters to limit matches to the beginning or end of the URL. */ + urlProcessPattern?: string; + /** @description Specify a regular expression to limit pages **processed** to those URLs that contain a match to your expression. This will override any `urlProcessPattern` value. */ + urlProcessRegEx?: string; + /** @description Specify ||-separated strings to limit pages **processed** to those whose HTML contains *any* of the content strings. */ + pageProcessPattern?: string; + /** + * @description Specify max pages to process through Diffbot APIs. Default: 100,000. + * @default 100000 + */ + maxToProcess?: number; + /** + * @description Specify max pages to process per subdomain. Default: no limit (-1) + * @default -1 + */ + maxToProcessPerSubdomain?: number; + /** + * @description By default repeat crawls will only process new (previously unprocessed) pages. Set to 0 (`onlyProcessIfNew=0`) to process all content on repeat crawls. + * @default 1 + */ + onlyProcessIfNew?: number; + }; + }; + }; + responses: { + /** @description Crawl job created */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example Successfully added urls for spidering. */ + response?: string; + jobs?: { + /** @example -1 */ + maxToProcessPerSubdomain?: number; + jobStatus?: { + /** @example Job is initializing. */ + message?: string; + /** @example 0 */ + status?: number; + }; + /** @example -1 */ + maxHops?: number; + /** @example 1 */ + onlyProcessIfNew?: number; + /** @example https://api.diffbot.com/v3/crawl/download/.json */ + downloadJson?: string; + /** @example */ + urlProcessPattern?: string; + /** @example 0 */ + jobCompletionTimeUTC?: number; + /** @example -1 */ + maxRounds?: number; + /** @example 1 */ + restrictDomain?: number; + /** @example crawl */ + type?: string; + /** @example 0 */ + pageCrawlSuccessesThisRound?: number; + /** @example */ + urlCrawlRegEx?: string; + /** @example */ + pageProcessPattern?: string; + /** @example https://api.diffbot.com/v3/analyze */ + apiUrl?: string; + /** @example 1 */ + useCanonical?: number; + /** @example -1 */ + maxToCrawlPerSubdomain?: number; + /** @example 1726163287 */ + jobCreationTimeUTC?: number; + /** @example 0 */ + repeat?: number; + /** @example https://api.diffbot.com/v3/crawl/download/.csv */ + downloadUrls?: string; + /** @example 1 */ + obeyRobots?: number; + /** @example 0 */ + roundsCompleted?: number; + /** @example 0 */ + pageCrawlAttempts?: number; + /** @example */ + notifyWebhook?: string; + /** @example 0 */ + pageProcessSuccessesThisRound?: number; + customHeaders?: Record; + /** @example 0 */ + objectsFound?: number; + /** @example 0 */ + roundStartTime?: number; + /** @example */ + urlCrawlPattern?: string; + /** @example 100000 */ + maxToProcess?: number; + /** @example -1 */ + seedRecrawlFrequency?: number; + /** @example https://example.com */ + seeds?: string; + /** @example */ + urlProcessRegEx?: string; + /** @example 0 */ + pageProcessSuccesses?: number; + /** @example 0 */ + urlsHarvested?: number; + /** @example 0.25 */ + crawlDelay?: number; + /** @example 1726163288 */ + currentTime?: number; + /** @example 0 */ + useProxies?: number; + /** @example 100000 */ + maxToCrawl?: number; + /** @example 0 */ + sentJobDoneNotification?: number; + /** @example 1726163288 */ + currentTimeUTC?: number; + /** @example 0 */ + restrictSubdomain?: number; + /** @example test-crawl */ + name?: string; + /** @example */ + notifyEmail?: string; + /** @example 0 */ + pageCrawlSuccesses?: number; + /** @example 0 */ + pageProcessAttempts?: number; + }[]; + }; + }; + }; + /** @description Too many collections for token — the 1,000-crawl limit has been reached */ + 505: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 505 */ + errorCode?: number; + /** @example Too many collections for token */ + error?: string; + }; + }; + }; + }; + }; + "retrieve-crawl-job-data": { + parameters: { + query: { + /** @description Name of the crawl job whose data you wish to download. */ + name: string; + /** @description Request `format=csv` to download the extracted data in CSV format (default: `json`). Note that CSV files will only contain top-level fields. */ + format?: string; + /** @description Request `type=urls` to retrieve the [URL Report (CSV)](/products/crawl/faq/url-report). */ + type?: string; + /** @description Pass an integer value (e.g. `num=100`) to request a subset of URLs, most recently processed first. */ + num?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description The crawl job's extracted data */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": ({ + /** + * @description Name of the Crawl or Bulk job this record was extracted from + * @example example-test + */ + col: string; + /** + * @description Immediate parent contained a link to this record + * @example https://www.example.com + */ + parentUrl: string; + /** + * @description The URL of this record + * @example https://www.example.com/ + */ + pageUrl: string; + /** + * @description A reference + * @example 1492074227214006500 + */ + id: number; + /** + * @description A standard entity type assigned either on job creation or classified by [Analyze API](/docs/extract/analyze). All other keys on this job record follow the [ontology](/docs/ontology/) set by its type. + * @example other + */ + type: string; + /** + * @description Your token + * @example + */ + token: string; + /** + * @description Time when record was crawled, in ISO 8601 format. + * @example 2026-07-24T23:15:09Z + */ + timestamp: string; + } & { + [key: string]: unknown; + })[]; + }; + }; + /** @description No crawl job by that name */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 400 */ + errorCode?: number; + /** @example No collection record */ + error?: string; + }; + }; + }; + }; + }; +} diff --git a/src/generated/dql.ts b/src/generated/dql.ts new file mode 100644 index 0000000..85e8750 --- /dev/null +++ b/src/generated/dql.ts @@ -0,0 +1,518 @@ +/** + * Generated from openapi/dql.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/kg/v3/dql": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Search with DQL + * @description Search the Knowledge Graph using DQL + */ + get: operations["dqlGet"]; + put?: never; + /** + * Search with DQL + * @description Search the Knowledge Graph using DQL + */ + post: operations["dqlPost"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/dql/report": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Coverage report by Query + * @description Download coverage report by DQL query + */ + get: operations["reportFind"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/dql/report/{id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Coverage report by ID + * @description Download coverage report by report ID + */ + get: operations["reportGet"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + /** @description A single result */ + Data: { + /** + * Format: double + * @description score + * @example 0.75 + */ + score?: number; + entity?: components["schemas"]["JSONObject"]; + /** @description Entity-level context */ + entity_ctx?: { + [key: string]: Record; + }; + errors?: string[]; + /** @description The query for the facet */ + callbackQuery?: string; + /** + * Format: double + * @description The upper bound of the facet + */ + upperBound?: number; + /** + * Format: double + * @description The lower bound of the facet + */ + lowerBound?: number; + /** + * Format: int64 + * @description The count of the facet + */ + count?: number; + /** @description The value of the facet */ + value?: string; + /** @description The uri of the facet value (for uri facets) */ + uri?: string; + }; + DqlResponse: { + /** + * Format: int32 + * @description Response version + * @example 3 + */ + version?: number; + /** Format: int64 */ + hits?: number; + /** + * Format: int64 + * @description Number of results in this response + * @example 50 + */ + results?: number; + /** + * @description Version of KG that was queried + * @example 250 + */ + kgversion?: string; + /** + * @description Diffbot type of returned results + * @example Organization + */ + diffbot_type?: string; + /** @description Was this a facet query? */ + facet?: boolean; + /** @description Results of the query */ + data?: components["schemas"]["Data"][]; + rewrites?: components["schemas"]["Rewrite"][]; + /** + * @description Whether the query fell back to text search. + * @example false + */ + textFallback?: boolean; + /** + * @description Identifier for the generated coverage report. Present only when `report=true` was passed. + * @example C-4d87a965-b43f + */ + reportId?: string; + }; + /** + * @description Entity. Can be null if no entity was found. + * @example { + * "id": "EYX1i02YVPsuT7fPLUYgRhQ", + * "diffbotUri": "http://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + * "name": "Diffbot", + * "origins": [ + * "angel.co/diffbot", + * "bloomberg.com/profiles/companies/0506286D:US-diffbot-corp", + * "crunchbase.com/organization/diffbot", + * "linkedin.com/company/diffbot", + * "en.wikipedia.org/wiki/Diffbot" + * ] + * } + */ + JSONObject: { + empty?: boolean; + }; + Rewrite: { + query?: string; + reason?: string; + }; + DqlPostRequest: { + /** + * @description Query type + * @default query + * @enum {string} + */ + type: "query" | "text" | "queryTextFallback" | "crawl"; + /** @description DQL Query */ + query: string; + /** @description Comma-delimited list of custom crawl collections to query. Specify `all` to query all collections for your token. This parameter is valid only when type=crawl and defaults to `all` */ + col?: string; + /** + * Format: int32 + * @description Return results starting at `from` index. `from+size <= 10_000` for facet queries. + * + * Please note that the from parameter is no longer applicable to Article queries and has been deprecated. In the future, using this parameter will result in an error. + * @default 0 + */ + from: number; + /** + * Format: int32 + * @description Maximum number of results to return. `-1` to return all results. `from+size <= 10_000` for facet queries. + * @default 50 + */ + size: number; + /** + * @description Output format + * @default json + * @enum {string} + */ + format: "json" | "jsonl" | "csv" | "xls" | "xlsx"; + /** + * @description The spec defines the columns to export. This is applicable for csv, xls and xslx formats. + * + * Export spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields). + */ + exportspec?: string; + /** @description For export endpoint, define separator for fields which contains multiple values */ + exportseparator?: string; + /** @description For export endpoint, The file name of the export file */ + exportfile?: string; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + /** + * @description `extended` returns origin information for facts. `id` returns just the diffbotIds and origins. + * @enum {string} + */ + jsonmode?: "extended" | "id"; + /** + * @description `nonCanonicalFacts=true` returns non-canonical facts. + * @default false + */ + nonCanonicalFacts: boolean; + /** + * @description `noDedupArticles=true` indicates that articles should not be deduplicated but all of them returned + * @default false + */ + noDedupArticles: boolean; + /** + * @description Articles only. Utilize Diffbot clustering algorithms to identify related or similar objects. + * - `cluster=all` to return each article with a clusterId identifying thematically-similar articles. + * - `cluster=best` to remove thematically-similar articles from the result set, leaving a single best representative from each cluster. + * - `cluster=dedupe` to remove exact and near-duplicate articles from the result set. Where applicable, clusterId is returned in the `entity_ctx` element of each response entity. + * @enum {string} + */ + cluster?: never; + /** + * @description Generate coverage report. See [Coverage Reports](https://docs.diffbot.com/docs/dql-coverage-reports) for more details on generating coverage reports for DQL queries. + * @default false + */ + report: boolean; + view?: string; + }; + /** @description Returned when the DQL query itself cannot be parsed. */ + QueryParseError: { + /** @example true */ + error?: boolean; + /** @example Encountered " ":" ": "" at line 1, column 6. */ + message?: string; + /** @example 1 */ + line?: number; + /** @example 6 */ + column?: number; + /** + * @description The message with the offending token wrapped in `` for display. + * @example type:::Bogus(( + */ + htmlMessage?: string; + }; + /** @description Returned when a request parameter is missing or invalid. */ + ParameterError: { + /** @example true */ + error?: boolean; + /** @example Parameter 'query' not specified */ + message?: string; + }; + AuthError: { + /** @example 401 */ + code?: number; + /** @example Unauthorized. Incorrect token. */ + message?: string; + /** @example a9cbf96bb0e2084d5c1595ae8f526576 */ + requestId?: string; + }; + ServerError: { + /** @example 500 */ + code?: number; + /** @example HTTP 500 Internal Server Error */ + message?: string; + }; + }; + responses: { + /** @description Query results */ + Results: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DqlResponse"]; + "application/json-lines": Record; + "text/csv": Record; + "application/ms-excel": Record; + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": Record; + }; + }; + /** @description Error parsing the query */ + QueryParseError: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["QueryParseError"]; + }; + }; + /** @description Unauthorized */ + Unauthorized: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Invalid or missing parameters */ + ParameterError: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ParameterError"]; + }; + }; + /** @description Internal Server Error */ + ServerError: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ServerError"]; + }; + }; + }; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + dqlGet: { + parameters: { + query: { + /** @description Query type */ + type?: "query" | "crawl"; + /** @description DQL Query */ + query: string; + /** @description Comma-delimited list of custom crawl collections to query. Specify `all` to query all collections for your token. This parameter is valid only when type=crawl and defaults to `all` */ + col?: string; + /** + * @description Return results starting at `from` index. `from+size <= 10_000` for facet queries. + * + * Please note that the from parameter is no longer applicable to Article queries and has been deprecated. In the future, using this parameter will result in an error. + */ + from?: number; + /** @description Maximum number of results to return. `-1` to return all results. `from+size <= 10_000` for facet queries. */ + size?: number; + /** @description Output format */ + format?: "json" | "jsonl" | "csv" | "xls" | "xlsx"; + /** + * @description The spec defines the columns to export. This is applicable for csv, xls and xslx formats. + * + * Export spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields). + */ + exportspec?: string; + /** @description The export separator */ + exportseparator?: string; + /** @description For export endpoint, The file name of the export file */ + exportfile?: string; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + /** @description `extended` returns origin information for facts. `id` returns just the diffbotIds and origins. */ + jsonmode?: "extended" | "id"; + /** @description `nonCanonicalFacts=true` returns non-canonical facts. */ + nonCanonicalFacts?: boolean; + /** @description `noDedupArticles=true` indicates that articles should not be deduplicated but all of them returned */ + noDedupArticles?: boolean; + /** + * @description Articles only. Utilize Diffbot clustering algorithms to identify related or similar objects. + * - `cluster=all` to return each article with a clusterId identifying thematically-similar articles. + * - `cluster=best` to remove thematically-similar articles from the result set, leaving a single best representative from each cluster. + * - `cluster=dedupe` to remove exact and near-duplicate articles from the result set. Where applicable, clusterId is returned in the `entity_ctx` element of each response entity. + */ + cluster?: "all" | "best" | "dedupe"; + /** @description Generate coverage report. See [Coverage Reports](https://docs.diffbot.com/docs/dql-coverage-reports) for more details on generating coverage reports for DQL queries. */ + report?: boolean; + view?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Results"]; + 400: components["responses"]["QueryParseError"]; + 401: components["responses"]["Unauthorized"]; + 422: components["responses"]["ParameterError"]; + 500: components["responses"]["ServerError"]; + }; + }; + dqlPost: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** @description DQL request */ + requestBody?: { + content: { + "application/json": components["schemas"]["DqlPostRequest"]; + }; + }; + responses: { + 200: components["responses"]["Results"]; + 400: components["responses"]["QueryParseError"]; + 401: components["responses"]["Unauthorized"]; + 422: components["responses"]["ParameterError"]; + 500: components["responses"]["ServerError"]; + }; + }; + reportFind: { + parameters: { + query: { + /** @description DQL Query */ + query: string; + /** + * @description Return results starting at `from` index. `from+size <= 10_000` for facet queries. + * + * Please note that the from parameter is no longer applicable to Article queries and has been deprecated. In the future, using this parameter will result in an error. + */ + from?: number; + /** @description Maximum number of results to return. `-1` to return all results. `from+size <= 10_000` for facet queries. */ + size?: number; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + /** + * @description The spec defines the columns to export. This is applicable for csv, xls and xslx formats. + * + * Export spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields). + */ + exportspec?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Coverage report */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example DiffbotID,Name,$.name,$.nbEmployees + * "",,1.0,1.0 + * El0wZJw_cOn-MAyp_hZZ5Tw,Yandex,1,1 + * EUFq-3WlpNsq0pvfUYWXOEA,Google,1,1 + * Ec5P22-6EPLOXo1-z6-4NzA,Instagram,1,1 + */ + "text/csv": string; + }; + }; + 401: components["responses"]["Unauthorized"]; + 500: components["responses"]["ServerError"]; + }; + }; + reportGet: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The report id */ + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Coverage report */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example DiffbotID,Name,$.name,$.nbEmployees + * "",,1.0,1.0 + * El0wZJw_cOn-MAyp_hZZ5Tw,Yandex,1,1 + * EUFq-3WlpNsq0pvfUYWXOEA,Google,1,1 + * Ec5P22-6EPLOXo1-z6-4NzA,Instagram,1,1 + */ + "text/csv": string; + }; + }; + /** @description Report not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ParameterError"]; + }; + }; + 500: components["responses"]["ServerError"]; + }; + }; +} diff --git a/src/generated/enhance.ts b/src/generated/enhance.ts new file mode 100644 index 0000000..7102962 --- /dev/null +++ b/src/generated/enhance.ts @@ -0,0 +1,1478 @@ +/** + * Generated from openapi/enhance.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/kg/v3/enhance": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Enhance + * @description Enrich a person or organization record with partial data input + */ + get: operations["enhance"]; + put?: never; + /** + * Enhance + * @description Enrich a person or organization record with partial data input (POST option) + */ + post: operations["enhancePost"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/combine": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Combine + * @description Enrich a person record and return both person and current employer data + */ + get: operations["combine"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Create a Bulkjob + * @description Enhance multiple records in bulk asynchronously + */ + post: operations["submitBulkjob"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List Bulkjobs for Token + * @description Poll the status of all Enhance Bulkjobs for a token + */ + get: operations["bulkjobStatusForToken"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/{bulkjobId}/status": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Poll bulkjob status + * @description Poll the status of an Enhance Bulkjob + */ + get: operations["bulkjobStatus"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/{bulkjobId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Download Results of Bulkjob + * @description Download the result of a completed Enhance Bulkjob + */ + get: operations["bulkjobResultGet"]; + put?: never; + /** + * Download Results of Bulkjob + * @description Download the result of a completed Enhance Bulkjob + */ + post: operations["bulkjobResultPost"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/report/{bulkjobId}/{reportId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Download Bulkjob Coverage Report + * @description Download the coverage report of a completed Bulk Enhance job + */ + get: operations["coverageReportGet"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/{bulkjobId}/delete": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Delete Bulkjob + * @description Delete bulkjob + */ + get: operations["deleteBulkjob"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/{bulkjobId}/{jobIdx}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Download Single Result of Bulkjob + * @description Use this API to download the result of a single job within a bulkjob by specifying the index of the job. + */ + get: operations["singleResult"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/kg/v3/enhance/bulk/{bulkjobId}/stop": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Stop Bulkjob + * @description Stop an active Enhance Bulkjob + */ + get: operations["stopBulkjob"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + /** @description Enhanced entity data */ + Data: { + /** + * Format: double + * @description Enhance score + */ + score?: number; + /** Format: double */ + esscore?: number; + entity?: components["schemas"]["ObjectNode"]; + errors?: string[]; + /** @description Refresh info for debug */ + refreshInfo?: { + [key: string]: components["schemas"]["RefreshDebugInformation"]; + }; + refreshDelta?: components["schemas"]["Delta"]; + /** @description Entity-level context */ + entity_ctx?: { + [key: string]: Record; + }; + }; + Delta: Record; + EnhanceResponse: { + /** + * Format: int32 + * @description Response version + */ + version?: number; + /** + * Format: int32 + * @description Number of matches returned + */ + hits?: number; + /** @description Version of KG that was queried */ + kgversion?: string; + request_ctx?: components["schemas"]["RequestContext"]; + data?: components["schemas"]["Data"][]; + errors?: string[]; + /** @description Search info for debug */ + searchInfo?: { + [key: string]: components["schemas"]["SearchCandidate"]; + }; + }; + /** + * @description Enhanced entity. Can be null if no entity was found. + * @example { + * "id": "EYX1i02YVPsuT7fPLUYgRhQ", + * "images": [ + * { + * "naturalHeight": 0, + * "width": 0, + * "url": "https://kg.diffbot.com/image/api/get?fetch=yes&url=g%3Cj7P0St%7D9u.LnOv%7BI%7C%40g7.pBYINs%3Ff%7CGZ7jr-tGZhWf%3DK%2FOvy%5Bp%3CSKbs8i%7CEyNzCa%3EYr%3Ef%3CRc%3CLr5gx_hOgiFuER5%3Cb%7C%3E.I%5Do", + * "isCached": true, + * "naturalWidth": 0, + * "primary": true, + * "height": 0 + * } + * ], + * "diffbotUri": "http://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + * "name": "Diffbot", + * "origins": [ + * "angel.co/diffbot", + * "bloomberg.com/profiles/companies/0506286D:US-diffbot-corp", + * "crunchbase.com/organization/diffbot", + * "linkedin.com/company/diffbot", + * "en.wikipedia.org/wiki/Diffbot" + * ] + * } + */ + ObjectNode: Record; + OrganizationRequest: { + /** @description Diffbot entity type */ + type?: string; + /** @description Name of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + name?: string[]; + /** @description Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + url?: string[]; + /** @description Location of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + location?: string; + /** @description IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + ip?: string; + /** @description Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + phone?: string; + /** @description Description of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + description?: string; + }; + PersonRequest: { + /** @description Diffbot entity type */ + type?: string; + /** @description Name of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + name?: string[]; + /** @description Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + url?: string[]; + /** @description Location of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + location?: string; + /** @description IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + ip?: string; + /** @description Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + phone?: string; + /** @description Description of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + description?: string; + /** @description Email of the entity to enhance. Parameter can be used only with type `Person` */ + email?: string[]; + /** @description Employer of the entity to enhance. Parameter can be used only with type `Person` */ + employer?: string; + /** @description Title of the entity to enhance. Parameter can be used only with type `Person` */ + title?: string; + /** @description School of the entity to enhance. Parameter can be used only with type `Person` */ + school?: string; + }; + RefreshDebugInformation: { + url?: string; + hasResult?: boolean; + /** Format: int64 */ + lastCrawlTime?: number; + message?: string; + }; + /** @description Request context */ + RequestContext: { + /** @description Submitted query */ + query?: components["schemas"]["OrganizationRequest"] | components["schemas"]["PersonRequest"] | string; + /** @description Query context. Example: submitted query parameters etc. */ + query_ctx?: { + [key: string]: Record; + }; + }; + /** @description Search info for debug */ + SearchCandidate: { + /** Format: double */ + mlScore?: number; + /** Format: double */ + baseMlScore?: number; + debugInfo?: components["schemas"]["RefreshDebugInformation"]; + }; + /** @description Bulkjob status */ + BulkjobStatus: { + /** @description Bulkjob Id */ + job_id?: string; + /** @description Bulkjob name */ + name?: string; + /** @description Is Refresh enabled for this job? */ + refresh?: boolean; + /** + * @description Is Search enabled for this job? + * @enum {string} + */ + search?: "ON" | "OFF" | "SEARCHONLY" | "FALLBACK"; + /** @description Bulkjob status message */ + message?: string; + /** + * Format: int32 + * @description Number of jobs in bulkjob + */ + jobs_total?: number; + /** + * Format: int32 + * @description Number of jobs completed in bulkjob. + */ + jobs_completed?: number; + /** + * @description Status of Bulkjob. One of {NOT_STARTED, IN_PROCESS, COMPLETE, COMPLETE_WITH_FAILURES, STOPPED, ERROR_FINALIZING, UNKNOWN} + * @enum {string} + */ + status?: "NOT_STARTED" | "IN_PROCESS" | "COMPLETE" | "COMPLETE_WITH_FAILURES" | "STOPPED" | "ERROR_FINALIZING" | "ERROR_SCHEDULING" | "UNKNOWN"; + /** + * Format: int64 + * @description Estimated remaining time for bulkjob to complete (in seconds). `null` if job is completed or not started. + */ + estimate_remaining_time?: number; + /** + * Format: int64 + * @description Bulkjob submit time + */ + submit_time_ms?: number; + /** + * Format: int64 + * @description Bulkjob start time + */ + start_time_ms?: number; + /** + * Format: int64 + * @description Bulkjob completed time + */ + completed_time_ms?: number; + /** + * Format: int32 + * @description Total results + */ + total_results?: number; + bulkjobId?: string; + /** Format: int32 */ + totalJobs?: number; + /** Format: int32 */ + finishedJobs?: number; + /** Format: int64 */ + estimateRemainingTimeSec?: number; + /** Format: int64 */ + submitTimeMs?: number; + /** Format: int64 */ + startTimeMs?: number; + /** Format: int64 */ + completedTimeMs?: number; + /** Format: int32 */ + totalResults?: number; + /** + * Format: int64 + * @description Size of download file in bytes + */ + fileSizeBytes?: number; + /** @description Coverage reports generated for this bulkjob */ + reports?: components["schemas"]["Report"][]; + }; + /** @description Coverage reports generated for this bulkjob */ + Report: { + /** @description Coverage Report ID */ + reportId?: string; + /** + * @description Status of report generation. One of {NOT_STARTED, IN_PROCESS, COMPLETE, UNKNOWN} + * @enum {string} + */ + status?: "NOT_STARTED" | "IN_PROCESS" | "COMPLETE" | "COMPLETE_WITH_FAILURES" | "STOPPED" | "ERROR_FINALIZING" | "ERROR_SCHEDULING" | "UNKNOWN"; + /** @description Export format */ + format?: string; + /** + * @description The spec defines the columns to export. This is applicable for csv, xls and xslx formats. + * + * Export spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields). + */ + spec?: string; + exportfile?: string; + /** @description Separator for multi-value fields when exporting columnar results */ + exportSeparator?: string; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Return only records that have a match */ + onlyMatches?: boolean; + }; + StatusResponseError: { + /** + * Format: int32 + * @description Status code (same as the HTTP status code) + */ + code?: number; + /** @description Status message */ + message?: string; + request_ctx?: components["schemas"]["RequestContext"]; + bulkjobStatus?: components["schemas"]["BulkjobStatus"]; + /** @description `true` if error */ + error?: boolean; + }; + AuthError: { + /** @description Identifier of the rejected request, also returned in the `X-Diffbot-RequestId` header */ + requestId?: string; + /** + * Format: int32 + * @description Status code (same as the HTTP status code) + */ + code?: number; + /** @description Status message */ + message?: string; + }; + JsonNode: Record; + BulkjobResultPostRequest: { + /** + * Format: int32 + * @description Return first n results + */ + head?: number; + /** + * Format: int32 + * @description 1-based index of the first result to return (1 = first result). + */ + from?: number; + /** + * Format: int32 + * @description Maximum number of results to return starting from the 'from' position. + */ + size?: number; + /** @description Return only records that have a match */ + onlyMatches?: boolean; + /** + * @description Export format + * @enum {string} + */ + format?: "json" | "jsonl" | "csv" | "xls" | "xlsx"; + /** + * @description The spec defines the columns to export. This is applicable for csv, xls and xslx formats. + * + * Export spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields). + */ + exportspec?: string; + /** @description File name of the export file */ + exportfile?: string; + /** @description Separator for multi-value fields when exporting columnar results */ + exportseparator?: string; + /** @description Prefixes the enhance query parameters to the CSV export result */ + exportquery?: boolean; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + /** + * Format: int32 + * @description Seconds to wait for bulkjob results to export. Results will continue to export in the background. To only trigger an export, use 0. + */ + wait?: number; + }; + BulkjobStatusResponse: { + /** + * Format: int32 + * @description Bulkjob status code (same as the HTTP status code) + */ + code?: number; + content?: components["schemas"]["BulkjobStatus"]; + }; + BulkjobRecoveryStatusResponse: { + /** + * @description Bulkjob recovery status + * @enum {string} + */ + status?: "NOT_STARTED" | "IN_PROCESS" | "COMPLETE" | "COMPLETE_WITH_FAILURES" | "STOPPED" | "ERROR_FINALIZING" | "ERROR_SCHEDULING" | "UNKNOWN"; + /** @description Bulkjob recovery status message */ + message?: string; + }; + BulkjobAccepted: { + /** @description Bulkjob Id for checking status and downloading results */ + job_id?: string; + }; + IdRequestEntity: { + /** @description DiffbotId of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + id?: string; + }; + EnhancePostRequest: { + /** + * @description Diffbot entity type + * @enum {string} + */ + type: "Person" | "Organization"; + /** @description DiffbotId of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + id?: string; + /** @description Name of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + name?: string[]; + /** @description Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + url?: string[]; + /** @description Location of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + location?: string; + /** @description IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + ip?: string; + /** @description Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + phone?: string; + /** @description Description of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + description?: string; + /** @description Email of the entity to enhance. Parameter can be used only with type `Person` */ + email?: string[]; + /** @description Employer of the entity to enhance. Parameter can be used only with type `Person` */ + employer?: string; + /** @description Title of the entity to enhance. Parameter can be used only with type `Person` */ + title?: string; + /** @description School of the entity to enhance. Parameter can be used only with type `Person` */ + school?: string; + /** @description User defined ID for correlation */ + customId?: string; + /** + * Format: int32 + * @description Maximum number of results to return + * @default 1 + */ + size: number; + /** + * Format: double + * @description Enhance similarity threshold + */ + threshold?: number; + /** + * @description `refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data. + * @default false + */ + refresh: boolean; + /** + * @description `search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG. + * @default false + */ + search: boolean; + /** + * @description `nonCanonicalFacts=true` returns non-canonical facts. + * @default false + */ + nonCanonicalFacts: boolean; + /** + * @description `jsonmode=extended` returns origin information for facts. + * @enum {string} + */ + jsonmode?: "extended"; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + enhance: { + parameters: { + query: { + /** @description Diffbot entity type */ + type: "Person" | "Organization"; + /** @description DiffbotId of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + id?: string; + /** @description Name of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + name?: string[]; + /** @description Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + url?: string[]; + /** @description Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + phone?: string; + /** @description Email of the entity to enhance. Parameter can be used only with type `Person` */ + email?: string[]; + /** @description Employer of the entity to enhance. Parameter can be used only with type `Person` */ + employer?: string; + /** @description Title of the entity to enhance. Parameter can be used only with type `Person` */ + title?: string; + /** @description School of the entity to enhance. Parameter can be used only with type `Person` */ + school?: string; + /** @description Location of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + location?: string; + /** @description Description of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + description?: string; + /** @description IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + ip?: string; + /** @description User defined ID for correlation */ + customId?: string; + /** @description Maximum number of results to return */ + size?: number; + /** @description Enhance similarity threshold */ + threshold?: number; + /** @description `refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data. */ + refresh?: boolean; + /** @description `search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG. */ + search?: boolean; + /** @description `nonCanonicalFacts=true` returns non-canonical facts. */ + nonCanonicalFacts?: boolean; + /** @description `jsonmode=extended` returns origin information for facts. */ + jsonmode?: "extended"; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Enhanced entity */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnhanceResponse"]; + }; + }; + /** @description Error parsing request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Insufficient credits */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + enhancePost: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/json": components["schemas"]["EnhancePostRequest"]; + }; + }; + responses: { + /** @description Enhanced entity */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnhanceResponse"]; + }; + }; + /** @description Error parsing request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Insufficient credits */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + combine: { + parameters: { + query: { + /** @description Diffbot entity type */ + type: "Person"; + /** @description Name of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + name?: string[]; + /** @description Origin or homepage URI of entity to enhance. Parameter can be used with types `Person` and `Organization` */ + url?: string[]; + /** @description Phone of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + phone?: string; + /** @description Email of the entity to enhance. Parameter can be used only with type `Person` */ + email?: string; + /** @description Employer of the entity to enhance. Parameter can be used only with type `Person` */ + employer?: string; + /** @description Title of the entity to enhance. Parameter can be used only with type `Person` */ + title?: string; + /** @description School of the entity to enhance. Parameter can be used only with type `Person` */ + school?: string; + /** @description Location of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + location?: string; + /** @description IP address of the entity to enhance. Parameter can be used with types `Person` and `Organization` */ + ip?: string; + /** @description User defined ID for correlation */ + customId?: string; + /** @description Enhance similarity threshold */ + threshold?: number; + /** @description `refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data. */ + refresh?: boolean; + /** @description `search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG. */ + search?: boolean; + /** @description `nonCanonicalFacts=true` returns non-canonical facts. */ + nonCanonicalFacts?: boolean; + /** @description `jsonmode=extended` returns origin information for facts. */ + jsonmode?: "extended"; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Enhanced entity */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnhanceResponse"]; + }; + }; + /** @description Error parsing request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Insufficient credits */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + submitBulkjob: { + parameters: { + query?: { + /** @description Enhance Bulkjob name */ + name?: string; + /** @description Maximum number of results to return */ + size?: number; + /** @description Enhance similarity threshold */ + threshold?: number; + /** @description `refresh=true` indicates that Diffbot will attempt to recrawl all the origins of the identified entity and reconstruct the returned entity from this refreshed data. */ + refresh?: boolean; + /** @description `search=true` indicates that Diffbot will attempt to search the web for origins for the search query and merge relevant results with whats found in the KG. */ + search?: boolean; + /** @description `nonCanonicalFacts=true` returns non-canonical facts. */ + nonCanonicalFacts?: boolean; + /** @description `jsonmode=extended` returns origin information for facts. */ + jsonmode?: "extended"; + /** @description Webhook URL */ + webhookurl?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + /** @description Bulk query payload */ + requestBody?: { + content: { + "application/json": (components["schemas"]["OrganizationRequest"] | components["schemas"]["PersonRequest"] | components["schemas"]["IdRequestEntity"])[]; + }; + }; + responses: { + /** @description Acknowledgement with bulkjobId */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkjobAccepted"]; + }; + }; + /** @description Error parsing request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Insufficient credits */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + bulkjobStatusForToken: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Success status */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkjobStatus"][]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + bulkjobStatus: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Bulkjob complete */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkjobStatusResponse"]; + }; + }; + /** @description Bulkjob is still executing or is stopped */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkjobStatusResponse"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Bulkjob not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + bulkjobResultGet: { + parameters: { + query?: { + /** @description Return first n results */ + head?: number; + /** @description 1-based index of the first result to return (1 = first result). */ + from?: number; + /** @description Maximum number of results to return starting from the 'from' position. */ + size?: number; + /** @description Return only records that have a match */ + onlyMatches?: "true" | "false"; + /** @description Export format */ + format?: "json" | "jsonl" | "csv" | "xls" | "xlsx"; + /** + * @description The spec defines the columns to export. This is applicable for csv, xls and xslx formats. + * + * Export spec specifies how to export the entity JSON to a csv. A simple export spec looks like `name;summary`. To specify more complex export spec, including specifying how many of a list of `industries` to return, or the ideal `employment` record, see [Exporting Columnar Format](https://docs.diffbot.com/reference/exporting-csv) and [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields). + */ + exportspec?: string; + /** @description File name of the export file */ + exportfile?: string; + /** @description Separator for multi-value fields when exporting columnar results */ + exportseparator?: string; + /** @description Prefixes the enhance query parameters to the CSV export result */ + exportquery?: "true" | "false"; + /** @description Semi-colon separated path filter to filter response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filter?: string; + /** @description Semi-colon separated path filter to filter and exclude data from response json. You can use a simple dot notation `skills.name` or a JsonPath expression such as `$.name;$.locations.country.name`. For more advanced options, see [JsonPath Filtering](https://docs.diffbot.com/reference/filtering-fields) for syntax. */ + filterExclude?: string; + /** @description Seconds to wait for bulkjob results to export. Results will continue to export in the background. To only trigger an export, use 0. */ + wait?: number; + }; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Result data. Depends on format (default is json lines). */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["JsonNode"]; + "application/json-lines": Record; + "text/csv": Record; + "application/ms-excel": Record; + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": Record; + }; + }; + /** @description Bulkjob or result export is still executing or is stopped */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Invalid bulkjobId */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Bulkjob not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + bulkjobResultPost: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + }; + cookie?: never; + }; + /** @description Bulk results request */ + requestBody?: { + content: { + "application/json": components["schemas"]["BulkjobResultPostRequest"]; + }; + }; + responses: { + /** @description Result data. Depends on format (default is json lines). */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["JsonNode"]; + "application/json-lines": Record; + "text/csv": Record; + "application/ms-excel": Record; + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": Record; + }; + }; + /** @description Bulkjob is still executing or is stopped */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Invalid bulkjobId */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Bulkjob not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + coverageReportGet: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + /** @description Enhance Report ID */ + reportId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Coverage report (csv) */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "text/csv": Record; + "application/json": components["schemas"]["JsonNode"]; + }; + }; + /** @description Report generation still in progress */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Report not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + deleteBulkjob: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Bulkjob was successfully deleted */ + 200: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Invalid bulkjobId */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Bulkjob not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + singleResult: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + /** @description Job index within the bulkjob */ + jobIdx: number; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Enhance result */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["EnhanceResponse"]; + }; + }; + /** @description Bulkjob is still executing or is stopped */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Invalid bulkjobId */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Bulkjob not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; + stopBulkjob: { + parameters: { + query?: never; + header?: never; + path: { + /** @description Enhance Bulkjob ID */ + bulkjobId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Bulkjob status after the stop request */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BulkjobRecoveryStatusResponse"]; + }; + }; + /** @description Invalid bulkjobId */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Token not specified or other client errors */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AuthError"]; + }; + }; + /** @description Bulkjob not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["StatusResponseError"]; + }; + }; + }; + }; +} diff --git a/src/generated/extract.ts b/src/generated/extract.ts new file mode 100644 index 0000000..9a5310e --- /dev/null +++ b/src/generated/extract.ts @@ -0,0 +1,2338 @@ +/** + * Generated from openapi/extract.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/analyze": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Analyze + * @description Automatically classify a page and extract data according to its type. + */ + get: operations["extract-analyze"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/article": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Article + * @description Automatically extract clean article text and other data from news articles, blog posts and other text-heavy pages. + */ + get: operations["article"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/product": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Product + * @description Automatically extract pricing, product specs, images, and more from an e-commerce product page. + */ + get: operations["product"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/discussion": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Discussion + * @description Automatically structure and extract entire threads of reviews/comments from articles, product pages, and forum threads. + */ + get: operations["discussion"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/job": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Job + * @description Automatically extracts structured information from job postings. + */ + get: operations["job"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/image": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Image + * @description Automatically identifies the primary image(s) on any web page and returns comprehensive information and metadata for each image. + */ + get: operations["image"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/video": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Video + * @description The Video API automatically extracts detailed video information—including most metadata, thumbnail images, direct video URL and embed code from nearly any video page or video platform on the web. + */ + get: operations["video"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/list": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * List + * @description Automatically structures a list of items from news index pages, product listings pages, search engine results pages, and other "list-like" pages. + */ + get: operations["list"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/event": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Event + * @description Automatically extracts dates, location and address information, images and event descriptions from event pages. + */ + get: operations["event"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/{api}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Extract with Custom API + * @description Extracts a page using a modified Extract API or a custom ruleset. + */ + get: operations["customget"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/custom": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Retrieve Custom APIs + * @description Get all the Custom APIs and their rules currently defined on your token + */ + get: operations["retrieve-a-custom-api"]; + put?: never; + /** + * Create or Update a Custom API + * @description Create or update the parameters and ruleset of an existing Custom API + */ + post: operations["create-a-custom-api"]; + /** + * Delete a Custom API + * @description Delete definitions of existing Custom APIs for a given URL pattern and API on your token + */ + delete: operations["delete-a-custom-api"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: never; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + "extract-analyze": { + parameters: { + query: { + /** @description Web page URL of the analyze to process (URL encoded) */ + url: string; + /** @description By default the Analyze API will fully extract all pages that match an existing Extract API. Set mode to a specific Extract API (e.g., `mode="article"`) to extract content only from that specific page-type. All other pages will simply return the default Analyze fields. */ + mode?: "article" | "product" | "discussion" | "image" | "video" | "list" | "event"; + /** @description If an appropriate API cannot be determined (pages classified with type "other"), fall back to this API. */ + fallback?: "article" | "product" | "discussion" | "image" | "video" | "list" | "event"; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Pass `discussion=false` to disable automatic extraction of comments or reviews from pages identified as articles or products. This will not affect pages identified as discussions. */ + discussion?: boolean; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + * "api": "analyze", + * "version": 3 + * }, + * "humanLanguage": "en", + * "objects": [ + * { + * "date": "Fri, 04 Sep 2020 00:00:00 GMT", + * "sentiment": 0.153, + * "images": [ + * { + * "naturalHeight": 869, + * "width": 654, + * "diffbotUri": "image|3|1663647584", + * "url": "https://wp.technologyreview.com/wp-content/uploads/2022/03/Flower-Trip-style.jpeg?resize=1006,640", + * "naturalWidth": 1366, + * "height": 418 + * }, + * { + * "naturalHeight": 1900, + * "width": 460, + * "diffbotUri": "image|3|683243517", + * "url": "https://wp.technologyreview.com/wp-content/uploads/2022/02/MA22_Demis-Hassabis-99-v1.jpg?resize=1006,1400", + * "naturalWidth": 1366, + * "height": 294 + * } + * ], + * "author": "Will Douglas Heaven", + * "estimatedDate": "Fri, 04 Sep 2020 00:00:00 GMT", + * "publisherRegion": "North America", + * "icon": "https://www.technologyreview.com/static/media/favicon.1cfcdb44.ico", + * "diffbotUri": "article|3|973247980", + * "siteName": "MIT Technology Review", + * "type": "article", + * "title": "This know-it-all AI learns by reading the entire web nonstop", + * "tags": [ + * { + * "score": 0.998680055141449, + * "sentiment": 0, + * "count": 10, + * "label": "artificial intelligence", + * "uri": "https://diffbot.com/entity/E_lYDrjmAMlKKwXaDf958zg", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Skill", + * "http://dbpedia.org/ontology/Activity" + * ] + * }, + * { + * "score": 0.9686350226402283, + * "sentiment": 0.889, + * "count": 7, + * "label": "Diffbot", + * "uri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Organisation" + * ] + * }, + * { + * "score": 0.9306924939155579, + * "sentiment": 0, + * "count": 2, + * "label": "Michigan", + * "uri": "https://diffbot.com/entity/E2eIrTt0jPUmGmuV6N2O3KQ", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Place", + * "http://dbpedia.org/ontology/PopulatedPlace", + * "http://dbpedia.org/ontology/State" + * ] + * }, + * { + * "score": 0.9025880098342896, + * "sentiment": 0, + * "count": 1, + * "label": "Paul Katsen", + * "uri": "https://diffbot.com/entity/EqUim_ci0ObmrK2gZM3UfNA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Person" + * ] + * }, + * { + * "score": 0.8933213353157043, + * "sentiment": 0.48, + * "count": 4, + * "label": "Katy Perry", + * "uri": "https://diffbot.com/entity/E_6rhi_PEOD6vGencwOxd2A", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Person" + * ] + * }, + * { + * "score": 0.8848651051521301, + * "sentiment": 0, + * "count": 4, + * "label": "Mike Tung", + * "uri": "https://diffbot.com/entity/ESGMaGV9uP0SuTmfPTtNEoA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Person" + * ] + * }, + * { + * "score": 0.8562507629394531, + * "sentiment": 0, + * "count": 4, + * "label": "Google", + * "uri": "https://diffbot.com/entity/EUFq-3WlpNsq0pvfUYWXOEA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Organisation" + * ] + * }, + * { + * "score": 0.7750672101974487, + * "sentiment": 0, + * "count": 2, + * "label": "Alaska", + * "uri": "https://diffbot.com/entity/E4odwkG_xMNeZTbHrnNrojA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Place", + * "http://dbpedia.org/ontology/PopulatedPlace", + * "http://dbpedia.org/ontology/State" + * ] + * }, + * { + * "score": 0.7653270959854126, + * "sentiment": 0, + * "count": 1, + * "label": "Zola", + * "uri": "https://diffbot.com/entity/E0qGTA2o5NjaeezggjMsoVw", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Organisation" + * ] + * }, + * { + * "score": 0.7643865942955017, + * "sentiment": 0.75, + * "count": 3, + * "label": "GUID Partition Table", + * "uri": "https://diffbot.com/entity/EReKbXuSJMYmoM8lawtgEsA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Skill", + * "http://dbpedia.org/ontology/Activity" + * ] + * } + * ], + * "publisherCountry": "United States", + * "humanLanguage": "en", + * "authorUrl": "https://www.technologyreview.com/author/will-douglas-heaven/", + * "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + * "html": "
\"knowledge
\n

Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.

\n

In one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.

\n

Except that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.

\n

Language models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”

\n

This is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.

\n

Like GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.

\n

Pointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.

\n

Knowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.

\n

A few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.

\n

But Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.

\n

Alongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”

\n

Super surfer

\n

To collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.

\n

Every three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”

\n

Browsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.

\n

Diffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.

\n

Researchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.

\n

Fake shoes

\n

Adidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.

\n

For now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.

\n

Tung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.

\n

Still, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”

\n
\"NLP
\n
\"Demis
", + * "categories": [ + * { + * "score": 0.962, + * "name": "Technology & Computing", + * "id": "iabv2-596" + * }, + * { + * "score": 0.962, + * "name": "Artificial Intelligence", + * "id": "iabv2-597" + * } + * ], + * "text": "Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.\nIn one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.\nExcept that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.\nLanguage models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”\nThis is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.\nLike GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.\nPointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.\nKnowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.\nA few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.\nBut Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.\nAlongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”\nSuper surfer\nTo collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.\nEvery three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”\nBrowsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.\nDiffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.\nResearchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.\nFake shoes\nAdidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.\nFor now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.\nTung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.\nStill, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”", + * "authors": [ + * { + * "name": "Will Douglas Heavenarchive page", + * "link": "technologyreview.com/author/will-douglas-heaven" + * } + * ] + * } + * ], + * "type": "article", + * "title": "This know-it-all AI learns by reading the entire web nonstop | MIT Technology Review" + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + humanLanguage?: string; + objects?: { + date?: string; + sentiment?: number; + images?: { + naturalHeight?: number; + width?: number; + diffbotUri?: string; + url?: string; + naturalWidth?: number; + height?: number; + }[]; + author?: string; + estimatedDate?: string; + publisherRegion?: string; + icon?: string; + diffbotUri?: string; + siteName?: string; + type?: string; + title?: string; + tags?: { + score?: number; + sentiment?: number; + count?: number; + label?: string; + uri?: string; + rdfTypes?: string[]; + }[]; + publisherCountry?: string; + humanLanguage?: string; + authorUrl?: string; + pageUrl?: string; + html?: string; + categories?: { + score?: number; + name?: string; + id?: string; + }[]; + text?: string; + authors?: { + name?: string; + link?: string; + }[]; + }[]; + type?: string; + title?: string; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + article: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb" | "quotes"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + /** @description Pass `paging=true` to enable automatic concatenation for multi-page articles. (default: false) */ + paging?: boolean; + /** @description Set the maximum number of automatically-generated tags to return. (Default: 10) */ + maxTags?: number; + /** @description Set the minimum relevance `score` of tags to return, between 0.0 and 1.0. By default only tags with a `score` equal to or above 0.5 will be returned. */ + tagConfidence?: number; + /** @description Set the minimum relevance `score` of categories to return, between 0.0 and 1.0. By default only categories with a `score` equal to or above 0.5 will be returned. */ + categoryConfidence?: number; + /** @description Pass `discussion=false` to disable automatic extraction of article comments. */ + discussion?: boolean; + /** @description Run extracted text and title through the Diffbot Natural Language API. Example: `&naturalLanguage=entities,facts,categories,sentiment`. */ + naturalLanguage?: "entities" | "sentiment" | "summary" | "facts" | "openFacts" | "records" | "categories" | "sentences" | "language"; + /** @description Sets the maximum number of sentences for summary generation when using naturalLanguage=summary (Default: 3). */ + summaryNumSentences?: number; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + * "api": "article", + * "version": 3 + * }, + * "humanLanguage": "en", + * "objects": [ + * { + * "date": "Fri, 04 Sep 2020 00:00:00 GMT", + * "sentiment": 0.153, + * "images": [ + * { + * "naturalHeight": 869, + * "width": 654, + * "diffbotUri": "image|3|1663647584", + * "url": "https://wp.technologyreview.com/wp-content/uploads/2022/03/Flower-Trip-style.jpeg?resize=1006,640", + * "naturalWidth": 1366, + * "height": 418 + * } + * ], + * "author": "Will Douglas Heaven", + * "estimatedDate": "Fri, 04 Sep 2020 00:00:00 GMT", + * "publisherRegion": "North America", + * "icon": "https://www.technologyreview.com/static/media/favicon.1cfcdb44.ico", + * "diffbotUri": "article|3|973247980", + * "siteName": "MIT Technology Review", + * "type": "article", + * "title": "This know-it-all AI learns by reading the entire web nonstop", + * "tags": [ + * { + * "score": 0.998680055141449, + * "sentiment": 0, + * "count": 10, + * "label": "artificial intelligence", + * "uri": "https://diffbot.com/entity/E_lYDrjmAMlKKwXaDf958zg", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Skill", + * "http://dbpedia.org/ontology/Activity" + * ] + * }, + * { + * "score": 0.9686350226402283, + * "sentiment": 0.889, + * "count": 7, + * "label": "Diffbot", + * "uri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Organisation" + * ] + * }, + * { + * "score": 0.9306924939155579, + * "sentiment": 0, + * "count": 2, + * "label": "Michigan", + * "uri": "https://diffbot.com/entity/E2eIrTt0jPUmGmuV6N2O3KQ", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Place", + * "http://dbpedia.org/ontology/PopulatedPlace", + * "http://dbpedia.org/ontology/State" + * ] + * }, + * { + * "score": 0.9025880098342896, + * "sentiment": 0, + * "count": 1, + * "label": "Paul Katsen", + * "uri": "https://diffbot.com/entity/EqUim_ci0ObmrK2gZM3UfNA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Person" + * ] + * }, + * { + * "score": 0.8933213353157043, + * "sentiment": 0.48, + * "count": 4, + * "label": "Katy Perry", + * "uri": "https://diffbot.com/entity/E_6rhi_PEOD6vGencwOxd2A", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Person" + * ] + * }, + * { + * "score": 0.8848651051521301, + * "sentiment": 0, + * "count": 4, + * "label": "Mike Tung", + * "uri": "https://diffbot.com/entity/ESGMaGV9uP0SuTmfPTtNEoA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Person" + * ] + * }, + * { + * "score": 0.8562507629394531, + * "sentiment": 0, + * "count": 4, + * "label": "Google", + * "uri": "https://diffbot.com/entity/EUFq-3WlpNsq0pvfUYWXOEA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Organisation" + * ] + * }, + * { + * "score": 0.7750672101974487, + * "sentiment": 0, + * "count": 2, + * "label": "Alaska", + * "uri": "https://diffbot.com/entity/E4odwkG_xMNeZTbHrnNrojA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Place", + * "http://dbpedia.org/ontology/PopulatedPlace", + * "http://dbpedia.org/ontology/State" + * ] + * }, + * { + * "score": 0.7653270959854126, + * "sentiment": 0, + * "count": 1, + * "label": "Zola", + * "uri": "https://diffbot.com/entity/E0qGTA2o5NjaeezggjMsoVw", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Organisation" + * ] + * }, + * { + * "score": 0.7643865942955017, + * "sentiment": 0.75, + * "count": 3, + * "label": "GUID Partition Table", + * "uri": "https://diffbot.com/entity/EReKbXuSJMYmoM8lawtgEsA", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Skill", + * "http://dbpedia.org/ontology/Activity" + * ] + * } + * ], + * "publisherCountry": "United States", + * "humanLanguage": "en", + * "authorUrl": "https://www.technologyreview.com/author/will-douglas-heaven/", + * "pageUrl": "https://www.technologyreview.com/2020/09/04/1008156/knowledge-graph-ai-reads-web-machine-learning-natural-language-processing/", + * "html": "
\"knowledge
\n

Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.

\n

In one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.

\n

Except that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.

\n

Language models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”

\n

This is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.

\n

Like GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.

\n

Pointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.

\n

Knowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.

\n

A few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.

\n

But Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.

\n

Alongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”

\n

Super surfer

\n

To collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.

\n

Every three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”

\n

Browsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.

\n

Diffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.

\n

Researchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.

\n

Fake shoes

\n

Adidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.

\n

For now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.

\n

Tung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.

\n

Still, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”

\n
\"NLP
\n
\"Demis
", + * "categories": [ + * { + * "score": 0.962, + * "name": "Technology & Computing", + * "id": "iabv2-596" + * }, + * { + * "score": 0.962, + * "name": "Artificial Intelligence", + * "id": "iabv2-597" + * } + * ], + * "text": "Back in July, OpenAI’s latest language model, GPT-3, dazzled with its ability to churn out paragraphs that look as if they could have been written by a human. People started showing off how GPT-3 could also autocomplete code or fill in blanks in spreadsheets.\nIn one example, Twitter employee Paul Katsen tweeted “the spreadsheet function to rule them all,” in which GPT-3 fills out columns by itself, pulling in data for US states: the population of Michigan is 10.3 million, Alaska became a state in 1906, and so on.\nExcept that GPT-3 can be a bit of a bullshitter. The population of Michigan has never been 10.3 million, and Alaska became a state in 1959.\nLanguage models like GPT-3 are amazing mimics, but they have little sense of what they’re actually saying. “They’re really good at generating stories about unicorns,” says Mike Tung, CEO of Stanford startup Diffbot. “But they’re not trained to be factual.”\nThis is a problem if we want AIs to be trustworthy. That’s why Diffbot takes a different approach. It is building an AI that reads every page on the entire public web, in multiple languages, and extracts as many facts from those pages as it can.\nLike GPT-3, Diffbot’s system learns by vacuuming up vast amounts of human-written text found online. But instead of using that data to train a language model, Diffbot turns what it reads into a series of three-part factoids that relate one thing to another: subject, verb, object.\nPointed at my bio, for example, Diffbot learns that Will Douglas Heaven is a journalist; Will Douglas Heaven works at MIT Technology Review; MIT Technology Review is a media company; and so on. Each of these factoids gets joined up with billions of others in a sprawling, interconnected network of facts. This is known as a knowledge graph.\nKnowledge graphs are not new. They have been around for decades, and were a fundamental concept in early AI research. But constructing and maintaining knowledge graphs has typically been done by hand, which is hard. This also stopped Tim Berners-Lee from realizing what he called the semantic web, which would have included information for machines as well as humans, so that bots could book our flights, do our shopping, or give smarter answers to questions than search engines.\nA few years ago, Google started using knowledge graphs too. Search for “Katy Perry” and you will get a box next to the main search results telling you that Katy Perry is an American singer-songwriter with music available on YouTube, Spotify, and Deezer. You can see at a glance that she is married to Orlando Bloom, she’s 35 and worth $125 million, and so on. Instead of giving you a list of links to pages about Katy Perry, Google gives you a set of facts about her drawn from its knowledge graph.\nBut Google only does this for its most popular search terms. Diffbot wants to do it for everything. By fully automating the construction process, Diffbot has been able to build what may be the largest knowledge graph ever.\nAlongside Google and Microsoft, it is one of only three US companies that crawl the entire public web. “It definitely makes sense to crawl the web,” says Victoria Lin, a research scientist at Salesforce who works on natural-language processing and knowledge representation. “A lot of human effort can otherwise go into making a large knowledge base.” Heiko Paulheim at the University of Mannheim in Germany agrees: “Automation is the only way to build large-scale knowledge graphs.”\nSuper surfer\nTo collect its facts, Diffbot’s AI reads the web as a human would—but much faster. Using a super-charged version of the Chrome browser, the AI views the raw pixels of a web page and uses image-recognition algorithms to categorize the page as one of 20 different types, including video, image, article, event, and discussion thread. It then identifies key elements on the page, such as headline, author, product description, or price, and uses NLP to extract facts from any text.\nEvery three-part factoid gets added to the knowledge graph. Diffbot extracts facts from pages written in any language, which means that it can answer queries about Katy Perry, say, using facts taken from articles in Chinese or Arabic even if they do not contain the term “Katy Perry.”\nBrowsing the web like a human lets the AI see the same facts that we see. It also means it has had to learn to navigate the web like us. The AI must scroll down, switch between tabs, and click away pop-ups. “The AI has to play the web like a video game just to experience the pages,” says Tung.\nDiffbot crawls the web nonstop and rebuilds its knowledge graph every four to five days. According to Tung, the AI adds 100 million to 150 million entities each month as new people pop up online, companies are created, and products are launched. It uses more machine-learning algorithms to fuse new facts with old, creating new connections or overwriting out-of-date ones. Diffbot has to add new hardware to its data center as the knowledge graph grows.\nResearchers can access Diffbot’s knowledge graph for free. But Diffbot also has around 400 paying customers. The search engine DuckDuckGo uses it to generate its own Google-like boxes. Snapchat uses it to extract highlights from news pages. The popular wedding-planner app Zola uses it to help people make wedding lists, pulling in images and prices. NASDAQ, which provides information about the stock market, uses it for financial research.\nFake shoes\nAdidas and Nike even use it to search the web for counterfeit shoes. A search engine will return a long list of sites that mention Nike trainers. But Diffbot lets these companies look for sites that are actually selling their shoes, rather just talking about them.\nFor now, these companies must interact with Diffbot using code. But Tung plans to add a natural-language interface. Ultimately, he wants to build what he calls a “universal factoid question answering system”: an AI that could answer almost anything you asked it, with sources to back up its response.\nTung and Lin agree that this kind of AI cannot be built with language models alone. But better yet would be to combine the technologies, using a language model like GPT-3 to craft a human-like front end for a know-it-all bot.\nStill, even an AI that has its facts straight is not necessarily smart. “We’re not trying to define what intelligence is, or anything like that,” says Tung. “We’re just trying to build something useful.”", + * "authors": [ + * { + * "name": "Will Douglas Heavenarchive page", + * "link": "technologyreview.com/author/will-douglas-heaven" + * } + * ] + * } + * ], + * "type": "article", + * "title": "This know-it-all AI learns by reading the entire web nonstop | MIT Technology Review" + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + humanLanguage?: string; + objects?: { + date?: string; + sentiment?: number; + images?: { + naturalHeight?: number; + width?: number; + diffbotUri?: string; + url?: string; + naturalWidth?: number; + height?: number; + }[]; + author?: string; + estimatedDate?: string; + publisherRegion?: string; + icon?: string; + diffbotUri?: string; + siteName?: string; + type?: string; + title?: string; + tags?: { + score?: number; + sentiment?: number; + count?: number; + label?: string; + uri?: string; + rdfTypes?: string[]; + }[]; + publisherCountry?: string; + humanLanguage?: string; + authorUrl?: string; + pageUrl?: string; + html?: string; + categories?: { + score?: number; + name?: string; + id?: string; + }[]; + text?: string; + authors?: { + name?: string; + link?: string; + }[]; + }[]; + type?: string; + title?: string; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + product: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + /** @description Pass `discussion=false` to disable automatic extraction of product reviews. */ + discussion?: boolean; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.microcenter.com/product/628738/evga-nvidia-geforce-rtx-3090-ftw3-ultra-triple-fan-24gb-gddr6x-pcie-40-graphics-card", + * "api": "product", + * "version": 3 + * }, + * "humanLanguage": "en", + * "objects": [ + * { + * "images": [ + * { + * "xpath": "/html[1]/body[1]/main[1]/article[1]/div[3]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/div[1]/a[1]/img[1]", + * "naturalHeight": 200, + * "width": 260, + * "diffbotUri": "image|3|735477840", + * "title": "Product Image View 2", + * "url": "https://90a1c75758623581b3f8-5c119c3de181c9857fcb2784776b17ef.ssl.cf2.rackcdn.com/628738_177923_01_front_thumbnail.jpg", + * "naturalWidth": 200, + * "primary": true, + * "height": 260 + * } + * ], + * "offerPrice": "$1,919.99", + * "productId": "843368067137", + * "diffbotUri": "product|3|-1003574237", + * "upc": "843368067137", + * "productOrigin": "us", + * "mpn": "24G-P5-3987-KR", + * "gs1Code": "U.S. and Canada", + * "multipleProducts": true, + * "availability": true, + * "type": "product", + * "title": "EVGA NVIDIA GeForce RTX 3090 FTW3 Ultra Triple-Fan 24GB GDDR6X PCIe 4.0 Graphics Card", + * "offerPriceDetails": { + * "symbol": "$", + * "amount": 1919.99, + * "text": "$1,919.99" + * }, + * "specs": { + * "mfr_part_": "24G-P5-3987-KR", + * "upc": "843368067137", + * "sku": "177923" + * }, + * "normalizedSpecs": { + * "sku": [ + * { + * "cleanLiteral": "843368067137" + * } + * ] + * }, + * "humanLanguage": "en", + * "pageUrl": "https://www.microcenter.com/product/628738/evga-nvidia-geforce-rtx-3090-ftw3-ultra-triple-fan-24gb-gddr6x-pcie-40-graphics-card", + * "text": "The EVGA GeForce RTX 3090 is colossally powerful in every way imaginable, giving you a whole new tier of performance at 8K resolution. It's powered by the NVIDIA Ampere architecture, which doubles down on ray tracing and AI performance with enhanced RT Cores, Tensor Cores, and new streaming multiprocessors. Combined with the next generation of design, cooling, and overclocking with EVGA Precision X1, the EVGA GeForce RTX 3090 Series redefines the definition of ultimate performance.", + * "category": "Computers", + * "sku": "177923", + * "brand": "EVGA" + * } + * ], + * "type": "product", + * "title": "EVGA NVIDIA GeForce RTX 3090 FTW3 Ultra Triple-Fan 24GB GDDR6X PCIe 4.0 Graphics Card - Micro Center" + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + humanLanguage?: string; + objects?: { + images?: { + xpath?: string; + naturalHeight?: number; + width?: number; + diffbotUri?: string; + title?: string; + url?: string; + naturalWidth?: number; + primary?: boolean; + height?: number; + }[]; + offerPrice?: string; + productId?: string; + diffbotUri?: string; + upc?: string; + productOrigin?: string; + mpn?: string; + gs1Code?: string; + multipleProducts?: boolean; + availability?: boolean; + type?: string; + title?: string; + offerPriceDetails?: { + symbol?: string; + amount?: number; + text?: string; + }; + specs?: { + mfr_part_?: string; + upc?: string; + sku?: string; + }; + normalizedSpecs?: { + sku?: { + cleanLiteral?: string; + }[]; + }; + humanLanguage?: string; + pageUrl?: string; + text?: string; + category?: string; + sku?: string; + brand?: string; + }[]; + type?: string; + title?: string; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + discussion: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb" | "sentiment"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + /** @description Pass `paging=true` to enable automatic concatenation for multi-page articles. (default: false) */ + paging?: boolean; + /** @description Set the maximum number of pages in a thread to automatically concatenate in a single response. Default = 1 (no concatenation). Set `maxPages=all` to retrieve all pages of a thread regardless of length. Each individual page will count as a separate API call. */ + maxPages?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "api": "discussion", + * "version": 3 + * }, + * "objects": [ + * { + * "numPages": 1, + * "humanLanguage": "en", + * "confidence": 0.05500000089407453, + * "diffbotUri": "discussion|3|-870809033", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "numPosts": 13, + * "type": "discussion", + * "title": "[OC] 66% of Top 50 Russian Exposed Companies Have Announced Sanctions", + * "posts": [ + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "images": [ + * { + * "naturalHeight": 767, + * "width": 457, + * "diffbotUri": "image|3|-804821395", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "url": "https://preview.redd.it/l76k59t8jsm81.png?width=457&auto=webp&s=632efa1f24e607358bbec99c161a6aa579aebfe1", + * "naturalWidth": 457, + * "height": 767 + * } + * ], + * "humanLanguage": "en", + * "author": "hicheoo", + * "authorUrl": "https://old.reddit.com/user/hicheoo", + * "diffbotUri": "post|3|29462830", + * "html": "
\n

Want to add to the discussion?

\n

Post a comment!

\n

Create an account

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 0, + * "text": "Want to add to the discussion?\nPost a comment!\n\n \nCreate an account", + * "type": "post", + * "title": "[OC] 66% of Top 50 Russian Exposed Companies Have Announced Sanctions" + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "not_mig", + * "authorUrl": "https://old.reddit.com/user/not_mig", + * "diffbotUri": "post|3|-720375378", + * "html": "

What's the difference between blue, yellow, and green?

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 1, + * "text": "What's the difference between blue, yellow, and green?", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "hicheoo", + * "authorUrl": "https://old.reddit.com/user/hicheoo", + * "diffbotUri": "post|3|-148816221", + * "html": "

They're exemptions. I should've clarified up top, but they're basically all in the description.

\n

Green: Typical Sanctions
\n Yellow: Sanctions, but might be a PR move.
\n Blue: Healthcare

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 2, + * "text": "They're exemptions. I should've clarified up top, but they're basically all in the description.\nGreen: Typical Sanctions\nYellow: Sanctions, but might be a PR move.\nBlue: Healthcare", + * "type": "post", + * "parentId": 1 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "Zealousideal-Lie7255", + * "authorUrl": "https://old.reddit.com/user/Zealousideal-Lie7255", + * "diffbotUri": "post|3|-683402068", + * "html": "

A lot of oil service companies have no reported sanctions. Like Schlumberger, Baker Hughes. Some Chinese companies too.

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 3, + * "text": "A lot of oil service companies have no reported sanctions. Like Schlumberger, Baker Hughes. Some Chinese companies too.", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "varnima", + * "authorUrl": "https://old.reddit.com/user/varnima", + * "diffbotUri": "post|3|-603833918", + * "html": "

JetBrains changed and imposed sanctions https://blog.jetbrains.com/blog/2022/03/11/jetbrains-statement-on-ukraine/

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 4, + * "text": "JetBrains changed and imposed sanctions https://blog.jetbrains.com/blog/2022/03/11/jetbrains-statement-on-ukraine/", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "hicheoo", + * "authorUrl": "https://old.reddit.com/user/hicheoo", + * "diffbotUri": "post|3|-296888207", + * "html": "

Yeah, they're green in the chart.

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 5, + * "text": "Yeah, they're green in the chart.", + * "type": "post", + * "parentId": 4 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "hicheoo", + * "authorUrl": "https://old.reddit.com/user/hicheoo", + * "diffbotUri": "post|3|624793084", + * "html": "

Sources: - Diffbot Sanctions Tracker (https://www.diffbot.com/insights/every-company-affected-by-sanctions/) - Diffbot Knowledge Graph (more detail on query below)

\n

Data Viz Tool: Infogram

\n

Disclaimer: I work for Diffbot

\n

I started by querying the Knowledge Graph for people who live in Russia but work for a non-Russian company. Faceting this query by their employer provides me with a list of non-Russian companies ranked by # of Russian employees.

\n

\ntype:Person location.country.name:"Russia" employments.{employer.{location.country.name!="Russia" nbLocations>0} isCurrent:true} facet:employments.{employer.name isCurrent:true}\n

\n

This data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 6, + * "text": "Sources: - Diffbot Sanctions Tracker (https://www.diffbot.com/insights/every-company-affected-by-sanctions/) - Diffbot Knowledge Graph (more detail on query below)\nData Viz Tool: Infogram\nDisclaimer: I work for Diffbot\nI started by querying the Knowledge Graph for people who live in Russia but work for a non-Russian company. Faceting this query by their employer provides me with a list of non-Russian companies ranked by # of Russian employees.\ntype:Person location.country.name:\"Russia\" employments.{employer.{location.country.name!=\"Russia\" nbLocations>0} isCurrent:true} facet:employments.{employer.name isCurrent:true}\nThis data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "zzzmick", + * "authorUrl": "https://old.reddit.com/user/zzzmick", + * "diffbotUri": "post|3|-130810969", + * "html": "

epam had over 10k employees in Russia

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 7, + * "text": "epam had over 10k employees in Russia", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "hicheoo", + * "authorUrl": "https://old.reddit.com/user/hicheoo", + * "diffbotUri": "post|3|-1458692070", + * "html": "

Yup. The data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 8, + * "text": "Yup. The data underrepresents actual employment figures, as there are many employees who do not maintain an internet presence linking them to their employer. Underrepresentation should be fairly equal across all companies, and relative position in the rankings should be accurate.", + * "type": "post", + * "parentId": 7 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "JanitorKarl", + * "authorUrl": "https://old.reddit.com/user/JanitorKarl", + * "diffbotUri": "post|3|-149138223", + * "html": "

Schlumberger and Baker Hughes are both in the oilfield services industry.

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 9, + * "text": "Schlumberger and Baker Hughes are both in the oilfield services industry.", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "flumenia", + * "authorUrl": "https://old.reddit.com/user/flumenia", + * "diffbotUri": "post|3|889762151", + * "html": "

What if Microsoft stops to extend licenses of Microsoft Office to Russia? That would make the biggest impact, I guess

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 10, + * "text": "What if Microsoft stops to extend licenses of Microsoft Office to Russia? That would make the biggest impact, I guess", + * "type": "post", + * "parentId": 0 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "Imperial_Empirical", + * "authorUrl": "https://old.reddit.com/user/Imperial_Empirical", + * "diffbotUri": "post|3|-179317804", + * "html": "

Putin ordered the development of Russian alternatives after the Crimean annexation due to dependancy/spying fears. I believe from 2016 onwards Microsoft was largely fased out internally.

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 11, + * "text": "Putin ordered the development of Russian alternatives after the Crimean annexation due to dependancy/spying fears. I believe from 2016 onwards Microsoft was largely fased out internally.", + * "type": "post", + * "parentId": 10 + * }, + * { + * "date": "Fri, 11 Mar 2022 00:00:00 GMT", + * "humanLanguage": "en", + * "author": "Nightblood83", + * "authorUrl": "https://old.reddit.com/user/Nightblood83", + * "diffbotUri": "post|3|-901046006", + * "html": "

A lot of accountants for commies...

", + * "pageUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/", + * "id": 12, + * "text": "A lot of accountants for commies...", + * "type": "post", + * "parentId": 0 + * } + * ], + * "tags": [ + * { + * "score": 0.8428076505661011, + * "count": 5, + * "label": "economic sanctions", + * "uri": "https://diffbot.com/entity/EWnXSPtH6Osi0pmx8-WPKAg", + * "rdfTypes": [ + * "http://dbpedia.org/ontology/Miscellaneous" + * ] + * } + * ], + * "participants": 9, + * "rssUrl": "https://old.reddit.com/r/dataisbeautiful/comments/tbvdhu/oc_66_of_top_50_russian_exposed_companies_have/.rss" + * } + * ] + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: { + numPages?: number; + humanLanguage?: string; + confidence?: number; + diffbotUri?: string; + pageUrl?: string; + numPosts?: number; + type?: string; + title?: string; + posts?: { + date?: string; + images?: { + naturalHeight?: number; + width?: number; + diffbotUri?: string; + pageUrl?: string; + url?: string; + naturalWidth?: number; + height?: number; + }[]; + humanLanguage?: string; + author?: string; + authorUrl?: string; + diffbotUri?: string; + html?: string; + pageUrl?: string; + id?: number; + text?: string; + type?: string; + title?: string; + }[]; + tags?: { + score?: number; + count?: number; + label?: string; + uri?: string; + rdfTypes?: string[]; + }[]; + participants?: number; + rssUrl?: string; + }[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + job: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.pathai.com/careers/DirectorClinicalOperations-6715804002", + * "api": "job", + * "version": 3 + * }, + * "objects": [ + * { + * "requirements": [ + * "Experience in histologic-based or other medical imaging in a regulated environment is strongly preferred." + * ], + * "diffbotUri": "job|3|1793551482", + * "type": "job", + * "title": "Director, Clinical Operations", + * "remote": "REMOTE", + * "skills": [ + * { + * "salience": 0.7316089, + * "skill": "DDT", + * "confidence": 0.6169359, + * "diffbotUri": "https://diffbot.com/entity/Ey44naXoPNamQWWt5PK0JEQ" + * } + * ], + * "humanLanguage": "en", + * "employer": { + * "name": "PathAI", + * "diffbotUri": "https://diffbot.com/entity/EIf6kAhRvMq2mBsM6DV2Krw" + * }, + * "pageUrl": "https://www.pathai.com/careers/DirectorClinicalOperations-6715804002", + * "text": "Who We Are\nPathAI is on a mission to improve patient outcomes with AI-powered pathology. We are transforming traditional pathology methods into powerful, new technologies. These innovations in pathology can help accelerate drug development, improve confidence in the accuracy of diagnosis, and get life-saving therapies to patients more quickly. At PathAI, you'll work with a diverse and talented team of people, who are dedicated to solving complex problems and making a huge impact.\nWhere You Fit\nThe Director of Clinical Operations will be responsible for programmatic tactical execution of regulated research and clinical products, including companion diagnostic (CDx), non-CDx-IVD, Drug Development Tools (DDT) and RUO strategies. This is a highly matrixed role that requires outstanding emotional intelligence combined with detailed subject matter expertise in order to help coordinate the program across all functional areas including quality, clinical, business development, scientific programs, engineering and product teams. The Director of Clinical Operations will bring together all stakeholders internally and externally to successfully design, execute on all aspects of our clinical development programs. Outstanding detailed organizational skills with a proactive mindset toward identification of all programmatic risks and mitigations is essential.\nWhat You’ll Do\nLead the execution of research and clinical development strategies for new clinical trial, research, and medical device product development, and operationalize these strategies from development to commercialization.\nAccountable for oversight of all assigned studies within the country in accordance with the overall development plan and clinical operations plan, including adherence to quality, timelines and budget.\nEstablish and continually communicate detailed program and project plans with critical paths and dependencies clearly identified across each area and communicate timeline progress and risks through defined operating mechanisms.\nEducate relevant leads from each functional area on the regulations impacting development programs.\nClearly communicate strategy, progress, and risks cross functionally and proactively drive/facilitate the relationship between internal project teams and external stakeholders.\nEffectively prioritize the projects within our clinical development programs to ensure adequate resourcing and efficient delivery on priority projects for PathAI.\nInform leadership of any identified gaps, risks, solutions to the program as a whole.\nWhat You Bring\nOur employees' skills come in all shapes and sizes, but to be successful in this role with us, you'll at least need:\nPh.D (preferred) or M.S. in a Life Science or equivalent field, plus at least 5-7 years of relevant industry experience, including at least 3 years in a role of primary responsibility for clinical trial execution and management.\nYou have a solid understanding of and experience in drug development (clinical trials, biomarker testing), assay development, and co-development for CDx programs.\nDirect management experience and supervisory responsibility.\nYou have knowledge (training preferred) of the software development life cycle (IEC 62304), design control, and experience in leading in vitro diagnostic development teams.\nYou have supported projects for FDA and other regulatory body submissions from development to validation to approval.\nYou have cross therapeutic area clinical biomarker testing and diagnostic testing experience in global clinical trials.\nYou have experience and a deep understanding of biomarker testing and reporting requirements globally.\nYou excel in leading multidisciplinary teams of scientists and engineers in highly matrixed organization is strongly preferred\nYou are a strategic thinker, pro-active and agile, and you have a strong interest in bringing teams together at PathAI, as well as external partners, to bring better treatments to patients.\nExperience in histologic-based or other medical imaging in a regulated environment is strongly preferred.\nWe Want To Hear From You\nAt PathAI, we are looking for individuals who are team players, are willing to do the work no matter how big or small it may be, and who are passionate about everything they do. If this sounds like you, even if you may not match the job description to a tee, we encourage you to apply. You could be exactly what we're looking for.\nPathAI is an equal opportunity employer, dedicated to creating a workplace that is free of harassment and discrimination. We base our employment decisions on business needs, job requirements, and qualifications — that's all. We do not discriminate based on race, gender, religion, health, personal beliefs, age, family or parental status, or any other status. We don't tolerate any kind of discrimination or bias, and we are looking for teammates who feel the same way.\n#LI-Remote\n", + * "datePosted": "Sat, 07 May 2022 00:00:00 GMT", + * "tasks": [ + * "Lead the execution of research and clinical development strategies for new clinical trial, research, and medical device product development, and operationalize these strategies from development to commercialization.", + * "Accountable for oversight of all assigned studies within the country in accordance with the overall development plan and clinical operations plan, including adherence to quality, timelines and budget.", + * "Establish and continually communicate detailed program and project plans with critical paths and dependencies clearly identified across each area and communicate timeline progress and risks through defined operating mechanisms.\n", + * "Educate relevant leads from each functional area on the regulations impacting development programs.\n", + * "Clearly communicate strategy, progress, and risks cross functionally and proactively drive/facilitate the relationship between internal project teams and external stakeholders.", + * "Effectively prioritize the projects within our clinical development programs to ensure adequate resourcing and efficient delivery on priority projects for PathAI.\n", + * "Inform leadership of any identified gaps, risks, solutions to the program as a whole.\n", + * "Ph.D (preferred) or M.S. in a Life Science or equivalent field, plus at least 5-7 years of relevant industry experience, including at least 3 years in a role of primary responsibility for clinical trial execution and management.", + * "You have a solid understanding of and experience in drug development (clinical trials, biomarker testing), assay development, and co-development for CDx programs.\n", + * "Direct management experience and supervisory responsibility.", + * "You have knowledge (training preferred) of the software development life cycle (IEC 62304), design control, and experience in leading in vitro diagnostic development teams.", + * "You have supported projects for FDA and other regulatory body submissions from development to validation to approval.", + * "You have cross therapeutic area clinical biomarker testing and diagnostic testing experience in global clinical trials.", + * "You have experience and a deep understanding of biomarker testing and reporting requirements globally.", + * "You excel in leading multidisciplinary teams of scientists and engineers in highly matrixed organization is strongly preferred", + * "You are a strategic thinker, pro-active and agile, and you have a strong interest in bringing teams together at PathAI, as well as external partners, to bring better treatments to patients.\n" + * ] + * } + * ] + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: { + requirements?: string[]; + diffbotUri?: string; + type?: string; + title?: string; + postedDate?: string; + remote?: string; + skills?: { + salience?: number; + skill?: string; + confidence?: number; + diffbotUri?: string; + }[]; + humanLanguage?: string; + employer?: { + name?: string; + diffbotUri?: string; + }; + text?: string; + pageUrl?: string; + tasks?: string[]; + locations?: { + country?: { + summary?: string; + image?: string; + types?: string[]; + name?: string; + diffbotUri?: string; + targetDiffbotUri?: string; + targetDiffbotId?: string; + type?: string; + }; + address?: string; + city?: { + summary?: string; + image?: string; + types?: string[]; + name?: string; + diffbotUri?: string; + targetDiffbotUri?: string; + targetDiffbotId?: string; + type?: string; + }; + region?: { + summary?: string; + image?: string; + types?: string[]; + name?: string; + diffbotUri?: string; + targetDiffbotUri?: string; + targetDiffbotId?: string; + type?: string; + }; + subregion?: { + summary?: string; + image?: string; + types?: string[]; + name?: string; + diffbotUri?: string; + targetDiffbotUri?: string; + targetDiffbotId?: string; + type?: string; + }; + latitude?: number; + precision?: number; + longitude?: number; + }[]; + }[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + image: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb" | "displayHeight" | "displayWidth"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.diffbot.com/products/extract/", + * "api": "image", + * "version": 3 + * }, + * "objects": [ + * { + * "xpath": "/HTML/BODY/MAIN/DIV[@id='slice-readslikehumans']/DIV[@class='container px-3 py-5 py-md-6 mx-auto']/DIV[@class='row justify-content-center']/DIV[@class='col-12 col-md d-flex justify-content-center align-items-center order-md-1']/IMG[@class='img-fluid mb-4 align-self-start']", + * "humanLanguage": "en", + * "naturalHeight": 1023, + * "diffbotUri": "image|3|666824882", + * "pageUrl": "https://www.diffbot.com/products/extract/", + * "type": "image", + * "url": "https://www.diffbot.com/assets/img/products/extract_screenshot.png", + * "naturalWidth": 897, + * "tags": [ + * { + * "typeHierarchy": [ + * "http://www.w3.org/2002/07/owl#Thing", + * "http://dbpedia.org/ontology/Work", + * "http://dbpedia.org/ontology/Website" + * ], + * "id": 33898, + * "label": "Website", + * "type": "http://dbpedia.org/ontology/Website", + * "uri": "https://www.diffbot.com/entity/Xd90vp_U4MJOoRHIja3quxg" + * }, + * { + * "id": 1404579, + * "label": "Ring binder", + * "uri": "http://diffbot.com/entity/XYwPij6UgPdaDEY6fXSrrRA" + * } + * ] + * }, + * { + * "xpath": "/HTML/BODY/MAIN/DIV[@id='slice-lookslikeahuman']/DIV[@class='container px-3 py-5 py-md-6 mx-auto']/DIV[@class='row justify-content-center']/DIV[@class='col-12 col-md d-flex justify-content-center align-items-center order-md-1']/IMG[@class='img-fluid mb-4']", + * "humanLanguage": "en", + * "naturalHeight": 270, + * "diffbotUri": "image|3|1865428953", + * "pageUrl": "https://www.diffbot.com/products/extract/", + * "type": "image", + * "url": "https://www.diffbot.com/assets/img/products/any_language.png", + * "naturalWidth": 554, + * "tags": [ + * { + * "id": 5462349, + * "label": "Sachet", + * "uri": "http://diffbot.com/entity/X1_7MIKBoPpuRnAPVQXqQxA" + * }, + * { + * "id": 479373, + * "label": "Eraser", + * "uri": "http://diffbot.com/entity/XGPnWNBUZPyuNyfK9rJr6vQ" + * }, + * { + * "id": 57260, + * "label": "Envelope", + * "uri": "http://diffbot.com/entity/XK5gEV93iP6SqKMDjDr0YBQ" + * } + * ] + * }, + * { + * "xpath": "/HTML/BODY/MAIN/DIV[@id='slice-one-click-crawling']/DIV[@class='container px-3 py-5 py-md-6 mx-auto']/DIV[@class='row justify-content-center']/DIV[@class='col-12 offset-md-1 col-md d-flex justify-content-center align-items-center order-md-2']/IMG[@class='img-fluid mb-4']", + * "humanLanguage": "en", + * "naturalHeight": 372, + * "diffbotUri": "image|3|1699007329", + * "pageUrl": "https://www.diffbot.com/products/extract/", + * "type": "image", + * "url": "https://www.diffbot.com/assets/img/products/analyze_two.png", + * "naturalWidth": 451, + * "tags": [ + * { + * "id": 268267, + * "label": "Tray", + * "uri": "http://diffbot.com/entity/XwXgQ3A7sNmypX2VX3oZCgg" + * }, + * { + * "id": 467731, + * "label": "Spatula", + * "uri": "http://diffbot.com/entity/XZohZtDoDPvq1wusJQCSZZA" + * }, + * { + * "id": 2649730, + * "label": "Measuring cup", + * "uri": "http://diffbot.com/entity/XhwdQ4HnKOsCVrf9YU3hyWw" + * } + * ] + * } + * ] + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: { + xpath?: string; + humanLanguage?: string; + naturalHeight?: number; + diffbotUri?: string; + pageUrl?: string; + type?: string; + url?: string; + naturalWidth?: number; + tags?: ({ + typeHierarchy?: string[]; + id?: number; + label?: string; + type?: string; + uri?: string; + } | { + id?: number; + label?: string; + uri?: string; + })[]; + }[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + video: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.youtube.com/watch?v=hFZFjoX2cGg", + * "api": "video", + * "version": 3 + * }, + * "objects": [ + * { + * "date": "Sun, 24 May 2020 07:00:00 GMT", + * "images": [ + * { + * "diffbotUri": "image|3|231854607", + * "title": "Backyard Squirrel Maze 1.0- Ninja Warrior Course", + * "url": "https://i.ytimg.com/vi/hFZFjoX2cGg/maxresdefault.jpg", + * "primary": true + * } + * ], + * "author": "Mark Rober", + * "mime": "video/mp4", + * "naturalHeight": 720, + * "diffbotUri": "video|3|1870173316", + * "type": "video", + * "title": "Backyard Squirrel Maze 1.0- Ninja Warrior Course", + * "url": "https://rr6---sn-5uaeznks.googlevideo.com/videoplayback?expire=1649742262&ei=Vr1UYoLhFpDm8wTUt76wAQ&ip=23.229.39.25&id=o-AOUTPxsk0l8eAnvqC6G9PtYnkFK4S-lYoQ1G-mt8W40U&itag=399&aitags=133%2C134%2C135%2C136%2C137%2C160%2C242%2C243%2C244%2C247%2C248%2C278%2C394%2C395%2C396%2C397%2C398%2C399&source=youtube&requiressl=yes&mh=s4&mm=31%2C29&mn=sn-5uaeznks%2Csn-5ualdnl7&ms=au%2Crdu&mv=u&mvi=6&pl=23&spc=4ocVC0JfKoZvX0FSPklAcHZ7LBTF&vprv=1&mime=video%2Fmp4&ns=ZSXKPHKFPvaGHi2sm3qErYcG&gir=yes&clen=246339629&dur=1220.093&lmt=1637823031873234&mt=1649720229&fvip=3&keepalive=yes&fexp=24001373%2C24007246&c=WEB&txp=5531432&n=b2VCd9pxXsFyoV_m&sparams=expire%2Cei%2Cip%2Cid%2Caitags%2Csource%2Crequiressl%2Cspc%2Cvprv%2Cmime%2Cns%2Cgir%2Cclen%2Cdur%2Clmt&sig=AOq0QJ8wRgIhAKiKoHk2DfvaKA3Adu0CkNNF_lOavT4Mk0CYkQvbG0wpAiEAgiAc4h_1GqjzjGXgdiKdgdWZXWM0M9ubY-ns1qhGyns%3D&lsparams=mh%2Cmm%2Cmn%2Cms%2Cmv%2Cmvi%2Cpl&lsig=AG3C_xAwRQIhAM5DsCeq2RzEblNQ3qRq-3xEydeAidszCKAg8PvahEK1AiAzkjRXQQmDItpKWMxLq1TTb_32NWjPxiTDIVeOrBvxLw%3D%3D", + * "naturalWidth": 1280, + * "duration": 1220, + * "provider": "YouTube", + * "humanLanguage": "en", + * "html": "", + * "pageUrl": "https://www.youtube.com/watch?v=hFZFjoX2cGg", + * "text": "Squirrels were stealing my bird seed so I solved the problem with mechanical engineering :)\n\nHere is an explanation of the illusion dish thing!- https://demos.smu.ca/index.php/demos/optics/69-mirage-mirror\n\nHere is a link to the illusion dish (not sponsored :) https://www.amazon.com/dp/B0718XCG7F/ref=cm_sw_em_r_mt_dp_U_D9PYEbC85X14F\n\n*MUSIC*\n0:02 - Arrow (Instrumental) - Andrew Applepie http://andrewapplepie.com/\n0:27 - Kalimba Jam - Blue Wednesday https://soundcloud.com/bluewednesday/\n3:21 - Zambo - Devil in Disguise https://danijel-zambo.bandcamp.com/track/devil-in-disguise-2 \n3:47 - Cereal Killa - Blue Wednesday https://soundcloud.com/bluewednesday/\n5:28 - J. Thompson - Real Quick Lovin' https://www.amazon.com/Real-Quick-Lovin/dp/B0010YGE3W\n5:39 - New Shoes - Blue Wednesday https://soundcloud.com/bluewednesday/\n7:51- Chi- Ponder- https://www.prodbyponder.com/5-free-beats32029775\n8:31 - Marimba Idea - Blue Wednesday https://soundcloud.com/bluewednesday/ \n9:25 - Josef Falkenskold - Tiny Tumble https://www.epidemicsound.com/artists/josef-falkenskold\n19:07 - Nik- Ponder- https://www.prodbyponder.com/5-free-beats32029775", + * "viewCount": 92199455 + * } + * ] + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: { + date?: string; + images?: { + diffbotUri?: string; + title?: string; + url?: string; + primary?: boolean; + }[]; + author?: string; + mime?: string; + naturalHeight?: number; + diffbotUri?: string; + type?: string; + title?: string; + url?: string; + naturalWidth?: number; + duration?: number; + provider?: string; + humanLanguage?: string; + html?: string; + pageUrl?: string; + text?: string; + viewCount?: number; + }[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + list: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://slashdot.org/", + * "api": "list", + * "version": 3 + * }, + * "objects": [ + * { + * "diffbotUri": "list|3|-768312769", + * "icon": "https://slashdot.org/favicon.ico", + * "pageUrl": "https://slashdot.org/", + * "type": "list", + * "title": "News for nerds, stuff that matters", + * "items": [ + * { + * "date": "Tue, 05 Oct 2021 00:00:00 GMT", + * "summary": "According to fluent Korean speaker Youngmi Mayer, Squid Game features \"botched\" subtitles that have changed the show's meaning for English-speaking viewers.\nFor those unaware of Squid Game, it's a Korean-language drama about an alternative world where people in debt compete in deadly games. The plot sees a group of people tempted into a survival game where they have the chance to walk away with 45.6 billion Korean won ($38 million) if they win a series of six games. According to a BBC article, it's currently on track to become Netflix's biggest original series. From the report: \"The dialogue was so well written and zero of it was preserved [in the subtitles],\" Youngmi said in a Twitter post. In a TikTok video that's had almost nine million views, Youngmi gave several examples of mistranslation. In one scene a character tries to convince people to play the game with her, and the closed-caption subtitles read: \"I'm not a genius, but I still got it worked out.\" But what the character actually says, Youngmi explains, is: \"I am very smart, I just never got a chance to study.\" That translation puts more emphasis on the wealth disparity in society -- which is also a theme in the Oscar-winning 2019 Korean film, Parasite. \"Almost everything she says is being botched translation-wise... the writers, all they want you to know about her is that,\" Youngmi said. \"[It] seems so small, but it's the entire character's purpose of being in the show.\" Youngmi later clarifies that her initial comments were about the automatically generated closed-caption subtitles rather than the English language subtitles, which are \"substantially better.\" But she added: \"The misses in the metaphors -- and what the writers were trying to actually say -- are still pretty present.\"", + * "link": "https://entertainment.slashdot.org/story/21/10/05/0146245/squid-game-subtitles-change-meaning-of-netflix-show", + * "title": "Squid Game Subtitles 'Change Meaning' of Netflix Show" + * }, + * { + * "date": "Mon, 04 Oct 2021 00:00:00 GMT", + * "summary": "An anonymous reader quotes a report from The Verge: Sarah was the patient in a proof-of-concept trial of a new approach to treating severe, treatment-resistant depression, published today in the journal Nature Medicine. The findings open up another possible strategy for helping people with the disorder. The study only involved Sarah, and it's still not clear how well it might work in other people. The lessons from the trial, though, helped the researchers understand more about the nature of depression and could apply to other efforts to treat the disease. The trial used a technique called deep brain stimulation, where electrodes implanted within the brain deliver electrical impulses in an attempt to change or regulate abnormal brain activity. It's common for conditions like epilepsy and Parkinson's disease. Research over the past decade has shown that it can sometimes help with depression, but the findings have been inconsistent. Most previous efforts delivered stimulation to individual regions of the brain thought to be involved in depression. This study, though, was targeted at regions that were part of specific brain circuits -- interconnected parts of the brain that are responsible for specific functions.\nIn addition, the circuits involved might be different for each person. So in this trial, the study team personalized the treatment approach to the specific patient's depression. They mapped out the type of brain activity that occurred when Sarah's depression symptoms flared. Then, they surgically implanted a device that could detect that brain activity and send stimulation to the circuit where the activity was happening. For Sarah, the procedure was highly effective. Her scores on depression rating scales dropped the morning after the device was turned on. And perhaps more importantly, she felt dramatic changes in her mood. During her first time getting the stimulation, she laughed out loud in the lab. \"And everyone in the room went, 'Oh my god,' because that's the first time I spontaneously laughed and smiled, where it wasn't faked, in five years,\" she said. Sarah's depression circuit flares up hundreds of times a day, and each time, the implanted device delivers a brief stimulating pulse. In total, she gets around 30 minutes of stimulation each day [...]. Sarah can't feel the pulses, but she said she does have a general idea of when they're happening throughout the day. \"There's a sense of alertness and energy or positivity that I'll feel,\" she said.", + * "link": "https://science.slashdot.org/story/21/10/04/2241244/a-surgically-implanted-brain-stimulation-device-could-help-treat-severe-depression", + * "title": "A Surgically Implanted Brain Stimulation Device Could Help Treat Severe Depression" + * }, + * { + * "date": "Mon, 04 Oct 2021 00:00:00 GMT", + * "summary": "German officials on Monday unveiled what they said is the world's first commercial plant for making synthetic kerosene, touted as a climate-friendly fuel of the future. The Associated Press reports: The facility in Werlte, near Germany's northwestern border with the Netherlands, will use water and electricity from four nearby wind farms to produce hydrogen. In a century-old process, the hydrogen is combined with carbon dioxide to make crude oil, which can then be refined into jet fuel. Burning that synthetic kerosene releases only as much CO2 into the atmosphere as was previously removed to produce the fuel, making it \"carbon neutral.\"\nThe amount of fuel that the plant can produce beginning early next year is modest: just eight barrels a day, or about 336 gallons of jet fuel. That would be enough to fill up one small passenger plane every three weeks. By comparison, total fuel consumption of commercial airlines worldwide reached 95 billion gallons in 2019, before the pandemic hit the travel industry, according to the International Air Transport Association, or IATA. But Atmosfair, a German non-profit group behind the project, says its purpose is to show that the process is technologically feasible and -- once it is scaled up and with sufficient demand -- economically viable.\nInitially the price of synthetic kerosene produced in Werlte will be far higher than that of regular jet fuel, though Atmosfair won't divulge how much it will be charging its first customer, the German airline Lufthansa. However Atmosfair's chief executive, Dietrich Brockhagen, says a price of 5 euros ($5.80) per liter (0.26 gallons) is possible. That's still several times what kerosene currently costs, but Atsmofair is banking on carbon taxes driving up the price of fossil fuels, making his product more competitive. Additionally, authorities at the national and European level are putting in place quotas for the amount of e-fuel that airlines will have to use in future. That will create demand, making it more attractive to invest in bigger and better plants. Ueckerdt said 5 euros per liter is feasible by 2030, when the European Union's executive may require airlines to meet 0.7% of their kerosene needs with e-fuels. Under current plans, that would rise to 28% by 2050.", + * "link": "https://news.slashdot.org/story/21/10/04/2236211/germany-unveils-worlds-first-commercial-plant-for-making-synthetic-kerosene", + * "title": "Germany Unveils World's First Commercial Plant For Making Synthetic Kerosene" + * }, + * { + * "date": "Mon, 04 Oct 2021 00:00:00 GMT", + * "summary": "We thought the carnage was over for popular decentralized finance, or DeFi, staking protocol Compound, but as it turns out, millions more than we thought are at risk. About $162 million is up for grabs after an upgrade gone very wrong, according to Robert Leshner, founder of Compound Labs. CNBC reports: At first, the Compound chief tweeted Friday that there was a cap to how many comp tokens could be accidentally distributed, noting that âoethe impact is bounded, at worst, 280,000 comp tokens,â or about $92.6 million. But on Sunday morning, Leshner revealed that the pool of cash that had already been emptied once had been replenished â\" exposing another 202,472.5 comp tokens to exploit, or roughly $66.9 million at its current price.\nOn Wednesday, Compound rolled out what should have been a pretty standard upgrade. Soon after implementation, however, it was clear that something had gone seriously wrong, once users started to receive millions of dollars in comp tokens. For example, $30 million worth of comp tokens were claimed in one transaction. The saving grace of the entire debacle, however, was the fact that the pool of cash that was open to exploit -- something called the Comptroller contract -- had a finite amount of tokens. The problem is that this leaky pool got a fresh influx of cash, and 0.5 comp tokens are being added roughly every 15 seconds, according to Gupta. \"When the drip() function was called this morning, it sent the backlog (202,472.5, about two months of COMP since the last time the function was called) into the protocol for distribution to users,\" Leshner wrote in a tweet Sunday morning. Leshner noted that this brought the total comp at risk to 490,000 comp tokens, or about $162 million.\nThere are a few proposals to fix the bug, but Compound's governance model is such that any changes to the protocol require a multiday voting window, and Gupta said it takes another week for the successful proposal to be executed. In the meantime, this pool of cash is once again up for grabs for users who know how to exploit the bug. Compound made clear that no supplied or borrowed funds were at risk, which is some consolation. \"No user funds are or were at risk so it's not that big of a deal,\" said Gupta. \"Everyone kinda got diluted but didn't lose anything directly.\"", + * "link": "https://news.slashdot.org/story/21/10/04/2228209/bug-puts-162-million-up-for-grabs-says-founder-of-defi-platform-compound", + * "title": "Bug Puts $162 Million Up For Grabs, Says Founder of DeFi Platform Compound" + * } + * ] + * } + * ] + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: { + diffbotUri?: string; + icon?: string; + pageUrl?: string; + type?: string; + title?: string; + items?: { + date?: string; + summary?: string; + link?: string; + title?: string; + }[]; + }[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + event: { + parameters: { + query: { + /** @description Target URL to extract */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "request": { + * "pageUrl": "https://www.eventbrite.com/e/graphql-summit-2018-tickets-46601841362", + * "api": "event", + * "version": 3 + * }, + * "objects": [ + * { + * "venue": "The Regency Center", + * "images": [ + * { + * "naturalHeight": 0, + * "width": 720, + * "diffbotUri": "image|3|2088675194", + * "url": "https://img.evbuc.com/https%3A%2F%2Fcdn.evbuc.com%2Fimages%2F45534724%2F182934487936%2F1%2Foriginal.jpg?w=800&auto=compress&rect=0%2C0%2C2160%2C1080&s=e73c0be202c2cd939dcd3e809d96e093", + * "naturalWidth": 0, + * "primary": true, + * "height": 360 + * } + * ], + * "timezoneOffset": "-07:00", + * "endDate": "Fri, 09 Nov 2018 01:00:00 GMT", + * "endDateLocalTimezone": "Thu, 08 Nov 2018 17:00:00", + * "humanLanguage": "en", + * "diffbotUri": "event|3|-2069254052", + * "description": "​Join over 850 developers at the 3rd annual GraphQL Summit, the world's largest conference dedicated to GraphQL. Whether you're just getting started or a veteran GraphQL practitioner, this is the must-attend developer conference of the year.\nChoose from 40+ sessions across two tracks at one of San Francisco's most historic event venues\n​See how teams at top startups and enterprises are modernizing their platforms by using GraphQL in complex production environments\nHear where the technology is heading next, directly from the leaders of GraphQL's major open source projects\nMeet top solution providers from around the world - all under one roof.\nEnroll in hands-on workshops and classroom training to boost your skills to get the most out of GraphQL.\nGraphQL Summit Day Code of Conduct\nWe believe our community and events should be truly open for everyone. As such, we are committed to providing a friendly, safe, and welcoming environment for all, regardless of gender, sexual orientation, disability, ethnicity, or religion. Our event code of conduct outlines our expectations for participant behavior, as well as the consequences for unacceptable behavior.\n*GraphQL Summit is hosted at the Regency Center which is an accessible space with elevator access*", + * "pageUrl": "https://archives.diffbot.com/replay/4F02A5A651C6767544362C27B1D9BAC1/", + * "location": { + * "country": { + * "name": "United States", + * "diffbotUri": "http://diffbot.com/entity/A01d4EK33MmCosgI2KXa4-A", + * "websiteUris": [ + * "wikidata.org/entity/Q30" + * ], + * "targetDiffbotUri": "http://diffbot.com/entity/A01d4EK33MmCosgI2KXa4-A", + * "surfaceForm": "United States" + * }, + * "isCurrent": true, + * "address": "1300 Van Ness Ave, San Francisco, California", + * "city": { + * "name": "San Francisco", + * "diffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + * "websiteUris": [ + * "wikidata.org/entity/Q62" + * ], + * "targetDiffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + * "surfaceForm": "San Francisco" + * }, + * "street": "1300 Van Ness Ave", + * "subregion": { + * "name": "San Francisco County", + * "diffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + * "websiteUris": [ + * "wikidata.org/entity/Q62" + * ], + * "targetDiffbotUri": "http://diffbot.com/entity/A8AV-O2v0MBuACAY7pFZzbg", + * "surfaceForm": "San Francisco County" + * }, + * "latitude": 37.78785705566406, + * "precision": 0.10000000149011612, + * "postalCode": "94109", + * "region": { + * "name": "California", + * "diffbotUri": "http://diffbot.com/entity/Al0_8ehooNxOPT36Y4RdV9w", + * "websiteUris": [ + * "wikidata.org/entity/Q99" + * ], + * "targetDiffbotUri": "http://diffbot.com/entity/Al0_8ehooNxOPT36Y4RdV9w", + * "surfaceForm": "California" + * }, + * "longitude": -122.42138671875 + * }, + * "type": "event", + * "title": "GraphQL Summit 2018", + * "startDate": "Wed, 07 Nov 2018 16:30:00 GMT", + * "startDateLocalTimezone": "Wed, 07 Nov 2018 08:30:00" + * } + * ] + * } + */ + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: { + venue?: string; + images?: { + naturalHeight?: number; + width?: number; + diffbotUri?: string; + url?: string; + naturalWidth?: number; + primary?: boolean; + height?: number; + }[]; + timezoneOffset?: string; + endDate?: string; + endDateLocalTimezone?: string; + humanLanguage?: string; + diffbotUri?: string; + description?: string; + pageUrl?: string; + location?: { + country?: { + name?: string; + diffbotUri?: string; + websiteUris?: string[]; + targetDiffbotUri?: string; + surfaceForm?: string; + }; + isCurrent?: boolean; + address?: string; + city?: { + name?: string; + diffbotUri?: string; + websiteUris?: string[]; + targetDiffbotUri?: string; + surfaceForm?: string; + }; + street?: string; + subregion?: { + name?: string; + diffbotUri?: string; + websiteUris?: string[]; + targetDiffbotUri?: string; + surfaceForm?: string; + }; + latitude?: number; + precision?: number; + postalCode?: string; + region?: { + name?: string; + diffbotUri?: string; + websiteUris?: string[]; + targetDiffbotUri?: string; + surfaceForm?: string; + }; + longitude?: number; + }; + type?: string; + title?: string; + startDate?: string; + startDateLocalTimezone?: string; + }[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + customget: { + parameters: { + query: { + /** @description Target URL to extract (url encoded) */ + url: string; + /** @description Specify optional fields to be returned from any fully-extracted pages (e.g. `fields=querystring,links`) */ + fields?: "links" | "extlinks" | "meta" | "querystring" | "breadcrumb"; + /** @description Sets a value in milliseconds to wait for the retrieval/fetch of content from the requested URL. The default timeout for the third-party response is 30 seconds (30000). */ + timeout?: number; + /** @description Use for jsonp requests. Needed for cross-domain ajax. */ + callback?: string; + /** @description Specify an IP address of a [custom proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) that will be used to fetch the target page. (Ex: `&proxy` or `&proxy=0.0.0.0`) */ + proxy?: string; + /** @description Used to specify the authentication parameters that will be used with a custom proxy specified in the &proxy parameter. (Ex: `proxyAuth=username:password`) */ + proxyAuth?: string; + /** @description Set to `default` to use [Diffbot's datacenter proxy](https://docs.diffbot.com/reference/using-proxies#how-to-use-proxies) for this request. `none` will instruct Extract to not use proxies, even if proxies have been enabled for this particular URL globally. */ + useProxy?: string; + }; + header?: never; + path: { + /** @description Name of your Custom API */ + api: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + request?: { + pageUrl?: string; + api?: string; + version?: number; + }; + objects?: Record[]; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + "retrieve-a-custom-api": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + notes?: string[]; + xForwardHeaders?: { + "X-Evaluate": string; + }; + rules?: { + name?: string; + selector?: string; + }[]; + api?: string; + urlPattern?: string; + testUrl?: string; + }[]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "errorCode": 500, + * "error": "Internal Server Error" + * } + */ + "application/json": { + errorCode?: number; + error?: string; + }; + }; + }; + }; + }; + "create-a-custom-api": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/json": { + /** @description An array of strings that can be added manually. The API automatically adds a notes specifying when the API was last updated. */ + notes?: string[]; + /** @description Allows you to pass in X-Forward headers by name, including X-Evaluate (aka. X-Forward-X-Evaluate; omit "X-Forward" in the header name) */ + xForwardHeaders?: { + "X-Evaluate"?: string; + }; + /** @description An object that defines a set of rules for a specific urlPattern-api combination */ + rules?: { + name?: string; + selector?: string; + filters?: { + args?: string[]; + type?: string; + }[]; + }[]; + /** @description The specific API being targeted. Always precede the API name with "/api/" as in `"/api/article"` except for "all") */ + api?: string; + /** @description A regex that defines the URLs for which the ruleset will be applied */ + urlPattern?: string; + /** @description A URL that can be used to check that the rule still works as intended. This is the page that will load automatically when editing the ruleset in the Dashboard UI */ + testUrl?: string; + /** @description Rendering options */ + renderOptions?: string; + /** @description An array of string selectors that should be omitted from the DOM before extraction occurs */ + prefilters?: string[]; + /** @description Used to disable proxies (when they have been set globally), by applying the value `"none"` */ + useProxy?: string; + }; + }; + }; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "hashes": [ + * "abcd1234" + * ] + * } + */ + "application/json": { + hashes?: string[]; + }; + }; + }; + }; + }; + "delete-a-custom-api": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: { + content: { + "application/json": { + /** @description The specific API being targeted. Always precede the API name with "/api/" as in `"/api/article"` except for "all") */ + api?: string; + /** @description A regex that defines the URLs corresponding to which the Custom API definitions will be deleted. Make sure that is json-escaped before url-encoding it. */ + urlPattern?: string; + /** @description Your token. The Custom API definitions only for this token will be deleted */ + token?: string; + }; + }; + }; + responses: { + /** @description Successful API Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "api": "/api/discussion", + * "token": "TOKEN", + * "urlPattern": "examplePattern.com", + * "status": "success" + * } + */ + "application/json": { + token?: string; + urlPattern?: string; + api?: string; + status?: string; + }; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "statusCode": 500, + * "error": "urlPattern and apiPrefix required to delete rule" + * } + */ + "application/json": { + statusCode?: number; + error?: string; + }; + }; + }; + }; + }; +} diff --git a/src/generated/natural-language.ts b/src/generated/natural-language.ts new file mode 100644 index 0000000..cf80063 --- /dev/null +++ b/src/generated/natural-language.ts @@ -0,0 +1,418 @@ +/** + * Generated from openapi/natural-language.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/v1/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Process Text */ + post: { + parameters: { + query?: { + /** @description Attributes to include in response */ + fields?: ("entities" | "sentiment" | "facts" | "records" | "categories" | "sentences" | "language" | "summary")[]; + }; + header?: never; + path?: never; + cookie?: never; + }; + /** @description documents to be processed */ + requestBody: { + content: { + "application/json": components["schemas"]["DocumentArray"]; + "application/x-www-form-urlencoded": components["schemas"]["DocumentArray"]; + }; + }; + responses: { + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["NaturalLanguageResponseArray"]; + }; + }; + /** @description Invalid input */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + /** + * @example { + * "content": "Diffbot is a startup based in Menlo Park." + * } + */ + Document: { + /** + * Format: blob + * @description The text you'd like Diffbot Natural Language to process. + */ + content: string; + /** + * @description Language of the text specified as an [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). Review our supported languages for more information. + * @default auto + */ + lang: string; + /** + * @description `plain text` for free text. `plain text with title` includes a title separated from the body by two line breaks (e.g. "This is a title\n\nThis is the body"). + * @default plain text + * @enum {string} + */ + format: "plain text" | "plain text with title"; + /** + * @description Optional. Provides context to the model and may produce better results. + * @enum {string} + */ + documentType?: "organization bio" | "person bio" | "news article" | "address" | "discussion"; + customSummary?: components["schemas"]["CustomSummary"]; + }; + /** + * @example [ + * { + * "content": "Diffbot is a startup based in Menlo Park." + * } + * ] + */ + DocumentArray: components["schemas"]["Document"][]; + EntityWithMentions: { + name?: string; + diffbotUri?: string; + /** Format: float */ + confidence?: number; + /** Format: float */ + salience?: number; + /** Format: float */ + sentiment?: number; + allUris?: string[]; + allTypes?: components["schemas"]["EntityType"][]; + mentions?: components["schemas"]["Mention"][]; + location?: components["schemas"]["Location"]; + }; + Mention: { + text?: string; + beginOffset?: number; + endOffset?: number; + isPronoun?: boolean; + /** Format: float */ + confidence?: number; + }; + Entity: { + name?: string; + diffbotUri?: string; + /** Format: float */ + confidence?: number; + allUris?: string[]; + allTypes?: components["schemas"]["EntityType"][]; + isCustom?: boolean; + /** @description Index of this entity in the field "entities" (if requested) */ + entityIndex?: number; + }; + Span: { + beginOffset?: number; + endOffset?: number; + }; + Fact: { + humanReadable?: string; + entity?: components["schemas"]["Entity"]; + property?: components["schemas"]["Property"]; + value?: components["schemas"]["Entity"]; + qualifiers?: components["schemas"]["Qualifier"][]; + /** Format: float */ + confidence?: number; + evidence?: components["schemas"]["Evidence"][]; + }; + /** @description https://demo.nl.diffbot.com/schema/#qualifiers */ + Qualifier: { + property?: components["schemas"]["Property"]; + value?: components["schemas"]["Qualifier_value"]; + /** Format: float */ + confidence?: number; + }; + /** @description https://demo.nl.diffbot.com/schema/#properties */ + Property: { + name?: string; + diffbotUri?: string; + description?: string; + }; + /** @description https://demo.nl.diffbot.com/schema/#types */ + EntityType: { + name?: string; + diffbotUri?: string; + dbpediaUri?: string; + }; + Evidence: { + /** @description One or more sentences where the fact was extracted from */ + passage?: string; + entityMentions?: components["schemas"]["Mention"][]; + valueMentions?: components["schemas"]["Mention"][]; + }; + Location: { + /** Format: float */ + latitude?: number; + /** Format: float */ + longitude?: number; + /** Format: float */ + precision?: number; + }; + Category: { + id?: string; + name?: string; + path?: string; + /** Format: float */ + confidence?: number; + isPrimary?: boolean; + }; + /** @description https://demo.nl.diffbot.com/schema/#categories */ + Categorization: { + iabv1?: components["schemas"]["Category"][]; + iabv2?: components["schemas"]["Category"][]; + diffbot?: components["schemas"]["Category"][]; + }; + /** @description Define custom parameters for summary generation. */ + CustomSummary: { + /** + * Format: int32 + * @description Maximum number of sentences for the generated summary. + * @default 3 + */ + maxNumberOfSentences: number; + }; + NaturalLanguageResponse: { + errors?: string[]; + entities?: components["schemas"]["EntityWithMentions"][]; + /** Format: float */ + sentiment?: number; + facts?: components["schemas"]["Fact"][]; + /** @description Array of entities with attributes extracted from inputDocument according to the KG schema (https://docs.diffbot.com/docs/ontology) */ + records?: Record[]; + categories?: components["schemas"]["Categorization"]; + sentences?: components["schemas"]["Span"][]; + /** @description Language of the text specified as an [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes). */ + language?: string; + /** @description Generated summary. */ + summary?: string; + }; + /** + * @example [ + * { + * "sentiment": 0, + * "entities": [ + * { + * "name": "Diffbot", + * "diffbotUri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + * "confidence": 0.99898225, + * "salience": 0.95394653, + * "sentiment": 0, + * "isCustom": false, + * "allUris": [ + * "http://www.wikidata.org/entity/Q17052069" + * ], + * "allTypes": [ + * { + * "name": "organization", + * "diffbotUri": "https://diffbot.com/entity/EN1ClYEdMMQCxB6AWTkT3mA", + * "dbpediaUri": "http://dbpedia.org/ontology/Organisation" + * } + * ], + * "mentions": [ + * { + * "text": "Diffbot", + * "beginOffset": 0, + * "endOffset": 7, + * "confidence": 0.99898225 + * } + * ], + * "location": { + * "latitude": 37.456104, + * "longitude": -122.1763, + * "precision": 0.1 + * } + * }, + * { + * "name": "Menlo Park", + * "diffbotUri": "https://diffbot.com/entity/EQiAUr370NpW1gvx-V4fTrQ", + * "confidence": 0.9996269, + * "salience": 0.9082091, + * "sentiment": 0, + * "isCustom": false, + * "allUris": [ + * "http://www.wikidata.org/entity/Q74195" + * ], + * "allTypes": [ + * { + * "name": "location", + * "diffbotUri": "https://diffbot.com/entity/EiCyWUm4lNziqgLHx47iAIQ", + * "dbpediaUri": "http://dbpedia.org/ontology/Place" + * }, + * { + * "name": "administrative area", + * "diffbotUri": "https://diffbot.com/entity/EcTIu1tWKPouIa6qZtSpc4A", + * "dbpediaUri": "http://dbpedia.org/ontology/PopulatedPlace" + * }, + * { + * "name": "city", + * "diffbotUri": "https://diffbot.com/entity/EzdJrGHiyMWu0XbSn101rFA", + * "dbpediaUri": "http://dbpedia.org/ontology/City" + * } + * ], + * "mentions": [ + * { + * "text": "Menlo Park", + * "beginOffset": 30, + * "endOffset": 40, + * "confidence": 0.9996269 + * } + * ], + * "location": { + * "latitude": 37.454166, + * "longitude": -122.17861, + * "precision": 6.708204 + * } + * }, + * { + * "name": "startup company", + * "diffbotUri": "https://diffbot.com/entity/EfX0EeXroOmaXrLe-SylIuw", + * "confidence": 0.81958973, + * "salience": 0.6474858, + * "sentiment": 0, + * "isCustom": false, + * "allUris": [], + * "mentions": [ + * { + * "text": "startup", + * "beginOffset": 13, + * "endOffset": 20, + * "confidence": 0.81958973 + * } + * ] + * } + * ], + * "facts": [ + * { + * "humanReadable": "[Diffbot] headquarters [Menlo Park]", + * "entity": { + * "name": "Diffbot", + * "diffbotUri": "https://diffbot.com/entity/EYX1i02YVPsuT7fPLUYgRhQ", + * "confidence": 0.99898225, + * "allUris": [ + * "http://www.wikidata.org/entity/Q17052069" + * ], + * "allTypes": [ + * { + * "name": "organization", + * "diffbotUri": "https://diffbot.com/entity/EN1ClYEdMMQCxB6AWTkT3mA", + * "dbpediaUri": "http://dbpedia.org/ontology/Organisation" + * } + * ], + * "isCustom": false, + * "entityIndex": 1 + * }, + * "property": { + * "name": "headquarters", + * "diffbotUri": "https://docs.diffbot.com/docs/ont-organization#location" + * }, + * "value": { + * "name": "Menlo Park", + * "diffbotUri": "https://diffbot.com/entity/EQiAUr370NpW1gvx-V4fTrQ", + * "confidence": 0.9996269, + * "allUris": [ + * "http://www.wikidata.org/entity/Q74195" + * ], + * "allTypes": [ + * { + * "name": "location", + * "diffbotUri": "https://diffbot.com/entity/EiCyWUm4lNziqgLHx47iAIQ", + * "dbpediaUri": "http://dbpedia.org/ontology/Place" + * }, + * { + * "name": "administrative area", + * "diffbotUri": "https://diffbot.com/entity/EcTIu1tWKPouIa6qZtSpc4A", + * "dbpediaUri": "http://dbpedia.org/ontology/PopulatedPlace" + * }, + * { + * "name": "city", + * "diffbotUri": "https://diffbot.com/entity/EzdJrGHiyMWu0XbSn101rFA", + * "dbpediaUri": "http://dbpedia.org/ontology/City" + * } + * ], + * "isCustom": false, + * "entityIndex": 0 + * }, + * "qualifiers": [ + * { + * "property": { + * "name": "is current" + * }, + * "value": { + * "name": "true" + * }, + * "confidence": 0 + * } + * ], + * "confidence": 0.95, + * "evidence": [ + * { + * "passage": "Diffbot is a startup based in Menlo Park.", + * "entityMentions": [ + * { + * "text": "Diffbot", + * "beginOffset": 0, + * "endOffset": 7, + * "confidence": 0.99898225 + * } + * ], + * "valueMentions": [ + * { + * "text": "Menlo Park", + * "beginOffset": 30, + * "endOffset": 40, + * "confidence": 0.9996269 + * } + * ] + * } + * ] + * } + * ] + * } + * ] + */ + NaturalLanguageResponseArray: components["schemas"]["NaturalLanguageResponse"][]; + ErrorResponse: { + messages?: string[]; + }; + Qualifier_value: { + name?: string; + diffbotUri?: string; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export type operations = Record; diff --git a/src/generated/web-search.ts b/src/generated/web-search.ts new file mode 100644 index 0000000..9c2a1d2 --- /dev/null +++ b/src/generated/web-search.ts @@ -0,0 +1,202 @@ +/** + * Generated from openapi/web-search.json by `pnpm gen:types`. Do not hand-edit. + * Source: openapi-typescript v7.13.0 + */ +export interface paths { + "/": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Run a Web Search + * @description Search Diffbot's Web Index + */ + get: operations["get_"]; + put?: never; + /** Run a Web Search */ + post: operations["post_"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + /** + * @description Your search query. + * + * Use the following operators to refine your search. + * + * | Operator | Description | Example | + * | --- | --- | --- | + * | `after` | Documents published after the provided date | `after:2026-06-10` | + * | `before` | Documents published before the provided date | `before:2026-05-10` | + * | `site` | Documents matching the provided domain | `site:diffbot.com` | + * | `url` | Documents matching the exact URL | `url:diffbot.com` | + */ + Query: string[]; + /** + * @description Number of results to return in the search response + * @default 10 + */ + Size: number; + /** @description Cap the search response to this value. Highlight chunks will be reduced in length equitably. */ + MaxTokens: number; + }; + responses: { + /** @description Successful response */ + Results: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @description The parsed query terms. */ + query?: string[]; + /** @description The ranked results. */ + search_results?: { + /** + * @description Represents the confidence of the result from 0 (lowest) to highest (1). 3 decimal places of precision. + * @default 0 + * @example 0.973 + */ + score: number; + /** + * @description URL of the page. + * @example https://www.diffbot.com/ + */ + pageUrl?: string; + /** + * @description Title of the page. + * @example Web Data for your AI + */ + title?: string; + /** + * @description A spliced chunk highlight from the original full page contents. + * @example Imagine if your app could access the web like a structured database. + * + * Your browser does not support the video tag. + * - 50+ data fields, including categories, revenue, locations, and investments + * - Over 246M companies and non-profits in the Knowledge Graph + * - Extract and refresh orgs on demand + * + * - More than just text — entity matching, topic-level sentiment, and more + * - Over 1.6B news articles, blog posts, and press releases in the Knowledge Graph + * + * + * ## The Web is Noisy, + * Diffbot Straightens it Out + * + * The world's largest compendium of human knowledge is buried in the code of 1.2 billion public websites. Diffbot reads it all like a human, then transforms it into usable data. + */ + content?: string; + /** + * @description Publication date of the page, when known. RFC 1123 format. Omitted when the page has no discoverable publication date. + * @example Thu, 12 Mar 2026 00:00:00 GMT + */ + date?: string; + }[]; + /** + * @description Time taken for retrieval, reranking, and response generation. + * @default 0 + * @example 154 + */ + timeMs: number; + }; + }; + }; + /** @description Unauthorized */ + Unauthorized: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** + * @default 0 + * @example 401 + */ + code: number; + /** @example Missing or invalid Authorization header. Expected: Bearer */ + message?: string; + }; + }; + }; + }; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export interface operations { + get_: { + parameters: { + query: { + text: components["schemas"]["Query"]; + size?: components["schemas"]["Size"]; + maxTokens?: components["schemas"]["MaxTokens"]; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Results"]; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + errors?: string[]; + }; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; + post_: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + text: components["schemas"]["Query"]; + size?: components["schemas"]["Size"]; + maxTokens?: components["schemas"]["MaxTokens"]; + }; + }; + }; + responses: { + 200: components["responses"]["Results"]; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** @example 400 */ + code?: number; + /** @example Missing text parameter */ + message?: string; + }; + }; + }; + 401: components["responses"]["Unauthorized"]; + }; + }; +} diff --git a/src/http.ts b/src/http.ts index 30e4949..8f1cf19 100644 --- a/src/http.ts +++ b/src/http.ts @@ -1,22 +1,146 @@ -import { APIError, AuthError, RateLimitError } from "./errors.js"; +import { headersToRecord } from "./decode.js"; +import { APIError, AuthError, NotFoundError, RateLimitError } from "./errors.js"; export type FetchFn = typeof fetch; +export type QueryParamValue = string | number | boolean; +export type QueryParams = Record; + +/** + * Retry policy for transient failures (429, 5xx, connection errors). `maxAttempts` + * counts the initial try, so `3` means up to 2 retries. Disable per client with + * `retry: false` on `DiffbotClientOptions`; disable per call with `retryable: false` + * on a handler's options — required for GET-shaped mutations (e.g. `crawl?delete=1`) + * that must never be retried blindly. + * + * `maxDelayMs` caps the exponential backoff and is also the ceiling for a server's + * `Retry-After`: if a 429/5xx asks for a longer wait (seconds or HTTP-date form), the + * SDK does not sleep or retry and returns that response at once, so the caller gets + * `RateLimitError` (or `APIError` for a 5xx) with the header intact + * (`RateLimitError.retryAfter`, `APIError.headers["retry-after"]`). + */ +export interface RetryPolicy { + maxAttempts?: number; + baseDelayMs?: number; + maxDelayMs?: number; + jitter?: boolean; +} + +interface ResolvedRetryPolicy { + maxAttempts: number; + baseDelayMs: number; + maxDelayMs: number; + jitter: boolean; +} + +export const DEFAULT_RETRY_POLICY: Required = { + maxAttempts: 3, + baseDelayMs: 250, + maxDelayMs: 5_000, + jitter: true, +}; + +function resolveRetryPolicy(policy: RetryPolicy): ResolvedRetryPolicy { + return { + maxAttempts: policy.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts, + baseDelayMs: policy.baseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs, + maxDelayMs: policy.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs, + jitter: policy.jitter ?? DEFAULT_RETRY_POLICY.jitter, + }; +} + +function isRetryableStatus(status: number): boolean { + return status === 429 || status >= 500; +} + +/** Parses `Retry-After` as seconds or an HTTP date into a non-negative delay (ms). */ +function parseRetryAfterMs(retryAfter: string | null): number | undefined { + if (!retryAfter) return undefined; + const seconds = Number(retryAfter); + if (Number.isFinite(seconds)) { + return Math.max(0, seconds * 1000); + } + const dateMs = Date.parse(retryAfter); + if (!Number.isNaN(dateMs)) { + return Math.max(0, dateMs - Date.now()); + } + return undefined; +} + +/** + * Delay before the next attempt: a parseable `Retry-After` is honored as given, + * otherwise exponential backoff capped at `maxDelayMs`. Returns `null` when + * `Retry-After` exceeds `maxDelayMs`, meaning "do not retry; surface the response". + */ +function computeDelayMs(attempt: number, retryAfter: string | null, policy: ResolvedRetryPolicy): number | null { + const retryAfterMs = parseRetryAfterMs(retryAfter); + if (retryAfterMs !== undefined) { + return retryAfterMs > policy.maxDelayMs ? null : retryAfterMs; + } + const exponential = Math.min(policy.maxDelayMs, policy.baseDelayMs * 2 ** (attempt - 1)); + return policy.jitter ? exponential * (0.5 + Math.random() * 0.5) : exponential; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export interface HttpRequestOptions { - params?: Record; + params?: QueryParams; headers?: Record; json?: unknown; + /** Raw request body (string, Uint8Array, URLSearchParams, Blob, FormData, ...). Ignored if `json` is set. */ + body?: RequestInit["body"]; + /** Overrides the inferred `Content-Type`. */ + contentType?: string; redirect?: "follow" | "error" | "manual"; + /** Overrides the client's default timeout for this call only. */ + timeout?: number; + /** Set `false` to opt a call out of retries, e.g. a GET-shaped mutation. Default `true`. */ + retryable?: boolean; +} + +function buildUrl(url: string, params?: QueryParams): URL { + const parsed = new URL(url); + if (!params) return parsed; + for (const [key, value] of Object.entries(params)) { + if (value === undefined) continue; + if (Array.isArray(value)) { + for (const item of value) { + parsed.searchParams.append(key, String(item)); + } + } else { + parsed.searchParams.append(key, String(value)); + } + } + return parsed; +} + +function resolveContentType(options: HttpRequestOptions, headers: Headers): string | undefined { + if (options.contentType) return options.contentType; + if (headers.has("content-type")) return undefined; + if (options.json !== undefined) return "application/json"; + if (options.body instanceof URLSearchParams) return "application/x-www-form-urlencoded;charset=UTF-8"; + return undefined; } export class HttpClient { private closed = false; + private readonly retryPolicy: ResolvedRetryPolicy | null; constructor( private readonly fetchFn: FetchFn, private readonly timeoutMs: number, readonly userAgent: string, - ) {} + retry: RetryPolicy | false = {}, + ) { + this.retryPolicy = retry === false ? null : resolveRetryPolicy(retry); + } + + /** The client-level request timeout (ms), used when a call does not set its own. */ + get defaultTimeout(): number { + return this.timeoutMs; + } async get(url: string, options: HttpRequestOptions = {}): Promise { return this.request(url, { ...options, method: "GET" }); @@ -26,41 +150,70 @@ export class HttpClient { return this.request(url, { ...options, method: "POST" }); } + async delete(url: string, options: HttpRequestOptions = {}): Promise { + return this.request(url, { ...options, method: "DELETE" }); + } + async request(url: string, options: HttpRequestOptions & { method?: string } = {}): Promise { if (this.closed) { throw new Error("HTTP client is closed"); } - const parsed = new URL(url); - if (options.params) { - for (const [key, value] of Object.entries(options.params)) { - if (value !== undefined) { - parsed.searchParams.set(key, String(value)); - } - } - } - + const parsed = buildUrl(url, options.params); const headers = new Headers(options.headers); headers.set("User-Agent", this.userAgent); - if (options.json !== undefined) { - headers.set("Content-Type", "application/json"); - } + const contentType = resolveContentType(options, headers); + if (contentType) headers.set("Content-Type", contentType); const init: RequestInit = { method: options.method ?? "GET", headers, redirect: options.redirect, - signal: AbortSignal.timeout(this.timeoutMs), }; if (options.json !== undefined) { init.body = JSON.stringify(options.json); + } else if (options.body !== undefined) { + init.body = options.body; } - return this.fetchFn(parsed.toString(), init); + const timeoutMs = options.timeout ?? this.timeoutMs; + const retryable = options.retryable ?? true; + const policy = retryable ? this.retryPolicy : null; + const maxAttempts = policy ? policy.maxAttempts : 1; + + for (let attempt = 1; ; attempt++) { + let response: Response; + try { + response = await this.fetchFn(parsed.toString(), { + ...init, + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (error) { + if (policy && attempt < maxAttempts) { + await sleep(computeDelayMs(attempt, null, policy) ?? 0); + continue; + } + throw error; + } + + if (policy && attempt < maxAttempts && isRetryableStatus(response.status)) { + const delayMs = computeDelayMs(attempt, response.headers.get("retry-after"), policy); + // Retry-After beyond maxDelayMs: don't sleep or retry; the caller raises + // RateLimitError/APIError from this response with the header intact. + if (delayMs === null) return response; + // Release the discarded attempt's connection; an unread body pins it in undici. + await response.body?.cancel().catch(() => undefined); + await sleep(delayMs); + continue; + } + + return response; + } } - async close(): Promise { + close(): Promise { this.closed = true; + return Promise.resolve(); } get isClosed(): boolean { @@ -74,15 +227,22 @@ export function raiseForStatus(response: Response, bodyText?: string): void { } const status = response.status; const body = bodyText ?? ""; + const headers = headersToRecord(response.headers); if (status === 401 || status === 403) { - throw new AuthError(status, body); + throw new AuthError(status, body, headers); + } + if (status === 404) { + throw new NotFoundError(status, body, headers); } if (status === 429) { - throw new RateLimitError(status, body, response.headers.get("retry-after")); + throw new RateLimitError(status, body, response.headers.get("retry-after"), headers); } - throw new APIError(status, body); + throw new APIError(status, body, headers); } -export async function readBodyText(response: Response): Promise { - return response.text(); +/** Reads the body as text and raises when the response is not ok; a no-op on success. */ +export async function ensureOk(response: Response): Promise { + if (response.ok) return; + const body = await response.text(); + raiseForStatus(response, body); } diff --git a/src/index.ts b/src/index.ts index e24638b..7e40376 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,20 @@ export { DiffbotClient, type DiffbotClientOptions } from "./client.js"; export { resolveTokenFromEnv, type TokenSource } from "./auth.js"; -export { extract } from "./extract.js"; +export { account, verifyToken, type AccountOptions } from "./account.js"; +export { + extract, + type AnalyzeExtractOptions, + type ArticleExtractOptions, + type CustomExtractOptions, + type DiscussionExtractOptions, + type EventExtractOptions, + type ExtractOptions, + type ImageExtractOptions, + type JobExtractOptions, + type ListExtractOptions, + type ProductExtractOptions, + type VideoExtractOptions, +} from "./extract.js"; export { ANY_OBJECT_SCHEMA, RESPONSE_FORMAT_TYPES, @@ -8,17 +22,22 @@ export { askJson, extractJson, jsonSchemaFormat, + type AskJsonOptions, type AskOptions, type ChatMessage, + type ChatRole, type ResponseFormat, type ResponseFormatType, } from "./ask.js"; export { crawl, crawlDeleteJob, + crawlDownload, crawlGetJob, crawlListJobs, CrawlEventType, + type CrawlDownloadOptions, + type CrawlDownloadResult, type CrawlEvent, type CrawlOptions, } from "./crawl.js"; @@ -28,8 +47,15 @@ export { dqlFetchOntologyText, dqlParallel, type DqlOptions, + type DqlType, } from "./kg.js"; -export { entities } from "./nlp.js"; +export { + entities, + nlp, + type EntitiesResponse, + type NlpOptions, + type NlpResponse, +} from "./nlp.js"; export { Ontology } from "./ontology.js"; export { OntologyStore, @@ -44,9 +70,36 @@ export { ValidationError, APIError, AuthError, + NotFoundError, RateLimitError, + JobError, ExtractionError, } from "./errors.js"; export { VERSION } from "./constants.js"; -export type { FetchFn } from "./http.js"; +export type { FetchFn, QueryParams, RetryPolicy } from "./http.js"; export type { JsonObject } from "./json.js"; + +// Response/shared types (generated-backed where a spec defines a schema; hand-written +// in src/types/undocumented.ts otherwise — see that file's header for which is which). +export type { DqlEntity, DqlResponse, DqlRewrite, ExportFormat } from "./types/dql.js"; +export type { AccountResponse, AccountUsage } from "./types/account.js"; +export type { NaturalLanguageResponse, NlpDocument, NlpField } from "./types/natural-language.js"; +export type { WebSearchResponse, WebSearchResult } from "./types/web-search.js"; +export type { + AnalyzeObject, + ArticleObject, + CrawlDataRecord, + CrawlJob, + CustomApiObject, + DiscussionObject, + EventObject, + ExtractResponse, + FieldMeta, + ImageObject, + JobObject, + JobStatus, + ListObject, + OntologyDocument, + ProductObject, + VideoObject, +} from "./types/undocumented.js"; diff --git a/src/kg.ts b/src/kg.ts index a0e9f28..d1023ba 100644 --- a/src/kg.ts +++ b/src/kg.ts @@ -1,67 +1,158 @@ import type { DiffbotClient } from "./client.js"; -import { readBodyText, raiseForStatus } from "./http.js"; -import type { JsonObject } from "./json.js"; +import { parseJsonText, readBytes, readNdjson, readText } from "./decode.js"; +import { ValidationError } from "./errors.js"; +import { ensureOk } from "./http.js"; import { Ontology } from "./ontology.js"; +import type { DqlEntity, DqlResponse, ExportFormat } from "./types/dql.js"; +import type { ExtraParams } from "./types/extra.js"; + +const DQL_NAMED_PARAMS = [ + "token", + "query", + "type", + "col", + "size", + "from", + "format", + "filter", + "filterExclude", + "exportspec", + "exportseparator", + "exportfile", + "jsonmode", + "nonCanonicalFacts", + "noDedupArticles", + "cluster", +] as const; + +type DqlNamedParam = (typeof DQL_NAMED_PARAMS)[number]; + +/** `query` and `queryTextFallback` are DQL; `text` is plain-text search; `crawl` searches a crawl collection (see `col`). */ +export type DqlType = "query" | "text" | "queryTextFallback" | "crawl"; export interface DqlOptions { + /** Search type. Omitted from the body by default (the API default is `query`). */ + type?: DqlType; + /** Crawl collection to search when `type` is `"crawl"`. */ + col?: string; size?: number; from?: number; - format?: string; + format?: ExportFormat; filter?: string; + /** DQL clause whose matches are excluded from the results. */ + filterExclude?: string; exportspec?: string; - extra?: Record; + /** Column separator for `csv`/`xls`/`xlsx` exports. */ + exportseparator?: string; + /** File name hint for exports. */ + exportfile?: string; + jsonmode?: "extended" | "id"; + nonCanonicalFacts?: boolean; + noDedupArticles?: boolean; + cluster?: string; + /** Free-form body fields for anything not named above (e.g. `report`, `view`). A named key here throws `ValidationError`. */ + extra?: ExtraParams; raw?: boolean; } -function buildDqlParams( - client: DiffbotClient, - query: string, - options: DqlOptions, -): Record { - const params: Record = { - token: client.token, - query, - size: options.size ?? 10, - }; - if (options.from) params.from = options.from; - if (options.format && options.format !== "json") params.format = options.format; - if (options.filter !== undefined) params.filter = options.filter; - if (options.exportspec !== undefined) params.exportspec = options.exportspec; - if (options.extra) Object.assign(params, options.extra); - return params; +type DqlResultFor = O extends { format: "jsonl" } + ? AsyncGenerator + : O extends { raw: true } + ? Uint8Array + : O extends { format: "xls" | "xlsx" } + ? Uint8Array + : O extends { format: "csv" } + ? string + : DqlResponse; + +/** JSON body for `POST /kg/v3/dql` (`DqlPostRequest`). The token stays in the query string. */ +function buildDqlBody(query: string, options: DqlOptions): Record { + if (options.extra) { + const named: readonly string[] = DQL_NAMED_PARAMS; + const clash = Object.keys(options.extra).filter((key) => named.includes(key)); + if (clash.length) { + throw new ValidationError( + `dql(): ${clash.join(", ")} cannot be passed through extra; use the named option instead`, + ); + } + } + const body: Record = { query, size: options.size ?? 10 }; + if (options.type !== undefined) body.type = options.type; + if (options.col !== undefined) body.col = options.col; + if (options.from) body.from = options.from; + if (options.format && options.format !== "json") body.format = options.format; + if (options.filter !== undefined) body.filter = options.filter; + if (options.filterExclude !== undefined) body.filterExclude = options.filterExclude; + if (options.exportspec !== undefined) body.exportspec = options.exportspec; + if (options.exportseparator !== undefined) body.exportseparator = options.exportseparator; + if (options.exportfile !== undefined) body.exportfile = options.exportfile; + if (options.jsonmode !== undefined) body.jsonmode = options.jsonmode; + if (options.nonCanonicalFacts !== undefined) body.nonCanonicalFacts = options.nonCanonicalFacts; + if (options.noDedupArticles !== undefined) body.noDedupArticles = options.noDedupArticles; + if (options.cluster !== undefined) body.cluster = options.cluster; + if (options.extra) Object.assign(body, options.extra); + return body; +} + +function resolveDqlKind(format: ExportFormat | undefined, raw: boolean | undefined): "json" | "text" | "bytes" | "ndjson" { + if (format === "jsonl") return "ndjson"; + if (raw) return "bytes"; + if (format === "xls" || format === "xlsx") return "bytes"; + if (format === "csv") return "text"; + return "json"; } +export async function dql>( + client: DiffbotClient, + query: string, + options?: O, +): Promise>; export async function dql( client: DiffbotClient, query: string, options: DqlOptions = {}, -): Promise { - const params = buildDqlParams(client, query, options); - const response = await client.http.get(client.dqlUrl, { params }); - const body = await readBodyText(response); - raiseForStatus(response, body); - if (options.raw) { - return new TextEncoder().encode(body); +): Promise> { + const response = await client.http.post(client.dqlUrl, { + params: { token: client.token }, + json: buildDqlBody(query, options), + }); + await ensureOk(response); + switch (resolveDqlKind(options.format, options.raw)) { + case "ndjson": + return readNdjson(response); + case "bytes": + return readBytes(response); + case "text": + return readText(response); + default: + return parseJsonText(await readText(response)); } - return JSON.parse(body) as JsonObject; } +export async function dqlParallel>( + client: DiffbotClient, + queries: Q, + workers?: number, +): Promise<{ [K in keyof Q]: DqlResultFor }>; export async function dqlParallel( client: DiffbotClient, - queries: Array, + queries: ReadonlyArray, workers = 8, -): Promise> { +): Promise>> { if (!queries.length) return []; const limit = Math.min(workers, queries.length); - const results: Array = new Array(queries.length); + const results = new Array>( + queries.length, + ); let index = 0; async function worker(): Promise { - while (true) { + for (;;) { const current = index++; - if (current >= queries.length) return; - const { query, ...opts } = queries[current]!; + const entry = queries[current]; + if (entry === undefined) return; + const { query, ...opts } = entry; results[current] = await dql(client, query, opts); } } @@ -73,9 +164,8 @@ export async function dqlParallel( /** Fetch the raw ontology JSON text. For a parsed, cached `Ontology`, see `OntologyStore`. */ export async function dqlFetchOntologyText(client: DiffbotClient): Promise { const response = await client.http.get(client.ontologyUrl); - const body = await readBodyText(response); - raiseForStatus(response, body); - return body; + await ensureOk(response); + return readText(response); } export async function dqlFetchOntology(client: DiffbotClient): Promise { diff --git a/src/nlp.ts b/src/nlp.ts index 349e71e..b6697ab 100644 --- a/src/nlp.ts +++ b/src/nlp.ts @@ -1,19 +1,61 @@ import type { DiffbotClient } from "./client.js"; -import { readBodyText, raiseForStatus } from "./http.js"; -import type { JsonObject } from "./json.js"; +import { parseJsonText, readText, unsafeCast } from "./decode.js"; +import { ValidationError } from "./errors.js"; +import { ensureOk } from "./http.js"; +import type { NaturalLanguageResponse, NlpDocument, NlpField } from "./types/natural-language.js"; + +export type { NaturalLanguageResponse, NlpDocument, NlpField } from "./types/natural-language.js"; const NLP_FIELDS = "entities,sentiment"; -export async function entities( - client: DiffbotClient, - text: string, - lang = "auto", -): Promise { +/** Narrowed to the two fields `entities()` always requests. */ +export type EntitiesResponse = Pick; + +export async function entities(client: DiffbotClient, text: string, lang = "auto"): Promise { const params = { token: client.token, fields: NLP_FIELDS }; const payload = [{ lang, format: "plain text", content: text }]; const response = await client.http.post(client.nlpUrl, { params, json: payload }); - const body = await readBodyText(response); - raiseForStatus(response, body); - const data = JSON.parse(body) as JsonObject | JsonObject[]; + await ensureOk(response); + const data = parseJsonText(await readText(response)); return Array.isArray(data) ? (data[0] ?? {}) : data; } + +export interface NlpOptions { + /** Which analyses to run (wire `fields`, comma-joined). At least one. */ + fields: readonly F[]; + /** ISO 639-1 code, or `"auto"` (default). */ + lang?: string; + /** `"plain text with title"` treats the first line as the title. Default `"plain text"`. */ + format?: NlpDocument["format"]; + documentType?: NonNullable; + /** Used with `fields: ["summary"]`. */ + customSummary?: { maxNumberOfSentences: number }; +} + +/** The response narrowed to the requested fields. */ +export type NlpResponse = Pick; + +/** + * Full-fidelity call on the NL endpoint for a single document. `entities()` is the + * fixed `entities,sentiment` shortcut; multi-document batches are not supported. + */ +export async function nlp( + client: DiffbotClient, + text: string, + options: NlpOptions, +): Promise> { + if (!options.fields.length) throw new ValidationError("nlp(): fields must name at least one field"); + const params = { token: client.token, fields: options.fields.join(",") }; + const document: Record = { + lang: options.lang ?? "auto", + format: options.format ?? "plain text", + content: text, + }; + if (options.documentType !== undefined) document.documentType = options.documentType; + if (options.customSummary !== undefined) document.customSummary = options.customSummary; + const response = await client.http.post(client.nlpUrl, { params, json: [document] }); + await ensureOk(response); + const data = parseJsonText | NlpResponse[]>(await readText(response)); + // Every response field is optional, so `{}` is a valid (empty) result. + return Array.isArray(data) ? (data[0] ?? unsafeCast>({})) : data; +} diff --git a/src/ontology.ts b/src/ontology.ts index 86cb5e5..6d0f05c 100644 --- a/src/ontology.ts +++ b/src/ontology.ts @@ -1,49 +1,55 @@ -import type { JsonObject } from "./json.js"; +import { parseJsonText } from "./decode.js"; +import type { + FieldMeta, + OntologyDocument, + OntologyTypeDef, + TaxonomyNode, +} from "./types/undocumented.js"; + +export type { FieldMeta, OntologyDocument } from "./types/undocumented.js"; -type FieldMeta = JsonObject; type FieldEntry = [string, FieldMeta]; export class Ontology { - readonly data: JsonObject; + readonly data: OntologyDocument; - constructor(data: JsonObject) { + constructor(data: OntologyDocument) { this.data = data; } static fromJson(raw: string | Uint8Array): Ontology { const text = typeof raw === "string" ? raw : new TextDecoder().decode(raw); - return new Ontology(JSON.parse(text) as JsonObject); + return new Ontology(parseJsonText(text)); } types(): string[] { - const types = this.data.types as Record | undefined; - return Object.keys(types ?? {}).sort(); + return Object.keys(this.data.types ?? {}).sort(); } composites(): string[] { - const composites = this.data.composites as Record | undefined; - return Object.keys(composites ?? {}).sort(); + return Object.keys(this.data.composites ?? {}).sort(); } enums(): string[] { - const enums = this.data.enums as Record | undefined; - return Object.keys(enums ?? {}).sort(); + return Object.keys(this.data.enums ?? {}).sort(); } taxonomies(): string[] { - const taxonomies = this.data.taxonomies as Record | undefined; - return Object.keys(taxonomies ?? {}).sort(); + return Object.keys(this.data.taxonomies ?? {}).sort(); } - private static fieldsOf(container: Record, typeName: string): Record { + private static fieldsOf( + container: Record, + typeName: string, + ): Record { const entry = container[typeName]; if (!entry) throw new Error(`Unknown name: ${typeName}`); - return (entry.fields as Record | undefined) ?? {}; + return entry.fields ?? {}; } fieldsFor(typeName: string): Record { - const types = (this.data.types ?? {}) as Record; - const composites = (this.data.composites ?? {}) as Record; + const types = this.data.types ?? {}; + const composites = this.data.composites ?? {}; if (typeName in types) return Ontology.fieldsOf(types, typeName); if (typeName in composites) return Ontology.fieldsOf(composites, typeName); throw new Error(`${typeName} is not a known entity type or composite`); @@ -65,34 +71,32 @@ export class Ontology { } taxonomyValues(name: string, search?: string | null): string[] { - const taxonomies = (this.data.taxonomies ?? {}) as Record; - const tax = taxonomies[name]; + const tax = (this.data.taxonomies ?? {})[name]; if (!tax) throw new Error(`Unknown taxonomy: ${name}`); const pattern = search ? new RegExp(search, "i") : null; const out: string[] = []; - const walk = (node: JsonObject): void => { + const walk = (node: TaxonomyNode): void => { const n = node.name; if (typeof n === "string" && (!pattern || pattern.test(n))) { out.push(n); } - for (const child of (node.children as JsonObject[] | undefined) ?? []) { + for (const child of node.children ?? []) { walk(child); } }; - for (const cat of (tax.categories as JsonObject[] | undefined) ?? []) { + for (const cat of tax.categories ?? []) { walk(cat); } return out; } enumValues(name: string): string[] { - const enums = (this.data.enums ?? {}) as Record; - const enumEntry = enums[name]; + const enumEntry = (this.data.enums ?? {})[name]; if (!enumEntry) throw new Error(`Unknown enum: ${name}`); - return [...((enumEntry.values as string[] | undefined) ?? [])]; + return [...(enumEntry.values ?? [])]; } findNamed(search: string): string[] { @@ -103,10 +107,12 @@ export class Ontology { if (Array.isArray(node)) { for (const v of node) walk(v); } else if (node && typeof node === "object") { - const obj = node as JsonObject; - const n = obj.name; - if (typeof n === "string" && pattern.test(n)) found.add(n); - for (const v of Object.values(obj)) walk(v); + for (const [key, value] of Object.entries(node)) { + if (key === "name" && typeof value === "string" && pattern.test(value)) { + found.add(value); + } + walk(value); + } } }; @@ -115,10 +121,10 @@ export class Ontology { } static formatField(name: string, meta: FieldMeta): string { - let typeName = String(meta.type ?? "?"); + let typeName = meta.type ?? "?"; if (typeName === "LinkedEntity") { - const le = meta.leType as string[] | undefined; - if (le?.length) typeName = `LinkedEntity (${le[0]})`; + const first = meta.leType?.[0]; + if (first) typeName = `LinkedEntity (${first})`; } const flags: string[] = []; if (meta.isList) flags.push("isList"); diff --git a/src/types/account.ts b/src/types/account.ts new file mode 100644 index 0000000..b3a6b00 --- /dev/null +++ b/src/types/account.ts @@ -0,0 +1,9 @@ +/** Hand-written facade over `src/generated/account.ts`. */ +import type { operations } from "../generated/account.js"; +import type { JsonObject } from "../json.js"; + +export type AccountResponse = operations["account"]["responses"][200]["content"]["application/json"]; +export type AccountUsage = NonNullable[number]; + +type _AccountResponseIsJsonObject = AccountResponse extends JsonObject ? true : never; +const _accountResponseIsJsonObject: _AccountResponseIsJsonObject = true; diff --git a/src/types/dql.ts b/src/types/dql.ts new file mode 100644 index 0000000..9329187 --- /dev/null +++ b/src/types/dql.ts @@ -0,0 +1,13 @@ +/** Hand-written facade over `src/generated/dql.ts`, kept assignable to `JsonObject`. */ +import type { components } from "../generated/dql.js"; +import type { JsonObject } from "../json.js"; + +export type DqlResponse = components["schemas"]["DqlResponse"]; +export type DqlEntity = components["schemas"]["Data"]; +export type DqlRewrite = components["schemas"]["Rewrite"]; +export type ExportFormat = "json" | "jsonl" | "csv" | "xls" | "xlsx"; + +// Compile-time guard: response overlays must stay assignable to JsonObject (see +// typesafety-revision.md) so a server-added field never breaks a caller. +type _DqlResponseIsJsonObject = DqlResponse extends JsonObject ? true : never; +const _dqlResponseIsJsonObject: _DqlResponseIsJsonObject = true; diff --git a/src/types/extra.ts b/src/types/extra.ts new file mode 100644 index 0000000..e8df2b8 --- /dev/null +++ b/src/types/extra.ts @@ -0,0 +1,7 @@ +/** + * The shape of a handler's `extra` escape hatch: a free-form bag of wire query + * parameters, minus the keys the handler already exposes as named options. Passing + * a named key through `extra` is a compile-time error instead of a silent no-op or an + * ambiguous "which one wins" footgun at runtime. + */ +export type ExtraParams = Record & Partial>; diff --git a/src/types/natural-language.ts b/src/types/natural-language.ts new file mode 100644 index 0000000..6535799 --- /dev/null +++ b/src/types/natural-language.ts @@ -0,0 +1,10 @@ +/** Hand-written facade over `src/generated/natural-language.ts`. */ +import type { components } from "../generated/natural-language.js"; +import type { JsonObject } from "../json.js"; + +export type NaturalLanguageResponse = components["schemas"]["NaturalLanguageResponse"]; +export type NlpDocument = components["schemas"]["Document"]; +export type NlpField = "entities" | "sentiment" | "facts" | "records" | "categories" | "sentences" | "language" | "summary"; + +type _NaturalLanguageResponseIsJsonObject = NaturalLanguageResponse extends JsonObject ? true : never; +const _naturalLanguageResponseIsJsonObject: _NaturalLanguageResponseIsJsonObject = true; diff --git a/src/types/undocumented.ts b/src/types/undocumented.ts new file mode 100644 index 0000000..d43bf0e --- /dev/null +++ b/src/types/undocumented.ts @@ -0,0 +1,162 @@ +/** + * Types for surfaces this SDK ships that no Diffbot OpenAPI spec or docs page + * describes: the ontology JSON document, the Ask/RAG endpoint, Extract's per-api + * object shapes (the spec gives examples only, no response schema), and the + * Crawl/Bulk job record (same — examples only, see `docs__crawl__create.md`). + * + * Every type here is weaker evidence than one generated from `src/generated/*`: + * the server is free to change these fields without it being a documented + * breaking change. Keep members optional and interfaces open (extending + * `JsonObject`) so an unexpected field never produces a type error. + */ +import type { JsonObject } from "../json.js"; + +// ---- Ontology (kg.diffbot.com/kg/ontology — not a documented API operation) ---- + +export interface FieldMeta extends JsonObject { + type?: string; + isList?: boolean; + isComposite?: boolean; + isEnum?: boolean; + isDeprecated?: boolean; + leType?: string[]; +} + +export interface OntologyTypeDef extends JsonObject { + fields?: Record; +} + +export interface TaxonomyNode extends JsonObject { + name?: string; + children?: TaxonomyNode[]; +} + +export interface Taxonomy extends JsonObject { + categories?: TaxonomyNode[]; +} + +export interface OntologyEnum extends JsonObject { + values?: string[]; +} + +export interface OntologyDocument extends JsonObject { + types?: Record; + composites?: Record; + enums?: Record; + taxonomies?: Record; +} + +// ---- Ask / RAG (llm.diffbot.com/rag/v1/chat/completions — undocumented) ---- + +export type ChatRole = "system" | "user" | "assistant"; + +export interface ChatCompletionRequest extends JsonObject { + model: string; + messages: Array<{ role: ChatRole; content: string }>; + stream?: boolean; + response_format?: JsonObject; +} + +export interface ChatCompletionChunk extends JsonObject { + choices?: Array<{ delta?: { content?: string } }>; +} + +// ---- Crawl / Bulk job record (crawl.json/bulk.json document examples only) ---- + +export interface JobStatus extends JsonObject { + status?: number; + message?: string; +} + +export interface CrawlJob extends JsonObject { + name?: string; + jobStatus?: JobStatus; + type?: string; + seeds?: string[]; + roundsCompleted?: number; + pageCrawlAttempts?: number; + pageCrawlSuccesses?: number; + pageCrawlSuccessesThisRound?: number; + pagesInQueue?: number; + downloadJson?: string; + downloadUrls?: string; +} + +/** + * One record of `GET /v3/crawl/data` (JSON). Shape from `crawl.json`'s response schema + * (the keys below are required there); all other keys follow the ontology of `type`. + * Note that the server includes your token in each record: don't log them verbatim. + */ +export interface CrawlDataRecord extends JsonObject { + col?: string; + parentUrl?: string; + pageUrl?: string; + id?: number; + type?: string; + title?: string; + content?: string; + token?: string; + timestamp?: string; +} + +// ---- Extract (extract.json documents examples only, no response schema) ---- + +export interface ExtractResponse extends JsonObject { + request?: JsonObject; + objects?: T[]; + humanLanguage?: string; + errorCode?: number; + error?: string; +} + +interface DiffbotObjectBase extends JsonObject { + type?: string; + title?: string; + pageUrl?: string; + diffbotUri?: string; +} + +export type AnalyzeObject = DiffbotObjectBase; + +export interface ArticleObject extends DiffbotObjectBase { + text?: string; + html?: string; + author?: string; + date?: string; + tags?: Array<{ label?: string; score?: number }>; +} + +export interface ProductObject extends DiffbotObjectBase { + offerPrice?: string; + regularPrice?: string; + availability?: boolean; + productId?: string; +} + +export interface DiscussionObject extends DiffbotObjectBase { + text?: string; + posts?: JsonObject[]; +} + +export type JobObject = DiffbotObjectBase; + +export interface ImageObject extends DiffbotObjectBase { + naturalHeight?: number; + naturalWidth?: number; +} + +export interface VideoObject extends DiffbotObjectBase { + duration?: number; +} + +export interface ListObject extends DiffbotObjectBase { + items?: JsonObject[]; +} + +export interface EventObject extends DiffbotObjectBase { + startDate?: string; + endDate?: string; +} + +/** A Custom API's object shape is caller-defined; keep it open by design. */ +export type CustomApiObject = JsonObject; diff --git a/src/types/web-search.ts b/src/types/web-search.ts new file mode 100644 index 0000000..d3f2510 --- /dev/null +++ b/src/types/web-search.ts @@ -0,0 +1,9 @@ +/** Hand-written facade over `src/generated/web-search.ts`. */ +import type { operations } from "../generated/web-search.js"; +import type { JsonObject } from "../json.js"; + +export type WebSearchResponse = operations["get_"]["responses"][200]["content"]["application/json"]; +export type WebSearchResult = NonNullable[number]; + +type _WebSearchResponseIsJsonObject = WebSearchResponse extends JsonObject ? true : never; +const _webSearchResponseIsJsonObject: _WebSearchResponseIsJsonObject = true; diff --git a/src/web-search.ts b/src/web-search.ts index 48cc25e..592ae04 100644 --- a/src/web-search.ts +++ b/src/web-search.ts @@ -1,13 +1,21 @@ import type { DiffbotClient } from "./client.js"; +import { parseJsonText, readText } from "./decode.js"; import { ValidationError } from "./errors.js"; -import { readBodyText, raiseForStatus } from "./http.js"; -import type { JsonObject } from "./json.js"; +import { ensureOk } from "./http.js"; +import type { WebSearchResponse } from "./types/web-search.js"; + +export type { WebSearchResponse, WebSearchResult } from "./types/web-search.js"; /** How much of each page's matching content the web search returns in `content`. */ export type SnippetLength = "short" | "medium" | "long" | "full"; export const SNIPPET_LENGTHS: readonly SnippetLength[] = ["short", "medium", "long", "full"]; +function isSnippetLength(value: string): value is SnippetLength { + const lengths: readonly string[] = SNIPPET_LENGTHS; + return lengths.includes(value); +} + export interface WebSearchOptions { numResults?: number; maxTokens?: number; @@ -19,18 +27,31 @@ export interface WebSearchOptions { snippetLength?: SnippetLength; } +/** The API accepts 1-5 `text` values per request. */ +export const WEB_SEARCH_MAX_TEXTS = 5; + +/** + * `text` may be one query or 1-5 queries (sent as repeated `text=` params on the GET). + * A single string is the default path. Any other count throws `ValidationError`. + */ export async function webSearch( client: DiffbotClient, - text: string, + text: string | readonly string[], options: WebSearchOptions = {}, -): Promise { +): Promise { + const texts = typeof text === "string" ? [text] : text; + if (texts.length < 1 || texts.length > WEB_SEARCH_MAX_TEXTS) { + throw new ValidationError(`webSearch text must have 1-${WEB_SEARCH_MAX_TEXTS.toString()} values; got ${texts.length.toString()}`); + } const headers = { Authorization: `Bearer ${client.token}` }; - const params: Record = { text }; + const params: Record = { + text: typeof text === "string" ? text : [...texts], + }; if (options.numResults !== undefined) params.size = options.numResults; if (options.maxTokens !== undefined) params.maxTokens = options.maxTokens; if (options.snippetLength !== undefined) { - const length = options.snippetLength.toLowerCase() as SnippetLength; - if (!SNIPPET_LENGTHS.includes(length)) { + const length = options.snippetLength.toLowerCase(); + if (!isSnippetLength(length)) { throw new ValidationError( `snippetLength must be one of ${SNIPPET_LENGTHS.join(", ")}; got ${JSON.stringify(options.snippetLength)}`, ); @@ -40,7 +61,6 @@ export async function webSearch( } const response = await client.http.get(client.webSearchUrl, { headers, params }); - const body = await readBodyText(response); - raiseForStatus(response, body); - return JSON.parse(body) as JsonObject; + await ensureOk(response); + return parseJsonText(await readText(response)); } diff --git a/tests/account.test.ts b/tests/account.test.ts new file mode 100644 index 0000000..0729f33 --- /dev/null +++ b/tests/account.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { DiffbotClient, account, verifyToken } from "../src/index.js"; +import { createMockFetch, jsonResponse } from "./helpers/mock-fetch.js"; + +function makeClient(handler: Parameters[0], extra: { retry?: false } = {}) { + return new DiffbotClient({ token: "test-token", fetch: createMockFetch(handler), ...extra }); +} + +describe("account", () => { + it("GETs /v4/account with the token and no days by default", async () => { + const db = makeClient((req) => { + expect(req.method).toBe("GET"); + expect(req.url).toBe("https://api.diffbot.com/v4/account"); + expect(req.params.get("token")).toBe("test-token"); + expect(req.params.has("days")).toBe(false); + return jsonResponse({ name: "Diffy", plan: "p", status: "active", usage: [{ date: "2022-05-04", credits: 1 }] }); + }); + const result = await account(db); + expect(result.status).toBe("active"); + expect(result.usage?.[0]?.credits).toBe(1); + }); + + it("forwards days", async () => { + const db = makeClient((req) => { + expect(req.params.get("days")).toBe("365"); + return jsonResponse({}); + }); + await account(db, { days: 365 }); + }); + + it("honors apiBase", async () => { + const db = new DiffbotClient({ + token: "t", + apiBase: "http://localhost:9000", + fetch: createMockFetch((req) => { + expect(req.url).toBe("http://localhost:9000/v4/account"); + return jsonResponse({}); + }), + }); + await account(db); + }); +}); + +describe("verifyToken", () => { + it("is true on success and asks for a single day", async () => { + const db = makeClient((req) => { + expect(req.params.get("days")).toBe("1"); + return jsonResponse({ status: "active" }); + }); + expect(await verifyToken(db)).toBe(true); + }); + + it.each([401, 403])("is false on %i (AuthError)", async (status) => { + const db = makeClient(() => jsonResponse({ code: status, message: "bad token" }, status)); + expect(await verifyToken(db)).toBe(false); + }); + + it("propagates non-auth errors", async () => { + const db = makeClient(() => jsonResponse({ message: "boom" }, 500), { retry: false }); + await expect(verifyToken(db)).rejects.toMatchObject({ statusCode: 500 }); + const limited = makeClient(() => jsonResponse({ message: "slow" }, 429), { retry: false }); + await expect(verifyToken(limited)).rejects.toMatchObject({ name: "RateLimitError" }); + }); + + it("propagates network errors", async () => { + const db = new DiffbotClient({ + token: "t", + retry: false, + fetch: () => Promise.reject(new TypeError("fetch failed")), + }); + await expect(verifyToken(db)).rejects.toThrow("fetch failed"); + }); +}); diff --git a/tests/ask.test.ts b/tests/ask.test.ts index 8dfb2ae..171c714 100644 --- a/tests/ask.test.ts +++ b/tests/ask.test.ts @@ -218,4 +218,31 @@ describe("askJson", () => { { city: "NYC" }, ]); }); + + it("checks the result with a parse hook instead of an unchecked cast", async () => { + const { db } = captureBody('{"capital": "Paris"}'); + const seen: unknown[] = []; + const result = await askJson(db, [{ role: "user", content: "x" }], undefined, { + parse: (value) => { + seen.push(value); + if (typeof value !== "object" || value === null || !("capital" in value)) { + throw new Error("missing capital"); + } + return { capital: String((value as { capital: unknown }).capital) }; + }, + }); + expect(seen).toEqual([{ capital: "Paris" }]); + expect(result).toEqual({ capital: "Paris" }); + }); + + it("propagates a parse hook's rejection", async () => { + const { db } = captureBody('{"capital": "Paris"}'); + await expect( + askJson(db, [{ role: "user", content: "x" }], undefined, { + parse: () => { + throw new Error("validation failed"); + }, + }), + ).rejects.toThrow("validation failed"); + }); }); diff --git a/tests/client.test.ts b/tests/client.test.ts new file mode 100644 index 0000000..4fc2c44 --- /dev/null +++ b/tests/client.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { DiffbotClient } from "../src/index.js"; + +describe("DiffbotClient per-host bases", () => { + it("defaults every endpoint to the public hosts", () => { + const db = new DiffbotClient({ token: "t" }); + expect(db.analyzeUrl).toBe("https://api.diffbot.com/v3"); + expect(db.crawlerUrl).toBe("https://api.diffbot.com/v3/crawl"); + expect(db.llmUrl).toBe("https://llm.diffbot.com/rag/v1/chat/completions"); + expect(db.webSearchUrl).toBe("https://llm.diffbot.com/api/v1/web_search"); + expect(db.nlpUrl).toBe("https://nl.diffbot.com/v1/"); + expect(db.dqlUrl).toBe("https://kg.diffbot.com/kg/v3/dql"); + expect(db.ontologyUrl).toBe("https://kg.diffbot.com/kg/ontology"); + }); + + it("derives every api.diffbot.com endpoint from a single apiBase override", () => { + const db = new DiffbotClient({ token: "t", apiBase: "https://staging.example.com" }); + expect(db.analyzeUrl).toBe("https://staging.example.com/v3"); + expect(db.crawlerUrl).toBe("https://staging.example.com/v3/crawl"); + }); + + it("derives every kg.diffbot.com endpoint from a single kgBase override", () => { + const db = new DiffbotClient({ token: "t", kgBase: "https://kg.staging.example.com" }); + expect(db.dqlUrl).toBe("https://kg.staging.example.com/kg/v3/dql"); + expect(db.ontologyUrl).toBe("https://kg.staging.example.com/kg/ontology"); + }); + + it("derives every llm.diffbot.com endpoint from a single llmBase override", () => { + const db = new DiffbotClient({ token: "t", llmBase: "https://llm.staging.example.com" }); + expect(db.llmUrl).toBe("https://llm.staging.example.com/rag/v1/chat/completions"); + expect(db.webSearchUrl).toBe("https://llm.staging.example.com/api/v1/web_search"); + }); + + it("derives the NL endpoint from nlpBase", () => { + const db = new DiffbotClient({ token: "t", nlpBase: "https://nl.staging.example.com" }); + expect(db.nlpUrl).toBe("https://nl.staging.example.com/v1/"); + }); + + it("lets the legacy per-endpoint option win over a host base", () => { + const db = new DiffbotClient({ + token: "t", + apiBase: "https://staging.example.com", + analyzeUrl: "https://custom.example.com/v3", + }); + expect(db.analyzeUrl).toBe("https://custom.example.com/v3"); + expect(db.crawlerUrl).toBe("https://staging.example.com/v3/crawl"); + }); +}); diff --git a/tests/crawl.test.ts b/tests/crawl.test.ts index d86dc19..fde5926 100644 --- a/tests/crawl.test.ts +++ b/tests/crawl.test.ts @@ -1,5 +1,16 @@ import { describe, expect, it } from "vitest"; -import { AuthError, CrawlEventType, DiffbotClient, crawl } from "../src/index.js"; +import { + APIError, + AuthError, + CrawlEventType, + DiffbotClient, + JobError, + crawl, + crawlDeleteJob, + crawlDownload, + crawlGetJob, + crawlListJobs, +} from "../src/index.js"; import { createMockFetch, jsonResponse, type MockHandler } from "./helpers/mock-fetch.js"; function makeClient(handler: MockHandler): DiffbotClient { @@ -31,6 +42,78 @@ describe("crawl", () => { expect(params!.get("seeds")).toBe("https://example.com"); }); + it("sends the named create options as query params", async () => { + let params: URLSearchParams | undefined; + const db = makeClient((req) => { + params = req.params; + return jsonResponse({ jobs: [{ name: "test-job" }] }); + }); + await crawl(db, "https://example.com", { + jobName: "test-job", + urlCrawlRegEx: "^https://example\\.com/a", + urlProcessRegEx: "/p/", + pageProcessPattern: "In stock", + maxToCrawlPerSubdomain: 50, + maxToProcessPerSubdomain: 0, + useCanonical: false, + notifyWebhook: "https://hooks.example/x", + notifyEmail: "me@example.com", + }).next(); + expect(params!.get("urlCrawlRegEx")).toBe("^https://example\\.com/a"); + expect(params!.get("urlProcessRegEx")).toBe("/p/"); + expect(params!.get("pageProcessPattern")).toBe("In stock"); + expect(params!.get("maxToCrawlPerSubdomain")).toBe("50"); + expect(params!.get("maxToProcessPerSubdomain")).toBe("0"); + expect(params!.get("useCanonical")).toBe("0"); + expect(params!.get("notifyWebhook")).toBe("https://hooks.example/x"); + expect(params!.get("notifyEmail")).toBe("me@example.com"); + }); + + it("omits the new options when unset", async () => { + let params: URLSearchParams | undefined; + const db = makeClient((req) => { + params = req.params; + return jsonResponse({ jobs: [{ name: "test-job" }] }); + }); + await crawl(db, "https://example.com", { jobName: "test-job" }).next(); + for (const key of ["urlCrawlRegEx", "useCanonical", "notifyEmail", "customHeaders", "maxToCrawlPerSubdomain"]) { + expect(params!.has(key)).toBe(false); + } + }); + + it("sends a single customHeaders string once", async () => { + let params: URLSearchParams | undefined; + const db = makeClient((req) => { + params = req.params; + return jsonResponse({ jobs: [{ name: "test-job" }] }); + }); + await crawl(db, "https://example.com", { jobName: "test-job", customHeaders: "Accept-Language:en-us" }).next(); + expect(params!.getAll("customHeaders")).toEqual(["Accept-Language:en-us"]); + }); + + it("sends customHeaders arrays as repeated params", async () => { + let params: URLSearchParams | undefined; + const db = makeClient((req) => { + params = req.params; + return jsonResponse({ jobs: [{ name: "test-job" }] }); + }); + await crawl(db, "https://example.com", { + jobName: "test-job", + customHeaders: ["Accept-Language:en-us", "X-Test:1"], + }).next(); + expect(params!.getAll("customHeaders")).toEqual(["Accept-Language:en-us", "X-Test:1"]); + }); + + it("omits customHeaders for an empty array", async () => { + let params: URLSearchParams | undefined; + const db = makeClient((req) => { + params = req.params; + return jsonResponse({ jobs: [{ name: "test-job" }] }); + }); + await crawl(db, "https://example.com", { jobName: "test-job", customHeaders: [] }).next(); + expect(params!.has("customHeaders")).toBe(false); + }); + it("normalizes the url", async () => { let seeds = ""; const db = makeClient((req) => { @@ -41,6 +124,14 @@ describe("crawl", () => { expect(seeds).toBe("https://example.com"); }); + it("raises APIError when create answers HTTP 200 with an errorCode", async () => { + const db = makeClient(() => jsonResponse({ errorCode: 505, error: "name is over 30 chars" })); + const iterator = crawl(db, "https://example.com", { jobName: "x".repeat(31) }); + const error: unknown = await iterator.next().catch((e: unknown) => e); + expect(error).toBeInstanceOf(APIError); + expect(error).toMatchObject({ statusCode: 200, apiMessage: "name is over 30 chars" }); + }); + it("raises AuthError on 401", async () => { const db = makeClient(() => jsonResponse( @@ -62,4 +153,125 @@ describe("crawl", () => { }); await expect(crawl(db, "https://example.com").next()).rejects.toBeInstanceOf(AuthError); }); + + it("raises JobError when a watched job reports failure, parsing quoted CSV fields", async () => { + let call = 0; + const db = makeClient(() => { + call++; + switch (call) { + case 1: // crawl creation + return jsonResponse({ jobs: [{ name: "test-job" }] }); + case 2: // status poll, iteration 1: still in progress + return jsonResponse({ jobs: [{ name: "test-job", jobStatus: { status: 7 } }] }); + case 3: // URL report, iteration 1 + return new Response( + 'Url,Crawled Time,Crawl Status\n"https://example.com/a,b",t,200\n', + { status: 200, headers: { "Content-Type": "text/csv" } }, + ); + case 4: // status poll, iteration 2: failed + return jsonResponse({ + jobs: [{ name: "test-job", jobStatus: { status: 10, message: "crawl failed: timeout" } }], + }); + default: // URL report, iteration 2 — read before the failure is raised + return new Response("Url,Crawled Time,Crawl Status\n", { + status: 200, + headers: { "Content-Type": "text/csv" }, + }); + } + }); + + const events: Array<{ details: Record }> = []; + await expect( + (async () => { + for await (const event of crawl(db, "https://example.com", { + jobName: "test-job", + watch: true, + pollInterval: 0, + })) { + events.push(event); + } + })(), + ).rejects.toBeInstanceOf(JobError); + + const urlEvent = events.find((e) => "url" in e.details); + // The quoted field "https://example.com/a,b" must survive as one URL, not be split on its comma. + expect(urlEvent?.details.url).toBe("https://example.com/a,b"); + }); + + it("crawlGetJob returns undefined for a missing job, not {}", async () => { + const db = makeClient(() => jsonResponse({ jobs: [] })); + expect(await crawlGetJob(db, "nope")).toBeUndefined(); + }); + + it("crawlGetJob returns the job record when found", async () => { + const db = makeClient(() => jsonResponse({ jobs: [{ name: "test-job" }] })); + expect(await crawlGetJob(db, "test-job")).toEqual({ name: "test-job" }); + }); + + it("crawlListJobs returns an empty array when there are no jobs", async () => { + const db = makeClient(() => jsonResponse({ jobs: [] })); + expect(await crawlListJobs(db)).toEqual([]); + }); + + it("does not retry crawlDeleteJob (a GET-shaped mutation)", async () => { + let calls = 0; + const db = new DiffbotClient({ + token: "test-token", + retry: { baseDelayMs: 1, maxDelayMs: 1 }, + fetch: createMockFetch(() => { + calls++; + return new Response("", { status: 503 }); + }), + }); + await expect(crawlDeleteJob(db, "test-job")).rejects.toMatchObject({ statusCode: 503 }); + expect(calls).toBe(1); + }); +}); + +describe("crawlDownload", () => { + it("GETs /v3/crawl/data with name only by default and parses JSON records", async () => { + const db = makeClient((req) => { + expect(req.method).toBe("GET"); + expect(req.url).toBe("https://api.diffbot.com/v3/crawl/data"); + expect(req.params.get("token")).toBe("test-token"); + expect(req.params.get("name")).toBe("job1"); + expect(req.params.has("format")).toBe(false); + expect(req.params.has("type")).toBe(false); + expect(req.params.has("num")).toBe(false); + return jsonResponse([{ col: "job1", pageUrl: "https://example.com/", type: "other" }]); + }); + const records = await crawlDownload(db, "job1"); + expect(records).toEqual([{ col: "job1", pageUrl: "https://example.com/", type: "other" }]); + }); + + it("returns CSV text for format: csv and forwards type/num", async () => { + const db = makeClient((req) => { + expect(req.params.get("format")).toBe("csv"); + expect(req.params.get("num")).toBe("5"); + return new Response("pageUrl,title\nhttps://example.com/,Example\n", { status: 200 }); + }); + const csv = await crawlDownload(db, "job1", { format: "csv", num: 5 }); + expect(csv).toBe("pageUrl,title\nhttps://example.com/,Example\n"); + }); + + it("returns the URL report as text (type: urls)", async () => { + const db = makeClient((req) => { + expect(req.params.get("type")).toBe("urls"); + expect(req.params.has("format")).toBe(false); + return new Response("Url,Crawled Time,Crawl Status\n", { status: 200 }); + }); + expect(await crawlDownload(db, "job1", { type: "urls" })).toBe("Url,Crawled Time,Crawl Status\n"); + }); + + it("returns [] for an empty JSON body", async () => { + const db = makeClient(() => new Response("", { status: 200 })); + expect(await crawlDownload(db, "job1")).toEqual([]); + }); + + it("raises on a 400 for an unknown job and on a 200 error object", async () => { + const missing = makeClient(() => jsonResponse({ errorCode: 400, error: "No collection record" }, 400)); + await expect(crawlDownload(missing, "nope")).rejects.toMatchObject({ statusCode: 400 }); + const odd = makeClient(() => jsonResponse({ errorCode: 400, error: "No collection record" })); + await expect(crawlDownload(odd, "nope")).rejects.toBeInstanceOf(JobError); + }); }); diff --git a/tests/extract.test.ts b/tests/extract.test.ts index 71d2c6c..a0beb83 100644 --- a/tests/extract.test.ts +++ b/tests/extract.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { DiffbotClient, ExtractionError, ValidationError, extract } from "../src/index.js"; import { createMockFetch, jsonResponse } from "./helpers/mock-fetch.js"; @@ -81,3 +81,111 @@ describe("extract", () => { expect(db.http.isClosed).toBe(true); }); }); + +describe("extract options", () => { + afterEach(() => vi.restoreAllMocks()); + + function capture(clientTimeout?: number) { + const seen: { params?: URLSearchParams } = {}; + const db = new DiffbotClient({ + token: "test-token", + timeout: clientTimeout, + fetch: createMockFetch((req) => { + seen.params = req.params; + return jsonResponse({ objects: [] }); + }), + }); + return { db, seen }; + } + + it("keeps the legacy wire defaults when no options are given", async () => { + const { db, seen } = capture(); + await extract(db, "example.com"); + expect(Object.fromEntries(seen.params!)).toEqual({ + token: "test-token", + url: "https://example.com", + timeout: "30000", + mode: "llm", + }); + }); + + it("sends named options, joining fields with commas", async () => { + const { db, seen } = capture(); + await extract(db, "example.com", "article", "json", { + fields: ["links", "meta"], + paging: false, + maxTags: 5, + tagConfidence: 0.5, + categoryConfidence: 0.25, + discussion: true, + naturalLanguage: ["entities", "sentiment"], + summaryNumSentences: 3, + useProxy: "default", + }); + expect(Object.fromEntries(seen.params!)).toMatchObject({ + fields: "links,meta", + paging: "false", + maxTags: "5", + tagConfidence: "0.5", + categoryConfidence: "0.25", + discussion: "true", + naturalLanguage: "entities,sentiment", + summaryNumSentences: "3", + useProxy: "default", + }); + expect(seen.params!.has("mode")).toBe(false); + }); + + it("sends discussion and analyze-only options", async () => { + const { db, seen } = capture(); + await extract(db, "example.com", "discussion", "json", { maxPages: 3 }); + expect(seen.params!.get("maxPages")).toBe("3"); + await extract(db, "example.com", "analyze", "json", { fallback: "article" }); + expect(seen.params!.get("fallback")).toBe("article"); + }); + + it("an explicit mode overrides the fmt=markdown mapping", async () => { + const { db, seen } = capture(); + await extract(db, "example.com", "analyze", "markdown", { mode: "article" }); + expect(seen.params!.get("mode")).toBe("article"); + await extract(db, "example.com", "analyze", "markdown"); + expect(seen.params!.get("mode")).toBe("llm"); + }); + + it("passes proxy/proxyAuth through typed extra", async () => { + const { db, seen } = capture(); + await extract(db, "example.com", "article", "json", { extra: { proxy: "host:80", proxyAuth: "u:p" } }); + expect(seen.params!.get("proxy")).toBe("host:80"); + expect(seen.params!.get("proxyAuth")).toBe("u:p"); + }); + + describe("timeout vs transport timeout", () => { + function spyTransport() { + const spy = vi.spyOn(AbortSignal, "timeout"); + return () => spy.mock.calls.map((c) => c[0]); + } + + it("sends a caller timeout on the wire and raises the transport timeout above it", async () => { + const calls = spyTransport(); + const { db, seen } = capture(30_000); + await extract(db, "example.com", "article", "json", { timeout: 90_000 }); + expect(seen.params!.get("timeout")).toBe("90000"); + expect(calls()).toEqual([100_000]); + }); + + it("keeps a larger client timeout when the caller's timeout is small", async () => { + const calls = spyTransport(); + const { db, seen } = capture(120_000); + await extract(db, "example.com", "article", "json", { timeout: 5_000 }); + expect(seen.params!.get("timeout")).toBe("5000"); + expect(calls()).toEqual([120_000]); + }); + + it("never aborts before the default 30s wire timeout can answer", async () => { + const calls = spyTransport(); + const { db } = capture(10_000); + await extract(db, "example.com"); + expect(calls()).toEqual([40_000]); + }); + }); +}); diff --git a/tests/helpers/mock-fetch.ts b/tests/helpers/mock-fetch.ts index 01dabec..7813b47 100644 --- a/tests/helpers/mock-fetch.ts +++ b/tests/helpers/mock-fetch.ts @@ -13,6 +13,8 @@ export type MockHandler = (request: MockRequest) => Response | Promise function toTextBody(body: RequestInit["body"]): string | undefined { if (body == null) return undefined; if (typeof body === "string") return body; + if (body instanceof URLSearchParams) return body.toString(); + if (body instanceof Uint8Array) return new TextDecoder().decode(body); return undefined; } diff --git a/tests/http.test.ts b/tests/http.test.ts new file mode 100644 index 0000000..4335352 --- /dev/null +++ b/tests/http.test.ts @@ -0,0 +1,275 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { APIError, NotFoundError, RateLimitError } from "../src/errors.js"; +import { ensureOk, HttpClient, raiseForStatus, type FetchFn } from "../src/http.js"; +import { createMockFetch, jsonResponse } from "./helpers/mock-fetch.js"; + +describe("HttpClient repeated params", () => { + it("appends array-valued params instead of overwriting", async () => { + const fetchFn = createMockFetch((req) => { + expect(req.params.getAll("tag")).toEqual(["a", "b", "c"]); + return jsonResponse({}); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", false); + await client.get("https://example.com/x", { params: { tag: ["a", "b", "c"] } }); + }); +}); + +describe("HttpClient DELETE, raw and form bodies", () => { + it("issues a DELETE with a JSON body", async () => { + const fetchFn = createMockFetch((req) => { + expect(req.method).toBe("DELETE"); + expect(req.headers.get("content-type")).toBe("application/json"); + expect(req.body).toBe(JSON.stringify({ api: "x" })); + return jsonResponse({}); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", false); + await client.delete("https://example.com/custom", { json: { api: "x" } }); + }); + + it("form-encodes a URLSearchParams body and sets Content-Type", async () => { + const fetchFn = createMockFetch((req) => { + expect(req.headers.get("content-type")).toContain("application/x-www-form-urlencoded"); + expect(req.body).toBe("a=1&b=2"); + return jsonResponse({}); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", false); + await client.post("https://example.com/x", { body: new URLSearchParams({ a: "1", b: "2" }) }); + }); + + it("lets a caller override the inferred Content-Type and send a raw body", async () => { + const fetchFn = createMockFetch((req) => { + expect(req.headers.get("content-type")).toBe("text/html"); + expect(req.body).toBe(""); + return jsonResponse({}); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", false); + await client.post("https://example.com/x", { body: "", contentType: "text/html" }); + }); + + it("respects a Content-Type already present in headers", async () => { + const fetchFn = createMockFetch((req) => { + expect(req.headers.get("content-type")).toBe("application/pdf"); + return jsonResponse({}); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", false); + await client.post("https://example.com/x", { + body: "pdf-bytes", + headers: { "Content-Type": "application/pdf" }, + }); + }); +}); + +describe("HttpClient per-call timeout", () => { + it("aborts sooner than the client default when a shorter timeout is given", async () => { + const fetchFn: FetchFn = (_input, init) => + new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve(jsonResponse({})), 200); + init?.signal?.addEventListener("abort", () => { + clearTimeout(timer); + reject(new Error("aborted")); + }); + }); + const client = new HttpClient(fetchFn, 5_000, "test-agent", false); + await expect(client.get("https://example.com/x", { timeout: 10 })).rejects.toThrow(); + }); +}); + +describe("HttpClient retry policy", () => { + it("retries a 429 and honors a numeric Retry-After", async () => { + let calls = 0; + const fetchFn = createMockFetch(() => { + calls++; + if (calls === 1) { + return new Response("", { status: 429, headers: { "retry-after": "0" } }); + } + return jsonResponse({ ok: true }); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { baseDelayMs: 1, maxDelayMs: 1 }); + const response = await client.get("https://example.com/x"); + expect(calls).toBe(2); + expect(response.status).toBe(200); + }); + + it("retries 5xx responses", async () => { + let calls = 0; + const fetchFn = createMockFetch(() => { + calls++; + if (calls === 1) return new Response("", { status: 503 }); + return jsonResponse({ ok: true }); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { baseDelayMs: 1, maxDelayMs: 1 }); + const response = await client.get("https://example.com/x"); + expect(calls).toBe(2); + expect(response.status).toBe(200); + }); + + it("discards the body of a response it is about to retry", async () => { + const discarded = new Response("busy", { status: 503 }); + let calls = 0; + const fetchFn = createMockFetch(() => (++calls === 1 ? discarded : jsonResponse({ ok: true }))); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { baseDelayMs: 1, maxDelayMs: 1 }); + await client.get("https://example.com/x"); + await expect(discarded.text()).rejects.toThrow(); + }); + + it("gives up after maxAttempts", async () => { + let calls = 0; + const fetchFn = createMockFetch(() => { + calls++; + return new Response("", { status: 500 }); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { + maxAttempts: 2, + baseDelayMs: 1, + maxDelayMs: 1, + }); + const response = await client.get("https://example.com/x"); + expect(calls).toBe(2); + expect(response.status).toBe(500); + }); + + it("does not retry a call marked retryable: false", async () => { + let calls = 0; + const fetchFn = createMockFetch(() => { + calls++; + return new Response("", { status: 503 }); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { baseDelayMs: 1, maxDelayMs: 1 }); + const response = await client.get("https://example.com/x", { retryable: false }); + expect(calls).toBe(1); + expect(response.status).toBe(503); + }); + + it("does not retry at all when retry is disabled client-wide", async () => { + let calls = 0; + const fetchFn = createMockFetch(() => { + calls++; + return new Response("", { status: 500 }); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", false); + const response = await client.get("https://example.com/x"); + expect(calls).toBe(1); + expect(response.status).toBe(500); + }); + + it("does not retry a 4xx that isn't 429", async () => { + let calls = 0; + const fetchFn = createMockFetch(() => { + calls++; + return new Response("", { status: 400 }); + }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { baseDelayMs: 1, maxDelayMs: 1 }); + const response = await client.get("https://example.com/x"); + expect(calls).toBe(1); + expect(response.status).toBe(400); + }); +}); + +describe("HttpClient Retry-After ceiling (maxDelayMs)", () => { + const NOW = Date.UTC(2026, 0, 1, 0, 0, 0); + const httpDate = (offsetSeconds: number) => new Date(NOW + offsetSeconds * 1000).toUTCString(); + + afterEach(() => { + vi.useRealTimers(); + }); + + /** Fails once with `status` + `retryAfter`, then succeeds. */ + function failOnce(status: number, retryAfter: string) { + const state = { calls: 0 }; + const fetchFn = createMockFetch(() => { + state.calls++; + if (state.calls === 1) { + return new Response("slow down", { status, headers: { "retry-after": retryAfter } }); + } + return jsonResponse({ ok: true }); + }); + return { state, fetchFn }; + } + + async function expectSleepsThenRetries(fetchFn: FetchFn, state: { calls: number }, sleepMs: number) { + vi.useFakeTimers({ toFake: ["setTimeout", "Date"], now: NOW }); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { maxDelayMs: 10_000, jitter: false }); + const pending = client.get("https://example.com/x"); + await vi.advanceTimersByTimeAsync(sleepMs - 1); + expect(state.calls).toBe(1); + await vi.advanceTimersByTimeAsync(1); + const response = await pending; + expect(state.calls).toBe(2); + expect(response.status).toBe(200); + } + + it("honors a seconds Retry-After under the cap", async () => { + const { state, fetchFn } = failOnce(429, "3"); + await expectSleepsThenRetries(fetchFn, state, 3_000); + }); + + it("honors an HTTP-date Retry-After under the cap", async () => { + const { state, fetchFn } = failOnce(429, httpDate(4)); + await expectSleepsThenRetries(fetchFn, state, 4_000); + }); + + it("honors a Retry-After exactly at the cap", async () => { + const { state, fetchFn } = failOnce(503, "10"); + await expectSleepsThenRetries(fetchFn, state, 10_000); + }); + + async function expectNoRetry(status: number, retryAfter: string): Promise { + vi.useFakeTimers({ toFake: ["setTimeout", "Date"], now: NOW }); + const { state, fetchFn } = failOnce(status, retryAfter); + const client = new HttpClient(fetchFn, 30_000, "test-agent", { maxDelayMs: 10_000 }); + const response = await client.get("https://example.com/x"); + expect(state.calls).toBe(1); + expect(response.status).toBe(status); + expect(vi.getTimerCount()).toBe(0); + const error = await ensureOk(response).then( + () => undefined, + (e: unknown) => e, + ); + // The body was not discarded, so the error carries it. + expect((error as APIError).body).toBe("slow down"); + return error; + } + + it("returns a 429 without sleeping when a seconds Retry-After exceeds the cap", async () => { + const error = await expectNoRetry(429, "3600"); + expect(error).toBeInstanceOf(RateLimitError); + expect((error as RateLimitError).retryAfter).toBe("3600"); + }); + + it("returns a 429 without sleeping when an HTTP-date Retry-After exceeds the cap", async () => { + const header = httpDate(3600); + const error = await expectNoRetry(429, header); + expect(error).toBeInstanceOf(RateLimitError); + expect((error as RateLimitError).retryAfter).toBe(header); + }); + + it("returns a 5xx without sleeping when Retry-After exceeds the cap", async () => { + const error = await expectNoRetry(503, "11"); + expect(error).toBeInstanceOf(APIError); + expect(error).not.toBeInstanceOf(RateLimitError); + expect((error as APIError).statusCode).toBe(503); + expect((error as APIError).headers?.["retry-after"]).toBe("11"); + }); + + it.each(["soon", "Infinity", "NaN"])("falls back to capped backoff for an unusable Retry-After (%s)", async (header) => { + const { state, fetchFn } = failOnce(429, header); + await expectSleepsThenRetries(fetchFn, state, 250); + }); +}); + +describe("raiseForStatus", () => { + it("throws NotFoundError for a 404 and attaches response headers", () => { + const response = new Response("", { status: 404, headers: { "X-Test": "1" } }); + try { + raiseForStatus(response, "{}"); + throw new Error("expected raiseForStatus to throw"); + } catch (error) { + expect(error).toBeInstanceOf(NotFoundError); + expect((error as NotFoundError).headers?.["x-test"]).toBe("1"); + } + }); + + it("is a no-op for an ok response", () => { + expect(() => raiseForStatus(new Response("", { status: 200 }))).not.toThrow(); + }); +}); diff --git a/tests/kg.test.ts b/tests/kg.test.ts index 7817bbe..10bf408 100644 --- a/tests/kg.test.ts +++ b/tests/kg.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from "vitest"; -import { DiffbotClient, dql, dqlFetchOntologyText, dqlParallel } from "../src/index.js"; +import { DiffbotClient, ValidationError, dql, dqlFetchOntologyText, dqlParallel } from "../src/index.js"; import { createMockFetch, jsonResponse } from "./helpers/mock-fetch.js"; +function bodyOf(req: { body?: string }): Record { + return JSON.parse(req.body ?? "{}") as Record; +} + function makeClient(handler: Parameters[0]) { return new DiffbotClient({ token: "test-token", fetch: createMockFetch(handler) }); } @@ -10,22 +14,50 @@ describe("dql", () => { it("sends token, query, and default size", async () => { const db = makeClient((req) => { expect(req.params.get("token")).toBe("test-token"); - expect(req.params.get("query")).toBe('type:Organization'); - expect(req.params.get("size")).toBe("10"); + expect(bodyOf(req)).toEqual({ query: "type:Organization", size: 10 }); return jsonResponse({ data: [], hits: 0 }); }); const result = await dql(db, "type:Organization"); expect(result).toEqual({ data: [], hits: 0 }); }); + it("POSTs JSON with the token in the query string and the query out of the URL", async () => { + let seen = false; + const db = makeClient((req) => { + seen = true; + expect(req.method).toBe("POST"); + expect(req.headers.get("content-type")).toBe("application/json"); + expect([...req.params.keys()]).toEqual(["token"]); + expect(req.params.get("query")).toBeNull(); + expect(req.body).not.toContain("test-token"); + expect(bodyOf(req).query).toBe("type:Organization name:Acme"); + return jsonResponse({ data: [] }); + }); + await dql(db, "type:Organization name:Acme"); + expect(seen).toBe(true); + }); + + it("retries a 503 on the POST", async () => { + let calls = 0; + const db = new DiffbotClient({ + token: "test-token", + retry: { baseDelayMs: 1, maxDelayMs: 1 }, + fetch: createMockFetch(() => (++calls < 2 ? jsonResponse({}, 503) : jsonResponse({ data: [] }))), + }); + await dql(db, "type:Organization"); + expect(calls).toBe(2); + }); + it("forwards options and omits json format", async () => { const db = makeClient((req) => { - expect(req.params.get("size")).toBe("25"); - expect(req.params.get("from")).toBe("5"); - expect(req.params.get("filter")).toBe("foo"); - expect(req.params.get("exportspec")).toBe("bar"); - expect(req.params.get("format")).toBeNull(); - expect(req.params.get("extraKey")).toBe("extraVal"); + expect(bodyOf(req)).toEqual({ + query: "type:Person", + size: 25, + from: 5, + filter: "foo", + exportspec: "bar", + extraKey: "extraVal", + }); return jsonResponse({ data: [] }); }); await dql(db, "type:Person", { @@ -40,7 +72,7 @@ describe("dql", () => { it("sets non-json format", async () => { const db = makeClient((req) => { - expect(req.params.get("format")).toBe("csv"); + expect(bodyOf(req).format).toBe("csv"); return jsonResponse({ data: [] }); }); await dql(db, "type:Person", { format: "csv" }); @@ -54,6 +86,36 @@ describe("dql", () => { expect(JSON.parse(text)).toEqual({ data: [{ name: "Acme" }] }); }); + it("streams format: jsonl as an async generator of entities", async () => { + const lines = `${JSON.stringify({ value: "a" })}\n${JSON.stringify({ value: "b" })}\n`; + const db = makeClient((req) => { + expect(bodyOf(req).format).toBe("jsonl"); + return new Response(lines, { status: 200 }); + }); + const result = await dql(db, "type:Organization", { format: "jsonl" }); + const items: unknown[] = []; + for await (const item of result) items.push(item); + expect(items).toEqual([{ value: "a" }, { value: "b" }]); + }); + + it("returns format: csv as plain text, not a JSON.parse attempt", async () => { + const db = makeClient(() => new Response("name,hits\nAcme,3\n", { status: 200 })); + const result = await dql(db, "type:Organization", { format: "csv" }); + expect(result).toBe("name,hits\nAcme,3\n"); + }); + + it("preserves non-UTF-8 bytes in raw mode instead of corrupting them (xls/xlsx export)", async () => { + // Invalid-UTF-8 byte sequences, as in a binary xls/xlsx body. Previously the response + // was decoded as text (lossily, to U+FFFD) and then re-encoded, corrupting these bytes. + const binary = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x80, 0x81]); + const db = makeClient(() => + new Response(binary, { status: 200, headers: { "Content-Type": "application/vnd.ms-excel" } }), + ); + const result = await dql(db, "type:Organization", { format: "xlsx", raw: true }); + expect(result).toBeInstanceOf(Uint8Array); + expect(Array.from(result as Uint8Array)).toEqual(Array.from(binary)); + }); + it("raises AuthError on 401", async () => { const db = makeClient(() => jsonResponse({ code: 401, message: "Unauthorized. Incorrect token." }, 401), @@ -85,6 +147,63 @@ describe("dql", () => { }); }); +describe("dql named params", () => { + it("places every named param in the JSON body and keeps the token in the query", async () => { + const db = makeClient((req) => { + expect(req.params.get("token")).toBe("test-token"); + expect(req.params.has("type")).toBe(false); + expect(bodyOf(req)).toEqual({ + query: "q", + size: 10, + type: "crawl", + col: "my-crawl", + filterExclude: "x", + exportseparator: ";", + exportfile: "out.csv", + jsonmode: "id", + nonCanonicalFacts: true, + noDedupArticles: false, + cluster: "all", + report: "true", + }); + return jsonResponse({ data: [] }); + }); + await dql(db, "q", { + type: "crawl", + col: "my-crawl", + filterExclude: "x", + exportseparator: ";", + exportfile: "out.csv", + jsonmode: "id", + nonCanonicalFacts: true, + noDedupArticles: false, + cluster: "all", + extra: { report: "true" }, + }); + }); + + it("omits unset named params (defaults unchanged)", async () => { + const db = makeClient((req) => { + expect(bodyOf(req)).toEqual({ query: "q", size: 10 }); + return jsonResponse({ data: [] }); + }); + await dql(db, "q", { type: undefined, col: undefined, nonCanonicalFacts: undefined }); + }); + + it("raises ValidationError when extra duplicates a named param, without sending", async () => { + let called = false; + const db = makeClient(() => { + called = true; + return jsonResponse({}); + }); + const bad = { extra: { type: "text" } } as unknown as Parameters[2]; + await expect(dql(db, "q", bad)).rejects.toBeInstanceOf(ValidationError); + const bad2 = { size: 5, extra: { size: "7" } } as unknown as Parameters[2]; + await expect(dql(db, "q", bad2)).rejects.toBeInstanceOf(ValidationError); + expect(called).toBe(false); + }); +}); + describe("dqlParallel", () => { it("returns empty array for no queries", async () => { const db = makeClient(() => jsonResponse({})); @@ -93,8 +212,7 @@ describe("dqlParallel", () => { it("runs queries and preserves order", async () => { const db = makeClient((req) => { - const query = req.params.get("query")!; - return jsonResponse({ query }); + return jsonResponse({ query: bodyOf(req).query }); }); const results = await dqlParallel( db, @@ -105,9 +223,9 @@ describe("dqlParallel", () => { }); it("forwards per-query options", async () => { - const db = makeClient((req) => jsonResponse({ size: req.params.get("size") })); + const db = makeClient((req) => jsonResponse({ size: bodyOf(req).size })); const results = await dqlParallel(db, [{ query: "a", size: 50 }]); - expect(results).toEqual([{ size: "50" }]); + expect(results).toEqual([{ size: 50 }]); }); }); diff --git a/tests/live.live.test.ts b/tests/live.live.test.ts index dba42e2..07ead04 100644 --- a/tests/live.live.test.ts +++ b/tests/live.live.test.ts @@ -5,7 +5,24 @@ */ import { describe, expect, it } from "vitest"; -import { DiffbotClient, ask, askJson, dql, extract, jsonSchemaFormat, webSearch } from "../src/index.js"; +import { + DiffbotClient, + account, + ask, + askJson, + crawlDeleteJob, + crawlDownload, + crawlGetJob, + crawlListJobs, + crawl, + dql, + entities, + extract, + jsonSchemaFormat, + nlp, + verifyToken, + webSearch, +} from "../src/index.js"; import { resolveToken } from "../src/node/index.js"; import type { JsonObject } from "../src/json.js"; @@ -83,6 +100,105 @@ describe("live", () => { await db.close(); }); + // DQL is sent as POST (JSON body). Kept cheap: size 0 is a hit count, others size 1. + live("dql POST: size 0 hit count", async () => { + await using db = new DiffbotClient({ token }); + const result = await dql(db, "type:Organization", { size: 0 }); + expect(typeof result.hits).toBe("number"); + }); + + live("dql POST: csv returns text", async () => { + await using db = new DiffbotClient({ token }); + const csv = await dql(db, 'type:Organization name:"Diffbot"', { size: 1, format: "csv" }); + expect(typeof csv).toBe("string"); + expect(csv.length).toBeGreaterThan(0); + }); + + live("dql POST: jsonl streams entities", async () => { + await using db = new DiffbotClient({ token }); + const items: unknown[] = []; + for await (const item of await dql(db, 'type:Organization name:"Diffbot"', { size: 1, format: "jsonl" })) { + items.push(item); + } + expect(items.length).toBeGreaterThan(0); + }); + + live("dql POST: a query over 8 KB is accepted (no URL length limit)", async () => { + await using db = new DiffbotClient({ token }); + const clauses = Array.from({ length: 400 }, (_, i) => `name:"Acme Widget Company ${i}"`); + const query = `type:Organization or(${clauses.join(",")})`; + expect(query.length).toBeGreaterThan(8 * 1024); + const result = await dql(db, query, { size: 0 }); + expect(typeof result.hits).toBe("number"); + }); + + live("entities", async () => { + await using db = new DiffbotClient({ token }); + const result = await entities(db, "Tim Cook is the CEO of Apple, based in Cupertino."); + expect(Array.isArray(result.entities)).toBe(true); + expect(result.entities?.length ?? 0).toBeGreaterThan(0); + }); + + // The account body contains `token` and `childTokens`: assert on shape only and never + // put the body (or those fields) in an assertion message. + live("account + verifyToken (free)", async () => { + await using db = new DiffbotClient({ token }); + const body = await account(db, { days: 1 }); + expect(typeof body.status).toBe("string"); + expect(Array.isArray(body.usage)).toBe(true); + expect(await verifyToken(db)).toBe(true); + await using bad = new DiffbotClient({ token: "invalid-token-for-live-test" }); + expect(await verifyToken(bad)).toBe(false); + }); + + live("extract with fields and a caller timeout", async () => { + await using db = new DiffbotClient({ token }); + const result = await extract(db, "https://example.com", "article", "json", { + fields: ["links"], + timeout: 45_000, + }); + expect(Array.isArray(result.objects)).toBe(true); + expect(result.objects?.length ?? 0).toBeGreaterThan(0); + }); + + live("tiny crawl -> poll -> crawlDownload csv/json -> delete", async () => { + const prefix = "sdk-live-ts-"; + await using db = new DiffbotClient({ token }); + // Leaked-job check: anything from a prior run of this suite older than 1 h is deleted. + const cutoff = Date.now() - 60 * 60 * 1000; + for (const job of await crawlListJobs(db)) { + const created = Number(job.name?.slice(prefix.length)); + if (job.name?.startsWith(prefix) && Number.isFinite(created) && created < cutoff) { + await crawlDeleteJob(db, job.name); + } + } + const name = `${prefix}${Date.now().toString()}`; + try { + for await (const _ of crawl(db, "https://example.com", { + jobName: name, + maxToCrawl: 5, + maxToProcess: 5, + hops: 0, + })) { + void _; + } + let status = 0; + for (let i = 0; i < 40; i++) { + const job = await crawlGetJob(db, name); + status = job?.jobStatus?.status ?? 0; + if (status !== 0 && status !== 7) break; + await new Promise((r) => setTimeout(r, 3_000)); + } + expect(status).not.toBe(0); + const csv = await crawlDownload(db, name, { format: "csv" }); + expect(typeof csv).toBe("string"); + const records = await crawlDownload(db, name); + expect(Array.isArray(records)).toBe(true); + } finally { + await crawlDeleteJob(db, name); + } + }, 180_000); + live("web search", async () => { const db = new DiffbotClient({ token }); const result = await webSearch(db, "diffbot knowledge graph", { numResults: 3 }); @@ -91,6 +207,26 @@ describe("live", () => { await db.close(); }); + // S2: assert on shape only, never on web content. + live("web search: numResults 1, single and 2-value text", async () => { + const db = new DiffbotClient({ token }); + const one = await webSearch(db, "diffbot knowledge graph", { numResults: 1 }); + expect(Array.isArray(one.search_results)).toBe(true); + const two = await webSearch(db, ["diffbot knowledge graph", "diffbot extract api"], { numResults: 1 }); + expect(Array.isArray(two.search_results)).toBe(true); + await db.close(); + }); + + live("entities and nlp on one sentence", async () => { + await using db = new DiffbotClient({ token }); + const sentence = "Tim Cook is the CEO of Apple, based in Cupertino."; + const viaEntities = await entities(db, sentence); + expect(Array.isArray(viaEntities.entities)).toBe(true); + const viaNlp = await nlp(db, sentence, { fields: ["entities", "language"] }); + expect(Array.isArray(viaNlp.entities)).toBe(true); + expect(typeof viaNlp.language).toBe("string"); + }); + live("web search snippetLength", async () => { const db = new DiffbotClient({ token }); const total = (r: JsonObject) => diff --git a/tests/nlp.test.ts b/tests/nlp.test.ts index 117f36d..416b04e 100644 --- a/tests/nlp.test.ts +++ b/tests/nlp.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { DiffbotClient, entities } from "../src/index.js"; +import { DiffbotClient, ValidationError, entities, nlp } from "../src/index.js"; import { createMockFetch, jsonResponse } from "./helpers/mock-fetch.js"; function makeClient(handler: Parameters[0]) { @@ -45,3 +45,66 @@ describe("entities", () => { expect(JSON.parse(body)[0].lang).toBe("fr"); }); }); + +describe("nlp", () => { + function capture(response: unknown = [{}]) { + const seen: { params?: URLSearchParams; body?: Array>; method?: string; url?: string } = {}; + const db = makeClient((req) => { + seen.params = req.params; + seen.method = req.method; + seen.url = req.url; + seen.body = JSON.parse(req.body ?? "[]") as Array>; + return jsonResponse(response); + }); + return { db, seen }; + } + + it("posts one document with comma-joined fields and defaults", async () => { + const { db, seen } = capture([{ summary: "s", language: "en" }]); + const result = await nlp(db, "Hello world.", { fields: ["summary", "language"] }); + expect(seen.method).toBe("POST"); + expect(seen.url).toContain("nl.diffbot.com/v1/"); + expect(seen.params!.get("token")).toBe("test-token"); + expect(seen.params!.get("fields")).toBe("summary,language"); + expect(seen.body).toEqual([{ lang: "auto", format: "plain text", content: "Hello world." }]); + expect(result).toEqual({ summary: "s", language: "en" }); + }); + + it("forwards lang, format, documentType and customSummary", async () => { + const { db, seen } = capture(); + await nlp(db, "Title\nBody", { + fields: ["summary"], + lang: "en", + format: "plain text with title", + documentType: "news article", + customSummary: { maxNumberOfSentences: 2 }, + }); + expect(seen.body).toEqual([ + { + lang: "en", + format: "plain text with title", + content: "Title\nBody", + documentType: "news article", + customSummary: { maxNumberOfSentences: 2 }, + }, + ]); + }); + + it("accepts an object response and returns {} for an empty array", async () => { + expect(await nlp(capture({ sentiment: 0.1 }).db, "x", { fields: ["sentiment"] })).toEqual({ sentiment: 0.1 }); + expect(await nlp(capture([]).db, "x", { fields: ["sentiment"] })).toEqual({}); + }); + + it("rejects empty fields without sending", async () => { + const { db, seen } = capture(); + await expect(nlp(db, "x", { fields: [] })).rejects.toBeInstanceOf(ValidationError); + expect(seen.method).toBeUndefined(); + }); + + it("leaves entities() on entities,sentiment", async () => { + const { db, seen } = capture(); + await entities(db, "x"); + expect(seen.params!.get("fields")).toBe("entities,sentiment"); + expect(seen.body).toEqual([{ lang: "auto", format: "plain text", content: "x" }]); + }); +}); diff --git a/tests/readme-examples.test.ts b/tests/readme-examples.test.ts index cb1705b..2e501fd 100644 --- a/tests/readme-examples.test.ts +++ b/tests/readme-examples.test.ts @@ -2,11 +2,16 @@ import { describe, expect, it } from "vitest"; import { CrawlEventType, DiffbotClient, + account, ask, crawl, + crawlDownload, + crawlGetJob, dql, entities, + nlp, extract, + verifyToken, webSearch, } from "../src/index.js"; import { createMockFetch, jsonResponse, sseResponse } from "./helpers/mock-fetch.js"; @@ -52,11 +57,67 @@ describe("README examples", () => { expect(events[0]!.eventType).toBe(CrawlEventType.JobCreated); }); + it("extract with options", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(req.url).toBe("https://api.diffbot.com/v3/article"); + expect(req.params.get("fields")).toBe("links,meta"); + expect(req.params.get("timeout")).toBe("60000"); + expect(req.params.get("naturalLanguage")).toBe("entities,sentiment"); + return jsonResponse({ objects: [{ title: "T" }] }); + }), + }); + const article = await extract(db, "https://www.example.com", "article", "json", { + fields: ["links", "meta"], + timeout: 60_000, + naturalLanguage: ["entities", "sentiment"], + }); + expect(article.objects).toHaveLength(1); + }); + + it("verifyToken and account", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(req.url).toBe("https://api.diffbot.com/v4/account"); + return jsonResponse({ plan: "free", usage: [] }); + }), + }); + expect(await verifyToken(db)).toBe(true); + const { plan, usage } = await account(db, { days: 7 }); + expect(plan).toBe("free"); + expect(usage).toEqual([]); + }); + + it("crawl without watch, then crawlGetJob and crawlDownload", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + if (req.url.endsWith("/crawl/data")) { + return req.params.get("format") === "csv" + ? new Response("pageUrl\nhttps://example.com/\n") + : jsonResponse([{ pageUrl: "https://example.com/" }]); + } + return jsonResponse({ jobs: [{ name: "my-job", jobStatus: { status: 9 } }] }); + }), + }); + for await (const _ of crawl(db, "https://www.example.com", { jobName: "my-job" })) { + void _; + } + const job = await crawlGetJob(db, "my-job"); + expect(job?.jobStatus?.status).toBe(9); + const records = await crawlDownload(db, "my-job"); + expect(records).toEqual([{ pageUrl: "https://example.com/" }]); + expect(await crawlDownload(db, "my-job", { format: "csv" })).toContain("pageUrl"); + }); + it("dql", async () => { const db = new DiffbotClient({ token: "test-token", fetch: createMockFetch((req) => { - expect(req.params.get("query")).toContain("Diffbot"); + expect(req.method).toBe("POST"); + expect(req.body).toContain("Diffbot"); return jsonResponse({ data: [{ entity: { name: "Diffbot" } }] }); }), }); @@ -83,6 +144,45 @@ describe("README examples", () => { expect(searchResults[0]!.title).toBe("Diffbot"); }); + it("dql type crawl with col", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(JSON.parse(req.body ?? "{}")).toEqual({ query: "title:Pricing", size: 5, type: "crawl", col: "my-job" }); + return jsonResponse({ data: [] }); + }), + }); + const pages = await dql(db, "title:Pricing", { type: "crawl", col: "my-job", size: 5 }); + expect(pages.data).toEqual([]); + }); + + it("webSearch with multiple text values", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(req.params.getAll("text")).toEqual(["diffbot knowledge graph", "diffbot extract api"]); + expect(req.params.get("size")).toBe("3"); + return jsonResponse({ search_results: [] }); + }), + }); + await webSearch(db, ["diffbot knowledge graph", "diffbot extract api"], { numResults: 3 }); + }); + + it("crawl with customHeaders array", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(req.params.getAll("customHeaders")).toEqual(["Accept-Language:en-us", "X-Test:1"]); + expect(req.params.get("urlProcessRegEx")).toBe("/products/"); + return jsonResponse({ jobs: [{ name: "j" }] }); + }), + }); + await crawl(db, "https://www.example.com", { + urlProcessRegEx: "/products/", + customHeaders: ["Accept-Language:en-us", "X-Test:1"], + }).next(); + }); + it("entities", async () => { const db = new DiffbotClient({ token: "test-token", @@ -107,6 +207,53 @@ describe("README examples", () => { expect(result.sentiment).toBe(0.3); }); + it("nlp", async () => { + const db = new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(req.params.get("fields")).toBe("summary,language"); + const doc = (JSON.parse(req.body ?? "[]") as Array>)[0]!; + expect(doc.format).toBe("plain text with title"); + expect(doc.customSummary).toEqual({ maxNumberOfSentences: 1 }); + return jsonResponse([{ summary: "Apple reports record earnings.", language: "en" }]); + }), + }); + const result = await nlp(db, "Headline\nApple CEO Tim Cook announced record quarterly earnings.", { + fields: ["summary", "language"], + format: "plain text with title", + documentType: "news article", + customSummary: { maxNumberOfSentences: 1 }, + }); + expect(result.summary).toBe("Apple reports record earnings."); + expect(result.language).toBe("en"); + }); + + it("apiBase points every api.diffbot.com endpoint at a staging host", async () => { + const staging = new DiffbotClient({ + token: "test-token", + apiBase: "https://staging.example.com", + fetch: createMockFetch((req) => { + expect(req.url).toBe("https://staging.example.com/v3/analyze"); + return jsonResponse({ objects: [] }); + }), + }); + await extract(staging, "https://news.ycombinator.com"); + }); + + it("retry: false disables retries", async () => { + let calls = 0; + const db = new DiffbotClient({ + token: "test-token", + retry: false, + fetch: createMockFetch(() => { + calls++; + return new Response("", { status: 503 }); + }), + }); + await expect(dql(db, "type:Organization")).rejects.toMatchObject({ statusCode: 503 }); + expect(calls).toBe(1); + }); + it("extract with await using", async () => { await using db = new DiffbotClient({ token: "test-token", diff --git a/tests/types/account.test-d.ts b/tests/types/account.test-d.ts new file mode 100644 index 0000000..b13a146 --- /dev/null +++ b/tests/types/account.test-d.ts @@ -0,0 +1,17 @@ +import { describe, expectTypeOf, it } from "vitest"; +import { DiffbotClient, account, verifyToken, type AccountResponse } from "../../src/index.js"; + +declare const db: DiffbotClient; + +describe("account()", () => { + it("returns AccountResponse and verifyToken a boolean", () => { + expectTypeOf(account(db)).resolves.toEqualTypeOf(); + expectTypeOf(account(db, { days: 1 })).resolves.toEqualTypeOf(); + expectTypeOf(verifyToken(db)).resolves.toEqualTypeOf(); + }); + + it("exposes the spec's fields", () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/tests/types/ask.test-d.ts b/tests/types/ask.test-d.ts new file mode 100644 index 0000000..92489e7 --- /dev/null +++ b/tests/types/ask.test-d.ts @@ -0,0 +1,32 @@ +import { describe, expectTypeOf, it } from "vitest"; +import { DiffbotClient, ask, askJson, type ChatMessage } from "../../src/index.js"; + +declare const db: DiffbotClient; + +describe("ask()/askJson() types", () => { + it("ChatMessage.role is a closed union, not a bare string", () => { + expectTypeOf().toEqualTypeOf<"system" | "user" | "assistant">(); + }); + + it("ask() yields strings", () => { + expectTypeOf(ask(db, [{ role: "user", content: "hi" }])).toEqualTypeOf>(); + }); + + it("askJson defaults to unknown without a parse hook or explicit T", () => { + expectTypeOf(askJson(db, [{ role: "user", content: "hi" }])).resolves.toEqualTypeOf(); + }); + + it("askJson narrows T from an explicit type parameter", () => { + expectTypeOf( + askJson<{ capital: string }>(db, [{ role: "user", content: "hi" }]), + ).resolves.toEqualTypeOf<{ capital: string }>(); + }); + + it("askJson narrows T from the parse hook's return type", () => { + expectTypeOf( + askJson(db, [{ role: "user", content: "hi" }], undefined, { + parse: (value: unknown) => ({ capital: String(value) }), + }), + ).resolves.toEqualTypeOf<{ capital: string }>(); + }); +}); diff --git a/tests/types/contract.test-d.ts b/tests/types/contract.test-d.ts new file mode 100644 index 0000000..63fc7f0 --- /dev/null +++ b/tests/types/contract.test-d.ts @@ -0,0 +1,130 @@ +import { describe, expectTypeOf, it } from "vitest"; +import type { AccountOptions } from "../../src/account.js"; +import type { CrawlDownloadOptions, CrawlOptions } from "../../src/crawl.js"; +import type { + AnalyzeExtractOptions, + ArticleExtractOptions, + CustomExtractOptions, + DiscussionExtractOptions, + EventExtractOptions, + ImageExtractOptions, + JobExtractOptions, + ListExtractOptions, + ProductExtractOptions, + VideoExtractOptions, +} from "../../src/extract.js"; +import type { operations as AccountOps } from "../../src/generated/account.js"; +import type { operations as CrawlOps } from "../../src/generated/crawl.js"; +import type { components as DqlComponents } from "../../src/generated/dql.js"; +import type { DqlOptions } from "../../src/kg.js"; +import type { operations as ExtractOps } from "../../src/generated/extract.js"; + +/* + * Contract: for every in-scope operation, the option keys the SDK names, plus an explicit + * INTENTIONALLY_UNNAMED list, must equal the parameter keys in the vendored spec. A param + * added to a spec (after `pnpm gen:types`) fails here until it is named or consciously + * allow-listed; a name that is not in the spec fails too. `extra` is not a wire key. + * + */ + +type Named = Exclude; +type SpecKeys = keyof NonNullable & string; + +// Cut or positional parameters. Why each is not a named option: +// url / name positional arguments of the function +// callback JSONP, pointless for an SDK (plan §6) +// proxy, proxyAuth credential-bearing and niche; typed `extra` only +type ExtractUnnamed = "url" | "callback" | "proxy" | "proxyAuth"; + +// DQL (POST body, `DqlPostRequest`): +// query positional argument of dql() +// report DQL coverage reports are out of scope (plan §2); reachable via `extra` +// view same as report +// `raw` is SDK-only (not on the wire) and is excluded below. +type DqlUnnamed = "query" | "report" | "view"; + +describe("spec <-> named option contract", () => { + it("extract: analyze", () => { + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + }); + it("extract: article", () => { + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + }); + it("extract: product", () => { + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + }); + it("extract: discussion", () => { + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + }); + it("extract: job, image, video, list, event", () => { + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + }); + it("extract: custom API", () => { + expectTypeOf | ExtractUnnamed>().toEqualTypeOf< + SpecKeys + >(); + }); + + it("account", () => { + expectTypeOf>().toEqualTypeOf>(); + }); + + it("crawl data download", () => { + // `name` is the positional job name. + expectTypeOf | "name">().toEqualTypeOf< + SpecKeys + >(); + }); + + it("dql", () => { + expectTypeOf> | DqlUnnamed>().toEqualTypeOf< + keyof DqlComponents["schemas"]["DqlPostRequest"] & string + >(); + }); + + it("crawl create: new S2 named options are real spec fields", () => { + // crawl() keeps its older, SDK-shaped names (hops, jobName, ...), so this is a subset + // check, not equality. Recurring-crawl admin stays out (plan §2/§6): + // repeat, maxRounds, seedRecrawlFrequency, onlyProcessIfNew + type CreateBody = CrawlOps["create-a-crawl"]["requestBody"] extends infer R + ? NonNullable extends { content: { "application/x-www-form-urlencoded": infer B } } + ? keyof B & string + : never + : never; + type S2Named = + | "urlCrawlRegEx" + | "urlProcessRegEx" + | "pageProcessPattern" + | "maxToCrawlPerSubdomain" + | "maxToProcessPerSubdomain" + | "useCanonical" + | "notifyWebhook" + | "notifyEmail"; + expectTypeOf().toExtend(); + expectTypeOf().toExtend(); + expectTypeOf<"repeat" | "maxRounds" | "seedRecrawlFrequency" | "onlyProcessIfNew">().toExtend(); + expectTypeOf>().toEqualTypeOf(); + }); +}); diff --git a/tests/types/crawl.test-d.ts b/tests/types/crawl.test-d.ts new file mode 100644 index 0000000..10c3f3a --- /dev/null +++ b/tests/types/crawl.test-d.ts @@ -0,0 +1,33 @@ +import { describe, expectTypeOf, it } from "vitest"; +import { DiffbotClient, crawlDownload, crawlGetJob, crawlListJobs, type CrawlOptions } from "../../src/index.js"; +import type { CrawlDataRecord, CrawlJob } from "../../src/types/undocumented.js"; + +declare const db: DiffbotClient; + +describe("crawl job accessors", () => { + it("crawlListJobs resolves to an array of CrawlJob", () => { + expectTypeOf(crawlListJobs(db)).resolves.toEqualTypeOf(); + }); + + it("crawlGetJob resolves to CrawlJob | undefined, not {}", () => { + expectTypeOf(crawlGetJob(db, "name")).resolves.toEqualTypeOf(); + }); +}); + +describe("crawlDownload overloads", () => { + it("returns records by default and for format: json", () => { + expectTypeOf(crawlDownload(db, "n")).resolves.toEqualTypeOf(); + expectTypeOf(crawlDownload(db, "n", { format: "json", num: 3 })).resolves.toEqualTypeOf(); + }); + + it("returns a string for format: csv and for the CSV-only URL report", () => { + expectTypeOf(crawlDownload(db, "n", { format: "csv" })).resolves.toEqualTypeOf(); + expectTypeOf(crawlDownload(db, "n", { type: "urls" })).resolves.toEqualTypeOf(); + }); +}); + +describe("crawl() customHeaders", () => { + it("accepts a string or a readonly string array", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); diff --git a/tests/types/dql.test-d.ts b/tests/types/dql.test-d.ts new file mode 100644 index 0000000..53bb786 --- /dev/null +++ b/tests/types/dql.test-d.ts @@ -0,0 +1,86 @@ +import { describe, expectTypeOf, it } from "vitest"; +import type * as publicApi from "../../src/index.js"; +import { DiffbotClient, dql, dqlParallel, type DqlOptions } from "../../src/index.js"; +import type { DqlEntity, DqlResponse } from "../../src/types/dql.js"; + +declare const db: DiffbotClient; + +describe("dql() format overloads", () => { + it("defaults to a parsed DqlResponse", () => { + expectTypeOf(dql(db, "type:Organization")).resolves.toEqualTypeOf(); + expectTypeOf(dql(db, "type:Organization", { format: "json" })).resolves.toEqualTypeOf(); + }); + + it("returns a string for csv", () => { + expectTypeOf(dql(db, "type:Organization", { format: "csv" })).resolves.toEqualTypeOf(); + }); + + it("returns bytes for xls/xlsx or raw", () => { + expectTypeOf(dql(db, "type:Organization", { format: "xls" })).resolves.toEqualTypeOf(); + expectTypeOf(dql(db, "type:Organization", { format: "xlsx" })).resolves.toEqualTypeOf(); + expectTypeOf(dql(db, "type:Organization", { raw: true })).resolves.toEqualTypeOf(); + }); + + it("returns an async generator of entities for jsonl", () => { + expectTypeOf(dql(db, "type:Organization", { format: "jsonl" })).resolves.toEqualTypeOf< + AsyncGenerator + >(); + }); + + it("rejects a named option passed through extra", () => { + // @ts-expect-error -- "size" collides with the named `size` option. + void dql(db, "type:Organization", { extra: { size: "5" } }); + }); + + it("types the new named params and rejects them in extra", () => { + expectTypeOf().toEqualTypeOf<"query" | "text" | "queryTextFallback" | "crawl" | undefined>(); + // @ts-expect-error -- "col" is a named option now. + void dql(db, "q", { extra: { col: "x" } }); + // report/view stay reachable through extra. + void dql(db, "q", { extra: { report: "true", view: "v" } }); + }); + + it("dqlParallel maps each query's own format to its own result type", async () => { + const results = await dqlParallel(db, [ + { query: "a" }, + { query: "b", format: "csv" }, + { query: "c", format: "xlsx" }, + ]); + expectTypeOf(results[0]).toEqualTypeOf(); + expectTypeOf(results[1]).toEqualTypeOf(); + expectTypeOf(results[2]).toEqualTypeOf(); + }); +}); + +describe("dql() public contract is unchanged by the GET -> POST switch", () => { + it("keeps the same option keys (no `method` option, no `dqlPost`)", () => { + expectTypeOf().toEqualTypeOf< + | "size" + | "from" + | "format" + | "filter" + | "exportspec" + | "extra" + | "raw" + | "type" + | "col" + | "filterExclude" + | "exportseparator" + | "exportfile" + | "jsonmode" + | "nonCanonicalFacts" + | "noDedupArticles" + | "cluster" + >(); + expectTypeOf<"dqlPost" extends keyof typeof publicApi ? true : false>().toEqualTypeOf(); + }); + + it("keeps size/from as numbers", () => { + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf(); + }); + + it("dqlParallel still accepts per-query options and a worker count (result types: see above)", () => { + void dqlParallel(db, [{ query: "a", size: 1 }], 4); + }); +}); diff --git a/tests/types/extract.test-d.ts b/tests/types/extract.test-d.ts new file mode 100644 index 0000000..cdad343 --- /dev/null +++ b/tests/types/extract.test-d.ts @@ -0,0 +1,90 @@ +import { describe, expectTypeOf, it } from "vitest"; +import { DiffbotClient, extract } from "../../src/index.js"; +import type { + AnalyzeObject, + ArticleObject, + CustomApiObject, + DiscussionObject, + EventObject, + ExtractResponse, + ImageObject, + JobObject, + ListObject, + ProductObject, + VideoObject, +} from "../../src/types/undocumented.js"; + +declare const db: DiffbotClient; + +describe("extract() overloads", () => { + it("defaults to the analyze api", () => { + expectTypeOf(extract(db, "https://example.com")).resolves.toEqualTypeOf< + ExtractResponse + >(); + }); + + it("narrows the return type per built-in api", () => { + expectTypeOf(extract(db, "https://example.com", "article")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "product")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "discussion")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "job")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "image")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "video")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "list")).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "https://example.com", "event")).resolves.toEqualTypeOf< + ExtractResponse + >(); + }); + + it("falls back to an open object shape for a Custom API name", () => { + expectTypeOf(extract(db, "https://example.com", "my-custom-api")).resolves.toEqualTypeOf< + ExtractResponse + >(); + }); +}); + +describe("extract() options", () => { + it("types fields per API", () => { + void extract(db, "u", "article", "json", { fields: ["links", "quotes"] }); + // @ts-expect-error -- "quotes" is an article-only field. + void extract(db, "u", "product", "json", { fields: ["quotes"] }); + // @ts-expect-error -- fields is a list, not a comma-joined string. + void extract(db, "u", "article", "json", { fields: "links,meta" }); + }); + + it("rejects options that belong to another API", () => { + // @ts-expect-error -- `paging` is not an option of the product API. + void extract(db, "u", "product", "json", { paging: true }); + // @ts-expect-error -- `callback` (JSONP) is intentionally not exposed. + void extract(db, "u", "article", "json", { callback: "cb" }); + }); + + it("rejects named options in extra but allows proxy/proxyAuth", () => { + void extract(db, "u", "article", "json", { extra: { proxy: "h:1", proxyAuth: "u:p" } }); + // @ts-expect-error -- `timeout` is a named option. + void extract(db, "u", "article", "json", { extra: { timeout: "1" } }); + }); + + it("keeps the return type per API with options", () => { + expectTypeOf(extract(db, "u", "article", "json", { timeout: 1 })).resolves.toEqualTypeOf< + ExtractResponse + >(); + expectTypeOf(extract(db, "u", undefined, undefined, { mode: "article" })).resolves.toEqualTypeOf< + ExtractResponse + >(); + }); +}); diff --git a/tests/types/ontology.test-d.ts b/tests/types/ontology.test-d.ts new file mode 100644 index 0000000..29880d7 --- /dev/null +++ b/tests/types/ontology.test-d.ts @@ -0,0 +1,22 @@ +import { describe, expectTypeOf, it } from "vitest"; +import { Ontology } from "../../src/index.js"; +import type { FieldMeta, OntologyDocument } from "../../src/types/undocumented.js"; + +describe("Ontology types", () => { + it("fieldsFor resolves to a Record", () => { + const ontology = new Ontology({} as OntologyDocument); + expectTypeOf(ontology.fieldsFor("Article")).toEqualTypeOf>(); + }); + + it("a hand-built ontology document satisfies OntologyDocument", () => { + // The `satisfies` clause is the assertion: this fails to compile if the literal + // below doesn't conform to OntologyDocument's shape. + const doc = { + types: { + Article: { fields: { title: { type: "String" } } }, + }, + enums: { Gender: { values: ["Male", "Female"] } }, + } satisfies OntologyDocument; + expectTypeOf(doc.enums.Gender.values).toEqualTypeOf(); + }); +}); diff --git a/tests/types/public-surface.test-d.ts b/tests/types/public-surface.test-d.ts new file mode 100644 index 0000000..c1be3c1 --- /dev/null +++ b/tests/types/public-surface.test-d.ts @@ -0,0 +1,17 @@ +import { describe, expectTypeOf, it } from "vitest"; +import type * as publicApi from "../../src/index.js"; + +// Enhance / Bulk Extract / Custom API are infra only (vendored specs + src/generated), +// not part of the public SDK surface. This fails if one leaks into src/index.ts. +type ForbiddenExport = Extract< + keyof typeof publicApi, + `${string}${"enhance" | "Enhance" | "bulk" | "Bulk" | "custom" | "Custom"}${string}` +>; + +describe("public surface", () => { + it("exports no enhance/bulk/custom symbols", () => { + // `CustomApiObject` (the open-shape extract result for custom APIs) is the one allowed + // name; it is a response type, not a Custom API management function. + expectTypeOf>().toEqualTypeOf(); + }); +}); diff --git a/tests/types/web-search-nlp.test-d.ts b/tests/types/web-search-nlp.test-d.ts new file mode 100644 index 0000000..3ede47a --- /dev/null +++ b/tests/types/web-search-nlp.test-d.ts @@ -0,0 +1,57 @@ +import { describe, expectTypeOf, it } from "vitest"; +import { DiffbotClient, entities, nlp, webSearch } from "../../src/index.js"; +import type { EntitiesResponse } from "../../src/nlp.js"; +import type { NaturalLanguageResponse } from "../../src/types/natural-language.js"; +import type { WebSearchResponse } from "../../src/types/web-search.js"; + +declare const db: DiffbotClient; + +describe("webSearch()/entities() types", () => { + it("webSearch accepts a string or a readonly string array, nothing else", () => { + void webSearch(db, "a"); + void webSearch(db, ["a", "b"]); + void webSearch(db, ["a", "b"] as const); + // @ts-expect-error -- text must be strings. + void webSearch(db, [1, 2]); + // @ts-expect-error -- a bare number is not a query. + void webSearch(db, 5); + expectTypeOf(webSearch(db, ["a"])).resolves.toEqualTypeOf(); + }); + + it("webSearch resolves to a typed WebSearchResponse", () => { + expectTypeOf(webSearch(db, "diffbot")).resolves.toEqualTypeOf(); + }); + + it("entities resolves to just the two fields it always requests", () => { + expectTypeOf(entities(db, "some text")).resolves.toEqualTypeOf(); + expectTypeOf().toEqualTypeOf<"entities" | "sentiment">(); + }); +}); + +describe("nlp() types", () => { + it("narrows the response to the requested fields", () => { + expectTypeOf(nlp(db, "t", { fields: ["summary", "language"] })).resolves.toEqualTypeOf< + Pick + >(); + expectTypeOf(nlp(db, "t", { fields: ["sentiment"] })).resolves.toEqualTypeOf< + Pick + >(); + }); + + it("requires fields and rejects unknown ones, formats and document types", () => { + // @ts-expect-error -- fields is required. + void nlp(db, "t", {}); + // @ts-expect-error -- not an NL field. + void nlp(db, "t", { fields: ["nope"] }); + // @ts-expect-error -- not a supported format. + void nlp(db, "t", { fields: ["summary"], format: "html" }); + // @ts-expect-error -- not a supported documentType. + void nlp(db, "t", { fields: ["summary"], documentType: "blog" }); + void nlp(db, "t", { + fields: ["summary"], + format: "plain text with title", + documentType: "news article", + customSummary: { maxNumberOfSentences: 3 }, + }); + }); +}); diff --git a/tests/version.test.ts b/tests/version.test.ts new file mode 100644 index 0000000..e8e869e --- /dev/null +++ b/tests/version.test.ts @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { VERSION } from "../src/index.js"; + +// The user agent reports VERSION, not package.json, so a release must bump both. +describe("VERSION", () => { + it("matches package.json", () => { + const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { + version: string; + }; + expect(VERSION).toBe(pkg.version); + }); +}); diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts index cab629c..8229c4b 100644 --- a/tests/web-search.test.ts +++ b/tests/web-search.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { DiffbotClient, webSearch } from "../src/index.js"; +import { DiffbotClient, ValidationError, webSearch } from "../src/index.js"; import { createMockFetch, jsonResponse } from "./helpers/mock-fetch.js"; function makeClient(handler: Parameters[0]) { @@ -68,3 +68,49 @@ describe("webSearch", () => { await expect(webSearch(db, "diffbot")).rejects.toMatchObject({ name: "AuthError" }); }); }); + +describe("webSearch multi-value text", () => { + function capture() { + const seen: { texts?: string[]; method?: string; called: boolean } = { called: false }; + const db = makeClient((req) => { + seen.called = true; + seen.method = req.method; + seen.texts = req.params.getAll("text"); + return jsonResponse({ results: [] }); + }); + return { db, seen }; + } + + it("sends one string as a single text param (GET)", async () => { + const { db, seen } = capture(); + await webSearch(db, "diffbot"); + expect(seen.method).toBe("GET"); + expect(seen.texts).toEqual(["diffbot"]); + }); + + it("treats a one-element array like a string", async () => { + const { db, seen } = capture(); + await webSearch(db, ["diffbot"]); + expect(seen.texts).toEqual(["diffbot"]); + }); + + it("sends 2 values as repeated text params", async () => { + const { db, seen } = capture(); + await webSearch(db, ["a b", "c&d"], { numResults: 1 }); + expect(seen.method).toBe("GET"); + expect(seen.texts).toEqual(["a b", "c&d"]); + }); + + it("accepts 5 values", async () => { + const { db, seen } = capture(); + await webSearch(db, ["1", "2", "3", "4", "5"]); + expect(seen.texts).toEqual(["1", "2", "3", "4", "5"]); + }); + + it("rejects 0 and 6 values without sending", async () => { + const { db, seen } = capture(); + await expect(webSearch(db, [])).rejects.toBeInstanceOf(ValidationError); + await expect(webSearch(db, ["1", "2", "3", "4", "5", "6"])).rejects.toBeInstanceOf(ValidationError); + expect(seen.called).toBe(false); + }); +});