diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ba10e2..417b594 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,9 @@ name: Release # tests, build, `pnpm pack`) and nothing else. Every step that publishes to npm # or creates a GitHub release is gated on a real tag push, so a dispatch — on # any branch, by anyone with write access — can never ship a version. +# +# npm auth is trusted publishing (OIDC). No NPM_TOKEN and no GitHub environment. +# The package must already exist on npm before a trusted publisher can be added. on: push: tags: ["v*"] @@ -93,22 +96,35 @@ jobs: registry-url: https://registry.npmjs.org - run: pnpm install --frozen-lockfile - run: pnpm build + # Trusted publishing needs npm >= 11.5.1; Node 22 ships older. Use npm, + # not pnpm, so the OIDC token exchange works. No token is set. + - name: Update npm + run: | + npm install -g npm@latest + npm --version + # The dist-tag is derived from the version, not from the tag name: a # prerelease (anything containing a `-`, e.g. 0.1.0-rc.1) publishes as # `next` so a plain `npm install @diffbot/ai-sdk` keeps resolving to the # last stable release. The guards above already proved the git tag equals # this version. - - name: pnpm publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Publish to npm run: | + set -euo pipefail + name="$(node -p "require('./package.json').name")" version="$(node -p "require('./package.json').version")" case "$version" in *-*) dist_tag=next ;; *) dist_tag=latest ;; esac - echo "Publishing $version with --tag $dist_tag." - pnpm publish --access public --provenance --no-git-checks --tag "$dist_tag" + if npm view "${name}@${version}" version >/dev/null 2>&1; then + echo "${name}@${version} is already on npm; skipping publish." + exit 0 + fi + echo "Publishing $version with --tag $dist_tag (npm trusted publishing)." + # --ignore-scripts: dist/ was built above. No NODE_AUTH_TOKEN; npm + # authenticates with the trusted publisher on npmjs.com. + npm publish --access public --provenance --ignore-scripts --tag "$dist_tag" github-release: name: create GitHub release @@ -125,7 +141,12 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - gh release create "$GITHUB_REF_NAME" \ - --repo "$GITHUB_REPOSITORY" \ - --title "$GITHUB_REF_NAME" \ - --generate-notes + set -euo pipefail + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "Release $GITHUB_REF_NAME already exists; leaving it." + else + gh release create "$GITHUB_REF_NAME" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$GITHUB_REF_NAME" \ + --generate-notes + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1cd74..74329e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.1.0 — Unreleased +## 0.1.0 - 2026-11-17 First API of `@diffbot/ai-sdk`: Diffbot tools, retrieval helpers, and an OpenAI-compatible provider for the Vercel AI SDK (`ai` ^6 or ^7). @@ -24,3 +24,14 @@ for the Vercel AI SDK (`ai` ^6 or ^7). timeout. - Examples: `examples/company-research` (tool-calling agent), `examples/manual-rag` (retrieve helpers, no tool loop), and `examples/worker` (Workers `env` bag and streamed RAG). +- `extractOptions` on `diffbotExtract`, `loadExtract`, and `streamExtract` passes the SDK's + `ExtractOptions` (`timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, …) through to + `extract`. The tool's model-facing input schema is unchanged, and `api`/`fmt` defaults stay. +- Opt-in `download` on `loadCrawl` / `streamCrawl`: start the job without watching, poll it, and + map the records from `crawlDownload` to documents (`defaultCrawlRecordMapper`). The default watch + stream is unchanged. + +### Changed + +- Depends on `@diffbot/typescript` `^0.6.0` (cumulative 0.4.0–0.6.0). DQL is POST with a JSON body; the token stays the query parameter. Unit-test mocks already assert that body. The dev-only `.pnpmfile.cjs` sibling link is removed. Refresh `pnpm-lock.yaml` from the registry once 0.6.0 is on npm; do not commit a `link:` or `file:` override. +- `retrieveKnowledgeGraph` takes `nonCanonicalFacts` as a named boolean. Passing that key through `extra` throws, matching the SDK. diff --git a/CLAUDE.md b/CLAUDE.md index 1d4aa13..652c661 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ Precedence inside `execute` (lazy): injected `client` → `apiKey` → `resolveT ## Phase 4 scope (current) -Seven tools plus retrieval/loader helpers: `retrieveKnowledgeGraph`, `retrieveWebSearch`, `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper` (crawl is helpers-only, never a tool). Provider at `@diffbot/ai-sdk/provider` (`createDiffbot`). `diffbotAsk` defaults to `timeout: 120_000` when this package builds the client; injected clients keep their own timeout. `diffbotTools()` still builds one shared client, so that default covers the whole bundle when `diffbot_ask` is included. No examples yet (Phase 5). Live coverage is complete: every tool, both retrievers, `loadExtract`, and the provider (`generateText` + `streamText`). +Seven tools plus retrieval/loader helpers: `retrieveKnowledgeGraph`, `retrieveWebSearch`, `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper`, `defaultCrawlRecordMapper` (crawl is helpers-only, never a tool; `download` opts into start → poll → `crawlDownload`). Provider at `@diffbot/ai-sdk/provider` (`createDiffbot`). `diffbotAsk` defaults to `timeout: 120_000` when this package builds the client; injected clients keep their own timeout. `diffbotTools()` still builds one shared client, so that default covers the whole bundle when `diffbot_ask` is included. No examples yet (Phase 5). Live coverage is complete: every tool, both retrievers, `loadExtract`, and the provider (`generateText` + `streamText`). ## CI and live tests @@ -22,7 +22,7 @@ Four workflows: - `ci.yml` — unit-only, **no secrets** (lint, typecheck, unit tests + build on Node 20/22/24, the `ai@6`/`ai@7` peer matrix, empty-token live-skip guard, Workers bundle check). Keep it secret-free so it can run on fork PRs. - `integration.yml` — `pnpm test:live`, on push to `main`, a twice-monthly drift schedule (1st and 15th at 14:17 UTC), and `workflow_dispatch`. Never `pull_request`/`pull_request_target`, so `DIFFBOT_API_TOKEN` is never exposed to fork PR code. A guard step fails the run on a missing or blank secret rather than skipping everything green. - `canary.yml` — weekly `ai@latest` + `@ai-sdk/openai-compatible@latest` typecheck + unit tests. No secrets. -- `release.yml` — `v*` tag only; `workflow_dispatch` is a build-only rehearsal and every publish/release step is gated on the tag push. It fails early unless the tag equals `package.json`'s version and `"private"` is no longer `true` (Phase 6 flips that flag — do not change it before then). +- `release.yml` — `v*` tag only; `workflow_dispatch` is a build-only rehearsal and every publish/release step is gated on the tag push. It fails early unless the tag equals `package.json`'s version and `"private"` is not `true`. Publish uses npm trusted publishing (OIDC). There is no `NPM_TOKEN`. `@diffbot/ai-sdk` is not on npm yet, so a trusted publisher cannot be added until a one-time manual bootstrap publish creates the package. `pnpm test:workers` dry-run bundles `tests/fixtures/worker/` for workerd and then scans the bundle: `wrangler deploy --dry-run` alone leaves static `node:*` imports in the output as externals, so the scan is what actually enforces Workers-cleanliness. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d68f89..d4aefbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,18 +22,25 @@ After `pnpm test:peer:ai6` (or `:ai7`, or `:latest`) restore the lockfile — th git checkout package.json pnpm-lock.yaml && pnpm install --frozen-lockfile ``` +## Local SDK link (temporary) + +The release dependency is `@diffbot/typescript` `^0.6.0`. Do not commit a `file:` / `link:` specifier +or a `.pnpmfile.cjs`. Until 0.6.0 is on npm, `pnpm-lock.yaml` still points at a sibling link and +cannot be refreshed from the registry. For a local unit-test run, link a built 0.6.0 checkout and +restore `pnpm-lock.yaml` before committing. + ## Workflows -| Workflow | Triggers | Secrets | -| ----------------- | ------------------------------------------------- | ------------------------- | -| `ci.yml` | push to `main`, every `pull_request`, dispatch | **none** | -| `integration.yml` | push to `main`, 1st + 15th at 14:17 UTC, dispatch | `DIFFBOT_API_TOKEN` | -| `canary.yml` | Mondays at 14:17 UTC, dispatch | **none** | -| `release.yml` | `v*` tag; dispatch is a build-only rehearsal | `NPM_TOKEN` (publish job) | +| Workflow | Triggers | Secrets | +| ----------------- | ------------------------------------------------- | --------------------------------- | +| `ci.yml` | push to `main`, every `pull_request`, dispatch | **none** | +| `integration.yml` | push to `main`, 1st + 15th at 14:17 UTC, dispatch | `DIFFBOT_API_TOKEN` | +| `canary.yml` | Mondays at 14:17 UTC, dispatch | **none** | +| `release.yml` | `v*` tag; dispatch is a build-only rehearsal | **none** (npm trusted publishing) | `ci.yml` runs lint + typecheck, unit tests + build on Node 20 / 22 / 24, the peer matrix (`ai@6` + `@ai-sdk/openai-compatible@2`, `ai@7` + `@3`), the empty-token live-skip guard, and the Workers bundle check. **It must stay secret-free** — that is what lets every job run on fork PRs. -`release.yml` publishes only from a `v*` tag push: it checks the tag equals `package.json`'s `version` and that `"private"` is no longer `true`, runs lint/typecheck/tests/build, then `pnpm publish --access public --provenance`, then creates the GitHub release. `workflow_dispatch` stops after `pnpm pack`; every publishing step is gated on the tag push, so a dispatch cannot ship a version. To release: bump `version`, commit, tag `vX.Y.Z`, push the tag. +`release.yml` publishes only from a `v*` tag push: it checks the tag equals `package.json`'s `version` and that `"private"` is not `true`, runs lint/typecheck/tests/build, then `npm publish --access public --provenance` via trusted publishing (OIDC, no `NPM_TOKEN`), then creates the GitHub release. `workflow_dispatch` stops after `pnpm pack`; every publishing step is gated on the tag push, so a dispatch cannot ship a version. To release: bump `version`, commit, tag `vX.Y.Z`, push the tag. npm cannot attach a trusted publisher until the package exists; the first publish of `@diffbot/ai-sdk` needs a one-time manual bootstrap (see the release report) before that tag will succeed. ## Live tests diff --git a/README.md b/README.md index b5fd633..ab25e56 100644 --- a/README.md +++ b/README.md @@ -527,19 +527,26 @@ Each returns (or yields) `DiffbotDocument` objects: `{ content: string; metadata: Record }`. Options beyond the auth ones: - `retrieveKnowledgeGraph` — `query` (DQL, required), `k` (default `10`), `from`, `filter`, - `exportspec`, `extra`, `format` (default `json`; other formats throw, since only JSON can be - shaped), plus `fields` / `contentFields` (default `DEFAULT_KG_CONTENT_FIELDS`) / `documentMapper`. + `exportspec`, `nonCanonicalFacts`, `extra` (only keys that are not named DQL options; 0.6.0 + throws if a named key such as `nonCanonicalFacts` is passed through `extra`), `format` (default + `json`; other formats throw, since only JSON can be shaped), plus `fields` / `contentFields` + (default `DEFAULT_KG_CONTENT_FIELDS`) / `documentMapper`. - `retrieveWebSearch` — `query` (required), `k` (default `10`), `maxTokens`, `snippetLength`, plus `fields` / `contentFields` (default `DEFAULT_WEB_CONTENT_FIELDS`) / `documentMapper`. - `loadExtract` / `streamExtract` — `urls` (required), `api` (default `analyze`), `fmt` (default - `markdown`), `documentMapper(url, raw)`. `metadata` carries `url`, `title`, `pageUrl`, + `markdown`), `extractOptions` (the SDK's `ExtractOptions`: `timeout`, `mode`, `fields`, `fallback`, + `discussion`, `paging`, `extra`, …), `documentMapper(url, raw)`. `metadata` carries `url`, `title`, `pageUrl`, `resolvedPageUrl` and `type`. URLs are fetched one at a time, in order, and an extraction failure throws (as in `@diffbot/langchain`): `loadExtract` rejects, while `streamExtract` has already yielded the documents before the failing URL. - `loadCrawl` / `streamCrawl` — `site` (required), `crawlOptions` (the SDK's `CrawlOptions`; `watch` defaults to `true` so URL events are emitted), `eventMapper(event)` returning a document or `null` to skip. The default `content` is the crawled URL itself, with `url`, `status` and - `crawl_timestamp` in `metadata`. + `crawl_timestamp` in `metadata`. Pass `download: true` (or `{ num, type, recordMapper }`) to skip + the watch stream: the job is started without `watch`, polled until it finishes, and its extracted + records are fetched with the SDK's `crawlDownload`, one document per record (`content` is the + record's `content`, falling back to its title; `eventMapper` does not apply). `type: "urls"` is + rejected, since that report is CSV. `defaultCrawlMapper` drops the `job_created` event, so with `watch: false` you get no documents at all — that combination only makes sense with an `eventMapper`. Pass `crawlOptions.jobName` if you @@ -647,7 +654,7 @@ Every runtime export of `@diffbot/ai-sdk`: | `diffbotWebSearch`, `diffbotExtract`, `diffbotKnowledgeGraph`, `diffbotEntities`, `diffbotOntology`, `diffbotDqlProbe`, `diffbotAsk` | Model-facing `tool()` | | `diffbotTools` | Bundle | | `retrieveKnowledgeGraph`, `retrieveWebSearch` | Retrieval | -| `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper` | Loaders | +| `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper`, `defaultCrawlRecordMapper` | Loaders | | `dictToDocument`, `shapeExtractResponse`, `DEFAULT_KG_CONTENT_FIELDS`, `DEFAULT_WEB_CONTENT_FIELDS` | Shaping | | `CrawlEventType` | Re-export from the SDK | diff --git a/package.json b/package.json index 4516ce2..82f737d 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "@diffbot/ai-sdk", - "version": "0.1.0-dev.0", + "version": "0.1.0", "description": "Diffbot tools and LLM provider for the Vercel AI SDK — web search, extract, DQL, ontology, and RAG", "type": "module", - "private": true, + "private": false, "packageManager": "pnpm@10.33.0", "main": "./dist/index.cjs", "module": "./dist/index.js", @@ -61,7 +61,7 @@ "test:workers": "tests/fixtures/worker/dry-run.sh" }, "dependencies": { - "@diffbot/typescript": "^0.3.0" + "@diffbot/typescript": "^0.6.0" }, "peerDependencies": { "@ai-sdk/openai-compatible": "^2.0.0 || ^3.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 59af73a..8f137e8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@diffbot/typescript': - specifier: ^0.3.0 - version: 0.3.0 + specifier: ^0.6.0 + version: 0.6.0 devDependencies: '@ai-sdk/openai-compatible': specifier: ^3.0.0 @@ -73,8 +73,8 @@ packages: resolution: {integrity: sha512-4uPgZ3qiX/JaQFr0dQoibUzHImZit9u05ro2BxGUnJwU4fV28BOAbKUa2OjQvqtM1tMac3bTKkaXdyzGovQXqg==} engines: {node: '>=22'} - '@diffbot/typescript@0.3.0': - resolution: {integrity: sha512-VkHiLohAbhUzTNbLL8gwNXAIoTrSfyenAIam5sVLhUGysSkkxuVbNX78BVoU4AlM5pwF3FNEA6Z/5RBRl+TLbQ==} + '@diffbot/typescript@0.6.0': + resolution: {integrity: sha512-Sdrnk3+4QHo9BzP/hheAweI4L067EKEHXOrK26Lc6F0gLY/RMsifFiJVeolEAnKdI8HfDrUb7K/qBMQkU6DfiA==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.27.7': @@ -1430,7 +1430,7 @@ snapshots: dependencies: json-schema: 0.4.0 - '@diffbot/typescript@0.3.0': {} + '@diffbot/typescript@0.6.0': {} '@esbuild/aix-ppc64@0.27.7': optional: true diff --git a/src/index.ts b/src/index.ts index 31f9df0..7684b48 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,7 +39,10 @@ export { loadCrawl, streamCrawl, defaultCrawlMapper, + defaultCrawlRecordMapper, + type CrawlDownloadLoadOptions, type CrawlEventMapper, + type CrawlRecordMapper, type StreamCrawlOptions, } from "./retrieve/crawl.js"; diff --git a/src/retrieve/crawl.ts b/src/retrieve/crawl.ts index ab482e1..9cf66b0 100644 --- a/src/retrieve/crawl.ts +++ b/src/retrieve/crawl.ts @@ -1,4 +1,13 @@ -import { crawl, CrawlEventType, type CrawlEvent, type CrawlOptions } from "@diffbot/typescript"; +import { + crawl, + crawlDownload, + crawlGetJob, + CrawlEventType, + JobError, + type CrawlDataRecord, + type CrawlEvent, + type CrawlOptions, +} from "@diffbot/typescript"; import { type DiffbotAuth, createClientResolver } from "../client.js"; import type { DiffbotDocument } from "../documents.js"; @@ -19,10 +28,51 @@ export function defaultCrawlMapper(event: CrawlEvent): DiffbotDocument | null { }; } +export type CrawlRecordMapper = (record: CrawlDataRecord) => DiffbotDocument | null; + +/** + * Maps one `crawlDownload` JSON record. `content` is the extracted body (falling back to the + * title), and metadata is picked field by field: the server echoes the caller's token in each + * record, so the record is never spread wholesale. + */ +export function defaultCrawlRecordMapper(record: CrawlDataRecord): DiffbotDocument { + const url = typeof record.pageUrl === "string" ? record.pageUrl : ""; + const body = typeof record.content === "string" && record.content ? record.content : undefined; + const title = typeof record.title === "string" ? record.title : undefined; + return { + content: body ?? title ?? url, + metadata: { + url, + title, + type: record.type, + parentUrl: record.parentUrl, + id: record.id, + crawl_timestamp: record.timestamp, + }, + }; +} + +/** Options for the opt-in poll + download path (`crawlDownload`). */ +export interface CrawlDownloadLoadOptions { + /** Only the N most recently processed URLs (the SDK's `num`). */ + num?: number; + /** Downloaded record type, passed through as the SDK's `type`. */ + type?: string; + /** Maps each downloaded record; return `null` to drop it. */ + recordMapper?: CrawlRecordMapper; +} + export interface StreamCrawlOptions extends DiffbotAuth { site: string; crawlOptions?: CrawlOptions; eventMapper?: CrawlEventMapper; + /** + * Opt in to the non-watch path: start the job without watching, poll `crawlGetJob` until it + * reaches a terminal status, then fetch the extracted records with `crawlDownload` (JSON). + * Yields one document per record instead of one per crawled URL. Omit for the default + * watch-stream behavior, where `eventMapper` applies; `eventMapper` is ignored here. + */ + download?: true | CrawlDownloadLoadOptions; } function resolvedCrawlOptions(crawlOptions?: CrawlOptions): CrawlOptions { @@ -33,11 +83,75 @@ function resolvedCrawlOptions(crawlOptions?: CrawlOptions): CrawlOptions { return opts; } +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function* streamCrawlDownload( + options: StreamCrawlOptions, +): AsyncGenerator { + // `createClientResolver` reads only the `DiffbotAuth` keys, so the options pass through whole. + const { site, crawlOptions, download } = options; + const { num, type, recordMapper }: CrawlDownloadLoadOptions = + download === true ? {} : (download ?? {}); + if (type === "urls") { + // The URL report is CSV text, not records; the default watch path already covers it. + throw new TypeError('`download.type` "urls" returns CSV, not records; omit `download` instead'); + } + const client = createClientResolver(options)(); + const mapper = recordMapper ?? defaultCrawlRecordMapper; + const pollInterval = crawlOptions?.pollInterval ?? 2000; + + let jobName = crawlOptions?.jobName; + for await (const event of crawl(client, site, { ...crawlOptions, watch: false })) { + if ( + event.eventType === CrawlEventType.JobCreated && + typeof event.details.job_name === "string" + ) { + jobName = event.details.job_name; + } + } + if (jobName === undefined) { + throw new JobError("", "Crawl did not report a job name"); + } + + for (;;) { + const job = await crawlGetJob(client, jobName); + if (!job) { + throw new JobError(jobName, "Crawler job not found while polling"); + } + const status = job.jobStatus?.status ?? 0; + if (status !== 0 && status !== 7) { + const message = job.jobStatus?.message ?? ""; + if (/fail|error/i.test(message)) { + throw new JobError(jobName, `Crawler job failed: ${message}`); + } + break; + } + await sleep(pollInterval); + } + + const records = await crawlDownload(client, jobName, { + ...(type !== undefined ? { type } : {}), + ...(num !== undefined ? { num } : {}), + }); + for (const record of records) { + const doc = mapper(record); + if (doc !== null) { + yield doc; + } + } +} + export async function* streamCrawl( options: StreamCrawlOptions, ): AsyncGenerator { - const { site, crawlOptions, eventMapper, ...auth } = options; - const client = createClientResolver(auth)(); + if (options.download) { + yield* streamCrawlDownload(options); + return; + } + const { site, crawlOptions, eventMapper } = options; + const client = createClientResolver(options)(); const mapper = eventMapper ?? defaultCrawlMapper; const opts = resolvedCrawlOptions(crawlOptions); diff --git a/src/retrieve/extract.ts b/src/retrieve/extract.ts index c301e79..0a7e3dd 100644 --- a/src/retrieve/extract.ts +++ b/src/retrieve/extract.ts @@ -1,4 +1,4 @@ -import { extract, type JsonObject } from "@diffbot/typescript"; +import { extract, type ExtractOptions, type JsonObject } from "@diffbot/typescript"; import { type DiffbotAuth, createClientResolver } from "../client.js"; import { shapeExtractResponse, type DiffbotDocument } from "../documents.js"; @@ -8,6 +8,8 @@ export interface LoadExtractOptions extends DiffbotAuth { urls: string[]; api?: string; fmt?: string; + /** Extra SDK extract options (`timeout`, `mode`, `fields`, `paging`, `extra`, …) for every URL. */ + extractOptions?: ExtractOptions; documentMapper?: ExtractDocumentMapper; } @@ -28,12 +30,25 @@ function defaultExtractMapper(url: string, raw: JsonObject): DiffbotDocument { export async function* streamExtract( options: LoadExtractOptions, ): AsyncGenerator { - const { urls, api = "analyze", fmt = "markdown", documentMapper, ...auth } = options; + const { + urls, + api = "analyze", + fmt = "markdown", + extractOptions, + documentMapper, + ...auth + } = options; const client = createClientResolver(auth)(); const mapper = documentMapper ?? defaultExtractMapper; for (const url of urls) { - const raw = (await extract(client, url, api, fmt)) as JsonObject; + const raw = (await extract( + client, + url, + api, + fmt, + extractOptions as Parameters>[4], + )) as JsonObject; yield mapper(url, raw); } } diff --git a/src/retrieve/knowledge-graph.ts b/src/retrieve/knowledge-graph.ts index 91ec5c3..d8c8b60 100644 --- a/src/retrieve/knowledge-graph.ts +++ b/src/retrieve/knowledge-graph.ts @@ -21,6 +21,12 @@ export interface RetrieveKnowledgeGraphOptions extends DiffbotAuth, DiffbotDocum exportspec?: string; /** Only `"json"` can be shaped into documents; anything else throws before the request. */ format?: string; + /** + * Named DQL option. Do not also put it in `extra` — `@diffbot/typescript` 0.6.0 + * rejects named keys passed through `extra`. + */ + nonCanonicalFacts?: boolean; + /** Body fields that are not named DQL options (`report`, `view`, …). */ extra?: Record; } @@ -34,6 +40,7 @@ export async function retrieveKnowledgeGraph( filter, exportspec, format = "json", + nonCanonicalFacts, extra, fields, contentFields = DEFAULT_KG_CONTENT_FIELDS, @@ -59,6 +66,7 @@ export async function retrieveKnowledgeGraph( format, ...(filter !== undefined ? { filter } : {}), ...(exportspec !== undefined ? { exportspec } : {}), + ...(nonCanonicalFacts !== undefined ? { nonCanonicalFacts } : {}), ...(extra !== undefined ? { extra } : {}), }); diff --git a/src/tools/extract.ts b/src/tools/extract.ts index b1a9220..9b64a3a 100644 --- a/src/tools/extract.ts +++ b/src/tools/extract.ts @@ -1,4 +1,10 @@ -import { extract, ExtractionError, type DiffbotClient, type JsonObject } from "@diffbot/typescript"; +import { + extract, + ExtractionError, + type DiffbotClient, + type ExtractOptions, + type JsonObject, +} from "@diffbot/typescript"; import { tool, type Tool } from "ai"; import { z } from "zod"; import { type DiffbotAuth, createClientResolver } from "../client.js"; @@ -7,6 +13,12 @@ import { shapeExtractResponse, type ShapedExtractResponse } from "../documents.j export interface DiffbotExtractOptions extends DiffbotAuth { api?: string; fmt?: string; + /** + * Extra `@diffbot/typescript` extract options (`timeout`, `mode`, `fields`, `fallback`, + * `discussion`, `paging`, `extra`, …) applied to every call. Set by the developer, not the + * model; the tool's input schema is unchanged. + */ + extractOptions?: ExtractOptions; } export interface DiffbotExtractToolError { @@ -50,7 +62,14 @@ export function createExtractTool( const fmt = input.fmt ?? options?.fmt ?? "markdown"; try { - const raw = (await extract(client, input.url, api, fmt)) as JsonObject; + const raw = (await extract( + client, + input.url, + api, + fmt, + // `api` is a runtime string, so the per-API option union cannot be narrowed here. + options?.extractOptions as Parameters>[4], + )) as JsonObject; return shapeExtractResponse(raw); } catch (e) { if (e instanceof ExtractionError) { diff --git a/tests/helpers/mock-fetch.ts b/tests/helpers/mock-fetch.ts index 963086f..d881b6b 100644 --- a/tests/helpers/mock-fetch.ts +++ b/tests/helpers/mock-fetch.ts @@ -12,6 +12,8 @@ export interface MockRequest { headers: Headers; params: URLSearchParams; body?: string; + /** `body` parsed as JSON (POST bodies such as DQL); undefined if absent or not JSON. */ + json?: Record; } export type MockHandler = (request: MockRequest) => Response | Promise; @@ -22,6 +24,18 @@ function toTextBody(body: RequestInit["body"]): string | undefined { return undefined; } +function parseJsonBody(body: string | undefined): Record | undefined { + if (body === undefined) return undefined; + try { + const parsed: unknown = JSON.parse(body); + return parsed !== null && typeof parsed === "object" + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} + export function createMockFetch(handler: MockHandler): FetchFn { return async (input, init) => { const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; @@ -36,6 +50,7 @@ export function createMockFetch(handler: MockHandler): FetchFn { headers, params: parsed.searchParams, body, + json: parseJsonBody(body), }); }; } diff --git a/tests/imports.test.ts b/tests/imports.test.ts index e7a2280..cf2a07f 100644 --- a/tests/imports.test.ts +++ b/tests/imports.test.ts @@ -10,6 +10,7 @@ const EXPECTED_VALUES = [ "DEFAULT_KG_CONTENT_FIELDS", "DEFAULT_WEB_CONTENT_FIELDS", "defaultCrawlMapper", + "defaultCrawlRecordMapper", "dictToDocument", "diffbotAsk", "diffbotDqlProbe", diff --git a/tests/retrieve/crawl.test.ts b/tests/retrieve/crawl.test.ts index acbe1f7..a3f254d 100644 --- a/tests/retrieve/crawl.test.ts +++ b/tests/retrieve/crawl.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from "vitest"; import { CrawlEventType, type CrawlEvent } from "@diffbot/typescript"; import type { DiffbotDocument } from "../../src/documents.js"; -import { defaultCrawlMapper, loadCrawl, streamCrawl } from "../../src/retrieve/crawl.js"; +import { + defaultCrawlMapper, + defaultCrawlRecordMapper, + loadCrawl, + streamCrawl, +} from "../../src/retrieve/crawl.js"; import { createMockFetch, jsonResponse } from "../helpers/mock-fetch.js"; const CRAWL_URL = "https://api.diffbot.com/v3/crawl"; @@ -121,3 +126,125 @@ describe("streamCrawl watch defaulting", () => { expect(docs[0]!.content).toBe(CrawlEventType.JobCreated); }); }); + +describe("loadCrawl download path", () => { + const RECORDS = [ + { + pageUrl: "https://example.com/a", + title: "A", + content: "Body A", + type: "article", + id: 1, + token: "secret-echoed-by-server", + timestamp: "2026-01-01T00:00:00Z", + }, + { pageUrl: "https://example.com/b", title: "B" }, + ]; + + function crawlFetch(statuses: number[], log: Array<{ kind: string; params: URLSearchParams }>) { + let polls = 0; + return createMockFetch((req) => { + if (req.url.endsWith("/data")) { + log.push({ kind: "data", params: req.params }); + return jsonResponse(RECORDS); + } + if (req.params.get("seeds")) { + log.push({ kind: "create", params: req.params }); + return jsonResponse({ jobs: [{ name: "test-job" }] }); + } + log.push({ kind: "status", params: req.params }); + const status = statuses[Math.min(polls++, statuses.length - 1)]!; + return jsonResponse({ jobs: [{ name: "test-job", jobStatus: { status, message: "m" } }] }); + }); + } + + it("creates without watching, polls until terminal, then downloads JSON records", async () => { + const log: Array<{ kind: string; params: URLSearchParams }> = []; + const docs = await loadCrawl({ + apiKey: "test-token", + fetch: crawlFetch([0, 7, 9], log), + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: { num: 5 }, + }); + + expect(log.map((l) => l.kind)).toEqual(["create", "status", "status", "status", "data"]); + const data = log.at(-1)!.params; + expect(data.get("name")).toBe("test-job"); + expect(data.get("num")).toBe("5"); + expect(data.has("format")).toBe(false); + expect(data.has("type")).toBe(false); + + expect(docs).toHaveLength(2); + expect(docs[0]).toEqual({ + content: "Body A", + metadata: { + url: "https://example.com/a", + title: "A", + type: "article", + parentUrl: undefined, + id: 1, + crawl_timestamp: "2026-01-01T00:00:00Z", + }, + }); + expect(docs[1]!.content).toBe("B"); + expect(JSON.stringify(docs)).not.toContain("secret-echoed-by-server"); + }); + + it("accepts `download: true` and a custom recordMapper that can drop records", async () => { + const log: Array<{ kind: string; params: URLSearchParams }> = []; + const all = await loadCrawl({ + apiKey: "t", + fetch: crawlFetch([2], log), + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: true, + }); + expect(all).toHaveLength(2); + + const some = await loadCrawl({ + apiKey: "t", + fetch: crawlFetch([2], []), + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: { recordMapper: (r) => (r.title === "B" ? defaultCrawlRecordMapper(r) : null) }, + }); + expect(some.map((d) => d.metadata.title)).toEqual(["B"]); + }); + + it("throws a JobError when the job ends in failure, without downloading", async () => { + const log: Array<{ kind: string; params: URLSearchParams }> = []; + const fetch = createMockFetch((req) => { + log.push({ kind: req.url.endsWith("/data") ? "data" : "other", params: req.params }); + return req.params.get("seeds") + ? jsonResponse({ jobs: [{ name: "test-job" }] }) + : jsonResponse({ jobs: [{ jobStatus: { status: 4, message: "Job failed: bad seed" } }] }); + }); + await expect( + loadCrawl({ + apiKey: "t", + fetch, + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: true, + }), + ).rejects.toThrow(/Crawler job failed/); + expect(log.some((l) => l.kind === "data")).toBe(false); + }); + + it("rejects type: urls, which is CSV rather than records", async () => { + await expect( + loadCrawl({ + apiKey: "t", + fetch: createMockFetch(() => jsonResponse({})), + site: "https://example.com", + download: { type: "urls" }, + }), + ).rejects.toThrow(/returns CSV/); + }); + + it("falls back to title, then URL, for content", () => { + expect(defaultCrawlRecordMapper({ pageUrl: "u", title: "T" }).content).toBe("T"); + expect(defaultCrawlRecordMapper({ pageUrl: "u" }).content).toBe("u"); + }); +}); diff --git a/tests/retrieve/extract.test.ts b/tests/retrieve/extract.test.ts index 81fefd4..24f0387 100644 --- a/tests/retrieve/extract.test.ts +++ b/tests/retrieve/extract.test.ts @@ -104,6 +104,25 @@ describe("loadExtract", () => { expect(seen!.params.has("mode")).toBe(false); }); + it("forwards extractOptions to every request without dropping api/fmt defaults", async () => { + const seen: URLSearchParams[] = []; + await load( + (req) => { + seen.push(req.params); + return jsonResponse({ objects: [{ text: "x" }] }); + }, + ["https://example.com/a", "https://example.com/b"], + { extractOptions: { timeout: 60_000, fallback: "article", discussion: false } }, + ); + expect(seen).toHaveLength(2); + for (const params of seen) { + expect(params.get("timeout")).toBe("60000"); + expect(params.get("fallback")).toBe("article"); + expect(params.get("discussion")).toBe("false"); + expect(params.get("mode")).toBe("llm"); + } + }); + it("propagates an extraction failure, like the LangChain loader", async () => { const handler: MockHandler = (req) => req.params.get("url")?.endsWith("/bad") diff --git a/tests/retrieve/knowledge-graph.test.ts b/tests/retrieve/knowledge-graph.test.ts index e2a3fd0..f4ba5ef 100644 --- a/tests/retrieve/knowledge-graph.test.ts +++ b/tests/retrieve/knowledge-graph.test.ts @@ -66,8 +66,8 @@ describe("retrieveKnowledgeGraph document mapping", () => { const docs = await retrieve( (req) => { expect(req.url).toBe(DQL_URL); - expect(req.method).toBe("GET"); - expect(req.params.get("query")).toBe("type:Organization"); + expect(req.method).toBe("POST"); + expect(req.json?.query).toBe("type:Organization"); return jsonResponse(SAMPLE_BODY); }, { k: 2, query: "type:Organization" }, @@ -136,7 +136,7 @@ describe("retrieveKnowledgeGraph request wiring", () => { it("sends k as the size wire param", async () => { await retrieve( (req) => { - expect(req.params.get("size")).toBe("3"); + expect(req.json?.size).toBe(3); return jsonResponse(SAMPLE_BODY); }, { k: 3, query: "type:Organization" }, @@ -144,9 +144,9 @@ describe("retrieveKnowledgeGraph request wiring", () => { }); it("defaults to size 10 and forwards from, filter, exportspec and extra", async () => { - const seen: URLSearchParams[] = []; + const seen: Array<{ params: URLSearchParams; json: Record }> = []; const handler: MockHandler = (req) => { - seen.push(req.params); + seen.push({ params: req.params, json: req.json ?? {} }); return jsonResponse(SAMPLE_BODY); }; await retrieve(handler, { query: "q" }); @@ -155,16 +155,20 @@ describe("retrieveKnowledgeGraph request wiring", () => { from: 20, filter: "nbEmployees>50", exportspec: "name", - extra: { nonCanonicalFacts: "true" }, + nonCanonicalFacts: true, + extra: { report: "1" }, }); - expect(seen[0]!.get("size")).toBe("10"); - expect(seen[0]!.has("filter")).toBe(false); - expect(seen[1]!.get("from")).toBe("20"); - expect(seen[1]!.get("filter")).toBe("nbEmployees>50"); - expect(seen[1]!.get("exportspec")).toBe("name"); - expect(seen[1]!.get("nonCanonicalFacts")).toBe("true"); - expect(seen[1]!.get("token")).toBe("test-token"); + expect(seen[0]!.json.size).toBe(10); + expect("filter" in seen[0]!.json).toBe(false); + expect(seen[1]!.json).toMatchObject({ + from: 20, + filter: "nbEmployees>50", + exportspec: "name", + nonCanonicalFacts: true, + report: "1", + }); + expect(seen[1]!.params.get("token")).toBe("test-token"); }); it("caps the documents at k even if the server returns more", async () => { @@ -204,7 +208,7 @@ describe("retrieveKnowledgeGraph request wiring", () => { const docs = await retrieve( (req) => { /* The SDK omits the param for the default format. */ - expect(req.params.has("format")).toBe(false); + expect("format" in (req.json ?? {})).toBe(false); return jsonResponse(SAMPLE_BODY); }, { query: "q", format: "json" }, diff --git a/tests/tools/dql-probe.test.ts b/tests/tools/dql-probe.test.ts index 4f05174..16a4a2e 100644 --- a/tests/tools/dql-probe.test.ts +++ b/tests/tools/dql-probe.test.ts @@ -23,10 +23,10 @@ describe("diffbotDqlProbe", () => { const out = await runProbe( (req) => { expect(req.url).toBe(DQL_URL); - expect(req.method).toBe("GET"); + expect(req.method).toBe("POST"); expect(req.params.get("token")).toBe("test-token"); - expect(req.params.get("size")).toBe("0"); - const query = req.params.get("query") ?? ""; + expect(req.json?.size).toBe(0); + const query = String(req.json?.query ?? ""); const hits = query.includes("Diffbot") ? 5 : 100; return jsonResponse({ hits, results: 0 }); }, @@ -45,7 +45,7 @@ describe("diffbotDqlProbe", () => { const queries = ["q0", "q1", "q2", "q3"]; const out = await runProbe( async (req) => { - const query = req.params.get("query") ?? ""; + const query = String(req.json?.query ?? ""); const delay = (queries.length - queries.indexOf(query)) * 5; await new Promise((resolve) => setTimeout(resolve, delay)); return jsonResponse({ hits: queries.indexOf(query) }); diff --git a/tests/tools/extract.test.ts b/tests/tools/extract.test.ts index 187ebfa..bdecdba 100644 --- a/tests/tools/extract.test.ts +++ b/tests/tools/extract.test.ts @@ -43,6 +43,33 @@ describe("diffbotExtract", () => { ); }); + it("threads factory extractOptions onto the wire", async () => { + await runExtract( + (req) => { + expect(req.url).toBe("https://api.diffbot.com/v3/article"); + expect(req.params.get("timeout")).toBe("45000"); + expect(req.params.get("paging")).toBe("false"); + expect(req.params.get("fields")).toBe("links,meta"); + expect(req.params.get("mode")).toBeNull(); + return jsonResponse({ objects: [{ text: "Body." }] }); + }, + { url: "https://example.com", api: "article", fmt: "json" }, + { extractOptions: { timeout: 45_000, paging: false, fields: ["links", "meta"] } }, + ); + }); + + it("keeps markdown mode and default timeout when no extractOptions are set", async () => { + await runExtract( + (req) => { + expect(req.params.get("mode")).toBe("llm"); + expect(req.params.get("timeout")).toBe("30000"); + expect(req.params.has("fields")).toBe(false); + return jsonResponse({ objects: [{ text: "Body." }] }); + }, + { url: "https://example.com" }, + ); + }); + it("shapes successful responses", async () => { const out = await runExtract( () => diff --git a/tests/tools/knowledge-graph.test.ts b/tests/tools/knowledge-graph.test.ts index d309928..7c56f67 100644 --- a/tests/tools/knowledge-graph.test.ts +++ b/tests/tools/knowledge-graph.test.ts @@ -25,6 +25,7 @@ describe("diffbotKnowledgeGraph", () => { await runDql( (req) => { expect(req.url).toBe("https://kg.diffbot.com/kg/v3/dql"); + expect(req.method).toBe("POST"); expect(req.params.get("token")).toBe("test-token"); return jsonResponse({ data: [] }); }, @@ -35,10 +36,13 @@ describe("diffbotKnowledgeGraph", () => { it("sends size, from, and filter on the wire", async () => { await runDql( (req) => { - expect(req.params.get("query")).toBe("type:Organization"); - expect(req.params.get("size")).toBe("5"); - expect(req.params.get("from")).toBe("20"); - expect(req.params.get("filter")).toBe("nbEmployees>100"); + expect(req.json).toMatchObject({ + query: "type:Organization", + size: 5, + from: 20, + filter: "nbEmployees>100", + }); + expect(req.params.has("query")).toBe(false); return jsonResponse({ data: [] }); }, { query: "type:Organization", size: 5, from: 20, filter: "nbEmployees>100" }, @@ -48,7 +52,7 @@ describe("diffbotKnowledgeGraph", () => { it("defaults size to 10", async () => { await runDql( (req) => { - expect(req.params.get("size")).toBe("10"); + expect(req.json?.size).toBe(10); return jsonResponse({ data: [] }); }, { query: "type:Organization" },