From de3030a1ee2f9f162ac4687ba9ea80108695b44d Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:18:59 -1000 Subject: [PATCH 01/11] chore(deps): pin @diffbot/typescript ^0.5.0 with a dev-only sibling link Co-Authored-By: Claude Sonnet 5.5 --- .pnpmfile.cjs | 26 ++++++++++++++++++++++++++ package.json | 2 +- pnpm-lock.yaml | 12 ++++-------- 3 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 .pnpmfile.cjs diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs new file mode 100644 index 0000000..db45b18 --- /dev/null +++ b/.pnpmfile.cjs @@ -0,0 +1,26 @@ +// DEV-ONLY local SDK resolution. +// TODO(@diffbot/typescript 0.5.0): once 0.5.0 is on npm, delete this file, then run +// `pnpm install` to refresh pnpm-lock.yaml against the registry. +// +// While `@diffbot/typescript` ^0.5.0 is unpublished, resolve it to a sibling checkout +// (../diffbot-typescript, built to dist/) when that directory exists. Without the +// sibling this is a no-op and the production range in package.json applies. +const fs = require("node:fs"); +const path = require("node:path"); + +// DIFFBOT_TYPESCRIPT_DIR (relative to this repo or absolute) overrides the sibling path, e.g. +// to point at a separately built copy when the sibling's dist/ is stale. +const sibling = path.resolve(__dirname, process.env.DIFFBOT_TYPESCRIPT_DIR || "../diffbot-typescript"); +const hasSibling = fs.existsSync(path.join(sibling, "package.json")); +const linkTarget = "link:" + path.relative(__dirname, sibling); + +module.exports = { + hooks: { + readPackage(pkg) { + if (hasSibling && pkg.name === "@diffbot/ai-sdk" && pkg.dependencies?.["@diffbot/typescript"]) { + pkg.dependencies["@diffbot/typescript"] = linkTarget; + } + return pkg; + }, + }, +}; diff --git a/package.json b/package.json index 4516ce2..3a8475a 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "test:workers": "tests/fixtures/worker/dry-run.sh" }, "dependencies": { - "@diffbot/typescript": "^0.3.0" + "@diffbot/typescript": "^0.5.0" }, "peerDependencies": { "@ai-sdk/openai-compatible": "^2.0.0 || ^3.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 59af73a..79b5fe1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,13 +4,15 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +pnpmfileChecksum: sha256-fg5/4wz7hn/Yc5bVuAjD78yR5opJnyjgjKYsazChans= + importers: .: dependencies: '@diffbot/typescript': - specifier: ^0.3.0 - version: 0.3.0 + specifier: link:../diffbot-typescript + version: link:../diffbot-typescript devDependencies: '@ai-sdk/openai-compatible': specifier: ^3.0.0 @@ -73,10 +75,6 @@ packages: resolution: {integrity: sha512-4uPgZ3qiX/JaQFr0dQoibUzHImZit9u05ro2BxGUnJwU4fV28BOAbKUa2OjQvqtM1tMac3bTKkaXdyzGovQXqg==} engines: {node: '>=22'} - '@diffbot/typescript@0.3.0': - resolution: {integrity: sha512-VkHiLohAbhUzTNbLL8gwNXAIoTrSfyenAIam5sVLhUGysSkkxuVbNX78BVoU4AlM5pwF3FNEA6Z/5RBRl+TLbQ==} - engines: {node: '>=18'} - '@esbuild/aix-ppc64@0.27.7': resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} @@ -1430,8 +1428,6 @@ snapshots: dependencies: json-schema: 0.4.0 - '@diffbot/typescript@0.3.0': {} - '@esbuild/aix-ppc64@0.27.7': optional: true From f86a2c583e2779e413585625d0229db1d6effdc2 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:19:23 -1000 Subject: [PATCH 02/11] fix(test): assert DQL as POST with a JSON body Co-Authored-By: Claude Sonnet 5.5 --- tests/helpers/mock-fetch.ts | 15 ++++++++++++++ tests/retrieve/knowledge-graph.test.ts | 28 ++++++++++++++------------ tests/tools/dql-probe.test.ts | 8 ++++---- tests/tools/knowledge-graph.test.ts | 14 ++++++++----- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/tests/helpers/mock-fetch.ts b/tests/helpers/mock-fetch.ts index 963086f..d881b6b 100644 --- a/tests/helpers/mock-fetch.ts +++ b/tests/helpers/mock-fetch.ts @@ -12,6 +12,8 @@ export interface MockRequest { headers: Headers; params: URLSearchParams; body?: string; + /** `body` parsed as JSON (POST bodies such as DQL); undefined if absent or not JSON. */ + json?: Record; } export type MockHandler = (request: MockRequest) => Response | Promise; @@ -22,6 +24,18 @@ function toTextBody(body: RequestInit["body"]): string | undefined { return undefined; } +function parseJsonBody(body: string | undefined): Record | undefined { + if (body === undefined) return undefined; + try { + const parsed: unknown = JSON.parse(body); + return parsed !== null && typeof parsed === "object" + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} + export function createMockFetch(handler: MockHandler): FetchFn { return async (input, init) => { const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; @@ -36,6 +50,7 @@ export function createMockFetch(handler: MockHandler): FetchFn { headers, params: parsed.searchParams, body, + json: parseJsonBody(body), }); }; } diff --git a/tests/retrieve/knowledge-graph.test.ts b/tests/retrieve/knowledge-graph.test.ts index e2a3fd0..e8dbf0f 100644 --- a/tests/retrieve/knowledge-graph.test.ts +++ b/tests/retrieve/knowledge-graph.test.ts @@ -66,8 +66,8 @@ describe("retrieveKnowledgeGraph document mapping", () => { const docs = await retrieve( (req) => { expect(req.url).toBe(DQL_URL); - expect(req.method).toBe("GET"); - expect(req.params.get("query")).toBe("type:Organization"); + expect(req.method).toBe("POST"); + expect(req.json?.query).toBe("type:Organization"); return jsonResponse(SAMPLE_BODY); }, { k: 2, query: "type:Organization" }, @@ -136,7 +136,7 @@ describe("retrieveKnowledgeGraph request wiring", () => { it("sends k as the size wire param", async () => { await retrieve( (req) => { - expect(req.params.get("size")).toBe("3"); + expect(req.json?.size).toBe(3); return jsonResponse(SAMPLE_BODY); }, { k: 3, query: "type:Organization" }, @@ -144,9 +144,9 @@ describe("retrieveKnowledgeGraph request wiring", () => { }); it("defaults to size 10 and forwards from, filter, exportspec and extra", async () => { - const seen: URLSearchParams[] = []; + const seen: Array<{ params: URLSearchParams; json: Record }> = []; const handler: MockHandler = (req) => { - seen.push(req.params); + seen.push({ params: req.params, json: req.json ?? {} }); return jsonResponse(SAMPLE_BODY); }; await retrieve(handler, { query: "q" }); @@ -158,13 +158,15 @@ describe("retrieveKnowledgeGraph request wiring", () => { extra: { nonCanonicalFacts: "true" }, }); - expect(seen[0]!.get("size")).toBe("10"); - expect(seen[0]!.has("filter")).toBe(false); - expect(seen[1]!.get("from")).toBe("20"); - expect(seen[1]!.get("filter")).toBe("nbEmployees>50"); - expect(seen[1]!.get("exportspec")).toBe("name"); - expect(seen[1]!.get("nonCanonicalFacts")).toBe("true"); - expect(seen[1]!.get("token")).toBe("test-token"); + expect(seen[0]!.json.size).toBe(10); + expect("filter" in seen[0]!.json).toBe(false); + expect(seen[1]!.json).toMatchObject({ + from: 20, + filter: "nbEmployees>50", + exportspec: "name", + nonCanonicalFacts: "true", + }); + expect(seen[1]!.params.get("token")).toBe("test-token"); }); it("caps the documents at k even if the server returns more", async () => { @@ -204,7 +206,7 @@ describe("retrieveKnowledgeGraph request wiring", () => { const docs = await retrieve( (req) => { /* The SDK omits the param for the default format. */ - expect(req.params.has("format")).toBe(false); + expect("format" in (req.json ?? {})).toBe(false); return jsonResponse(SAMPLE_BODY); }, { query: "q", format: "json" }, diff --git a/tests/tools/dql-probe.test.ts b/tests/tools/dql-probe.test.ts index 4f05174..16a4a2e 100644 --- a/tests/tools/dql-probe.test.ts +++ b/tests/tools/dql-probe.test.ts @@ -23,10 +23,10 @@ describe("diffbotDqlProbe", () => { const out = await runProbe( (req) => { expect(req.url).toBe(DQL_URL); - expect(req.method).toBe("GET"); + expect(req.method).toBe("POST"); expect(req.params.get("token")).toBe("test-token"); - expect(req.params.get("size")).toBe("0"); - const query = req.params.get("query") ?? ""; + expect(req.json?.size).toBe(0); + const query = String(req.json?.query ?? ""); const hits = query.includes("Diffbot") ? 5 : 100; return jsonResponse({ hits, results: 0 }); }, @@ -45,7 +45,7 @@ describe("diffbotDqlProbe", () => { const queries = ["q0", "q1", "q2", "q3"]; const out = await runProbe( async (req) => { - const query = req.params.get("query") ?? ""; + const query = String(req.json?.query ?? ""); const delay = (queries.length - queries.indexOf(query)) * 5; await new Promise((resolve) => setTimeout(resolve, delay)); return jsonResponse({ hits: queries.indexOf(query) }); diff --git a/tests/tools/knowledge-graph.test.ts b/tests/tools/knowledge-graph.test.ts index d309928..7c56f67 100644 --- a/tests/tools/knowledge-graph.test.ts +++ b/tests/tools/knowledge-graph.test.ts @@ -25,6 +25,7 @@ describe("diffbotKnowledgeGraph", () => { await runDql( (req) => { expect(req.url).toBe("https://kg.diffbot.com/kg/v3/dql"); + expect(req.method).toBe("POST"); expect(req.params.get("token")).toBe("test-token"); return jsonResponse({ data: [] }); }, @@ -35,10 +36,13 @@ describe("diffbotKnowledgeGraph", () => { it("sends size, from, and filter on the wire", async () => { await runDql( (req) => { - expect(req.params.get("query")).toBe("type:Organization"); - expect(req.params.get("size")).toBe("5"); - expect(req.params.get("from")).toBe("20"); - expect(req.params.get("filter")).toBe("nbEmployees>100"); + expect(req.json).toMatchObject({ + query: "type:Organization", + size: 5, + from: 20, + filter: "nbEmployees>100", + }); + expect(req.params.has("query")).toBe(false); return jsonResponse({ data: [] }); }, { query: "type:Organization", size: 5, from: 20, filter: "nbEmployees>100" }, @@ -48,7 +52,7 @@ describe("diffbotKnowledgeGraph", () => { it("defaults size to 10", async () => { await runDql( (req) => { - expect(req.params.get("size")).toBe("10"); + expect(req.json?.size).toBe(10); return jsonResponse({ data: [] }); }, { query: "type:Organization" }, From 96fc2316926bf9717599734edd49ec89dd6fb3c4 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:19:47 -1000 Subject: [PATCH 03/11] feat(extract): pass SDK ExtractOptions through the tool and loaders Co-Authored-By: Claude Sonnet 5.5 --- pnpm-lock.yaml | 6 +++--- src/retrieve/extract.ts | 21 ++++++++++++++++++--- src/tools/extract.ts | 23 +++++++++++++++++++++-- tests/retrieve/extract.test.ts | 19 +++++++++++++++++++ tests/tools/extract.test.ts | 27 +++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 8 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79b5fe1..eea0fcf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,15 +4,15 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -pnpmfileChecksum: sha256-fg5/4wz7hn/Yc5bVuAjD78yR5opJnyjgjKYsazChans= +pnpmfileChecksum: sha256-gY/fcS52hpgZg2yTS1qqEdIOZw9YciuCOtugpxYnAV8= importers: .: dependencies: '@diffbot/typescript': - specifier: link:../diffbot-typescript - version: link:../diffbot-typescript + specifier: link:../../tmp/sdk-build/diffbot-typescript + version: link:../../tmp/sdk-build/diffbot-typescript devDependencies: '@ai-sdk/openai-compatible': specifier: ^3.0.0 diff --git a/src/retrieve/extract.ts b/src/retrieve/extract.ts index c301e79..0a7e3dd 100644 --- a/src/retrieve/extract.ts +++ b/src/retrieve/extract.ts @@ -1,4 +1,4 @@ -import { extract, type JsonObject } from "@diffbot/typescript"; +import { extract, type ExtractOptions, type JsonObject } from "@diffbot/typescript"; import { type DiffbotAuth, createClientResolver } from "../client.js"; import { shapeExtractResponse, type DiffbotDocument } from "../documents.js"; @@ -8,6 +8,8 @@ export interface LoadExtractOptions extends DiffbotAuth { urls: string[]; api?: string; fmt?: string; + /** Extra SDK extract options (`timeout`, `mode`, `fields`, `paging`, `extra`, …) for every URL. */ + extractOptions?: ExtractOptions; documentMapper?: ExtractDocumentMapper; } @@ -28,12 +30,25 @@ function defaultExtractMapper(url: string, raw: JsonObject): DiffbotDocument { export async function* streamExtract( options: LoadExtractOptions, ): AsyncGenerator { - const { urls, api = "analyze", fmt = "markdown", documentMapper, ...auth } = options; + const { + urls, + api = "analyze", + fmt = "markdown", + extractOptions, + documentMapper, + ...auth + } = options; const client = createClientResolver(auth)(); const mapper = documentMapper ?? defaultExtractMapper; for (const url of urls) { - const raw = (await extract(client, url, api, fmt)) as JsonObject; + const raw = (await extract( + client, + url, + api, + fmt, + extractOptions as Parameters>[4], + )) as JsonObject; yield mapper(url, raw); } } diff --git a/src/tools/extract.ts b/src/tools/extract.ts index b1a9220..9b64a3a 100644 --- a/src/tools/extract.ts +++ b/src/tools/extract.ts @@ -1,4 +1,10 @@ -import { extract, ExtractionError, type DiffbotClient, type JsonObject } from "@diffbot/typescript"; +import { + extract, + ExtractionError, + type DiffbotClient, + type ExtractOptions, + type JsonObject, +} from "@diffbot/typescript"; import { tool, type Tool } from "ai"; import { z } from "zod"; import { type DiffbotAuth, createClientResolver } from "../client.js"; @@ -7,6 +13,12 @@ import { shapeExtractResponse, type ShapedExtractResponse } from "../documents.j export interface DiffbotExtractOptions extends DiffbotAuth { api?: string; fmt?: string; + /** + * Extra `@diffbot/typescript` extract options (`timeout`, `mode`, `fields`, `fallback`, + * `discussion`, `paging`, `extra`, …) applied to every call. Set by the developer, not the + * model; the tool's input schema is unchanged. + */ + extractOptions?: ExtractOptions; } export interface DiffbotExtractToolError { @@ -50,7 +62,14 @@ export function createExtractTool( const fmt = input.fmt ?? options?.fmt ?? "markdown"; try { - const raw = (await extract(client, input.url, api, fmt)) as JsonObject; + const raw = (await extract( + client, + input.url, + api, + fmt, + // `api` is a runtime string, so the per-API option union cannot be narrowed here. + options?.extractOptions as Parameters>[4], + )) as JsonObject; return shapeExtractResponse(raw); } catch (e) { if (e instanceof ExtractionError) { diff --git a/tests/retrieve/extract.test.ts b/tests/retrieve/extract.test.ts index 81fefd4..24f0387 100644 --- a/tests/retrieve/extract.test.ts +++ b/tests/retrieve/extract.test.ts @@ -104,6 +104,25 @@ describe("loadExtract", () => { expect(seen!.params.has("mode")).toBe(false); }); + it("forwards extractOptions to every request without dropping api/fmt defaults", async () => { + const seen: URLSearchParams[] = []; + await load( + (req) => { + seen.push(req.params); + return jsonResponse({ objects: [{ text: "x" }] }); + }, + ["https://example.com/a", "https://example.com/b"], + { extractOptions: { timeout: 60_000, fallback: "article", discussion: false } }, + ); + expect(seen).toHaveLength(2); + for (const params of seen) { + expect(params.get("timeout")).toBe("60000"); + expect(params.get("fallback")).toBe("article"); + expect(params.get("discussion")).toBe("false"); + expect(params.get("mode")).toBe("llm"); + } + }); + it("propagates an extraction failure, like the LangChain loader", async () => { const handler: MockHandler = (req) => req.params.get("url")?.endsWith("/bad") diff --git a/tests/tools/extract.test.ts b/tests/tools/extract.test.ts index 187ebfa..bdecdba 100644 --- a/tests/tools/extract.test.ts +++ b/tests/tools/extract.test.ts @@ -43,6 +43,33 @@ describe("diffbotExtract", () => { ); }); + it("threads factory extractOptions onto the wire", async () => { + await runExtract( + (req) => { + expect(req.url).toBe("https://api.diffbot.com/v3/article"); + expect(req.params.get("timeout")).toBe("45000"); + expect(req.params.get("paging")).toBe("false"); + expect(req.params.get("fields")).toBe("links,meta"); + expect(req.params.get("mode")).toBeNull(); + return jsonResponse({ objects: [{ text: "Body." }] }); + }, + { url: "https://example.com", api: "article", fmt: "json" }, + { extractOptions: { timeout: 45_000, paging: false, fields: ["links", "meta"] } }, + ); + }); + + it("keeps markdown mode and default timeout when no extractOptions are set", async () => { + await runExtract( + (req) => { + expect(req.params.get("mode")).toBe("llm"); + expect(req.params.get("timeout")).toBe("30000"); + expect(req.params.has("fields")).toBe(false); + return jsonResponse({ objects: [{ text: "Body." }] }); + }, + { url: "https://example.com" }, + ); + }); + it("shapes successful responses", async () => { const out = await runExtract( () => From f3597a37f0171a967fca7939ae4ddbc512b156eb Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:21:10 -1000 Subject: [PATCH 04/11] chore(deps): refresh lockfile for the sibling link and lint-ignore the pnpmfile Co-Authored-By: Claude Sonnet 5.5 --- .pnpmfile.cjs | 11 +++++++++-- eslint.config.ts | 1 + pnpm-lock.yaml | 6 +++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs index db45b18..3676e93 100644 --- a/.pnpmfile.cjs +++ b/.pnpmfile.cjs @@ -10,14 +10,21 @@ const path = require("node:path"); // DIFFBOT_TYPESCRIPT_DIR (relative to this repo or absolute) overrides the sibling path, e.g. // to point at a separately built copy when the sibling's dist/ is stale. -const sibling = path.resolve(__dirname, process.env.DIFFBOT_TYPESCRIPT_DIR || "../diffbot-typescript"); +const sibling = path.resolve( + __dirname, + process.env.DIFFBOT_TYPESCRIPT_DIR || "../diffbot-typescript", +); const hasSibling = fs.existsSync(path.join(sibling, "package.json")); const linkTarget = "link:" + path.relative(__dirname, sibling); module.exports = { hooks: { readPackage(pkg) { - if (hasSibling && pkg.name === "@diffbot/ai-sdk" && pkg.dependencies?.["@diffbot/typescript"]) { + if ( + hasSibling && + pkg.name === "@diffbot/ai-sdk" && + pkg.dependencies?.["@diffbot/typescript"] + ) { pkg.dependencies["@diffbot/typescript"] = linkTarget; } return pkg; diff --git a/eslint.config.ts b/eslint.config.ts index d3f49e2..d22c06e 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -8,6 +8,7 @@ export default tseslint.config( tsc and vitest are what judge them, not eslint. */ ignores: [ "dist/**", + ".pnpmfile.cjs", "node_modules/**", "coverage/**", "**/.wrangler/**", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eea0fcf..4ffe9a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,15 +4,15 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -pnpmfileChecksum: sha256-gY/fcS52hpgZg2yTS1qqEdIOZw9YciuCOtugpxYnAV8= +pnpmfileChecksum: sha256-ufjlYadkGX1lk1TqT83j9UssZyTlwlfDBdRUTr2Vrwo= importers: .: dependencies: '@diffbot/typescript': - specifier: link:../../tmp/sdk-build/diffbot-typescript - version: link:../../tmp/sdk-build/diffbot-typescript + specifier: link:../diffbot-typescript + version: link:../diffbot-typescript devDependencies: '@ai-sdk/openai-compatible': specifier: ^3.0.0 From ec1880f2ead891ba06f5a3758809e4a0006f721f Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:21:10 -1000 Subject: [PATCH 05/11] feat(crawl): opt-in poll + crawlDownload path for loadCrawl/streamCrawl Co-Authored-By: Claude Sonnet 5.5 --- src/index.ts | 3 + src/retrieve/crawl.ts | 119 +++++++++++++++++++++++++++++++- tests/imports.test.ts | 1 + tests/retrieve/crawl.test.ts | 129 ++++++++++++++++++++++++++++++++++- 4 files changed, 249 insertions(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index 31f9df0..7684b48 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,7 +39,10 @@ export { loadCrawl, streamCrawl, defaultCrawlMapper, + defaultCrawlRecordMapper, + type CrawlDownloadLoadOptions, type CrawlEventMapper, + type CrawlRecordMapper, type StreamCrawlOptions, } from "./retrieve/crawl.js"; diff --git a/src/retrieve/crawl.ts b/src/retrieve/crawl.ts index ab482e1..02ff760 100644 --- a/src/retrieve/crawl.ts +++ b/src/retrieve/crawl.ts @@ -1,4 +1,13 @@ -import { crawl, CrawlEventType, type CrawlEvent, type CrawlOptions } from "@diffbot/typescript"; +import { + crawl, + crawlDownload, + crawlGetJob, + CrawlEventType, + JobError, + type CrawlDataRecord, + type CrawlEvent, + type CrawlOptions, +} from "@diffbot/typescript"; import { type DiffbotAuth, createClientResolver } from "../client.js"; import type { DiffbotDocument } from "../documents.js"; @@ -19,10 +28,51 @@ export function defaultCrawlMapper(event: CrawlEvent): DiffbotDocument | null { }; } +export type CrawlRecordMapper = (record: CrawlDataRecord) => DiffbotDocument | null; + +/** + * Maps one `crawlDownload` JSON record. `content` is the extracted body (falling back to the + * title), and metadata is picked field by field: the server echoes the caller's token in each + * record, so the record is never spread wholesale. + */ +export function defaultCrawlRecordMapper(record: CrawlDataRecord): DiffbotDocument { + const url = typeof record.pageUrl === "string" ? record.pageUrl : ""; + const body = typeof record.content === "string" && record.content ? record.content : undefined; + const title = typeof record.title === "string" ? record.title : undefined; + return { + content: body ?? title ?? url, + metadata: { + url, + title, + type: record.type, + parentUrl: record.parentUrl, + id: record.id, + crawl_timestamp: record.timestamp, + }, + }; +} + +/** Options for the opt-in poll + download path (`crawlDownload`). */ +export interface CrawlDownloadLoadOptions { + /** Only the N most recently processed URLs (the SDK's `num`). */ + num?: number; + /** Downloaded record type, passed through as the SDK's `type`. */ + type?: string; + /** Maps each downloaded record; return `null` to drop it. */ + recordMapper?: CrawlRecordMapper; +} + export interface StreamCrawlOptions extends DiffbotAuth { site: string; crawlOptions?: CrawlOptions; eventMapper?: CrawlEventMapper; + /** + * Opt in to the non-watch path: start the job without watching, poll `crawlGetJob` until it + * reaches a terminal status, then fetch the extracted records with `crawlDownload` (JSON). + * Yields one document per record instead of one per crawled URL. Omit for the default + * watch-stream behavior, where `eventMapper` applies; `eventMapper` is ignored here. + */ + download?: true | CrawlDownloadLoadOptions; } function resolvedCrawlOptions(crawlOptions?: CrawlOptions): CrawlOptions { @@ -33,10 +83,75 @@ function resolvedCrawlOptions(crawlOptions?: CrawlOptions): CrawlOptions { return opts; } +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function* streamCrawlDownload( + options: StreamCrawlOptions, +): AsyncGenerator { + const { site, crawlOptions, download, eventMapper: _unused, ...auth } = options; + void _unused; + const { num, type, recordMapper }: CrawlDownloadLoadOptions = + download === true ? {} : (download ?? {}); + if (type === "urls") { + // The URL report is CSV text, not records; the default watch path already covers it. + throw new TypeError('`download.type` "urls" returns CSV, not records; omit `download` instead'); + } + const client = createClientResolver(auth)(); + const mapper = recordMapper ?? defaultCrawlRecordMapper; + const pollInterval = crawlOptions?.pollInterval ?? 2000; + + let jobName = crawlOptions?.jobName; + for await (const event of crawl(client, site, { ...crawlOptions, watch: false })) { + if ( + event.eventType === CrawlEventType.JobCreated && + typeof event.details.job_name === "string" + ) { + jobName = event.details.job_name; + } + } + if (jobName === undefined) { + throw new JobError("", "Crawl did not report a job name"); + } + + for (;;) { + const job = await crawlGetJob(client, jobName); + if (!job) { + throw new JobError(jobName, "Crawler job not found while polling"); + } + const status = job.jobStatus?.status ?? 0; + if (status !== 0 && status !== 7) { + const message = job.jobStatus?.message ?? ""; + if (/fail|error/i.test(message)) { + throw new JobError(jobName, `Crawler job failed: ${message}`); + } + break; + } + await sleep(pollInterval); + } + + const records = await crawlDownload(client, jobName, { + ...(type !== undefined ? { type } : {}), + ...(num !== undefined ? { num } : {}), + }); + for (const record of records as CrawlDataRecord[]) { + const doc = mapper(record); + if (doc !== null) { + yield doc; + } + } +} + export async function* streamCrawl( options: StreamCrawlOptions, ): AsyncGenerator { - const { site, crawlOptions, eventMapper, ...auth } = options; + if (options.download) { + yield* streamCrawlDownload(options); + return; + } + const { site, crawlOptions, eventMapper, download: _download, ...auth } = options; + void _download; const client = createClientResolver(auth)(); const mapper = eventMapper ?? defaultCrawlMapper; const opts = resolvedCrawlOptions(crawlOptions); diff --git a/tests/imports.test.ts b/tests/imports.test.ts index e7a2280..cf2a07f 100644 --- a/tests/imports.test.ts +++ b/tests/imports.test.ts @@ -10,6 +10,7 @@ const EXPECTED_VALUES = [ "DEFAULT_KG_CONTENT_FIELDS", "DEFAULT_WEB_CONTENT_FIELDS", "defaultCrawlMapper", + "defaultCrawlRecordMapper", "dictToDocument", "diffbotAsk", "diffbotDqlProbe", diff --git a/tests/retrieve/crawl.test.ts b/tests/retrieve/crawl.test.ts index acbe1f7..a3f254d 100644 --- a/tests/retrieve/crawl.test.ts +++ b/tests/retrieve/crawl.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from "vitest"; import { CrawlEventType, type CrawlEvent } from "@diffbot/typescript"; import type { DiffbotDocument } from "../../src/documents.js"; -import { defaultCrawlMapper, loadCrawl, streamCrawl } from "../../src/retrieve/crawl.js"; +import { + defaultCrawlMapper, + defaultCrawlRecordMapper, + loadCrawl, + streamCrawl, +} from "../../src/retrieve/crawl.js"; import { createMockFetch, jsonResponse } from "../helpers/mock-fetch.js"; const CRAWL_URL = "https://api.diffbot.com/v3/crawl"; @@ -121,3 +126,125 @@ describe("streamCrawl watch defaulting", () => { expect(docs[0]!.content).toBe(CrawlEventType.JobCreated); }); }); + +describe("loadCrawl download path", () => { + const RECORDS = [ + { + pageUrl: "https://example.com/a", + title: "A", + content: "Body A", + type: "article", + id: 1, + token: "secret-echoed-by-server", + timestamp: "2026-01-01T00:00:00Z", + }, + { pageUrl: "https://example.com/b", title: "B" }, + ]; + + function crawlFetch(statuses: number[], log: Array<{ kind: string; params: URLSearchParams }>) { + let polls = 0; + return createMockFetch((req) => { + if (req.url.endsWith("/data")) { + log.push({ kind: "data", params: req.params }); + return jsonResponse(RECORDS); + } + if (req.params.get("seeds")) { + log.push({ kind: "create", params: req.params }); + return jsonResponse({ jobs: [{ name: "test-job" }] }); + } + log.push({ kind: "status", params: req.params }); + const status = statuses[Math.min(polls++, statuses.length - 1)]!; + return jsonResponse({ jobs: [{ name: "test-job", jobStatus: { status, message: "m" } }] }); + }); + } + + it("creates without watching, polls until terminal, then downloads JSON records", async () => { + const log: Array<{ kind: string; params: URLSearchParams }> = []; + const docs = await loadCrawl({ + apiKey: "test-token", + fetch: crawlFetch([0, 7, 9], log), + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: { num: 5 }, + }); + + expect(log.map((l) => l.kind)).toEqual(["create", "status", "status", "status", "data"]); + const data = log.at(-1)!.params; + expect(data.get("name")).toBe("test-job"); + expect(data.get("num")).toBe("5"); + expect(data.has("format")).toBe(false); + expect(data.has("type")).toBe(false); + + expect(docs).toHaveLength(2); + expect(docs[0]).toEqual({ + content: "Body A", + metadata: { + url: "https://example.com/a", + title: "A", + type: "article", + parentUrl: undefined, + id: 1, + crawl_timestamp: "2026-01-01T00:00:00Z", + }, + }); + expect(docs[1]!.content).toBe("B"); + expect(JSON.stringify(docs)).not.toContain("secret-echoed-by-server"); + }); + + it("accepts `download: true` and a custom recordMapper that can drop records", async () => { + const log: Array<{ kind: string; params: URLSearchParams }> = []; + const all = await loadCrawl({ + apiKey: "t", + fetch: crawlFetch([2], log), + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: true, + }); + expect(all).toHaveLength(2); + + const some = await loadCrawl({ + apiKey: "t", + fetch: crawlFetch([2], []), + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: { recordMapper: (r) => (r.title === "B" ? defaultCrawlRecordMapper(r) : null) }, + }); + expect(some.map((d) => d.metadata.title)).toEqual(["B"]); + }); + + it("throws a JobError when the job ends in failure, without downloading", async () => { + const log: Array<{ kind: string; params: URLSearchParams }> = []; + const fetch = createMockFetch((req) => { + log.push({ kind: req.url.endsWith("/data") ? "data" : "other", params: req.params }); + return req.params.get("seeds") + ? jsonResponse({ jobs: [{ name: "test-job" }] }) + : jsonResponse({ jobs: [{ jobStatus: { status: 4, message: "Job failed: bad seed" } }] }); + }); + await expect( + loadCrawl({ + apiKey: "t", + fetch, + site: "https://example.com", + crawlOptions: { jobName: "test-job", pollInterval: 0 }, + download: true, + }), + ).rejects.toThrow(/Crawler job failed/); + expect(log.some((l) => l.kind === "data")).toBe(false); + }); + + it("rejects type: urls, which is CSV rather than records", async () => { + await expect( + loadCrawl({ + apiKey: "t", + fetch: createMockFetch(() => jsonResponse({})), + site: "https://example.com", + download: { type: "urls" }, + }), + ).rejects.toThrow(/returns CSV/); + }); + + it("falls back to title, then URL, for content", () => { + expect(defaultCrawlRecordMapper({ pageUrl: "u", title: "T" }).content).toBe("T"); + expect(defaultCrawlRecordMapper({ pageUrl: "u" }).content).toBe("u"); + }); +}); From 47890be68410912c3dde25b907decc87c3530c16 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:21:10 -1000 Subject: [PATCH 06/11] docs: document extractOptions, crawl download, and the dev SDK link Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 12 ++++++++++++ CONTRIBUTING.md | 8 ++++++++ README.md | 11 ++++++++--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1cd74..b651f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,3 +24,15 @@ for the Vercel AI SDK (`ai` ^6 or ^7). timeout. - Examples: `examples/company-research` (tool-calling agent), `examples/manual-rag` (retrieve helpers, no tool loop), and `examples/worker` (Workers `env` bag and streamed RAG). +- `extractOptions` on `diffbotExtract`, `loadExtract`, and `streamExtract` passes the SDK's + `ExtractOptions` (`timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, …) through to + `extract`. The tool's model-facing input schema is unchanged, and `api`/`fmt` defaults stay. +- Opt-in `download` on `loadCrawl` / `streamCrawl`: start the job without watching, poll it, and + map the records from `crawlDownload` to documents (`defaultCrawlRecordMapper`). The default watch + stream is unchanged. + +### Changed + +- Depends on `@diffbot/typescript` `^0.5.0`, which POSTs DQL. Unit-test mocks now assert the POST + JSON body. Until 0.5.0 is on npm, `.pnpmfile.cjs` links a sibling `../diffbot-typescript` for + local development (see CONTRIBUTING). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d68f89..a1ca0c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,14 @@ After `pnpm test:peer:ai6` (or `:ai7`, or `:latest`) restore the lockfile — th git checkout package.json pnpm-lock.yaml && pnpm install --frozen-lockfile ``` +## Local SDK link (temporary) + +`@diffbot/typescript` `^0.5.0` is not on npm yet. `.pnpmfile.cjs` resolves it to a sibling +`../diffbot-typescript` checkout when that directory exists (build it first; it is dev-only and a +no-op without the sibling). Set `DIFFBOT_TYPESCRIPT_DIR` to use a different build. CI cannot resolve +the range until the release. TODO: once 0.5.0 is published, delete `.pnpmfile.cjs` and refresh +`pnpm-lock.yaml`. + ## Workflows | Workflow | Triggers | Secrets | diff --git a/README.md b/README.md index b5fd633..7b842af 100644 --- a/README.md +++ b/README.md @@ -532,14 +532,19 @@ Each returns (or yields) `DiffbotDocument` objects: - `retrieveWebSearch` — `query` (required), `k` (default `10`), `maxTokens`, `snippetLength`, plus `fields` / `contentFields` (default `DEFAULT_WEB_CONTENT_FIELDS`) / `documentMapper`. - `loadExtract` / `streamExtract` — `urls` (required), `api` (default `analyze`), `fmt` (default - `markdown`), `documentMapper(url, raw)`. `metadata` carries `url`, `title`, `pageUrl`, + `markdown`), `extractOptions` (the SDK's `ExtractOptions`: `timeout`, `mode`, `fields`, `fallback`, + `discussion`, `paging`, `extra`, …), `documentMapper(url, raw)`. `metadata` carries `url`, `title`, `pageUrl`, `resolvedPageUrl` and `type`. URLs are fetched one at a time, in order, and an extraction failure throws (as in `@diffbot/langchain`): `loadExtract` rejects, while `streamExtract` has already yielded the documents before the failing URL. - `loadCrawl` / `streamCrawl` — `site` (required), `crawlOptions` (the SDK's `CrawlOptions`; `watch` defaults to `true` so URL events are emitted), `eventMapper(event)` returning a document or `null` to skip. The default `content` is the crawled URL itself, with `url`, `status` and - `crawl_timestamp` in `metadata`. + `crawl_timestamp` in `metadata`. Pass `download: true` (or `{ num, type, recordMapper }`) to skip + the watch stream: the job is started without `watch`, polled until it finishes, and its extracted + records are fetched with the SDK's `crawlDownload`, one document per record (`content` is the + record's `content`, falling back to its title; `eventMapper` does not apply). `type: "urls"` is + rejected, since that report is CSV. `defaultCrawlMapper` drops the `job_created` event, so with `watch: false` you get no documents at all — that combination only makes sense with an `eventMapper`. Pass `crawlOptions.jobName` if you @@ -647,7 +652,7 @@ Every runtime export of `@diffbot/ai-sdk`: | `diffbotWebSearch`, `diffbotExtract`, `diffbotKnowledgeGraph`, `diffbotEntities`, `diffbotOntology`, `diffbotDqlProbe`, `diffbotAsk` | Model-facing `tool()` | | `diffbotTools` | Bundle | | `retrieveKnowledgeGraph`, `retrieveWebSearch` | Retrieval | -| `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper` | Loaders | +| `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper`, `defaultCrawlRecordMapper` | Loaders | | `dictToDocument`, `shapeExtractResponse`, `DEFAULT_KG_CONTENT_FIELDS`, `DEFAULT_WEB_CONTENT_FIELDS` | Shaping | | `CrawlEventType` | Re-export from the SDK | From 00b2eb1104a74f131ab62e3f94dbbdfdd6067803 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:28:28 -1000 Subject: [PATCH 07/11] refactor(crawl): pass options straight to the client resolver and drop a redundant cast Co-Authored-By: Claude Opus 5.5 --- src/retrieve/crawl.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/retrieve/crawl.ts b/src/retrieve/crawl.ts index 02ff760..9cf66b0 100644 --- a/src/retrieve/crawl.ts +++ b/src/retrieve/crawl.ts @@ -90,15 +90,15 @@ function sleep(ms: number): Promise { async function* streamCrawlDownload( options: StreamCrawlOptions, ): AsyncGenerator { - const { site, crawlOptions, download, eventMapper: _unused, ...auth } = options; - void _unused; + // `createClientResolver` reads only the `DiffbotAuth` keys, so the options pass through whole. + const { site, crawlOptions, download } = options; const { num, type, recordMapper }: CrawlDownloadLoadOptions = download === true ? {} : (download ?? {}); if (type === "urls") { // The URL report is CSV text, not records; the default watch path already covers it. throw new TypeError('`download.type` "urls" returns CSV, not records; omit `download` instead'); } - const client = createClientResolver(auth)(); + const client = createClientResolver(options)(); const mapper = recordMapper ?? defaultCrawlRecordMapper; const pollInterval = crawlOptions?.pollInterval ?? 2000; @@ -135,7 +135,7 @@ async function* streamCrawlDownload( ...(type !== undefined ? { type } : {}), ...(num !== undefined ? { num } : {}), }); - for (const record of records as CrawlDataRecord[]) { + for (const record of records) { const doc = mapper(record); if (doc !== null) { yield doc; @@ -150,9 +150,8 @@ export async function* streamCrawl( yield* streamCrawlDownload(options); return; } - const { site, crawlOptions, eventMapper, download: _download, ...auth } = options; - void _download; - const client = createClientResolver(auth)(); + const { site, crawlOptions, eventMapper } = options; + const client = createClientResolver(options)(); const mapper = eventMapper ?? defaultCrawlMapper; const opts = resolvedCrawlOptions(crawlOptions); From fae1d8b9566390eacdfa33dc1f1cafe25d3ee181 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:28:29 -1000 Subject: [PATCH 08/11] docs(agents): list defaultCrawlRecordMapper and the crawl download path Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1d4aa13..f9e8b6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ Precedence inside `execute` (lazy): injected `client` → `apiKey` → `resolveT ## Phase 4 scope (current) -Seven tools plus retrieval/loader helpers: `retrieveKnowledgeGraph`, `retrieveWebSearch`, `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper` (crawl is helpers-only, never a tool). Provider at `@diffbot/ai-sdk/provider` (`createDiffbot`). `diffbotAsk` defaults to `timeout: 120_000` when this package builds the client; injected clients keep their own timeout. `diffbotTools()` still builds one shared client, so that default covers the whole bundle when `diffbot_ask` is included. No examples yet (Phase 5). Live coverage is complete: every tool, both retrievers, `loadExtract`, and the provider (`generateText` + `streamText`). +Seven tools plus retrieval/loader helpers: `retrieveKnowledgeGraph`, `retrieveWebSearch`, `loadExtract`, `streamExtract`, `loadCrawl`, `streamCrawl`, `defaultCrawlMapper`, `defaultCrawlRecordMapper` (crawl is helpers-only, never a tool; `download` opts into start → poll → `crawlDownload`). Provider at `@diffbot/ai-sdk/provider` (`createDiffbot`). `diffbotAsk` defaults to `timeout: 120_000` when this package builds the client; injected clients keep their own timeout. `diffbotTools()` still builds one shared client, so that default covers the whole bundle when `diffbot_ask` is included. No examples yet (Phase 5). Live coverage is complete: every tool, both retrievers, `loadExtract`, and the provider (`generateText` + `streamText`). ## CI and live tests From 97c7019c4db49bfbbc01ec9b620dbf0e1df6934e Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 17:59:07 -1000 Subject: [PATCH 09/11] Prepare 0.1.0 for @diffbot/typescript 0.6.0 and trusted publishing. Drop the sibling pnpmfile, stop using NPM_TOKEN, and pass nonCanonicalFacts as a named DQL option. The lockfile stays unresolved until 0.6.0 is on npm. --- .github/workflows/release.yml | 39 ++++++++++++++++++++------ .pnpmfile.cjs | 33 ---------------------- CHANGELOG.md | 7 ++--- CLAUDE.md | 2 +- CONTRIBUTING.md | 23 ++++++++------- README.md | 6 ++-- eslint.config.ts | 1 - package.json | 6 ++-- src/retrieve/knowledge-graph.ts | 8 ++++++ tests/retrieve/knowledge-graph.test.ts | 6 ++-- 10 files changed, 64 insertions(+), 67 deletions(-) delete mode 100644 .pnpmfile.cjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ba10e2..417b594 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,9 @@ name: Release # tests, build, `pnpm pack`) and nothing else. Every step that publishes to npm # or creates a GitHub release is gated on a real tag push, so a dispatch — on # any branch, by anyone with write access — can never ship a version. +# +# npm auth is trusted publishing (OIDC). No NPM_TOKEN and no GitHub environment. +# The package must already exist on npm before a trusted publisher can be added. on: push: tags: ["v*"] @@ -93,22 +96,35 @@ jobs: registry-url: https://registry.npmjs.org - run: pnpm install --frozen-lockfile - run: pnpm build + # Trusted publishing needs npm >= 11.5.1; Node 22 ships older. Use npm, + # not pnpm, so the OIDC token exchange works. No token is set. + - name: Update npm + run: | + npm install -g npm@latest + npm --version + # The dist-tag is derived from the version, not from the tag name: a # prerelease (anything containing a `-`, e.g. 0.1.0-rc.1) publishes as # `next` so a plain `npm install @diffbot/ai-sdk` keeps resolving to the # last stable release. The guards above already proved the git tag equals # this version. - - name: pnpm publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - name: Publish to npm run: | + set -euo pipefail + name="$(node -p "require('./package.json').name")" version="$(node -p "require('./package.json').version")" case "$version" in *-*) dist_tag=next ;; *) dist_tag=latest ;; esac - echo "Publishing $version with --tag $dist_tag." - pnpm publish --access public --provenance --no-git-checks --tag "$dist_tag" + if npm view "${name}@${version}" version >/dev/null 2>&1; then + echo "${name}@${version} is already on npm; skipping publish." + exit 0 + fi + echo "Publishing $version with --tag $dist_tag (npm trusted publishing)." + # --ignore-scripts: dist/ was built above. No NODE_AUTH_TOKEN; npm + # authenticates with the trusted publisher on npmjs.com. + npm publish --access public --provenance --ignore-scripts --tag "$dist_tag" github-release: name: create GitHub release @@ -125,7 +141,12 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - gh release create "$GITHUB_REF_NAME" \ - --repo "$GITHUB_REPOSITORY" \ - --title "$GITHUB_REF_NAME" \ - --generate-notes + set -euo pipefail + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "Release $GITHUB_REF_NAME already exists; leaving it." + else + gh release create "$GITHUB_REF_NAME" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$GITHUB_REF_NAME" \ + --generate-notes + fi diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs deleted file mode 100644 index 3676e93..0000000 --- a/.pnpmfile.cjs +++ /dev/null @@ -1,33 +0,0 @@ -// DEV-ONLY local SDK resolution. -// TODO(@diffbot/typescript 0.5.0): once 0.5.0 is on npm, delete this file, then run -// `pnpm install` to refresh pnpm-lock.yaml against the registry. -// -// While `@diffbot/typescript` ^0.5.0 is unpublished, resolve it to a sibling checkout -// (../diffbot-typescript, built to dist/) when that directory exists. Without the -// sibling this is a no-op and the production range in package.json applies. -const fs = require("node:fs"); -const path = require("node:path"); - -// DIFFBOT_TYPESCRIPT_DIR (relative to this repo or absolute) overrides the sibling path, e.g. -// to point at a separately built copy when the sibling's dist/ is stale. -const sibling = path.resolve( - __dirname, - process.env.DIFFBOT_TYPESCRIPT_DIR || "../diffbot-typescript", -); -const hasSibling = fs.existsSync(path.join(sibling, "package.json")); -const linkTarget = "link:" + path.relative(__dirname, sibling); - -module.exports = { - hooks: { - readPackage(pkg) { - if ( - hasSibling && - pkg.name === "@diffbot/ai-sdk" && - pkg.dependencies?.["@diffbot/typescript"] - ) { - pkg.dependencies["@diffbot/typescript"] = linkTarget; - } - return pkg; - }, - }, -}; diff --git a/CHANGELOG.md b/CHANGELOG.md index b651f5e..74329e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.1.0 — Unreleased +## 0.1.0 - 2026-11-17 First API of `@diffbot/ai-sdk`: Diffbot tools, retrieval helpers, and an OpenAI-compatible provider for the Vercel AI SDK (`ai` ^6 or ^7). @@ -33,6 +33,5 @@ for the Vercel AI SDK (`ai` ^6 or ^7). ### Changed -- Depends on `@diffbot/typescript` `^0.5.0`, which POSTs DQL. Unit-test mocks now assert the POST - JSON body. Until 0.5.0 is on npm, `.pnpmfile.cjs` links a sibling `../diffbot-typescript` for - local development (see CONTRIBUTING). +- Depends on `@diffbot/typescript` `^0.6.0` (cumulative 0.4.0–0.6.0). DQL is POST with a JSON body; the token stays the query parameter. Unit-test mocks already assert that body. The dev-only `.pnpmfile.cjs` sibling link is removed. Refresh `pnpm-lock.yaml` from the registry once 0.6.0 is on npm; do not commit a `link:` or `file:` override. +- `retrieveKnowledgeGraph` takes `nonCanonicalFacts` as a named boolean. Passing that key through `extra` throws, matching the SDK. diff --git a/CLAUDE.md b/CLAUDE.md index f9e8b6a..652c661 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ Four workflows: - `ci.yml` — unit-only, **no secrets** (lint, typecheck, unit tests + build on Node 20/22/24, the `ai@6`/`ai@7` peer matrix, empty-token live-skip guard, Workers bundle check). Keep it secret-free so it can run on fork PRs. - `integration.yml` — `pnpm test:live`, on push to `main`, a twice-monthly drift schedule (1st and 15th at 14:17 UTC), and `workflow_dispatch`. Never `pull_request`/`pull_request_target`, so `DIFFBOT_API_TOKEN` is never exposed to fork PR code. A guard step fails the run on a missing or blank secret rather than skipping everything green. - `canary.yml` — weekly `ai@latest` + `@ai-sdk/openai-compatible@latest` typecheck + unit tests. No secrets. -- `release.yml` — `v*` tag only; `workflow_dispatch` is a build-only rehearsal and every publish/release step is gated on the tag push. It fails early unless the tag equals `package.json`'s version and `"private"` is no longer `true` (Phase 6 flips that flag — do not change it before then). +- `release.yml` — `v*` tag only; `workflow_dispatch` is a build-only rehearsal and every publish/release step is gated on the tag push. It fails early unless the tag equals `package.json`'s version and `"private"` is not `true`. Publish uses npm trusted publishing (OIDC). There is no `NPM_TOKEN`. `@diffbot/ai-sdk` is not on npm yet, so a trusted publisher cannot be added until a one-time manual bootstrap publish creates the package. `pnpm test:workers` dry-run bundles `tests/fixtures/worker/` for workerd and then scans the bundle: `wrangler deploy --dry-run` alone leaves static `node:*` imports in the output as externals, so the scan is what actually enforces Workers-cleanliness. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1ca0c9..d4aefbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,24 +24,23 @@ git checkout package.json pnpm-lock.yaml && pnpm install --frozen-lockfile ## Local SDK link (temporary) -`@diffbot/typescript` `^0.5.0` is not on npm yet. `.pnpmfile.cjs` resolves it to a sibling -`../diffbot-typescript` checkout when that directory exists (build it first; it is dev-only and a -no-op without the sibling). Set `DIFFBOT_TYPESCRIPT_DIR` to use a different build. CI cannot resolve -the range until the release. TODO: once 0.5.0 is published, delete `.pnpmfile.cjs` and refresh -`pnpm-lock.yaml`. +The release dependency is `@diffbot/typescript` `^0.6.0`. Do not commit a `file:` / `link:` specifier +or a `.pnpmfile.cjs`. Until 0.6.0 is on npm, `pnpm-lock.yaml` still points at a sibling link and +cannot be refreshed from the registry. For a local unit-test run, link a built 0.6.0 checkout and +restore `pnpm-lock.yaml` before committing. ## Workflows -| Workflow | Triggers | Secrets | -| ----------------- | ------------------------------------------------- | ------------------------- | -| `ci.yml` | push to `main`, every `pull_request`, dispatch | **none** | -| `integration.yml` | push to `main`, 1st + 15th at 14:17 UTC, dispatch | `DIFFBOT_API_TOKEN` | -| `canary.yml` | Mondays at 14:17 UTC, dispatch | **none** | -| `release.yml` | `v*` tag; dispatch is a build-only rehearsal | `NPM_TOKEN` (publish job) | +| Workflow | Triggers | Secrets | +| ----------------- | ------------------------------------------------- | --------------------------------- | +| `ci.yml` | push to `main`, every `pull_request`, dispatch | **none** | +| `integration.yml` | push to `main`, 1st + 15th at 14:17 UTC, dispatch | `DIFFBOT_API_TOKEN` | +| `canary.yml` | Mondays at 14:17 UTC, dispatch | **none** | +| `release.yml` | `v*` tag; dispatch is a build-only rehearsal | **none** (npm trusted publishing) | `ci.yml` runs lint + typecheck, unit tests + build on Node 20 / 22 / 24, the peer matrix (`ai@6` + `@ai-sdk/openai-compatible@2`, `ai@7` + `@3`), the empty-token live-skip guard, and the Workers bundle check. **It must stay secret-free** — that is what lets every job run on fork PRs. -`release.yml` publishes only from a `v*` tag push: it checks the tag equals `package.json`'s `version` and that `"private"` is no longer `true`, runs lint/typecheck/tests/build, then `pnpm publish --access public --provenance`, then creates the GitHub release. `workflow_dispatch` stops after `pnpm pack`; every publishing step is gated on the tag push, so a dispatch cannot ship a version. To release: bump `version`, commit, tag `vX.Y.Z`, push the tag. +`release.yml` publishes only from a `v*` tag push: it checks the tag equals `package.json`'s `version` and that `"private"` is not `true`, runs lint/typecheck/tests/build, then `npm publish --access public --provenance` via trusted publishing (OIDC, no `NPM_TOKEN`), then creates the GitHub release. `workflow_dispatch` stops after `pnpm pack`; every publishing step is gated on the tag push, so a dispatch cannot ship a version. To release: bump `version`, commit, tag `vX.Y.Z`, push the tag. npm cannot attach a trusted publisher until the package exists; the first publish of `@diffbot/ai-sdk` needs a one-time manual bootstrap (see the release report) before that tag will succeed. ## Live tests diff --git a/README.md b/README.md index 7b842af..ab25e56 100644 --- a/README.md +++ b/README.md @@ -527,8 +527,10 @@ Each returns (or yields) `DiffbotDocument` objects: `{ content: string; metadata: Record }`. Options beyond the auth ones: - `retrieveKnowledgeGraph` — `query` (DQL, required), `k` (default `10`), `from`, `filter`, - `exportspec`, `extra`, `format` (default `json`; other formats throw, since only JSON can be - shaped), plus `fields` / `contentFields` (default `DEFAULT_KG_CONTENT_FIELDS`) / `documentMapper`. + `exportspec`, `nonCanonicalFacts`, `extra` (only keys that are not named DQL options; 0.6.0 + throws if a named key such as `nonCanonicalFacts` is passed through `extra`), `format` (default + `json`; other formats throw, since only JSON can be shaped), plus `fields` / `contentFields` + (default `DEFAULT_KG_CONTENT_FIELDS`) / `documentMapper`. - `retrieveWebSearch` — `query` (required), `k` (default `10`), `maxTokens`, `snippetLength`, plus `fields` / `contentFields` (default `DEFAULT_WEB_CONTENT_FIELDS`) / `documentMapper`. - `loadExtract` / `streamExtract` — `urls` (required), `api` (default `analyze`), `fmt` (default diff --git a/eslint.config.ts b/eslint.config.ts index d22c06e..d3f49e2 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -8,7 +8,6 @@ export default tseslint.config( tsc and vitest are what judge them, not eslint. */ ignores: [ "dist/**", - ".pnpmfile.cjs", "node_modules/**", "coverage/**", "**/.wrangler/**", diff --git a/package.json b/package.json index 3a8475a..82f737d 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "@diffbot/ai-sdk", - "version": "0.1.0-dev.0", + "version": "0.1.0", "description": "Diffbot tools and LLM provider for the Vercel AI SDK — web search, extract, DQL, ontology, and RAG", "type": "module", - "private": true, + "private": false, "packageManager": "pnpm@10.33.0", "main": "./dist/index.cjs", "module": "./dist/index.js", @@ -61,7 +61,7 @@ "test:workers": "tests/fixtures/worker/dry-run.sh" }, "dependencies": { - "@diffbot/typescript": "^0.5.0" + "@diffbot/typescript": "^0.6.0" }, "peerDependencies": { "@ai-sdk/openai-compatible": "^2.0.0 || ^3.0.0", diff --git a/src/retrieve/knowledge-graph.ts b/src/retrieve/knowledge-graph.ts index 91ec5c3..d8c8b60 100644 --- a/src/retrieve/knowledge-graph.ts +++ b/src/retrieve/knowledge-graph.ts @@ -21,6 +21,12 @@ export interface RetrieveKnowledgeGraphOptions extends DiffbotAuth, DiffbotDocum exportspec?: string; /** Only `"json"` can be shaped into documents; anything else throws before the request. */ format?: string; + /** + * Named DQL option. Do not also put it in `extra` — `@diffbot/typescript` 0.6.0 + * rejects named keys passed through `extra`. + */ + nonCanonicalFacts?: boolean; + /** Body fields that are not named DQL options (`report`, `view`, …). */ extra?: Record; } @@ -34,6 +40,7 @@ export async function retrieveKnowledgeGraph( filter, exportspec, format = "json", + nonCanonicalFacts, extra, fields, contentFields = DEFAULT_KG_CONTENT_FIELDS, @@ -59,6 +66,7 @@ export async function retrieveKnowledgeGraph( format, ...(filter !== undefined ? { filter } : {}), ...(exportspec !== undefined ? { exportspec } : {}), + ...(nonCanonicalFacts !== undefined ? { nonCanonicalFacts } : {}), ...(extra !== undefined ? { extra } : {}), }); diff --git a/tests/retrieve/knowledge-graph.test.ts b/tests/retrieve/knowledge-graph.test.ts index e8dbf0f..f4ba5ef 100644 --- a/tests/retrieve/knowledge-graph.test.ts +++ b/tests/retrieve/knowledge-graph.test.ts @@ -155,7 +155,8 @@ describe("retrieveKnowledgeGraph request wiring", () => { from: 20, filter: "nbEmployees>50", exportspec: "name", - extra: { nonCanonicalFacts: "true" }, + nonCanonicalFacts: true, + extra: { report: "1" }, }); expect(seen[0]!.json.size).toBe(10); @@ -164,7 +165,8 @@ describe("retrieveKnowledgeGraph request wiring", () => { from: 20, filter: "nbEmployees>50", exportspec: "name", - nonCanonicalFacts: "true", + nonCanonicalFacts: true, + report: "1", }); expect(seen[1]!.params.get("token")).toBe("test-token"); }); From 332487805bc4620d5603b7222b54364ff32401d9 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 18:38:26 -1000 Subject: [PATCH 10/11] Point @diffbot/typescript at 0.6.0-rc.0 for CI verification. --- package.json | 2 +- pnpm-lock.yaml | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 82f737d..466ff0f 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "test:workers": "tests/fixtures/worker/dry-run.sh" }, "dependencies": { - "@diffbot/typescript": "^0.6.0" + "@diffbot/typescript": "^0.6.0-rc.0" }, "peerDependencies": { "@ai-sdk/openai-compatible": "^2.0.0 || ^3.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4ffe9a5..41b1214 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,15 +4,13 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false -pnpmfileChecksum: sha256-ufjlYadkGX1lk1TqT83j9UssZyTlwlfDBdRUTr2Vrwo= - importers: .: dependencies: '@diffbot/typescript': - specifier: link:../diffbot-typescript - version: link:../diffbot-typescript + specifier: ^0.6.0-rc.0 + version: 0.6.0-rc.0 devDependencies: '@ai-sdk/openai-compatible': specifier: ^3.0.0 @@ -75,6 +73,10 @@ packages: resolution: {integrity: sha512-4uPgZ3qiX/JaQFr0dQoibUzHImZit9u05ro2BxGUnJwU4fV28BOAbKUa2OjQvqtM1tMac3bTKkaXdyzGovQXqg==} engines: {node: '>=22'} + '@diffbot/typescript@0.6.0-rc.0': + resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} + engines: {node: '>=18'} + '@esbuild/aix-ppc64@0.27.7': resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} @@ -1428,6 +1430,8 @@ snapshots: dependencies: json-schema: 0.4.0 + '@diffbot/typescript@0.6.0-rc.0': {} + '@esbuild/aix-ppc64@0.27.7': optional: true From 4354a2c4403af9b61648de36ef22322e02f4b6f7 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 11:57:17 -1000 Subject: [PATCH 11/11] Pin @diffbot/typescript ^0.6.0 (stable) and relock from npm. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 466ff0f..82f737d 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "test:workers": "tests/fixtures/worker/dry-run.sh" }, "dependencies": { - "@diffbot/typescript": "^0.6.0-rc.0" + "@diffbot/typescript": "^0.6.0" }, "peerDependencies": { "@ai-sdk/openai-compatible": "^2.0.0 || ^3.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 41b1214..8f137e8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@diffbot/typescript': - specifier: ^0.6.0-rc.0 - version: 0.6.0-rc.0 + specifier: ^0.6.0 + version: 0.6.0 devDependencies: '@ai-sdk/openai-compatible': specifier: ^3.0.0 @@ -73,8 +73,8 @@ packages: resolution: {integrity: sha512-4uPgZ3qiX/JaQFr0dQoibUzHImZit9u05ro2BxGUnJwU4fV28BOAbKUa2OjQvqtM1tMac3bTKkaXdyzGovQXqg==} engines: {node: '>=22'} - '@diffbot/typescript@0.6.0-rc.0': - resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} + '@diffbot/typescript@0.6.0': + resolution: {integrity: sha512-Sdrnk3+4QHo9BzP/hheAweI4L067EKEHXOrK26Lc6F0gLY/RMsifFiJVeolEAnKdI8HfDrUb7K/qBMQkU6DfiA==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.27.7': @@ -1430,7 +1430,7 @@ snapshots: dependencies: json-schema: 0.4.0 - '@diffbot/typescript@0.6.0-rc.0': {} + '@diffbot/typescript@0.6.0': {} '@esbuild/aix-ppc64@0.27.7': optional: true