From 92b50d292d8b0bd6ff5adb67772adffdd588e6e4 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:18:42 -1000 Subject: [PATCH 01/11] chore(deps): require @diffbot/typescript ^0.5.0, link local SDK for dev Co-Authored-By: Claude Sonnet 5.5 --- .pnpmfile.cjs | 21 +++++++++++++++++++++ package.json | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 .pnpmfile.cjs diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs new file mode 100644 index 0000000..e83e133 --- /dev/null +++ b/.pnpmfile.cjs @@ -0,0 +1,21 @@ +/* + DEV-ONLY: resolve @diffbot/typescript to a sibling checkout when one exists. + + TODO(sdk-0.5.0): delete this file (and refresh pnpm-lock.yaml) once + @diffbot/typescript 0.5.0 is published to npm. Until then `^0.5.0` in + package.json cannot be resolved from the registry, so CI on this branch + cannot install; local installs link ../diffbot-typescript instead. +*/ +const fs = require("node:fs"); +const path = require("node:path"); + +const SIBLING = path.resolve(__dirname, "..", "diffbot-typescript"); + +function readPackage(pkg) { + if (pkg.name === "@diffbot/langchain" && fs.existsSync(path.join(SIBLING, "package.json"))) { + pkg.devDependencies = { ...pkg.devDependencies, "@diffbot/typescript": "link:../diffbot-typescript" }; + } + return pkg; +} + +module.exports = { hooks: { readPackage } }; diff --git a/package.json b/package.json index ecd16ff..6ae6b3c 100644 --- a/package.json +++ b/package.json @@ -74,11 +74,11 @@ "zod": "^3.25.76 || ^4" }, "peerDependencies": { - "@diffbot/typescript": "^0.2.0", + "@diffbot/typescript": "^0.5.0", "@langchain/core": "^1.0.0" }, "devDependencies": { - "@diffbot/typescript": "^0.2.0", + "@diffbot/typescript": "^0.5.0", "@eslint/js": "^9.20.0", "@langchain/core": "^1.2.5", "@types/node": "^22.15.21", From e358d4ff8dd3ea65bf04be24d5e0d7effab539de Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:19:08 -1000 Subject: [PATCH 02/11] test: expect DQL POST with JSON body in fetch mocks Co-Authored-By: Claude Sonnet 5.5 --- tests/helpers/mock-fetch.ts | 15 +++++++++++++++ tests/retrievers/knowledge-graph.test.ts | 22 +++++++++++----------- tests/tools/dql-probe.test.ts | 8 ++++---- tests/tools/knowledge-graph.test.ts | 14 +++++++------- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/tests/helpers/mock-fetch.ts b/tests/helpers/mock-fetch.ts index 3d41441..999925c 100644 --- a/tests/helpers/mock-fetch.ts +++ b/tests/helpers/mock-fetch.ts @@ -12,6 +12,8 @@ export interface MockRequest { headers: Headers; params: URLSearchParams; body?: string; + /* Parsed JSON request body, when the body is a JSON string (DQL POSTs). */ + json?: Record; } export type MockHandler = (request: MockRequest) => Response | Promise; @@ -30,12 +32,25 @@ export function createMockFetch(handler: MockHandler): FetchFn { const headers = new Headers(init?.headers); const body = toTextBody(init?.body ?? null); + let json: Record | undefined; + if (body !== undefined) { + try { + const parsedBody: unknown = JSON.parse(body); + if (parsedBody !== null && typeof parsedBody === "object") { + json = parsedBody as Record; + } + } catch { + /* Not JSON; leave `json` undefined. */ + } + } + return handler({ url: `${parsed.origin}${parsed.pathname}`, method, headers, params: parsed.searchParams, body, + json, }); }; } diff --git a/tests/retrievers/knowledge-graph.test.ts b/tests/retrievers/knowledge-graph.test.ts index 4654b04..fb85474 100644 --- a/tests/retrievers/knowledge-graph.test.ts +++ b/tests/retrievers/knowledge-graph.test.ts @@ -67,8 +67,8 @@ describe("DiffbotKnowledgeGraphRetriever document mapping", () => { it("maps nested {score, entity} hits into documents", async () => { const db = makeClient((req) => { expect(req.url).toBe(DQL_URL); - expect(req.method).toBe("GET"); - expect(req.params.get("query")).toBe("type:Organization"); + expect(req.method).toBe("POST"); + expect(req.json?.query).toBe("type:Organization"); return jsonResponse(SAMPLE_BODY); }); const docs = await new DiffbotKnowledgeGraphRetriever({ @@ -227,7 +227,7 @@ describe("DiffbotKnowledgeGraphRetriever document mapping", () => { describe("DiffbotKnowledgeGraphRetriever request wiring", () => { it("sends k as the size wire param", async () => { const db = makeClient((req) => { - expect(req.params.get("size")).toBe("3"); + expect(req.json?.size).toBe(3); return jsonResponse(SAMPLE_BODY); }); await new DiffbotKnowledgeGraphRetriever({ client: db, k: 3 }).invoke("type:Organization"); @@ -235,7 +235,7 @@ describe("DiffbotKnowledgeGraphRetriever request wiring", () => { it("defaults size to 10 when k is not given", async () => { const db = makeClient((req) => { - expect(req.params.get("size")).toBe("10"); + expect(req.json?.size).toBe(10); return jsonResponse(SAMPLE_BODY); }); await new DiffbotKnowledgeGraphRetriever({ client: db }).invoke("type:Organization"); @@ -260,13 +260,13 @@ describe("DiffbotKnowledgeGraphRetriever request wiring", () => { it("sends from, filter, exportspec and extra as wire params", async () => { const db = makeClient((req) => { - expect(req.params.get("size")).toBe("5"); - expect(req.params.get("from")).toBe("10"); - expect(req.params.get("filter")).toBe("type:Organization"); - expect(req.params.get("exportspec")).toBe("csv-spec"); - expect(req.params.get("revision")).toBe("2024-01"); + expect(req.json?.size).toBe(5); + expect(req.json?.from).toBe(10); + expect(req.json?.filter).toBe("type:Organization"); + expect(req.json?.exportspec).toBe("csv-spec"); + expect(req.json?.revision).toBe("2024-01"); /* `format` defaults to json, which the SDK omits from the wire. */ - expect(req.params.get("format")).toBeNull(); + expect(req.json?.format).toBeUndefined(); return jsonResponse(SAMPLE_BODY); }); await new DiffbotKnowledgeGraphRetriever({ @@ -302,7 +302,7 @@ describe("DiffbotKnowledgeGraphRetriever request wiring", () => { it("throws when the endpoint returns a non-JSON-object body", async () => { const db = makeClient((req) => { - expect(req.params.get("format")).toBe("csv"); + expect(req.json?.format).toBe("csv"); /* A CSV export is not a JSON object, so the retriever refuses it. */ return jsonResponse("id,name\nE1,Acme AI"); }); diff --git a/tests/tools/dql-probe.test.ts b/tests/tools/dql-probe.test.ts index 6403748..97aedb7 100644 --- a/tests/tools/dql-probe.test.ts +++ b/tests/tools/dql-probe.test.ts @@ -28,10 +28,10 @@ describe("DiffbotDQLProbeTool", () => { it("probes each variant at size=0 and returns hit counts in input order", async () => { const db = makeClient((req) => { expect(req.url).toBe(DQL_URL); - expect(req.method).toBe("GET"); + expect(req.method).toBe("POST"); /* size=0 is the whole point: match counts, no entity payload. */ - expect(req.params.get("size")).toBe("0"); - const query = req.params.get("query") ?? ""; + expect(req.json?.size).toBe(0); + const query = String(req.json?.query ?? ""); const hits = query.includes("Diffbot") ? 5 : 100; return jsonResponse({ hits, results: 0 }); }); @@ -49,7 +49,7 @@ describe("DiffbotDQLProbeTool", () => { it("preserves input order when responses come back out of order", async () => { const queries = ["q0", "q1", "q2", "q3"]; const db = makeClient(async (req) => { - const query = req.params.get("query") ?? ""; + const query = String(req.json?.query ?? ""); /* Later queries answer first, so positional zipping is really tested. */ const delay = (queries.length - queries.indexOf(query)) * 5; await new Promise((resolve) => setTimeout(resolve, delay)); diff --git a/tests/tools/knowledge-graph.test.ts b/tests/tools/knowledge-graph.test.ts index 854bb60..f8a5da3 100644 --- a/tests/tools/knowledge-graph.test.ts +++ b/tests/tools/knowledge-graph.test.ts @@ -30,10 +30,10 @@ describe("DiffbotKnowledgeGraphTool", () => { it("sends query, size, from and filter on the wire", async () => { const tool = toolWith((req) => { expect(req.url).toBe("https://kg.diffbot.com/kg/v3/dql"); - expect(req.params.get("query")).toBe("type:Organization"); - expect(req.params.get("size")).toBe("5"); - expect(req.params.get("from")).toBe("20"); - expect(req.params.get("filter")).toBe("nbEmployees>100"); + expect(req.json?.query).toBe("type:Organization"); + expect(req.json?.size).toBe(5); + expect(req.json?.from).toBe(20); + expect(req.json?.filter).toBe("nbEmployees>100"); return jsonResponse({ data: [] }); }); @@ -47,9 +47,9 @@ describe("DiffbotKnowledgeGraphTool", () => { it("defaults to size 10 and omits from and filter", async () => { const tool = toolWith((req) => { - expect(req.params.get("size")).toBe("10"); - expect(req.params.get("from")).toBeNull(); - expect(req.params.get("filter")).toBeNull(); + expect(req.json?.size).toBe(10); + expect(req.json?.from).toBeUndefined(); + expect(req.json?.filter).toBeUndefined(); return jsonResponse({ data: [] }); }); From 5621799e1b0596ade87291082ac113c3c41a4f1a Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:20:14 -1000 Subject: [PATCH 03/11] feat(extract): pass SDK ExtractOptions through extract tool and loader Also adapts to the SDK's tighter ExportFormat and ChatRole types, and lets .pnpmfile.cjs take DIFFBOT_TS_SDK_PATH for a scratch SDK build. Co-Authored-By: Claude Sonnet 5.5 --- .pnpmfile.cjs | 10 ++++++++-- src/chat_models.ts | 11 +++++++---- src/loaders/extract.ts | 13 +++++++++++-- src/retrievers/knowledge-graph.ts | 6 +++--- src/tools/extract.ts | 20 +++++++++++++++++--- tests/loaders/extract.test.ts | 17 +++++++++++++++++ tests/tools/extract.test.ts | 22 ++++++++++++++++++++++ 7 files changed, 85 insertions(+), 14 deletions(-) diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs index e83e133..e21fc7f 100644 --- a/.pnpmfile.cjs +++ b/.pnpmfile.cjs @@ -9,11 +9,17 @@ const fs = require("node:fs"); const path = require("node:path"); -const SIBLING = path.resolve(__dirname, "..", "diffbot-typescript"); +/* + DIFFBOT_TS_SDK_PATH overrides the sibling checkout, e.g. to point at a scratch + build when the sibling's dist/ is stale. +*/ +const SIBLING = process.env.DIFFBOT_TS_SDK_PATH + ? path.resolve(process.env.DIFFBOT_TS_SDK_PATH) + : path.resolve(__dirname, "..", "diffbot-typescript"); function readPackage(pkg) { if (pkg.name === "@diffbot/langchain" && fs.existsSync(path.join(SIBLING, "package.json"))) { - pkg.devDependencies = { ...pkg.devDependencies, "@diffbot/typescript": "link:../diffbot-typescript" }; + pkg.devDependencies = { ...pkg.devDependencies, "@diffbot/typescript": `link:${path.relative(__dirname, SIBLING)}` }; } return pkg; } diff --git a/src/chat_models.ts b/src/chat_models.ts index bb908c9..1c87f69 100644 --- a/src/chat_models.ts +++ b/src/chat_models.ts @@ -12,7 +12,7 @@ other knobs. */ -import { ask, type ChatMessage, type DiffbotClient } from "@diffbot/typescript"; +import { ask, type ChatMessage, type ChatRole, type DiffbotClient } from "@diffbot/typescript"; import type { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager"; import { BaseChatModel, @@ -24,7 +24,7 @@ import { assertClient, type DiffbotComponentFields } from "./base.js"; function toDiffbotMessages(messages: BaseMessage[]): ChatMessage[] { return messages.map((message) => { - let role: string; + let role: ChatRole; switch (message.getType()) { case "human": role = "user"; @@ -36,8 +36,11 @@ function toDiffbotMessages(messages: BaseMessage[]): ChatMessage[] { role = "system"; break; default: - /* Fall back on the message's own type for anything exotic. */ - role = message.getType(); + /* + The endpoint only knows system/user/assistant, so anything exotic + (tool, function, ...) is sent as a user turn. + */ + role = "user"; } const content = typeof message.content === "string" ? message.content : JSON.stringify(message.content); diff --git a/src/loaders/extract.ts b/src/loaders/extract.ts index 10876ec..194de05 100644 --- a/src/loaders/extract.ts +++ b/src/loaders/extract.ts @@ -6,7 +6,12 @@ and pass it in. URLs are extracted serially, in order. */ -import { extract, type DiffbotClient, type JsonObject } from "@diffbot/typescript"; +import { + extract, + type DiffbotClient, + type ExtractOptions, + type JsonObject, +} from "@diffbot/typescript"; import { BaseDocumentLoader } from "@langchain/core/document_loaders/base"; import { Document } from "@langchain/core/documents"; import { assertClient, shapeExtractResponse, type DiffbotComponentFields } from "../base.js"; @@ -35,6 +40,8 @@ export interface DiffbotExtractLoaderFields extends DiffbotComponentFields { api?: string; /* Output format. `markdown` uses Diffbot's LLM-optimized mode. */ fmt?: string; + /* SDK `ExtractOptions` (`timeout`, `mode`, `fields`, `paging`, ...) applied to every URL. */ + extractOptions?: ExtractOptions; /* Optional `(url, rawResponse) -> Document` override. */ documentMapper?: ExtractDocumentMapper; } @@ -59,6 +66,7 @@ export class DiffbotExtractLoader extends BaseDocumentLoader { urls: string[]; api: string; fmt: string; + extractOptions?: ExtractOptions; documentMapper?: ExtractDocumentMapper; constructor(fields: DiffbotExtractLoaderFields) { @@ -67,6 +75,7 @@ export class DiffbotExtractLoader extends BaseDocumentLoader { this.urls = fields.urls; this.api = fields.api ?? "analyze"; this.fmt = fields.fmt ?? "markdown"; + this.extractOptions = fields.extractOptions; this.documentMapper = fields.documentMapper; } @@ -84,7 +93,7 @@ export class DiffbotExtractLoader extends BaseDocumentLoader { */ async *lazyLoad(): AsyncGenerator { for (const url of this.urls) { - const raw = await extract(this.client, url, this.api, this.fmt); + const raw = await extract(this.client, url, this.api, this.fmt, this.extractOptions); yield this.toDoc(url, raw); } } diff --git a/src/retrievers/knowledge-graph.ts b/src/retrievers/knowledge-graph.ts index dbdd241..99f3b11 100644 --- a/src/retrievers/knowledge-graph.ts +++ b/src/retrievers/knowledge-graph.ts @@ -1,4 +1,4 @@ -import { dql, type DiffbotClient, type JsonObject } from "@diffbot/typescript"; +import { dql, type DiffbotClient, type ExportFormat, type JsonObject } from "@diffbot/typescript"; import { type Document } from "@langchain/core/documents"; import { BaseRetriever, type BaseRetrieverInput } from "@langchain/core/retrievers"; import { @@ -25,7 +25,7 @@ export interface DiffbotKnowledgeGraphRetrieverFields /* Export spec for non-JSON formats. */ exportspec?: string; /* Response format. Defaults to `json`; other values return raw bytes. */ - format?: string; + format?: ExportFormat; /* Extra query params merged into the DQL request. */ extra?: Record; /* @@ -70,7 +70,7 @@ export class DiffbotKnowledgeGraphRetriever extends BaseRetriever { from: number; filter?: string; exportspec?: string; - format: string; + format: ExportFormat; extra?: Record; fields?: string[]; contentFields: string[]; diff --git a/src/tools/extract.ts b/src/tools/extract.ts index 8d127e1..0f13d31 100644 --- a/src/tools/extract.ts +++ b/src/tools/extract.ts @@ -1,4 +1,9 @@ -import { extract, ExtractionError, type DiffbotClient } from "@diffbot/typescript"; +import { + extract, + ExtractionError, + type DiffbotClient, + type ExtractOptions, +} from "@diffbot/typescript"; import { StructuredTool, type ToolParams } from "@langchain/core/tools"; import type { InferInteropZodOutput } from "@langchain/core/utils/types"; import { z } from "zod/v3"; @@ -29,7 +34,14 @@ export interface DiffbotExtractToolError { export type DiffbotExtractToolOutput = ShapedExtractResponse | DiffbotExtractToolError; -export interface DiffbotExtractToolFields extends ToolParams, DiffbotComponentFields {} +export interface DiffbotExtractToolFields extends ToolParams, DiffbotComponentFields { + /* + SDK `ExtractOptions` (`timeout`, `mode`, `fields`, `fallback`, `paging`, ...) + applied to every call. Fixed at construction rather than exposed in the call + schema: they are operator knobs, and the agent only picks the URL. + */ + extractOptions?: ExtractOptions; +} /* Tool that extracts structured content from a URL via Diffbot's analyze API. @@ -64,15 +76,17 @@ export class DiffbotExtractTool extends StructuredTool< schema = schema; client: DiffbotClient; + extractOptions?: ExtractOptions; constructor(fields: DiffbotExtractToolFields) { super(fields); this.client = assertClient(fields.client); + this.extractOptions = fields.extractOptions; } async _call(input: InferInteropZodOutput): Promise { try { - const raw = await extract(this.client, input.url, input.api, input.fmt); + const raw = await extract(this.client, input.url, input.api, input.fmt, this.extractOptions); return shapeExtractResponse(raw); } catch (e) { /* diff --git a/tests/loaders/extract.test.ts b/tests/loaders/extract.test.ts index 62fdd71..3518764 100644 --- a/tests/loaders/extract.test.ts +++ b/tests/loaders/extract.test.ts @@ -148,4 +148,21 @@ describe("DiffbotExtractLoader.lazyLoad", () => { const docs = await loader.load(); expect(docs).toHaveLength(2); }); + + it("passes extractOptions to every extract call", async () => { + const timeouts: Array = []; + const db = makeClient((req) => { + timeouts.push(req.params.get("timeout")); + expect(req.params.get("useProxy")).toBe("true"); + return jsonResponse({ objects: [{ text: "Body." }] }); + }); + const loader = new DiffbotExtractLoader({ + client: db, + urls: ["https://example.com/a", "https://example.com/b"], + extractOptions: { timeout: 7000, useProxy: "true" }, + }); + + await loader.load(); + expect(timeouts).toEqual(["7000", "7000"]); + }); }); diff --git a/tests/tools/extract.test.ts b/tests/tools/extract.test.ts index 2d28125..9ce4f45 100644 --- a/tests/tools/extract.test.ts +++ b/tests/tools/extract.test.ts @@ -105,4 +105,26 @@ describe("DiffbotExtractTool", () => { const tool = toolWith(() => jsonResponse({ message: "bad token" }, 401)); await expect(tool.invoke({ url: "https://example.com" })).rejects.toThrow(/401/); }); + + it("threads constructor extractOptions through to the wire", async () => { + const tool = new DiffbotExtractTool({ + client: new DiffbotClient({ + token: "test-token", + fetch: createMockFetch((req) => { + expect(req.params.get("timeout")).toBe("5000"); + expect(req.params.get("fields")).toBe("title,text"); + expect(req.params.get("paging")).toBe("false"); + /* Defaults for api / fmt are unchanged. */ + expect(req.url).toBe("https://api.diffbot.com/v3/analyze"); + expect(req.params.get("mode")).toBe("llm"); + return jsonResponse({ objects: [{ text: "Body." }] }); + }), + }), + extractOptions: { timeout: 5000, fields: ["title", "text"], paging: false }, + }); + + await expect(tool.invoke({ url: "https://example.com" })).resolves.toMatchObject({ + content: "Body.", + }); + }); }); From 15dd19c744d16acb7d83e9ef6c9badd658e7911d Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:21:21 -1000 Subject: [PATCH 04/11] feat(crawl): opt-in crawlDownload mode on DiffbotCrawlLoader Co-Authored-By: Claude Sonnet 5.5 --- README.md | 25 +++++++ src/index.ts | 3 + src/loaders/crawl.ts | 137 ++++++++++++++++++++++++++++++++++-- tests/loaders/crawl.test.ts | 130 ++++++++++++++++++++++++++++++++++ 4 files changed, 291 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3143e0a..2d2ea04 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,8 @@ for await (const doc of loader.lazyLoad()) { `DiffbotExtractTool` returns a structured `{ error, errorCode }` object when Diffbot reports an extraction failure (a 200 response carrying an `errorCode`), so an agent can react and try another URL instead of having to catch an exception. Auth and rate-limit failures still throw — those are infrastructure problems, not per-call signals. +Both accept `extractOptions`, the SDK's `ExtractOptions` (`timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `useProxy`, `extra`, ...), applied to every call. `api` / `fmt` keep their `analyze` / `markdown` defaults. On the tool, `extractOptions` is fixed at construction and is not part of the agent-facing call schema — the agent only chooses the URL. + `DiffbotExtractLoader` implements `lazyLoad()` as well as `load()`. Prefer `lazyLoad()` for long URL lists: it yields each `Document` as its request finishes, instead of buffering the whole batch. ## Crawl a site @@ -175,6 +177,29 @@ for await (const doc of loader.lazyLoad()) { The loader defaults `watch: true` into its `crawlOptions`, because without it the SDK emits a single `job_created` event and nothing else. Pass `crawlOptions: { watch: false, ... }` if you want to fire a job and walk away, and `eventMapper` to keep events the default mapper skips. +### Non-watch mode: `crawlDownload` + +Pass `download` to skip the watch-style stream. The loader creates the job (always with `watch: false`), polls `crawlGetJob` until it finishes, then fetches the extracted records once with the SDK's `crawlDownload`. Each record becomes a `Document` whose `pageContent` is the record's extracted `content`. This suits runtimes that can't sit in a watch loop. It is opt-in; without `download` the loader behaves exactly as above. + +```ts +import { DiffbotClient, resolveTokenFromEnv } from "@diffbot/typescript"; +import { DiffbotCrawlLoader } from "@diffbot/langchain"; + +const client = new DiffbotClient({ token: resolveTokenFromEnv() }); + +/* Create a job, wait for it, download its records. */ +const fresh = new DiffbotCrawlLoader({ + client, + site: "https://www.diffbot.com", + download: { num: 50, pollInterval: 5000, maxWait: 600_000 }, +}); + +/* Or download a job that already finished (no `site` needed). */ +const existing = new DiffbotCrawlLoader({ client, download: { jobName: "my-finished-job" } }); +``` + +`download` options: `jobName` (reuse an existing job instead of creating one), `num` (only the N most recently processed URLs), `pollInterval` (ms, default 2000), `maxWait` (ms, default unlimited) and `recordMapper` (`(record) => Document | null`; the default is exported as `defaultCrawlRecordMapper`). A job that finishes with a failure message throws a `JobError`. The record's echoed `token` is never copied into metadata. + ## ChatDiffbot ```ts diff --git a/src/index.ts b/src/index.ts index ca476e2..b45a3e4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,10 @@ export { export { DiffbotCrawlLoader, defaultCrawlMapper, + defaultCrawlRecordMapper, type CrawlEventMapper, + type CrawlRecordMapper, + type DiffbotCrawlDownloadOptions, type DiffbotCrawlLoaderFields, } from "./loaders/crawl.js"; diff --git a/src/loaders/crawl.ts b/src/loaders/crawl.ts index 23ddd32..db4f25c 100644 --- a/src/loaders/crawl.ts +++ b/src/loaders/crawl.ts @@ -12,7 +12,11 @@ import { crawl, + crawlDownload, + crawlGetJob, CrawlEventType, + JobError, + type CrawlDataRecord, type CrawlEvent, type CrawlOptions, type DiffbotClient, @@ -49,9 +53,64 @@ export function defaultCrawlMapper(event: CrawlEvent): Document | null { }); } +/* `(record) -> Document | null`. Return null to skip the record. */ +export type CrawlRecordMapper = (record: CrawlDataRecord) => Document | null; + +/* + Default mapping for downloaded crawl records: `content` becomes `pageContent` + and the rest of the identifying fields become metadata. The record's `token` + is deliberately never copied — the crawl API echoes the caller's token into + every record, and metadata tends to end up in logs and vector stores. +*/ +export function defaultCrawlRecordMapper(record: CrawlDataRecord): Document | null { + const url = record.pageUrl ?? ""; + if (!url && !record.content) { + return null; + } + return new Document({ + pageContent: record.content ?? "", + metadata: { + url, + title: record.title, + type: record.type, + parentUrl: record.parentUrl, + id: record.id, + crawl_timestamp: record.timestamp, + }, + }); +} + +/* + Opt-in non-watch path. Instead of streaming URL events from `crawl`, create + the job (or reuse a finished one), poll `crawlGetJob` until it reaches a + terminal status, then fetch the extracted records once with `crawlDownload`. + Suited to runtimes that can't sit in a watch loop. +*/ +export interface DiffbotCrawlDownloadOptions { + /* + Name of an existing crawl job to download. When set, no job is created and + `site` / `crawlOptions` are ignored (the job is still polled, so passing an + unfinished job waits for it). + */ + jobName?: string; + /* Only the N most recently processed URLs (`crawlDownload`'s `num`). */ + num?: number; + /* Milliseconds between status polls. Defaults to 2000. */ + pollInterval?: number; + /* Give up polling after this many milliseconds. Defaults to no limit. */ + maxWait?: number; + /* Optional `(record) -> Document | null` override. Return null to skip. */ + recordMapper?: CrawlRecordMapper; +} + export interface DiffbotCrawlLoaderFields extends DiffbotComponentFields { - /* Seed URL for the crawl. */ - site: string; + /* Seed URL for the crawl. Optional only when `download.jobName` is set. */ + site?: string; + /* + Switches the loader to the poll-then-`crawlDownload` path. Omit (the + default) for the existing watch-style stream. + */ + download?: DiffbotCrawlDownloadOptions; /* Extra options passed to `crawl(client, site, options)`. @@ -80,14 +139,21 @@ export class DiffbotCrawlLoader extends BaseDocumentLoader { lc_namespace = ["langchain", "document_loaders", "diffbot"]; client: DiffbotClient; - site: string; + site?: string; + download?: DiffbotCrawlDownloadOptions; crawlOptions: CrawlOptions; eventMapper?: CrawlEventMapper; constructor(fields: DiffbotCrawlLoaderFields) { super(); this.client = assertClient(fields.client); + if (fields.site === undefined && fields.download?.jobName === undefined) { + throw new Error( + "DiffbotCrawlLoader needs `site`, or `download.jobName` for an existing job.", + ); + } this.site = fields.site; + this.download = fields.download; this.crawlOptions = { ...(fields.crawlOptions ?? {}) }; if (this.crawlOptions.watch === undefined) { this.crawlOptions.watch = true; @@ -105,7 +171,11 @@ export class DiffbotCrawlLoader extends BaseDocumentLoader { an `AsyncGenerator`, so this is a natural passthrough. */ async *lazyLoad(): AsyncGenerator { - for await (const event of crawl(this.client, this.site, this.crawlOptions)) { + if (this.download) { + yield* this.lazyLoadDownload(this.download); + return; + } + for await (const event of crawl(this.client, this.site!, this.crawlOptions)) { const doc = this.mapEvent(event); if (doc !== null) { yield doc; @@ -113,6 +183,65 @@ export class DiffbotCrawlLoader extends BaseDocumentLoader { } } + private async *lazyLoadDownload(opts: DiffbotCrawlDownloadOptions): AsyncGenerator { + let jobName = opts.jobName; + if (jobName === undefined) { + /* `watch: false` always: polling is done here, not inside the SDK stream. */ + for await (const event of crawl(this.client, this.site!, { + ...this.crawlOptions, + watch: false, + })) { + if (event.eventType === CrawlEventType.JobCreated) { + const name = event.details.job_name; + if (typeof name === "string") jobName = name; + } + } + if (jobName === undefined) { + throw new Error("Crawl did not report a job name."); + } + } + await this.waitForJob(jobName, opts); + + const records = await crawlDownload(this.client, jobName, { + ...(opts.num !== undefined ? { num: opts.num } : {}), + }); + const mapper = opts.recordMapper ?? defaultCrawlRecordMapper; + for (const record of records) { + const doc = mapper(record); + if (doc !== null) { + yield doc; + } + } + } + + /* + Poll until the job leaves the pending/in-progress states (status 0 / 7, + matching the SDK's own watch loop). A terminal message mentioning + fail/error is a failed job, not an empty result. + */ + private async waitForJob(jobName: string, opts: DiffbotCrawlDownloadOptions): Promise { + const pollInterval = opts.pollInterval ?? 2000; + const deadline = opts.maxWait === undefined ? undefined : Date.now() + opts.maxWait; + for (;;) { + const job = await crawlGetJob(this.client, jobName); + if (!job) { + throw new JobError(jobName, "Crawler job not found"); + } + const status = job.jobStatus?.status ?? 0; + if (status !== 0 && status !== 7) { + const message = job.jobStatus?.message ?? ""; + if (/fail|error/i.test(message)) { + throw new JobError(jobName, `Crawler job failed: ${message}`); + } + return; + } + if (deadline !== undefined && Date.now() + pollInterval > deadline) { + throw new JobError(jobName, `Crawler job did not finish within ${opts.maxWait}ms`); + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + } + } + async load(): Promise { const docs: Document[] = []; for await (const doc of this.lazyLoad()) { diff --git a/tests/loaders/crawl.test.ts b/tests/loaders/crawl.test.ts index 3628a00..65ca469 100644 --- a/tests/loaders/crawl.test.ts +++ b/tests/loaders/crawl.test.ts @@ -157,3 +157,133 @@ describe("DiffbotCrawlLoader end-to-end with watch: true (the default)", () => { expect(docs[1]!.pageContent).toBe("https://example.com/page2"); }); }); + +describe("DiffbotCrawlLoader download mode (non-watch crawlDownload)", () => { + const RECORDS = [ + { + pageUrl: "https://example.com/a", + title: "A", + type: "article", + content: "Body A", + token: "test-token", + timestamp: "2026-01-01T00:00:00Z", + }, + { pageUrl: "https://example.com/b", title: "B", type: "article", content: "Body B" }, + ]; + + it("creates the job, polls until finished, then downloads records as Documents", async () => { + const kinds: string[] = []; + let statusCalls = 0; + const db = makeClient((req) => { + if (req.url.endsWith("/data")) { + kinds.push("data"); + expect(req.params.get("name")).toBe("dl-job"); + expect(req.params.get("num")).toBe("5"); + expect(req.params.get("type")).toBeNull(); + return jsonResponse(RECORDS); + } + if (req.params.get("seeds")) { + kinds.push("create"); + return jsonResponse({ jobs: [{ name: "dl-job" }] }); + } + kinds.push("status"); + statusCalls += 1; + /* First poll still running (status 7), second poll done. */ + return jsonResponse({ + jobs: [{ jobStatus: statusCalls === 1 ? { status: 7 } : { status: 9, message: "done" } }], + }); + }); + + const docs = await new DiffbotCrawlLoader({ + client: db, + site: "https://example.com", + crawlOptions: { jobName: "dl-job" }, + download: { num: 5, pollInterval: 1 }, + }).load(); + + expect(kinds).toEqual(["create", "status", "status", "data"]); + expect(docs.map((d) => d.pageContent)).toEqual(["Body A", "Body B"]); + expect(docs[0]!.metadata.url).toBe("https://example.com/a"); + /* The echoed token must never reach metadata. */ + expect(JSON.stringify(docs)).not.toContain("test-token"); + }); + + it("downloads an already-finished job without creating one", async () => { + const kinds: string[] = []; + const db = makeClient((req) => { + if (req.url.endsWith("/data")) { + kinds.push("data"); + return jsonResponse(RECORDS); + } + expect(req.params.get("seeds")).toBeNull(); + kinds.push("status"); + return jsonResponse({ jobs: [{ jobStatus: { status: 9, message: "done" } }] }); + }); + + const docs = await new DiffbotCrawlLoader({ + client: db, + download: { jobName: "old-job" }, + }).load(); + + expect(kinds).toEqual(["status", "data"]); + expect(docs).toHaveLength(2); + }); + + it("applies a custom recordMapper and skips nulls", async () => { + const db = makeClient((req) => + req.url.endsWith("/data") + ? jsonResponse(RECORDS) + : jsonResponse({ jobs: [{ jobStatus: { status: 9, message: "done" } }] }), + ); + const docs = await new DiffbotCrawlLoader({ + client: db, + download: { + jobName: "old-job", + recordMapper: (r) => + r.title === "A" ? new Document({ pageContent: String(r.title) }) : null, + }, + }).load(); + expect(docs.map((d) => d.pageContent)).toEqual(["A"]); + }); + + it("throws when the finished job reports a failure", async () => { + const db = makeClient(() => + jsonResponse({ jobs: [{ jobStatus: { status: 5, message: "Job failed: bad seed" } }] }), + ); + await expect( + new DiffbotCrawlLoader({ client: db, download: { jobName: "bad" } }).load(), + ).rejects.toThrow(/failed/); + }); + + it("throws when the job does not exist", async () => { + const db = makeClient(() => jsonResponse({ jobs: [] })); + await expect( + new DiffbotCrawlLoader({ client: db, download: { jobName: "nope" } }).load(), + ).rejects.toThrow(/not found/); + }); + + it("gives up after maxWait", async () => { + const db = makeClient(() => jsonResponse({ jobs: [{ jobStatus: { status: 7 } }] })); + await expect( + new DiffbotCrawlLoader({ + client: db, + download: { jobName: "slow", pollInterval: 5, maxWait: 1 }, + }).load(), + ).rejects.toThrow(/did not finish/); + }); + + it("requires site or download.jobName", () => { + expect(() => new DiffbotCrawlLoader({ client: makeClient(() => jsonResponse({})) })).toThrow( + /site/, + ); + }); + + it("leaves the default watch behavior alone when download is unset", () => { + const loader = new DiffbotCrawlLoader({ + client: makeClient(() => jsonResponse({})), + site: "https://example.com", + }); + expect(loader.download).toBeUndefined(); + expect(loader.crawlOptions.watch).toBe(true); + }); +}); From f6f9bb906cfd29ba1fb3c5096143d94d18a909ee Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:32:30 -1000 Subject: [PATCH 05/11] fix(crawl): read record text fallback and bound download polling by default defaultCrawlRecordMapper read only `content`; Analyze-classified article records carry `text` instead, so they became empty Documents. maxWait now defaults to 600000 ms to match the Python loader's poll_timeout instead of polling forever. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 2 +- src/loaders/crawl.ts | 34 ++++++++++++++++++++++++---------- tests/loaders/crawl.test.ts | 13 +++++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8656f1d..2032638 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ src/ ├── chat_models.ts # ChatDiffbot (wraps `ask`) ├── loaders/ │ ├── extract.ts # DiffbotExtractLoader -│ └── crawl.ts # DiffbotCrawlLoader, defaultCrawlMapper +│ └── crawl.ts # DiffbotCrawlLoader, defaultCrawlMapper, defaultCrawlRecordMapper ├── retrievers/ │ ├── knowledge-graph.ts # DiffbotKnowledgeGraphRetriever │ └── web-search.ts # DiffbotWebSearchRetriever diff --git a/README.md b/README.md index 2d2ea04..a1757f0 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ const fresh = new DiffbotCrawlLoader({ const existing = new DiffbotCrawlLoader({ client, download: { jobName: "my-finished-job" } }); ``` -`download` options: `jobName` (reuse an existing job instead of creating one), `num` (only the N most recently processed URLs), `pollInterval` (ms, default 2000), `maxWait` (ms, default unlimited) and `recordMapper` (`(record) => Document | null`; the default is exported as `defaultCrawlRecordMapper`). A job that finishes with a failure message throws a `JobError`. The record's echoed `token` is never copied into metadata. +`download` options: `jobName` (reuse an existing job instead of creating one), `num` (only the N most recently processed URLs), `pollInterval` (ms, default 2000), `maxWait` (ms, default 600000 like the Python loader's `poll_timeout`; `Infinity` waits indefinitely) and `recordMapper` (`(record) => Document | null`; the default is exported as `defaultCrawlRecordMapper` and reads `content`, falling back to `text`). A job that finishes with a failure message throws a `JobError`. The record's echoed `token` is never copied into metadata. ## ChatDiffbot diff --git a/src/loaders/crawl.ts b/src/loaders/crawl.ts index db4f25c..9a8aa56 100644 --- a/src/loaders/crawl.ts +++ b/src/loaders/crawl.ts @@ -57,18 +57,25 @@ export function defaultCrawlMapper(event: CrawlEvent): Document | null { export type CrawlRecordMapper = (record: CrawlDataRecord) => Document | null; /* - Default mapping for downloaded crawl records: `content` becomes `pageContent` - and the rest of the identifying fields become metadata. The record's `token` - is deliberately never copied — the crawl API echoes the caller's token into - every record, and metadata tends to end up in logs and vector stores. + Default mapping for downloaded crawl records: `content` (or, failing that, + `text`) becomes `pageContent` and the rest of the identifying fields become + metadata. Records follow the ontology of their `type`: the API's own example + (`type: "other"`) carries `content`, while Analyze-classified article records + carry `text`, so reading only one of them yields empty documents for the other. + + The record's `token` is deliberately never copied — the crawl API echoes the + caller's token into every record, and metadata tends to end up in logs and + vector stores. */ export function defaultCrawlRecordMapper(record: CrawlDataRecord): Document | null { const url = record.pageUrl ?? ""; - if (!url && !record.content) { + const content = + record.content || (typeof record.text === "string" ? record.text : undefined) || ""; + if (!url && !content) { return null; } return new Document({ - pageContent: record.content ?? "", + pageContent: content, metadata: { url, title: record.title, @@ -86,6 +93,8 @@ export function defaultCrawlRecordMapper(record: CrawlDataRecord): Document | nu terminal status, then fetch the extracted records once with `crawlDownload`. Suited to runtimes that can't sit in a watch loop. */ +const DEFAULT_DOWNLOAD_MAX_WAIT = 600_000; + export interface DiffbotCrawlDownloadOptions { /* Name of an existing crawl job to download. When set, no job is created and @@ -97,7 +106,11 @@ export interface DiffbotCrawlDownloadOptions { num?: number; /* Milliseconds between status polls. Defaults to 2000. */ pollInterval?: number; - /* Give up polling after this many milliseconds. Defaults to no limit. */ + /* + Give up polling after this many milliseconds. Defaults to 600000 (ten + minutes, matching the Python loader's `poll_timeout`); pass `Infinity` to + wait indefinitely. + */ maxWait?: number; /* Optional `(record) -> Document | null` override. Return null to skip. */ recordMapper?: CrawlRecordMapper; @@ -221,7 +234,8 @@ export class DiffbotCrawlLoader extends BaseDocumentLoader { */ private async waitForJob(jobName: string, opts: DiffbotCrawlDownloadOptions): Promise { const pollInterval = opts.pollInterval ?? 2000; - const deadline = opts.maxWait === undefined ? undefined : Date.now() + opts.maxWait; + const maxWait = opts.maxWait ?? DEFAULT_DOWNLOAD_MAX_WAIT; + const deadline = Date.now() + maxWait; for (;;) { const job = await crawlGetJob(this.client, jobName); if (!job) { @@ -235,8 +249,8 @@ export class DiffbotCrawlLoader extends BaseDocumentLoader { } return; } - if (deadline !== undefined && Date.now() + pollInterval > deadline) { - throw new JobError(jobName, `Crawler job did not finish within ${opts.maxWait}ms`); + if (Date.now() + pollInterval > deadline) { + throw new JobError(jobName, `Crawler job did not finish within ${maxWait}ms`); } await new Promise((resolve) => setTimeout(resolve, pollInterval)); } diff --git a/tests/loaders/crawl.test.ts b/tests/loaders/crawl.test.ts index 65ca469..6bee25e 100644 --- a/tests/loaders/crawl.test.ts +++ b/tests/loaders/crawl.test.ts @@ -286,4 +286,17 @@ describe("DiffbotCrawlLoader download mode (non-watch crawlDownload)", () => { expect(loader.download).toBeUndefined(); expect(loader.crawlOptions.watch).toBe(true); }); + + it("falls back to `text` when a record has no `content`", async () => { + const db = makeClient((req) => + req.url.endsWith("/data") + ? jsonResponse([{ pageUrl: "https://example.com/c", type: "article", text: "Body C" }]) + : jsonResponse({ jobs: [{ jobStatus: { status: 9, message: "done" } }] }), + ); + const docs = await new DiffbotCrawlLoader({ + client: db, + download: { jobName: "old-job" }, + }).load(); + expect(docs.map((d) => d.pageContent)).toEqual(["Body C"]); + }); }); From ad932fccae6e9061721a1055a72670eb56dc2b03 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:32:30 -1000 Subject: [PATCH 06/11] test(chat): cover exotic message roles mapping to user Co-Authored-By: Claude Opus 5.5 --- tests/chat_models.test.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/chat_models.test.ts b/tests/chat_models.test.ts index f97fd37..05b0ee9 100644 --- a/tests/chat_models.test.ts +++ b/tests/chat_models.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { DiffbotClient } from "@diffbot/typescript"; -import { AIMessageChunk, HumanMessage, SystemMessage } from "@langchain/core/messages"; +import { AIMessageChunk, HumanMessage, SystemMessage, ToolMessage } from "@langchain/core/messages"; import { ChatDiffbot } from "../src/chat_models.js"; import { createMockFetch, sseResponse, type MockHandler } from "./helpers/mock-fetch.js"; @@ -74,4 +74,24 @@ describe("ChatDiffbot message role mapping", () => { expect(payload.messages[0]!.content).toBe("you are a bot"); expect(payload.messages[1]!.content).toBe("hi"); }); + + it("sends roles the endpoint does not know (tool, ...) as user turns", async () => { + let body: string | undefined; + const db = makeClient((req) => { + body = req.body; + return sseResponse(SSE_BODY); + }); + const llm = new ChatDiffbot({ client: db }); + + await llm.invoke([ + new HumanMessage("hi"), + new ToolMessage({ content: "tool output", tool_call_id: "call-1" }), + ]); + + const payload = JSON.parse(body ?? "{}") as { + messages: Array<{ role: string; content: string }>; + }; + expect(payload.messages.map((m) => m.role)).toEqual(["user", "user"]); + expect(payload.messages[1]!.content).toBe("tool output"); + }); }); From 7dad859ae55c20b198be76eb374d4d9a7d726e18 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:32:30 -1000 Subject: [PATCH 07/11] style: prettier-format .pnpmfile.cjs Co-Authored-By: Claude Opus 5.5 --- .pnpmfile.cjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs index e21fc7f..a609091 100644 --- a/.pnpmfile.cjs +++ b/.pnpmfile.cjs @@ -19,7 +19,10 @@ const SIBLING = process.env.DIFFBOT_TS_SDK_PATH function readPackage(pkg) { if (pkg.name === "@diffbot/langchain" && fs.existsSync(path.join(SIBLING, "package.json"))) { - pkg.devDependencies = { ...pkg.devDependencies, "@diffbot/typescript": `link:${path.relative(__dirname, SIBLING)}` }; + pkg.devDependencies = { + ...pkg.devDependencies, + "@diffbot/typescript": `link:${path.relative(__dirname, SIBLING)}`, + }; } return pkg; } From f939c176a9bc65397030d2520ec11536de6e32a4 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:30:27 -1000 Subject: [PATCH 08/11] ci(release): publish to npm via trusted publishing (OIDC) Use npm >= 11.5.1 with npm publish --provenance (no NPM_TOKEN), and never cancel in-flight publishes. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c2b28d..723778d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,11 +8,11 @@ on: tags: ["v*"] workflow_dispatch: -# Re-pushing a tag (or re-dispatching) cancels an in-flight run for the same -# ref so a stuck build can't pile up behind a newer attempt. +# Never cancel an in-flight run: cancelling a publish midway could leave npm +# and the GitHub Release out of sync. Runs for the same ref queue instead. concurrency: group: release-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: build: @@ -48,7 +48,7 @@ jobs: - name: Build run: pnpm build - # Bundle everything `pnpm publish` needs (package.json, README, LICENSE, + # Bundle everything `npm publish` needs (package.json, README, LICENSE, # dist/) so the publish job below can run without checking out the repo. - name: Assemble publishable package run: | @@ -83,17 +83,24 @@ jobs: - name: List artifact contents run: ls -lR package - - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v4 with: node-version: 22 registry-url: "https://registry.npmjs.org" + # npm trusted publishing (OIDC) needs npm >= 11.5.1; Node 22 ships older. + - name: Update npm + run: | + npm install -g npm@latest + npm --version + + # Uses npm (not pnpm) so the OIDC token exchange works. No token is set: + # npm authenticates via the trusted publisher configured on npmjs.com. + # --ignore-scripts: the package was already built in the build job, and + # package.json's prepublishOnly calls pnpm/tsup, absent in this job. - name: Publish to npm working-directory: package - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: pnpm publish --access public --no-git-checks --provenance + run: npm publish --access public --provenance --ignore-scripts - name: Publish GitHub Release env: From ef9db3ab55dd1a6b10a7c12110efe6c461a3d49b Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 17:59:07 -1000 Subject: [PATCH 09/11] Point 0.2.0 at @diffbot/typescript ^0.6.0. Peer, devDependency, examples, and the worker fixture move off the older ranges, and the dev-only pnpmfile link is removed. Lock refresh waits until 0.6.0 is on npm. --- .pnpmfile.cjs | 30 -------------------------- CHANGELOG.md | 11 ++++++++++ examples/company-research/package.json | 2 +- examples/dql-explorer/package.json | 2 +- fixtures/worker/package.json | 2 +- package.json | 4 ++-- 6 files changed, 16 insertions(+), 35 deletions(-) delete mode 100644 .pnpmfile.cjs create mode 100644 CHANGELOG.md diff --git a/.pnpmfile.cjs b/.pnpmfile.cjs deleted file mode 100644 index a609091..0000000 --- a/.pnpmfile.cjs +++ /dev/null @@ -1,30 +0,0 @@ -/* - DEV-ONLY: resolve @diffbot/typescript to a sibling checkout when one exists. - - TODO(sdk-0.5.0): delete this file (and refresh pnpm-lock.yaml) once - @diffbot/typescript 0.5.0 is published to npm. Until then `^0.5.0` in - package.json cannot be resolved from the registry, so CI on this branch - cannot install; local installs link ../diffbot-typescript instead. -*/ -const fs = require("node:fs"); -const path = require("node:path"); - -/* - DIFFBOT_TS_SDK_PATH overrides the sibling checkout, e.g. to point at a scratch - build when the sibling's dist/ is stale. -*/ -const SIBLING = process.env.DIFFBOT_TS_SDK_PATH - ? path.resolve(process.env.DIFFBOT_TS_SDK_PATH) - : path.resolve(__dirname, "..", "diffbot-typescript"); - -function readPackage(pkg) { - if (pkg.name === "@diffbot/langchain" && fs.existsSync(path.join(SIBLING, "package.json"))) { - pkg.devDependencies = { - ...pkg.devDependencies, - "@diffbot/typescript": `link:${path.relative(__dirname, SIBLING)}`, - }; - } - return pkg; -} - -module.exports = { hooks: { readPackage } }; diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5ba950c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## 0.2.0 - 2026-11-17 + +Ship the unpublished 0.2.0 line against `@diffbot/typescript` ^0.6.0. + +Peer, devDependency, `examples/company-research`, `examples/dql-explorer`, and `fixtures/worker` move off ^0.5.0 and ^0.2.0. DQL tests already assert POST with a JSON body; the token stays on the query string. + +The dev-only `.pnpmfile.cjs` sibling link is removed. Refresh the root lock and the example/fixture locks after 0.6.0 is on npm (`pnpm install` / `pnpm update @diffbot/typescript`). Example `link:../..` entries for `@diffbot/langchain` itself stay local dev links. + +Publishing is npm trusted publishing from `.github/workflows/release.yml` (no token, no GitHub environment). Tag `v0.2.0`. diff --git a/examples/company-research/package.json b/examples/company-research/package.json index b32b274..1e6e2ba 100644 --- a/examples/company-research/package.json +++ b/examples/company-research/package.json @@ -9,7 +9,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "^0.2.0", + "@diffbot/typescript": "^0.6.0", "@langchain/anthropic": "^1.5.4", "@langchain/core": "^1.2.5", "dotenv": "^17.2.3", diff --git a/examples/dql-explorer/package.json b/examples/dql-explorer/package.json index 555a041..c0c3cfe 100644 --- a/examples/dql-explorer/package.json +++ b/examples/dql-explorer/package.json @@ -14,7 +14,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "^0.2.0", + "@diffbot/typescript": "^0.6.0", "@hono/node-server": "^1.19.0", "@langchain/anthropic": "^1.5.4", "@langchain/core": "^1.2.5", diff --git a/fixtures/worker/package.json b/fixtures/worker/package.json index d495b14..0a9af78 100644 --- a/fixtures/worker/package.json +++ b/fixtures/worker/package.json @@ -8,7 +8,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "^0.2.0", + "@diffbot/typescript": "^0.6.0", "@langchain/core": "^1.2.5" }, "devDependencies": { diff --git a/package.json b/package.json index 6ae6b3c..27d1edf 100644 --- a/package.json +++ b/package.json @@ -74,11 +74,11 @@ "zod": "^3.25.76 || ^4" }, "peerDependencies": { - "@diffbot/typescript": "^0.5.0", + "@diffbot/typescript": "^0.6.0", "@langchain/core": "^1.0.0" }, "devDependencies": { - "@diffbot/typescript": "^0.5.0", + "@diffbot/typescript": "^0.6.0", "@eslint/js": "^9.20.0", "@langchain/core": "^1.2.5", "@types/node": "^22.15.21", From 4958777b2e1eaa35d10c1e393fd872ea0e65e7af Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 18:38:26 -1000 Subject: [PATCH 10/11] Point @diffbot/typescript at 0.6.0-rc.0 for CI verification. --- examples/company-research/package.json | 2 +- examples/company-research/pnpm-lock.yaml | 10 +++++----- examples/dql-explorer/package.json | 2 +- examples/dql-explorer/pnpm-lock.yaml | 10 +++++----- fixtures/worker/package.json | 2 +- fixtures/worker/pnpm-lock.yaml | 10 +++++----- package.json | 4 ++-- pnpm-lock.yaml | 10 +++++----- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/company-research/package.json b/examples/company-research/package.json index 1e6e2ba..2f19949 100644 --- a/examples/company-research/package.json +++ b/examples/company-research/package.json @@ -9,7 +9,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "^0.6.0", + "@diffbot/typescript": "0.6.0-rc.0", "@langchain/anthropic": "^1.5.4", "@langchain/core": "^1.2.5", "dotenv": "^17.2.3", diff --git a/examples/company-research/pnpm-lock.yaml b/examples/company-research/pnpm-lock.yaml index b07e1f6..110cb6e 100644 --- a/examples/company-research/pnpm-lock.yaml +++ b/examples/company-research/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: link:../.. version: link:../.. '@diffbot/typescript': - specifier: ^0.1.0 - version: 0.1.0 + specifier: 0.6.0-rc.0 + version: 0.6.0-rc.0 '@langchain/anthropic': specifier: ^1.5.4 version: 1.5.4(@langchain/core@1.2.5) @@ -58,8 +58,8 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} - '@diffbot/typescript@0.1.0': - resolution: {integrity: sha512-rlP1Eh1t03cQwtY2q8AP8b5xVUEDnKCFkSh6lxCf4eGgXP9mYESh76c2FribcBtKvxZSBrAZt7+Z8cZrtJDgKg==} + '@diffbot/typescript@0.6.0-rc.0': + resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.28.2': @@ -400,7 +400,7 @@ snapshots: '@cfworker/json-schema@4.1.1': {} - '@diffbot/typescript@0.1.0': {} + '@diffbot/typescript@0.6.0-rc.0': {} '@esbuild/aix-ppc64@0.28.2': optional: true diff --git a/examples/dql-explorer/package.json b/examples/dql-explorer/package.json index c0c3cfe..18b44ed 100644 --- a/examples/dql-explorer/package.json +++ b/examples/dql-explorer/package.json @@ -14,7 +14,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "^0.6.0", + "@diffbot/typescript": "0.6.0-rc.0", "@hono/node-server": "^1.19.0", "@langchain/anthropic": "^1.5.4", "@langchain/core": "^1.2.5", diff --git a/examples/dql-explorer/pnpm-lock.yaml b/examples/dql-explorer/pnpm-lock.yaml index a73ad38..00a147b 100644 --- a/examples/dql-explorer/pnpm-lock.yaml +++ b/examples/dql-explorer/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: link:../.. version: link:../.. '@diffbot/typescript': - specifier: ^0.1.0 - version: 0.1.0 + specifier: 0.6.0-rc.0 + version: 0.6.0-rc.0 '@hono/node-server': specifier: ^1.19.0 version: 1.19.17(hono@4.13.1) @@ -67,8 +67,8 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} - '@diffbot/typescript@0.1.0': - resolution: {integrity: sha512-rlP1Eh1t03cQwtY2q8AP8b5xVUEDnKCFkSh6lxCf4eGgXP9mYESh76c2FribcBtKvxZSBrAZt7+Z8cZrtJDgKg==} + '@diffbot/typescript@0.6.0-rc.0': + resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.28.2': @@ -419,7 +419,7 @@ snapshots: '@cfworker/json-schema@4.1.1': {} - '@diffbot/typescript@0.1.0': {} + '@diffbot/typescript@0.6.0-rc.0': {} '@esbuild/aix-ppc64@0.28.2': optional: true diff --git a/fixtures/worker/package.json b/fixtures/worker/package.json index 0a9af78..9413508 100644 --- a/fixtures/worker/package.json +++ b/fixtures/worker/package.json @@ -8,7 +8,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "^0.6.0", + "@diffbot/typescript": "0.6.0-rc.0", "@langchain/core": "^1.2.5" }, "devDependencies": { diff --git a/fixtures/worker/pnpm-lock.yaml b/fixtures/worker/pnpm-lock.yaml index 4be4b57..93f54e7 100644 --- a/fixtures/worker/pnpm-lock.yaml +++ b/fixtures/worker/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: link:../.. version: link:../.. '@diffbot/typescript': - specifier: ^0.2.0 - version: 0.2.0 + specifier: 0.6.0-rc.0 + version: 0.6.0-rc.0 '@langchain/core': specifier: ^1.2.5 version: 1.2.8(ws@8.21.0) @@ -139,8 +139,8 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} - '@diffbot/typescript@0.2.0': - resolution: {integrity: sha512-CnIeBZ+CvGBPFS9GiZj7x7j+yCqmz856DBh95/PWcDEeurD4nSCF1W8DYv+jdNML81mz5aA74a3Z2BOIqE3pLg==} + '@diffbot/typescript@0.6.0-rc.0': + resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} engines: {node: '>=18'} '@emnapi/runtime@1.11.3': @@ -1647,7 +1647,7 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.9 - '@diffbot/typescript@0.2.0': {} + '@diffbot/typescript@0.6.0-rc.0': {} '@emnapi/runtime@1.11.3': dependencies: diff --git a/package.json b/package.json index 27d1edf..c21d585 100644 --- a/package.json +++ b/package.json @@ -74,11 +74,11 @@ "zod": "^3.25.76 || ^4" }, "peerDependencies": { - "@diffbot/typescript": "^0.6.0", + "@diffbot/typescript": "^0.6.0-rc.0", "@langchain/core": "^1.0.0" }, "devDependencies": { - "@diffbot/typescript": "^0.6.0", + "@diffbot/typescript": "^0.6.0-rc.0", "@eslint/js": "^9.20.0", "@langchain/core": "^1.2.5", "@types/node": "^22.15.21", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f0cf5c3..af36035 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,8 +13,8 @@ importers: version: 4.4.3 devDependencies: '@diffbot/typescript': - specifier: ^0.2.0 - version: 0.2.0 + specifier: ^0.6.0-rc.0 + version: 0.6.0-rc.0 '@eslint/js': specifier: ^9.20.0 version: 9.39.5 @@ -82,8 +82,8 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} - '@diffbot/typescript@0.2.0': - resolution: {integrity: sha512-CnIeBZ+CvGBPFS9GiZj7x7j+yCqmz856DBh95/PWcDEeurD4nSCF1W8DYv+jdNML81mz5aA74a3Z2BOIqE3pLg==} + '@diffbot/typescript@0.6.0-rc.0': + resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.27.7': @@ -1609,7 +1609,7 @@ snapshots: '@cfworker/json-schema@4.1.1': {} - '@diffbot/typescript@0.2.0': {} + '@diffbot/typescript@0.6.0-rc.0': {} '@esbuild/aix-ppc64@0.27.7': optional: true From e38e5c16c8a65814c14e3faac1b8f2d5d14b2820 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 11:57:59 -1000 Subject: [PATCH 11/11] Pin @diffbot/typescript ^0.6.0 (stable) and relock from npm. Co-Authored-By: Claude Opus 5.5 --- examples/company-research/package.json | 2 +- examples/company-research/pnpm-lock.yaml | 10 +++++----- examples/dql-explorer/package.json | 2 +- examples/dql-explorer/pnpm-lock.yaml | 10 +++++----- fixtures/worker/package.json | 2 +- fixtures/worker/pnpm-lock.yaml | 10 +++++----- package.json | 4 ++-- pnpm-lock.yaml | 10 +++++----- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/company-research/package.json b/examples/company-research/package.json index 2f19949..1e6e2ba 100644 --- a/examples/company-research/package.json +++ b/examples/company-research/package.json @@ -9,7 +9,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "0.6.0-rc.0", + "@diffbot/typescript": "^0.6.0", "@langchain/anthropic": "^1.5.4", "@langchain/core": "^1.2.5", "dotenv": "^17.2.3", diff --git a/examples/company-research/pnpm-lock.yaml b/examples/company-research/pnpm-lock.yaml index 110cb6e..36620dd 100644 --- a/examples/company-research/pnpm-lock.yaml +++ b/examples/company-research/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: link:../.. version: link:../.. '@diffbot/typescript': - specifier: 0.6.0-rc.0 - version: 0.6.0-rc.0 + specifier: ^0.6.0 + version: 0.6.0 '@langchain/anthropic': specifier: ^1.5.4 version: 1.5.4(@langchain/core@1.2.5) @@ -58,8 +58,8 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} - '@diffbot/typescript@0.6.0-rc.0': - resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} + '@diffbot/typescript@0.6.0': + resolution: {integrity: sha512-Sdrnk3+4QHo9BzP/hheAweI4L067EKEHXOrK26Lc6F0gLY/RMsifFiJVeolEAnKdI8HfDrUb7K/qBMQkU6DfiA==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.28.2': @@ -400,7 +400,7 @@ snapshots: '@cfworker/json-schema@4.1.1': {} - '@diffbot/typescript@0.6.0-rc.0': {} + '@diffbot/typescript@0.6.0': {} '@esbuild/aix-ppc64@0.28.2': optional: true diff --git a/examples/dql-explorer/package.json b/examples/dql-explorer/package.json index 18b44ed..c0c3cfe 100644 --- a/examples/dql-explorer/package.json +++ b/examples/dql-explorer/package.json @@ -14,7 +14,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "0.6.0-rc.0", + "@diffbot/typescript": "^0.6.0", "@hono/node-server": "^1.19.0", "@langchain/anthropic": "^1.5.4", "@langchain/core": "^1.2.5", diff --git a/examples/dql-explorer/pnpm-lock.yaml b/examples/dql-explorer/pnpm-lock.yaml index 00a147b..5e7a79f 100644 --- a/examples/dql-explorer/pnpm-lock.yaml +++ b/examples/dql-explorer/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: link:../.. version: link:../.. '@diffbot/typescript': - specifier: 0.6.0-rc.0 - version: 0.6.0-rc.0 + specifier: ^0.6.0 + version: 0.6.0 '@hono/node-server': specifier: ^1.19.0 version: 1.19.17(hono@4.13.1) @@ -67,8 +67,8 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} - '@diffbot/typescript@0.6.0-rc.0': - resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} + '@diffbot/typescript@0.6.0': + resolution: {integrity: sha512-Sdrnk3+4QHo9BzP/hheAweI4L067EKEHXOrK26Lc6F0gLY/RMsifFiJVeolEAnKdI8HfDrUb7K/qBMQkU6DfiA==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.28.2': @@ -419,7 +419,7 @@ snapshots: '@cfworker/json-schema@4.1.1': {} - '@diffbot/typescript@0.6.0-rc.0': {} + '@diffbot/typescript@0.6.0': {} '@esbuild/aix-ppc64@0.28.2': optional: true diff --git a/fixtures/worker/package.json b/fixtures/worker/package.json index 9413508..0a9af78 100644 --- a/fixtures/worker/package.json +++ b/fixtures/worker/package.json @@ -8,7 +8,7 @@ }, "dependencies": { "@diffbot/langchain": "link:../..", - "@diffbot/typescript": "0.6.0-rc.0", + "@diffbot/typescript": "^0.6.0", "@langchain/core": "^1.2.5" }, "devDependencies": { diff --git a/fixtures/worker/pnpm-lock.yaml b/fixtures/worker/pnpm-lock.yaml index 93f54e7..e84139b 100644 --- a/fixtures/worker/pnpm-lock.yaml +++ b/fixtures/worker/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: link:../.. version: link:../.. '@diffbot/typescript': - specifier: 0.6.0-rc.0 - version: 0.6.0-rc.0 + specifier: ^0.6.0 + version: 0.6.0 '@langchain/core': specifier: ^1.2.5 version: 1.2.8(ws@8.21.0) @@ -139,8 +139,8 @@ packages: resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} - '@diffbot/typescript@0.6.0-rc.0': - resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} + '@diffbot/typescript@0.6.0': + resolution: {integrity: sha512-Sdrnk3+4QHo9BzP/hheAweI4L067EKEHXOrK26Lc6F0gLY/RMsifFiJVeolEAnKdI8HfDrUb7K/qBMQkU6DfiA==} engines: {node: '>=18'} '@emnapi/runtime@1.11.3': @@ -1647,7 +1647,7 @@ snapshots: dependencies: '@jridgewell/trace-mapping': 0.3.9 - '@diffbot/typescript@0.6.0-rc.0': {} + '@diffbot/typescript@0.6.0': {} '@emnapi/runtime@1.11.3': dependencies: diff --git a/package.json b/package.json index c21d585..27d1edf 100644 --- a/package.json +++ b/package.json @@ -74,11 +74,11 @@ "zod": "^3.25.76 || ^4" }, "peerDependencies": { - "@diffbot/typescript": "^0.6.0-rc.0", + "@diffbot/typescript": "^0.6.0", "@langchain/core": "^1.0.0" }, "devDependencies": { - "@diffbot/typescript": "^0.6.0-rc.0", + "@diffbot/typescript": "^0.6.0", "@eslint/js": "^9.20.0", "@langchain/core": "^1.2.5", "@types/node": "^22.15.21", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af36035..6a5e9e0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,8 +13,8 @@ importers: version: 4.4.3 devDependencies: '@diffbot/typescript': - specifier: ^0.6.0-rc.0 - version: 0.6.0-rc.0 + specifier: ^0.6.0 + version: 0.6.0 '@eslint/js': specifier: ^9.20.0 version: 9.39.5 @@ -82,8 +82,8 @@ packages: '@cfworker/json-schema@4.1.1': resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} - '@diffbot/typescript@0.6.0-rc.0': - resolution: {integrity: sha512-04XHLNRhi/4JoQ2a2d4RChBGS1ZParIWeDqfjL+mcdkhUanlVJiLwvSBbQO3ttKd1gsPUA7WdcQ0+HaaH6z5Ng==} + '@diffbot/typescript@0.6.0': + resolution: {integrity: sha512-Sdrnk3+4QHo9BzP/hheAweI4L067EKEHXOrK26Lc6F0gLY/RMsifFiJVeolEAnKdI8HfDrUb7K/qBMQkU6DfiA==} engines: {node: '>=18'} '@esbuild/aix-ppc64@0.27.7': @@ -1609,7 +1609,7 @@ snapshots: '@cfworker/json-schema@4.1.1': {} - '@diffbot/typescript@0.6.0-rc.0': {} + '@diffbot/typescript@0.6.0': {} '@esbuild/aix-ppc64@0.27.7': optional: true