From 966d63edbdbaa740da805e95a49b5859e14f9992 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:20:30 -1000 Subject: [PATCH 01/12] chore(deps): require diffbot>=3.3.0; add dev-only sibling SDK install Co-Authored-By: Claude Sonnet 5.5 --- Makefile | 10 +++++++++- pyproject.toml | 11 +++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4c5df69..3a970f2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,12 @@ -.PHONY: format lint test test_integration typing build clean bump-patch bump-minor bump-major set-token-pypi set-token-testpypi release-test release verify-release-test verify-release +.PHONY: dev-sdk format lint test test_integration typing build clean bump-patch bump-minor bump-major set-token-pypi set-token-testpypi release-test release verify-release-test verify-release + +# Dev-only: install the unreleased Diffbot SDK from a sibling checkout over the +# synced env (no-op if ../diffbot-python is absent). TODO(sdk-release): remove +# once diffbot>=3.3.0 is on PyPI. Pair with UV_NO_SYNC=true so `make test` etc. +# don't re-resolve and replace it. +dev-sdk: + @if [ -d ../diffbot-python ]; then uv pip install -e ../diffbot-python; \ + else echo "../diffbot-python not found; skipping"; fi format: uv run --group lint ruff format . diff --git a/pyproject.toml b/pyproject.toml index aa45052..f83b836 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,9 @@ requires-python = ">=3.10,<4.0" dependencies = [ "langchain-core>=1.0,<2.0", "httpx>=0.27,<1.0", - "diffbot>=3.0.0", + # TODO(sdk-release): diffbot 3.3.0 is not on PyPI yet. Once published, relock + # (`uv lock`) and drop the dev-only sibling install (`make dev-sdk`). + "diffbot>=3.3.0", ] [project.optional-dependencies] @@ -55,6 +57,10 @@ test = [ lint = ["ruff>=0.6,<1.0"] typing = ["mypy>=1.10,<2.0"] +# Dev-only: the unreleased Diffbot SDK is installed from a sibling checkout by +# `make dev-sdk` (`uv pip install -e ../diffbot-python`, skipped when that +# directory is absent). It is deliberately NOT a uv source: a path source can't +# be conditional on the checkout existing and would break every other install. [tool.uv.sources] langchain-core = { path = "../langchain/libs/core", editable = true } langchain-tests = { path = "../langchain/libs/standard-tests", editable = true } @@ -69,7 +75,8 @@ include = ["/langchain_diffbot", "/README.md", "/LICENSE", "/pyproject.toml"] disallow_untyped_defs = true [[tool.mypy.overrides]] -# The diffbot SDK doesn't ship a `py.typed` marker yet. Follow-up: upstream the marker. +# The diffbot SDK doesn't ship a `py.typed` marker yet (checked against 3.3.0 on +# feat/sdk-s1). TODO: drop this override once the SDK ships `py.typed`. module = "diffbot.*" ignore_missing_imports = true From 44f6cd3cf1b46774447c36c8a6a32c0ad9600dba Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:20:30 -1000 Subject: [PATCH 02/12] fix(test): mock DQL as POST with JSON body Co-Authored-By: Claude Sonnet 5.5 --- tests/unit_tests/test_readme_examples.py | 2 +- tests/unit_tests/test_retriever.py | 34 +++++++++++++----------- tests/unit_tests/test_tools.py | 11 +++++--- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/tests/unit_tests/test_readme_examples.py b/tests/unit_tests/test_readme_examples.py index d40479b..90f12d2 100644 --- a/tests/unit_tests/test_readme_examples.py +++ b/tests/unit_tests/test_readme_examples.py @@ -127,7 +127,7 @@ def test_readme_example_runs( pytest.skip(f"imports {sorted(unmockable)} — covered by the live suite") monkeypatch.setenv("DIFFBOT_API_TOKEN", "test-token") - respx_mock.get(DQL_URL).mock(return_value=httpx.Response(200, json=DQL_BODY)) + respx_mock.post(DQL_URL).mock(return_value=httpx.Response(200, json=DQL_BODY)) respx_mock.get(WEB_SEARCH_URL).mock( return_value=httpx.Response(200, json=WEB_SEARCH_BODY) ) diff --git a/tests/unit_tests/test_retriever.py b/tests/unit_tests/test_retriever.py index 99729b4..929f1e8 100644 --- a/tests/unit_tests/test_retriever.py +++ b/tests/unit_tests/test_retriever.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + import httpx import pytest import respx @@ -56,7 +58,7 @@ async def test_async_call_requires_async_client() -> None: @respx.mock def test_invoke_maps_entities_to_documents() -> None: - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) r = DiffbotKnowledgeGraphRetriever(client=Diffbot(token="t"), k=2) docs = r.invoke("type:Organization") assert len(docs) == 2 @@ -81,7 +83,7 @@ def test_invoke_handles_flat_entity_shape() -> None: {"id": "E9", "type": "Organization", "name": "Flat Co", "description": "x"} ] } - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=flat_body)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=flat_body)) r = DiffbotKnowledgeGraphRetriever(client=Diffbot(token="t")) [doc] = r.invoke("type:Organization") assert doc.page_content == "x" @@ -90,17 +92,17 @@ def test_invoke_handles_flat_entity_shape() -> None: @respx.mock def test_invoke_k_kwarg_overrides_default() -> None: - route = respx.get(f"{DQL_URL}").mock( + route = respx.post(f"{DQL_URL}").mock( return_value=httpx.Response(200, json=SAMPLE_BODY) ) r = DiffbotKnowledgeGraphRetriever(client=Diffbot(token="t"), k=10) r.invoke("type:Organization", k=3) - assert route.calls.last.request.url.params["size"] == "3" + assert json.loads(route.calls.last.request.content)["size"] == 3 @respx.mock def test_invoke_rejects_invalid_k() -> None: - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) r = DiffbotKnowledgeGraphRetriever(client=Diffbot(token="t")) with pytest.raises(ValueError, match="positive integer"): r.invoke("type:Organization", k=0) @@ -108,7 +110,7 @@ def test_invoke_rejects_invalid_k() -> None: @respx.mock async def test_ainvoke_maps_entities_to_documents() -> None: - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) r = DiffbotKnowledgeGraphRetriever(async_client=DiffbotAsync(token="t"), k=2) docs = await r.ainvoke("type:Organization") assert [d.metadata["id"] for d in docs] == ["E1", "E2"] @@ -137,7 +139,7 @@ async def test_ainvoke_maps_entities_to_documents() -> None: @respx.mock def test_fields_projection_narrows_metadata() -> None: - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=FAT_BODY)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=FAT_BODY)) r = DiffbotKnowledgeGraphRetriever( client=Diffbot(token="t"), fields=["id", "name", "homepageUri", "nbEmployees"], @@ -156,7 +158,7 @@ def test_fields_projection_narrows_metadata() -> None: def test_fields_excludes_chosen_content_field() -> None: # If `description` is in `fields` it should still be excluded from metadata # because it was promoted to `page_content`. - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=FAT_BODY)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=FAT_BODY)) r = DiffbotKnowledgeGraphRetriever( client=Diffbot(token="t"), fields=["id", "name", "description"], @@ -178,7 +180,7 @@ def test_content_fields_priority_is_configurable() -> None: } ] } - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=body)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=body)) r = DiffbotKnowledgeGraphRetriever( client=Diffbot(token="t"), content_fields=["summary", "description", "name"], @@ -192,7 +194,7 @@ def test_content_fields_priority_is_configurable() -> None: @respx.mock def test_document_mapper_overrides_default() -> None: - respx.get(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=FAT_BODY)) + respx.post(f"{DQL_URL}").mock(return_value=httpx.Response(200, json=FAT_BODY)) def mapper(entity: dict) -> Document: return Document( @@ -214,10 +216,10 @@ def mapper(entity: dict) -> Document: @respx.mock def test_sdk_kwargs_pass_through() -> None: - # The SDK serializes from_, filter, format, exportspec onto the request URL. + # The SDK POSTs from_, filter, format, exportspec in the JSON body. # We don't want to assert the SDK's wire format byte-for-byte (that's its # job to test), but the params should at least show up. - route = respx.get(DQL_URL).mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) + route = respx.post(DQL_URL).mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) r = DiffbotKnowledgeGraphRetriever( client=Diffbot(token="t"), k=5, @@ -226,9 +228,9 @@ def test_sdk_kwargs_pass_through() -> None: exportspec="csv-spec", ) r.invoke("name:Acme") - params = route.calls.last.request.url.params - assert params["size"] == "5" - assert params["from"] == "10" + params = json.loads(route.calls.last.request.content) + assert params["size"] == 5 + assert params["from"] == 10 assert params["filter"] == "type:Organization" assert params["exportspec"] == "csv-spec" @@ -238,7 +240,7 @@ def test_client_is_reused_and_not_closed() -> None: # The component uses the supplied client as-is and never closes it. After two # retrieve calls the client should still be usable, and the same client can be # shared across components (one connection pool). - respx.get(DQL_URL).mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) + respx.post(DQL_URL).mock(return_value=httpx.Response(200, json=SAMPLE_BODY)) db = Diffbot(token="t", timeout=5.0) r = DiffbotKnowledgeGraphRetriever(client=db, k=1) r.invoke("name:Acme") diff --git a/tests/unit_tests/test_tools.py b/tests/unit_tests/test_tools.py index 5d11fae..6c6d395 100644 --- a/tests/unit_tests/test_tools.py +++ b/tests/unit_tests/test_tools.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + import httpx import respx from diffbot import Diffbot @@ -144,7 +146,7 @@ def test_entities_tool_returns_response_dict() -> None: @respx.mock def test_kg_tool_returns_raw_body() -> None: body = {"data": [{"score": 1.0, "entity": {"id": "E1", "name": "Acme"}}]} - respx.get(DQL_URL).mock(return_value=httpx.Response(200, json=body)) + respx.post(DQL_URL).mock(return_value=httpx.Response(200, json=body)) tool = DiffbotKnowledgeGraphTool(client=Diffbot(token="t")) out = tool.invoke({"query": "type:Organization", "size": 1}) assert out["data"][0]["entity"]["id"] == "E1" @@ -186,11 +188,12 @@ def test_ontology_tool_returns_error_dict_on_unknown_type() -> None: @respx.mock def test_dql_probe_tool_returns_hit_counts() -> None: def handler(request: httpx.Request) -> httpx.Response: - assert request.url.params["size"] == "0" - hits = 5 if "Diffbot" in request.url.params["query"] else 100 + body = json.loads(request.content) + assert body["size"] == 0 + hits = 5 if "Diffbot" in body["query"] else 100 return httpx.Response(200, json={"hits": hits, "results": 0}) - respx.get(DQL_URL).mock(side_effect=handler) + respx.post(DQL_URL).mock(side_effect=handler) tool = DiffbotDQLProbeTool(client=Diffbot(token="t")) out = tool.invoke( {"queries": ['type:Organization name:"Diffbot"', "type:Organization"]} From 3f1c5c8a68b102ce1b0e39bb6f7dffebf5f722f7 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:20:50 -1000 Subject: [PATCH 03/12] feat(extract): forward SDK extract options from the extract tool and loader Co-Authored-By: Claude Sonnet 5.5 --- langchain_diffbot/document_loaders.py | 17 +++++++-- langchain_diffbot/tools.py | 14 ++++++-- tests/unit_tests/test_document_loaders.py | 42 +++++++++++++++++++++++ tests/unit_tests/test_tools.py | 32 ++++++++++++++++- 4 files changed, 100 insertions(+), 5 deletions(-) diff --git a/langchain_diffbot/document_loaders.py b/langchain_diffbot/document_loaders.py index 54a5b36..0907e8a 100644 --- a/langchain_diffbot/document_loaders.py +++ b/langchain_diffbot/document_loaders.py @@ -74,6 +74,15 @@ class DiffbotExtractLoader(_BaseDiffbotComponent, BaseLoader): fmt: str = "markdown" """Output format. `markdown` uses Diffbot's LLM-optimized mode.""" + extract_options: dict[str, Any] = Field(default_factory=dict) + """Keyword options forwarded to `Diffbot.extract` for every URL. + + Any of the SDK's extract options (`timeout`, `mode`, `fields`, `fallback`, + `discussion`, `paging`, `max_pages`, `natural_language`, + `summary_num_sentences`, `max_tags`, `tag_confidence`, + `category_confidence`, `use_proxy`, `extra`). Empty by default. + """ + document_mapper: ExtractDocumentMapper | None = None """Optional `(url, raw_response) -> Document` override.""" @@ -85,14 +94,18 @@ def lazy_load(self) -> Iterator[Document]: """Yield one `Document` per URL, calling `extract` synchronously.""" with self._sync_db() as db: for url in self.urls: - raw = db.extract(url, api=self.api, fmt=self.fmt) + raw = db.extract( + url, api=self.api, fmt=self.fmt, **self.extract_options + ) yield self._to_doc(url, raw) async def alazy_load(self) -> AsyncIterator[Document]: """Yield one `Document` per URL, calling `extract` asynchronously.""" async with self._async_db() as db: for url in self.urls: - raw = await db.extract(url, api=self.api, fmt=self.fmt) + raw = await db.extract( + url, api=self.api, fmt=self.fmt, **self.extract_options + ) yield self._to_doc(url, raw) diff --git a/langchain_diffbot/tools.py b/langchain_diffbot/tools.py index c53017d..1999bf9 100644 --- a/langchain_diffbot/tools.py +++ b/langchain_diffbot/tools.py @@ -53,6 +53,16 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool): ) args_schema: type[BaseModel] = _DiffbotExtractInput + extract_options: dict[str, Any] = Field(default_factory=dict) + """Keyword options forwarded to `Diffbot.extract` on every call. + + Any of the SDK's extract options (`timeout`, `mode`, `fields`, `fallback`, + `discussion`, `paging`, `max_pages`, `natural_language`, + `summary_num_sentences`, `max_tags`, `tag_confidence`, + `category_confidence`, `use_proxy`, `extra`). Empty by default. Set by the + developer, not the agent. + """ + @staticmethod def _shape_response(raw: dict[str, Any]) -> dict[str, Any]: objects = raw.get("objects") or [] @@ -74,7 +84,7 @@ def _run( ) -> dict[str, Any]: try: with self._sync_db() as db: - raw = db.extract(url, api=api, fmt=fmt) + raw = db.extract(url, api=api, fmt=fmt, **self.extract_options) except ExtractionError as e: return {"error": str(e), "errorCode": e.error_code} return self._shape_response(raw) @@ -88,7 +98,7 @@ async def _arun( ) -> dict[str, Any]: try: async with self._async_db() as db: - raw = await db.extract(url, api=api, fmt=fmt) + raw = await db.extract(url, api=api, fmt=fmt, **self.extract_options) except ExtractionError as e: return {"error": str(e), "errorCode": e.error_code} return self._shape_response(raw) diff --git a/tests/unit_tests/test_document_loaders.py b/tests/unit_tests/test_document_loaders.py index f0e574c..3e7ba87 100644 --- a/tests/unit_tests/test_document_loaders.py +++ b/tests/unit_tests/test_document_loaders.py @@ -116,3 +116,45 @@ def test_crawl_loader_user_can_override_watch() -> None: crawl_kwargs={"watch": False, "hops": 3}, ) assert loader._kwargs() == {"watch": False, "hops": 3} + + +@respx.mock +def test_extract_loader_passes_extract_options() -> None: + route = respx.get(ANALYZE_URL).mock( + return_value=httpx.Response(200, json={"objects": [{"text": "x"}]}) + ) + loader = DiffbotExtractLoader( + client=Diffbot(token="t"), + urls=["https://example.com"], + extract_options={"timeout": 5000, "fields": ["links", "meta"], "paging": False}, + ) + assert len(list(loader.lazy_load())) == 1 + params = route.calls.last.request.url.params + assert params["timeout"] == "5000" + assert params["fields"] == "links,meta" + + +@respx.mock +async def test_extract_loader_alazy_load_passes_extract_options() -> None: + route = respx.get(ANALYZE_URL).mock( + return_value=httpx.Response(200, json={"objects": [{"text": "x"}]}) + ) + loader = DiffbotExtractLoader( + async_client=DiffbotAsync(token="t"), + urls=["https://example.com"], + extract_options={"timeout": 7000, "mode": "article"}, + ) + assert len([d async for d in loader.alazy_load()]) == 1 + params = route.calls.last.request.url.params + assert params["timeout"] == "7000" + assert params["mode"] == "article" + + +@respx.mock +def test_extract_loader_default_sends_no_extra_options() -> None: + route = respx.get(ANALYZE_URL).mock( + return_value=httpx.Response(200, json={"objects": [{"text": "x"}]}) + ) + loader = DiffbotExtractLoader(client=Diffbot(token="t"), urls=["https://e.com"]) + list(loader.lazy_load()) + assert "fields" not in route.calls.last.request.url.params diff --git a/tests/unit_tests/test_tools.py b/tests/unit_tests/test_tools.py index 6c6d395..234b3b7 100644 --- a/tests/unit_tests/test_tools.py +++ b/tests/unit_tests/test_tools.py @@ -6,7 +6,7 @@ import httpx import respx -from diffbot import Diffbot +from diffbot import Diffbot, DiffbotAsync from langchain_diffbot import ( DiffbotAskTool, @@ -217,3 +217,33 @@ def test_ask_tool_aggregates_streamed_answer() -> None: tool = DiffbotAskTool(client=Diffbot(token="t")) out = tool.invoke({"question": "When was Diffbot founded?"}) assert out == "Diffbot was founded in 2008." + + +@respx.mock +def test_extract_tool_passes_extract_options() -> None: + route = respx.get(ANALYZE_URL).mock( + return_value=httpx.Response(200, json={"objects": [{"text": "x"}]}) + ) + tool = DiffbotExtractTool( + client=Diffbot(token="t"), + extract_options={"timeout": 5000, "fields": ["links", "meta"]}, + ) + tool.invoke({"url": "https://example.com"}) + params = route.calls.last.request.url.params + assert params["timeout"] == "5000" + assert params["fields"] == "links,meta" + + +@respx.mock +async def test_extract_tool_arun_passes_extract_options() -> None: + route = respx.get(ANALYZE_URL).mock( + return_value=httpx.Response(200, json={"objects": [{"text": "x"}]}) + ) + tool = DiffbotExtractTool( + async_client=DiffbotAsync(token="t"), + extract_options={"mode": "article", "timeout": 9000}, + ) + await tool.ainvoke({"url": "https://example.com"}) + params = route.calls.last.request.url.params + assert params["mode"] == "article" + assert params["timeout"] == "9000" From d319f3a147c20ba16ac7ecfedc608efffb1d96c6 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 14:46:52 -1000 Subject: [PATCH 04/12] feat(crawl): opt-in start/poll/download path on DiffbotCrawlLoader Co-Authored-By: Claude Sonnet 5.5 --- langchain_diffbot/document_loaders.py | 139 +++++++++++++++++++++- tests/unit_tests/test_document_loaders.py | 109 +++++++++++++++++ 2 files changed, 246 insertions(+), 2 deletions(-) diff --git a/langchain_diffbot/document_loaders.py b/langchain_diffbot/document_loaders.py index 0907e8a..78bed02 100644 --- a/langchain_diffbot/document_loaders.py +++ b/langchain_diffbot/document_loaders.py @@ -2,10 +2,13 @@ from __future__ import annotations +import asyncio +import time from collections.abc import AsyncIterator, Callable, Iterator from typing import Any from diffbot.crawl import CrawlEvent, CrawlEventType +from diffbot.errors import JobError from langchain_core.document_loaders import BaseLoader from langchain_core.documents import Document from pydantic import Field @@ -33,6 +36,34 @@ def _default_extract_mapper(url: str, raw: dict[str, Any]) -> Document: return Document(page_content=page_content, metadata=metadata) +CrawlRecordMapper = Callable[[dict[str, Any]], Document | None] +"""`(record) -> Document | None` for `crawl_download` records. None skips.""" + +# Crawler job status codes that mean "still running" (matches the SDK's own +# watch loop); anything else is terminal. +_CRAWL_RUNNING_STATUSES = (0, 7) + + +def _default_record_mapper(record: dict[str, Any]) -> Document | None: + url = record.get("pageUrl") or record.get("resolvedPageUrl") or "" + return _default_extract_mapper(url, {"objects": [record]}) + + +def _check_job_done(job_name: str, job: dict[str, Any]) -> bool: + """Return True when the job is terminal; raise `JobError` if it failed.""" + status = job.get("jobStatus") or {} + if status.get("status", 0) in _CRAWL_RUNNING_STATUSES: + return False + message = status.get("message", "") + if "fail" in message.lower() or "error" in message.lower(): + raise JobError(job_name, f"Crawler job failed: {message}") + return True + + +def _poll_timeout_error(job_name: str, timeout: float) -> TimeoutError: + return TimeoutError(f"Crawl job {job_name!r} not finished after {timeout}s") + + def _default_crawl_mapper(event: CrawlEvent) -> Document | None: # Non-URL events (e.g. JOB_CREATED) are skipped by default. The Document's # page_content is the URL itself — the crawl SDK only surfaces URLs, not @@ -133,6 +164,38 @@ class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader): event_mapper: CrawlEventMapper | None = None """Optional `(event) -> Document | None` override. Return None to skip.""" + download: bool = False + """Opt in to the start / poll / download path instead of watching. + + When True the loader starts a job (or reuses `job_name`), polls + `crawl_get_job` until it finishes, then calls `crawl_download` and yields a + `Document` per extracted record with real page content. `watch` and + `event_mapper` are not used in this mode. Default False keeps the + URL-per-event watch behavior. + """ + + job_name: str | None = None + """Existing crawl job to poll and download (only with `download=True`). + + If unset, a new job is started for `site` (using `crawl_kwargs`, with + `watch=False`; set `job_name` inside `crawl_kwargs` to choose its name). + """ + + poll_interval: float = 2.0 + """Seconds between job-status polls (only with `download=True`).""" + + poll_timeout: float | None = 600.0 + """Max seconds to wait for the job to finish; None waits forever.""" + + download_type: str | None = None + """Report passed to `crawl_download(type=...)`, e.g. `"urls"`.""" + + download_num: int | None = None + """Cap on records passed to `crawl_download(num=...)`.""" + + record_mapper: CrawlRecordMapper | None = None + """Optional `(record) -> Document | None` override for downloaded records.""" + def _kwargs(self) -> dict[str, Any]: kw = dict(self.crawl_kwargs) kw.setdefault("watch", True) @@ -142,8 +205,28 @@ def _map_event(self, event: CrawlEvent) -> Document | None: mapper = self.event_mapper or _default_crawl_mapper return mapper(event) + def _map_record(self, record: dict[str, Any]) -> Document | None: + return (self.record_mapper or _default_record_mapper)(record) + + def _deadline(self) -> float | None: + if self.poll_timeout is None: + return None + return time.monotonic() + self.poll_timeout + + def _start_kwargs(self) -> dict[str, Any]: + kw = dict(self.crawl_kwargs) + kw["watch"] = False + return kw + def lazy_load(self) -> Iterator[Document]: - """Drive a crawl synchronously and yield a `Document` per mapped event.""" + """Drive a crawl synchronously and yield a `Document` per mapped event. + + With `download=True`, start (or reuse) a job, poll until it finishes, + and yield a `Document` per downloaded record instead. + """ + if self.download: + yield from self._download_sync() + return with self._sync_db() as db: for event in db.crawl(self.site, **self._kwargs()): doc = self._map_event(event) @@ -151,9 +234,61 @@ def lazy_load(self) -> Iterator[Document]: yield doc async def alazy_load(self) -> AsyncIterator[Document]: - """Drive a crawl asynchronously and yield a `Document` per mapped event.""" + """Drive a crawl asynchronously and yield a `Document` per mapped event. + + With `download=True`, start (or reuse) a job, poll until it finishes, + and yield a `Document` per downloaded record instead. + """ + if self.download: + async for downloaded in self._download_async(): + yield downloaded + return async with self._async_db() as db: async for event in db.crawl(self.site, **self._kwargs()): doc = self._map_event(event) if doc is not None: yield doc + + def _download_sync(self) -> Iterator[Document]: + with self._sync_db() as db: + name = self.job_name + if name is None: + for event in db.crawl(self.site, **self._start_kwargs()): + if event.event_type == CrawlEventType.JOB_CREATED: + name = event.details["job_name"] + if name is None: + raise RuntimeError("Diffbot crawl did not report a job name") + deadline = self._deadline() + while not _check_job_done(name, db.crawl_get_job(name)): + if deadline is not None and time.monotonic() >= deadline: + raise _poll_timeout_error(name, self.poll_timeout or 0) + time.sleep(self.poll_interval) + records = db.crawl_download( + name, format="json", type=self.download_type, num=self.download_num + ) + for record in records: + rec_doc = self._map_record(record) # type: ignore[arg-type] + if rec_doc is not None: + yield rec_doc + + async def _download_async(self) -> AsyncIterator[Document]: + async with self._async_db() as db: + name = self.job_name + if name is None: + async for event in db.crawl(self.site, **self._start_kwargs()): + if event.event_type == CrawlEventType.JOB_CREATED: + name = event.details["job_name"] + if name is None: + raise RuntimeError("Diffbot crawl did not report a job name") + deadline = self._deadline() + while not _check_job_done(name, await db.crawl_get_job(name)): + if deadline is not None and time.monotonic() >= deadline: + raise _poll_timeout_error(name, self.poll_timeout or 0) + await asyncio.sleep(self.poll_interval) + records = await db.crawl_download( + name, format="json", type=self.download_type, num=self.download_num + ) + for record in records: + rec_doc = self._map_record(record) # type: ignore[arg-type] + if rec_doc is not None: + yield rec_doc diff --git a/tests/unit_tests/test_document_loaders.py b/tests/unit_tests/test_document_loaders.py index 3e7ba87..93dfbf3 100644 --- a/tests/unit_tests/test_document_loaders.py +++ b/tests/unit_tests/test_document_loaders.py @@ -3,9 +3,11 @@ from __future__ import annotations import httpx +import pytest import respx from diffbot import Diffbot, DiffbotAsync from diffbot.crawl import CrawlEvent, CrawlEventType +from diffbot.errors import JobError from langchain_diffbot import DiffbotCrawlLoader, DiffbotExtractLoader @@ -158,3 +160,110 @@ def test_extract_loader_default_sends_no_extra_options() -> None: loader = DiffbotExtractLoader(client=Diffbot(token="t"), urls=["https://e.com"]) list(loader.lazy_load()) assert "fields" not in route.calls.last.request.url.params + + +def _crawl_stubs(mocker, statuses: list[dict], records: list[dict]): # type: ignore[no-untyped-def] + db = Diffbot(token="t") + started = mocker.patch.object( + db, + "crawl", + return_value=iter( + [CrawlEvent(CrawlEventType.JOB_CREATED, "now", {"job_name": "job1"})] + ), + ) + get_job = mocker.patch.object(db, "crawl_get_job", side_effect=statuses) + download = mocker.patch.object(db, "crawl_download", return_value=records) + return db, started, get_job, download + + +_RECORDS = [ + {"text": "Page one", "title": "One", "pageUrl": "https://example.com/1"}, + {"text": "Page two", "title": "Two", "pageUrl": "https://example.com/2"}, +] + + +def test_crawl_loader_download_starts_polls_and_downloads(mocker) -> None: # type: ignore[no-untyped-def] + db, started, get_job, download = _crawl_stubs( + mocker, + [{"jobStatus": {"status": 0}}, {"jobStatus": {"status": 9, "message": "done"}}], + _RECORDS, + ) + loader = DiffbotCrawlLoader( + client=db, + site="https://example.com", + download=True, + poll_interval=0, + download_num=5, + crawl_kwargs={"hops": 1}, + ) + docs = list(loader.lazy_load()) + started.assert_called_once_with("https://example.com", hops=1, watch=False) + assert get_job.call_count == 2 + download.assert_called_once_with("job1", format="json", type=None, num=5) + assert [d.page_content for d in docs] == ["Page one", "Page two"] + assert docs[0].metadata["url"] == "https://example.com/1" + assert docs[0].metadata["title"] == "One" + + +def test_crawl_loader_download_reuses_existing_job(mocker) -> None: # type: ignore[no-untyped-def] + db, started, _, download = _crawl_stubs( + mocker, [{"jobStatus": {"status": 9, "message": "done"}}], _RECORDS + ) + loader = DiffbotCrawlLoader( + client=db, site="https://example.com", download=True, job_name="existing" + ) + assert len(list(loader.lazy_load())) == 2 + started.assert_not_called() + assert download.call_args.args == ("existing",) + + +def test_crawl_loader_download_raises_on_failed_job(mocker) -> None: # type: ignore[no-untyped-def] + db, *_ = _crawl_stubs( + mocker, [{"jobStatus": {"status": 4, "message": "Job failed"}}], [] + ) + loader = DiffbotCrawlLoader(client=db, site="https://example.com", download=True) + with pytest.raises(JobError): + list(loader.lazy_load()) + + +def test_crawl_loader_download_times_out(mocker) -> None: # type: ignore[no-untyped-def] + db, *_ = _crawl_stubs(mocker, [{"jobStatus": {"status": 0}}] * 50, []) + loader = DiffbotCrawlLoader( + client=db, + site="https://example.com", + download=True, + poll_interval=0, + poll_timeout=0, + ) + with pytest.raises(TimeoutError): + list(loader.lazy_load()) + + +async def test_crawl_loader_download_async(mocker) -> None: # type: ignore[no-untyped-def] + db = DiffbotAsync(token="t") + + async def fake_crawl(site, **kwargs): # type: ignore[no-untyped-def] + assert kwargs["watch"] is False + yield CrawlEvent(CrawlEventType.JOB_CREATED, "now", {"job_name": "job1"}) + + mocker.patch.object(db, "crawl", fake_crawl) + mocker.patch.object( + db, + "crawl_get_job", + side_effect=[{"jobStatus": {"status": 0}}, {"jobStatus": {"status": 9}}], + ) + mocker.patch.object(db, "crawl_download", return_value=_RECORDS) + loader = DiffbotCrawlLoader( + async_client=db, site="https://example.com", download=True, poll_interval=0 + ) + docs = [d async for d in loader.alazy_load()] + assert [d.page_content for d in docs] == ["Page one", "Page two"] + + +def test_crawl_loader_default_path_does_not_download(mocker) -> None: # type: ignore[no-untyped-def] + db = Diffbot(token="t") + crawl = mocker.patch.object(db, "crawl", return_value=iter([])) + download = mocker.patch.object(db, "crawl_download") + assert list(DiffbotCrawlLoader(client=db, site="https://e.com").lazy_load()) == [] + crawl.assert_called_once_with("https://e.com", watch=True) + download.assert_not_called() From 913a5fa8aac3f60ae711d7c63072058b94546e3b Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 14:47:01 -1000 Subject: [PATCH 05/12] docs(readme): extract_options and crawl download mode Co-Authored-By: Claude Sonnet 5.5 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 33b584c..f40bb06 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,8 @@ for doc in loader.lazy_load(): `DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`. +Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.3.0`. + ## Crawl a site `DiffbotCrawlLoader` drives a Diffbot crawl job and yields one `Document` per crawled URL. The `page_content` is the URL itself (the crawl API surfaces URLs, not page contents) — chain it with `DiffbotExtractLoader` to fetch the content of each URL. @@ -139,6 +141,8 @@ for doc in loader.lazy_load(): print(doc.metadata["url"], doc.metadata["status"]) ``` +By default the loader watches the job. Pass `download=True` for an opt-in start / poll / download path: it starts a job (or reuses `job_name`), polls until the job finishes (`poll_interval`, `poll_timeout`), then calls `crawl_download` and yields one `Document` per extracted record with real page content. `download_type`, `download_num`, and `record_mapper` tune the download. + ## ChatDiffbot ```python From 841506f8544d324c33793bd1a39e5980683c0dea Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:35:01 -1000 Subject: [PATCH 06/12] fix(crawl): fail fast on missing job and reject the CSV urls report In download mode, crawl_get_job returns {} for an unknown job, which was read as status 0 ("running"). The loader then polled until poll_timeout, or forever when poll_timeout=None. It now raises JobError. download_type="urls" makes crawl_download return CSV text. The loader iterated that string one character at a time and passed each character to the record mapper. That value is now rejected at construction, and the docstring no longer gives it as the example. Also share the record-mapping loop between the sync and async paths, which drops the two type: ignore comments. Co-Authored-By: Claude Opus 5.5 --- langchain_diffbot/document_loaders.py | 45 ++++++++++++++++------- tests/unit_tests/test_document_loaders.py | 39 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 13 deletions(-) diff --git a/langchain_diffbot/document_loaders.py b/langchain_diffbot/document_loaders.py index 78bed02..e20dc53 100644 --- a/langchain_diffbot/document_loaders.py +++ b/langchain_diffbot/document_loaders.py @@ -11,7 +11,7 @@ from diffbot.errors import JobError from langchain_core.document_loaders import BaseLoader from langchain_core.documents import Document -from pydantic import Field +from pydantic import Field, field_validator from langchain_diffbot._base import _BaseDiffbotComponent @@ -50,7 +50,13 @@ def _default_record_mapper(record: dict[str, Any]) -> Document | None: def _check_job_done(job_name: str, job: dict[str, Any]) -> bool: - """Return True when the job is terminal; raise `JobError` if it failed.""" + """Return True when the job is terminal; raise `JobError` if it failed. + + `crawl_get_job` returns `{}` for an unknown job; that is an error, not a + job that is "still running" (otherwise we'd poll until `poll_timeout`). + """ + if not job: + raise JobError(job_name, f"Crawler job {job_name!r} not found") status = job.get("jobStatus") or {} if status.get("status", 0) in _CRAWL_RUNNING_STATUSES: return False @@ -188,7 +194,11 @@ class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader): """Max seconds to wait for the job to finish; None waits forever.""" download_type: str | None = None - """Report passed to `crawl_download(type=...)`, e.g. `"urls"`.""" + """Report passed to `crawl_download(type=...)`. + + `"urls"` is rejected: that report is CSV text, not records. Use the default + watch mode (`download=False`) for a URL-per-`Document` crawl. + """ download_num: int | None = None """Cap on records passed to `crawl_download(num=...)`.""" @@ -196,6 +206,16 @@ class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader): record_mapper: CrawlRecordMapper | None = None """Optional `(record) -> Document | None` override for downloaded records.""" + @field_validator("download_type") + @classmethod + def _reject_urls_report(cls, v: str | None) -> str | None: + if v == "urls": + raise ValueError( + 'download_type="urls" returns CSV text, not records; use the ' + "default watch mode (download=False) to get one Document per URL" + ) + return v + def _kwargs(self) -> dict[str, Any]: kw = dict(self.crawl_kwargs) kw.setdefault("watch", True) @@ -205,8 +225,12 @@ def _map_event(self, event: CrawlEvent) -> Document | None: mapper = self.event_mapper or _default_crawl_mapper return mapper(event) - def _map_record(self, record: dict[str, Any]) -> Document | None: - return (self.record_mapper or _default_record_mapper)(record) + def _records_to_docs(self, records: list[dict[str, Any]]) -> Iterator[Document]: + mapper = self.record_mapper or _default_record_mapper + for record in records: + doc = mapper(record) + if doc is not None: + yield doc def _deadline(self) -> float | None: if self.poll_timeout is None: @@ -266,10 +290,7 @@ def _download_sync(self) -> Iterator[Document]: records = db.crawl_download( name, format="json", type=self.download_type, num=self.download_num ) - for record in records: - rec_doc = self._map_record(record) # type: ignore[arg-type] - if rec_doc is not None: - yield rec_doc + yield from self._records_to_docs(records) async def _download_async(self) -> AsyncIterator[Document]: async with self._async_db() as db: @@ -288,7 +309,5 @@ async def _download_async(self) -> AsyncIterator[Document]: records = await db.crawl_download( name, format="json", type=self.download_type, num=self.download_num ) - for record in records: - rec_doc = self._map_record(record) # type: ignore[arg-type] - if rec_doc is not None: - yield rec_doc + for rec_doc in self._records_to_docs(records): + yield rec_doc diff --git a/tests/unit_tests/test_document_loaders.py b/tests/unit_tests/test_document_loaders.py index 93dfbf3..82938d5 100644 --- a/tests/unit_tests/test_document_loaders.py +++ b/tests/unit_tests/test_document_loaders.py @@ -8,6 +8,7 @@ from diffbot import Diffbot, DiffbotAsync from diffbot.crawl import CrawlEvent, CrawlEventType from diffbot.errors import JobError +from langchain_core.documents import Document from langchain_diffbot import DiffbotCrawlLoader, DiffbotExtractLoader @@ -267,3 +268,41 @@ def test_crawl_loader_default_path_does_not_download(mocker) -> None: # type: i assert list(DiffbotCrawlLoader(client=db, site="https://e.com").lazy_load()) == [] crawl.assert_called_once_with("https://e.com", watch=True) download.assert_not_called() + + +def test_crawl_loader_download_raises_on_missing_job(mocker) -> None: # type: ignore[no-untyped-def] + # crawl_get_job returns {} for an unknown job; don't poll it until timeout. + db, _, get_job, download = _crawl_stubs(mocker, [{}], []) + loader = DiffbotCrawlLoader( + client=db, site="https://example.com", download=True, job_name="typo" + ) + with pytest.raises(JobError, match="not found"): + list(loader.lazy_load()) + assert get_job.call_count == 1 + download.assert_not_called() + + +def test_crawl_loader_rejects_urls_download_type() -> None: + # The `urls` report is CSV text; iterating it as records would be garbage. + with pytest.raises(ValueError, match="urls"): + DiffbotCrawlLoader( + client=Diffbot(token="t"), + site="https://example.com", + download=True, + download_type="urls", + ) + + +def test_crawl_loader_record_mapper_can_skip(mocker) -> None: # type: ignore[no-untyped-def] + db, *_ = _crawl_stubs( + mocker, [{"jobStatus": {"status": 9, "message": "done"}}], _RECORDS + ) + loader = DiffbotCrawlLoader( + client=db, + site="https://example.com", + download=True, + record_mapper=lambda r: ( + None if r["title"] == "One" else Document(page_content=r["text"]) + ), + ) + assert [d.page_content for d in loader.lazy_load()] == ["Page two"] From 390361a2810aec1bf820d071f20e2d5de1996859 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:35:10 -1000 Subject: [PATCH 07/12] docs(claude): correct SDK dependency name and document dev-sdk flow Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index f45d1bf..0c104f5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ LangChain integration package for the Diffbot APIs (Knowledge Graph, Extract, We ## Stack - **Build/deps**: `uv` (not Poetry/pip). `pyproject.toml` is PEP 621 with `hatchling` as the build backend. Dependency groups (`test`, `lint`, `typing`) are declared under `[dependency-groups]` and invoked via `uv run --group ...`. -- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `https://kg.diffbot.com/kg/v3/dql`, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). `diffbot-python` is published on PyPI and resolved from there as a normal dependency (`diffbot-python>=0.1.0`); `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. +- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.3.0`. Until 3.3.0 is published, `make dev-sdk` installs the sibling `../diffbot-python` checkout editable over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Drop this step once 3.3.0 is published (`TODO(sdk-release)` in `pyproject.toml` / `Makefile`). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Models**: `pydantic` v2. - **LangChain**: targets `langchain-core >=1.0,<2.0`. During local dev `langchain-core` and `langchain-tests` are resolved from a sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Test runner**: `pytest` with `asyncio_mode = "auto"`. From 5f3c4c0c355ff72cc17f5456abd8aafce3044ad5 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:30:27 -1000 Subject: [PATCH 08/12] ci(release): publish to PyPI via trusted publishing (OIDC) Add environment: pypi and id-token: write, use uv publish --trusted-publishing always, drop PYPI_TOKEN, least-privilege permissions, and never cancel in-flight publishes. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91da798..53b9973 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,20 +7,22 @@ on: tags: ["v*"] workflow_dispatch: -permissions: - contents: write +# Least privilege by default; the release job elevates its own permissions. +permissions: {} -# Re-pushing a tag (or re-dispatching) cancels an in-flight run for the same -# ref so a stuck build can't pile up behind a newer attempt. +# Never cancel an in-flight run: cancelling a publish midway could leave PyPI +# and the GitHub Release out of sync. Runs for the same ref queue instead. concurrency: group: release-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: build: name: Build distribution runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read steps: - uses: actions/checkout@v4 @@ -58,6 +60,13 @@ jobs: runs-on: ubuntu-latest # Only publish for real tag pushes; workflow_dispatch runs just build + verify. if: startsWith(github.ref, 'refs/tags/') + # Must match the environment configured on PyPI's trusted publisher. + environment: pypi + permissions: + # Required for PyPI trusted publishing (OIDC). + id-token: write + # Required to create the GitHub Release. + contents: write steps: - name: Download artifacts uses: actions/download-artifact@v4 @@ -77,9 +86,7 @@ jobs: python-version: "3.12" - name: Publish to PyPI - env: - UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: uv publish + run: uv publish --trusted-publishing always - name: Publish GitHub Release env: From 1e7693920336fb73acfc5131d85360a4ce1cd209 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:39:48 -1000 Subject: [PATCH 09/12] ci: drop pypi environment from release publish job Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 53b9973..979aa12 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,8 +60,7 @@ jobs: runs-on: ubuntu-latest # Only publish for real tag pushes; workflow_dispatch runs just build + verify. if: startsWith(github.ref, 'refs/tags/') - # Must match the environment configured on PyPI's trusted publisher. - environment: pypi + # No GitHub environment: the PyPI trusted publisher has none configured. permissions: # Required for PyPI trusted publishing (OIDC). id-token: write From 14fbd7cdcc5586b204080fd503d91772376c0ea6 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 17:59:07 -1000 Subject: [PATCH 10/12] Require diffbot>=3.4.0,<4 and release 0.3.0. 0.2.0 is already on PyPI. The mypy override stays because 3.4.0 has no py.typed marker. uv.lock is refreshed after 3.4.0 is published. --- CHANGELOG.md | 11 +++++++++++ CLAUDE.md | 2 +- Makefile | 8 ++++---- README.md | 2 +- pyproject.toml | 13 +++++++------ 5 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2f1f9af --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## 0.3.0 - 2026-11-17 + +Require `diffbot>=3.4.0,<4`. 0.2.0 is already on PyPI, so this release is 0.3.0. + +DQL follows the SDK: POST `https://kg.diffbot.com/kg/v3/dql` with a JSON body; the token stays the `token` query parameter. GET remains the low-level escape hatch. + +Relock with `uv lock --upgrade-package diffbot` once 3.4.0 is on PyPI. That command needs the sibling `../langchain` checkout so the editable `langchain-core` / `langchain-tests` sources stay in `uv.lock`. `make dev-sdk` is only a pre-publish local install; do not commit it. + +Publishing is PyPI trusted publishing from `.github/workflows/release.yml` (no token, no GitHub environment). Tag `v0.3.0`. The SDK still has no `py.typed` marker, so the mypy override stays. diff --git a/CLAUDE.md b/CLAUDE.md index 0c104f5..6ca0137 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ LangChain integration package for the Diffbot APIs (Knowledge Graph, Extract, We ## Stack - **Build/deps**: `uv` (not Poetry/pip). `pyproject.toml` is PEP 621 with `hatchling` as the build backend. Dependency groups (`test`, `lint`, `typing`) are declared under `[dependency-groups]` and invoked via `uv run --group ...`. -- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.3.0`. Until 3.3.0 is published, `make dev-sdk` installs the sibling `../diffbot-python` checkout editable over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Drop this step once 3.3.0 is published (`TODO(sdk-release)` in `pyproject.toml` / `Makefile`). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. +- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.4.0,<4`. Until 3.4.0 is on PyPI, `make dev-sdk` can install a sibling checkout over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Do not commit that editable install. Relock with `uv lock --upgrade-package diffbot` once 3.4.0 is published (that command needs the sibling `../langchain` checkout). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Models**: `pydantic` v2. - **LangChain**: targets `langchain-core >=1.0,<2.0`. During local dev `langchain-core` and `langchain-tests` are resolved from a sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Test runner**: `pytest` with `asyncio_mode = "auto"`. diff --git a/Makefile b/Makefile index 3a970f2..446098f 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ .PHONY: dev-sdk format lint test test_integration typing build clean bump-patch bump-minor bump-major set-token-pypi set-token-testpypi release-test release verify-release-test verify-release -# Dev-only: install the unreleased Diffbot SDK from a sibling checkout over the -# synced env (no-op if ../diffbot-python is absent). TODO(sdk-release): remove -# once diffbot>=3.3.0 is on PyPI. Pair with UV_NO_SYNC=true so `make test` etc. -# don't re-resolve and replace it. +# Dev-only: install a local Diffbot SDK checkout over the synced env (no-op if +# ../diffbot-python is absent). The release dependency is diffbot>=3.4.0,<4 from +# PyPI. Pair with UV_NO_SYNC=true so `make test` doesn't re-resolve and replace it. +# Do not commit the editable install. dev-sdk: @if [ -d ../diffbot-python ]; then uv pip install -e ../diffbot-python; \ else echo "../diffbot-python not found; skipping"; fi diff --git a/README.md b/README.md index f40bb06..001c08d 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ for doc in loader.lazy_load(): `DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`. -Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.3.0`. +Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`. ## Crawl a site diff --git a/pyproject.toml b/pyproject.toml index f83b836..3050c8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "langchain-diffbot" -version = "0.2.0" +version = "0.3.0" description = "LangChain integration for the Diffbot Knowledge Graph and Extract APIs" license = { text = "MIT" } readme = "README.md" @@ -24,9 +24,10 @@ requires-python = ">=3.10,<4.0" dependencies = [ "langchain-core>=1.0,<2.0", "httpx>=0.27,<1.0", - # TODO(sdk-release): diffbot 3.3.0 is not on PyPI yet. Once published, relock - # (`uv lock`) and drop the dev-only sibling install (`make dev-sdk`). - "diffbot>=3.3.0", + # Relock after 3.4.0 is on PyPI: `uv lock --upgrade-package diffbot` + # (needs the sibling ../langchain checkout). `make dev-sdk` is only for a + # pre-publish local install; do not commit that path. + "diffbot>=3.4.0,<4", ] [project.optional-dependencies] @@ -75,8 +76,8 @@ include = ["/langchain_diffbot", "/README.md", "/LICENSE", "/pyproject.toml"] disallow_untyped_defs = true [[tool.mypy.overrides]] -# The diffbot SDK doesn't ship a `py.typed` marker yet (checked against 3.3.0 on -# feat/sdk-s1). TODO: drop this override once the SDK ships `py.typed`. +# The diffbot SDK doesn't ship a `py.typed` marker yet (checked against 3.4.0). +# Drop this override once the SDK ships `py.typed`. module = "diffbot.*" ignore_missing_imports = true From 030962adf542c153bf3ce477fb8d7643578be80e Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 18:38:26 -1000 Subject: [PATCH 11/12] Point diffbot at 3.4.0rc0 for CI verification. --- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- Makefile | 2 +- README.md | 2 +- pyproject.toml | 4 +-- uv.lock | 94 ++++++++------------------------------------------ 6 files changed, 21 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f1f9af..86994ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.3.0 - 2026-11-17 -Require `diffbot>=3.4.0,<4`. 0.2.0 is already on PyPI, so this release is 0.3.0. +Require `diffbot>=3.4.0rc0,<4` so CI can verify the SDK release candidate. Switch to `>=3.4.0,<4` for the unison publish. 0.2.0 is already on PyPI, so this release is 0.3.0. DQL follows the SDK: POST `https://kg.diffbot.com/kg/v3/dql` with a JSON body; the token stays the `token` query parameter. GET remains the low-level escape hatch. diff --git a/CLAUDE.md b/CLAUDE.md index 6ca0137..38dd673 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ LangChain integration package for the Diffbot APIs (Knowledge Graph, Extract, We ## Stack - **Build/deps**: `uv` (not Poetry/pip). `pyproject.toml` is PEP 621 with `hatchling` as the build backend. Dependency groups (`test`, `lint`, `typing`) are declared under `[dependency-groups]` and invoked via `uv run --group ...`. -- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.4.0,<4`. Until 3.4.0 is on PyPI, `make dev-sdk` can install a sibling checkout over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Do not commit that editable install. Relock with `uv lock --upgrade-package diffbot` once 3.4.0 is published (that command needs the sibling `../langchain` checkout). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. +- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.4.0rc0,<4` on this verification branch (switch to `>=3.4.0,<4` for the unison publish). `make dev-sdk` can still install a sibling checkout over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Do not commit that editable install. Relock with `uv lock --upgrade-package diffbot` (that command needs the sibling `../langchain` checkout, or use `--no-sources`). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Models**: `pydantic` v2. - **LangChain**: targets `langchain-core >=1.0,<2.0`. During local dev `langchain-core` and `langchain-tests` are resolved from a sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Test runner**: `pytest` with `asyncio_mode = "auto"`. diff --git a/Makefile b/Makefile index 446098f..c1a674d 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: dev-sdk format lint test test_integration typing build clean bump-patch bump-minor bump-major set-token-pypi set-token-testpypi release-test release verify-release-test verify-release # Dev-only: install a local Diffbot SDK checkout over the synced env (no-op if -# ../diffbot-python is absent). The release dependency is diffbot>=3.4.0,<4 from +# ../diffbot-python is absent). The current verification dependency is diffbot>=3.4.0rc0,<4 from # PyPI. Pair with UV_NO_SYNC=true so `make test` doesn't re-resolve and replace it. # Do not commit the editable install. dev-sdk: diff --git a/README.md b/README.md index 001c08d..d47a5d5 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ for doc in loader.lazy_load(): `DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`. -Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`. +Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0rc0,<4` (switch to `>=3.4.0,<4` for the unison publish). ## Crawl a site diff --git a/pyproject.toml b/pyproject.toml index 3050c8c..6e5d171 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,10 @@ requires-python = ">=3.10,<4.0" dependencies = [ "langchain-core>=1.0,<2.0", "httpx>=0.27,<1.0", - # Relock after 3.4.0 is on PyPI: `uv lock --upgrade-package diffbot` + # Relock after the SDK RC/stable is on PyPI: `uv lock --upgrade-package diffbot` # (needs the sibling ../langchain checkout). `make dev-sdk` is only for a # pre-publish local install; do not commit that path. - "diffbot>=3.4.0,<4", + "diffbot>=3.4.0rc0,<4", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index d6af649..bc6afbe 100644 --- a/uv.lock +++ b/uv.lock @@ -205,16 +205,16 @@ wheels = [ [[package]] name = "diffbot" -version = "3.0.0" +version = "3.4.0rc0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "httpx" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ff/d2/4d2433eec2b49e97ddb167549272a68ca7fec312a8b3678c7843edb09ddd/diffbot-3.0.0.tar.gz", hash = "sha256:9ca2e77328d27242ab13046cea948514ff97de7498fcc074f586c305701d87e1", size = 33200, upload-time = "2026-09-28T22:38:06.326Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/33/db23518d8d602bee409e3dc50c42d17d6b432c580298a6a5587c8cdfdce4/diffbot-3.4.0rc0.tar.gz", hash = "sha256:5866529c085c2ce3064153bf135b40b5cd97f493d536d5415acc5dab18487256", size = 65653, upload-time = "2026-11-18T04:02:24.561Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/c2/a51b7a1b726288f8c9e2dc4d08fcda31fd2f3d7c9ce4c59d1d696452e8d5/diffbot-3.0.0-py3-none-any.whl", hash = "sha256:3465e1d59f112051cf62e8a8ccf4523196d9941429eb1bf9c9b6f60630db7cc2", size = 33662, upload-time = "2026-09-28T22:38:07.531Z" }, + { url = "https://files.pythonhosted.org/packages/39/c3/5fc7df5d9f8fd309f386f425a8bee9fcb07722a216acc5bdaf20c7eab7b2/diffbot-3.4.0rc0-py3-none-any.whl", hash = "sha256:5cd55605f864b19e3ac0e3b9d335bbb95b2038a5eadd8dbd274668087d09daa1", size = 54080, upload-time = "2026-11-18T04:02:23.241Z" }, ] [[package]] @@ -523,7 +523,7 @@ wheels = [ [[package]] name = "langchain-core" version = "1.4.0" -source = { editable = "../langchain/libs/core" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, { name = "langchain-protocol" }, @@ -535,56 +535,14 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] - -[package.metadata] -requires-dist = [ - { name = "jsonpatch", specifier = ">=1.33.0,<2.0.0" }, - { name = "langchain-protocol", specifier = ">=0.0.14" }, - { name = "langsmith", specifier = ">=0.3.45,<1.0.0" }, - { name = "packaging", specifier = ">=23.2.0" }, - { name = "pydantic", specifier = ">=2.7.4,<3.0.0" }, - { name = "pyyaml", specifier = ">=5.3.0,<7.0.0" }, - { name = "tenacity", specifier = ">=8.1.0,!=8.4.0,<10.0.0" }, - { name = "typing-extensions", specifier = ">=4.7.0,<5.0.0" }, - { name = "uuid-utils", specifier = ">=0.12.0,<1.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "grandalf", specifier = ">=0.8.0,<1.0.0" }, - { name = "jupyter", specifier = ">=1.0.0,<2.0.0" }, - { name = "setuptools", specifier = ">=67.6.1,<83.0.0" }, -] -lint = [{ name = "ruff", specifier = ">=0.15.0,<0.16.0" }] -test = [ - { name = "blockbuster", specifier = ">=1.5.18,<1.6.0" }, - { name = "freezegun", specifier = ">=1.2.2,<2.0.0" }, - { name = "grandalf", specifier = ">=0.8.0,<1.0.0" }, - { name = "langchain-tests", directory = "../langchain/libs/standard-tests" }, - { name = "numpy", marker = "python_full_version < '3.13'", specifier = ">=1.26.4" }, - { name = "numpy", marker = "python_full_version >= '3.13'", specifier = ">=2.1.0" }, - { name = "pytest", specifier = ">=9.0.3,<10.0.0" }, - { name = "pytest-asyncio", specifier = ">=1.3.0,<2.0.0" }, - { name = "pytest-benchmark" }, - { name = "pytest-codspeed" }, - { name = "pytest-mock", specifier = ">=3.10.0,<4.0.0" }, - { name = "pytest-socket", specifier = ">=0.7.0,<1.0.0" }, - { name = "pytest-watcher", specifier = ">=0.3.4,<1.0.0" }, - { name = "pytest-xdist", specifier = ">=3.6.1,<4.0.0" }, - { name = "responses", specifier = ">=0.25.0,<1.0.0" }, - { name = "syrupy", specifier = ">=5.0.0,<6.0.0" }, -] -test-integration = [] -typing = [ - { name = "langchain-text-splitters", directory = "../langchain/libs/text-splitters" }, - { name = "mypy", specifier = ">=1.19.1,<1.20.0" }, - { name = "types-pyyaml", specifier = ">=6.0.12.2,<7.0.0.0" }, - { name = "types-requests", specifier = ">=2.28.11.5,<3.0.0.0" }, +sdist = { url = "https://files.pythonhosted.org/packages/59/de/679a53472c25860837e32c0442c962fa86e95317a36460e2c9d5c91b17c2/langchain_core-1.4.0.tar.gz", hash = "sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f", size = 920260, upload-time = "2026-05-11T18:42:35.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/1a/86c38c27b81913a1c6c12448cab55defb5a1097c7dc9a4cea83f55477a2d/langchain_core-1.4.0-py3-none-any.whl", hash = "sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c", size = 548120, upload-time = "2026-05-11T18:42:33.992Z" }, ] [[package]] name = "langchain-diffbot" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "diffbot" }, @@ -620,12 +578,12 @@ typing = [ [package.metadata] requires-dist = [ - { name = "diffbot", specifier = ">=3.0.0" }, + { name = "diffbot", specifier = ">=3.4.0rc0,<4" }, { name = "fastapi", marker = "extra == 'examples'", specifier = ">=0.115,<1.0" }, { name = "httpx", specifier = ">=0.27,<1.0" }, { name = "langchain", marker = "extra == 'examples'", specifier = ">=1.3,<2.0" }, { name = "langchain-anthropic", marker = "extra == 'examples'", specifier = ">=1.4,<2.0" }, - { name = "langchain-core", editable = "../langchain/libs/core" }, + { name = "langchain-core", specifier = ">=1.0,<2.0" }, { name = "langsmith", marker = "extra == 'examples'", specifier = ">=0.1,<1.0" }, { name = "python-dotenv", marker = "extra == 'examples'", specifier = ">=1.0,<2.0" }, { name = "uvicorn", extras = ["standard"], marker = "extra == 'examples'", specifier = ">=0.30,<1.0" }, @@ -636,7 +594,7 @@ provides-extras = ["examples"] lint = [{ name = "ruff", specifier = ">=0.6,<1.0" }] test = [ { name = "langchain", specifier = ">=1.3,<2.0" }, - { name = "langchain-tests", editable = "../langchain/libs/standard-tests" }, + { name = "langchain-tests", specifier = ">=1.0,<2.0" }, { name = "pytest", specifier = ">=8.0,<10.0" }, { name = "pytest-asyncio", specifier = ">=0.23,<2.0" }, { name = "pytest-mock", specifier = ">=3.10,<4.0" }, @@ -659,7 +617,7 @@ wheels = [ [[package]] name = "langchain-tests" version = "1.1.9" -source = { editable = "../langchain/libs/standard-tests" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, { name = "langchain-core" }, @@ -674,31 +632,9 @@ dependencies = [ { name = "syrupy" }, { name = "vcrpy" }, ] - -[package.metadata] -requires-dist = [ - { name = "httpx", specifier = ">=0.28.1,<1.0.0" }, - { name = "langchain-core", editable = "../langchain/libs/core" }, - { name = "numpy", marker = "python_full_version < '3.13'", specifier = ">=1.26.2" }, - { name = "numpy", marker = "python_full_version >= '3.13'", specifier = ">=2.1.0" }, - { name = "pytest", specifier = ">=9.0.3,<10.0.0" }, - { name = "pytest-asyncio", specifier = ">=1.3.0,<2.0.0" }, - { name = "pytest-benchmark" }, - { name = "pytest-codspeed" }, - { name = "pytest-recording" }, - { name = "pytest-socket", specifier = ">=0.7.0,<1.0.0" }, - { name = "syrupy", specifier = ">=5.0.0,<6.0.0" }, - { name = "vcrpy", specifier = ">=8.0.0,<9.0.0" }, -] - -[package.metadata.requires-dev] -lint = [{ name = "ruff", specifier = ">=0.15.0,<0.16.0" }] -test = [{ name = "langchain-core", editable = "../langchain/libs/core" }] -test-integration = [] -typing = [ - { name = "langchain-core", editable = "../langchain/libs/core" }, - { name = "mypy", specifier = ">=1.19.1,<1.20.0" }, - { name = "types-pyyaml", specifier = ">=6.0.12.2,<7.0.0.0" }, +sdist = { url = "https://files.pythonhosted.org/packages/94/0e/6eef4ddcc3503d40de4486abc923a48d5853a310d2b2752123e7a3c713b1/langchain_tests-1.1.9.tar.gz", hash = "sha256:a977a0c8f9a42ba2021a5ae59d468e4023f98a83818a3cb30e12c1dfab830203", size = 184443, upload-time = "2026-05-21T18:24:15.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/bb/ef5ccff7c22938e28716ecf3c4cc468082843df7f3b2214a931cfcaae98a/langchain_tests-1.1.9-py3-none-any.whl", hash = "sha256:f33e36e2015da437df2001ec6949b443de09dd37d021dea6160cf85912ccf05f", size = 66312, upload-time = "2026-05-21T18:24:14.387Z" }, ] [[package]] From 90c3e4e6935974e148d819ab9c06ccd1e52232b2 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 11:58:39 -1000 Subject: [PATCH 12/12] Require diffbot>=3.4.0,<4 (stable) and relock from PyPI. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- Makefile | 2 +- README.md | 2 +- pyproject.toml | 4 ++-- uv.lock | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86994ad..2f1f9af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.3.0 - 2026-11-17 -Require `diffbot>=3.4.0rc0,<4` so CI can verify the SDK release candidate. Switch to `>=3.4.0,<4` for the unison publish. 0.2.0 is already on PyPI, so this release is 0.3.0. +Require `diffbot>=3.4.0,<4`. 0.2.0 is already on PyPI, so this release is 0.3.0. DQL follows the SDK: POST `https://kg.diffbot.com/kg/v3/dql` with a JSON body; the token stays the `token` query parameter. GET remains the low-level escape hatch. diff --git a/CLAUDE.md b/CLAUDE.md index 38dd673..6ca0137 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ LangChain integration package for the Diffbot APIs (Knowledge Graph, Extract, We ## Stack - **Build/deps**: `uv` (not Poetry/pip). `pyproject.toml` is PEP 621 with `hatchling` as the build backend. Dependency groups (`test`, `lint`, `typing`) are declared under `[dependency-groups]` and invoked via `uv run --group ...`. -- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.4.0rc0,<4` on this verification branch (switch to `>=3.4.0,<4` for the unison publish). `make dev-sdk` can still install a sibling checkout over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Do not commit that editable install. Relock with `uv lock --upgrade-package diffbot` (that command needs the sibling `../langchain` checkout, or use `--no-sources`). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. +- **HTTP / Diffbot transport**: the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. It wraps `httpx` under the hood, so `respx` continues to work for unit tests — mocks target the real upstream URLs (KG: `POST https://kg.diffbot.com/kg/v3/dql` with a JSON body, web search: `https://llm.diffbot.com/api/v1/web_search`, extract: `https://api.diffbot.com/v3/analyze`, ask: `https://llm.diffbot.com/rag/v1/chat/completions`, NLP: `https://nl.diffbot.com/v1/`, crawl: `https://api.diffbot.com/v3/crawl`, ontology: `https://kg.diffbot.com/kg/ontology`). It is published on PyPI as `diffbot` and declared as `diffbot>=3.4.0,<4`. Until 3.4.0 is on PyPI, `make dev-sdk` can install a sibling checkout over the synced env; run the `make` targets with `UV_NO_SYNC=true` so `uv run` doesn't re-resolve and replace it. Do not commit that editable install. Relock with `uv lock --upgrade-package diffbot` once 3.4.0 is published (that command needs the sibling `../langchain` checkout). `langchain-core` / `langchain-tests` are still resolved during local dev from the sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Models**: `pydantic` v2. - **LangChain**: targets `langchain-core >=1.0,<2.0`. During local dev `langchain-core` and `langchain-tests` are resolved from a sibling `../langchain/` checkout via `[tool.uv.sources]` — release builds use the published versions. - **Test runner**: `pytest` with `asyncio_mode = "auto"`. diff --git a/Makefile b/Makefile index c1a674d..446098f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: dev-sdk format lint test test_integration typing build clean bump-patch bump-minor bump-major set-token-pypi set-token-testpypi release-test release verify-release-test verify-release # Dev-only: install a local Diffbot SDK checkout over the synced env (no-op if -# ../diffbot-python is absent). The current verification dependency is diffbot>=3.4.0rc0,<4 from +# ../diffbot-python is absent). The release dependency is diffbot>=3.4.0,<4 from # PyPI. Pair with UV_NO_SYNC=true so `make test` doesn't re-resolve and replace it. # Do not commit the editable install. dev-sdk: diff --git a/README.md b/README.md index d47a5d5..001c08d 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ for doc in loader.lazy_load(): `DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`. -Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0rc0,<4` (switch to `>=3.4.0,<4` for the unison publish). +Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`. ## Crawl a site diff --git a/pyproject.toml b/pyproject.toml index 6e5d171..3050c8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,10 @@ requires-python = ">=3.10,<4.0" dependencies = [ "langchain-core>=1.0,<2.0", "httpx>=0.27,<1.0", - # Relock after the SDK RC/stable is on PyPI: `uv lock --upgrade-package diffbot` + # Relock after 3.4.0 is on PyPI: `uv lock --upgrade-package diffbot` # (needs the sibling ../langchain checkout). `make dev-sdk` is only for a # pre-publish local install; do not commit that path. - "diffbot>=3.4.0rc0,<4", + "diffbot>=3.4.0,<4", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index bc6afbe..86e60b5 100644 --- a/uv.lock +++ b/uv.lock @@ -205,16 +205,16 @@ wheels = [ [[package]] name = "diffbot" -version = "3.4.0rc0" +version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "httpx" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/33/db23518d8d602bee409e3dc50c42d17d6b432c580298a6a5587c8cdfdce4/diffbot-3.4.0rc0.tar.gz", hash = "sha256:5866529c085c2ce3064153bf135b40b5cd97f493d536d5415acc5dab18487256", size = 65653, upload-time = "2026-11-18T04:02:24.561Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/e9/479fd0f96144405f9fdce380c8fb933bfb7010a198d4f0c2a3e37e7e7d1c/diffbot-3.4.0.tar.gz", hash = "sha256:ce5ac1e5d896be79e5e46fcedf451e725b129bba45a88d82b44f151c6d538dc3", size = 65650, upload-time = "2026-11-18T21:56:46.012Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/c3/5fc7df5d9f8fd309f386f425a8bee9fcb07722a216acc5bdaf20c7eab7b2/diffbot-3.4.0rc0-py3-none-any.whl", hash = "sha256:5cd55605f864b19e3ac0e3b9d335bbb95b2038a5eadd8dbd274668087d09daa1", size = 54080, upload-time = "2026-11-18T04:02:23.241Z" }, + { url = "https://files.pythonhosted.org/packages/39/d6/af72e476093aafb3604591b597d86c21278ba109e8f71322e4d252c74c78/diffbot-3.4.0-py3-none-any.whl", hash = "sha256:bfc72bb50a9f1e885f04b823ed84436a7cad9802f2d6b1cb1e03edd373025057", size = 54041, upload-time = "2026-11-18T21:56:44.608Z" }, ] [[package]] @@ -578,7 +578,7 @@ typing = [ [package.metadata] requires-dist = [ - { name = "diffbot", specifier = ">=3.4.0rc0,<4" }, + { name = "diffbot", specifier = ">=3.4.0,<4" }, { name = "fastapi", marker = "extra == 'examples'", specifier = ">=0.115,<1.0" }, { name = "httpx", specifier = ">=0.27,<1.0" }, { name = "langchain", marker = "extra == 'examples'", specifier = ">=1.3,<2.0" },