From a6059451f5a6373d845407a136f1093b48364fc2 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Fri, 13 Nov 2026 16:09:44 -1000 Subject: [PATCH 01/31] =?UTF-8?q?feat(transport):=20Phase=200=20foundation?= =?UTF-8?q?=20=E2=80=94=20per-host=20bases,=20full=20HTTP=20capability=20s?= =?UTF-8?q?et,=20retry=20policy,=20and=20error=20hierarchy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the TypeScript SDK's Phase 0 transport work (same branch) in Python idiom: - Shared sync/async request-shaping and response-handling helpers in a new _transport.py, replacing the duplicated Diffbot/DiffbotAsync._raise_for_status methods and the repeated client._http.get(...) + raise pattern across every handler module with a single client.request(method, url, ...) used by both. - Per-host base URLs (api_base/kg_base/llm_base/nlp_base) with the existing per-endpoint *_url constructor options kept as winning aliases (additive). No standalone "endpoint registry" data structure — Phase 0 adds zero new endpoints, so the only thing it would serve today is base-URL resolution (done directly in client.py) and a retryable flag (done ad hoc), matching the TypeScript SDK's same call on this branch (see its PHASE0_LEFTOVERS.md). - Repeated query parameters (declared via a Sequence-typed QueryParams; httpx already encodes list-valued params as repeated keys). - DELETE + arbitrary method/body/Content-Type, form-encoded (data=) and raw (content=) request bodies, per-call timeout override, and a follow_redirects override, all threaded through client.request(). - A default-on retry policy (RetryPolicy dataclass) for 429/5xx/connection errors with backoff, jitter, and Retry-After support, capped at 3 attempts; disable client-wide with retry=False or per call with retryable=False. crawl_delete_job/_async now pass retryable=False since it's a GET-shaped mutation that must not be retried blindly. - Response headers surfaced on APIError (and subclasses) via a new headers field, populated from the error response. - NotFoundError (404) and JobError, both subclassing APIError so existing `except APIError` call sites are unaffected. crawl's job-failure path now raises JobError instead of a synthesized APIError(500, ...). - fix(ask): read the response body before raising on a non-2xx streaming response (sync and async). Previously client._raise_for_status accessed response.text immediately after opening the stream, which raises httpx.ResponseNotRead instead of AuthError/RateLimitError once the connection is genuinely still streaming (tests use a real byte-stream mock transport, since httpx.MockTransport's content= pre-fills the body and masks this). ask()/ask_json() now go through client.request(..., stream=True), which also means they pick up the retry policy like every other handler. - web_search.py's sync/async bodies, previously 100% duplicated, now share a single _build_web_search_request() helper. Also adds additive TypedDict/Literal type aliases (types.py) for existing surfaces only — ChatMessage, ResponseFormat, ExportFormat, SnippetLength — used in ask()/dql()/web_search() signatures. No Pydantic, no runtime behavior change; Python floor stays >=3.9 (stdlib typing only). Co-Authored-By: Claude Sonnet 5 --- src/diffbot/__init__.py | 11 ++ src/diffbot/_constants.py | 12 ++ src/diffbot/_transport.py | 207 ++++++++++++++++++++++++++++++++++ src/diffbot/ask.py | 41 ++++--- src/diffbot/client.py | 211 ++++++++++++++++++++++------------ src/diffbot/crawl.py | 54 ++++----- src/diffbot/errors.py | 32 +++++- src/diffbot/extract.py | 8 +- src/diffbot/kg.py | 27 ++--- src/diffbot/nlp.py | 10 +- src/diffbot/types.py | 43 +++++++ src/diffbot/web_search.py | 52 ++++----- tests/test_ask.py | 94 +++++++++++++++- tests/test_client.py | 61 ++++++++++ tests/test_crawl.py | 72 +++++++++++- tests/test_dql.py | 18 ++- tests/test_transport.py | 231 ++++++++++++++++++++++++++++++++++++++ 17 files changed, 1012 insertions(+), 172 deletions(-) create mode 100644 src/diffbot/_constants.py create mode 100644 src/diffbot/_transport.py create mode 100644 src/diffbot/types.py create mode 100644 tests/test_client.py create mode 100644 tests/test_transport.py diff --git a/src/diffbot/__init__.py b/src/diffbot/__init__.py index d35d41b..db7c900 100644 --- a/src/diffbot/__init__.py +++ b/src/diffbot/__init__.py @@ -41,10 +41,14 @@ def _installed_version(dist: str): AuthError, DiffbotError, ExtractionError, + JobError, + NotFoundError, RateLimitError, ValidationError, ) from .ontology import Ontology +from ._transport import RetryPolicy +from .types import ChatMessage, ExportFormat, ResponseFormat, SnippetLength __all__ = [ "Diffbot", @@ -54,9 +58,16 @@ def _installed_version(dist: str): "CrawlEvent", "CrawlEventType", "Ontology", + "RetryPolicy", + "ChatMessage", + "ResponseFormat", + "ExportFormat", + "SnippetLength", "DiffbotError", "AuthError", "ExtractionError", + "JobError", + "NotFoundError", "RateLimitError", "APIError", "ValidationError", diff --git a/src/diffbot/_constants.py b/src/diffbot/_constants.py new file mode 100644 index 0000000..a930a42 --- /dev/null +++ b/src/diffbot/_constants.py @@ -0,0 +1,12 @@ +"""Per-host base URLs and shared transport defaults. + +Overridable via ``api_base``/``kg_base``/``llm_base``/``nlp_base`` on the client +constructors; the older per-endpoint ``*_url`` options still win over these when set. +""" + +API_BASE = "https://api.diffbot.com" +KG_BASE = "https://kg.diffbot.com" +LLM_BASE = "https://llm.diffbot.com" +NLP_HOST_BASE = "https://nl.diffbot.com" + +DEFAULT_TIMEOUT = 30.0 diff --git a/src/diffbot/_transport.py b/src/diffbot/_transport.py new file mode 100644 index 0000000..de1d683 --- /dev/null +++ b/src/diffbot/_transport.py @@ -0,0 +1,207 @@ +"""Shared request-shaping and response-handling helpers for the sync and async clients. + +Both ``Diffbot`` and ``DiffbotAsync`` delegate here so the two don't carry duplicate +copies of retry/timeout/error-raising logic; the only difference between them is +whether the underlying ``httpx.Client``/``httpx.AsyncClient`` call is awaited. +""" + +import asyncio +import random +import time +from dataclasses import dataclass +from email.utils import parsedate_to_datetime +from typing import Any, Dict, Optional, Sequence, Union + +import httpx + +from .errors import APIError, AuthError, NotFoundError, RateLimitError + +QueryParamValue = Union[str, int, float, bool] +QueryParams = Dict[str, Union[QueryParamValue, Sequence[QueryParamValue], None]] + + +@dataclass +class RetryPolicy: + """Retry policy for transient failures (429, 5xx, connection errors). + + ``max_attempts`` counts the initial try, so ``3`` means up to 2 retries. Disable + per client with ``retry=False`` on the client constructor; disable per call with + ``retryable=False`` — required for GET-shaped mutations (e.g. ``crawl?delete=1``) + that must never be retried blindly. + """ + + max_attempts: int = 3 + base_delay: float = 0.25 + max_delay: float = 5.0 + jitter: bool = True + + +DEFAULT_RETRY_POLICY = RetryPolicy() + + +def resolve_retry_policy(retry: Union[RetryPolicy, bool, None]) -> Optional[RetryPolicy]: + """``None``/``True`` -> default policy (retries on by default); ``False`` -> disabled.""" + if retry is False: + return None + if retry is None or retry is True: + return DEFAULT_RETRY_POLICY + return retry + + +def _is_retryable_status(status_code: int) -> bool: + return status_code == 429 or status_code >= 500 + + +def _compute_delay(attempt: int, retry_after: Optional[str], policy: RetryPolicy) -> float: + """Parses ``Retry-After`` as seconds or an HTTP date; falls back to exponential backoff.""" + if retry_after: + try: + return max(0.0, float(retry_after)) + except ValueError: + pass + try: + dt = parsedate_to_datetime(retry_after) + return max(0.0, dt.timestamp() - time.time()) + except (TypeError, ValueError): + pass + exponential = min(policy.max_delay, policy.base_delay * (2 ** (attempt - 1))) + return exponential * (0.5 + random.random() * 0.5) if policy.jitter else exponential + + +def raise_for_status(response: httpx.Response) -> None: + if response.is_success: + return + status = response.status_code + body = response.text + headers = dict(response.headers) + if status in (401, 403): + raise AuthError(status, body, headers=headers) + if status == 404: + raise NotFoundError(status, body, headers=headers) + if status == 429: + raise RateLimitError(status, body, retry_after=response.headers.get("retry-after"), headers=headers) + raise APIError(status, body, headers=headers) + + +def _build_kwargs( + params: Optional[QueryParams], + headers: Optional[Dict[str, str]], + json: Optional[Any], + content: Optional[Union[str, bytes]], + data: Optional[Dict[str, Any]], + timeout: Optional[float], +) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {} + if params is not None: + kwargs["params"] = params + if headers is not None: + kwargs["headers"] = headers + if json is not None: + kwargs["json"] = json + if content is not None: + kwargs["content"] = content + if data is not None: + kwargs["data"] = data + if timeout is not None: + kwargs["timeout"] = timeout + return kwargs + + +def request( + http: httpx.Client, + retry_policy: Optional[RetryPolicy], + method: str, + url: str, + *, + params: Optional[QueryParams] = None, + headers: Optional[Dict[str, str]] = None, + json: Optional[Any] = None, + content: Optional[Union[str, bytes]] = None, + data: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + retryable: bool = True, + follow_redirects: Optional[bool] = None, + stream: bool = False, +) -> httpx.Response: + """Send a request, retrying on 429/5xx/connection errors per ``retry_policy``, then + raise for a non-2xx status. The request is rebuilt fresh on each attempt so a + request body is never resent from an already-consumed stream. + + With ``stream=True`` the response body is left unread on success (the caller + must iterate it and call ``.close()``), but is still read before raising on a + non-2xx status — without that, accessing the error body (e.g. via + ``RateLimitError.message``) would raise ``httpx.ResponseNotRead``. + """ + kwargs = _build_kwargs(params, headers, json, content, data, timeout) + send_kwargs: Dict[str, Any] = {} if follow_redirects is None else {"follow_redirects": follow_redirects} + if stream: + send_kwargs["stream"] = True + policy = retry_policy if retryable else None + max_attempts = policy.max_attempts if policy else 1 + attempt = 1 + while True: + req = http.build_request(method, url, **kwargs) + try: + response = http.send(req, **send_kwargs) + except httpx.TransportError: + if policy and attempt < max_attempts: + time.sleep(_compute_delay(attempt, None, policy)) + attempt += 1 + continue + raise + if policy and attempt < max_attempts and _is_retryable_status(response.status_code): + if stream: + response.close() + time.sleep(_compute_delay(attempt, response.headers.get("retry-after"), policy)) + attempt += 1 + continue + if response.is_success: + return response + response.read() + raise_for_status(response) + + +async def request_async( + http: httpx.AsyncClient, + retry_policy: Optional[RetryPolicy], + method: str, + url: str, + *, + params: Optional[QueryParams] = None, + headers: Optional[Dict[str, str]] = None, + json: Optional[Any] = None, + content: Optional[Union[str, bytes]] = None, + data: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + retryable: bool = True, + follow_redirects: Optional[bool] = None, + stream: bool = False, +) -> httpx.Response: + """Async twin of :func:`request`.""" + kwargs = _build_kwargs(params, headers, json, content, data, timeout) + send_kwargs: Dict[str, Any] = {} if follow_redirects is None else {"follow_redirects": follow_redirects} + if stream: + send_kwargs["stream"] = True + policy = retry_policy if retryable else None + max_attempts = policy.max_attempts if policy else 1 + attempt = 1 + while True: + req = http.build_request(method, url, **kwargs) + try: + response = await http.send(req, **send_kwargs) + except httpx.TransportError: + if policy and attempt < max_attempts: + await asyncio.sleep(_compute_delay(attempt, None, policy)) + attempt += 1 + continue + raise + if policy and attempt < max_attempts and _is_retryable_status(response.status_code): + if stream: + await response.aclose() + await asyncio.sleep(_compute_delay(attempt, response.headers.get("retry-after"), policy)) + attempt += 1 + continue + if response.is_success: + return response + await response.aread() + raise_for_status(response) diff --git a/src/diffbot/ask.py b/src/diffbot/ask.py index 727ad25..917fbe7 100644 --- a/src/diffbot/ask.py +++ b/src/diffbot/ask.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional from .errors import ValidationError +from .types import ChatMessage, ResponseFormat if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -20,7 +21,7 @@ _JSON_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE) -def json_schema_format(schema: Dict[str, Any], *, name: str = "response") -> Dict[str, Any]: +def json_schema_format(schema: Dict[str, Any], *, name: str = "response") -> ResponseFormat: """Build a ``response_format`` value that constrains output to ``schema``. The endpoint requires the schema nested under ``json_schema.schema``; passing @@ -36,7 +37,7 @@ def json_schema_format(schema: Dict[str, Any], *, name: str = "response") -> Dic return {"type": "json_schema", "json_schema": {"name": name, "schema": schema}} -def _validate_response_format(response_format: Optional[Dict[str, Any]]) -> None: +def _validate_response_format(response_format: Optional[ResponseFormat]) -> None: """Reject shapes the endpoint would accept but silently not enforce.""" if response_format is None: return @@ -61,9 +62,9 @@ def _validate_response_format(response_format: Optional[Dict[str, Any]]) -> None def _build_payload( client: Any, - messages: List[Dict[str, str]], + messages: List[ChatMessage], *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> tuple: _validate_response_format(response_format) headers = {"Authorization": f"Bearer {client.token}"} @@ -119,8 +120,8 @@ def _extract_json(text: str) -> Any: def _resolve_format( schema: Optional[Dict[str, Any]], - response_format: Optional[Dict[str, Any]], -) -> Dict[str, Any]: + response_format: Optional[ResponseFormat], +) -> ResponseFormat: if schema is not None and response_format is not None: raise ValidationError("pass either schema or response_format, not both") if response_format is not None: @@ -141,42 +142,46 @@ def _check_tool_call_leak(parsed: Any) -> Any: def ask( client: "Diffbot", - messages: List[Dict[str, str]], + messages: List[ChatMessage], *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> Iterator[str]: headers, payload = _build_payload(client, messages, response_format=response_format) - with client._http.stream("POST", client.llm_url, headers=headers, json=payload) as response: - client._raise_for_status(response) + response = client.request("POST", client.llm_url, headers=headers, json=payload, stream=True) + try: for line in response.iter_lines(): if line: content = _parse_chunk(line) if content: yield content + finally: + response.close() async def ask_async( client: "DiffbotAsync", - messages: List[Dict[str, str]], + messages: List[ChatMessage], *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> AsyncIterator[str]: headers, payload = _build_payload(client, messages, response_format=response_format) - async with client._http.stream("POST", client.llm_url, headers=headers, json=payload) as response: - client._raise_for_status(response) + response = await client.request("POST", client.llm_url, headers=headers, json=payload, stream=True) + try: async for line in response.aiter_lines(): if line: content = _parse_chunk(line) if content: yield content + finally: + await response.aclose() def ask_json( client: "Diffbot", - messages: List[Dict[str, str]], + messages: List[ChatMessage], schema: Optional[Dict[str, Any]] = None, *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> Any: fmt = _resolve_format(schema, response_format) text = "".join(ask(client, messages, response_format=fmt)) @@ -185,10 +190,10 @@ def ask_json( async def ask_json_async( client: "DiffbotAsync", - messages: List[Dict[str, str]], + messages: List[ChatMessage], schema: Optional[Dict[str, Any]] = None, *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> Any: fmt = _resolve_format(schema, response_format) chunks = [chunk async for chunk in ask_async(client, messages, response_format=fmt)] diff --git a/src/diffbot/client.py b/src/diffbot/client.py index d098650..a83c182 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -7,7 +7,16 @@ import httpx from . import __version__ -from .errors import APIError, AuthError, RateLimitError, ValidationError +from ._constants import API_BASE, DEFAULT_TIMEOUT, KG_BASE, LLM_BASE, NLP_HOST_BASE +from ._transport import ( + QueryParams, + RetryPolicy, + request as _transport_request, + request_async as _transport_request_async, + resolve_retry_policy, +) +from .errors import ValidationError +from .types import ChatMessage, ExportFormat, ResponseFormat, SnippetLength from .extract import extract as _extract, extract_async as _extract_async from .ask import ( ask as _ask, @@ -27,8 +36,6 @@ crawl_list_jobs_async as _crawl_list_jobs_async, ) from .kg import ( - KG_DQL_ENDPOINT, - KG_ONTOLOGY_ENDPOINT, dql as _dql, dql_async as _dql_async, dql_fetch_ontology as _dql_fetch_ontology, @@ -39,21 +46,14 @@ ) from .ontology import Ontology from .web_search import ( - WEB_SEARCH_BASE, web_search as _web_search, web_search_async as _web_search_async, ) from .nlp import ( - NLP_BASE, entities as _entities, entities_async as _entities_async, ) -EXTRACT_BASE = "https://api.diffbot.com/v3" -CRAWL_BASE = "https://api.diffbot.com/v3/crawl" -DIFFBOT_LLM_BASE = "https://llm.diffbot.com/rag/v1/chat/completions" -DEFAULT_TIMEOUT = 30.0 - class Diffbot: """Client for the Diffbot APIs. @@ -69,25 +69,39 @@ def __init__( token: str, *, timeout: float = DEFAULT_TIMEOUT, - analyze_url: str = EXTRACT_BASE, - llm_url: str = DIFFBOT_LLM_BASE, - crawler_url: str = CRAWL_BASE, - web_search_url: str = WEB_SEARCH_BASE, - nlp_url: str = NLP_BASE, - dql_url: str = KG_DQL_ENDPOINT, - ontology_url: str = KG_ONTOLOGY_ENDPOINT, + retry: Union[RetryPolicy, bool, None] = None, + api_base: str = API_BASE, + kg_base: str = KG_BASE, + llm_base: str = LLM_BASE, + nlp_base: str = NLP_HOST_BASE, + analyze_url: Optional[str] = None, + llm_url: Optional[str] = None, + crawler_url: Optional[str] = None, + web_search_url: Optional[str] = None, + nlp_url: Optional[str] = None, + dql_url: Optional[str] = None, + ontology_url: Optional[str] = None, transport: Optional[httpx.BaseTransport] = None, ): if not token: raise ValidationError("token is required") self.token = token - self.analyze_url = analyze_url - self.llm_url = llm_url - self.crawler_url = crawler_url - self.web_search_url = web_search_url - self.nlp_url = nlp_url - self.dql_url = dql_url - self.ontology_url = ontology_url + self.api_base = api_base + self.kg_base = kg_base + self.llm_base = llm_base + self.nlp_base = nlp_base + # analyze_url/crawler_url are bases the SDK appends to (/{api}, /data); the + # rest are complete endpoints. Each *_url option wins over its matching base + # when set, so pointing the whole SDK at a staging host is a one-liner while + # still allowing a single endpoint to be overridden independently. + self.analyze_url = analyze_url or f"{api_base}/v3" + self.crawler_url = crawler_url or f"{api_base}/v3/crawl" + self.llm_url = llm_url or f"{llm_base}/rag/v1/chat/completions" + self.web_search_url = web_search_url or f"{llm_base}/api/v1/web_search" + self.nlp_url = nlp_url or f"{nlp_base}/v1/" + self.dql_url = dql_url or f"{kg_base}/kg/v3/dql" + self.ontology_url = ontology_url or f"{kg_base}/kg/ontology" + self._retry_policy = resolve_retry_policy(retry) self._http = httpx.Client( timeout=timeout, headers={"User-Agent": f"diffbot-python/{__version__}"}, @@ -108,16 +122,42 @@ def __exit__( def close(self) -> None: self._http.close() - def _raise_for_status(self, response: httpx.Response) -> None: - if response.is_success: - return - status = response.status_code - body = response.text - if status in (401, 403): - raise AuthError(status, body) - if status == 429: - raise RateLimitError(status, body, retry_after=response.headers.get("retry-after")) - raise APIError(status, body) + def request( + self, + method: str, + url: str, + *, + params: Optional[QueryParams] = None, + headers: Optional[Dict[str, str]] = None, + json: Optional[Any] = None, + content: Optional[Union[str, bytes]] = None, + data: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + retryable: bool = True, + follow_redirects: Optional[bool] = None, + stream: bool = False, + ) -> httpx.Response: + """Send a request through this client's transport (retry policy, timeout) + and raise for a non-2xx status. Returns the raw :class:`httpx.Response`. + + With ``stream=True`` the response body is left unread on success — the + caller must iterate it and call ``.close()``. + """ + return _transport_request( + self._http, + self._retry_policy, + method, + url, + params=params, + headers=headers, + json=json, + content=content, + data=data, + timeout=timeout, + retryable=retryable, + follow_redirects=follow_redirects, + stream=stream, + ) def extract(self, url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: """Extract structured content from a URL. Returns the raw Diffbot API response.""" @@ -125,9 +165,9 @@ def extract(self, url: str, api: str = "analyze", fmt: str = "markdown") -> Dict def ask( self, - messages: List[Dict[str, str]], + messages: List[ChatMessage], *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> Iterator[str]: """Stream a response from the Diffbot LLM RAG API. @@ -138,10 +178,10 @@ def ask( def ask_json( self, - messages: List[Dict[str, str]], + messages: List[ChatMessage], schema: Optional[Dict[str, Any]] = None, *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> Any: """Ask the Diffbot LLM and return the answer parsed as JSON. @@ -172,7 +212,7 @@ def dql( *, size: int = 10, from_: int = 0, - format: str = "json", + format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, extra: Optional[Dict[str, str]] = None, @@ -203,7 +243,7 @@ def web_search( *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, - snippet_length: Optional[str] = None, + snippet_length: Optional[SnippetLength] = None, ) -> Dict[str, Any]: """Search the web via the Diffbot LLM web search API. @@ -236,25 +276,35 @@ def __init__( token: str, *, timeout: float = DEFAULT_TIMEOUT, - analyze_url: str = EXTRACT_BASE, - llm_url: str = DIFFBOT_LLM_BASE, - crawler_url: str = CRAWL_BASE, - web_search_url: str = WEB_SEARCH_BASE, - nlp_url: str = NLP_BASE, - dql_url: str = KG_DQL_ENDPOINT, - ontology_url: str = KG_ONTOLOGY_ENDPOINT, + retry: Union[RetryPolicy, bool, None] = None, + api_base: str = API_BASE, + kg_base: str = KG_BASE, + llm_base: str = LLM_BASE, + nlp_base: str = NLP_HOST_BASE, + analyze_url: Optional[str] = None, + llm_url: Optional[str] = None, + crawler_url: Optional[str] = None, + web_search_url: Optional[str] = None, + nlp_url: Optional[str] = None, + dql_url: Optional[str] = None, + ontology_url: Optional[str] = None, transport: Optional[httpx.AsyncBaseTransport] = None, ): if not token: raise ValidationError("token is required") self.token = token - self.analyze_url = analyze_url - self.llm_url = llm_url - self.crawler_url = crawler_url - self.web_search_url = web_search_url - self.nlp_url = nlp_url - self.dql_url = dql_url - self.ontology_url = ontology_url + self.api_base = api_base + self.kg_base = kg_base + self.llm_base = llm_base + self.nlp_base = nlp_base + self.analyze_url = analyze_url or f"{api_base}/v3" + self.crawler_url = crawler_url or f"{api_base}/v3/crawl" + self.llm_url = llm_url or f"{llm_base}/rag/v1/chat/completions" + self.web_search_url = web_search_url or f"{llm_base}/api/v1/web_search" + self.nlp_url = nlp_url or f"{nlp_base}/v1/" + self.dql_url = dql_url or f"{kg_base}/kg/v3/dql" + self.ontology_url = ontology_url or f"{kg_base}/kg/ontology" + self._retry_policy = resolve_retry_policy(retry) self._http = httpx.AsyncClient( timeout=timeout, headers={"User-Agent": f"diffbot-python/{__version__}"}, @@ -275,16 +325,37 @@ async def __aexit__( async def close(self) -> None: await self._http.aclose() - def _raise_for_status(self, response: httpx.Response) -> None: - if response.is_success: - return - status = response.status_code - body = response.text - if status in (401, 403): - raise AuthError(status, body) - if status == 429: - raise RateLimitError(status, body, retry_after=response.headers.get("retry-after")) - raise APIError(status, body) + async def request( + self, + method: str, + url: str, + *, + params: Optional[QueryParams] = None, + headers: Optional[Dict[str, str]] = None, + json: Optional[Any] = None, + content: Optional[Union[str, bytes]] = None, + data: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + retryable: bool = True, + follow_redirects: Optional[bool] = None, + stream: bool = False, + ) -> httpx.Response: + """Async twin of :meth:`Diffbot.request`.""" + return await _transport_request_async( + self._http, + self._retry_policy, + method, + url, + params=params, + headers=headers, + json=json, + content=content, + data=data, + timeout=timeout, + retryable=retryable, + follow_redirects=follow_redirects, + stream=stream, + ) async def extract(self, url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: """Extract structured content from a URL. Returns the raw Diffbot API response.""" @@ -292,9 +363,9 @@ async def extract(self, url: str, api: str = "analyze", fmt: str = "markdown") - async def ask( self, - messages: List[Dict[str, str]], + messages: List[ChatMessage], *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> AsyncIterator[str]: """Stream a response from the Diffbot LLM RAG API. @@ -306,10 +377,10 @@ async def ask( async def ask_json( self, - messages: List[Dict[str, str]], + messages: List[ChatMessage], schema: Optional[Dict[str, Any]] = None, *, - response_format: Optional[Dict[str, Any]] = None, + response_format: Optional[ResponseFormat] = None, ) -> Any: """Ask the Diffbot LLM and return the answer parsed as JSON. @@ -341,7 +412,7 @@ async def dql( *, size: int = 10, from_: int = 0, - format: str = "json", + format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, extra: Optional[Dict[str, str]] = None, @@ -368,7 +439,7 @@ async def web_search( *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, - snippet_length: Optional[str] = None, + snippet_length: Optional[SnippetLength] = None, ) -> Dict[str, Any]: """Search the web via the Diffbot LLM web search API. diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index 4fb9e10..c543a24 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -9,7 +9,7 @@ from enum import Enum from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional -from .errors import APIError +from .errors import JobError if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -58,8 +58,7 @@ def crawl( api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, ) - response = client._http.get(client.crawler_url, params=params) - client._raise_for_status(response) + client.request("GET", client.crawler_url, params=params) yield CrawlEvent( event_type=CrawlEventType.JOB_CREATED, @@ -73,23 +72,23 @@ def crawl( seen_urls: set = set() while True: - status_response = client._http.get( + status_response = client.request( + "GET", client.crawler_url, params={"token": client.token, "name": job_name}, ) - client._raise_for_status(status_response) jobs = status_response.json().get("jobs", []) if not jobs: break job_status = jobs[0].get("jobStatus", {}) status_code = job_status.get("status", 0) - urls_response = client._http.get( + urls_response = client.request( + "GET", f"{client.crawler_url}/data", params={"token": client.token, "name": job_name, "type": "urls"}, follow_redirects=True, ) - client._raise_for_status(urls_response) if urls_response.content: for event in _parse_url_csv(urls_response.text, seen_urls): yield event @@ -97,7 +96,7 @@ def crawl( if status_code not in (0, 7): message = job_status.get("message", "") if "fail" in message.lower() or "error" in message.lower(): - raise APIError(500, f"Crawler job failed: {message}") + raise JobError(job_name, f"Crawler job failed: {message}") break time.sleep(poll_interval) @@ -177,8 +176,7 @@ async def crawl_async( api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, ) - response = await client._http.get(client.crawler_url, params=params) - client._raise_for_status(response) + await client.request("GET", client.crawler_url, params=params) yield CrawlEvent( event_type=CrawlEventType.JOB_CREATED, @@ -192,23 +190,23 @@ async def crawl_async( seen_urls: set = set() while True: - status_response = await client._http.get( + status_response = await client.request( + "GET", client.crawler_url, params={"token": client.token, "name": job_name}, ) - client._raise_for_status(status_response) jobs = status_response.json().get("jobs", []) if not jobs: break job_status = jobs[0].get("jobStatus", {}) status_code = job_status.get("status", 0) - urls_response = await client._http.get( + urls_response = await client.request( + "GET", f"{client.crawler_url}/data", params={"token": client.token, "name": job_name, "type": "urls"}, follow_redirects=True, ) - client._raise_for_status(urls_response) if urls_response.content: for event in _parse_url_csv(urls_response.text, seen_urls): yield event @@ -216,55 +214,57 @@ async def crawl_async( if status_code not in (0, 7): message = job_status.get("message", "") if "fail" in message.lower() or "error" in message.lower(): - raise APIError(500, f"Crawler job failed: {message}") + raise JobError(job_name, f"Crawler job failed: {message}") break await asyncio.sleep(poll_interval) async def crawl_list_jobs_async(client: "DiffbotAsync") -> List[Dict[str, Any]]: - response = await client._http.get(client.crawler_url, params={"token": client.token}) - client._raise_for_status(response) + response = await client.request("GET", client.crawler_url, params={"token": client.token}) return response.json().get("jobs", []) async def crawl_get_job_async(client: "DiffbotAsync", job_name: str) -> Dict[str, Any]: - response = await client._http.get( + response = await client.request( + "GET", client.crawler_url, params={"token": client.token, "name": job_name}, ) - client._raise_for_status(response) jobs = response.json().get("jobs", []) return jobs[0] if jobs else {} async def crawl_delete_job_async(client: "DiffbotAsync", job_name: str) -> None: - response = await client._http.get( + # A GET-shaped mutation: retrying it blindly risks issuing the delete twice. + await client.request( + "GET", client.crawler_url, params={"token": client.token, "name": job_name, "delete": 1}, + retryable=False, ) - client._raise_for_status(response) def crawl_list_jobs(client: "Diffbot") -> List[Dict[str, Any]]: - response = client._http.get(client.crawler_url, params={"token": client.token}) - client._raise_for_status(response) + response = client.request("GET", client.crawler_url, params={"token": client.token}) return response.json().get("jobs", []) def crawl_get_job(client: "Diffbot", job_name: str) -> Dict[str, Any]: - response = client._http.get( + response = client.request( + "GET", client.crawler_url, params={"token": client.token, "name": job_name}, ) - client._raise_for_status(response) jobs = response.json().get("jobs", []) return jobs[0] if jobs else {} def crawl_delete_job(client: "Diffbot", job_name: str) -> None: - response = client._http.get( + # A GET-shaped mutation: retrying it blindly risks issuing the delete twice. + client.request( + "GET", client.crawler_url, params={"token": client.token, "name": job_name, "delete": 1}, + retryable=False, ) - client._raise_for_status(response) diff --git a/src/diffbot/errors.py b/src/diffbot/errors.py index c21873e..30cc9de 100644 --- a/src/diffbot/errors.py +++ b/src/diffbot/errors.py @@ -1,7 +1,7 @@ """Exception hierarchy for the Diffbot SDK.""" import json -from typing import Optional +from typing import Dict, Optional class DiffbotError(Exception): @@ -15,9 +15,10 @@ class ValidationError(DiffbotError): class APIError(DiffbotError): """The Diffbot API returned an error response.""" - def __init__(self, status_code: int, body: str): + def __init__(self, status_code: int, body: str, headers: Optional[Dict[str, str]] = None): self.status_code = status_code self.body = body + self.headers = headers self.message: Optional[str] = None self.request_id: Optional[str] = None try: @@ -37,11 +38,34 @@ class AuthError(APIError): class RateLimitError(APIError): """Rate limit exceeded (429).""" - def __init__(self, status_code: int, body: str, retry_after: Optional[str] = None): - super().__init__(status_code, body) + def __init__( + self, + status_code: int, + body: str, + retry_after: Optional[str] = None, + headers: Optional[Dict[str, str]] = None, + ): + super().__init__(status_code, body, headers=headers) self.retry_after = retry_after +class NotFoundError(APIError): + """A 404 from an endpoint with a documented "not found" shape (e.g. a missing job or report).""" + + +class JobError(APIError): + """A job (crawl or bulk) that completed its HTTP request successfully but reports a + failed status internally. + + Subclasses :class:`APIError` so existing ``except APIError`` sites keep working; + ``isinstance(err, JobError)`` distinguishes it from a transport-level failure. + """ + + def __init__(self, job_name: str, message: str): + super().__init__(500, message) + self.job_name = job_name + + class ExtractionError(DiffbotError): """The Diffbot API returned a 200 but reported an extraction failure.""" diff --git a/src/diffbot/extract.py b/src/diffbot/extract.py index ffef455..7a4c2f3 100644 --- a/src/diffbot/extract.py +++ b/src/diffbot/extract.py @@ -27,19 +27,19 @@ def _normalize_url(url: str) -> str: def extract(client: "Diffbot", url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: url = _normalize_url(url) - response = client._http.get( + response = client.request( + "GET", f"{client.analyze_url}/{api}", params=_build_params(client, url, fmt), ) - client._raise_for_status(response) return _parse_response(client, response.json()) async def extract_async(client: "DiffbotAsync", url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: url = _normalize_url(url) - response = await client._http.get( + response = await client.request( + "GET", f"{client.analyze_url}/{api}", params=_build_params(client, url, fmt), ) - client._raise_for_status(response) return _parse_response(client, response.json()) diff --git a/src/diffbot/kg.py b/src/diffbot/kg.py index 92e72a6..3f2a72b 100644 --- a/src/diffbot/kg.py +++ b/src/diffbot/kg.py @@ -5,13 +5,15 @@ from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union +from ._constants import KG_BASE from .ontology import Ontology +from .types import ExportFormat if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync -KG_DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql" -KG_ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology" +KG_DQL_ENDPOINT = f"{KG_BASE}/kg/v3/dql" +KG_ONTOLOGY_ENDPOINT = f"{KG_BASE}/kg/ontology" def _build_dql_params( @@ -19,7 +21,7 @@ def _build_dql_params( query: str, size: int, from_: int, - format: str, + format: ExportFormat, filter: Optional[str], exportspec: Optional[str], extra: Optional[Dict[str, str]], @@ -44,15 +46,14 @@ def dql( *, size: int = 10, from_: int = 0, - format: str = "json", + format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, extra: Optional[Dict[str, str]] = None, raw: bool = False, ) -> Union[Dict[str, Any], bytes]: params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra) - response = client._http.get(client.dql_url, params=params) - client._raise_for_status(response) + response = client.request("GET", client.dql_url, params=params) return response.content if raw else response.json() @@ -62,15 +63,14 @@ async def dql_async( *, size: int = 10, from_: int = 0, - format: str = "json", + format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, extra: Optional[Dict[str, str]] = None, raw: bool = False, ) -> Union[Dict[str, Any], bytes]: params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra) - response = await client._http.get(client.dql_url, params=params) - client._raise_for_status(response) + response = await client.request("GET", client.dql_url, params=params) return response.content if raw else response.json() @@ -104,8 +104,7 @@ async def _one(q: Dict[str, Any]) -> Union[Dict[str, Any], bytes]: def dql_refresh_ontology(client: "Diffbot", dest: pathlib.Path) -> None: - response = client._http.get(client.ontology_url) - client._raise_for_status(response) + response = client.request("GET", client.ontology_url) dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(response.content) @@ -116,13 +115,11 @@ def dql_fetch_ontology(client: "Diffbot") -> Ontology: Performs no caching — the caller decides whether and where to hold onto the result. Use :func:`dql_refresh_ontology` instead to persist raw bytes to disk. """ - response = client._http.get(client.ontology_url) - client._raise_for_status(response) + response = client.request("GET", client.ontology_url) return Ontology.from_json(response.content) async def dql_fetch_ontology_async(client: "DiffbotAsync") -> Ontology: """Async variant of :func:`dql_fetch_ontology`.""" - response = await client._http.get(client.ontology_url) - client._raise_for_status(response) + response = await client.request("GET", client.ontology_url) return Ontology.from_json(response.content) diff --git a/src/diffbot/nlp.py b/src/diffbot/nlp.py index 33f79bd..ac24e96 100644 --- a/src/diffbot/nlp.py +++ b/src/diffbot/nlp.py @@ -2,10 +2,12 @@ from typing import TYPE_CHECKING, Any, Dict +from ._constants import NLP_HOST_BASE + if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync -NLP_BASE = "https://nl.diffbot.com/v1/" +NLP_BASE = f"{NLP_HOST_BASE}/v1/" NLP_FIELDS = "entities,sentiment" @@ -17,8 +19,7 @@ def entities( ) -> Dict[str, Any]: params = {"token": client.token, "fields": NLP_FIELDS} payload = [{"lang": lang, "format": "plain text", "content": text}] - response = client._http.post(client.nlp_url, params=params, json=payload) - client._raise_for_status(response) + response = client.request("POST", client.nlp_url, params=params, json=payload) data = response.json() return data[0] if isinstance(data, list) else data @@ -31,7 +32,6 @@ async def entities_async( ) -> Dict[str, Any]: params = {"token": client.token, "fields": NLP_FIELDS} payload = [{"lang": lang, "format": "plain text", "content": text}] - response = await client._http.post(client.nlp_url, params=params, json=payload) - client._raise_for_status(response) + response = await client.request("POST", client.nlp_url, params=params, json=payload) data = response.json() return data[0] if isinstance(data, list) else data diff --git a/src/diffbot/types.py b/src/diffbot/types.py new file mode 100644 index 0000000..690e760 --- /dev/null +++ b/src/diffbot/types.py @@ -0,0 +1,43 @@ +"""Additive type aliases for existing SDK surfaces. + +These are structural overlays over the plain ``dict``/``str`` values the SDK already +accepts and returns — no parsing or validation changes at runtime. ``TypedDict`` +members are ``total=False`` where the wire shape genuinely allows a field to be +absent, so a server response missing a field never breaks a type-checked caller. +""" + +from typing import Any, Dict, Literal, TypedDict + + +#: Roles accepted by the Diffbot LLM RAG endpoint's chat messages. +ChatRole = Literal["system", "user", "assistant"] + + +class ChatMessage(TypedDict): + role: ChatRole + content: str + + +#: `response_format` types accepted by :func:`diffbot.ask.ask`/`ask_json`. +ResponseFormatType = Literal["text", "json_object", "json_schema"] + + +class JsonSchemaSpec(TypedDict, total=False): + name: str + schema: Dict[str, Any] + + +class ResponseFormat(TypedDict, total=False): + type: ResponseFormatType + json_schema: JsonSchemaSpec + + +#: DQL / Enhance-bulk export formats. +ExportFormat = Literal["json", "jsonl", "csv", "xls", "xlsx"] + +#: How much of each page's matching content `web_search` returns. +SnippetLength = Literal["short", "medium", "long", "full"] + +#: Documented crawl/bulk job status codes (``docs__crawl__manage.md``): +#: 0 initializing, 6 paused, 7 in progress, 8 admin-paused, 9 complete. +CrawlJobStatusCode = Literal[0, 6, 7, 8, 9] diff --git a/src/diffbot/web_search.py b/src/diffbot/web_search.py index 326fe3e..787a916 100644 --- a/src/diffbot/web_search.py +++ b/src/diffbot/web_search.py @@ -1,21 +1,23 @@ """Diffbot web search API.""" -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional + +from ._constants import LLM_BASE +from .types import SnippetLength if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync -WEB_SEARCH_BASE = "https://llm.diffbot.com/api/v1/web_search" +WEB_SEARCH_BASE = f"{LLM_BASE}/api/v1/web_search" -def web_search( - client: "Diffbot", +def _build_web_search_request( + client: Any, text: str, - *, - num_results: Optional[int] = None, - max_tokens: Optional[int] = None, - snippet_length: Optional[str] = None, -) -> Dict[str, Any]: + num_results: Optional[int], + max_tokens: Optional[int], + snippet_length: Optional[SnippetLength], +) -> Any: headers = {"Authorization": f"Bearer {client.token}"} params: Dict[str, Any] = {"text": text} if num_results is not None: @@ -27,8 +29,19 @@ def web_search( if params["snippetLength"] == "FULL": # the reranker always chunks; only the unranked path returns the stored page params["rerank"] = "none" - response = client._http.get(client.web_search_url, headers=headers, params=params) - client._raise_for_status(response) + return headers, params + + +def web_search( + client: "Diffbot", + text: str, + *, + num_results: Optional[int] = None, + max_tokens: Optional[int] = None, + snippet_length: Optional[SnippetLength] = None, +) -> Dict[str, Any]: + headers, params = _build_web_search_request(client, text, num_results, max_tokens, snippet_length) + response = client.request("GET", client.web_search_url, headers=headers, params=params) return response.json() @@ -38,19 +51,8 @@ async def web_search_async( *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, - snippet_length: Optional[str] = None, + snippet_length: Optional[SnippetLength] = None, ) -> Dict[str, Any]: - headers = {"Authorization": f"Bearer {client.token}"} - params: Dict[str, Any] = {"text": text} - if num_results is not None: - params["size"] = num_results - if max_tokens is not None: - params["maxTokens"] = max_tokens - if snippet_length is not None: - params["snippetLength"] = snippet_length.upper() - if params["snippetLength"] == "FULL": - # the reranker always chunks; only the unranked path returns the stored page - params["rerank"] = "none" - response = await client._http.get(client.web_search_url, headers=headers, params=params) - client._raise_for_status(response) + headers, params = _build_web_search_request(client, text, num_results, max_tokens, snippet_length) + response = await client.request("GET", client.web_search_url, headers=headers, params=params) return response.json() diff --git a/tests/test_ask.py b/tests/test_ask.py index 647c234..cf3a265 100644 --- a/tests/test_ask.py +++ b/tests/test_ask.py @@ -1,8 +1,9 @@ import json +import httpx import pytest -from diffbot import DiffbotAsync, json_schema_format +from diffbot import AuthError, Diffbot, DiffbotAsync, RateLimitError, json_schema_format from diffbot.ask import ( ANY_OBJECT_SCHEMA, _build_payload, @@ -192,6 +193,97 @@ def test_leak_guard_passes_lists(): assert _check_tool_call_leak(answer) is answer +""" +Streaming error path + +ask() opens a streaming response, so reading the error body must happen before +raising -- otherwise httpx raises ResponseNotRead instead of AuthError/RateLimitError. +httpx.MockTransport with content= pre-fills the body, which masks this bug (the body +looks "already read"), so these use a real SyncByteStream/AsyncByteStream that defers +content the way a genuine chunked network response would. +""" + + +class _SyncSlowStream(httpx.SyncByteStream): + def __init__(self, chunks): + self._chunks = chunks + + def __iter__(self): + for chunk in self._chunks: + yield chunk + + def close(self): + pass + + +class _AsyncSlowStream(httpx.AsyncByteStream): + def __init__(self, chunks): + self._chunks = chunks + + async def __aiter__(self): + for chunk in self._chunks: + yield chunk + + async def aclose(self): + pass + + +def test_ask_streaming_401_raises_auth_error_with_message(): + body = b'{"message": "Unauthorized. Incorrect token.", "requestId": "abc"}' + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(401, stream=_SyncSlowStream([body])) + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + with pytest.raises(AuthError) as exc_info: + list(db.ask([{"role": "user", "content": "hi"}])) + assert exc_info.value.message == "Unauthorized. Incorrect token." + assert exc_info.value.request_id == "abc" + + +def test_ask_streaming_429_raises_rate_limit_error_with_retry_after(): + body = b'{"message": "slow down"}' + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(429, headers={"retry-after": "5"}, stream=_SyncSlowStream([body])) + + db = Diffbot(token="test-token", retry=False, transport=httpx.MockTransport(handler)) + with pytest.raises(RateLimitError) as exc_info: + list(db.ask([{"role": "user", "content": "hi"}])) + assert exc_info.value.retry_after == "5" + assert exc_info.value.message == "slow down" + + +@pytest.mark.anyio +async def test_async_ask_streaming_401_raises_auth_error_with_message(): + body = b'{"message": "Unauthorized. Incorrect token.", "requestId": "abc"}' + + async def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(401, stream=_AsyncSlowStream([body])) + + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) + with pytest.raises(AuthError) as exc_info: + async for _ in db.ask([{"role": "user", "content": "hi"}]): + pass + assert exc_info.value.message == "Unauthorized. Incorrect token." + assert exc_info.value.request_id == "abc" + + +@pytest.mark.anyio +async def test_async_ask_streaming_429_raises_rate_limit_error_with_retry_after(): + body = b'{"message": "slow down"}' + + async def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(429, headers={"retry-after": "5"}, stream=_AsyncSlowStream([body])) + + db = DiffbotAsync(token="test-token", retry=False, transport=httpx.MockTransport(handler)) + with pytest.raises(RateLimitError) as exc_info: + async for _ in db.ask([{"role": "user", "content": "hi"}]): + pass + assert exc_info.value.retry_after == "5" + assert exc_info.value.message == "slow down" + + """ Live """ diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..93eef1c --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,61 @@ +import pytest + +from diffbot import Diffbot, DiffbotAsync + + +def test_defaults_every_endpoint_to_the_public_hosts(): + db = Diffbot(token="t") + assert db.analyze_url == "https://api.diffbot.com/v3" + assert db.crawler_url == "https://api.diffbot.com/v3/crawl" + assert db.llm_url == "https://llm.diffbot.com/rag/v1/chat/completions" + assert db.web_search_url == "https://llm.diffbot.com/api/v1/web_search" + assert db.nlp_url == "https://nl.diffbot.com/v1/" + assert db.dql_url == "https://kg.diffbot.com/kg/v3/dql" + assert db.ontology_url == "https://kg.diffbot.com/kg/ontology" + + +def test_derives_every_api_diffbot_com_endpoint_from_a_single_api_base_override(): + db = Diffbot(token="t", api_base="https://staging.example.com") + assert db.analyze_url == "https://staging.example.com/v3" + assert db.crawler_url == "https://staging.example.com/v3/crawl" + + +def test_derives_every_kg_diffbot_com_endpoint_from_a_single_kg_base_override(): + db = Diffbot(token="t", kg_base="https://kg.staging.example.com") + assert db.dql_url == "https://kg.staging.example.com/kg/v3/dql" + assert db.ontology_url == "https://kg.staging.example.com/kg/ontology" + + +def test_derives_every_llm_diffbot_com_endpoint_from_a_single_llm_base_override(): + db = Diffbot(token="t", llm_base="https://llm.staging.example.com") + assert db.llm_url == "https://llm.staging.example.com/rag/v1/chat/completions" + assert db.web_search_url == "https://llm.staging.example.com/api/v1/web_search" + + +def test_derives_the_nl_endpoint_from_nlp_base(): + db = Diffbot(token="t", nlp_base="https://nl.staging.example.com") + assert db.nlp_url == "https://nl.staging.example.com/v1/" + + +def test_legacy_per_endpoint_option_wins_over_a_host_base(): + db = Diffbot( + token="t", + api_base="https://staging.example.com", + analyze_url="https://custom.example.com/v3", + ) + assert db.analyze_url == "https://custom.example.com/v3" + assert db.crawler_url == "https://staging.example.com/v3/crawl" + + +def test_async_client_resolves_bases_the_same_way(): + db = DiffbotAsync(token="t", api_base="https://staging.example.com") + assert db.analyze_url == "https://staging.example.com/v3" + assert db.crawler_url == "https://staging.example.com/v3/crawl" + assert db.dql_url == "https://kg.diffbot.com/kg/v3/dql" + + +def test_token_required_async(): + from diffbot import ValidationError + + with pytest.raises(ValidationError): + DiffbotAsync(token="") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 375c10f..124e44d 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,7 +1,7 @@ import httpx import pytest -from diffbot import APIError, AuthError, Diffbot, RateLimitError +from diffbot import APIError, AuthError, Diffbot, DiffbotAsync, JobError, RateLimitError from diffbot import CrawlEvent, CrawlEventType @@ -60,4 +60,72 @@ def handler(request: httpx.Request) -> httpx.Response: err = exc_info.value assert err.status_code == 401 assert err.message == "Unauthorized" - assert err.request_id == "123456789" \ No newline at end of file + assert err.request_id == "123456789" + + +""" +Job failure -> JobError (not a synthesized APIError(500, ...)) +""" + + +def test_crawl_watch_raises_job_error_on_failed_status(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/data"): + return httpx.Response(200, text="") + if "seeds" in request.url.params: + return httpx.Response(200, json={"jobs": [{"name": "test-job"}]}) + return httpx.Response( + 200, + json={"jobs": [{"name": "test-job", "jobStatus": {"status": 9, "message": "Crawl failed: bad seed"}}]}, + ) + + db = make_client(handler) + with pytest.raises(JobError) as exc_info: + list(db.crawl("https://example.com", job_name="test-job", watch=True, poll_interval=0)) + assert isinstance(exc_info.value, APIError) + assert exc_info.value.job_name == "test-job" + assert "bad seed" in str(exc_info.value) + + +@pytest.mark.anyio +async def test_async_crawl_watch_raises_job_error_on_failed_status(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/data"): + return httpx.Response(200, text="") + if "seeds" in request.url.params: + return httpx.Response(200, json={"jobs": [{"name": "test-job"}]}) + return httpx.Response( + 200, + json={"jobs": [{"name": "test-job", "jobStatus": {"status": 9, "message": "Crawl failed: bad seed"}}]}, + ) + + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) + with pytest.raises(JobError) as exc_info: + events = [] + async for event in db.crawl("https://example.com", job_name="test-job", watch=True, poll_interval=0): + events.append(event) + assert exc_info.value.job_name == "test-job" + + +""" +crawl_delete_job is a GET-shaped mutation and must not be retried +""" + + +def test_crawl_delete_job_is_not_retried(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(503) + + from diffbot import RetryPolicy + + db = Diffbot( + token="test-token", + retry=RetryPolicy(base_delay=0.001, max_delay=0.001), + transport=httpx.MockTransport(handler), + ) + with pytest.raises(APIError): + db.crawl_delete_job("test-job") + assert calls["n"] == 1 \ No newline at end of file diff --git a/tests/test_dql.py b/tests/test_dql.py index 72b6747..b894bde 100644 --- a/tests/test_dql.py +++ b/tests/test_dql.py @@ -1,7 +1,7 @@ import httpx import pytest -from diffbot import Diffbot, DiffbotAsync +from diffbot import APIError, Diffbot, DiffbotAsync, NotFoundError """ @@ -87,6 +87,22 @@ def handler(request: httpx.Request) -> httpx.Response: assert ont.types() == ["Person"] +""" +Errors +""" + + +def test_dql_404_raises_not_found_error(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(404, json={"message": "no such report"}) + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + with pytest.raises(NotFoundError) as exc_info: + db.dql("type:Organization") + assert isinstance(exc_info.value, APIError) + assert exc_info.value.message == "no such report" + + """ Live """ diff --git a/tests/test_transport.py b/tests/test_transport.py new file mode 100644 index 0000000..271b5d3 --- /dev/null +++ b/tests/test_transport.py @@ -0,0 +1,231 @@ +import httpx +import pytest + +from diffbot import APIError, AuthError, Diffbot, DiffbotAsync, NotFoundError, RateLimitError, RetryPolicy +from diffbot._transport import raise_for_status + + +def make_client(handler, **kwargs) -> Diffbot: + return Diffbot(token="test-token", transport=httpx.MockTransport(handler), **kwargs) + + +""" +Repeated query parameters +""" + + +def test_request_appends_array_valued_params_instead_of_overwriting(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params.get_list("tag") == ["a", "b", "c"] + return httpx.Response(200, json={}) + + db = make_client(handler) + db.request("GET", "https://example.com/x", params={"tag": ["a", "b", "c"]}) + + +""" +DELETE, raw and form bodies +""" + + +def test_request_issues_a_delete_with_a_json_body(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "DELETE" + assert request.headers["content-type"] == "application/json" + assert request.content == b'{"api":"x"}' + return httpx.Response(200, json={}) + + db = make_client(handler) + db.request("DELETE", "https://example.com/custom", json={"api": "x"}) + + +def test_request_form_encodes_a_data_body_and_sets_content_type(): + def handler(request: httpx.Request) -> httpx.Response: + assert "application/x-www-form-urlencoded" in request.headers["content-type"] + assert request.content == b"a=1&b=2" + return httpx.Response(200, json={}) + + db = make_client(handler) + db.request("POST", "https://example.com/x", data={"a": "1", "b": "2"}) + + +def test_request_lets_a_caller_override_content_type_with_a_raw_body(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.headers["content-type"] == "text/html" + assert request.content == b"" + return httpx.Response(200, json={}) + + db = make_client(handler) + db.request( + "POST", + "https://example.com/x", + content="", + headers={"Content-Type": "text/html"}, + ) + + +""" +Per-call timeout +""" + + +def test_request_per_call_timeout_overrides_the_client_default(): + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["timeout"] = request.extensions.get("timeout") + return httpx.Response(200, json={}) + + db = make_client(handler, timeout=30.0) + db.request("GET", "https://example.com/x", timeout=5.0) + assert captured["timeout"]["read"] == 5.0 + + +""" +Retry policy +""" + + +def test_retries_a_429_and_honors_a_numeric_retry_after(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(429, headers={"retry-after": "0"}) + return httpx.Response(200, json={"ok": True}) + + db = make_client(handler, retry=RetryPolicy(base_delay=0.001, max_delay=0.001)) + response = db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + assert response.status_code == 200 + + +def test_retries_5xx_responses(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(503) + return httpx.Response(200, json={"ok": True}) + + db = make_client(handler, retry=RetryPolicy(base_delay=0.001, max_delay=0.001)) + response = db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + assert response.status_code == 200 + + +def test_retries_connection_errors(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.ConnectError("boom") + return httpx.Response(200, json={"ok": True}) + + db = make_client(handler, retry=RetryPolicy(base_delay=0.001, max_delay=0.001)) + response = db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + assert response.status_code == 200 + + +def test_gives_up_after_max_attempts(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(500) + + db = make_client(handler, retry=RetryPolicy(max_attempts=2, base_delay=0.001, max_delay=0.001)) + with pytest.raises(APIError): + db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + + +def test_does_not_retry_a_call_marked_retryable_false(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(503) + + db = make_client(handler, retry=RetryPolicy(base_delay=0.001, max_delay=0.001)) + with pytest.raises(APIError): + db.request("GET", "https://example.com/x", retryable=False) + assert calls["n"] == 1 + + +def test_does_not_retry_at_all_when_retry_is_disabled_client_wide(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(500) + + db = make_client(handler, retry=False) + with pytest.raises(APIError): + db.request("GET", "https://example.com/x") + assert calls["n"] == 1 + + +def test_does_not_retry_a_4xx_that_isnt_429(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + return httpx.Response(400) + + db = make_client(handler, retry=RetryPolicy(base_delay=0.001, max_delay=0.001)) + with pytest.raises(APIError): + db.request("GET", "https://example.com/x") + assert calls["n"] == 1 + + +@pytest.mark.anyio +async def test_async_retries_a_429(): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(429, headers={"retry-after": "0"}) + return httpx.Response(200, json={"ok": True}) + + db = DiffbotAsync( + token="test-token", + transport=httpx.MockTransport(handler), + retry=RetryPolicy(base_delay=0.001, max_delay=0.001), + ) + response = await db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + assert response.status_code == 200 + + +""" +raise_for_status +""" + + +def test_raise_for_status_throws_not_found_error_for_a_404_and_attaches_headers(): + response = httpx.Response(404, headers={"X-Test": "1"}) + with pytest.raises(NotFoundError) as exc_info: + raise_for_status(response) + assert exc_info.value.headers.get("x-test") == "1" + + +def test_raise_for_status_is_a_noop_for_an_ok_response(): + raise_for_status(httpx.Response(200)) + + +def test_raise_for_status_throws_auth_error_for_401(): + with pytest.raises(AuthError): + raise_for_status(httpx.Response(401)) + + +def test_raise_for_status_throws_rate_limit_error_for_429_with_retry_after(): + response = httpx.Response(429, headers={"retry-after": "7"}) + with pytest.raises(RateLimitError) as exc_info: + raise_for_status(response) + assert exc_info.value.retry_after == "7" From 9902bc3a8c9c45f471769af0920f95a25540c494 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Fri, 13 Nov 2026 16:09:54 -1000 Subject: [PATCH 02/31] docs: document per-host bases and retry policy in the README Updates the Client configuration table and adds examples for api_base/kg_base/ llm_base/nlp_base (pointing the whole SDK at a staging host with one option) and retry/RetryPolicy, matching the TypeScript SDK's README for the same Phase 0 work. Adds the matching tests per AGENTS.md. Co-Authored-By: Claude Sonnet 5 --- README.md | 52 +++++++++++++++++++++++++++-------- tests/test_readme_examples.py | 20 +++++++++++++- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index fd9d2ad..9a3e060 100644 --- a/README.md +++ b/README.md @@ -75,14 +75,19 @@ place to configure the SDK. |----------|---------|---------| | `token` | — (required) | all | | `timeout` | `30.0` (seconds) | all | +| `retry` | on, 3 attempts | all; `False` disables, or pass a `RetryPolicy` to tune it | | `transport` | httpx default | all | -| `analyze_url` | `https://api.diffbot.com/v3` | `extract` | -| `crawler_url` | `https://api.diffbot.com/v3/crawl` | `crawl`, `crawl_list_jobs`, `crawl_get_job`, `crawl_delete_job` | -| `llm_url` | `https://llm.diffbot.com/rag/v1/chat/completions` | `ask`, `ask_json` | -| `web_search_url` | `https://llm.diffbot.com/api/v1/web_search` | `web_search` | -| `nlp_url` | `https://nl.diffbot.com/v1/` | `entities` | -| `dql_url` | `https://kg.diffbot.com/kg/v3/dql` | `dql`, `dql_parallel` | -| `ontology_url` | `https://kg.diffbot.com/kg/ontology` | `dql_fetch_ontology`, `dql_refresh_ontology` | +| `api_base` | `https://api.diffbot.com` | `extract`, `crawl`, `crawl_list_jobs`, `crawl_get_job`, `crawl_delete_job` | +| `kg_base` | `https://kg.diffbot.com` | `dql`, `dql_parallel`, `dql_fetch_ontology`, `dql_refresh_ontology` | +| `llm_base` | `https://llm.diffbot.com` | `ask`, `ask_json`, `web_search` | +| `nlp_base` | `https://nl.diffbot.com` | `entities` | +| `analyze_url` | derived from `api_base` | `extract` | +| `crawler_url` | derived from `api_base` | `crawl`, `crawl_list_jobs`, `crawl_get_job`, `crawl_delete_job` | +| `llm_url` | derived from `llm_base` | `ask`, `ask_json` | +| `web_search_url` | derived from `llm_base` | `web_search` | +| `nlp_url` | derived from `nlp_base` | `entities` | +| `dql_url` | derived from `kg_base` | `dql`, `dql_parallel` | +| `ontology_url` | derived from `kg_base` | `dql_fetch_ontology`, `dql_refresh_ontology` | ```python from diffbot import Diffbot @@ -94,12 +99,35 @@ db = Diffbot( ) ``` -`analyze_url` is a base the SDK appends `/{api}` to, and `crawler_url` is used as given -for job management and with `/data` appended for crawl results. The rest are complete -endpoints, used as given. +Point the whole SDK at a staging environment with one option instead of one per endpoint: -Passing `transport` replaces the httpx transport, which is the hook for retries, proxies, -or mocking in tests. `Diffbot` takes an `httpx.BaseTransport`, `DiffbotAsync` an +```python +staging = Diffbot(token="YOUR_TOKEN", api_base="https://staging.example.com") +``` + +`api_base`/`kg_base`/`llm_base`/`nlp_base` set the host for every endpoint on that host +at once. The older per-endpoint options (`analyze_url`, `crawler_url`, `llm_url`, +`web_search_url`, `nlp_url`, `dql_url`, `ontology_url`) still work and win over the +matching base when both are set; `analyze_url` is a base the SDK appends `/{api}` to, +`crawler_url` is used as given for job management and with `/data` appended for crawl +results, and the rest are complete endpoints used as given. + +Requests retry `429` and `5xx` responses (and connection errors) with backoff and +jitter, honoring a `Retry-After` header, by default. Disable retries or tune the policy: + +```python +from diffbot import Diffbot, RetryPolicy + +db = Diffbot(token="YOUR_TOKEN", retry=False) +# or: retry=RetryPolicy(max_attempts=5) +``` + +The transport also accepts a per-call `retryable=False` escape hatch; the SDK uses it +internally for `crawl_delete_job`, a GET-shaped mutation that must not be retried +blindly. + +Passing `transport` replaces the httpx transport, which is the hook for proxies or +mocking in tests. `Diffbot` takes an `httpx.BaseTransport`, `DiffbotAsync` an `httpx.AsyncBaseTransport`: ```python diff --git a/tests/test_readme_examples.py b/tests/test_readme_examples.py index c0235bd..31a4588 100644 --- a/tests/test_readme_examples.py +++ b/tests/test_readme_examples.py @@ -4,7 +4,7 @@ import httpx import pytest -from diffbot import CrawlEventType, Diffbot, DiffbotAsync, json_schema_format, resolve_token +from diffbot import CrawlEventType, Diffbot, DiffbotAsync, RetryPolicy, json_schema_format, resolve_token SSE_PARIS = 'data: {"choices": [{"delta": {"content": "Paris"}}]}\n' @@ -48,6 +48,24 @@ def test_readme_client_configuration_transport(): assert db._http._transport is transport +def test_readme_client_configuration_api_base(): + staging = Diffbot(token="YOUR_TOKEN", api_base="https://staging.example.com") + assert staging.analyze_url == "https://staging.example.com/v3" + assert staging.crawler_url == "https://staging.example.com/v3/crawl" + + +def test_readme_client_configuration_retry(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + db = Diffbot(token="YOUR_TOKEN", retry=False, transport=httpx.MockTransport(handler)) + with pytest.raises(Exception): + db.extract("https://www.example.com") + + tuned = Diffbot(token="YOUR_TOKEN", retry=RetryPolicy(max_attempts=5)) + assert tuned._retry_policy.max_attempts == 5 + + # --------------------------------------------------------------------------- # Sync Usage # --------------------------------------------------------------------------- From b02290703a3e22dda7e5f8a4d8305b4672e4e11f Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Fri, 13 Nov 2026 16:11:01 -1000 Subject: [PATCH 03/31] docs: note two deliberately deferred Phase 0 items; add an Unreleased changelog entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PHASE0_LEFTOVERS.md: no standalone endpoint-registry data structure (same call as the TypeScript SDK on this branch — Phase 0 adds zero new endpoints, so a registry has no second consumer yet), response headers surfaced on the error path only (success-path headers deferred to Phase 6's dql_report, which doesn't exist yet to shape it against), confirmation that Python's dql(raw=True) never had the TypeScript SDK's binary-corruption bug, the ontology async/text asymmetry being Phase 1 scope rather than Phase 0, and the scope of the TypedDict/Literal additions (existing request parameters, not response shapes — this repo has no type-checking CI gate to shape response overlays against). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 52 ++++++++++++++++++++++ PHASE0_LEFTOVERS.md | 102 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 PHASE0_LEFTOVERS.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b3ffef2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,52 @@ +# Changelog + +## Unreleased + +Phase 0 of the Diffbot SDKs' full HTTP API coverage plan: shared transport +foundation and a sync/async de-duplication refactor. No new API endpoints +land in this phase, and the package version is unchanged (per the plan's +scope, version bumps are deferred). + +### Added + +- Per-host base URL options: `api_base`, `kg_base`, `llm_base`, `nlp_base` on + both `Diffbot` and `DiffbotAsync`. The older per-endpoint options + (`analyze_url`, `crawler_url`, `llm_url`, `web_search_url`, `nlp_url`, + `dql_url`, `ontology_url`) still work and win over the matching base when + both are set. +- Repeated query parameters: handler param builders now declare + `Sequence`-valued params explicitly (httpx already encoded them as repeated + keys; this makes it a documented, tested capability). +- `client.request(method, url, ...)` on both clients: `DELETE` + arbitrary + method/body/`Content-Type`, form-encoded (`data=`) and raw (`content=`) + bodies, a per-call `timeout` override separate from the client default, and + a `follow_redirects` override. +- A default-on retry policy (`RetryPolicy` dataclass, `retry` client option) + for `429`/`5xx`/connection errors, with backoff, jitter, and `Retry-After` + support, capped at 3 attempts by default. Disable with `retry=False`, or + opt a single call out with `retryable=False` (used internally by + `crawl_delete_job`/`crawl_delete_job_async`, a GET-shaped mutation that must + never be retried blindly). +- `NotFoundError` (404) and `JobError`, both subclassing `APIError` so + existing `except APIError` call sites are unaffected. +- `APIError.headers` (and subclasses), populated from the error response. +- Additive `TypedDict`/`Literal` type aliases in `diffbot.types`: + `ChatMessage`, `ResponseFormat`, `ExportFormat`, `SnippetLength` — used in + `ask()`/`ask_json()`/`dql()`/`web_search()` signatures. No Pydantic, no + runtime behavior change. + +### Fixed + +- `ask()`/`ask_json()` (sync and async): a non-2xx response (e.g. 401, 429) + while streaming now raises `AuthError`/`RateLimitError` as documented, + instead of `httpx.ResponseNotRead`. The error body is now read before + raising. +- `crawl()`'s job-failure path now raises `JobError` (carrying the job name) + instead of a synthesized `APIError(500, ...)`. + +### Internal + +- Sync/async de-duplication: `Diffbot`/`DiffbotAsync._raise_for_status` and + the repeated `client._http.get(...)` + raise pattern in every handler + module are replaced by shared helpers in a new `_transport.py`, used by + both clients through `client.request()`. diff --git a/PHASE0_LEFTOVERS.md b/PHASE0_LEFTOVERS.md new file mode 100644 index 0000000..60fadac --- /dev/null +++ b/PHASE0_LEFTOVERS.md @@ -0,0 +1,102 @@ +# Phase 0 leftovers (Python) + +Everything in the Phase 0 scope (`py-prompt.md`, opus-plan.md §6 "Phase 0 — Foundation", +§3.2–3.4, §3.7) landed. A few items are deliberately smaller than the plan's framing, or +were already true on the Python side without a code change — documented here rather than +silently shipped (or silently skipped) without explanation, mirroring the TypeScript +SDK's own `PHASE0_LEFTOVERS.md` on this branch. + +## 1. No literal "endpoint registry" data structure + +opus-plan §3.2/§6 0.1 frames the per-host base URLs as part of "an endpoint registry + +per-host base URLs." What landed is the per-host base URL resolution +(`api_base`/`kg_base`/`llm_base`/`nlp_base`, old `*_url` options winning aliases) directly +in `client.py` — there's no separate data structure mapping `(operation) -> {path, +auth_style, retryable}`. + +This mirrors the TypeScript SDK's same call on this branch: Phase 0 adds zero new +endpoints, so the only thing a registry would currently serve is base-URL resolution +(done) and a `retryable` flag (done ad hoc, as an explicit `retryable=False` at the one +call site that needs it — `crawl_delete_job`/`crawl_delete_job_async`). Building a +generic per-operation table with no second or third consumer risked guessing the wrong +shape for Phase 1+ (Account, Enhance, Bulk, Custom API, ...), which is where a real +registry starts paying for itself. Revisit when Phase 1 lands. + +## 2. Response headers: error path only, not success path + +opus-plan §6 0.6 asks for "response headers surfaced to callers," calling out an +`include_headers` option or a `*_raw` variant returning `(body, headers)`. + +What landed: every error response's headers are attached to `APIError` (and its +subclasses `AuthError`, `NotFoundError`, `RateLimitError`, `JobError`) via a new +`headers` field, populated in `_transport.raise_for_status()`. Tested in +`tests/test_transport.py`'s `raise_for_status` tests. + +What didn't land: a generic opt-in mechanism for a *successful* response's headers. The +plan's own stated motivation is `X-Diffbot-CoverageReportId` on a DQL coverage report +(opus-plan §3.4, §6 0.6's "prerequisite for DQL coverage reports") — that's a Phase 6 +operation (`dql_report`) that doesn't exist yet. Building the success-path mechanism now, +with no real caller to shape it against, risked the same guessing problem as the +registry above. Build it alongside `dql_report`/`dql_report_by_id` in Phase 6. Matches +the TypeScript SDK's identical call on this branch. + +## 3. `dql(raw=True)` binary correctness: no bug existed on the Python side + +opus-plan §3.3/§6 0.5 frames this as "binary + text response reads" plus a concrete bug: +the TypeScript SDK's `dql()` read the body as text and re-encoded it with +`TextEncoder`, corrupting `xls`/`xlsx` exports. The audit explicitly notes Python's byte +path was already correct (`kg.py` returning `response.content` directly). No code change +was needed here beyond routing the call through the shared `client.request()` helper, +which preserves the same `response.content` / `response.json()` split. Covered by the +existing `test_dql.py` raw-mode tests (unchanged) plus the new per-host-base tests. + +## 4. Ontology asymmetries are Phase 1, not Phase 0 + +opus-plan §5.2's core-surface-mapping row for the ontology endpoint lists adding +`dql_fetch_ontology_text()` and `dql_refresh_ontology_async()` to close the existing +Python/TypeScript asymmetry (`dql_refresh_ontology` has no async twin; TypeScript has no +raw-text fetch). That row is scoped under "Phase 1 — Account + token verification," not +Phase 0, and the `py-prompt.md` Phase 0 scope list does not mention it. Left alone. + +## 5. TypedDict/Literal annotations: existing-surface request parameters only + +`py-prompt.md` asks for "matching TypedDict(total=False)/Literal annotations, additive, +no Pydantic" as the Python counterpart to the TypeScript SDK's much larger Phase 0 type +safety push (vendored OpenAPI, generated response schemas, per-handler overloads). The +typesafety-revision page's override on this point is specifically about TypeScript's +approach; opus-plan's own phasing (§6, Phase 7 "typed responses... land as a dedicated +minor, not bundled with each endpoint" — the risk being a `TypedDict` claiming a field +that a future server response omits) argues for keeping Python's response shapes as +plain `dict`/`bytes` for now, which is what shipped. + +What landed: `src/diffbot/types.py` adds `ChatMessage`, `ResponseFormatType`, +`ResponseFormat`, `ExportFormat`, `SnippetLength`, and `CrawlJobStatusCode` (the last +documents the numeric status codes `crawl.py` already branches on; not yet wired into a +public signature since no handler takes a status code as an input). These are wired into +the existing `ask()`/`ask_json()`/`dql()`/`web_search()` request-parameter signatures in +both `client.py` and the handler modules — purely additive, no runtime behavior change +(verified by the full test suite staying green before and after). + +What didn't land: `TypedDict` overlays for *response* shapes (`DqlResponse`, +`WebSearchResponse`, `NlpResponse`, `ExtractResponse`, a `CrawlJob`/`JobStatus` pair, +`Ontology`'s wire format). This repo has no mypy/type-checking gate in CI today (checked +`.github/workflows/tests.yml`), so there is no second consumer to shape these against +yet, and retrofitting them risks exactly the staleness opus-plan §3.5/§6 Phase 7 warns +about. Revisit if/when Enhance (Phase 2) introduces the first genuinely new typed +response and a type-checking CI gate is added. + +## Everything else + +The sync/async de-duplication refactor (shared `_transport.py` request/response +helpers), per-host bases, repeated query parameters, DELETE + arbitrary method/body/ +Content-Type, form-encoded and raw bodies, per-call timeout, the default-on retry policy +with per-call `retryable=False` opt-out, `NotFoundError`/`JobError` (both subclassing +`APIError`), and the streaming `ask()` `ResponseNotRead` fix (sync and async, with tests +using a real byte-stream mock transport so the fix is actually exercised) all landed — +see the commit log for the breakdown and `tests/test_transport.py`, `tests/test_client.py`, +`tests/test_ask.py`, `tests/test_crawl.py`, `tests/test_dql.py` for coverage. + +The TypeScript SDK's two fixes (dql raw-mode binary corruption, crawl CSV quote-aware +parsing) and its full type-safety push (vendored OpenAPI, generated types, per-handler +overloads, typescript-eslint strict-type-checked) are not part of this checkout; they +were implemented first on the matching TypeScript branch per the assignment. From 92fc982e8b636cefa47d76014ccca6a78e150ea9 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:51:48 -1000 Subject: [PATCH 04/31] feat(dql): send DQL as POST with a JSON body dql()/dql_parallel() and async twins now POST to the DQL endpoint with a JSON body; the token stays in the query string. Public contract unchanged. extra widens to Mapping[str, Any]. Co-Authored-By: Claude Sonnet 5.5 --- src/diffbot/client.py | 6 +- src/diffbot/kg.py | 35 ++++++------ tests/test_dql.py | 101 ++++++++++++++++++++++++++++++++++ tests/test_dql_cli.py | 6 +- tests/test_ontology.py | 2 +- tests/test_readme_examples.py | 2 +- uv.lock | 56 +++++++++---------- 7 files changed, 155 insertions(+), 53 deletions(-) diff --git a/src/diffbot/client.py b/src/diffbot/client.py index a83c182..89247c4 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -2,7 +2,7 @@ import pathlib from types import TracebackType -from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, Sequence, Type, Union +from typing import Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, Sequence, Type, Union import httpx @@ -215,7 +215,7 @@ def dql( format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, - extra: Optional[Dict[str, str]] = None, + extra: Optional[Mapping[str, Any]] = None, raw: bool = False, ) -> Union[Dict[str, Any], bytes]: """Run a DQL query against the Diffbot Knowledge Graph. @@ -415,7 +415,7 @@ async def dql( format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, - extra: Optional[Dict[str, str]] = None, + extra: Optional[Mapping[str, Any]] = None, raw: bool = False, ) -> Union[Dict[str, Any], bytes]: """Run a DQL query against the Diffbot Knowledge Graph. diff --git a/src/diffbot/kg.py b/src/diffbot/kg.py index 3f2a72b..1d79766 100644 --- a/src/diffbot/kg.py +++ b/src/diffbot/kg.py @@ -3,7 +3,7 @@ import asyncio import pathlib from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence, Union from ._constants import KG_BASE from .ontology import Ontology @@ -16,28 +16,29 @@ KG_ONTOLOGY_ENDPOINT = f"{KG_BASE}/kg/ontology" -def _build_dql_params( - client: Any, +def _build_dql_body( query: str, size: int, from_: int, format: ExportFormat, filter: Optional[str], exportspec: Optional[str], - extra: Optional[Dict[str, str]], + extra: Optional[Mapping[str, Any]], ) -> Dict[str, Any]: - params: Dict[str, Any] = {"token": client.token, "query": query, "size": size} + """JSON body for ``POST /kg/v3/dql`` (``DqlPostRequest``). The token is *not* part + of the body; it travels in the query string, as the spec's security scheme says.""" + body: Dict[str, Any] = {"query": query, "size": size} if from_: - params["from"] = from_ + body["from"] = from_ if format != "json": - params["format"] = format + body["format"] = format if filter is not None: - params["filter"] = filter + body["filter"] = filter if exportspec is not None: - params["exportspec"] = exportspec + body["exportspec"] = exportspec if extra: - params.update(extra) - return params + body.update(extra) + return body def dql( @@ -49,11 +50,11 @@ def dql( format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, - extra: Optional[Dict[str, str]] = None, + extra: Optional[Mapping[str, Any]] = None, raw: bool = False, ) -> Union[Dict[str, Any], bytes]: - params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra) - response = client.request("GET", client.dql_url, params=params) + body = _build_dql_body(query, size, from_, format, filter, exportspec, extra) + response = client.request("POST", client.dql_url, params={"token": client.token}, json=body) return response.content if raw else response.json() @@ -66,11 +67,11 @@ async def dql_async( format: ExportFormat = "json", filter: Optional[str] = None, exportspec: Optional[str] = None, - extra: Optional[Dict[str, str]] = None, + extra: Optional[Mapping[str, Any]] = None, raw: bool = False, ) -> Union[Dict[str, Any], bytes]: - params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra) - response = await client.request("GET", client.dql_url, params=params) + body = _build_dql_body(query, size, from_, format, filter, exportspec, extra) + response = await client.request("POST", client.dql_url, params={"token": client.token}, json=body) return response.content if raw else response.json() diff --git a/tests/test_dql.py b/tests/test_dql.py index b894bde..6d01fff 100644 --- a/tests/test_dql.py +++ b/tests/test_dql.py @@ -1,3 +1,5 @@ +import json + import httpx import pytest @@ -114,3 +116,102 @@ def test_live_dql(db): assert len(result["data"]) > 0 entity = result["data"][0]["entity"] assert entity.get("name") == "Diffbot" + + +""" +DQL is sent as POST with a JSON body +""" + + +def _capture(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"data": [], "hits": 0}) + + return seen, handler + + +def test_dql_posts_json_body_with_token_in_query(): + seen, handler = _capture() + db = Diffbot(token="secret-tok", transport=httpx.MockTransport(handler)) + db.dql('type:Organization name:"Diffbot"') + (req,) = seen + assert req.method == "POST" + assert req.headers["content-type"] == "application/json" + assert req.url.params["token"] == "secret-tok" + assert "query" not in req.url.params + assert "Diffbot" not in str(req.url) + body = json.loads(req.content) + assert body == {"query": 'type:Organization name:"Diffbot"', "size": 10} + assert "token" not in body + + +def test_dql_body_defaults_and_options(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.dql("q", size=0, from_=5, format="csv", filter="f", exportspec="a;b", extra={"jsonmode": "extended", "n": 3}) + body = json.loads(seen[0].content) + assert body == { + "query": "q", "size": 0, "from": 5, "format": "csv", "filter": "f", + "exportspec": "a;b", "jsonmode": "extended", "n": 3, + } + seen.clear() + db.dql("q", format="json", from_=0) + assert json.loads(seen[0].content) == {"query": "q", "size": 10} + + +def test_dql_long_query_is_not_in_url(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + query = " OR ".join(f'name:"Company {i}"' for i in range(1000)) + assert len(query) > 8000 + db.dql(query, size=0) + assert len(str(seen[0].url)) < 200 + assert json.loads(seen[0].content)["query"] == query + + +def test_dql_retries_post_on_503(): + calls = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request.method) + if len(calls) == 1: + return httpx.Response(503, text="busy") + return httpx.Response(200, json={"hits": 1}) + + from diffbot import RetryPolicy + + db = Diffbot(token="t", transport=httpx.MockTransport(handler), retry=RetryPolicy(base_delay=0, jitter=False)) + assert db.dql("q", size=0) == {"hits": 1} + assert calls == ["POST", "POST"] + + +def test_dql_get_escape_hatch_still_works(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.request("GET", db.dql_url, params={"token": "t", "query": "q", "size": 1}) + assert seen[0].method == "GET" + assert seen[0].url.params["query"] == "q" + + +@pytest.mark.anyio +async def test_async_dql_posts_json_body_with_token_in_query(): + seen, handler = _capture() + db = DiffbotAsync(token="secret-tok", transport=httpx.MockTransport(handler)) + await db.dql("type:Person", size=0, from_=2) + (req,) = seen + assert req.method == "POST" + assert req.url.params["token"] == "secret-tok" + assert "query" not in req.url.params + assert json.loads(req.content) == {"query": "type:Person", "size": 0, "from": 2} + + +@pytest.mark.anyio +async def test_async_dql_parallel_posts(): + seen, handler = _capture() + db = DiffbotAsync(token="t", transport=httpx.MockTransport(handler)) + await db.dql_parallel([{"query": "a", "size": 0}, {"query": "b", "size": 0}]) + assert {r.method for r in seen} == {"POST"} + assert sorted(json.loads(r.content)["query"] for r in seen) == ["a", "b"] diff --git a/tests/test_dql_cli.py b/tests/test_dql_cli.py index 2592039..c96e6b3 100644 --- a/tests/test_dql_cli.py +++ b/tests/test_dql_cli.py @@ -79,8 +79,8 @@ def test_cli_ontology_types(ontology_cache): def test_cli_probe_table(monkeypatch): def handler(request: httpx.Request) -> httpx.Response: - assert request.url.params["size"] == "0" - hits = 5 if "Diffbot" in request.url.params["query"] else 100 + assert json.loads(request.content)["size"] == 0 + hits = 5 if "Diffbot" in json.loads(request.content)["query"] else 100 return httpx.Response(200, json={"hits": hits, "results": 0}) use_mock_client(monkeypatch, handler) @@ -107,7 +107,7 @@ def handler(request): def test_cli_export_writes_file(monkeypatch, tmp_path): def handler(request: httpx.Request) -> httpx.Response: - assert request.url.params["format"] == "csv" + assert json.loads(request.content)["format"] == "csv" return httpx.Response(200, content=b"Name\nDiffbot\n") use_mock_client(monkeypatch, handler) diff --git a/tests/test_ontology.py b/tests/test_ontology.py index a6bcea1..a91c4dc 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -105,7 +105,7 @@ def handler(request: httpx.Request) -> httpx.Response: @pytest.mark.anyio async def test_async_dql_parallel_runs_all_queries(): def handler(request: httpx.Request) -> httpx.Response: - q = request.url.params["query"] + q = json.loads(request.content)["query"] hits = 5 if "Diffbot" in q else 100 return httpx.Response(200, json={"hits": hits, "results": 0}) diff --git a/tests/test_readme_examples.py b/tests/test_readme_examples.py index 31a4588..003a14a 100644 --- a/tests/test_readme_examples.py +++ b/tests/test_readme_examples.py @@ -144,7 +144,7 @@ def handler(request: httpx.Request) -> httpx.Response: def test_readme_sync_dql(): def handler(request: httpx.Request) -> httpx.Response: - assert "Diffbot" in request.url.params["query"] + assert "Diffbot" in json.loads(request.content)["query"] return httpx.Response(200, json={"data": [{"entity": {"name": "Diffbot"}}]}) db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) diff --git a/uv.lock b/uv.lock index 86a39f0..165aa2c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.10'", @@ -14,9 +14,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "idna", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "exceptiongroup" }, + { name = "idna" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ @@ -31,9 +31,9 @@ resolution-markers = [ "python_full_version >= '3.10'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and python_full_version < '3.15'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } wheels = [ @@ -57,7 +57,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } wheels = [ @@ -72,7 +72,7 @@ resolution-markers = [ "python_full_version >= '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } wheels = [ @@ -90,7 +90,7 @@ wheels = [ [[package]] name = "diffbot" -version = "3.0.1" +version = "3.1.0" source = { editable = "." } dependencies = [ { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, @@ -119,8 +119,8 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -206,7 +206,7 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "mdurl", marker = "python_full_version < '3.10'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload-time = "2023-06-03T06:41:14.443Z" } wheels = [ @@ -221,7 +221,7 @@ resolution-markers = [ "python_full_version >= '3.10'", ] dependencies = [ - { name = "mdurl", marker = "python_full_version >= '3.10'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ @@ -272,13 +272,13 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } wheels = [ @@ -293,13 +293,13 @@ resolution-markers = [ "python_full_version >= '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ From 0d08d8ed4482de0ae1a91625fcab7fac84319707 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:52:10 -1000 Subject: [PATCH 05/31] ci: live tests on push to main, workflow_dispatch, cron 17 14 1,15 * * Separate workflow, never on pull requests, concurrency of 1. Adds a cheap live DQL POST/entities suite asserting on shape only. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/live-tests.yml | 38 +++++++++++++++++++++++++++++ tests/test_live.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 .github/workflows/live-tests.yml create mode 100644 tests/test_live.py diff --git a/.github/workflows/live-tests.yml b/.github/workflows/live-tests.yml new file mode 100644 index 0000000..3d2847a --- /dev/null +++ b/.github/workflows/live-tests.yml @@ -0,0 +1,38 @@ +name: Live tests + +# Real-API tests (`pytest -m live`). Deliberately never run on pull requests, +# so a fork or PR can neither spend credits nor reach the secret. +on: + push: + branches: [main] + workflow_dispatch: + schedule: + - cron: "17 14 1,15 * *" + +permissions: + contents: read + +jobs: + live: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + concurrency: + group: live-tests + cancel-in-progress: false + env: + DIFFBOT_TOKEN: ${{ secrets.DIFFBOT_TOKEN }} + # The SDK's own credential lookup reads DIFFBOT_API_TOKEN. + DIFFBOT_API_TOKEN: ${{ secrets.DIFFBOT_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Install dependencies (locked) + run: uv sync --frozen --extra dev + + - name: Run live tests + run: uv run pytest -m live -v diff --git a/tests/test_live.py b/tests/test_live.py new file mode 100644 index 0000000..afcfde8 --- /dev/null +++ b/tests/test_live.py @@ -0,0 +1,41 @@ +"""Cheap live checks against the real API (``pytest -m live``). + +Skipped by default. Assertions are on response *shape* only, never web content. +Never include token values in assertion messages. +""" +import pytest + +pytestmark = pytest.mark.live + + +def test_live_dql_post_hit_count(db): + result = db.dql("type:Organization name:Diffbot", size=0) + assert isinstance(result, dict) + assert "hits" in result + + +def test_live_dql_post_single_read(db): + result = db.dql('type:Organization name:"Diffbot"', size=1) + assert len(result["data"]) == 1 + + +def test_live_dql_post_csv(db): + body = db.dql('type:Organization name:"Diffbot"', size=1, format="csv", raw=True) + assert isinstance(body, bytes) and len(body) > 0 + + +def test_live_dql_post_jsonl(db): + body = db.dql('type:Organization name:"Diffbot"', size=1, format="jsonl", raw=True) + assert isinstance(body, bytes) and len(body) > 0 + + +def test_live_dql_long_query_beyond_url_limit(db): + query = "type:Organization or(" + ",".join(f'name:"Acme Widgets {i}"' for i in range(600)) + ")" + assert len(query) > 8192 + result = db.dql(query, size=0) + assert "hits" in result + + +def test_live_entities(db): + result = db.entities("Diffbot is a company based in Menlo Park.") + assert "entities" in result From 2eecaf41f7a5c06727610bd19f574c5aabf1f978 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:52:18 -1000 Subject: [PATCH 06/31] docs: close Phase 0 leftovers under the reduced-scope plan Co-Authored-By: Claude Sonnet 5.5 --- PHASE0_LEFTOVERS.md | 116 +++++++++++--------------------------------- 1 file changed, 27 insertions(+), 89 deletions(-) diff --git a/PHASE0_LEFTOVERS.md b/PHASE0_LEFTOVERS.md index 60fadac..564e7d7 100644 --- a/PHASE0_LEFTOVERS.md +++ b/PHASE0_LEFTOVERS.md @@ -1,102 +1,40 @@ # Phase 0 leftovers (Python) -Everything in the Phase 0 scope (`py-prompt.md`, opus-plan.md §6 "Phase 0 — Foundation", -§3.2–3.4, §3.7) landed. A few items are deliberately smaller than the plan's framing, or -were already true on the Python side without a code change — documented here rather than -silently shipped (or silently skipped) without explanation, mirroring the TypeScript -SDK's own `PHASE0_LEFTOVERS.md` on this branch. +Everything in the Phase 0 scope landed. This file records the items that are +deliberately smaller than the original plan's framing, and how each was resolved +under the reduced-scope coverage plan (`sdk-reduced/plan.md`). It mirrors the +TypeScript SDK's own `PHASE0_LEFTOVERS.md`. -## 1. No literal "endpoint registry" data structure +## 1. Literal "endpoint registry" data structure: won't do -opus-plan §3.2/§6 0.1 frames the per-host base URLs as part of "an endpoint registry + -per-host base URLs." What landed is the per-host base URL resolution -(`api_base`/`kg_base`/`llm_base`/`nlp_base`, old `*_url` options winning aliases) directly -in `client.py` — there's no separate data structure mapping `(operation) -> {path, -auth_style, retryable}`. +The original plan framed per-host base URLs as part of "an endpoint registry". +What landed is per-host base URL resolution (`api_base`/`kg_base`/`llm_base`/ +`nlp_base`, with the older `*_url` options winning) directly in `client.py`, plus +an ad hoc `retryable=False` at the one call site that needs it +(`crawl_delete_job`). The reduced-scope plan adds no new API areas, so a +per-operation table would never get a second consumer. **Won't do.** -This mirrors the TypeScript SDK's same call on this branch: Phase 0 adds zero new -endpoints, so the only thing a registry would currently serve is base-URL resolution -(done) and a `retryable` flag (done ad hoc, as an explicit `retryable=False` at the one -call site that needs it — `crawl_delete_job`/`crawl_delete_job_async`). Building a -generic per-operation table with no second or third consumer risked guessing the wrong -shape for Phase 1+ (Account, Enhance, Bulk, Custom API, ...), which is where a real -registry starts paying for itself. Revisit when Phase 1 lands. +## 2. Success-path response headers: won't do (DQL reports cut) -## 2. Response headers: error path only, not success path - -opus-plan §6 0.6 asks for "response headers surfaced to callers," calling out an -`include_headers` option or a `*_raw` variant returning `(body, headers)`. - -What landed: every error response's headers are attached to `APIError` (and its -subclasses `AuthError`, `NotFoundError`, `RateLimitError`, `JobError`) via a new -`headers` field, populated in `_transport.raise_for_status()`. Tested in -`tests/test_transport.py`'s `raise_for_status` tests. - -What didn't land: a generic opt-in mechanism for a *successful* response's headers. The -plan's own stated motivation is `X-Diffbot-CoverageReportId` on a DQL coverage report -(opus-plan §3.4, §6 0.6's "prerequisite for DQL coverage reports") — that's a Phase 6 -operation (`dql_report`) that doesn't exist yet. Building the success-path mechanism now, -with no real caller to shape it against, risked the same guessing problem as the -registry above. Build it alongside `dql_report`/`dql_report_by_id` in Phase 6. Matches -the TypeScript SDK's identical call on this branch. +Every *error* response's headers are attached to `APIError` (and subclasses) via +`headers`, populated in `_transport.raise_for_status()`. A generic opt-in for a +*successful* response's headers was only motivated by DQL coverage reports +(`X-Diffbot-CoverageReportId`), which are out of scope. **Won't do.** ## 3. `dql(raw=True)` binary correctness: no bug existed on the Python side -opus-plan §3.3/§6 0.5 frames this as "binary + text response reads" plus a concrete bug: -the TypeScript SDK's `dql()` read the body as text and re-encoded it with -`TextEncoder`, corrupting `xls`/`xlsx` exports. The audit explicitly notes Python's byte -path was already correct (`kg.py` returning `response.content` directly). No code change -was needed here beyond routing the call through the shared `client.request()` helper, -which preserves the same `response.content` / `response.json()` split. Covered by the -existing `test_dql.py` raw-mode tests (unchanged) plus the new per-host-base tests. - -## 4. Ontology asymmetries are Phase 1, not Phase 0 - -opus-plan §5.2's core-surface-mapping row for the ontology endpoint lists adding -`dql_fetch_ontology_text()` and `dql_refresh_ontology_async()` to close the existing -Python/TypeScript asymmetry (`dql_refresh_ontology` has no async twin; TypeScript has no -raw-text fetch). That row is scoped under "Phase 1 — Account + token verification," not -Phase 0, and the `py-prompt.md` Phase 0 scope list does not mention it. Left alone. - -## 5. TypedDict/Literal annotations: existing-surface request parameters only - -`py-prompt.md` asks for "matching TypedDict(total=False)/Literal annotations, additive, -no Pydantic" as the Python counterpart to the TypeScript SDK's much larger Phase 0 type -safety push (vendored OpenAPI, generated response schemas, per-handler overloads). The -typesafety-revision page's override on this point is specifically about TypeScript's -approach; opus-plan's own phasing (§6, Phase 7 "typed responses... land as a dedicated -minor, not bundled with each endpoint" — the risk being a `TypedDict` claiming a field -that a future server response omits) argues for keeping Python's response shapes as -plain `dict`/`bytes` for now, which is what shipped. - -What landed: `src/diffbot/types.py` adds `ChatMessage`, `ResponseFormatType`, -`ResponseFormat`, `ExportFormat`, `SnippetLength`, and `CrawlJobStatusCode` (the last -documents the numeric status codes `crawl.py` already branches on; not yet wired into a -public signature since no handler takes a status code as an input). These are wired into -the existing `ask()`/`ask_json()`/`dql()`/`web_search()` request-parameter signatures in -both `client.py` and the handler modules — purely additive, no runtime behavior change -(verified by the full test suite staying green before and after). +The TypeScript SDK corrupted `xls`/`xlsx` exports by re-encoding text. Python's +byte path (`response.content`) was already correct; no code change was needed. +Unchanged by the move to POST. Covered by the existing raw-mode tests. -What didn't land: `TypedDict` overlays for *response* shapes (`DqlResponse`, -`WebSearchResponse`, `NlpResponse`, `ExtractResponse`, a `CrawlJob`/`JobStatus` pair, -`Ontology`'s wire format). This repo has no mypy/type-checking gate in CI today (checked -`.github/workflows/tests.yml`), so there is no second consumer to shape these against -yet, and retrofitting them risks exactly the staleness opus-plan §3.5/§6 Phase 7 warns -about. Revisit if/when Enhance (Phase 2) introduces the first genuinely new typed -response and a type-checking CI gate is added. +## 4. Ontology parity: scheduled for S2 -## Everything else +Adding `dql_fetch_ontology_text()` and `dql_refresh_ontology_async()` to close the +Python/TypeScript asymmetry is scheduled for **S2** (Python 3.4.0). -The sync/async de-duplication refactor (shared `_transport.py` request/response -helpers), per-host bases, repeated query parameters, DELETE + arbitrary method/body/ -Content-Type, form-encoded and raw bodies, per-call timeout, the default-on retry policy -with per-call `retryable=False` opt-out, `NotFoundError`/`JobError` (both subclassing -`APIError`), and the streaming `ask()` `ResponseNotRead` fix (sync and async, with tests -using a real byte-stream mock transport so the fix is actually exercised) all landed — -see the commit log for the breakdown and `tests/test_transport.py`, `tests/test_client.py`, -`tests/test_ask.py`, `tests/test_crawl.py`, `tests/test_dql.py` for coverage. +## 5. Response TypedDicts: scheduled for S1 -The TypeScript SDK's two fixes (dql raw-mode binary corruption, crawl CSV quote-aware -parsing) and its full type-safety push (vendored OpenAPI, generated types, per-handler -overloads, typescript-eslint strict-type-checked) are not part of this checkout; they -were implemented first on the matching TypeScript branch per the assignment. +Request-side `TypedDict`/`Literal` aliases landed in `src/diffbot/types.py`. +Response shapes (`DqlResponse`, `WebSearchResponse`, `ExtractResponse`, `CrawlJob`, +`AccountResponse`, `AskChunk`, ...) are scheduled for **S1** (Python 3.3.0), +together with a pyright CI gate so they are checked from day one. From b0ae50b28ce3d37732fcd90e16fcfa6af92cfad9 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:52:18 -1000 Subject: [PATCH 07/31] docs(changelog): name 3.2.0, add Changed section and DQL POST migration note Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 47 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ffef2..c6e7518 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,48 @@ # Changelog -## Unreleased +## 3.2.0 (unreleased) -Phase 0 of the Diffbot SDKs' full HTTP API coverage plan: shared transport -foundation and a sync/async de-duplication refactor. No new API endpoints -land in this phase, and the package version is unchanged (per the plan's -scope, version bumps are deferred). +Shared transport foundation, a sync/async de-duplication refactor, and DQL sent +as a POST. The package version in `pyproject.toml` is bumped at release time. + +### Changed + +- **DQL is now sent as `POST` with a JSON body.** `dql()`, `dql_parallel()` and + their async twins keep the same names, options, defaults and return values; + only the wire format changes: `POST {dql_url}?token=…` with + `{"query", "size", "from"?, "format"?, "filter"?, "exportspec"?, …extra}`. + The token stays in the query string; the query text no longer appears in the + URL, and the URL-length limit on long queries is gone. `extra` is now typed + `Mapping[str, Any]` (values go into the JSON body verbatim). +- `crawl()` failure now raises `JobError` (carrying the job name) instead of a + synthesized `APIError(500, ...)`. `JobError` subclasses `APIError`. + +#### Migration note: mocks and proxies + +Test mocks keyed on `GET` and `?query=` stop matching. Before/after: + +```python +# before (3.1.x) +def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.params["query"] == "type:Organization" + return httpx.Response(200, json={"data": []}) + +# after (3.2.0) +def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert json.loads(request.content)["query"] == "type:Organization" + return httpx.Response(200, json={"data": []}) +``` + +GET-only proxies, caches, or a custom `dql_url` mirror may reject or not cache +POSTs. If you need the old wire format, use the low-level call (no new API): + +```python +db.request("GET", db.dql_url, params={"token": db.token, "query": q, "size": 10}) +``` + +or pin `diffbot<3.2`. ### Added From 2652c892cc138592e25dac030be55950a2f47b92 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:02:21 -1000 Subject: [PATCH 08/31] fix(errors): fall back to the error key for APIError.message Several Diffbot APIs answer with {errorCode, error} rather than {message}. Mirror the TypeScript SDK's tryParseErrorBody so APIError.message (and str(err)) carry that text, and ignore non-string message/requestId values. Co-Authored-By: Claude Opus 5.5 --- src/diffbot/errors.py | 15 +++++++++++---- tests/test_transport.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/diffbot/errors.py b/src/diffbot/errors.py index 30cc9de..cc54689 100644 --- a/src/diffbot/errors.py +++ b/src/diffbot/errors.py @@ -23,10 +23,17 @@ def __init__(self, status_code: int, body: str, headers: Optional[Dict[str, str] self.request_id: Optional[str] = None try: data = json.loads(body) - self.message = data.get("message") - self.request_id = data.get("requestId") - except (ValueError, AttributeError): - pass + except ValueError: + data = None + if isinstance(data, dict): + # Several Diffbot APIs use ``{"errorCode": ..., "error": "..."}`` instead of + # ``message``; fall back to it (parity with the TypeScript SDK). + message = data.get("message") + if not isinstance(message, str): + message = data.get("error") + self.message = message if isinstance(message, str) else None + request_id = data.get("requestId") + self.request_id = request_id if isinstance(request_id, str) else None display = self.message or (body[:200] + ("..." if len(body) > 200 else "")) super().__init__(f"Diffbot API error {status_code}: {display}") diff --git a/tests/test_transport.py b/tests/test_transport.py index 271b5d3..6ec5456 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -229,3 +229,20 @@ def test_raise_for_status_throws_rate_limit_error_for_429_with_retry_after(): with pytest.raises(RateLimitError) as exc_info: raise_for_status(response) assert exc_info.value.retry_after == "7" + + +def test_api_error_message_falls_back_to_error_key(): + from diffbot.errors import APIError + + err = APIError(400, '{"errorCode": 505, "error": "name is over 30 chars"}') + assert err.message == "name is over 30 chars" + assert "name is over 30 chars" in str(err) + + +def test_api_error_message_prefers_message_and_ignores_non_dict_bodies(): + from diffbot.errors import APIError + + assert APIError(400, '{"message": "m", "error": "e", "requestId": "r1"}').message == "m" + assert APIError(400, '{"message": "m"}').request_id is None + assert APIError(400, '["not", "a", "dict"]').message is None + assert APIError(500, "plain text").message is None From 58ebff4a8b64e2bb7c85bf2a36cc4bd35334d90f Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:02:21 -1000 Subject: [PATCH 09/31] docs(kg): drop the stale entity-enhancement mention from the module docstring Co-Authored-By: Claude Opus 5.5 --- src/diffbot/kg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffbot/kg.py b/src/diffbot/kg.py index 1d79766..0cccf85 100644 --- a/src/diffbot/kg.py +++ b/src/diffbot/kg.py @@ -1,4 +1,4 @@ -"""Diffbot Knowledge Graph APIs: DQL search and entity enhancement.""" +"""Diffbot Knowledge Graph APIs: DQL search and the ontology.""" import asyncio import pathlib From a2913773e1098df8239b3e564f36d138373c6339 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:02:30 -1000 Subject: [PATCH 10/31] docs(changelog): note the APIError.message fallback in 3.2.0 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6e7518..12b1e25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,9 @@ or pin `diffbot<3.2`. raising. - `crawl()`'s job-failure path now raises `JobError` (carrying the job name) instead of a synthesized `APIError(500, ...)`. +- `APIError.message` falls back to the body's `error` field when there is no + `message` (the `{errorCode, error}` shape several Diffbot APIs use), matching + the TypeScript SDK. ### Internal From 85a9e5cc0a09c98459d234dafa496bc143510f9a Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:59:13 -1000 Subject: [PATCH 11/31] feat: response TypedDicts, dql overloads and csv/jsonl fix, account/verify_token, crawl_download, extract options - types.py: stdlib TypedDict(total=False) response shapes (DqlResponse, WebSearchResponse, EntitiesResponse, ExtractResponse + per-API, CrawlJob, AccountResponse, AskChunk) and per-API extract option/field Literals. - dql: @overloads by format/raw; csv -> str, jsonl -> iterator of entities (previously both crashed in response.json() without raw=True). - account(days=None) / verify_token() (GET /v4/account); only AuthError -> False. - crawl_download(name, format, type, num); the crawl watcher reuses it. - extract: keyword-only timeout/mode/fields/etc; transport timeout raised to at least timeout + 10s; extra for proxy/proxyAuth. - pyright dev dependency + config (basic on src/, strict on types.py), CLI annotation fixes, tests/typing assert_type checks. Co-Authored-By: Claude Sonnet 5.5 --- pyproject.toml | 7 + src/diffbot/__init__.py | 97 ++++++++- src/diffbot/account.py | 44 ++++ src/diffbot/cli/__init__.py | 6 +- src/diffbot/cli/dql.py | 5 +- src/diffbot/cli/entities.py | 2 +- src/diffbot/client.py | 348 +++++++++++++++++++++++++++--- src/diffbot/crawl.py | 91 ++++++-- src/diffbot/extract.py | 84 +++++++- src/diffbot/kg.py | 49 ++++- src/diffbot/nlp.py | 9 +- src/diffbot/types.py | 417 +++++++++++++++++++++++++++++++++++- src/diffbot/web_search.py | 6 +- tests/test_account.py | 92 ++++++++ tests/test_crawl.py | 109 +++++++++- tests/test_dql.py | 57 +++++ tests/test_extract.py | 132 +++++++++++- tests/typing/test_types.py | 112 ++++++++++ uv.lock | 41 ++++ 19 files changed, 1622 insertions(+), 86 deletions(-) create mode 100644 src/diffbot/account.py create mode 100644 tests/test_account.py create mode 100644 tests/typing/test_types.py diff --git a/pyproject.toml b/pyproject.toml index 054b5b5..1e03c13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ [project.optional-dependencies] dev = [ "pytest>=8.0.0", + "pyright>=1.1.390", ] [project.urls] @@ -77,3 +78,9 @@ include = [ [tool.pytest.ini_options] markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_API_TOKEN"] addopts = "-m 'not live'" + +[tool.pyright] +include = ["src", "tests/typing"] +pythonVersion = "3.9" +typeCheckingMode = "basic" +strict = ["src/diffbot/types.py"] diff --git a/src/diffbot/__init__.py b/src/diffbot/__init__.py index db7c900..462ea20 100644 --- a/src/diffbot/__init__.py +++ b/src/diffbot/__init__.py @@ -48,7 +48,57 @@ def _installed_version(dist: str): ) from .ontology import Ontology from ._transport import RetryPolicy -from .types import ChatMessage, ExportFormat, ResponseFormat, SnippetLength +from .types import ( + ChatMessage, + CrawlJobStatusCode, + ExportFormat, + ResponseFormat, + SnippetLength, + AccountResponse, + AccountUsage, + AnalyzeExtractOptions, + ArticleExtractOptions, + ArticleExtractResponse, + ArticleObject, + ArticleTag, + AskChunk, + CrawlDataRecord, + CrawlDownloadFormat, + CrawlJob, + CrawlJobStatus, + CustomExtractOptions, + DiscussionExtractOptions, + DiscussionExtractResponse, + DiscussionObject, + DqlEntity, + DqlResponse, + DqlRewrite, + EntitiesResponse, + EventExtractOptions, + EventExtractResponse, + EventObject, + ExtractApi, + ExtractObject, + ExtractResponse, + ImageExtractOptions, + ImageExtractResponse, + ImageObject, + JobExtractOptions, + JobExtractResponse, + JobObject, + ListExtractOptions, + ListExtractResponse, + ListObject, + NlpEntity, + ProductExtractOptions, + ProductExtractResponse, + ProductObject, + VideoExtractOptions, + VideoExtractResponse, + VideoObject, + WebSearchResponse, + WebSearchResult, +) __all__ = [ "Diffbot", @@ -63,6 +113,51 @@ def _installed_version(dist: str): "ResponseFormat", "ExportFormat", "SnippetLength", + "CrawlJobStatusCode", + "AccountResponse", + "AccountUsage", + "AnalyzeExtractOptions", + "ArticleExtractOptions", + "ArticleExtractResponse", + "ArticleObject", + "ArticleTag", + "AskChunk", + "CrawlDataRecord", + "CrawlDownloadFormat", + "CrawlJob", + "CrawlJobStatus", + "CustomExtractOptions", + "DiscussionExtractOptions", + "DiscussionExtractResponse", + "DiscussionObject", + "DqlEntity", + "DqlResponse", + "DqlRewrite", + "EntitiesResponse", + "EventExtractOptions", + "EventExtractResponse", + "EventObject", + "ExtractApi", + "ExtractObject", + "ExtractResponse", + "ImageExtractOptions", + "ImageExtractResponse", + "ImageObject", + "JobExtractOptions", + "JobExtractResponse", + "JobObject", + "ListExtractOptions", + "ListExtractResponse", + "ListObject", + "NlpEntity", + "ProductExtractOptions", + "ProductExtractResponse", + "ProductObject", + "VideoExtractOptions", + "VideoExtractResponse", + "VideoObject", + "WebSearchResponse", + "WebSearchResult", "DiffbotError", "AuthError", "ExtractionError", diff --git a/src/diffbot/account.py b/src/diffbot/account.py new file mode 100644 index 0000000..a7b4352 --- /dev/null +++ b/src/diffbot/account.py @@ -0,0 +1,44 @@ +"""Diffbot Account API: usage and token verification (``GET /v4/account``).""" + +from typing import TYPE_CHECKING, Any, Dict, Optional + +from .errors import AuthError +from .types import AccountResponse + +if TYPE_CHECKING: + from .client import Diffbot, DiffbotAsync + + +def _params(client: Any, days: Optional[int]) -> Dict[str, Any]: + params: Dict[str, Any] = {"token": client.token} + if days is not None: + params["days"] = days + return params + + +def account(client: "Diffbot", days: Optional[int] = None) -> AccountResponse: + response = client.request("GET", client.account_url, params=_params(client, days)) + return response.json() + + +async def account_async(client: "DiffbotAsync", days: Optional[int] = None) -> AccountResponse: + response = await client.request("GET", client.account_url, params=_params(client, days)) + return response.json() + + +def verify_token(client: "Diffbot") -> bool: + """``True`` if the token is accepted. Only :class:`AuthError` maps to ``False``; + every other failure (network, 5xx, rate limit) propagates.""" + try: + account(client, days=1) + except AuthError: + return False + return True + + +async def verify_token_async(client: "DiffbotAsync") -> bool: + try: + await account_async(client, days=1) + except AuthError: + return False + return True diff --git a/src/diffbot/cli/__init__.py b/src/diffbot/cli/__init__.py index c26a035..a1a7dc8 100644 --- a/src/diffbot/cli/__init__.py +++ b/src/diffbot/cli/__init__.py @@ -2,6 +2,7 @@ import re import sys import textwrap +from typing import List, Optional from email.utils import parsedate_to_datetime import click @@ -17,6 +18,7 @@ AuthError, ExtractionError, APIError, + ChatMessage, ValidationError, json_schema_format, ) @@ -100,7 +102,7 @@ def extract(url, output, fmt, api): type=click.Path(exists=True, dir_okay=False), help="JSON Schema file constraining the output (implies --json)", ) -def ask(prompt: str, output: str = None, as_json: bool = False, schema_path: str = None): +def ask(prompt: str, output: Optional[str] = None, as_json: bool = False, schema_path: Optional[str] = None): """Ask a question to the Diffbot LLM""" db = get_client() stdin_content = sys.stdin.read() if not sys.stdin.isatty() else None @@ -119,7 +121,7 @@ def ask(prompt: str, output: str = None, as_json: bool = False, schema_path: str # permissive default; --json alone must not become {"type": "json_object"}. interactive_mode = is_interactive and not output and not as_json - messages = [ + messages: List[ChatMessage] = [ { "role": "system", "content": f"Current local time: {datetime.now().strftime('%A, %B %d, %Y, %I:%M:%S %p %Z')}", diff --git a/src/diffbot/cli/dql.py b/src/diffbot/cli/dql.py index fdb0203..1f8c7a9 100644 --- a/src/diffbot/cli/dql.py +++ b/src/diffbot/cli/dql.py @@ -5,6 +5,7 @@ import re import shutil import time +from typing import cast import click import sys @@ -12,7 +13,7 @@ from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table -from diffbot import DiffbotError +from diffbot import DiffbotError, ExportFormat from . import ontology from diffbot import resolve_token @@ -258,7 +259,7 @@ def export(query: str, out, fmt: str, exportspec, size: int, from_: int, filter_ query, size=size, from_=from_, - format=fmt, + format=cast(ExportFormat, fmt), exportspec=resolved_spec or None, filter=filter_, raw=True, diff --git a/src/diffbot/cli/entities.py b/src/diffbot/cli/entities.py index 255ca15..e4592d5 100644 --- a/src/diffbot/cli/entities.py +++ b/src/diffbot/cli/entities.py @@ -129,7 +129,7 @@ def entities(text, fmt, lang): color = _PALETTE[i % len(_PALETTE)] name = e.get("name") or e.get("label") or "" all_types = e.get("allTypes") or [] - etype = all_types[0]["name"] if all_types else (e.get("type") or "") + etype = all_types[0].get("name", "") if all_types else (e.get("type") or "") confidence = e.get("confidence") salience = e.get("salience") sentiment = e.get("sentiment") diff --git a/src/diffbot/client.py b/src/diffbot/client.py index 89247c4..a67c4ef 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -2,7 +2,7 @@ import pathlib from types import TracebackType -from typing import Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, Sequence, Type, Union +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Literal, Mapping, Optional, Sequence, Type, Union, overload import httpx @@ -16,7 +16,45 @@ resolve_retry_policy, ) from .errors import ValidationError -from .types import ChatMessage, ExportFormat, ResponseFormat, SnippetLength +from .types import ( + AccountResponse, + AnalyzeExtractOptions, + ArticleExtractOptions, + ArticleExtractResponse, + ChatMessage, + CrawlDataRecord, + CrawlDownloadFormat, + CrawlJob, + CustomExtractOptions, + DiscussionExtractOptions, + DiscussionExtractResponse, + DqlEntity, + DqlResponse, + EntitiesResponse, + EventExtractOptions, + EventExtractResponse, + ExportFormat, + ExtractResponse, + ImageExtractOptions, + ImageExtractResponse, + JobExtractOptions, + JobExtractResponse, + ListExtractOptions, + ListExtractResponse, + ProductExtractOptions, + ProductExtractResponse, + ResponseFormat, + SnippetLength, + VideoExtractOptions, + VideoExtractResponse, + WebSearchResponse, +) +from .account import ( + account as _account, + account_async as _account_async, + verify_token as _verify_token, + verify_token_async as _verify_token_async, +) from .extract import extract as _extract, extract_async as _extract_async from .ask import ( ask as _ask, @@ -30,6 +68,8 @@ crawl_async as _crawl_async, crawl_delete_job as _crawl_delete_job, crawl_delete_job_async as _crawl_delete_job_async, + crawl_download as _crawl_download, + crawl_download_async as _crawl_download_async, crawl_get_job as _crawl_get_job, crawl_get_job_async as _crawl_get_job_async, crawl_list_jobs as _crawl_list_jobs, @@ -55,6 +95,11 @@ ) +if TYPE_CHECKING: + # Only the type checker needs this (``**options: Unpack[...]`` in the overloads below). + from typing_extensions import Unpack + + class Diffbot: """Client for the Diffbot APIs. @@ -96,6 +141,7 @@ def __init__( # still allowing a single endpoint to be overridden independently. self.analyze_url = analyze_url or f"{api_base}/v3" self.crawler_url = crawler_url or f"{api_base}/v3/crawl" + self.account_url = f"{api_base}/v4/account" self.llm_url = llm_url or f"{llm_base}/rag/v1/chat/completions" self.web_search_url = web_search_url or f"{llm_base}/api/v1/web_search" self.nlp_url = nlp_url or f"{nlp_base}/v1/" @@ -159,9 +205,57 @@ def request( stream=stream, ) - def extract(self, url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: - """Extract structured content from a URL. Returns the raw Diffbot API response.""" - return _extract(self, url, api=api, fmt=fmt) + @overload + def extract( + self, url: str, api: Literal["analyze"] = "analyze", fmt: str = ..., **options: "Unpack[AnalyzeExtractOptions]" + ) -> ExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["article"], fmt: str = ..., **options: "Unpack[ArticleExtractOptions]" + ) -> ArticleExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["product"], fmt: str = ..., **options: "Unpack[ProductExtractOptions]" + ) -> ProductExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["discussion"], fmt: str = ..., **options: "Unpack[DiscussionExtractOptions]" + ) -> DiscussionExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["job"], fmt: str = ..., **options: "Unpack[JobExtractOptions]" + ) -> JobExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["image"], fmt: str = ..., **options: "Unpack[ImageExtractOptions]" + ) -> ImageExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["video"], fmt: str = ..., **options: "Unpack[VideoExtractOptions]" + ) -> VideoExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["list"], fmt: str = ..., **options: "Unpack[ListExtractOptions]" + ) -> ListExtractResponse: ... + @overload + def extract( + self, url: str, api: Literal["event"], fmt: str = ..., **options: "Unpack[EventExtractOptions]" + ) -> EventExtractResponse: ... + @overload + def extract(self, url: str, api: str, fmt: str = ..., **options: "Unpack[CustomExtractOptions]") -> ExtractResponse: ... + def extract(self, url: str, api: str = "analyze", fmt: str = "markdown", **options: Any) -> Any: + """Extract structured content from a URL. Returns the raw Diffbot API response. + + Keyword options (all optional, additive): ``timeout`` (server-side fetch + timeout in ms, default 30000; the transport timeout is raised to at least + ``timeout + 10s``), ``mode`` (overrides the ``fmt="markdown"`` -> ``llm`` + mapping), ``fields`` (sequence of field names, comma-joined), ``fallback``, + ``discussion``, ``paging``, ``max_pages``, ``natural_language``, + ``summary_num_sentences``, ``max_tags``, ``tag_confidence``, + ``category_confidence``, ``use_proxy``, and ``extra`` (a mapping of extra wire + parameters, e.g. ``proxy``/``proxyAuth``). Which apply depends on ``api``. + """ + return _extract(self, url, api=api, fmt=fmt, **options) def ask( self, @@ -191,21 +285,77 @@ def ask_json( return _ask_json(self, messages, schema, response_format=response_format) def crawl(self, site: str, **kwargs: Any) -> Iterator[CrawlEvent]: - """Start a crawl job.""" + """Start a crawl job. + + For tools that can't block on a long crawl, use ``crawl(site, watch=False)`` + (the default), poll :meth:`crawl_get_job`, then fetch results with + :meth:`crawl_download`. + """ yield from _crawl(self, site, **kwargs) - def crawl_list_jobs(self) -> List[Dict[str, Any]]: + def crawl_list_jobs(self) -> List[CrawlJob]: """List all crawler jobs for this token.""" return _crawl_list_jobs(self) - def crawl_get_job(self, job_name: str) -> Dict[str, Any]: - """Get the status of a crawler job.""" + def crawl_get_job(self, job_name: str) -> CrawlJob: + """Get a crawler job's record, or ``{}`` if no such job exists (the TypeScript SDK + returns ``undefined`` there; Python keeps ``{}`` so ``job.get(...)`` stays safe).""" return _crawl_get_job(self, job_name) + @overload + def crawl_download(self, name: str, *, format: Literal["csv"], type: Optional[str] = ..., num: Optional[int] = ...) -> str: ... + @overload + def crawl_download( + self, name: str, *, format: Literal["json"] = ..., type: Optional[str] = ..., num: Optional[int] = ... + ) -> List[CrawlDataRecord]: ... + @overload + def crawl_download( + self, name: str, *, format: CrawlDownloadFormat, type: Optional[str] = ..., num: Optional[int] = ... + ) -> Union[str, List[CrawlDataRecord]]: ... + def crawl_download( + self, + name: str, + *, + format: CrawlDownloadFormat = "json", + type: Optional[str] = None, + num: Optional[int] = None, + ) -> Any: + """Download a crawl job's results (``GET /v3/crawl/data``). + + ``format="json"`` (default) returns a list of records; ``format="csv"`` + returns the CSV text. ``type`` selects the report (e.g. ``"urls"``) and + ``num`` caps the number of records. + """ + return _crawl_download(self, name, format=format, type=type, num=num) + def crawl_delete_job(self, job_name: str) -> None: """Delete a crawler job.""" _crawl_delete_job(self, job_name) + @overload + def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[True], + ) -> bytes: ... + @overload + def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["csv"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> str: ... + @overload + def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["xls", "xlsx"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> bytes: ... + @overload + def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["jsonl"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> Iterator[DqlEntity]: ... + @overload + def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["json"] = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> DqlResponse: ... + @overload + def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: bool = ... + ) -> Union[DqlResponse, str, bytes, Iterator[DqlEntity]]: ... def dql( self, query: str, @@ -217,16 +367,19 @@ def dql( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, - ) -> Union[Dict[str, Any], bytes]: - """Run a DQL query against the Diffbot Knowledge Graph. + ) -> Any: + """Run a DQL query against the Diffbot Knowledge Graph (sent as ``POST`` with a JSON body). - Returns the parsed JSON response, or the raw response bytes when raw=True - (e.g. to retrieve CSV/export formats undecoded). + Return type follows the arguments: ``DqlResponse`` for ``format="json"`` + (default); ``str`` for ``"csv"``; ``bytes`` for ``"xls"``/``"xlsx"`` or any + format with ``raw=True`` (the undecoded response); an iterator of + ``DqlEntity`` for ``"jsonl"``. """ return _dql(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw) - def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Union[Dict[str, Any], bytes]]: - """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args.""" + def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Any]: + """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args; + each result's type follows that call, as for dql().""" return _dql_parallel(self, queries, workers=workers) def dql_refresh_ontology(self, dest: pathlib.Path) -> None: @@ -244,7 +397,7 @@ def web_search( num_results: Optional[int] = None, max_tokens: Optional[int] = None, snippet_length: Optional[SnippetLength] = None, - ) -> Dict[str, Any]: + ) -> WebSearchResponse: """Search the web via the Diffbot LLM web search API. snippet_length: "short", "medium", "long" or "full" — how much of each @@ -254,7 +407,22 @@ def web_search( """ return _web_search(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length) - def entities(self, text: str, *, lang: str = "auto") -> Dict[str, Any]: + def account(self, days: Optional[int] = None) -> AccountResponse: + """Account details and usage (``GET /v4/account``; free). ``days`` limits the usage window. + + The response includes ``token`` and ``childTokens``: never log or print it whole. + """ + return _account(self, days) + + def verify_token(self) -> bool: + """``True`` if the token is accepted, ``False`` on an auth failure (401/403). + + Calls :meth:`account` with ``days=1``. Any other failure (network, 5xx, rate + limit) propagates rather than reading as "invalid token". + """ + return _verify_token(self) + + def entities(self, text: str, *, lang: str = "auto") -> EntitiesResponse: """Identify and resolve entities and sentiment in text using the Diffbot NLP API. Entity IDs can be looked up in the Knowledge Graph via dql() using @@ -299,6 +467,7 @@ def __init__( self.nlp_base = nlp_base self.analyze_url = analyze_url or f"{api_base}/v3" self.crawler_url = crawler_url or f"{api_base}/v3/crawl" + self.account_url = f"{api_base}/v4/account" self.llm_url = llm_url or f"{llm_base}/rag/v1/chat/completions" self.web_search_url = web_search_url or f"{llm_base}/api/v1/web_search" self.nlp_url = nlp_url or f"{nlp_base}/v1/" @@ -357,9 +526,57 @@ async def request( stream=stream, ) - async def extract(self, url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: - """Extract structured content from a URL. Returns the raw Diffbot API response.""" - return await _extract_async(self, url, api=api, fmt=fmt) + @overload + async def extract( + self, url: str, api: Literal["analyze"] = "analyze", fmt: str = ..., **options: "Unpack[AnalyzeExtractOptions]" + ) -> ExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["article"], fmt: str = ..., **options: "Unpack[ArticleExtractOptions]" + ) -> ArticleExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["product"], fmt: str = ..., **options: "Unpack[ProductExtractOptions]" + ) -> ProductExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["discussion"], fmt: str = ..., **options: "Unpack[DiscussionExtractOptions]" + ) -> DiscussionExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["job"], fmt: str = ..., **options: "Unpack[JobExtractOptions]" + ) -> JobExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["image"], fmt: str = ..., **options: "Unpack[ImageExtractOptions]" + ) -> ImageExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["video"], fmt: str = ..., **options: "Unpack[VideoExtractOptions]" + ) -> VideoExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["list"], fmt: str = ..., **options: "Unpack[ListExtractOptions]" + ) -> ListExtractResponse: ... + @overload + async def extract( + self, url: str, api: Literal["event"], fmt: str = ..., **options: "Unpack[EventExtractOptions]" + ) -> EventExtractResponse: ... + @overload + async def extract(self, url: str, api: str, fmt: str = ..., **options: "Unpack[CustomExtractOptions]") -> ExtractResponse: ... + async def extract(self, url: str, api: str = "analyze", fmt: str = "markdown", **options: Any) -> Any: + """Extract structured content from a URL. Returns the raw Diffbot API response. + + Keyword options (all optional, additive): ``timeout`` (server-side fetch + timeout in ms, default 30000; the transport timeout is raised to at least + ``timeout + 10s``), ``mode`` (overrides the ``fmt="markdown"`` -> ``llm`` + mapping), ``fields`` (sequence of field names, comma-joined), ``fallback``, + ``discussion``, ``paging``, ``max_pages``, ``natural_language``, + ``summary_num_sentences``, ``max_tags``, ``tag_confidence``, + ``category_confidence``, ``use_proxy``, and ``extra`` (a mapping of extra wire + parameters, e.g. ``proxy``/``proxyAuth``). Which apply depends on ``api``. + """ + return await _extract_async(self, url, api=api, fmt=fmt, **options) async def ask( self, @@ -394,18 +611,69 @@ async def crawl(self, site: str, **kwargs: Any) -> AsyncIterator[CrawlEvent]: async for event in _crawl_async(self, site, **kwargs): yield event - async def crawl_list_jobs(self) -> List[Dict[str, Any]]: + async def crawl_list_jobs(self) -> List[CrawlJob]: """List all crawler jobs for this token.""" return await _crawl_list_jobs_async(self) - async def crawl_get_job(self, job_name: str) -> Dict[str, Any]: - """Get the status of a crawler job.""" + async def crawl_get_job(self, job_name: str) -> CrawlJob: + """Get a crawler job's record, or ``{}`` if no such job exists (the TypeScript SDK + returns ``undefined`` there; Python keeps ``{}`` so ``job.get(...)`` stays safe).""" return await _crawl_get_job_async(self, job_name) + @overload + async def crawl_download(self, name: str, *, format: Literal["csv"], type: Optional[str] = ..., num: Optional[int] = ...) -> str: ... + @overload + async def crawl_download( + self, name: str, *, format: Literal["json"] = ..., type: Optional[str] = ..., num: Optional[int] = ... + ) -> List[CrawlDataRecord]: ... + @overload + async def crawl_download( + self, name: str, *, format: CrawlDownloadFormat, type: Optional[str] = ..., num: Optional[int] = ... + ) -> Union[str, List[CrawlDataRecord]]: ... + async def crawl_download( + self, + name: str, + *, + format: CrawlDownloadFormat = "json", + type: Optional[str] = None, + num: Optional[int] = None, + ) -> Any: + """Download a crawl job's results (``GET /v3/crawl/data``). + + ``format="json"`` (default) returns a list of records; ``format="csv"`` + returns the CSV text. ``type`` selects the report (e.g. ``"urls"``) and + ``num`` caps the number of records. + """ + return await _crawl_download_async(self, name, format=format, type=type, num=num) + async def crawl_delete_job(self, job_name: str) -> None: """Delete a crawler job.""" await _crawl_delete_job_async(self, job_name) + @overload + async def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[True], + ) -> bytes: ... + @overload + async def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["csv"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> str: ... + @overload + async def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["xls", "xlsx"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> bytes: ... + @overload + async def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["jsonl"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> AsyncIterator[DqlEntity]: ... + @overload + async def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["json"] = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + ) -> DqlResponse: ... + @overload + async def dql( + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: bool = ... + ) -> Union[DqlResponse, str, bytes, AsyncIterator[DqlEntity]]: ... async def dql( self, query: str, @@ -417,16 +685,19 @@ async def dql( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, - ) -> Union[Dict[str, Any], bytes]: - """Run a DQL query against the Diffbot Knowledge Graph. + ) -> Any: + """Run a DQL query against the Diffbot Knowledge Graph (sent as ``POST`` with a JSON body). - Returns the parsed JSON response, or the raw response bytes when raw=True - (e.g. to retrieve CSV/export formats undecoded). + Return type follows the arguments: ``DqlResponse`` for ``format="json"`` + (default); ``str`` for ``"csv"``; ``bytes`` for ``"xls"``/``"xlsx"`` or any + format with ``raw=True`` (the undecoded response); an iterator of + ``DqlEntity`` for ``"jsonl"``. """ return await _dql_async(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw) - async def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Union[Dict[str, Any], bytes]]: - """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args.""" + async def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Any]: + """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args; + each result's type follows that call, as for dql().""" return await _dql_parallel_async(self, queries, workers=workers) async def dql_fetch_ontology(self) -> Ontology: @@ -440,7 +711,7 @@ async def web_search( num_results: Optional[int] = None, max_tokens: Optional[int] = None, snippet_length: Optional[SnippetLength] = None, - ) -> Dict[str, Any]: + ) -> WebSearchResponse: """Search the web via the Diffbot LLM web search API. snippet_length: "short", "medium", "long" or "full" — how much of each @@ -450,7 +721,22 @@ async def web_search( """ return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length) - async def entities(self, text: str, *, lang: str = "auto") -> Dict[str, Any]: + async def account(self, days: Optional[int] = None) -> AccountResponse: + """Account details and usage (``GET /v4/account``; free). ``days`` limits the usage window. + + The response includes ``token`` and ``childTokens``: never log or print it whole. + """ + return await _account_async(self, days) + + async def verify_token(self) -> bool: + """``True`` if the token is accepted, ``False`` on an auth failure (401/403). + + Calls :meth:`account` with ``days=1``. Any other failure (network, 5xx, rate + limit) propagates rather than reading as "invalid token". + """ + return await _verify_token_async(self) + + async def entities(self, text: str, *, lang: str = "auto") -> EntitiesResponse: """Identify and resolve entities and sentiment in text using the Diffbot NLP API. Entity IDs can be looked up in the Knowledge Graph via dql() using diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index c543a24..cac393c 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -3,6 +3,7 @@ import asyncio import csv import io +import json import time from dataclasses import dataclass from datetime import datetime @@ -10,6 +11,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional from .errors import JobError +from .types import CrawlDownloadFormat, CrawlJob if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -47,7 +49,11 @@ def crawl( poll_interval: float = 2.0, ) -> Iterator[CrawlEvent]: """Create a crawler job and yield a JOB_CREATED event. If watch=True, poll until - completion and yield URL_PROCESSED events for each crawled URL.""" + completion and yield URL_PROCESSED events for each crawled URL. + + Recommended pattern for tools that can't block on a long crawl: start with + ``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its + ``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`.""" if not site.startswith("http"): site = f"https://{site}" if not job_name: @@ -83,14 +89,9 @@ def crawl( job_status = jobs[0].get("jobStatus", {}) status_code = job_status.get("status", 0) - urls_response = client.request( - "GET", - f"{client.crawler_url}/data", - params={"token": client.token, "name": job_name, "type": "urls"}, - follow_redirects=True, - ) - if urls_response.content: - for event in _parse_url_csv(urls_response.text, seen_urls): + urls_csv = crawl_download(client, job_name, format="csv", type="urls") + if urls_csv: + for event in _parse_url_csv(urls_csv, seen_urls): yield event if status_code not in (0, 7): @@ -201,14 +202,9 @@ async def crawl_async( job_status = jobs[0].get("jobStatus", {}) status_code = job_status.get("status", 0) - urls_response = await client.request( - "GET", - f"{client.crawler_url}/data", - params={"token": client.token, "name": job_name, "type": "urls"}, - follow_redirects=True, - ) - if urls_response.content: - for event in _parse_url_csv(urls_response.text, seen_urls): + urls_csv = await crawl_download_async(client, job_name, format="csv", type="urls") + if urls_csv: + for event in _parse_url_csv(urls_csv, seen_urls): yield event if status_code not in (0, 7): @@ -220,12 +216,12 @@ async def crawl_async( await asyncio.sleep(poll_interval) -async def crawl_list_jobs_async(client: "DiffbotAsync") -> List[Dict[str, Any]]: +async def crawl_list_jobs_async(client: "DiffbotAsync") -> List[CrawlJob]: response = await client.request("GET", client.crawler_url, params={"token": client.token}) return response.json().get("jobs", []) -async def crawl_get_job_async(client: "DiffbotAsync", job_name: str) -> Dict[str, Any]: +async def crawl_get_job_async(client: "DiffbotAsync", job_name: str) -> CrawlJob: response = await client.request( "GET", client.crawler_url, @@ -245,12 +241,12 @@ async def crawl_delete_job_async(client: "DiffbotAsync", job_name: str) -> None: ) -def crawl_list_jobs(client: "Diffbot") -> List[Dict[str, Any]]: +def crawl_list_jobs(client: "Diffbot") -> List[CrawlJob]: response = client.request("GET", client.crawler_url, params={"token": client.token}) return response.json().get("jobs", []) -def crawl_get_job(client: "Diffbot", job_name: str) -> Dict[str, Any]: +def crawl_get_job(client: "Diffbot", job_name: str) -> CrawlJob: response = client.request( "GET", client.crawler_url, @@ -268,3 +264,56 @@ def crawl_delete_job(client: "Diffbot", job_name: str) -> None: params={"token": client.token, "name": job_name, "delete": 1}, retryable=False, ) + + +def _download_params(client: Any, name: str, format: CrawlDownloadFormat, type: Optional[str], num: Optional[int]) -> Dict[str, Any]: + params: Dict[str, Any] = {"token": client.token, "name": name} + if format == "csv": + params["format"] = "csv" + if type is not None: + params["type"] = type + if num is not None: + params["num"] = num + return params + + +def _decode_download(text: str, format: CrawlDownloadFormat) -> Any: + if format == "csv": + return text + return json.loads(text) if text.strip() else [] + + +# The public, overloaded signatures live on ``Diffbot.crawl_download`` / +# ``DiffbotAsync.crawl_download`` (``format="csv"`` -> ``str``, else records). +def crawl_download( + client: "Diffbot", + name: str, + *, + format: CrawlDownloadFormat = "json", + type: Optional[str] = None, + num: Optional[int] = None, +) -> Any: + response = client.request( + "GET", + f"{client.crawler_url}/data", + params=_download_params(client, name, format, type, num), + follow_redirects=True, + ) + return _decode_download(response.text, format) + + +async def crawl_download_async( + client: "DiffbotAsync", + name: str, + *, + format: CrawlDownloadFormat = "json", + type: Optional[str] = None, + num: Optional[int] = None, +) -> Any: + response = await client.request( + "GET", + f"{client.crawler_url}/data", + params=_download_params(client, name, format, type, num), + follow_redirects=True, + ) + return _decode_download(response.text, format) diff --git a/src/diffbot/extract.py b/src/diffbot/extract.py index 7a4c2f3..a54928d 100644 --- a/src/diffbot/extract.py +++ b/src/diffbot/extract.py @@ -1,20 +1,82 @@ """Diffbot Analyze API: extract structured content from a URL.""" -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync -from .errors import ExtractionError +from .errors import ExtractionError, ValidationError +#: Keyword options -> wire parameter names. ``fields``/``natural_language`` take a +#: sequence and are comma-joined; everything else is sent as-is. +_OPTION_WIRE: Dict[str, str] = { + "timeout": "timeout", + "mode": "mode", + "fields": "fields", + "fallback": "fallback", + "discussion": "discussion", + "paging": "paging", + "max_pages": "maxPages", + "natural_language": "naturalLanguage", + "summary_num_sentences": "summaryNumSentences", + "max_tags": "maxTags", + "tag_confidence": "tagConfidence", + "category_confidence": "categoryConfidence", + "use_proxy": "useProxy", +} +_COMMA_JOINED = ("fields", "natural_language") +#: Wire names a caller may not smuggle in through ``extra`` (named options win). +_RESERVED_WIRE = {"token", "url"} | set(_OPTION_WIRE.values()) -def _build_params(client: Any, url: str, fmt: str) -> Dict[str, Any]: - params = {"token": client.token, "url": url, "timeout": 30000} +#: The server-side fetch timeout (ms) when the caller doesn't set one. +DEFAULT_EXTRACT_TIMEOUT_MS = 30000 +#: Headroom the transport timeout keeps over the server-side one so the client +#: doesn't abort before the server has a chance to answer. +TRANSPORT_TIMEOUT_MARGIN_S = 10.0 + + +def _build_params(client: Any, url: str, fmt: str, options: Mapping[str, Any]) -> Dict[str, Any]: + unknown = sorted(set(options) - set(_OPTION_WIRE) - {"extra"}) + if unknown: + raise TypeError(f"extract() got unexpected keyword argument(s): {', '.join(unknown)}") + params: Dict[str, Any] = {"token": client.token, "url": url, "timeout": DEFAULT_EXTRACT_TIMEOUT_MS} + # An explicit ``mode`` overrides the markdown -> llm mapping. if fmt == "markdown": params["mode"] = "llm" + for name, value in options.items(): + if name == "extra" or value is None: + continue + if name in _COMMA_JOINED and not isinstance(value, str): + value = ",".join(value) + params[_OPTION_WIRE[name]] = value + extra = options.get("extra") + if extra: + clash = sorted(_RESERVED_WIRE & set(extra)) + if clash: + raise ValidationError( + f"extra may not set {', '.join(clash)}: use the named option instead (token/url are set by the SDK)" + ) + params.update(extra) return params +def _transport_timeout(client: Any, options: Mapping[str, Any]) -> Optional[float]: + """Per-call transport timeout (seconds), or ``None`` to keep the client's. + + Only raised, never lowered: a caller-set server-side ``timeout`` (ms) needs the + transport to wait at least that long plus a margin. A client configured with no + read timeout is left alone. + """ + timeout_ms = options.get("timeout") + if timeout_ms is None: + return None + client_read = client._http.timeout.read + needed = timeout_ms / 1000 + TRANSPORT_TIMEOUT_MARGIN_S + if client_read is not None and needed > client_read: + return needed + return None + + def _parse_response(client: Any, data: Dict[str, Any]) -> Dict[str, Any]: if "errorCode" in data: raise ExtractionError(data["errorCode"], data.get("error", "")) @@ -25,21 +87,27 @@ def _normalize_url(url: str) -> str: return url if url.startswith("http") else f"https://{url}" -def extract(client: "Diffbot", url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: +# The public, per-API overloaded signatures live on ``Diffbot.extract`` / +# ``DiffbotAsync.extract``; these shared implementations return ``Any``. +def extract(client: "Diffbot", url: str, api: str = "analyze", fmt: str = "markdown", **options: Any) -> Any: url = _normalize_url(url) response = client.request( "GET", f"{client.analyze_url}/{api}", - params=_build_params(client, url, fmt), + params=_build_params(client, url, fmt, options), + timeout=_transport_timeout(client, options), ) return _parse_response(client, response.json()) -async def extract_async(client: "DiffbotAsync", url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]: +async def extract_async( + client: "DiffbotAsync", url: str, api: str = "analyze", fmt: str = "markdown", **options: Any +) -> Any: url = _normalize_url(url) response = await client.request( "GET", f"{client.analyze_url}/{api}", - params=_build_params(client, url, fmt), + params=_build_params(client, url, fmt, options), + timeout=_transport_timeout(client, options), ) return _parse_response(client, response.json()) diff --git a/src/diffbot/kg.py b/src/diffbot/kg.py index 0cccf85..e016f51 100644 --- a/src/diffbot/kg.py +++ b/src/diffbot/kg.py @@ -1,13 +1,14 @@ """Diffbot Knowledge Graph APIs: DQL search and the ontology.""" import asyncio +import json import pathlib from concurrent.futures import ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, Sequence from ._constants import KG_BASE from .ontology import Ontology -from .types import ExportFormat +from .types import DqlEntity, ExportFormat if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -41,6 +42,32 @@ def _build_dql_body( return body +def _decode_dql(response: Any, format: ExportFormat, raw: bool) -> Any: + """Decode a DQL response by ``format``: json -> dict, csv -> str, xls/xlsx -> bytes, + jsonl -> iterator of entities. ``raw=True`` always returns the undecoded bytes.""" + if raw or format in ("xls", "xlsx"): + return response.content + if format == "csv": + return response.text + if format == "jsonl": + return _iter_jsonl(response.text) + return response.json() + + +def _iter_jsonl(text: str) -> Iterator[DqlEntity]: + for line in text.splitlines(): + if line.strip(): + yield json.loads(line) + + +async def _aiter_jsonl(text: str) -> AsyncIterator[DqlEntity]: + for line in text.splitlines(): + if line.strip(): + yield json.loads(line) + + +# The public, overloaded signatures live on ``Diffbot.dql`` / ``DiffbotAsync.dql``; +# these module-level implementations return ``Any`` and are shared by both. def dql( client: "Diffbot", query: str, @@ -52,10 +79,10 @@ def dql( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, -) -> Union[Dict[str, Any], bytes]: +) -> Any: body = _build_dql_body(query, size, from_, format, filter, exportspec, extra) response = client.request("POST", client.dql_url, params={"token": client.token}, json=body) - return response.content if raw else response.json() + return _decode_dql(response, format, raw) async def dql_async( @@ -69,10 +96,12 @@ async def dql_async( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, -) -> Union[Dict[str, Any], bytes]: +) -> Any: body = _build_dql_body(query, size, from_, format, filter, exportspec, extra) response = await client.request("POST", client.dql_url, params={"token": client.token}, json=body) - return response.content if raw else response.json() + if not raw and format == "jsonl": + return _aiter_jsonl(response.text) + return _decode_dql(response, format, raw) def dql_parallel( @@ -80,7 +109,7 @@ def dql_parallel( queries: Sequence[Dict[str, Any]], *, workers: int = 8, -) -> List[Union[Dict[str, Any], bytes]]: +) -> List[Any]: if not queries: return [] with ThreadPoolExecutor(max_workers=min(workers, len(queries))) as ex: @@ -92,16 +121,16 @@ async def dql_parallel_async( queries: Sequence[Dict[str, Any]], *, workers: int = 8, -) -> List[Union[Dict[str, Any], bytes]]: +) -> List[Any]: if not queries: return [] sem = asyncio.Semaphore(workers) - async def _one(q: Dict[str, Any]) -> Union[Dict[str, Any], bytes]: + async def _one(q: Dict[str, Any]) -> Any: async with sem: return await dql_async(client, **q) - return await asyncio.gather(*(_one(q) for q in queries)) + return list(await asyncio.gather(*(_one(q) for q in queries))) def dql_refresh_ontology(client: "Diffbot", dest: pathlib.Path) -> None: diff --git a/src/diffbot/nlp.py b/src/diffbot/nlp.py index ac24e96..a14951d 100644 --- a/src/diffbot/nlp.py +++ b/src/diffbot/nlp.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict from ._constants import NLP_HOST_BASE +from .types import EntitiesResponse if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -16,8 +17,8 @@ def entities( text: str, *, lang: str = "auto", -) -> Dict[str, Any]: - params = {"token": client.token, "fields": NLP_FIELDS} +) -> EntitiesResponse: + params: Dict[str, Any] = {"token": client.token, "fields": NLP_FIELDS} payload = [{"lang": lang, "format": "plain text", "content": text}] response = client.request("POST", client.nlp_url, params=params, json=payload) data = response.json() @@ -29,8 +30,8 @@ async def entities_async( text: str, *, lang: str = "auto", -) -> Dict[str, Any]: - params = {"token": client.token, "fields": NLP_FIELDS} +) -> EntitiesResponse: + params: Dict[str, Any] = {"token": client.token, "fields": NLP_FIELDS} payload = [{"lang": lang, "format": "plain text", "content": text}] response = await client.request("POST", client.nlp_url, params=params, json=payload) data = response.json() diff --git a/src/diffbot/types.py b/src/diffbot/types.py index 690e760..b5bef62 100644 --- a/src/diffbot/types.py +++ b/src/diffbot/types.py @@ -6,7 +6,7 @@ absent, so a server response missing a field never breaks a type-checked caller. """ -from typing import Any, Dict, Literal, TypedDict +from typing import Any, Dict, List, Literal, Mapping, Sequence, TypedDict, Union #: Roles accepted by the Diffbot LLM RAG endpoint's chat messages. @@ -41,3 +41,418 @@ class ResponseFormat(TypedDict, total=False): #: Documented crawl/bulk job status codes (``docs__crawl__manage.md``): #: 0 initializing, 6 paused, 7 in progress, 8 admin-paused, 9 complete. CrawlJobStatusCode = Literal[0, 6, 7, 8, 9] + + +# --------------------------------------------------------------------------- +# Response shapes (type-level only) +# +# Every response type is a ``TypedDict(total=False)`` overlay over the plain dict the +# SDK has always returned: nothing is parsed, copied or validated at runtime, and a +# field the server omits is simply absent. Shapes mirror the TypeScript SDK's facades. +# Sources: the OpenAPI specs (DQL, web search, NL, account); Extract, Crawl and Ask are +# undocumented/example-only, so those members are weaker evidence and may lag the +# server. If you index a key not declared here, ``typing.cast(Dict[str, Any], resp)`` +# is the escape hatch. +# --------------------------------------------------------------------------- + +# ---- DQL (kg.diffbot.com/kg/v3/dql) ---- + + +class DqlEntity(TypedDict, total=False): + score: float + entity: Dict[str, Any] + entity_ctx: Dict[str, Any] + errors: List[str] + callbackQuery: str + upperBound: float + lowerBound: float + count: int + value: str + uri: str + + +class DqlRewrite(TypedDict, total=False): + query: str + reason: str + + +class DqlResponse(TypedDict, total=False): + version: int + hits: int + results: int + kgversion: str + diffbot_type: str + facet: bool + data: List[DqlEntity] + rewrites: List[DqlRewrite] + textFallback: bool + reportId: str + + +# ---- Web search (llm.diffbot.com/api/v1/web_search) ---- + + +class WebSearchResult(TypedDict, total=False): + score: float + pageUrl: str + title: str + content: str + date: str + + +class WebSearchResponse(TypedDict, total=False): + query: List[str] + search_results: List[WebSearchResult] + timeMs: float + + +# ---- Natural language (nl.diffbot.com/v1) ---- + + +class NlpMention(TypedDict, total=False): + text: str + beginOffset: int + endOffset: int + isPronoun: bool + confidence: float + + +class NlpEntityType(TypedDict, total=False): + name: str + diffbotUri: str + dbpediaUri: str + + +class NlpLocation(TypedDict, total=False): + latitude: float + longitude: float + precision: float + + +class NlpEntity(TypedDict, total=False): + name: str + diffbotUri: str + confidence: float + salience: float + sentiment: float + allUris: List[str] + allTypes: List[NlpEntityType] + mentions: List[NlpMention] + location: NlpLocation + + +class EntitiesResponse(TypedDict, total=False): + """What :meth:`Diffbot.entities` returns (``entities`` + ``sentiment``).""" + + entities: List[NlpEntity] + sentiment: float + + +# ---- Extract (api.diffbot.com/v3/{api}) ---- + +#: Built-in Extract APIs; any other string is a Custom API name. +ExtractApi = Literal["analyze", "article", "product", "discussion", "job", "image", "video", "list", "event"] + + +class ExtractObject(TypedDict, total=False): + """Fields common to every extracted object (also what a Custom API returns).""" + + type: str + title: str + pageUrl: str + diffbotUri: str + + +class ArticleTag(TypedDict, total=False): + label: str + score: float + + +class ArticleObject(ExtractObject, total=False): + text: str + html: str + author: str + date: str + tags: List[ArticleTag] + + +class ProductObject(ExtractObject, total=False): + offerPrice: str + regularPrice: str + availability: bool + productId: str + + +class DiscussionObject(ExtractObject, total=False): + text: str + posts: List[Dict[str, Any]] + + +class JobObject(ExtractObject, total=False): + pass + + +class ImageObject(ExtractObject, total=False): + naturalHeight: int + naturalWidth: int + + +class VideoObject(ExtractObject, total=False): + duration: float + + +class ListObject(ExtractObject, total=False): + items: List[Dict[str, Any]] + + +class EventObject(ExtractObject, total=False): + startDate: str + endDate: str + + +# TypedDict cannot be generic on Python 3.9, so each API gets its own response type +# (the object list is the only thing that differs). +class ExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[ExtractObject] + humanLanguage: str + errorCode: int + error: str + + +class ArticleExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[ArticleObject] + humanLanguage: str + errorCode: int + error: str + + +class ProductExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[ProductObject] + humanLanguage: str + errorCode: int + error: str + + +class DiscussionExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[DiscussionObject] + humanLanguage: str + errorCode: int + error: str + + +class JobExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[JobObject] + humanLanguage: str + errorCode: int + error: str + + +class ImageExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[ImageObject] + humanLanguage: str + errorCode: int + error: str + + +class VideoExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[VideoObject] + humanLanguage: str + errorCode: int + error: str + + +class ListExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[ListObject] + humanLanguage: str + errorCode: int + error: str + + +class EventExtractResponse(TypedDict, total=False): + request: Dict[str, Any] + objects: List[EventObject] + humanLanguage: str + errorCode: int + error: str + + +# Per-API ``fields=`` values (``extract.json``). ``links``/``extlinks``/``meta``/ +# ``querystring``/``breadcrumb`` are common to all of them. +_CommonField = Literal["links", "extlinks", "meta", "querystring", "breadcrumb"] +AnalyzeField = _CommonField +ArticleField = Literal["links", "extlinks", "meta", "querystring", "breadcrumb", "quotes"] +ProductField = _CommonField +DiscussionField = Literal["links", "extlinks", "meta", "querystring", "breadcrumb", "sentiment"] +JobField = _CommonField +ImageField = Literal["links", "extlinks", "meta", "querystring", "breadcrumb", "displayHeight", "displayWidth"] +VideoField = _CommonField +ListField = _CommonField +EventField = _CommonField +#: Custom API names get the common set. +CustomField = _CommonField + +#: ``mode=`` for Analyze: restrict to one Extract API, or ``llm`` for markdown output. +ExtractMode = Literal["article", "product", "discussion", "image", "video", "list", "event", "llm"] +#: ``naturalLanguage=`` fields accepted by Article extract. +NaturalLanguageField = Literal[ + "entities", "sentiment", "summary", "facts", "openFacts", "records", "categories", "sentences", "language" +] +#: ``useProxy=``: Diffbot's datacenter proxy, or explicitly none. +UseProxy = Literal["default", "none"] + + +#: ``fallback=`` for Analyze: the API to use when a page classifies as ``other``. +ExtractFallback = Literal["article", "product", "discussion", "image", "video", "list", "event"] + + +# Keyword options of ``extract()`` per API, used as ``**options: Unpack[...]`` in the +# client overloads (type-level only). ``timeout`` is the server-side fetch timeout in +# milliseconds; the transport timeout is raised to at least ``timeout + 10s``. +# ``proxy``/``proxyAuth`` are deliberately not named: pass them through ``extra``. +class _ExtractCommonOptions(TypedDict, total=False): + timeout: int + use_proxy: UseProxy + extra: Mapping[str, Any] + + +class AnalyzeExtractOptions(_ExtractCommonOptions, total=False): + mode: ExtractMode + fallback: ExtractFallback + fields: Sequence[AnalyzeField] + discussion: bool + + +class ArticleExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[ArticleField] + paging: bool + max_tags: int + tag_confidence: float + category_confidence: float + discussion: bool + natural_language: Sequence[NaturalLanguageField] + summary_num_sentences: int + + +class ProductExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[ProductField] + discussion: bool + + +class DiscussionExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[DiscussionField] + paging: bool + max_pages: Union[int, Literal["all"]] + + +class JobExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[JobField] + + +class ImageExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[ImageField] + + +class VideoExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[VideoField] + + +class ListExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[ListField] + + +class EventExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[EventField] + + +class CustomExtractOptions(_ExtractCommonOptions, total=False): + fields: Sequence[CustomField] + + +# ---- Crawl (api.diffbot.com/v3/crawl) ---- + + +class CrawlJobStatus(TypedDict, total=False): + status: int + message: str + + +class CrawlJob(TypedDict, total=False): + """A crawler job record (``crawl_get_job`` returns ``{}`` for a missing job).""" + + name: str + jobStatus: CrawlJobStatus + type: str + seeds: List[str] + roundsCompleted: int + pageCrawlAttempts: int + pageCrawlSuccesses: int + pageCrawlSuccessesThisRound: int + pagesInQueue: int + downloadJson: str + downloadUrls: str + + +class CrawlDataRecord(ExtractObject, total=False): + """One extracted record from ``crawl_download(..., format="json")``.""" + + text: str + html: str + + +#: ``crawl_download`` formats. +CrawlDownloadFormat = Literal["json", "csv"] + + +# ---- Account (api.diffbot.com/v4/account) ---- + + +class AccountUsage(TypedDict, total=False): + date: str + nlp: int + credits: int + extractions: int + proxies: int + entities: int + refresh: int + facets: int + videos: int + subtitles: int + + +class AccountResponse(TypedDict, total=False): + """``token`` and ``childTokens`` are secrets: never log or print this whole dict.""" + + name: str + email: str + plan: str + planCredits: int + status: str + created: str + token: str + childTokens: List[str] + usage: List[AccountUsage] + + +# ---- Ask / RAG (llm.diffbot.com/rag/v1/chat/completions, undocumented) ---- + + +class AskDelta(TypedDict, total=False): + content: str + + +class AskChoice(TypedDict, total=False): + delta: AskDelta + + +class AskChunk(TypedDict, total=False): + """One parsed SSE chunk of the streaming chat completion.""" + + choices: List[AskChoice] diff --git a/src/diffbot/web_search.py b/src/diffbot/web_search.py index 787a916..be7e21c 100644 --- a/src/diffbot/web_search.py +++ b/src/diffbot/web_search.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional from ._constants import LLM_BASE -from .types import SnippetLength +from .types import SnippetLength, WebSearchResponse if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -39,7 +39,7 @@ def web_search( num_results: Optional[int] = None, max_tokens: Optional[int] = None, snippet_length: Optional[SnippetLength] = None, -) -> Dict[str, Any]: +) -> WebSearchResponse: headers, params = _build_web_search_request(client, text, num_results, max_tokens, snippet_length) response = client.request("GET", client.web_search_url, headers=headers, params=params) return response.json() @@ -52,7 +52,7 @@ async def web_search_async( num_results: Optional[int] = None, max_tokens: Optional[int] = None, snippet_length: Optional[SnippetLength] = None, -) -> Dict[str, Any]: +) -> WebSearchResponse: headers, params = _build_web_search_request(client, text, num_results, max_tokens, snippet_length) response = await client.request("GET", client.web_search_url, headers=headers, params=params) return response.json() diff --git a/tests/test_account.py b/tests/test_account.py new file mode 100644 index 0000000..053d88a --- /dev/null +++ b/tests/test_account.py @@ -0,0 +1,92 @@ +import httpx +import pytest + +from diffbot import APIError, AuthError, Diffbot, DiffbotAsync + +ACCOUNT = {"name": "Acme", "plan": "plus", "usage": [{"date": "2026-11-16", "credits": 3}]} + + +def make(handler, **kw) -> Diffbot: + return Diffbot(token="tok", transport=httpx.MockTransport(handler), retry=False, **kw) + + +def test_account_sends_get_v4_with_days(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json=ACCOUNT) + + assert make(handler).account(days=7) == ACCOUNT + (req,) = seen + assert req.method == "GET" + assert str(req.url).startswith("https://api.diffbot.com/v4/account?") + assert req.url.params["token"] == "tok" + assert req.url.params["days"] == "7" + + +def test_account_omits_days_by_default(): + def handler(request: httpx.Request) -> httpx.Response: + assert "days" not in request.url.params + return httpx.Response(200, json=ACCOUNT) + + make(handler).account() + + +def test_account_honors_api_base(): + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url).startswith("http://localhost:9/v4/account") + return httpx.Response(200, json={}) + + make(handler, api_base="http://localhost:9").account() + + +def test_verify_token_true_uses_days_1(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params["days"] == "1" + return httpx.Response(200, json=ACCOUNT) + + assert make(handler).verify_token() is True + + +@pytest.mark.parametrize("status", [401, 403]) +def test_verify_token_false_on_auth_error(status): + db = make(lambda r: httpx.Response(status, json={"errorCode": status, "error": "bad token"})) + assert db.verify_token() is False + with pytest.raises(AuthError): + db.account() + + +@pytest.mark.parametrize("status", [429, 500, 404]) +def test_verify_token_propagates_other_errors(status): + db = make(lambda r: httpx.Response(status, text="nope")) + with pytest.raises(APIError): + db.verify_token() + + +def test_verify_token_propagates_network_errors(): + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("down") + + with pytest.raises(httpx.ConnectError): + make(handler).verify_token() + + +@pytest.mark.anyio +async def test_async_account_and_verify_token(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" and request.url.params["days"] == "1" + return httpx.Response(200, json=ACCOUNT) + + db = DiffbotAsync(token="tok", transport=httpx.MockTransport(handler)) + assert await db.account(days=1) == ACCOUNT + assert await db.verify_token() is True + + +@pytest.mark.anyio +async def test_async_verify_token_false_and_propagation(): + bad = DiffbotAsync(token="tok", transport=httpx.MockTransport(lambda r: httpx.Response(401, text="no")), retry=False) + assert await bad.verify_token() is False + boom = DiffbotAsync(token="tok", transport=httpx.MockTransport(lambda r: httpx.Response(500, text="x")), retry=False) + with pytest.raises(APIError): + await boom.verify_token() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 124e44d..9ebbe9a 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -128,4 +128,111 @@ def handler(request: httpx.Request) -> httpx.Response: ) with pytest.raises(APIError): db.crawl_delete_job("test-job") - assert calls["n"] == 1 \ No newline at end of file + assert calls["n"] == 1 + +""" +crawl_download (GET /v3/crawl/data) and the watcher that reuses it +""" + +RECORDS = [{"type": "article", "title": "A", "pageUrl": "https://example.com/a"}] +URLS_CSV = 'Url,Crawl Status,Crawled Time\n"https://example.com/a",Successfully Crawled,2026-11-17\n' + + +def _data_handler(seen, *, json_body=None, text=None): + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + if json_body is not None: + return httpx.Response(200, json=json_body) + return httpx.Response(200, text=text or "") + + return handler + + +def test_crawl_download_json_default(): + seen = [] + db = make_client(_data_handler(seen, json_body=RECORDS)) + assert db.crawl_download("job1") == RECORDS + (req,) = seen + assert req.method == "GET" + assert req.url.path == "/v3/crawl/data" + assert dict(req.url.params) == {"token": "test-token", "name": "job1"} + + +def test_crawl_download_csv_type_and_num(): + seen = [] + db = make_client(_data_handler(seen, text=URLS_CSV)) + out = db.crawl_download("job1", format="csv", type="urls", num=5) + assert out == URLS_CSV + assert dict(seen[0].url.params) == { + "token": "test-token", "name": "job1", "format": "csv", "type": "urls", "num": "5", + } + + +def test_crawl_download_empty_json_body_is_empty_list(): + assert make_client(_data_handler([], text="")).crawl_download("job1") == [] + + +def test_crawl_download_follows_redirects(): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v3/crawl/data": + return httpx.Response(302, headers={"location": "https://files.example.com/out.json"}) + return httpx.Response(200, json=RECORDS) + + assert make_client(handler).crawl_download("job1") == RECORDS + + +def test_crawl_download_missing_job_raises_not_found(): + from diffbot import NotFoundError + + db = Diffbot(token="t", transport=httpx.MockTransport(lambda r: httpx.Response(404, text="no")), retry=False) + with pytest.raises(NotFoundError): + db.crawl_download("nope") + + +@pytest.mark.anyio +async def test_async_crawl_download(): + seen = [] + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(_data_handler(seen, json_body=RECORDS))) + assert await db.crawl_download("job1", num=1) == RECORDS + assert seen[0].url.params["num"] == "1" + db2 = DiffbotAsync(token="test-token", transport=httpx.MockTransport(_data_handler([], text=URLS_CSV))) + assert await db2.crawl_download("job1", format="csv") == URLS_CSV + + +def _watch_handler(seen): + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/data"): + seen.append(request) + return httpx.Response(200, text=URLS_CSV) + if "seeds" in request.url.params: + return httpx.Response(200, json={"jobs": []}) + return httpx.Response(200, json={"jobs": [{"name": "j", "jobStatus": {"status": 9, "message": "done"}}]}) + + return handler + + +def test_crawl_watch_uses_download_for_urls(): + seen = [] + events = list(make_client(_watch_handler(seen)).crawl("example.com", job_name="j", watch=True, poll_interval=0)) + urls = [e.details["url"] for e in events if e.event_type == CrawlEventType.URL_PROCESSED] + assert urls == ["https://example.com/a"] + assert seen[0].url.params["type"] == "urls" + assert seen[0].url.params["format"] == "csv" + + +@pytest.mark.anyio +async def test_async_crawl_watch_uses_download_for_urls(): + seen = [] + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(_watch_handler(seen))) + urls = [ + e.details["url"] + async for e in db.crawl("example.com", job_name="j", watch=True, poll_interval=0) + if e.event_type == CrawlEventType.URL_PROCESSED + ] + assert urls == ["https://example.com/a"] + assert seen[0].url.params["type"] == "urls" + + +def test_crawl_get_job_missing_is_empty_dict(): + db = make_client(lambda r: httpx.Response(200, json={"jobs": []})) + assert db.crawl_get_job("nope") == {} diff --git a/tests/test_dql.py b/tests/test_dql.py index 6d01fff..4fded9a 100644 --- a/tests/test_dql.py +++ b/tests/test_dql.py @@ -215,3 +215,60 @@ async def test_async_dql_parallel_posts(): await db.dql_parallel([{"query": "a", "size": 0}, {"query": "b", "size": 0}]) assert {r.method for r in seen} == {"POST"} assert sorted(json.loads(r.content)["query"] for r in seen) == ["a", "b"] + + +""" +Return decoding per format (csv/jsonl used to crash in response.json()) +""" + +JSONL = b'{"score": 1, "entity": {"name": "A"}}\n\n{"score": 2, "entity": {"name": "B"}}\n' + + +def _mk(content, **kw): + return Diffbot(token="t", transport=httpx.MockTransport(lambda r: httpx.Response(200, content=content, **kw))) + + +def test_dql_csv_returns_text(): + assert _mk(b"Name\nDiffbot\n").dql("q", format="csv") == "Name\nDiffbot\n" + + +def test_dql_csv_raw_returns_bytes(): + assert _mk(b"Name\nDiffbot\n").dql("q", format="csv", raw=True) == b"Name\nDiffbot\n" + + +@pytest.mark.parametrize("fmt", ["xls", "xlsx"]) +def test_dql_excel_returns_bytes(fmt): + payload = bytes(range(256)) + assert _mk(payload).dql("q", format=fmt) == payload + + +def test_dql_jsonl_returns_entity_iterator(): + result = _mk(JSONL).dql("q", format="jsonl") + assert not isinstance(result, (list, dict, bytes, str)) + assert [e["score"] for e in result] == [1, 2] + + +def test_dql_jsonl_raw_returns_bytes(): + assert _mk(JSONL).dql("q", format="jsonl", raw=True) == JSONL + + +def test_dql_json_raw_returns_bytes(): + assert _mk(b'{"hits": 1}').dql("q", raw=True) == b'{"hits": 1}' + + +def test_dql_parallel_decodes_each_format(): + results = _mk(b"a,b\n").dql_parallel([{"query": "q", "format": "csv"}] * 2) + assert results == ["a,b\n", "a,b\n"] + + +@pytest.mark.anyio +async def test_async_dql_formats(): + def adb(content): + return DiffbotAsync(token="t", transport=httpx.MockTransport(lambda r: httpx.Response(200, content=content))) + + assert await adb(b"x\n").dql("q", format="csv") == "x\n" + assert await adb(b"\x00\x01").dql("q", format="xlsx") == b"\x00\x01" + assert await adb(b"x\n").dql("q", format="csv", raw=True) == b"x\n" + entities = [e async for e in await adb(JSONL).dql("q", format="jsonl")] + assert [e["score"] for e in entities] == [1, 2] + assert await adb(JSONL).dql("q", format="jsonl", raw=True) == JSONL diff --git a/tests/test_extract.py b/tests/test_extract.py index f273d51..0c47e56 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -5,6 +5,7 @@ APIError, AuthError, Diffbot, + DiffbotAsync, ExtractionError, RateLimitError, ValidationError, @@ -167,4 +168,133 @@ def test_live_extract(db): result = db.extract("https://example.com") assert "objects" in result assert len(result["objects"]) > 0 - assert "content" in result["objects"][0] \ No newline at end of file + assert "content" in result["objects"][0] + +""" +Caller-set options (additive keyword-only args after fmt) +""" + + +def _capture_client(**kw): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"objects": []}) + + return seen, Diffbot(token="test-token", transport=httpx.MockTransport(handler), **kw) + + +def test_extract_defaults_unchanged(): + seen, db = _capture_client() + db.extract("example.com") + assert dict(seen[0].url.params) == {"token": "test-token", "url": "https://example.com", "timeout": "30000", "mode": "llm"} + + +def test_extract_wire_names_and_encoding(): + seen, db = _capture_client() + db.extract( + "example.com", + "article", + fmt="json", + timeout=45000, + fields=["links", "quotes"], + discussion=False, + paging=True, + max_tags=3, + tag_confidence=0.7, + category_confidence=0.6, + natural_language=["entities", "summary"], + summary_num_sentences=2, + use_proxy="default", + ) + params = dict(seen[0].url.params) + assert params == { + "token": "test-token", "url": "https://example.com", "timeout": "45000", + "fields": "links,quotes", "discussion": "false", "paging": "true", "maxTags": "3", + "tagConfidence": "0.7", "categoryConfidence": "0.6", "naturalLanguage": "entities,summary", + "summaryNumSentences": "2", "useProxy": "default", + } + + +def test_extract_discussion_max_pages_and_analyze_fallback(): + seen, db = _capture_client() + db.extract("example.com", "discussion", max_pages="all") + assert seen[0].url.params["maxPages"] == "all" + db.extract("example.com", fallback="article", mode="product") + assert seen[1].url.params["fallback"] == "article" + + +def test_extract_mode_overrides_markdown_mapping(): + seen, db = _capture_client() + db.extract("example.com", mode="article") # fmt defaults to markdown + assert seen[0].url.params["mode"] == "article" + db.extract("example.com") + assert seen[1].url.params["mode"] == "llm" + + +def test_extract_none_options_are_not_sent(): + seen, db = _capture_client() + db.extract("example.com", fmt="json", fields=None, timeout=None) + assert "fields" not in seen[0].url.params + assert seen[0].url.params["timeout"] == "30000" + + +def test_extract_extra_passes_through_and_rejects_collisions(): + seen, db = _capture_client() + db.extract("example.com", extra={"proxy": "1.2.3.4:80", "proxyAuth": "u:p"}) + assert seen[0].url.params["proxy"] == "1.2.3.4:80" + assert seen[0].url.params["proxyAuth"] == "u:p" + for bad in ({"token": "x"}, {"url": "x"}, {"maxPages": "2"}, {"timeout": 1}): + with pytest.raises(ValidationError): + db.extract("example.com", extra=bad) + + +def test_extract_unknown_option_is_type_error(): + _, db = _capture_client() + with pytest.raises(TypeError): + db.extract("example.com", bogus=1) + + +def _read_timeout(request: httpx.Request) -> float: + return request.extensions["timeout"]["read"] + + +def test_transport_timeout_is_raised_above_a_long_caller_timeout(): + seen, db = _capture_client(timeout=30) + db.extract("example.com", timeout=60000) + assert seen[0].url.params["timeout"] == "60000" # server-side, ms + assert _read_timeout(seen[0]) == 70 # transport (s) = timeout + 10 + + +def test_transport_timeout_is_never_lowered_by_a_short_caller_timeout(): + seen, db = _capture_client(timeout=120) + db.extract("example.com", timeout=5000) + assert seen[0].url.params["timeout"] == "5000" + assert _read_timeout(seen[0]) == 120 # client default kept; 5s + 10s < 120s + + +def test_transport_timeout_untouched_without_caller_timeout(): + seen, db = _capture_client(timeout=30) + db.extract("example.com") + assert _read_timeout(seen[0]) == 30 + + +@pytest.mark.anyio +async def test_async_extract_options_and_timeouts(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"objects": []}) + + db = DiffbotAsync(token="test-token", timeout=30, transport=httpx.MockTransport(handler)) + await db.extract("example.com", "article", fmt="json", fields=["links"], max_tags=2, timeout=60000) + assert seen[0].url.params["fields"] == "links" + assert seen[0].url.params["maxTags"] == "2" + assert _read_timeout(seen[0]) == 70 + db2 = DiffbotAsync(token="test-token", timeout=120, transport=httpx.MockTransport(handler)) + await db2.extract("example.com", timeout=5000) + assert _read_timeout(seen[1]) == 120 + with pytest.raises(ValidationError): + await db2.extract("example.com", extra={"token": "x"}) diff --git a/tests/typing/test_types.py b/tests/typing/test_types.py new file mode 100644 index 0000000..a6d969b --- /dev/null +++ b/tests/typing/test_types.py @@ -0,0 +1,112 @@ +"""Type-level assertions, checked by ``pyright`` in CI (``uv run --extra dev pyright``). + +``assert_type`` is a no-op at runtime, so these also run under pytest as a smoke test +that the calls themselves work against a mock transport. +""" +import json +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, cast + +import httpx + +from diffbot import ( + AccountResponse, + ArticleExtractResponse, + CrawlDataRecord, + CrawlJob, + Diffbot, + DiffbotAsync, + DqlEntity, + DqlResponse, + EntitiesResponse, + ExtractResponse, + ImageExtractResponse, + ProductExtractResponse, + WebSearchResponse, +) + +if TYPE_CHECKING: + from typing_extensions import assert_type +else: + + def assert_type(val, typ, /): # type: ignore[no-untyped-def] + return val + + +def _handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/dql"): + fmt = json.loads(request.content).get("format", "json") + if fmt == "json": + return httpx.Response(200, json={"hits": 1, "data": []}) + return httpx.Response(200, content=b'{"score": 1}\n' if fmt == "jsonl" else b"a,b\n") + return httpx.Response(200, json={"objects": [], "jobs": [], "search_results": [], "entities": []}) + + +db = Diffbot(token="t", transport=httpx.MockTransport(_handler)) +adb = DiffbotAsync(token="t", transport=httpx.MockTransport(_handler)) + + +def test_dql_overloads(): + assert_type(db.dql("q"), DqlResponse) + assert_type(db.dql("q", format="json"), DqlResponse) + assert_type(db.dql("q", size=0, from_=2, extra={"jsonmode": "extended", "n": 1}), DqlResponse) + assert_type(db.dql("q", format="csv"), str) + assert_type(db.dql("q", format="csv", raw=True), bytes) + assert_type(db.dql("q", raw=True), bytes) + assert_type(db.dql("q", format="xls"), bytes) + assert_type(db.dql("q", format="xlsx"), bytes) + assert_type(db.dql("q", format="jsonl"), Iterator[DqlEntity]) + hits: int = db.dql("q").get("hits", 0) + assert hits == 1 + + +async def _async_dql_types() -> None: + assert_type(await adb.dql("q"), DqlResponse) + assert_type(await adb.dql("q", format="csv"), str) + assert_type(await adb.dql("q", format="xlsx"), bytes) + assert_type(await adb.dql("q", raw=True), bytes) + assert_type(await adb.dql("q", format="jsonl"), AsyncIterator[DqlEntity]) + + +def test_extract_overloads(): + assert_type(db.extract("u"), ExtractResponse) + assert_type(db.extract("u", "analyze", mode="article", fallback="product"), ExtractResponse) + assert_type(db.extract("u", "article", fields=["links", "quotes"], max_tags=3), ArticleExtractResponse) + assert_type(db.extract("u", "image", fields=["displayWidth"]), ImageExtractResponse) + assert_type(db.extract("u", "my-custom-api"), ExtractResponse) + assert_type(db.extract("u", "article", fmt="json", timeout=60000, extra={"proxy": "x"}), ArticleExtractResponse) + + +def test_other_response_types(): + assert_type(db.web_search("q"), WebSearchResponse) + assert_type(db.entities("x"), EntitiesResponse) + assert_type(db.crawl_list_jobs(), List[CrawlJob]) + assert_type(db.crawl_get_job("j"), CrawlJob) + + +def _crawl_download_and_account_types() -> None: + assert_type(db.crawl_download("j"), List[CrawlDataRecord]) + assert_type(db.crawl_download("j", format="json", num=1), List[CrawlDataRecord]) + assert_type(db.crawl_download("j", format="csv", type="urls"), str) + assert_type(db.account(), AccountResponse) + assert_type(db.account(days=1), AccountResponse) + assert_type(db.verify_token(), bool) + + +async def _async_misc_types() -> None: + assert_type(await adb.web_search("q"), WebSearchResponse) + assert_type(await adb.entities("x"), EntitiesResponse) + assert_type(await adb.crawl_get_job("j"), CrawlJob) + assert_type(await adb.crawl_download("j", format="csv"), str) + assert_type(await adb.crawl_download("j"), List[CrawlDataRecord]) + assert_type(await adb.account(), AccountResponse) + assert_type(await adb.verify_token(), bool) + assert_type(await adb.extract("u", "product", fields=["meta"]), ProductExtractResponse) + + +def test_known_keys_type_check_and_cast_escape_hatch(): + resp = db.web_search("q") + results = resp.get("search_results", []) + assert results == [] + # undeclared keys: cast is the documented escape hatch + loose = cast(Dict[str, Any], db.dql("q")) + assert loose["hits"] == 1 diff --git a/uv.lock b/uv.lock index 165aa2c..a29be13 100644 --- a/uv.lock +++ b/uv.lock @@ -101,6 +101,7 @@ dependencies = [ [package.optional-dependencies] dev = [ + { name = "pyright" }, { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] @@ -109,6 +110,7 @@ dev = [ requires-dist = [ { name = "click", specifier = ">=8.1.0" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.390" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "rich", specifier = ">=13.0.0" }, ] @@ -237,6 +239,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "nodeenv" +version = "1.11.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ab/27/79acfd3d6f526a4a16f3d209dedcf7b5bc1ea15c5bf2bbb82f2c2285057e/nodeenv-1.11.1.tar.gz", hash = "sha256:d7fa027c1df437bdd974b0502a8dc76fa1b3183e369ac9539602bc6bab9100ba", size = 99853, upload-time = "2026-10-29T20:54:04.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/35/a09e0b1dac4027e35b872e4f123a9f9e7bbc39e6424bdc9776920d3a6491/nodeenv-1.11.1-py2.py3-none-any.whl", hash = "sha256:bc0fb085d8ed0d67a6471892535f3d9b795eb911b7cd129d3a24967aab7a538f", size = 34473, upload-time = "2026-10-29T20:54:02.752Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/99/d27cfc9518f49e4cbc25c683ec7cf8704a9d31311341c291c0e78ddf791b/nodeenv-1.12.0.tar.gz", hash = "sha256:4ea7f88339310afebc3253f2c7a91b94bacf97d73a36a5c108962abdfaca501e", size = 131292, upload-time = "2026-10-31T19:03:18.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/6a/8a9ac7d5e2f7cf53c459bfef90ebc17a2f8cd54b665af96799d4c534840e/nodeenv-1.12.0-py3-none-any.whl", hash = "sha256:80644d65c10cc2be12e3e4b54b3367ff288c7ad24ae75c2dcc65454fc7449d4d", size = 39442, upload-time = "2026-10-31T19:03:17.427Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -264,6 +290,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyright" +version = "1.1.414" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv", version = "1.11.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "nodeenv", version = "1.12.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, +] + [[package]] name = "pytest" version = "8.4.2" From 51f6700bf23411baffb97d1fac27d1f950e9e662 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:59:13 -1000 Subject: [PATCH 12/31] ci: run pyright in tests.yml; extend live suite over account, extract, crawl_download Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/tests.yml | 18 ++++++++++ tests/test_live.py | 69 ++++++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 276f88a..44fb401 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,24 @@ jobs: - name: Run tests run: uv run pytest + # Static typing gate: pyright (basic on src/, strict on types.py) plus the + # assert_type checks in tests/typing. One Python version is enough. + typecheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Install dependencies (locked) + run: uv sync --frozen --extra dev + + - name: Run pyright + run: uv run pyright + # Catch breakage against the lowest dependency versions our ranges allow, # which the pinned lockfile would otherwise hide. test-lowest: diff --git a/tests/test_live.py b/tests/test_live.py index afcfde8..35f6235 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -3,6 +3,9 @@ Skipped by default. Assertions are on response *shape* only, never web content. Never include token values in assertion messages. """ +import time +import uuid + import pytest pytestmark = pytest.mark.live @@ -20,13 +23,14 @@ def test_live_dql_post_single_read(db): def test_live_dql_post_csv(db): - body = db.dql('type:Organization name:"Diffbot"', size=1, format="csv", raw=True) - assert isinstance(body, bytes) and len(body) > 0 + text = db.dql('type:Organization name:"Diffbot"', size=1, format="csv") + assert isinstance(text, str) and len(text.splitlines()) >= 2 + assert isinstance(db.dql('type:Organization name:"Diffbot"', size=1, format="csv", raw=True), bytes) def test_live_dql_post_jsonl(db): - body = db.dql('type:Organization name:"Diffbot"', size=1, format="jsonl", raw=True) - assert isinstance(body, bytes) and len(body) > 0 + entities = list(db.dql('type:Organization name:"Diffbot"', size=1, format="jsonl")) + assert len(entities) >= 1 def test_live_dql_long_query_beyond_url_limit(db): @@ -39,3 +43,60 @@ def test_live_dql_long_query_beyond_url_limit(db): def test_live_entities(db): result = db.entities("Diffbot is a company based in Menlo Park.") assert "entities" in result + + +def _redacted(account) -> str: + """Account bodies carry ``token``/``childTokens``: only ever show the other keys.""" + return f"keys={sorted(k for k in account if k not in ('token', 'childTokens'))}" + + +def test_live_account_and_verify_token(db): + account = db.account(days=1) + assert isinstance(account, dict), _redacted({}) + assert "plan" in account or "name" in account or "usage" in account, _redacted(account) + assert db.verify_token() is True + + +def test_live_verify_token_rejects_a_bad_token(): + from diffbot import Diffbot + + assert Diffbot(token="not-a-real-token").verify_token() is False + + +def test_live_extract_with_fields_and_timeout(db): + result = db.extract("https://example.com", "article", fmt="json", fields=["links"], timeout=45000) + assert "objects" in result and len(result["objects"]) > 0 + + +CRAWL_PREFIX = "sdk-live-" + + +def _purge_leaked_crawls(db) -> None: + cutoff = time.time() - 3600 + for job in db.crawl_list_jobs(): + name = job.get("name", "") + if name.startswith(CRAWL_PREFIX): + try: + created = int(name.rsplit("-", 1)[-1]) + except ValueError: + continue + if created < cutoff: + db.crawl_delete_job(name) + + +def test_live_crawl_start_poll_download(db): + _purge_leaked_crawls(db) + name = f"{CRAWL_PREFIX}py-{uuid.uuid4().hex[:8]}-{int(time.time())}" + try: + for _ in db.crawl("https://example.com", job_name=name, hops=0, max_to_crawl=5, max_to_process=5): + pass + deadline = time.time() + 180 + while time.time() < deadline: + if db.crawl_get_job(name).get("jobStatus", {}).get("status", 0) not in (0, 7): + break + time.sleep(5) + records = db.crawl_download(name) + assert isinstance(records, list) + assert isinstance(db.crawl_download(name, format="csv"), str) + finally: + db.crawl_delete_job(name) From 97d53e4265471660a30a0999b47d0771d1880bb4 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:59:13 -1000 Subject: [PATCH 13/31] docs(readme): extract options, dql formats, crawl start/poll/download, account/verify_token Co-Authored-By: Claude Sonnet 5.5 --- README.md | 87 ++++++++++++++++++++++++ tests/test_readme_examples.py | 123 ++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) diff --git a/README.md b/README.md index 9a3e060..dbc948b 100644 --- a/README.md +++ b/README.md @@ -143,8 +143,20 @@ from diffbot import Diffbot db = Diffbot(token="YOUR_TOKEN") data = db.extract("https://www.example.com") + +# Optional keyword-only options (per-API field names are type-checked): +article = db.extract( + "https://www.example.com/post", + "article", + fmt="json", + timeout=60000, # server-side fetch timeout in ms (default 30000) + fields=["links", "quotes"], + max_tags=5, +) ``` +`timeout` is the Diffbot-side fetch timeout in milliseconds; the SDK raises the HTTP transport timeout to at least `timeout + 10s` so the client never aborts first. `mode` overrides the `fmt="markdown"` → `mode=llm` mapping, and `extra={"proxy": ..., "proxyAuth": ...}` passes anything not named. + ### Ask Diffbot LLM ```python from diffbot import Diffbot @@ -199,12 +211,47 @@ for event in db.crawl("https://www.example.com", hops=1): print(event) ``` +For tools that can't block on a long crawl, start the job without watching, poll it, then download the results: + +```python +import time +from diffbot import Diffbot + +db = Diffbot(token="YOUR_TOKEN") +for _ in db.crawl("https://www.example.com", job_name="my-crawl", hops=1): + pass # watch=False (the default): only creates the job + +while db.crawl_get_job("my-crawl").get("jobStatus", {}).get("status", 0) in (0, 7): + time.sleep(5) # 0 initializing, 7 in progress; 9 complete + +records = db.crawl_download("my-crawl") # list of extracted records +csv_text = db.crawl_download("my-crawl", format="csv") # str +``` + +`crawl_get_job` returns `{}` for a job that doesn't exist (the TypeScript SDK returns `undefined`). + ### Query the Knowledge Graph ```python from diffbot import Diffbot db = Diffbot(token="YOUR_TOKEN") results = db.dql('type:Organization name:"Diffbot"') + +# The return type follows `format`: DqlResponse (json), str (csv), bytes (xls/xlsx +# or raw=True), or an iterator of entities (jsonl). +csv_text = db.dql('type:Organization name:"Diffbot"', size=5, format="csv") +for entity in db.dql('type:Organization name:"Diffbot"', size=5, format="jsonl"): + print(entity["entity"]["name"]) +``` + +### Account and token check +```python +from diffbot import Diffbot + +db = Diffbot(token="YOUR_TOKEN") +if not db.verify_token(): # True/False; only an auth failure means False + raise SystemExit("Diffbot token rejected") +usage = db.account(days=7).get("usage", []) # free call; never print the whole response ``` ### Web Search @@ -248,6 +295,10 @@ async def main(): data = await db.extract("https://www.example.com") print(data) + article = await db.extract( + "https://www.example.com/post", "article", fmt="json", timeout=60000, fields=["links"] + ) + asyncio.run(main()) ``` @@ -301,6 +352,24 @@ async def main(): asyncio.run(main()) ``` +Start, poll, then download, without blocking inside one call: + +```python +import asyncio +from diffbot import DiffbotAsync + +async def main(): + async with DiffbotAsync(token="YOUR_TOKEN") as db: + async for _ in db.crawl("https://www.example.com", job_name="my-crawl", hops=1): + pass # watch=False (the default): only creates the job + while (await db.crawl_get_job("my-crawl")).get("jobStatus", {}).get("status", 0) in (0, 7): + await asyncio.sleep(5) + records = await db.crawl_download("my-crawl") + csv_text = await db.crawl_download("my-crawl", format="csv") + +asyncio.run(main()) +``` + ### Query the Knowledge Graph ```python import asyncio @@ -311,6 +380,24 @@ async def main(): results = await db.dql('type:Organization name:"Diffbot"') print(results) + csv_text = await db.dql('type:Organization name:"Diffbot"', size=5, format="csv") + async for entity in await db.dql('type:Organization name:"Diffbot"', size=5, format="jsonl"): + print(entity["entity"]["name"]) + +asyncio.run(main()) +``` + +### Account and token check +```python +import asyncio +from diffbot import DiffbotAsync + +async def main(): + async with DiffbotAsync(token="YOUR_TOKEN") as db: + if not await db.verify_token(): + raise SystemExit("Diffbot token rejected") + usage = (await db.account(days=7)).get("usage", []) + asyncio.run(main()) ``` diff --git a/tests/test_readme_examples.py b/tests/test_readme_examples.py index 003a14a..b78912b 100644 --- a/tests/test_readme_examples.py +++ b/tests/test_readme_examples.py @@ -306,3 +306,126 @@ def handler(request: httpx.Request) -> httpx.Response: db = Diffbot(token=resolve_token(), transport=httpx.MockTransport(handler)) data = db.extract("https://www.example.com") assert "objects" in data + + +# --------------------------------------------------------------------------- +# Extract options, DQL formats, crawl start/poll/download, account +# --------------------------------------------------------------------------- + +def test_readme_sync_extract_options(): + def handler(request: httpx.Request) -> httpx.Response: + params = request.url.params + assert request.url.path == "/v3/article" + assert params["timeout"] == "60000" + assert params["fields"] == "links,quotes" + assert params["maxTags"] == "5" + assert "mode" not in params # fmt="json" -> no llm mapping + assert request.extensions["timeout"]["read"] == 70 + return httpx.Response(200, json={"objects": [{"title": "Post"}]}) + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + article = db.extract( + "https://www.example.com/post", + "article", + fmt="json", + timeout=60000, + fields=["links", "quotes"], + max_tags=5, + ) + assert article["objects"][0]["title"] == "Post" + + +def test_readme_sync_dql_formats(): + def handler(request: httpx.Request) -> httpx.Response: + fmt = json.loads(request.content)["format"] + if fmt == "csv": + return httpx.Response(200, text="name\nDiffbot\n") + return httpx.Response(200, text='{"entity": {"name": "Diffbot"}}\n') + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + csv_text = db.dql('type:Organization name:"Diffbot"', size=5, format="csv") + assert csv_text == "name\nDiffbot\n" + names = [e["entity"]["name"] for e in db.dql('type:Organization name:"Diffbot"', size=5, format="jsonl")] + assert names == ["Diffbot"] + + +def _crawl_flow_handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v3/crawl/data": + if request.url.params.get("format") == "csv": + return httpx.Response(200, text="Url\nhttps://www.example.com\n") + return httpx.Response(200, json=[{"pageUrl": "https://www.example.com"}]) + if "seeds" in request.url.params: + return httpx.Response(200, json={"jobs": [{"name": "my-crawl"}]}) + return httpx.Response(200, json={"jobs": [{"name": "my-crawl", "jobStatus": {"status": 9}}]}) + + +def test_readme_sync_crawl_start_poll_download(): + db = Diffbot(token="test-token", transport=httpx.MockTransport(_crawl_flow_handler)) + for _ in db.crawl("https://www.example.com", job_name="my-crawl", hops=1): + pass + while db.crawl_get_job("my-crawl").get("jobStatus", {}).get("status", 0) in (0, 7): + raise AssertionError("job should already be complete") + records = db.crawl_download("my-crawl") + csv_text = db.crawl_download("my-crawl", format="csv") + assert records == [{"pageUrl": "https://www.example.com"}] + assert csv_text.startswith("Url") + assert db.crawl_get_job.__doc__ and "{}" in db.crawl_get_job.__doc__ + + +def test_readme_sync_account_and_verify_token(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v4/account" + return httpx.Response(200, json={"usage": [{"date": "2026-11-16", "credits": 1}]}) + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + assert db.verify_token() + usage = db.account(days=7).get("usage", []) + assert usage[0]["credits"] == 1 + + +@pytest.mark.anyio +async def test_readme_async_extract_options(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params["fields"] == "links" + return httpx.Response(200, json={"objects": []}) + + async with DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) as db: + await db.extract("https://www.example.com", "article", fmt="json", timeout=60000, fields=["links"]) + + +@pytest.mark.anyio +async def test_readme_async_dql_formats(): + def handler(request: httpx.Request) -> httpx.Response: + if json.loads(request.content)["format"] == "csv": + return httpx.Response(200, text="name\nDiffbot\n") + return httpx.Response(200, text='{"entity": {"name": "Diffbot"}}\n') + + async with DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) as db: + csv_text = await db.dql('type:Organization name:"Diffbot"', size=5, format="csv") + names = [e["entity"]["name"] async for e in await db.dql('type:Organization name:"Diffbot"', size=5, format="jsonl")] + assert csv_text == "name\nDiffbot\n" + assert names == ["Diffbot"] + + +@pytest.mark.anyio +async def test_readme_async_crawl_start_poll_download(): + async with DiffbotAsync(token="test-token", transport=httpx.MockTransport(_crawl_flow_handler)) as db: + async for _ in db.crawl("https://www.example.com", job_name="my-crawl", hops=1): + pass + while (await db.crawl_get_job("my-crawl")).get("jobStatus", {}).get("status", 0) in (0, 7): + raise AssertionError("job should already be complete") + records = await db.crawl_download("my-crawl") + csv_text = await db.crawl_download("my-crawl", format="csv") + assert records == [{"pageUrl": "https://www.example.com"}] + assert csv_text.startswith("Url") + + +@pytest.mark.anyio +async def test_readme_async_account_and_verify_token(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"usage": []}) + + async with DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) as db: + assert await db.verify_token() + usage = (await db.account(days=7)).get("usage", []) + assert usage == [] From 13cd90d36c74e0f29b431b1886ea3af9382a46bc Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 12:59:39 -1000 Subject: [PATCH 14/31] docs: 3.3.0 changelog, S1 leftovers, mark response TypedDicts landed; type AskChunk in the SSE parser Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 50 +++++++++++++++++++++++++++++++++++++++++++++ PHASE0_LEFTOVERS.md | 8 ++++---- S1_LEFTOVERS.md | 37 +++++++++++++++++++++++++++++++++ src/diffbot/ask.py | 10 +++++---- 4 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 S1_LEFTOVERS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b1e25..eed9f9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## 3.3.0 (unreleased) + +Response types, a pyright gate, and two small methods that complete existing +workflows. Additive minor: no wire changes. The package version in +`pyproject.toml` is bumped at release time. + +### Added + +- Response `TypedDict`s (stdlib, `total=False`, no Pydantic, Python 3.9-compatible), + exported from `diffbot`: `DqlResponse`/`DqlEntity`, `WebSearchResponse`/ + `WebSearchResult`, `EntitiesResponse`, `ExtractResponse` and per-API + `ArticleExtractResponse`, `ProductExtractResponse`, ... with their object types, + `CrawlJob`, `CrawlDataRecord`, `AccountResponse`, `AskChunk`. +- `account(days=None)` and `verify_token()` (sync and async): `GET /v4/account`. + `verify_token()` calls it with `days=1`; only an auth failure (401/403) returns + `False`, every other error propagates. The account response carries `token` and + `childTokens`: don't log it whole. +- `crawl_download(name, *, format="json", type=None, num=None)` (sync and async): + `GET /v3/crawl/data`. `format="csv"` returns `str`, otherwise a list of records. + The crawl watcher now uses it. For tools that can't block, the recommended + pattern is `crawl(..., watch=False)` -> `crawl_get_job` -> `crawl_download`. +- `extract()` keyword-only options after `fmt`: `timeout` (server-side fetch timeout + in ms, default still 30000), `mode` (overrides the `fmt="markdown"` -> `llm` + mapping), `fields` (sequence, comma-joined; per-API `Literal` names), `fallback`, + `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, + `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` + for anything else (`proxy`/`proxyAuth`; keys that collide with a named option or + `token`/`url` raise `ValidationError`). When you set `timeout`, the per-call + transport timeout is raised to at least `timeout + 10s` (never lowered). +- Dev: `pyright` (basic on `src/`, strict on `types.py`) plus `tests/typing` + `assert_type` checks, run in `tests.yml`. + +### Fixed + +- `dql(format="csv")` and `dql(format="jsonl")` without `raw=True` crashed in + `response.json()`. `csv` now returns `str`, `xls`/`xlsx` return `bytes`, and + `jsonl` returns an iterator of entities (`AsyncIterator` from the async client). + `raw=True` is unchanged (undecoded `bytes`). + +### Changed (type-level only) + +- Return annotations changed from `Dict[str, Any]` to the TypedDicts above + (`dql`, `extract`, `web_search`, `entities`, `crawl_list_jobs`, `crawl_get_job`), + and `dql`/`extract`/`crawl_download` gained `@overload`s. Runtime values are + unchanged. A type checker may now flag keys the types don't declare; wrap the + value in `typing.cast(Dict[str, Any], resp)` as the escape hatch. `dql_parallel` + returns `List[Any]` (each element's type follows its own call). +- `crawl_get_job` still returns `{}` for a missing job (the TypeScript SDK returns + `undefined`); unchanged on purpose. + ## 3.2.0 (unreleased) Shared transport foundation, a sync/async de-duplication refactor, and DQL sent diff --git a/PHASE0_LEFTOVERS.md b/PHASE0_LEFTOVERS.md index 564e7d7..fd6e9dd 100644 --- a/PHASE0_LEFTOVERS.md +++ b/PHASE0_LEFTOVERS.md @@ -32,9 +32,9 @@ Unchanged by the move to POST. Covered by the existing raw-mode tests. Adding `dql_fetch_ontology_text()` and `dql_refresh_ontology_async()` to close the Python/TypeScript asymmetry is scheduled for **S2** (Python 3.4.0). -## 5. Response TypedDicts: scheduled for S1 +## 5. Response TypedDicts: landed in S1 -Request-side `TypedDict`/`Literal` aliases landed in `src/diffbot/types.py`. +Request-side `TypedDict`/`Literal` aliases landed in `src/diffbot/types.py` with Phase 0. Response shapes (`DqlResponse`, `WebSearchResponse`, `ExtractResponse`, `CrawlJob`, -`AccountResponse`, `AskChunk`, ...) are scheduled for **S1** (Python 3.3.0), -together with a pyright CI gate so they are checked from day one. +`AccountResponse`, `AskChunk`, ...) land in S1 (Python 3.3.0, branch `feat/sdk-s1`) +together with the pyright CI gate, so they are checked from day one. diff --git a/S1_LEFTOVERS.md b/S1_LEFTOVERS.md new file mode 100644 index 0000000..e3763e7 --- /dev/null +++ b/S1_LEFTOVERS.md @@ -0,0 +1,37 @@ +# S1 leftovers (Python) + +S1 (plan §5, §6, §8) landed on `feat/sdk-s1`. Notes on what is deliberately smaller than +the plan's wording, and what is unverified. + +## Deliberately narrower + +- **Overloads live on the client methods** (`Diffbot`/`DiffbotAsync` `dql`, `extract`, + `crawl_download`), the public surface. The module-level shared implementations in + `kg.py`/`extract.py`/`crawl.py` return `Any` to avoid duplicating ~50 overloads. +- **Per-API extract options use `**options: Unpack[...]` in the overloads** (a + `TYPE_CHECKING`-only import of `typing_extensions.Unpack`; no runtime dependency). + At runtime the implementation takes `**options` and rejects unknown names with + `TypeError`. Every option is accepted for every API at runtime; only the type + level restricts them per API. +- **Transport timeout is raised only when the caller sets `timeout`.** With the default + 30000 ms and the default 30 s client timeout the two are still equal, as before. +- `dql_parallel` returns `List[Any]` (heterogeneous per-call formats). +- `EntitiesResponse` mirrors TS `Pick<..., "entities" | "sentiment">`; `errors` is not declared. +- `CrawlDataRecord` is a best-effort shape (the crawl data JSON is example-only in the + docs); keep it open to change. The TypeScript counterpart did not exist yet when this + was written, so field-for-field parity with its `CrawlDataRecord` is unchecked. + +## Unverified here + +- Live tests (`pytest -m live`) were not run against the real API (no token resolved in + the build environment); `test_live_verify_token_rejects_a_bad_token` was. + The live crawl test (start/poll/download/delete) is therefore unexercised. +- Open question from plan §11: DQL POST parity (jsonl/csv/xlsx with `exportspec`, + JSON strings vs numbers in `extra`) should be confirmed with the Diffbot contact + before release. + +## Not in S1 (see plan §2/§8) + +S2 items (DQL named params, crawl-create options, `nlp()`, web-search multi-value +`text` + `snippet_length` validation, ontology text/async parity) and everything marked +Out. diff --git a/src/diffbot/ask.py b/src/diffbot/ask.py index 917fbe7..54a1044 100644 --- a/src/diffbot/ask.py +++ b/src/diffbot/ask.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional from .errors import ValidationError -from .types import ChatMessage, ResponseFormat +from .types import AskChunk, ChatMessage, ResponseFormat if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -76,12 +76,14 @@ def _build_payload( def _parse_chunk(line: str): try: - chunk = json.loads(line.replace("data: ", "")) + chunk: AskChunk = json.loads(line.replace("data: ", "")) except json.JSONDecodeError: return None choices = chunk.get("choices") - if choices and choices[0].get("delta", {}).get("content"): - return choices[0]["delta"]["content"] + if choices: + content = choices[0].get("delta", {}).get("content") + if content: + return content return None From 432ce4a0ecb13bc1cb666c8c0396a89056b5a167 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 14:48:51 -1000 Subject: [PATCH 15/31] fix(test): keep live crawl job name within the 30-char API limit Co-Authored-By: Claude Sonnet 5.5 --- tests/test_live.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_live.py b/tests/test_live.py index 35f6235..416d48b 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -86,7 +86,7 @@ def _purge_leaked_crawls(db) -> None: def test_live_crawl_start_poll_download(db): _purge_leaked_crawls(db) - name = f"{CRAWL_PREFIX}py-{uuid.uuid4().hex[:8]}-{int(time.time())}" + name = f"{CRAWL_PREFIX}{uuid.uuid4().hex[:8]}-{int(time.time())}" # 28 chars; API limit is 30 try: for _ in db.crawl("https://example.com", job_name=name, hops=0, max_to_crawl=5, max_to_process=5): pass From 653915be4b45be51fc7d717987b9948c3af5722e Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 14:48:51 -1000 Subject: [PATCH 16/31] fix(crawl): raise APIError when create returns HTTP 200 with errorCode Co-Authored-By: Claude Sonnet 5.5 --- src/diffbot/crawl.py | 17 ++++++++++++++--- tests/test_crawl.py | 20 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index cac393c..4dafdea 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -10,7 +10,7 @@ from enum import Enum from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional -from .errors import JobError +from .errors import APIError, JobError from .types import CrawlDownloadFormat, CrawlJob if TYPE_CHECKING: @@ -64,7 +64,7 @@ def crawl( api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, ) - client.request("GET", client.crawler_url, params=params) + _raise_on_create_error(client.request("GET", client.crawler_url, params=params)) yield CrawlEvent( event_type=CrawlEventType.JOB_CREATED, @@ -103,6 +103,17 @@ def crawl( time.sleep(poll_interval) +def _raise_on_create_error(response: Any) -> None: + """The crawler reports some create failures (e.g. a job name over 30 chars) as HTTP 200 + with ``errorCode``/``error`` in the body; surface them instead of returning JOB_CREATED.""" + try: + data = response.json() + except ValueError: + return + if isinstance(data, dict) and data.get("errorCode"): + raise APIError(200, json.dumps({"message": data.get("error") or f"errorCode {data['errorCode']}"})) + + def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int, max_to_process: int, hops: int, restrict_domain: bool, api_url: str, crawl_delay: float, url_crawl_pattern: Optional[str], @@ -177,7 +188,7 @@ async def crawl_async( api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, ) - await client.request("GET", client.crawler_url, params=params) + _raise_on_create_error(await client.request("GET", client.crawler_url, params=params)) yield CrawlEvent( event_type=CrawlEventType.JOB_CREATED, diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 9ebbe9a..878e356 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -236,3 +236,23 @@ async def test_async_crawl_watch_uses_download_for_urls(): def test_crawl_get_job_missing_is_empty_dict(): db = make_client(lambda r: httpx.Response(200, json={"jobs": []})) assert db.crawl_get_job("nope") == {} + + +def test_crawl_raises_when_create_returns_200_with_error_code(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"errorCode": 505, "error": "name is over 30 chars"}) + + db = make_client(handler) + with pytest.raises(APIError, match="name is over 30 chars"): + next(db.crawl("https://example.com", job_name="x" * 31)) + + +@pytest.mark.anyio +async def test_async_crawl_raises_when_create_returns_200_with_error_code(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"errorCode": 505, "error": "name is over 30 chars"}) + + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) + with pytest.raises(APIError, match="name is over 30 chars"): + async for _ in db.crawl("https://example.com", job_name="x" * 31): + pass From 45e7cdc18ac612ccfcc5d1a036f5070772086e37 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:06:54 -1000 Subject: [PATCH 17/31] fix(crawl): crawl_download urls report and error bodies; keep the real create-error body - crawl_download(type="urls") returned a JSON decode error: that report is always CSV, whatever format says. Return the text (matching the TypeScript SDK) and add a type="urls" -> str overload. - crawl_download raised nothing for a non-list JSON body such as {errorCode, error} for an unknown job and handed the dict back as records. Raise JobError carrying the job name, as TypeScript does. - crawl create with HTTP 200 + errorCode now raises APIError with the real status, body and headers instead of a synthesized body; message comes from the error field via the APIError fallback. - Document the 30-character job name limit (README, crawl docstring) and add the create-error fix to the 3.3.0 changelog. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++++++-- README.md | 2 ++ src/diffbot/client.py | 18 ++++++++++++++---- src/diffbot/crawl.py | 29 +++++++++++++++++++++-------- tests/test_crawl.py | 21 ++++++++++++++++++++- tests/typing/test_types.py | 2 ++ 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eed9f9c..d16df99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,9 @@ workflows. Additive minor: no wire changes. The package version in `False`, every other error propagates. The account response carries `token` and `childTokens`: don't log it whole. - `crawl_download(name, *, format="json", type=None, num=None)` (sync and async): - `GET /v3/crawl/data`. `format="csv"` returns `str`, otherwise a list of records. - The crawl watcher now uses it. For tools that can't block, the recommended + `GET /v3/crawl/data`. `format="csv"` or `type="urls"` (that report is always CSV) + returns `str`, otherwise a list of records; a JSON body that is not a list (e.g. an + unknown job) raises `JobError`. The crawl watcher now uses it. For tools that can't block, the recommended pattern is `crawl(..., watch=False)` -> `crawl_get_job` -> `crawl_download`. - `extract()` keyword-only options after `fmt`: `timeout` (server-side fetch timeout in ms, default still 30000), `mode` (overrides the `fmt="markdown"` -> `llm` @@ -38,6 +39,10 @@ workflows. Additive minor: no wire changes. The package version in `response.json()`. `csv` now returns `str`, `xls`/`xlsx` return `bytes`, and `jsonl` returns an iterator of entities (`AsyncIterator` from the async client). `raw=True` is unchanged (undecoded `bytes`). +- `crawl()` create failures that the crawler reports as HTTP 200 with `errorCode`/ + `error` in the body (e.g. errorCode 505 for a job name over 30 characters, which + creates nothing) now raise `APIError` (status 200, the real body and headers, + `message` from `error`) instead of yielding `JOB_CREATED` for a job that doesn't exist. ### Changed (type-level only) diff --git a/README.md b/README.md index dbc948b..9f32ec6 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,8 @@ for event in db.crawl("https://www.example.com", hops=1): print(event) ``` +Job names are limited to 30 characters. The crawler reports a create failure such as an overlong name as HTTP 200 with an `errorCode` in the body; `crawl()` raises `APIError` for it rather than yielding `JOB_CREATED`. + For tools that can't block on a long crawl, start the job without watching, poll it, then download the results: ```python diff --git a/src/diffbot/client.py b/src/diffbot/client.py index a67c4ef..d140edd 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -305,6 +305,10 @@ def crawl_get_job(self, job_name: str) -> CrawlJob: @overload def crawl_download(self, name: str, *, format: Literal["csv"], type: Optional[str] = ..., num: Optional[int] = ...) -> str: ... @overload + def crawl_download( + self, name: str, *, format: CrawlDownloadFormat = ..., type: Literal["urls"], num: Optional[int] = ... + ) -> str: ... + @overload def crawl_download( self, name: str, *, format: Literal["json"] = ..., type: Optional[str] = ..., num: Optional[int] = ... ) -> List[CrawlDataRecord]: ... @@ -323,8 +327,9 @@ def crawl_download( """Download a crawl job's results (``GET /v3/crawl/data``). ``format="json"`` (default) returns a list of records; ``format="csv"`` - returns the CSV text. ``type`` selects the report (e.g. ``"urls"``) and - ``num`` caps the number of records. + returns the CSV text. ``type`` selects the report: ``"urls"`` is always CSV + text, whatever ``format`` says. ``num`` caps the number of records. A JSON + body that is not a list (e.g. an unknown job) raises ``JobError``. """ return _crawl_download(self, name, format=format, type=type, num=num) @@ -623,6 +628,10 @@ async def crawl_get_job(self, job_name: str) -> CrawlJob: @overload async def crawl_download(self, name: str, *, format: Literal["csv"], type: Optional[str] = ..., num: Optional[int] = ...) -> str: ... @overload + async def crawl_download( + self, name: str, *, format: CrawlDownloadFormat = ..., type: Literal["urls"], num: Optional[int] = ... + ) -> str: ... + @overload async def crawl_download( self, name: str, *, format: Literal["json"] = ..., type: Optional[str] = ..., num: Optional[int] = ... ) -> List[CrawlDataRecord]: ... @@ -641,8 +650,9 @@ async def crawl_download( """Download a crawl job's results (``GET /v3/crawl/data``). ``format="json"`` (default) returns a list of records; ``format="csv"`` - returns the CSV text. ``type`` selects the report (e.g. ``"urls"``) and - ``num`` caps the number of records. + returns the CSV text. ``type`` selects the report: ``"urls"`` is always CSV + text, whatever ``format`` says. ``num`` caps the number of records. A JSON + body that is not a list (e.g. an unknown job) raises ``JobError``. """ return await _crawl_download_async(self, name, format=format, type=type, num=num) diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index 4dafdea..8b4b645 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -51,6 +51,9 @@ def crawl( """Create a crawler job and yield a JOB_CREATED event. If watch=True, poll until completion and yield URL_PROCESSED events for each crawled URL. + ``job_name`` must be at most 30 characters; the crawler answers an overlong name + with HTTP 200 + ``errorCode`` 505 and creates nothing, which raises ``APIError``. + Recommended pattern for tools that can't block on a long crawl: start with ``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its ``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`.""" @@ -104,14 +107,16 @@ def crawl( def _raise_on_create_error(response: Any) -> None: - """The crawler reports some create failures (e.g. a job name over 30 chars) as HTTP 200 - with ``errorCode``/``error`` in the body; surface them instead of returning JOB_CREATED.""" + """The crawler reports some create failures (e.g. a job name over 30 chars, errorCode + 505) as HTTP 200 with ``errorCode``/``error`` in the body and creates nothing; surface + them instead of returning JOB_CREATED. The error keeps the real status, body and + headers (``APIError.message`` reads the body's ``error`` field).""" try: data = response.json() except ValueError: return if isinstance(data, dict) and data.get("errorCode"): - raise APIError(200, json.dumps({"message": data.get("error") or f"errorCode {data['errorCode']}"})) + raise APIError(response.status_code, response.text, headers=dict(response.headers)) def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int, @@ -288,10 +293,18 @@ def _download_params(client: Any, name: str, format: CrawlDownloadFormat, type: return params -def _decode_download(text: str, format: CrawlDownloadFormat) -> Any: - if format == "csv": +def _decode_download(name: str, text: str, format: CrawlDownloadFormat, type: Optional[str]) -> Any: + # The ``urls`` report is always CSV text, whatever ``format`` says. + if format == "csv" or type == "urls": return text - return json.loads(text) if text.strip() else [] + if not text.strip(): + return [] + data = json.loads(text) + if not isinstance(data, list): + # e.g. ``{"errorCode": ..., "error": ...}`` for an unknown job + error = data.get("error") if isinstance(data, dict) else None + raise JobError(name, f"Crawl data download failed: {error or 'unexpected response'}") + return data # The public, overloaded signatures live on ``Diffbot.crawl_download`` / @@ -310,7 +323,7 @@ def crawl_download( params=_download_params(client, name, format, type, num), follow_redirects=True, ) - return _decode_download(response.text, format) + return _decode_download(name, response.text, format, type) async def crawl_download_async( @@ -327,4 +340,4 @@ async def crawl_download_async( params=_download_params(client, name, format, type, num), follow_redirects=True, ) - return _decode_download(response.text, format) + return _decode_download(name, response.text, format, type) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 878e356..31f5327 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,3 +1,5 @@ +import json + import httpx import pytest @@ -189,6 +191,20 @@ def test_crawl_download_missing_job_raises_not_found(): db.crawl_download("nope") +def test_crawl_download_urls_report_is_csv_whatever_the_format(): + seen = [] + db = make_client(_data_handler(seen, text=URLS_CSV)) + assert db.crawl_download("job1", type="urls") == URLS_CSV + assert "format" not in seen[0].url.params + + +def test_crawl_download_non_list_json_raises_job_error(): + db = make_client(_data_handler([], json_body={"errorCode": 500, "error": "No collection record"})) + with pytest.raises(JobError, match="No collection record") as exc_info: + db.crawl_download("nope") + assert exc_info.value.job_name == "nope" + + @pytest.mark.anyio async def test_async_crawl_download(): seen = [] @@ -243,8 +259,11 @@ def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(200, json={"errorCode": 505, "error": "name is over 30 chars"}) db = make_client(handler) - with pytest.raises(APIError, match="name is over 30 chars"): + with pytest.raises(APIError, match="name is over 30 chars") as exc_info: next(db.crawl("https://example.com", job_name="x" * 31)) + assert exc_info.value.status_code == 200 + assert exc_info.value.message == "name is over 30 chars" + assert json.loads(exc_info.value.body)["errorCode"] == 505 @pytest.mark.anyio diff --git a/tests/typing/test_types.py b/tests/typing/test_types.py index a6d969b..a0d603b 100644 --- a/tests/typing/test_types.py +++ b/tests/typing/test_types.py @@ -87,6 +87,7 @@ def _crawl_download_and_account_types() -> None: assert_type(db.crawl_download("j"), List[CrawlDataRecord]) assert_type(db.crawl_download("j", format="json", num=1), List[CrawlDataRecord]) assert_type(db.crawl_download("j", format="csv", type="urls"), str) + assert_type(db.crawl_download("j", type="urls"), str) assert_type(db.account(), AccountResponse) assert_type(db.account(days=1), AccountResponse) assert_type(db.verify_token(), bool) @@ -97,6 +98,7 @@ async def _async_misc_types() -> None: assert_type(await adb.entities("x"), EntitiesResponse) assert_type(await adb.crawl_get_job("j"), CrawlJob) assert_type(await adb.crawl_download("j", format="csv"), str) + assert_type(await adb.crawl_download("j", type="urls"), str) assert_type(await adb.crawl_download("j"), List[CrawlDataRecord]) assert_type(await adb.account(), AccountResponse) assert_type(await adb.verify_token(), bool) From 4c7d899b59f9fe1096f25df3c7c2734ce6aaea8e Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:16:11 -1000 Subject: [PATCH 18/31] feat(dql): named params type, col, filter_exclude, exportseparator, exportfile, jsonmode, non_canonical_facts, no_dedup_articles, cluster Keyword-only options on dql()/dql_async() (and so dql_parallel), sent in the POST body. Explicit option plus the same wire key in extra raises ValidationError. report/view stay reachable via extra only. Co-Authored-By: Claude Sonnet 5.5 --- src/diffbot/__init__.py | 8 ++++ src/diffbot/client.py | 72 ++++++++++++++++++++++++------ src/diffbot/kg.py | 69 +++++++++++++++++++++++++++-- src/diffbot/types.py | 27 ++++++++++++ tests/test_dql.py | 90 +++++++++++++++++++++++++++++++++++++- tests/typing/test_types.py | 8 ++++ 6 files changed, 256 insertions(+), 18 deletions(-) diff --git a/src/diffbot/__init__.py b/src/diffbot/__init__.py index 462ea20..79c026a 100644 --- a/src/diffbot/__init__.py +++ b/src/diffbot/__init__.py @@ -51,6 +51,10 @@ def _installed_version(dist: str): from .types import ( ChatMessage, CrawlJobStatusCode, + DqlCluster, + DqlJsonMode, + DqlOptions, + DqlType, ExportFormat, ResponseFormat, SnippetLength, @@ -111,6 +115,10 @@ def _installed_version(dist: str): "RetryPolicy", "ChatMessage", "ResponseFormat", + "DqlCluster", + "DqlJsonMode", + "DqlOptions", + "DqlType", "ExportFormat", "SnippetLength", "CrawlJobStatusCode", diff --git a/src/diffbot/client.py b/src/diffbot/client.py index d140edd..ffe2f4f 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -27,6 +27,10 @@ CrawlJob, CustomExtractOptions, DiscussionExtractOptions, + DqlCluster, + DqlJsonMode, + DqlOptions, + DqlType, DiscussionExtractResponse, DqlEntity, DqlResponse, @@ -339,27 +343,27 @@ def crawl_delete_job(self, job_name: str) -> None: @overload def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[True], + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[True], **options: "Unpack[DqlOptions]" ) -> bytes: ... @overload def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["csv"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["csv"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> str: ... @overload def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["xls", "xlsx"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["xls", "xlsx"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> bytes: ... @overload def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["jsonl"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["jsonl"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> Iterator[DqlEntity]: ... @overload def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["json"] = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["json"] = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> DqlResponse: ... @overload def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: bool = ... + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: bool = ..., **options: "Unpack[DqlOptions]" ) -> Union[DqlResponse, str, bytes, Iterator[DqlEntity]]: ... def dql( self, @@ -372,6 +376,15 @@ def dql( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, + type: Optional[DqlType] = None, + col: Optional[str] = None, + filter_exclude: Optional[str] = None, + exportseparator: Optional[str] = None, + exportfile: Optional[str] = None, + jsonmode: Optional[DqlJsonMode] = None, + non_canonical_facts: Optional[bool] = None, + no_dedup_articles: Optional[bool] = None, + cluster: Optional[DqlCluster] = None, ) -> Any: """Run a DQL query against the Diffbot Knowledge Graph (sent as ``POST`` with a JSON body). @@ -379,8 +392,19 @@ def dql( (default); ``str`` for ``"csv"``; ``bytes`` for ``"xls"``/``"xlsx"`` or any format with ``raw=True`` (the undecoded response); an iterator of ``DqlEntity`` for ``"jsonl"``. + + Named options (all optional, sent in the POST body): ``type`` + (``"query"``/``"text"``/``"queryTextFallback"``/``"crawl"``; ``"crawl"`` searches + crawl collections, selected with ``col``), ``filter_exclude``, ``exportseparator``, + ``exportfile``, ``jsonmode``, ``non_canonical_facts``, ``no_dedup_articles`` and + ``cluster``. A key given both as a named option and in ``extra`` raises + ``ValidationError``. ``report`` and ``view`` are not named; use ``extra``. """ - return _dql(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw) + return _dql(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw, + type=type, col=col, filter_exclude=filter_exclude, exportseparator=exportseparator, + exportfile=exportfile, jsonmode=jsonmode, non_canonical_facts=non_canonical_facts, + no_dedup_articles=no_dedup_articles, cluster=cluster, + ) def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Any]: """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args; @@ -662,27 +686,27 @@ async def crawl_delete_job(self, job_name: str) -> None: @overload async def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[True], + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[True], **options: "Unpack[DqlOptions]" ) -> bytes: ... @overload async def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["csv"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["csv"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> str: ... @overload async def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["xls", "xlsx"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["xls", "xlsx"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> bytes: ... @overload async def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["jsonl"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["jsonl"], filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> AsyncIterator[DqlEntity]: ... @overload async def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: Literal["json"] = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ... + self, query: str, *, size: int = ..., from_: int = ..., format: Literal["json"] = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: Literal[False] = ..., **options: "Unpack[DqlOptions]" ) -> DqlResponse: ... @overload async def dql( - self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: bool = ... + self, query: str, *, size: int = ..., from_: int = ..., format: ExportFormat = ..., filter: Optional[str] = ..., exportspec: Optional[str] = ..., extra: Optional[Mapping[str, Any]] = ..., raw: bool = ..., **options: "Unpack[DqlOptions]" ) -> Union[DqlResponse, str, bytes, AsyncIterator[DqlEntity]]: ... async def dql( self, @@ -695,6 +719,15 @@ async def dql( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, + type: Optional[DqlType] = None, + col: Optional[str] = None, + filter_exclude: Optional[str] = None, + exportseparator: Optional[str] = None, + exportfile: Optional[str] = None, + jsonmode: Optional[DqlJsonMode] = None, + non_canonical_facts: Optional[bool] = None, + no_dedup_articles: Optional[bool] = None, + cluster: Optional[DqlCluster] = None, ) -> Any: """Run a DQL query against the Diffbot Knowledge Graph (sent as ``POST`` with a JSON body). @@ -702,8 +735,19 @@ async def dql( (default); ``str`` for ``"csv"``; ``bytes`` for ``"xls"``/``"xlsx"`` or any format with ``raw=True`` (the undecoded response); an iterator of ``DqlEntity`` for ``"jsonl"``. + + Named options (all optional, sent in the POST body): ``type`` + (``"query"``/``"text"``/``"queryTextFallback"``/``"crawl"``; ``"crawl"`` searches + crawl collections, selected with ``col``), ``filter_exclude``, ``exportseparator``, + ``exportfile``, ``jsonmode``, ``non_canonical_facts``, ``no_dedup_articles`` and + ``cluster``. A key given both as a named option and in ``extra`` raises + ``ValidationError``. ``report`` and ``view`` are not named; use ``extra``. """ - return await _dql_async(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw) + return await _dql_async(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw, + type=type, col=col, filter_exclude=filter_exclude, exportseparator=exportseparator, + exportfile=exportfile, jsonmode=jsonmode, non_canonical_facts=non_canonical_facts, + no_dedup_articles=no_dedup_articles, cluster=cluster, + ) async def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Any]: """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args; diff --git a/src/diffbot/kg.py b/src/diffbot/kg.py index e016f51..4a7dc71 100644 --- a/src/diffbot/kg.py +++ b/src/diffbot/kg.py @@ -7,8 +7,9 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, Sequence from ._constants import KG_BASE +from .errors import ValidationError from .ontology import Ontology -from .types import DqlEntity, ExportFormat +from .types import DqlCluster, DqlEntity, DqlJsonMode, DqlType, ExportFormat if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -25,6 +26,7 @@ def _build_dql_body( filter: Optional[str], exportspec: Optional[str], extra: Optional[Mapping[str, Any]], + named: Optional[Mapping[str, Any]] = None, ) -> Dict[str, Any]: """JSON body for ``POST /kg/v3/dql`` (``DqlPostRequest``). The token is *not* part of the body; it travels in the query string, as the spec's security scheme says.""" @@ -37,11 +39,46 @@ def _build_dql_body( body["filter"] = filter if exportspec is not None: body["exportspec"] = exportspec + if named: + # ``named`` maps wire names to values; ``None`` means "not set". + set_named = {k: v for k, v in named.items() if v is not None} + if extra: + dupes = sorted(set(set_named) & set(extra)) + if dupes: + raise ValidationError( + f"{', '.join(dupes)} given both as a named option and in extra; pass it once" + ) + body.update(set_named) if extra: body.update(extra) return body +def _named_dql_options( + type: Optional[DqlType], + col: Optional[str], + filter_exclude: Optional[str], + exportseparator: Optional[str], + exportfile: Optional[str], + jsonmode: Optional[DqlJsonMode], + non_canonical_facts: Optional[bool], + no_dedup_articles: Optional[bool], + cluster: Optional[DqlCluster], +) -> Dict[str, Any]: + """Python names -> wire names for the named DQL options.""" + return { + "type": type, + "col": col, + "filterExclude": filter_exclude, + "exportseparator": exportseparator, + "exportfile": exportfile, + "jsonmode": jsonmode, + "nonCanonicalFacts": non_canonical_facts, + "noDedupArticles": no_dedup_articles, + "cluster": cluster, + } + + def _decode_dql(response: Any, format: ExportFormat, raw: bool) -> Any: """Decode a DQL response by ``format``: json -> dict, csv -> str, xls/xlsx -> bytes, jsonl -> iterator of entities. ``raw=True`` always returns the undecoded bytes.""" @@ -79,8 +116,21 @@ def dql( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, + type: Optional[DqlType] = None, + col: Optional[str] = None, + filter_exclude: Optional[str] = None, + exportseparator: Optional[str] = None, + exportfile: Optional[str] = None, + jsonmode: Optional[DqlJsonMode] = None, + non_canonical_facts: Optional[bool] = None, + no_dedup_articles: Optional[bool] = None, + cluster: Optional[DqlCluster] = None, ) -> Any: - body = _build_dql_body(query, size, from_, format, filter, exportspec, extra) + named = _named_dql_options( + type, col, filter_exclude, exportseparator, exportfile, jsonmode, + non_canonical_facts, no_dedup_articles, cluster, + ) + body = _build_dql_body(query, size, from_, format, filter, exportspec, extra, named) response = client.request("POST", client.dql_url, params={"token": client.token}, json=body) return _decode_dql(response, format, raw) @@ -96,8 +146,21 @@ async def dql_async( exportspec: Optional[str] = None, extra: Optional[Mapping[str, Any]] = None, raw: bool = False, + type: Optional[DqlType] = None, + col: Optional[str] = None, + filter_exclude: Optional[str] = None, + exportseparator: Optional[str] = None, + exportfile: Optional[str] = None, + jsonmode: Optional[DqlJsonMode] = None, + non_canonical_facts: Optional[bool] = None, + no_dedup_articles: Optional[bool] = None, + cluster: Optional[DqlCluster] = None, ) -> Any: - body = _build_dql_body(query, size, from_, format, filter, exportspec, extra) + named = _named_dql_options( + type, col, filter_exclude, exportseparator, exportfile, jsonmode, + non_canonical_facts, no_dedup_articles, cluster, + ) + body = _build_dql_body(query, size, from_, format, filter, exportspec, extra, named) response = await client.request("POST", client.dql_url, params={"token": client.token}, json=body) if not raw and format == "jsonl": return _aiter_jsonl(response.text) diff --git a/src/diffbot/types.py b/src/diffbot/types.py index b5bef62..6a47eb3 100644 --- a/src/diffbot/types.py +++ b/src/diffbot/types.py @@ -35,6 +35,33 @@ class ResponseFormat(TypedDict, total=False): #: DQL / Enhance-bulk export formats. ExportFormat = Literal["json", "jsonl", "csv", "xls", "xlsx"] +#: DQL ``type``: ``crawl`` searches your crawl collections (see ``col``). +DqlType = Literal["query", "text", "queryTextFallback", "crawl"] + +#: DQL ``jsonmode``: ``extended`` adds origin info for facts, ``id`` returns only ids and origins. +DqlJsonMode = Literal["extended", "id"] + +#: DQL ``cluster`` (Article queries only). +DqlCluster = Literal["all", "best", "dedupe"] + + +class DqlOptions(TypedDict, total=False): + """Named DQL options sent in the POST JSON body (keyword-only on ``dql()``). + + ``report`` and ``view`` are deliberately not named; pass them through ``extra``. + """ + + type: DqlType + col: str + filter_exclude: str + exportseparator: str + exportfile: str + jsonmode: DqlJsonMode + non_canonical_facts: bool + no_dedup_articles: bool + cluster: DqlCluster + + #: How much of each page's matching content `web_search` returns. SnippetLength = Literal["short", "medium", "long", "full"] diff --git a/tests/test_dql.py b/tests/test_dql.py index 4fded9a..4e0f843 100644 --- a/tests/test_dql.py +++ b/tests/test_dql.py @@ -3,7 +3,7 @@ import httpx import pytest -from diffbot import APIError, Diffbot, DiffbotAsync, NotFoundError +from diffbot import APIError, Diffbot, DiffbotAsync, NotFoundError, ValidationError """ @@ -272,3 +272,91 @@ def adb(content): entities = [e async for e in await adb(JSONL).dql("q", format="jsonl")] assert [e["score"] for e in entities] == [1, 2] assert await adb(JSONL).dql("q", format="jsonl", raw=True) == JSONL + + +""" +Named DQL options (S2) +""" + +ALL_NAMED = dict( + type="crawl", col="a,b", filter_exclude="skills.name", exportseparator="|", exportfile="out.csv", + jsonmode="extended", non_canonical_facts=True, no_dedup_articles=True, cluster="best", +) +ALL_WIRE = { + "type": "crawl", "col": "a,b", "filterExclude": "skills.name", "exportseparator": "|", + "exportfile": "out.csv", "jsonmode": "extended", "nonCanonicalFacts": True, + "noDedupArticles": True, "cluster": "best", +} + + +def test_dql_named_options_go_in_body_not_url(): + seen, handler = _capture() + db = Diffbot(token="tok", transport=httpx.MockTransport(handler)) + db.dql("q", **ALL_NAMED) + (req,) = seen + assert dict(req.url.params) == {"token": "tok"} + assert json.loads(req.content) == {"query": "q", "size": 10, **ALL_WIRE} + + +def test_dql_named_options_default_to_absent(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.dql("q") + assert json.loads(seen[0].content) == {"query": "q", "size": 10} + seen.clear() + db.dql("q", non_canonical_facts=False, type="query") + assert json.loads(seen[0].content) == {"query": "q", "size": 10, "nonCanonicalFacts": False, "type": "query"} + + +def test_dql_extra_collision_with_named_raises(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + with pytest.raises(ValidationError, match="jsonmode"): + db.dql("q", jsonmode="id", extra={"jsonmode": "extended"}) + with pytest.raises(ValidationError, match="filterExclude"): + db.dql("q", filter_exclude="a", extra={"filterExclude": "b"}) + assert seen == [] + + +def test_dql_report_and_view_still_reachable_via_extra(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.dql("q", type="text", extra={"report": True, "view": "x"}) + assert json.loads(seen[0].content) == {"query": "q", "size": 10, "type": "text", "report": True, "view": "x"} + + +def test_dql_parallel_forwards_named_options(): + seen, handler = _capture() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.dql_parallel([{"query": "a", "type": "crawl", "col": "all"}]) + assert json.loads(seen[0].content) == {"query": "a", "size": 10, "type": "crawl", "col": "all"} + + +@pytest.mark.anyio +async def test_async_dql_named_options_go_in_body_not_url(): + seen, handler = _capture() + db = DiffbotAsync(token="tok", transport=httpx.MockTransport(handler)) + await db.dql("q", **ALL_NAMED) + (req,) = seen + assert dict(req.url.params) == {"token": "tok"} + assert json.loads(req.content) == {"query": "q", "size": 10, **ALL_WIRE} + seen.clear() + await db.dql("q") + assert json.loads(seen[0].content) == {"query": "q", "size": 10} + + +@pytest.mark.anyio +async def test_async_dql_extra_collision_with_named_raises(): + seen, handler = _capture() + db = DiffbotAsync(token="t", transport=httpx.MockTransport(handler)) + with pytest.raises(ValidationError, match="cluster"): + await db.dql("q", cluster="all", extra={"cluster": "best"}) + assert seen == [] + + +@pytest.mark.anyio +async def test_async_dql_parallel_forwards_named_options(): + seen, handler = _capture() + db = DiffbotAsync(token="t", transport=httpx.MockTransport(handler)) + await db.dql_parallel([{"query": "a", "type": "crawl", "col": "all"}]) + assert json.loads(seen[0].content) == {"query": "a", "size": 10, "type": "crawl", "col": "all"} diff --git a/tests/typing/test_types.py b/tests/typing/test_types.py index a0d603b..7509e54 100644 --- a/tests/typing/test_types.py +++ b/tests/typing/test_types.py @@ -112,3 +112,11 @@ def test_known_keys_type_check_and_cast_escape_hatch(): # undeclared keys: cast is the documented escape hatch loose = cast(Dict[str, Any], db.dql("q")) assert loose["hits"] == 1 + + +def _dql_named_option_types() -> None: + assert_type(db.dql("q", type="crawl", col="all", cluster="best", jsonmode="id"), DqlResponse) + assert_type(db.dql("q", format="csv", exportseparator="|", exportfile="x.csv"), str) + assert_type(db.dql("q", non_canonical_facts=True, no_dedup_articles=True, filter_exclude="a"), DqlResponse) + db.dql("q", type="bogus") # type: ignore[call-overload] + db.dql("q", cluster="nope") # type: ignore[call-overload] From e3570afbe6889a7d705d710fea34bcea7f623a9e Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:16:40 -1000 Subject: [PATCH 19/31] feat(crawl): named create options and multi-value custom_headers url_crawl_regex, url_process_regex, page_process_pattern, max_to_crawl_per_subdomain, max_to_process_per_subdomain, use_canonical, notify_webhook, notify_email on crawl() (query params on the existing GET). custom_headers accepts a str or a sequence, sent as repeated params. Recurring-crawl admin stays out. Co-Authored-By: Claude Sonnet 5.5 --- src/diffbot/crawl.py | 58 +++++++++++++++++++++++++--- tests/test_crawl.py | 92 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 6 deletions(-) diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index 8b4b645..c73643e 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional, Sequence, Union from .errors import APIError, JobError from .types import CrawlDownloadFormat, CrawlJob @@ -44,9 +44,18 @@ def crawl( url_process_pattern: Optional[str] = None, obey_robots: bool = False, use_proxies: bool = False, - custom_headers: Optional[str] = None, + custom_headers: Optional[Union[str, Sequence[str]]] = None, watch: bool = False, poll_interval: float = 2.0, + *, + url_crawl_regex: Optional[str] = None, + url_process_regex: Optional[str] = None, + page_process_pattern: Optional[str] = None, + max_to_crawl_per_subdomain: Optional[int] = None, + max_to_process_per_subdomain: Optional[int] = None, + use_canonical: Optional[bool] = None, + notify_webhook: Optional[str] = None, + notify_email: Optional[str] = None, ) -> Iterator[CrawlEvent]: """Create a crawler job and yield a JOB_CREATED event. If watch=True, poll until completion and yield URL_PROCESSED events for each crawled URL. @@ -56,7 +65,11 @@ def crawl( Recommended pattern for tools that can't block on a long crawl: start with ``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its - ``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`.""" + ``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`. + + Create options are sent as query params on the existing GET. ``custom_headers`` is + one ``"Name: value"`` string or a sequence of them (sent as repeated params). + Recurring-crawl admin (``repeat``, ``maxRounds``, ...) is not exposed.""" if not site.startswith("http"): site = f"https://{site}" if not job_name: @@ -66,6 +79,16 @@ def crawl( client, job_name, site, max_to_crawl, max_to_process, hops, restrict_domain, api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, + { + "urlCrawlRegEx": url_crawl_regex, + "urlProcessRegEx": url_process_regex, + "pageProcessPattern": page_process_pattern, + "maxToCrawlPerSubdomain": max_to_crawl_per_subdomain, + "maxToProcessPerSubdomain": max_to_process_per_subdomain, + "useCanonical": None if use_canonical is None else int(use_canonical), + "notifyWebhook": notify_webhook, + "notifyEmail": notify_email, + }, ) _raise_on_create_error(client.request("GET", client.crawler_url, params=params)) @@ -123,7 +146,8 @@ def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int max_to_process: int, hops: int, restrict_domain: bool, api_url: str, crawl_delay: float, url_crawl_pattern: Optional[str], url_process_pattern: Optional[str], obey_robots: bool, - use_proxies: bool, custom_headers: Optional[str]) -> Dict[str, Any]: + use_proxies: bool, custom_headers: Optional[Union[str, Sequence[str]]], + named: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: params: Dict[str, Any] = { "token": client.token, "name": job_name, @@ -146,7 +170,10 @@ def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int if use_proxies: params["useProxies"] = 1 if custom_headers: - params["customHeaders"] = custom_headers + # a sequence becomes repeated ``customHeaders=`` params; a lone string stays a plain value + params["customHeaders"] = custom_headers if isinstance(custom_headers, str) else list(custom_headers) + if named: + params.update({k: v for k, v in named.items() if v is not None}) return params @@ -179,9 +206,18 @@ async def crawl_async( url_process_pattern: Optional[str] = None, obey_robots: bool = False, use_proxies: bool = False, - custom_headers: Optional[str] = None, + custom_headers: Optional[Union[str, Sequence[str]]] = None, watch: bool = False, poll_interval: float = 2.0, + *, + url_crawl_regex: Optional[str] = None, + url_process_regex: Optional[str] = None, + page_process_pattern: Optional[str] = None, + max_to_crawl_per_subdomain: Optional[int] = None, + max_to_process_per_subdomain: Optional[int] = None, + use_canonical: Optional[bool] = None, + notify_webhook: Optional[str] = None, + notify_email: Optional[str] = None, ) -> AsyncIterator[CrawlEvent]: if not site.startswith("http"): site = f"https://{site}" @@ -192,6 +228,16 @@ async def crawl_async( client, job_name, site, max_to_crawl, max_to_process, hops, restrict_domain, api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, + { + "urlCrawlRegEx": url_crawl_regex, + "urlProcessRegEx": url_process_regex, + "pageProcessPattern": page_process_pattern, + "maxToCrawlPerSubdomain": max_to_crawl_per_subdomain, + "maxToProcessPerSubdomain": max_to_process_per_subdomain, + "useCanonical": None if use_canonical is None else int(use_canonical), + "notifyWebhook": notify_webhook, + "notifyEmail": notify_email, + }, ) _raise_on_create_error(await client.request("GET", client.crawler_url, params=params)) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 31f5327..75817b3 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -275,3 +275,95 @@ def handler(request: httpx.Request) -> httpx.Response: with pytest.raises(APIError, match="name is over 30 chars"): async for _ in db.crawl("https://example.com", job_name="x" * 31): pass + +""" +Named create options and multi-value custom_headers (S2) +""" + + +def _capture_create(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"jobs": [{"name": "j"}]}) + + return seen, handler + + +NAMED = dict( + url_crawl_regex="^https://a/", url_process_regex="/p/", page_process_pattern=" Date: Tue, 17 Nov 2026 13:17:28 -1000 Subject: [PATCH 20/31] feat(nlp): nlp() on the existing NL endpoint, entities() unchanged nlp(text, *, fields, lang, format, document_type, custom_summary) plus async twin, single document. Adds NlpField/NlpFormat/NlpCustomSummary/NlpResponse types, README snippet and its test. entities() keeps hard-coding fields=entities,sentiment. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 17 +++++++++ src/diffbot/__init__.py | 8 +++++ src/diffbot/client.py | 50 ++++++++++++++++++++++++++ src/diffbot/nlp.py | 58 ++++++++++++++++++++++++++++-- src/diffbot/types.py | 30 ++++++++++++++++ tests/test_nlp.py | 68 +++++++++++++++++++++++++++++++++++ tests/test_readme_examples.py | 25 +++++++++++++ tests/typing/test_types.py | 13 +++++++ 8 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 tests/test_nlp.py diff --git a/README.md b/README.md index 9f32ec6..e94c916 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,23 @@ for entity in result["entities"]: print("sentiment:", result.get("sentiment")) ``` +### NLP: other fields (`nlp`) +`entities()` always requests `entities` and `sentiment`. `nlp()` reaches the rest of the +same endpoint for a single document. +```python +from diffbot import Diffbot + +db = Diffbot(token="YOUR_TOKEN") +result = db.nlp( + "Acme Launches Rocket\n\nAcme announced a new rocket today.", + fields=["summary", "categories", "language"], + format="plain text with title", + document_type="news article", + custom_summary={"max_number_of_sentences": 1}, +) +print(result.get("summary")) +``` + ## Async Usage ### Extract structured content diff --git a/src/diffbot/__init__.py b/src/diffbot/__init__.py index 79c026a..a8816fa 100644 --- a/src/diffbot/__init__.py +++ b/src/diffbot/__init__.py @@ -56,6 +56,10 @@ def _installed_version(dist: str): DqlOptions, DqlType, ExportFormat, + NlpCustomSummary, + NlpField, + NlpFormat, + NlpResponse, ResponseFormat, SnippetLength, AccountResponse, @@ -120,6 +124,10 @@ def _installed_version(dist: str): "DqlOptions", "DqlType", "ExportFormat", + "NlpCustomSummary", + "NlpField", + "NlpFormat", + "NlpResponse", "SnippetLength", "CrawlJobStatusCode", "AccountResponse", diff --git a/src/diffbot/client.py b/src/diffbot/client.py index ffe2f4f..13557fc 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -45,6 +45,10 @@ JobExtractResponse, ListExtractOptions, ListExtractResponse, + NlpCustomSummary, + NlpField, + NlpFormat, + NlpResponse, ProductExtractOptions, ProductExtractResponse, ResponseFormat, @@ -96,6 +100,8 @@ from .nlp import ( entities as _entities, entities_async as _entities_async, + nlp as _nlp, + nlp_async as _nlp_async, ) @@ -459,6 +465,28 @@ def entities(self, text: str, *, lang: str = "auto") -> EntitiesResponse: """ return _entities(self, text, lang=lang) + def nlp( + self, + text: str, + *, + fields: Optional[Sequence[NlpField]] = None, + lang: str = "auto", + format: NlpFormat = "plain text", + document_type: Optional[str] = None, + custom_summary: Optional[NlpCustomSummary] = None, + ) -> NlpResponse: + """Run the NL endpoint on a single document and return the requested ``fields``. + + fields: any of ``"entities"``, ``"sentiment"``, ``"facts"``, ``"records"``, + ``"categories"``, ``"sentences"``, ``"language"``, ``"summary"``; omit for the API + default (``entities,sentiment,facts,records,sentences``). format: + ``"plain text"`` or ``"plain text with title"`` (title, two line breaks, body). + document_type gives the model context; custom_summary + ``{"max_number_of_sentences": 1..10}`` shapes the ``summary`` field. + Multi-document batches are not supported; :meth:`entities` is unchanged. + """ + return _nlp(self, text, fields=fields, lang=lang, format=format, document_type=document_type, custom_summary=custom_summary) + class DiffbotAsync: """Async client for Diffbot APIs. @@ -797,3 +825,25 @@ async def entities(self, text: str, *, lang: str = "auto") -> EntitiesResponse: id:or("id1","id2","id3") — no type: declaration required. """ return await _entities_async(self, text, lang=lang) + + async def nlp( + self, + text: str, + *, + fields: Optional[Sequence[NlpField]] = None, + lang: str = "auto", + format: NlpFormat = "plain text", + document_type: Optional[str] = None, + custom_summary: Optional[NlpCustomSummary] = None, + ) -> NlpResponse: + """Run the NL endpoint on a single document and return the requested ``fields``. + + fields: any of ``"entities"``, ``"sentiment"``, ``"facts"``, ``"records"``, + ``"categories"``, ``"sentences"``, ``"language"``, ``"summary"``; omit for the API + default (``entities,sentiment,facts,records,sentences``). format: + ``"plain text"`` or ``"plain text with title"`` (title, two line breaks, body). + document_type gives the model context; custom_summary + ``{"max_number_of_sentences": 1..10}`` shapes the ``summary`` field. + Multi-document batches are not supported; :meth:`entities` is unchanged. + """ + return await _nlp_async(self, text, fields=fields, lang=lang, format=format, document_type=document_type, custom_summary=custom_summary) diff --git a/src/diffbot/nlp.py b/src/diffbot/nlp.py index a14951d..e801b25 100644 --- a/src/diffbot/nlp.py +++ b/src/diffbot/nlp.py @@ -1,9 +1,9 @@ """Diffbot NLP API: entity identification, resolution, and sentiment.""" -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence from ._constants import NLP_HOST_BASE -from .types import EntitiesResponse +from .types import EntitiesResponse, NlpCustomSummary, NlpField, NlpFormat, NlpResponse if TYPE_CHECKING: from .client import Diffbot, DiffbotAsync @@ -36,3 +36,57 @@ async def entities_async( response = await client.request("POST", client.nlp_url, params=params, json=payload) data = response.json() return data[0] if isinstance(data, list) else data + + +def _build_nlp_request( + client: Any, + text: str, + fields: Optional[Sequence[NlpField]], + lang: str, + format: NlpFormat, + document_type: Optional[str], + custom_summary: Optional[NlpCustomSummary], +) -> Any: + params: Dict[str, Any] = {"token": client.token} + if fields is not None: + params["fields"] = fields if isinstance(fields, str) else ",".join(fields) + document: Dict[str, Any] = {"lang": lang, "format": format, "content": text} + if document_type is not None: + document["documentType"] = document_type + if custom_summary is not None: + max_sentences = custom_summary.get("max_number_of_sentences") + if max_sentences is not None: + document["customSummary"] = {"maxNumberOfSentences": max_sentences} + return params, [document] + + +def nlp( + client: "Diffbot", + text: str, + *, + fields: Optional[Sequence[NlpField]] = None, + lang: str = "auto", + format: NlpFormat = "plain text", + document_type: Optional[str] = None, + custom_summary: Optional[NlpCustomSummary] = None, +) -> NlpResponse: + params, payload = _build_nlp_request(client, text, fields, lang, format, document_type, custom_summary) + response = client.request("POST", client.nlp_url, params=params, json=payload) + data = response.json() + return data[0] if isinstance(data, list) else data + + +async def nlp_async( + client: "DiffbotAsync", + text: str, + *, + fields: Optional[Sequence[NlpField]] = None, + lang: str = "auto", + format: NlpFormat = "plain text", + document_type: Optional[str] = None, + custom_summary: Optional[NlpCustomSummary] = None, +) -> NlpResponse: + params, payload = _build_nlp_request(client, text, fields, lang, format, document_type, custom_summary) + response = await client.request("POST", client.nlp_url, params=params, json=payload) + data = response.json() + return data[0] if isinstance(data, list) else data diff --git a/src/diffbot/types.py b/src/diffbot/types.py index 6a47eb3..2b4e2d5 100644 --- a/src/diffbot/types.py +++ b/src/diffbot/types.py @@ -175,6 +175,36 @@ class EntitiesResponse(TypedDict, total=False): sentiment: float +#: Fields ``nlp()`` can request (``fields`` query param of the NL endpoint). +NlpField = Literal["entities", "sentiment", "facts", "records", "categories", "sentences", "language", "summary"] + +#: ``format`` of the submitted document. +NlpFormat = Literal["plain text", "plain text with title"] + + +class NlpCustomSummary(TypedDict, total=False): + """Summary generation options (wire: ``customSummary.maxNumberOfSentences``).""" + + max_number_of_sentences: int + + +class NlpResponse(TypedDict, total=False): + """What :meth:`Diffbot.nlp` returns: any of the requested ``fields`` may be present. + + Only the ones with a documented shape here are typed in detail; the rest are + left loose (``Any``) because the spec examples are the only source. + """ + + entities: List[NlpEntity] + sentiment: float + facts: List[Any] + records: List[Any] + categories: List[Any] + sentences: List[Any] + language: str + summary: str + + # ---- Extract (api.diffbot.com/v3/{api}) ---- #: Built-in Extract APIs; any other string is a Custom API name. diff --git a/tests/test_nlp.py b/tests/test_nlp.py new file mode 100644 index 0000000..526862d --- /dev/null +++ b/tests/test_nlp.py @@ -0,0 +1,68 @@ +import json + +import httpx +import pytest + +from diffbot import Diffbot, DiffbotAsync + +DOC = [{"entities": [], "sentiment": 0.1, "summary": "s"}] + + +def _capture(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json=DOC) + + return seen, handler + + +def test_nlp_defaults_omit_fields_and_unwraps_list(): + seen, handler = _capture() + db = Diffbot(token="tok", transport=httpx.MockTransport(handler)) + assert db.nlp("hello") == DOC[0] + (req,) = seen + assert req.method == "POST" + assert str(req.url).startswith("https://nl.diffbot.com/v1/") + assert dict(req.url.params) == {"token": "tok"} + assert json.loads(req.content) == [{"lang": "auto", "format": "plain text", "content": "hello"}] + + +def test_nlp_options_shape_request(): + seen, handler = _capture() + db = Diffbot(token="tok", transport=httpx.MockTransport(handler)) + db.nlp("t", fields=["summary", "facts"], lang="en", document_type="blog", + custom_summary={"max_number_of_sentences": 2}) + (req,) = seen + assert req.url.params["fields"] == "summary,facts" + assert json.loads(req.content) == [{ + "lang": "en", "format": "plain text", "content": "t", "documentType": "blog", + "customSummary": {"maxNumberOfSentences": 2}, + }] + + +def test_entities_unchanged_still_hardcodes_fields(): + seen, handler = _capture() + db = Diffbot(token="tok", transport=httpx.MockTransport(handler)) + db.entities("x") + assert seen[0].url.params["fields"] == "entities,sentiment" + assert json.loads(seen[0].content) == [{"lang": "auto", "format": "plain text", "content": "x"}] + + +@pytest.mark.anyio +async def test_async_nlp_options_shape_request(): + seen, handler = _capture() + db = DiffbotAsync(token="tok", transport=httpx.MockTransport(handler)) + assert await db.nlp("t") == DOC[0] + await db.nlp("Title\n\nBody", fields=["language"], format="plain text with title", + custom_summary={"max_number_of_sentences": 3}) + assert "fields" not in seen[0].url.params + assert seen[1].url.params["fields"] == "language" + assert json.loads(seen[1].content) == [{ + "lang": "auto", "format": "plain text with title", "content": "Title\n\nBody", + "customSummary": {"maxNumberOfSentences": 3}, + }] + seen.clear() + await db.entities("x") + assert seen[0].url.params["fields"] == "entities,sentiment" diff --git a/tests/test_readme_examples.py b/tests/test_readme_examples.py index b78912b..7933b62 100644 --- a/tests/test_readme_examples.py +++ b/tests/test_readme_examples.py @@ -290,6 +290,31 @@ def handler(request: httpx.Request) -> httpx.Response: assert result["sentiment"] == 0.3 +def test_readme_sync_nlp(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json=[{"summary": "Acme launched a rocket.", "language": "en"}]) + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + result = db.nlp( + "Acme Launches Rocket\n\nAcme announced a new rocket today.", + fields=["summary", "categories", "language"], + format="plain text with title", + document_type="news article", + custom_summary={"max_number_of_sentences": 1}, + ) + assert result["summary"] == "Acme launched a rocket." + (req,) = seen + assert req.url.params["fields"] == "summary,categories,language" + assert json.loads(req.content) == [{ + "lang": "auto", "format": "plain text with title", + "content": "Acme Launches Rocket\n\nAcme announced a new rocket today.", + "documentType": "news article", "customSummary": {"maxNumberOfSentences": 1}, + }] + + # --------------------------------------------------------------------------- # Authentication # --------------------------------------------------------------------------- diff --git a/tests/typing/test_types.py b/tests/typing/test_types.py index 7509e54..4b2c399 100644 --- a/tests/typing/test_types.py +++ b/tests/typing/test_types.py @@ -18,6 +18,7 @@ DqlEntity, DqlResponse, EntitiesResponse, + NlpResponse, ExtractResponse, ImageExtractResponse, ProductExtractResponse, @@ -120,3 +121,15 @@ def _dql_named_option_types() -> None: assert_type(db.dql("q", non_canonical_facts=True, no_dedup_articles=True, filter_exclude="a"), DqlResponse) db.dql("q", type="bogus") # type: ignore[call-overload] db.dql("q", cluster="nope") # type: ignore[call-overload] + + +def _nlp_types() -> None: + assert_type(db.nlp("x"), NlpResponse) + assert_type(db.nlp("x", fields=["summary", "language"], custom_summary={"max_number_of_sentences": 2}), NlpResponse) + assert_type(db.entities("x"), EntitiesResponse) + db.nlp("x", fields=["bogus"]) # type: ignore[list-item] + db.nlp("x", format="html") # type: ignore[arg-type] + + +async def _nlp_async_types() -> None: + assert_type(await adb.nlp("x", format="plain text with title"), NlpResponse) From 4a838985b485ecda7cb9d54bb42f0a3e38939860 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:18:02 -1000 Subject: [PATCH 21/31] feat(web-search): accept 1-5 text values; snippet_length validation parity text may be a str or 1-5 strings (repeated text= params on the existing GET); 0 or >5 raises ValidationError. snippet_length is case-normalized and validated like the TS SDK; invalid values now raise ValidationError. Docstrings label snippet_length/rerank as undocumented. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 3 ++ src/diffbot/client.py | 20 ++++++--- src/diffbot/web_search.py | 28 ++++++++++--- tests/test_readme_examples.py | 11 +++++ tests/test_web_search.py | 77 +++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e94c916..99b73de 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,9 @@ for r in results["search_results"]: # "short" (~1-1.5 KB per result), "medium", "long" (default), or "full" results = db.web_search("diffbot knowledge graph", num_results=5, snippet_length="short") +# 1-5 queries in one call (sent as repeated text= params); 0 or >5 raises ValidationError +results = db.web_search(["diffbot knowledge graph", "diffbot crawl api"], num_results=3) + # "full" with a url: query returns the whole stored page page = db.web_search("url:https://en.wikipedia.org/wiki/Knowledge_graph", snippet_length="full") ``` diff --git a/src/diffbot/client.py b/src/diffbot/client.py index 13557fc..6b721db 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -427,7 +427,7 @@ def dql_fetch_ontology(self) -> Ontology: def web_search( self, - text: str, + text: Union[str, Sequence[str]], *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, @@ -435,10 +435,14 @@ def web_search( ) -> WebSearchResponse: """Search the web via the Diffbot LLM web search API. - snippet_length: "short", "medium", "long" or "full" — how much of each + text: one query string, or 1-5 strings (sent as repeated ``text=`` params); + anything else raises ``ValidationError``. + + snippet_length (undocumented upstream; ``rerank`` is set internally): "short", "medium", "long" or "full" — how much of each page's matching content to return. Omit for the API default (long). "full" returns the whole stored page (reranking is disabled for it); - use it with a url: query. + use it with a url: query. Values are case-insensitive; anything else + raises ``ValidationError``. """ return _web_search(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length) @@ -788,7 +792,7 @@ async def dql_fetch_ontology(self) -> Ontology: async def web_search( self, - text: str, + text: Union[str, Sequence[str]], *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, @@ -796,10 +800,14 @@ async def web_search( ) -> WebSearchResponse: """Search the web via the Diffbot LLM web search API. - snippet_length: "short", "medium", "long" or "full" — how much of each + text: one query string, or 1-5 strings (sent as repeated ``text=`` params); + anything else raises ``ValidationError``. + + snippet_length (undocumented upstream; ``rerank`` is set internally): "short", "medium", "long" or "full" — how much of each page's matching content to return. Omit for the API default (long). "full" returns the whole stored page (reranking is disabled for it); - use it with a url: query. + use it with a url: query. Values are case-insensitive; anything else + raises ``ValidationError``. """ return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length) diff --git a/src/diffbot/web_search.py b/src/diffbot/web_search.py index be7e21c..1227aa5 100644 --- a/src/diffbot/web_search.py +++ b/src/diffbot/web_search.py @@ -1,8 +1,9 @@ """Diffbot web search API.""" -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional, Sequence, Union, get_args from ._constants import LLM_BASE +from .errors import ValidationError from .types import SnippetLength, WebSearchResponse if TYPE_CHECKING: @@ -10,22 +11,37 @@ WEB_SEARCH_BASE = f"{LLM_BASE}/api/v1/web_search" +SNIPPET_LENGTHS = get_args(SnippetLength) +MAX_TEXT_VALUES = 5 + def _build_web_search_request( client: Any, - text: str, + text: Union[str, Sequence[str]], num_results: Optional[int], max_tokens: Optional[int], snippet_length: Optional[SnippetLength], ) -> Any: headers = {"Authorization": f"Bearer {client.token}"} - params: Dict[str, Any] = {"text": text} + params: Dict[str, Any] = {} + if isinstance(text, str): + params["text"] = text + else: + values = list(text) + if not 1 <= len(values) <= MAX_TEXT_VALUES: + raise ValidationError(f"text must be a string or 1-{MAX_TEXT_VALUES} strings; got {len(values)}") + params["text"] = values # sent as repeated text= params if num_results is not None: params["size"] = num_results if max_tokens is not None: params["maxTokens"] = max_tokens if snippet_length is not None: - params["snippetLength"] = snippet_length.upper() + length = snippet_length.lower() if isinstance(snippet_length, str) else snippet_length + if length not in SNIPPET_LENGTHS: + raise ValidationError( + f"snippet_length must be one of {', '.join(SNIPPET_LENGTHS)}; got {snippet_length!r}" + ) + params["snippetLength"] = length.upper() if params["snippetLength"] == "FULL": # the reranker always chunks; only the unranked path returns the stored page params["rerank"] = "none" @@ -34,7 +50,7 @@ def _build_web_search_request( def web_search( client: "Diffbot", - text: str, + text: Union[str, Sequence[str]], *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, @@ -47,7 +63,7 @@ def web_search( async def web_search_async( client: "DiffbotAsync", - text: str, + text: Union[str, Sequence[str]], *, num_results: Optional[int] = None, max_tokens: Optional[int] = None, diff --git a/tests/test_readme_examples.py b/tests/test_readme_examples.py index 7933b62..6ac54ab 100644 --- a/tests/test_readme_examples.py +++ b/tests/test_readme_examples.py @@ -229,6 +229,17 @@ def handler(request: httpx.Request) -> httpx.Response: assert r["content"] == "AI-powered web data." +def test_readme_sync_web_search_multiple_text(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.params.get_list("text") == ["diffbot knowledge graph", "diffbot crawl api"] + assert request.url.params["size"] == "3" + return httpx.Response(200, json={"search_results": []}) + + db = Diffbot(token="test-token", transport=httpx.MockTransport(handler)) + results = db.web_search(["diffbot knowledge graph", "diffbot crawl api"], num_results=3) + assert results["search_results"] == [] + + @pytest.mark.anyio async def test_readme_async_web_search(): def handler(request: httpx.Request) -> httpx.Response: diff --git a/tests/test_web_search.py b/tests/test_web_search.py index 9e95304..d87a75a 100644 --- a/tests/test_web_search.py +++ b/tests/test_web_search.py @@ -49,3 +49,80 @@ def handler(request: httpx.Request) -> httpx.Response: db.web_search("url:https://example.com", snippet_length="full") assert seen["snippetLength"] == "FULL" assert seen["rerank"] == "none" + + +""" +Multi-value text and snippet_length validation (S2) +""" + +import httpx # noqa: E402 + +from diffbot import Diffbot, DiffbotAsync, ValidationError # noqa: E402 + + +def _cap(): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200, json={"search_results": []}) + + return seen, handler + + +@pytest.mark.parametrize("n", [1, 2, 5]) +def test_web_search_text_sequence_sends_repeated_params(n): + seen, handler = _cap() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + texts = [f"q{i}" for i in range(n)] + db.web_search(texts) + (req,) = seen + assert req.method == "GET" + assert req.url.params.get_list("text") == texts + assert req.headers["authorization"] == "Bearer t" + + +def test_web_search_single_string_unchanged(): + seen, handler = _cap() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.web_search("one query", num_results=1) + assert seen[0].url.params.get_list("text") == ["one query"] + assert seen[0].url.params["size"] == "1" + + +@pytest.mark.parametrize("texts", [[], [f"q{i}" for i in range(6)]]) +def test_web_search_text_out_of_range_raises(texts): + seen, handler = _cap() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + with pytest.raises(ValidationError, match="1-5"): + db.web_search(texts) + assert seen == [] + + +def test_web_search_snippet_length_case_normalized_and_validated(): + seen, handler = _cap() + db = Diffbot(token="t", transport=httpx.MockTransport(handler)) + db.web_search("q", snippet_length="Medium") # type: ignore[arg-type] + assert seen[0].url.params["snippetLength"] == "MEDIUM" + with pytest.raises(ValidationError, match="snippet_length"): + db.web_search("q", snippet_length="huge") # type: ignore[arg-type] + assert len(seen) == 1 + + +@pytest.mark.anyio +async def test_async_web_search_multi_text_and_validation(): + seen, handler = _cap() + db = DiffbotAsync(token="t", transport=httpx.MockTransport(handler)) + await db.web_search(["a", "b"], snippet_length="full") + assert seen[0].url.params.get_list("text") == ["a", "b"] + assert seen[0].url.params["snippetLength"] == "FULL" + assert seen[0].url.params["rerank"] == "none" + await db.web_search("solo") + assert seen[1].url.params.get_list("text") == ["solo"] + with pytest.raises(ValidationError): + await db.web_search([]) + with pytest.raises(ValidationError): + await db.web_search(["x"] * 6) + with pytest.raises(ValidationError, match="snippet_length"): + await db.web_search("q", snippet_length="nope") # type: ignore[arg-type] + assert len(seen) == 2 From d2558f57f6c5942f6ff7b1444ad76bf71cd9220a Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:18:44 -1000 Subject: [PATCH 22/31] feat(ontology): dql_fetch_ontology_text and dql_refresh_ontology_async MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the Python/TypeScript ontology asymmetry (PHASE0_LEFTOVERS §4). Raw JSON text fetch on both clients; async refresh on DiffbotAsync (dql_refresh_ontology) and in diffbot.kg (dql_refresh_ontology_async). dql_fetch_ontology reuses the text fetch. Co-Authored-By: Claude Sonnet 5.5 --- PHASE0_LEFTOVERS.md | 9 ++++--- src/diffbot/client.py | 15 ++++++++++++ src/diffbot/kg.py | 33 +++++++++++++++++++------ tests/test_ontology.py | 49 ++++++++++++++++++++++++++++++++++++++ tests/typing/test_types.py | 7 ++++++ 5 files changed, 103 insertions(+), 10 deletions(-) diff --git a/PHASE0_LEFTOVERS.md b/PHASE0_LEFTOVERS.md index fd6e9dd..613a6c5 100644 --- a/PHASE0_LEFTOVERS.md +++ b/PHASE0_LEFTOVERS.md @@ -27,10 +27,13 @@ The TypeScript SDK corrupted `xls`/`xlsx` exports by re-encoding text. Python's byte path (`response.content`) was already correct; no code change was needed. Unchanged by the move to POST. Covered by the existing raw-mode tests. -## 4. Ontology parity: scheduled for S2 +## 4. Ontology parity: landed in S2 -Adding `dql_fetch_ontology_text()` and `dql_refresh_ontology_async()` to close the -Python/TypeScript asymmetry is scheduled for **S2** (Python 3.4.0). +`dql_fetch_ontology_text()` (sync + async) and `dql_refresh_ontology_async()` landed in +**S2** (Python 3.4.0, branch `feat/sdk-s2`). `dql_fetch_ontology()` reuses the text +fetch. On the `DiffbotAsync` client the refresh is named `dql_refresh_ontology` (the +async client drops the `_async` suffix, as for `dql_fetch_ontology`); the module-level +function in `diffbot.kg` is `dql_refresh_ontology_async`. ## 5. Response TypedDicts: landed in S1 diff --git a/src/diffbot/client.py b/src/diffbot/client.py index 6b721db..6c0fa60 100644 --- a/src/diffbot/client.py +++ b/src/diffbot/client.py @@ -88,9 +88,12 @@ dql_async as _dql_async, dql_fetch_ontology as _dql_fetch_ontology, dql_fetch_ontology_async as _dql_fetch_ontology_async, + dql_fetch_ontology_text as _dql_fetch_ontology_text, + dql_fetch_ontology_text_async as _dql_fetch_ontology_text_async, dql_parallel as _dql_parallel, dql_parallel_async as _dql_parallel_async, dql_refresh_ontology as _dql_refresh_ontology, + dql_refresh_ontology_async as _dql_refresh_ontology_async, ) from .ontology import Ontology from .web_search import ( @@ -425,6 +428,10 @@ def dql_fetch_ontology(self) -> Ontology: """Download the ontology and return it as a queryable Ontology (no caching).""" return _dql_fetch_ontology(self) + def dql_fetch_ontology_text(self) -> str: + """Download the ontology and return the raw JSON text (no parsing, no caching).""" + return _dql_fetch_ontology_text(self) + def web_search( self, text: Union[str, Sequence[str]], @@ -790,6 +797,14 @@ async def dql_fetch_ontology(self) -> Ontology: """Download the ontology and return it as a queryable Ontology (no caching).""" return await _dql_fetch_ontology_async(self) + async def dql_fetch_ontology_text(self) -> str: + """Download the ontology and return the raw JSON text (no parsing, no caching).""" + return await _dql_fetch_ontology_text_async(self) + + async def dql_refresh_ontology(self, dest: pathlib.Path) -> None: + """Download the Diffbot Knowledge Graph ontology and write it to dest.""" + await _dql_refresh_ontology_async(self, dest) + async def web_search( self, text: Union[str, Sequence[str]], diff --git a/src/diffbot/kg.py b/src/diffbot/kg.py index 4a7dc71..594659b 100644 --- a/src/diffbot/kg.py +++ b/src/diffbot/kg.py @@ -196,23 +196,42 @@ async def _one(q: Dict[str, Any]) -> Any: return list(await asyncio.gather(*(_one(q) for q in queries))) +def dql_fetch_ontology_text(client: "Diffbot") -> str: + """Download the ontology and return the raw JSON text, undecoded.""" + return client.request("GET", client.ontology_url).text + + +async def dql_fetch_ontology_text_async(client: "DiffbotAsync") -> str: + """Async variant of :func:`dql_fetch_ontology_text`.""" + return (await client.request("GET", client.ontology_url)).text + + +def _write_ontology(dest: pathlib.Path, content: bytes) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(content) + + def dql_refresh_ontology(client: "Diffbot", dest: pathlib.Path) -> None: response = client.request("GET", client.ontology_url) - dest.parent.mkdir(parents=True, exist_ok=True) - dest.write_bytes(response.content) + _write_ontology(dest, response.content) + + +async def dql_refresh_ontology_async(client: "DiffbotAsync", dest: pathlib.Path) -> None: + """Async variant of :func:`dql_refresh_ontology`.""" + response = await client.request("GET", client.ontology_url) + await asyncio.to_thread(_write_ontology, dest, response.content) def dql_fetch_ontology(client: "Diffbot") -> Ontology: """Download the ontology and return it as a queryable :class:`Ontology`. Performs no caching — the caller decides whether and where to hold onto the - result. Use :func:`dql_refresh_ontology` instead to persist raw bytes to disk. + result. Use :func:`dql_refresh_ontology` instead to persist raw bytes to disk, + or :func:`dql_fetch_ontology_text` for the raw JSON text. """ - response = client.request("GET", client.ontology_url) - return Ontology.from_json(response.content) + return Ontology.from_json(dql_fetch_ontology_text(client)) async def dql_fetch_ontology_async(client: "DiffbotAsync") -> Ontology: """Async variant of :func:`dql_fetch_ontology`.""" - response = await client.request("GET", client.ontology_url) - return Ontology.from_json(response.content) + return Ontology.from_json(await dql_fetch_ontology_text_async(client)) diff --git a/tests/test_ontology.py b/tests/test_ontology.py index a91c4dc..f0b780f 100644 --- a/tests/test_ontology.py +++ b/tests/test_ontology.py @@ -117,3 +117,52 @@ def handler(request: httpx.Request) -> httpx.Response: ] ) assert [r["hits"] for r in results] == [5, 100] + + +""" +Raw text fetch and async refresh (S2) +""" + +RAW = json.dumps(FIXTURE_ONTOLOGY) + + +def _raw_handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == "/kg/ontology" + return httpx.Response(200, text=RAW) + + +def test_dql_fetch_ontology_text_returns_raw_json_text(): + db = Diffbot(token="t", transport=httpx.MockTransport(_raw_handler)) + text = db.dql_fetch_ontology_text() + assert text == RAW + assert json.loads(text) == FIXTURE_ONTOLOGY + # the parsed variant still works on the same payload + assert db.dql_fetch_ontology().types() == ["Organization", "Person"] + + +@pytest.mark.anyio +async def test_async_dql_fetch_ontology_text_and_parsed(): + db = DiffbotAsync(token="t", transport=httpx.MockTransport(_raw_handler)) + assert await db.dql_fetch_ontology_text() == RAW + assert (await db.dql_fetch_ontology()).types() == ["Organization", "Person"] + + +def test_dql_refresh_ontology_writes_bytes(tmp_path): + db = Diffbot(token="t", transport=httpx.MockTransport(_raw_handler)) + dest = tmp_path / "nested" / "ontology.json" + db.dql_refresh_ontology(dest) + assert json.loads(dest.read_text()) == FIXTURE_ONTOLOGY + + +@pytest.mark.anyio +async def test_async_dql_refresh_ontology_writes_bytes(tmp_path): + db = DiffbotAsync(token="t", transport=httpx.MockTransport(_raw_handler)) + dest = tmp_path / "nested" / "ontology.json" + await db.dql_refresh_ontology(dest) + assert json.loads(dest.read_text()) == FIXTURE_ONTOLOGY + from diffbot.kg import dql_refresh_ontology_async + + other = tmp_path / "b.json" + await dql_refresh_ontology_async(db, other) + assert other.read_bytes() == dest.read_bytes() diff --git a/tests/typing/test_types.py b/tests/typing/test_types.py index 4b2c399..07133c5 100644 --- a/tests/typing/test_types.py +++ b/tests/typing/test_types.py @@ -4,6 +4,7 @@ that the calls themselves work against a mock transport. """ import json +import pathlib from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, cast import httpx @@ -133,3 +134,9 @@ def _nlp_types() -> None: async def _nlp_async_types() -> None: assert_type(await adb.nlp("x", format="plain text with title"), NlpResponse) + + +async def _ontology_text_types() -> None: + assert_type(db.dql_fetch_ontology_text(), str) + assert_type(await adb.dql_fetch_ontology_text(), str) + assert_type(await adb.dql_refresh_ontology(pathlib.Path("o.json")), None) From dcc975496c33f7a00ba6e8058b50f16909eabab9 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 13:19:12 -1000 Subject: [PATCH 23/31] feat(live,docs): S2 live cases, 3.4.0 changelog, S2 leftovers Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++ S1_LEFTOVERS.md | 5 ++--- S2_LEFTOVERS.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test_live.py | 18 +++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 S2_LEFTOVERS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d16df99..fbd5b8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,48 @@ # Changelog +## 3.4.0 (unreleased) + +Named options on endpoints the SDK already calls, plus `nlp()` and two ontology +helpers. Additive, except that invalid `snippet_length` values now raise (see +Changed). The package version in `pyproject.toml` is bumped at release time. + +### Added + +- `dql()`/`dql_async()` (and so `dql_parallel`) keyword-only options `type` + (`"query"`, `"text"`, `"queryTextFallback"`, `"crawl"`; `"crawl"` searches crawl + collections), `col`, `filter_exclude`, `exportseparator`, `exportfile`, + `jsonmode`, `non_canonical_facts`, `no_dedup_articles`, `cluster`. They go in the + POST JSON body (wire names `filterExclude`, `nonCanonicalFacts`, + `noDedupArticles`); the token stays in the query string. Passing the same key as a + named option and in `extra` raises `ValidationError`. `report` and `view` are not + named; use `extra`. `size` still defaults to 10. Types: `DqlType`, `DqlJsonMode`, + `DqlCluster`, `DqlOptions`. +- `crawl()` keyword-only create options `url_crawl_regex`, `url_process_regex`, + `page_process_pattern`, `max_to_crawl_per_subdomain`, `max_to_process_per_subdomain`, + `use_canonical`, `notify_webhook`, `notify_email` (query params on the existing + GET). `custom_headers` now accepts a string or a sequence of strings (sent as + repeated `customHeaders` params). Recurring-crawl admin (`repeat`, `maxRounds`, ...) + is not exposed. +- `nlp(text, *, fields, lang, format, document_type, custom_summary)` (sync and + async) on the existing NL endpoint, single document. `fields` is a sequence of + `NlpField`; `format` may be `"plain text with title"`; + `custom_summary={"max_number_of_sentences": n}`. `entities()` is unchanged. Types: + `NlpField`, `NlpFormat`, `NlpCustomSummary`, `NlpResponse`. +- `web_search(text=...)` accepts a string or 1-5 strings (repeated `text=` params on + the existing GET). Out-of-range lengths raise `ValidationError` before any request. +- `dql_fetch_ontology_text()` (sync and async) returns the raw ontology JSON text; + `dql_refresh_ontology` is now also available on `DiffbotAsync` (module function + `diffbot.kg.dql_refresh_ontology_async`). `dql_fetch_ontology()` reuses the text fetch. +- Live suite: `web_search` (single and two-value `text`) and `entities` + `nlp`. + +### Changed + +- **`web_search(snippet_length=...)` is validated** (parity with the TypeScript SDK): + the value is case-insensitive and must be one of `short`, `medium`, `long`, `full`; + anything else raises `ValidationError` instead of being sent upstream. Breaking only + for invalid input. `snippet_length` and `rerank` are undocumented upstream and + labelled so in the docstrings. + ## 3.3.0 (unreleased) Response types, a pyright gate, and two small methods that complete existing diff --git a/S1_LEFTOVERS.md b/S1_LEFTOVERS.md index e3763e7..abc1b86 100644 --- a/S1_LEFTOVERS.md +++ b/S1_LEFTOVERS.md @@ -32,6 +32,5 @@ the plan's wording, and what is unverified. ## Not in S1 (see plan §2/§8) -S2 items (DQL named params, crawl-create options, `nlp()`, web-search multi-value -`text` + `snippet_length` validation, ontology text/async parity) and everything marked -Out. +The S2 items that were listed here landed in S2: see `S2_LEFTOVERS.md`. Everything +marked Out in the plan is still out. diff --git a/S2_LEFTOVERS.md b/S2_LEFTOVERS.md new file mode 100644 index 0000000..57ced6d --- /dev/null +++ b/S2_LEFTOVERS.md @@ -0,0 +1,48 @@ +# S2 leftovers (Python) + +S2 (plan §2, §6, §8; Python 3.4.0) landed on `feat/sdk-s2`, stacked on `feat/sdk-s1`. +`pyproject.toml` is not bumped (release-time). + +## Landed + +- DQL named params (`type`, `col`, `filter_exclude`, `exportseparator`, `exportfile`, + `jsonmode`, `non_canonical_facts`, `no_dedup_articles`, `cluster`), POST body, `extra` + collision -> `ValidationError`, forwarded by `dql_parallel`. +- Crawl-create named options and `custom_headers: str | Sequence[str]`. +- `nlp()` (sync/async) on the NL endpoint; `entities()` untouched. +- Web search: 1-5 `text` values, `snippet_length` validation. +- Ontology: `dql_fetch_ontology_text`, `dql_refresh_ontology_async`. +- Live cases: web search (single/2-value `text`), entities + nlp. + +## Deliberately narrower than the plan + +- **DQL overloads use `**options: Unpack[DqlOptions]`** (type-level only, like extract); + the implementation has explicit keyword parameters. +- **`extra` collision applies to the new named keys only.** Existing keys (`size`, + `from`, `format`, `filter`, `exportspec`) can still be overridden through `extra`, as + before; changing that would be a behaviour change outside S2. A named option whose + value is `None` does not collide, so `extra={"jsonmode": ...}` keeps working. +- **`crawl()` has no `extra`**, so `repeat`/`maxRounds`/`seedRecrawlFrequency`/ + `onlyProcessIfNew` are simply unreachable (per the task, no new backdoor). + `Diffbot.crawl` still takes `**kwargs: Any`, so the new names are not type-checked + there. `use_canonical` is sent as `1`/`0` only when set. +- **`nlp()` response is one `NlpResponse` TypedDict** (all fields optional), not a + per-`fields` `Pick`. `facts`/`records`/`categories`/`sentences` are `List[Any]`. + `custom_summary` uses the Python key `max_number_of_sentences` (wire + `maxNumberOfSentences`); the 1-10 range is left to the server. +- **Async refresh naming:** on `DiffbotAsync` it is `dql_refresh_ontology` (no suffix, + like `dql_fetch_ontology`); `dql_refresh_ontology_async` is the `diffbot.kg` function. +- `snippet_length` stays typed as the lowercase `SnippetLength` Literal; uppercase input + is accepted at runtime only. +- `crawl_get_job` still returns `{}` for a missing job (TS: `undefined`); unchanged. + +## Unverified + +- Live tests (`pytest -m live`) were not run in this session, including the two new + cases. Not confirmed against the real API: `nlp()` `documentType`/`customSummary` + placement inside the document object (taken from the docs), repeated `text=` on the + live web search endpoint, repeated `customHeaders` on the crawl create, and + `type=crawl` DQL requests. +- The TypeScript S2 counterpart was not present when this was written, so field-for-field + parity (names, validation messages) is unchecked beyond `snippetLength`. +- Plan §11 open question about DQL POST parity is still open (see `S1_LEFTOVERS.md`). diff --git a/tests/test_live.py b/tests/test_live.py index 416d48b..0ea6798 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -45,6 +45,24 @@ def test_live_entities(db): assert "entities" in result +def test_live_nlp_on_one_sentence(db): + sentence = "Diffbot is a company based in Menlo Park." + ents = db.entities(sentence) + assert "entities" in ents + result = db.nlp(sentence, fields=["entities", "sentiment", "language"]) + assert isinstance(result, dict) + assert "entities" in result + assert "language" in result + + +def test_live_web_search_single_and_two_value_text(db): + one = db.web_search("diffbot knowledge graph", num_results=1) + assert "search_results" in one + assert len(one["search_results"]) <= 1 + two = db.web_search(["diffbot knowledge graph", "diffbot crawl api"], num_results=1) + assert "search_results" in two + + def _redacted(account) -> str: """Account bodies carry ``token``/``childTokens``: only ever show the other keys.""" return f"keys={sorted(k for k in account if k not in ('token', 'childTokens'))}" From dba418f0f23bc23168d8ce58fb14f5f5bd4d5502 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 14:49:08 -1000 Subject: [PATCH 24/31] docs: record Q2/Q3 live confirmations in S2 leftovers Co-Authored-By: Claude Sonnet 5.5 --- S2_LEFTOVERS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/S2_LEFTOVERS.md b/S2_LEFTOVERS.md index 57ced6d..122495b 100644 --- a/S2_LEFTOVERS.md +++ b/S2_LEFTOVERS.md @@ -46,3 +46,8 @@ S2 (plan §2, §6, §8; Python 3.4.0) landed on `feat/sdk-s2`, stacked on `feat/ - The TypeScript S2 counterpart was not present when this was written, so field-for-field parity (names, validation messages) is unchecked beyond `snippetLength`. - Plan §11 open question about DQL POST parity is still open (see `S1_LEFTOVERS.md`). + +## Live confirmations + +- Q2 confirmed live: repeated `text=` on GET web search is accepted (HTTP 200, results mix both values; 5 values also 200). +- Q3 confirmed live: GET crawl create works, but job names must be <= 30 chars (longer returns HTTP 200 + errorCode 505 and creates nothing; the SDK now raises APIError for that). From 427982d4f2509cabe3582d267d9a0cfc77243da9 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:07:27 -1000 Subject: [PATCH 25/31] refactor(crawl): share the named create-option mapping between sync and async Co-Authored-By: Claude Opus 5.5 --- src/diffbot/crawl.py | 51 +++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index c73643e..c03aa22 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -79,16 +79,10 @@ def crawl( client, job_name, site, max_to_crawl, max_to_process, hops, restrict_domain, api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, - { - "urlCrawlRegEx": url_crawl_regex, - "urlProcessRegEx": url_process_regex, - "pageProcessPattern": page_process_pattern, - "maxToCrawlPerSubdomain": max_to_crawl_per_subdomain, - "maxToProcessPerSubdomain": max_to_process_per_subdomain, - "useCanonical": None if use_canonical is None else int(use_canonical), - "notifyWebhook": notify_webhook, - "notifyEmail": notify_email, - }, + _named_create_options( + url_crawl_regex, url_process_regex, page_process_pattern, max_to_crawl_per_subdomain, + max_to_process_per_subdomain, use_canonical, notify_webhook, notify_email, + ), ) _raise_on_create_error(client.request("GET", client.crawler_url, params=params)) @@ -142,6 +136,29 @@ def _raise_on_create_error(response: Any) -> None: raise APIError(response.status_code, response.text, headers=dict(response.headers)) +def _named_create_options( + url_crawl_regex: Optional[str], + url_process_regex: Optional[str], + page_process_pattern: Optional[str], + max_to_crawl_per_subdomain: Optional[int], + max_to_process_per_subdomain: Optional[int], + use_canonical: Optional[bool], + notify_webhook: Optional[str], + notify_email: Optional[str], +) -> Dict[str, Any]: + """Python names -> wire names for the S2 create options; ``None`` means "not set".""" + return { + "urlCrawlRegEx": url_crawl_regex, + "urlProcessRegEx": url_process_regex, + "pageProcessPattern": page_process_pattern, + "maxToCrawlPerSubdomain": max_to_crawl_per_subdomain, + "maxToProcessPerSubdomain": max_to_process_per_subdomain, + "useCanonical": None if use_canonical is None else int(use_canonical), + "notifyWebhook": notify_webhook, + "notifyEmail": notify_email, + } + + def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int, max_to_process: int, hops: int, restrict_domain: bool, api_url: str, crawl_delay: float, url_crawl_pattern: Optional[str], @@ -228,16 +245,10 @@ async def crawl_async( client, job_name, site, max_to_crawl, max_to_process, hops, restrict_domain, api_url, crawl_delay, url_crawl_pattern, url_process_pattern, obey_robots, use_proxies, custom_headers, - { - "urlCrawlRegEx": url_crawl_regex, - "urlProcessRegEx": url_process_regex, - "pageProcessPattern": page_process_pattern, - "maxToCrawlPerSubdomain": max_to_crawl_per_subdomain, - "maxToProcessPerSubdomain": max_to_process_per_subdomain, - "useCanonical": None if use_canonical is None else int(use_canonical), - "notifyWebhook": notify_webhook, - "notifyEmail": notify_email, - }, + _named_create_options( + url_crawl_regex, url_process_regex, page_process_pattern, max_to_crawl_per_subdomain, + max_to_process_per_subdomain, use_canonical, notify_webhook, notify_email, + ), ) _raise_on_create_error(await client.request("GET", client.crawler_url, params=params)) From 5fe3c47acab3c1f2d7fa0f385a54a91c050cb8b5 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 15:53:55 -1000 Subject: [PATCH 26/31] fix(transport): stop retrying when Retry-After exceeds max_delay A 429/5xx Retry-After was honored verbatim, so `Retry-After: 3600` made one call sleep an hour; max_delay capped only the exponential backoff. Now a Retry-After (seconds or HTTP-date) above RetryPolicy.max_delay means no sleep and no retry: the response is raised at once as RateLimitError with retry_after populated (or APIError with the header for a 5xx), in the sync, async and streamed paths. Values at or under the cap are honored as before. Non-finite seconds (inf/nan) fall back to backoff instead of crashing time.sleep. Matches the TypeScript SDK. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 +++ README.md | 6 ++ src/diffbot/_transport.py | 77 ++++++++++++++++------ tests/test_transport.py | 130 +++++++++++++++++++++++++++++++++++++- 4 files changed, 203 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbd5b8b..4be42bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,15 @@ Changed). The package version in `pyproject.toml` is bumped at release time. ### Changed +- **`Retry-After` now has a ceiling** (parity with the TypeScript SDK). + `RetryPolicy.max_delay` (default 5.0 s) used to cap only the exponential backoff, so a + 429/5xx with `Retry-After: 3600` made one call sleep an hour. Now, if `Retry-After` + (seconds or HTTP-date form) exceeds `max_delay`, the SDK neither sleeps nor retries + and raises at once: `RateLimitError` with `retry_after` set, or `APIError` with the + header in `headers` for a 5xx. Sync, async, and streamed requests all behave this + way. A `Retry-After` at or under the cap is honored as before; a non-finite value + (`inf`, `nan`) now falls back to backoff (`inf` used to crash `time.sleep`). Raise + `max_delay` to keep waiting out longer server-requested delays. - **`web_search(snippet_length=...)` is validated** (parity with the TypeScript SDK): the value is case-insensitive and must be one of `short`, `medium`, `long`, `full`; anything else raises `ValidationError` instead of being sent upstream. Breaking only diff --git a/README.md b/README.md index 99b73de..e08374d 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,12 @@ db = Diffbot(token="YOUR_TOKEN", retry=False) # or: retry=RetryPolicy(max_attempts=5) ``` +`max_delay` (seconds, default 5.0) caps the backoff and is also the ceiling for +`Retry-After`. If the server asks for a longer wait (in seconds or as an HTTP date), the +SDK does not sleep or retry: the call raises at once with `RateLimitError` +(`retry_after` holds the raw header) or, for a 5xx, `APIError` (see +`headers["retry-after"]`), so you decide when to try again. + The transport also accepts a per-call `retryable=False` escape hatch; the SDK uses it internally for `crawl_delete_job`, a GET-shaped mutation that must not be retried blindly. diff --git a/src/diffbot/_transport.py b/src/diffbot/_transport.py index de1d683..fd1d4fb 100644 --- a/src/diffbot/_transport.py +++ b/src/diffbot/_transport.py @@ -6,6 +6,7 @@ """ import asyncio +import math import random import time from dataclasses import dataclass @@ -28,6 +29,12 @@ class RetryPolicy: per client with ``retry=False`` on the client constructor; disable per call with ``retryable=False`` — required for GET-shaped mutations (e.g. ``crawl?delete=1``) that must never be retried blindly. + + ``max_delay`` (seconds) caps the exponential backoff and is also the ceiling for + a server's ``Retry-After``: if a 429/5xx asks for a longer wait (seconds or + HTTP-date form), the SDK does not sleep or retry and raises at once — + ``RateLimitError`` with ``retry_after`` set, or ``APIError`` (header in + ``headers``) for a 5xx. """ max_attempts: int = 3 @@ -52,22 +59,54 @@ def _is_retryable_status(status_code: int) -> bool: return status_code == 429 or status_code >= 500 -def _compute_delay(attempt: int, retry_after: Optional[str], policy: RetryPolicy) -> float: - """Parses ``Retry-After`` as seconds or an HTTP date; falls back to exponential backoff.""" - if retry_after: - try: - return max(0.0, float(retry_after)) - except ValueError: - pass - try: - dt = parsedate_to_datetime(retry_after) - return max(0.0, dt.timestamp() - time.time()) - except (TypeError, ValueError): - pass +def _parse_retry_after(retry_after: Optional[str]) -> Optional[float]: + """Parses ``Retry-After`` as seconds or an HTTP date into a non-negative delay (seconds).""" + if not retry_after: + return None + try: + seconds = float(retry_after) + except ValueError: + pass + else: + if math.isfinite(seconds): + return max(0.0, seconds) + return None + try: + dt = parsedate_to_datetime(retry_after) + return max(0.0, dt.timestamp() - time.time()) + except (TypeError, ValueError): + return None + + +def _compute_delay(attempt: int, retry_after: Optional[str], policy: RetryPolicy) -> Optional[float]: + """Delay before the next attempt: a parseable ``Retry-After`` is honored as given, + otherwise exponential backoff capped at ``max_delay``. Returns ``None`` when + ``Retry-After`` exceeds ``max_delay``, meaning "do not retry; raise now".""" + retry_after_s = _parse_retry_after(retry_after) + if retry_after_s is not None: + return None if retry_after_s > policy.max_delay else retry_after_s + return _backoff_delay(attempt, policy) + + +def _backoff_delay(attempt: int, policy: RetryPolicy) -> float: exponential = min(policy.max_delay, policy.base_delay * (2 ** (attempt - 1))) return exponential * (0.5 + random.random() * 0.5) if policy.jitter else exponential +def _retry_delay( + policy: Optional[RetryPolicy], attempt: int, max_attempts: int, response: httpx.Response +) -> Optional[float]: + """Seconds to sleep before retrying ``response``, or ``None`` to stop and raise. + + ``None`` also covers a ``Retry-After`` beyond ``policy.max_delay``: the response is + then raised as-is (``RateLimitError.retry_after`` / ``APIError.headers``) instead of + sleeping for however long the server asked. + """ + if not policy or attempt >= max_attempts or not _is_retryable_status(response.status_code): + return None + return _compute_delay(attempt, response.headers.get("retry-after"), policy) + + def raise_for_status(response: httpx.Response) -> None: if response.is_success: return @@ -145,14 +184,15 @@ def request( response = http.send(req, **send_kwargs) except httpx.TransportError: if policy and attempt < max_attempts: - time.sleep(_compute_delay(attempt, None, policy)) + time.sleep(_backoff_delay(attempt, policy)) attempt += 1 continue raise - if policy and attempt < max_attempts and _is_retryable_status(response.status_code): + delay = _retry_delay(policy, attempt, max_attempts, response) + if delay is not None: if stream: response.close() - time.sleep(_compute_delay(attempt, response.headers.get("retry-after"), policy)) + time.sleep(delay) attempt += 1 continue if response.is_success: @@ -191,14 +231,15 @@ async def request_async( response = await http.send(req, **send_kwargs) except httpx.TransportError: if policy and attempt < max_attempts: - await asyncio.sleep(_compute_delay(attempt, None, policy)) + await asyncio.sleep(_backoff_delay(attempt, policy)) attempt += 1 continue raise - if policy and attempt < max_attempts and _is_retryable_status(response.status_code): + delay = _retry_delay(policy, attempt, max_attempts, response) + if delay is not None: if stream: await response.aclose() - await asyncio.sleep(_compute_delay(attempt, response.headers.get("retry-after"), policy)) + await asyncio.sleep(delay) attempt += 1 continue if response.is_success: diff --git a/tests/test_transport.py b/tests/test_transport.py index 6ec5456..215c2ba 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -1,8 +1,13 @@ +from email.utils import formatdate +from types import SimpleNamespace +from typing import List + import httpx import pytest from diffbot import APIError, AuthError, Diffbot, DiffbotAsync, NotFoundError, RateLimitError, RetryPolicy -from diffbot._transport import raise_for_status +from diffbot import _transport +from diffbot._transport import _compute_delay, raise_for_status def make_client(handler, **kwargs) -> Diffbot: @@ -203,6 +208,129 @@ def handler(request: httpx.Request) -> httpx.Response: assert response.status_code == 200 +""" +Retry-After ceiling (max_delay) +""" + +FAKE_NOW = 1767225600.0 # 2026-01-01T00:00:00Z +CAPPED = RetryPolicy(max_delay=10.0, jitter=False) + + +def http_date(offset_seconds: float) -> str: + return formatdate(FAKE_NOW + offset_seconds, usegmt=True) + + +@pytest.fixture +def sleeps(monkeypatch) -> List[float]: + """Freezes ``time.time()`` as seen by the transport and records sleeps instead of sleeping.""" + recorded: List[float] = [] + + async def fake_async_sleep(seconds: float) -> None: + recorded.append(seconds) + + monkeypatch.setattr(_transport, "time", SimpleNamespace(time=lambda: FAKE_NOW, sleep=recorded.append)) + monkeypatch.setattr(_transport, "asyncio", SimpleNamespace(sleep=fake_async_sleep)) + return recorded + + +def fail_once(status: int, retry_after: str): + calls = {"n": 0} + + def handler(request: httpx.Request) -> httpx.Response: + calls["n"] += 1 + if calls["n"] == 1: + return httpx.Response(status, headers={"retry-after": retry_after}, text="slow down") + return httpx.Response(200, json={"ok": True}) + + return calls, handler + + +@pytest.mark.parametrize( + "retry_after, expected", + [("3", 3.0), (http_date(4), 4.0), ("10", 10.0), ("0", 0.0), (http_date(-60), 0.0)], + ids=["seconds", "http-date", "seconds-at-cap", "zero", "past-date"], +) +def test_compute_delay_honors_retry_after_at_or_under_the_cap(sleeps, retry_after, expected): + assert _compute_delay(1, retry_after, CAPPED) == expected + + +@pytest.mark.parametrize("retry_after", ["3600", "10.5", http_date(3600)], ids=["seconds", "fraction", "http-date"]) +def test_compute_delay_returns_none_when_retry_after_exceeds_the_cap(sleeps, retry_after): + assert _compute_delay(1, retry_after, CAPPED) is None + + +@pytest.mark.parametrize("retry_after", ["soon", "inf", "nan"]) +def test_compute_delay_falls_back_to_backoff_for_unusable_retry_after(sleeps, retry_after): + assert _compute_delay(1, retry_after, CAPPED) == CAPPED.base_delay + + +@pytest.mark.parametrize("retry_after, expected", [("3", 3.0), (http_date(4), 4.0)], ids=["seconds", "http-date"]) +def test_sleeps_for_retry_after_under_the_cap_then_retries(sleeps, retry_after, expected): + calls, handler = fail_once(429, retry_after) + db = make_client(handler, retry=CAPPED) + response = db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + assert response.status_code == 200 + assert sleeps == [expected] + + +@pytest.mark.parametrize("retry_after", ["3600", http_date(3600)], ids=["seconds", "http-date"]) +def test_raises_rate_limit_error_without_sleeping_when_retry_after_exceeds_the_cap(sleeps, retry_after): + calls, handler = fail_once(429, retry_after) + db = make_client(handler, retry=CAPPED) + with pytest.raises(RateLimitError) as exc_info: + db.request("GET", "https://example.com/x") + assert calls["n"] == 1 + assert sleeps == [] + assert exc_info.value.retry_after == retry_after + assert exc_info.value.body == "slow down" + + +def test_raises_api_error_without_sleeping_when_a_5xx_retry_after_exceeds_the_cap(sleeps): + calls, handler = fail_once(503, "11") + db = make_client(handler, retry=CAPPED) + with pytest.raises(APIError) as exc_info: + db.request("GET", "https://example.com/x") + assert not isinstance(exc_info.value, RateLimitError) + assert calls["n"] == 1 + assert sleeps == [] + assert exc_info.value.status_code == 503 + assert (exc_info.value.headers or {})["retry-after"] == "11" + + +def test_streamed_request_raises_with_body_when_retry_after_exceeds_the_cap(sleeps): + calls, handler = fail_once(429, "3600") + db = make_client(handler, retry=CAPPED) + with pytest.raises(RateLimitError) as exc_info: + _transport.request(db._http, CAPPED, "GET", "https://example.com/x", stream=True) + assert calls["n"] == 1 + assert sleeps == [] + assert exc_info.value.body == "slow down" + + +@pytest.mark.anyio +@pytest.mark.parametrize("retry_after, expected", [("3", 3.0), (http_date(4), 4.0)], ids=["seconds", "http-date"]) +async def test_async_sleeps_for_retry_after_under_the_cap(sleeps, retry_after, expected): + calls, handler = fail_once(429, retry_after) + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler), retry=CAPPED) + response = await db.request("GET", "https://example.com/x") + assert calls["n"] == 2 + assert response.status_code == 200 + assert sleeps == [expected] + + +@pytest.mark.anyio +@pytest.mark.parametrize("retry_after", ["3600", http_date(3600)], ids=["seconds", "http-date"]) +async def test_async_raises_without_sleeping_when_retry_after_exceeds_the_cap(sleeps, retry_after): + calls, handler = fail_once(429, retry_after) + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler), retry=CAPPED) + with pytest.raises(RateLimitError) as exc_info: + await db.request("GET", "https://example.com/x") + assert calls["n"] == 1 + assert sleeps == [] + assert exc_info.value.retry_after == retry_after + + """ raise_for_status """ From 1ff55a016febe37ba3f5feee3aebddab0712c0e4 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:13:07 -1000 Subject: [PATCH 27/31] chore(release): 3.4.0 Bump pyproject.toml and the uv.lock entry for the project from 3.1.0 to 3.4.0. __version__, the db --version string, and the User-Agent all read installed metadata, so there is no second constant. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1e03c13..eceeb87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "diffbot" -version = "3.1.0" +version = "3.4.0" description = "Python client library for Diffbot APIs" readme = "README.md" requires-python = ">=3.9" diff --git a/uv.lock b/uv.lock index a29be13..32e82a7 100644 --- a/uv.lock +++ b/uv.lock @@ -90,7 +90,7 @@ wheels = [ [[package]] name = "diffbot" -version = "3.1.0" +version = "3.4.0" source = { editable = "." } dependencies = [ { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, From 7d0241b3590622e069d83058846a7c5dbf19290a Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:13:13 -1000 Subject: [PATCH 28/31] docs: one 3.4.0 changelog entry; drop the resolved leftovers files 3.2.0 and 3.3.0 were never published, so the changelog now has a single 3.4.0 entry (Keep a Changelog) covering everything since 3.1.x, with the breaking changes listed first: DQL over POST, crawl create HTTP 200 + errorCode now raises, Retry-After over max_delay is not retried, retries are on by default, snippet_length is validated, and dql extra collisions raise. PHASE0/S1/S2_LEFTOVERS.md recorded only closed items, deliberate narrowings, and live checks that have now run. The narrowings are under "Known limitations" in the changelog. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 275 ++++++++++++++++++-------------------------- PHASE0_LEFTOVERS.md | 43 ------- S1_LEFTOVERS.md | 36 ------ S2_LEFTOVERS.md | 53 --------- 4 files changed, 115 insertions(+), 292 deletions(-) delete mode 100644 PHASE0_LEFTOVERS.md delete mode 100644 S1_LEFTOVERS.md delete mode 100644 S2_LEFTOVERS.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be42bb..35bff35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,130 +1,132 @@ # Changelog -## 3.4.0 (unreleased) - -Named options on endpoints the SDK already calls, plus `nlp()` and two ontology -helpers. Additive, except that invalid `snippet_length` values now raise (see -Changed). The package version in `pyproject.toml` is bumped at release time. +All notable changes to the `diffbot` Python package are documented here. The format +follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses +[Semantic Versioning](https://semver.org/). + +## [3.4.0] - 2026-11-17 + +First release since 3.1.x. It rolls up three development lines that were never +published on their own: the shared transport, sync/async de-duplication and DQL over +POST (planned as 3.2.0), response types plus `account`/`verify_token`/`crawl_download`/ +extract options (3.3.0), and named options on existing endpoints plus `nlp()` (3.4.0). +No new Diffbot API areas are added. + +### ⚠ Breaking changes + +- **DQL is sent as `POST` with a JSON body.** `dql()`, `dql_parallel()` and their async + twins keep the same signature and results, but mocks, proxies, or mirrors keyed on + `GET` + `?query=` stop matching. See [Migrating DQL](#migrating-dql-mocks-and-proxies) + below. +- **`crawl()` raises `APIError` when the create call answers HTTP 200 with + `errorCode`/`error`** (for example errorCode 505, job name over 30 characters). It used + to yield `JOB_CREATED` for a job that was never created. The error keeps the real + status (200), body, and headers. +- **A `Retry-After` above `RetryPolicy.max_delay` (default 5.0 s) is no longer waited + out or retried.** The call raises at once: `RateLimitError` with `retry_after` = raw + header, or `APIError` with `headers["retry-after"]` for a 5xx (sync, async, and + streamed). Raise `max_delay` to keep waiting. +- **Requests are now retried by default** (`429`, `5xx`, connection errors; 3 attempts + with backoff and jitter). `crawl_delete_job` is never retried. Pass `retry=False` for + the old single-attempt behavior. +- **`web_search(snippet_length=...)` is validated:** anything other than `short`, + `medium`, `long`, `full` (case-insensitive) raises `ValidationError` instead of being + sent upstream. +- **`dql()` raises `ValidationError` when `extra` repeats one of the new named options** + (`type`, `col`, `filter_exclude`, ...). +- **Type-level:** `dql`, `extract`, `web_search`, `entities`, `crawl_list_jobs`, + `crawl_get_job` return `TypedDict`s instead of `Dict[str, Any]`, and several methods + gained `@overload`s. Runtime values are unchanged; a type checker may flag undeclared + keys (escape hatch: `typing.cast(Dict[str, Any], resp)`). ### Added -- `dql()`/`dql_async()` (and so `dql_parallel`) keyword-only options `type` - (`"query"`, `"text"`, `"queryTextFallback"`, `"crawl"`; `"crawl"` searches crawl - collections), `col`, `filter_exclude`, `exportseparator`, `exportfile`, - `jsonmode`, `non_canonical_facts`, `no_dedup_articles`, `cluster`. They go in the - POST JSON body (wire names `filterExclude`, `nonCanonicalFacts`, - `noDedupArticles`); the token stays in the query string. Passing the same key as a - named option and in `extra` raises `ValidationError`. `report` and `view` are not - named; use `extra`. `size` still defaults to 10. Types: `DqlType`, `DqlJsonMode`, - `DqlCluster`, `DqlOptions`. -- `crawl()` keyword-only create options `url_crawl_regex`, `url_process_regex`, +- **Transport:** per-host base URLs (`api_base`, `kg_base`, `llm_base`, `nlp_base` on + `Diffbot` and `DiffbotAsync`; the older `*_url` options still win when both are set); + documented repeated query params; `client.request(method, url, ...)` with `DELETE`, + form (`data=`) and raw (`content=`) bodies, a per-call `timeout`, and a + `follow_redirects` override; the `RetryPolicy` dataclass and `retry` option with + per-call `retryable=False`. +- **Errors:** `NotFoundError` (404) and `JobError`, both subclasses of `APIError`; + `APIError.headers` from the error response. +- **Types:** request aliases (`ChatMessage`, `ResponseFormat`, `ExportFormat`, + `SnippetLength`) and response `TypedDict`s exported from `diffbot` (`DqlResponse`/ + `DqlEntity`, `WebSearchResponse`/`WebSearchResult`, `EntitiesResponse`, + `ExtractResponse` and per-API variants, `CrawlJob`, `CrawlDataRecord`, + `AccountResponse`, `AskChunk`, ...). stdlib only, Python 3.9-compatible, no Pydantic. +- `account(days=None)` and `verify_token()` (sync and async) over `GET /v4/account`. + `verify_token()` returns `False` only on an auth failure. The account response carries + `token` and `childTokens`; don't log it whole. +- `crawl_download(name, *, format="json", type=None, num=None)` (sync and async) over + `GET /v3/crawl/data`. `format="csv"` or `type="urls"` returns `str`, otherwise a list + of records; a non-list JSON body (e.g. unknown job) raises `JobError`. The crawl + watcher uses it. For tools that can't block: `crawl(..., watch=False)` -> + `crawl_get_job` -> `crawl_download`. +- `extract()` keyword-only options: `timeout` (server-side ms, default 30000), `mode`, + `fields` (per-API `Literal` names), `fallback`, `discussion`, `paging`, `max_pages`, + `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, + `category_confidence`, `use_proxy`, and `extra` (`proxy`/`proxyAuth`; collisions with a + named option or `token`/`url` raise `ValidationError`). Setting `timeout` raises the + per-call transport timeout to at least `timeout + 10s`. +- `dql()`/`dql_async()` (and `dql_parallel`) keyword-only options `type` (`"query"`, + `"text"`, `"queryTextFallback"`, `"crawl"`), `col`, `filter_exclude`, + `exportseparator`, `exportfile`, `jsonmode`, `non_canonical_facts`, + `no_dedup_articles`, `cluster`, sent in the POST body. `report` and `view` stay + reachable via `extra`. Types: `DqlType`, `DqlJsonMode`, `DqlCluster`, `DqlOptions`. +- `crawl()` create options `url_crawl_regex`, `url_process_regex`, `page_process_pattern`, `max_to_crawl_per_subdomain`, `max_to_process_per_subdomain`, - `use_canonical`, `notify_webhook`, `notify_email` (query params on the existing - GET). `custom_headers` now accepts a string or a sequence of strings (sent as - repeated `customHeaders` params). Recurring-crawl admin (`repeat`, `maxRounds`, ...) - is not exposed. -- `nlp(text, *, fields, lang, format, document_type, custom_summary)` (sync and - async) on the existing NL endpoint, single document. `fields` is a sequence of - `NlpField`; `format` may be `"plain text with title"`; - `custom_summary={"max_number_of_sentences": n}`. `entities()` is unchanged. Types: + `use_canonical`, `notify_webhook`, `notify_email`; `custom_headers` accepts a string or + a sequence (repeated params). The 30-character job-name limit is documented. +- `nlp(text, *, fields, lang, format, document_type, custom_summary)` (sync and async) on + the existing NL endpoint, single document. `entities()` is unchanged. Types: `NlpField`, `NlpFormat`, `NlpCustomSummary`, `NlpResponse`. -- `web_search(text=...)` accepts a string or 1-5 strings (repeated `text=` params on - the existing GET). Out-of-range lengths raise `ValidationError` before any request. -- `dql_fetch_ontology_text()` (sync and async) returns the raw ontology JSON text; - `dql_refresh_ontology` is now also available on `DiffbotAsync` (module function - `diffbot.kg.dql_refresh_ontology_async`). `dql_fetch_ontology()` reuses the text fetch. -- Live suite: `web_search` (single and two-value `text`) and `entities` + `nlp`. +- `web_search(text=...)` accepts a string or 1-5 strings (repeated `text=`); + out-of-range lengths raise `ValidationError` before any request. +- `dql_fetch_ontology_text()` (sync and async); `dql_refresh_ontology` on + `DiffbotAsync` (module function `diffbot.kg.dql_refresh_ontology_async`). +- Dev: `pyright` gate (basic on `src/`, strict on `types.py`, plus `tests/typing`) in CI; + live test workflow (push to `main`, manual dispatch, twice-monthly cron; never on PRs). ### Changed -- **`Retry-After` now has a ceiling** (parity with the TypeScript SDK). - `RetryPolicy.max_delay` (default 5.0 s) used to cap only the exponential backoff, so a - 429/5xx with `Retry-After: 3600` made one call sleep an hour. Now, if `Retry-After` - (seconds or HTTP-date form) exceeds `max_delay`, the SDK neither sleeps nor retries - and raises at once: `RateLimitError` with `retry_after` set, or `APIError` with the - header in `headers` for a 5xx. Sync, async, and streamed requests all behave this - way. A `Retry-After` at or under the cap is honored as before; a non-finite value - (`inf`, `nan`) now falls back to backoff (`inf` used to crash `time.sleep`). Raise - `max_delay` to keep waiting out longer server-requested delays. -- **`web_search(snippet_length=...)` is validated** (parity with the TypeScript SDK): - the value is case-insensitive and must be one of `short`, `medium`, `long`, `full`; - anything else raises `ValidationError` instead of being sent upstream. Breaking only - for invalid input. `snippet_length` and `rerank` are undocumented upstream and - labelled so in the docstrings. - -## 3.3.0 (unreleased) - -Response types, a pyright gate, and two small methods that complete existing -workflows. Additive minor: no wire changes. The package version in -`pyproject.toml` is bumped at release time. - -### Added - -- Response `TypedDict`s (stdlib, `total=False`, no Pydantic, Python 3.9-compatible), - exported from `diffbot`: `DqlResponse`/`DqlEntity`, `WebSearchResponse`/ - `WebSearchResult`, `EntitiesResponse`, `ExtractResponse` and per-API - `ArticleExtractResponse`, `ProductExtractResponse`, ... with their object types, - `CrawlJob`, `CrawlDataRecord`, `AccountResponse`, `AskChunk`. -- `account(days=None)` and `verify_token()` (sync and async): `GET /v4/account`. - `verify_token()` calls it with `days=1`; only an auth failure (401/403) returns - `False`, every other error propagates. The account response carries `token` and - `childTokens`: don't log it whole. -- `crawl_download(name, *, format="json", type=None, num=None)` (sync and async): - `GET /v3/crawl/data`. `format="csv"` or `type="urls"` (that report is always CSV) - returns `str`, otherwise a list of records; a JSON body that is not a list (e.g. an - unknown job) raises `JobError`. The crawl watcher now uses it. For tools that can't block, the recommended - pattern is `crawl(..., watch=False)` -> `crawl_get_job` -> `crawl_download`. -- `extract()` keyword-only options after `fmt`: `timeout` (server-side fetch timeout - in ms, default still 30000), `mode` (overrides the `fmt="markdown"` -> `llm` - mapping), `fields` (sequence, comma-joined; per-API `Literal` names), `fallback`, - `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, - `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` - for anything else (`proxy`/`proxyAuth`; keys that collide with a named option or - `token`/`url` raise `ValidationError`). When you set `timeout`, the per-call - transport timeout is raised to at least `timeout + 10s` (never lowered). -- Dev: `pyright` (basic on `src/`, strict on `types.py`) plus `tests/typing` - `assert_type` checks, run in `tests.yml`. +- `dql()` request shape: `POST {dql_url}?token=…` with `{"query", "size", "from"?, + "format"?, "filter"?, "exportspec"?, …extra}`. The token stays in the query string; + the query text no longer appears in URLs, and the URL length limit is gone. `extra` is + typed `Mapping[str, Any]` and its values go into the JSON body verbatim. +- A failed crawl job raises `JobError` (carrying the job name) instead of a synthesized + `APIError(500, ...)`. `JobError` subclasses `APIError`, so `except APIError` still + catches it. +- Internal: shared sync/async transport helpers in `_transport.py` replace the + per-handler `_http.get(...)` + raise pattern. ### Fixed +- `ask()`/`ask_json()` (sync and async): a non-2xx response while streaming raises + `AuthError`/`RateLimitError`/`APIError` as documented instead of + `httpx.ResponseNotRead`. - `dql(format="csv")` and `dql(format="jsonl")` without `raw=True` crashed in - `response.json()`. `csv` now returns `str`, `xls`/`xlsx` return `bytes`, and - `jsonl` returns an iterator of entities (`AsyncIterator` from the async client). - `raw=True` is unchanged (undecoded `bytes`). -- `crawl()` create failures that the crawler reports as HTTP 200 with `errorCode`/ - `error` in the body (e.g. errorCode 505 for a job name over 30 characters, which - creates nothing) now raise `APIError` (status 200, the real body and headers, - `message` from `error`) instead of yielding `JOB_CREATED` for a job that doesn't exist. - -### Changed (type-level only) - -- Return annotations changed from `Dict[str, Any]` to the TypedDicts above - (`dql`, `extract`, `web_search`, `entities`, `crawl_list_jobs`, `crawl_get_job`), - and `dql`/`extract`/`crawl_download` gained `@overload`s. Runtime values are - unchanged. A type checker may now flag keys the types don't declare; wrap the - value in `typing.cast(Dict[str, Any], resp)` as the escape hatch. `dql_parallel` - returns `List[Any]` (each element's type follows its own call). + `response.json()`. `csv` now returns `str`, `xls`/`xlsx` return `bytes`, and `jsonl` + returns an iterator of entities (`AsyncIterator` on the async client). +- `crawl()` no longer reports `JOB_CREATED` for a create that failed with HTTP 200 + + `errorCode`. +- `APIError.message` falls back to the body's `error` field (the `{errorCode, error}` + shape), matching the TypeScript SDK. +- `Retry-After: inf` no longer crashes `time.sleep` with `OverflowError`; non-finite + values fall back to backoff. + +### Known limitations + +- `extra` can still override the pre-existing DQL keys (`size`, `from`, `format`, + `filter`, `exportspec`); only the new named keys are collision-checked. +- `crawl()` has no `extra`; recurring-crawl admin (`repeat`, `maxRounds`, ...) is not + exposed. `Diffbot.crawl(**kwargs)` does not type-check the new create options. +- `nlp()` returns one `NlpResponse` `TypedDict` (all fields optional), not a per-`fields` + type. `custom_summary` reads only the Python key `max_number_of_sentences`. - `crawl_get_job` still returns `{}` for a missing job (the TypeScript SDK returns - `undefined`); unchanged on purpose. - -## 3.2.0 (unreleased) - -Shared transport foundation, a sync/async de-duplication refactor, and DQL sent -as a POST. The package version in `pyproject.toml` is bumped at release time. - -### Changed + `undefined`). +- No generic success-path response headers. -- **DQL is now sent as `POST` with a JSON body.** `dql()`, `dql_parallel()` and - their async twins keep the same names, options, defaults and return values; - only the wire format changes: `POST {dql_url}?token=…` with - `{"query", "size", "from"?, "format"?, "filter"?, "exportspec"?, …extra}`. - The token stays in the query string; the query text no longer appears in the - URL, and the URL-length limit on long queries is gone. `extra` is now typed - `Mapping[str, Any]` (values go into the JSON body verbatim). -- `crawl()` failure now raises `JobError` (carrying the job name) instead of a - synthesized `APIError(500, ...)`. `JobError` subclasses `APIError`. - -#### Migration note: mocks and proxies +### Migrating DQL: mocks and proxies Test mocks keyed on `GET` and `?query=` stop matching. Before/after: @@ -135,7 +137,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert request.url.params["query"] == "type:Organization" return httpx.Response(200, json={"data": []}) -# after (3.2.0) +# after (3.4.0) def handler(request: httpx.Request) -> httpx.Response: assert request.method == "POST" assert json.loads(request.content)["query"] == "type:Organization" @@ -149,51 +151,4 @@ POSTs. If you need the old wire format, use the low-level call (no new API): db.request("GET", db.dql_url, params={"token": db.token, "query": q, "size": 10}) ``` -or pin `diffbot<3.2`. - -### Added - -- Per-host base URL options: `api_base`, `kg_base`, `llm_base`, `nlp_base` on - both `Diffbot` and `DiffbotAsync`. The older per-endpoint options - (`analyze_url`, `crawler_url`, `llm_url`, `web_search_url`, `nlp_url`, - `dql_url`, `ontology_url`) still work and win over the matching base when - both are set. -- Repeated query parameters: handler param builders now declare - `Sequence`-valued params explicitly (httpx already encoded them as repeated - keys; this makes it a documented, tested capability). -- `client.request(method, url, ...)` on both clients: `DELETE` + arbitrary - method/body/`Content-Type`, form-encoded (`data=`) and raw (`content=`) - bodies, a per-call `timeout` override separate from the client default, and - a `follow_redirects` override. -- A default-on retry policy (`RetryPolicy` dataclass, `retry` client option) - for `429`/`5xx`/connection errors, with backoff, jitter, and `Retry-After` - support, capped at 3 attempts by default. Disable with `retry=False`, or - opt a single call out with `retryable=False` (used internally by - `crawl_delete_job`/`crawl_delete_job_async`, a GET-shaped mutation that must - never be retried blindly). -- `NotFoundError` (404) and `JobError`, both subclassing `APIError` so - existing `except APIError` call sites are unaffected. -- `APIError.headers` (and subclasses), populated from the error response. -- Additive `TypedDict`/`Literal` type aliases in `diffbot.types`: - `ChatMessage`, `ResponseFormat`, `ExportFormat`, `SnippetLength` — used in - `ask()`/`ask_json()`/`dql()`/`web_search()` signatures. No Pydantic, no - runtime behavior change. - -### Fixed - -- `ask()`/`ask_json()` (sync and async): a non-2xx response (e.g. 401, 429) - while streaming now raises `AuthError`/`RateLimitError` as documented, - instead of `httpx.ResponseNotRead`. The error body is now read before - raising. -- `crawl()`'s job-failure path now raises `JobError` (carrying the job name) - instead of a synthesized `APIError(500, ...)`. -- `APIError.message` falls back to the body's `error` field when there is no - `message` (the `{errorCode, error}` shape several Diffbot APIs use), matching - the TypeScript SDK. - -### Internal - -- Sync/async de-duplication: `Diffbot`/`DiffbotAsync._raise_for_status` and - the repeated `client._http.get(...)` + raise pattern in every handler - module are replaced by shared helpers in a new `_transport.py`, used by - both clients through `client.request()`. +or pin `diffbot<3.4` (3.1.x). diff --git a/PHASE0_LEFTOVERS.md b/PHASE0_LEFTOVERS.md deleted file mode 100644 index 613a6c5..0000000 --- a/PHASE0_LEFTOVERS.md +++ /dev/null @@ -1,43 +0,0 @@ -# Phase 0 leftovers (Python) - -Everything in the Phase 0 scope landed. This file records the items that are -deliberately smaller than the original plan's framing, and how each was resolved -under the reduced-scope coverage plan (`sdk-reduced/plan.md`). It mirrors the -TypeScript SDK's own `PHASE0_LEFTOVERS.md`. - -## 1. Literal "endpoint registry" data structure: won't do - -The original plan framed per-host base URLs as part of "an endpoint registry". -What landed is per-host base URL resolution (`api_base`/`kg_base`/`llm_base`/ -`nlp_base`, with the older `*_url` options winning) directly in `client.py`, plus -an ad hoc `retryable=False` at the one call site that needs it -(`crawl_delete_job`). The reduced-scope plan adds no new API areas, so a -per-operation table would never get a second consumer. **Won't do.** - -## 2. Success-path response headers: won't do (DQL reports cut) - -Every *error* response's headers are attached to `APIError` (and subclasses) via -`headers`, populated in `_transport.raise_for_status()`. A generic opt-in for a -*successful* response's headers was only motivated by DQL coverage reports -(`X-Diffbot-CoverageReportId`), which are out of scope. **Won't do.** - -## 3. `dql(raw=True)` binary correctness: no bug existed on the Python side - -The TypeScript SDK corrupted `xls`/`xlsx` exports by re-encoding text. Python's -byte path (`response.content`) was already correct; no code change was needed. -Unchanged by the move to POST. Covered by the existing raw-mode tests. - -## 4. Ontology parity: landed in S2 - -`dql_fetch_ontology_text()` (sync + async) and `dql_refresh_ontology_async()` landed in -**S2** (Python 3.4.0, branch `feat/sdk-s2`). `dql_fetch_ontology()` reuses the text -fetch. On the `DiffbotAsync` client the refresh is named `dql_refresh_ontology` (the -async client drops the `_async` suffix, as for `dql_fetch_ontology`); the module-level -function in `diffbot.kg` is `dql_refresh_ontology_async`. - -## 5. Response TypedDicts: landed in S1 - -Request-side `TypedDict`/`Literal` aliases landed in `src/diffbot/types.py` with Phase 0. -Response shapes (`DqlResponse`, `WebSearchResponse`, `ExtractResponse`, `CrawlJob`, -`AccountResponse`, `AskChunk`, ...) land in S1 (Python 3.3.0, branch `feat/sdk-s1`) -together with the pyright CI gate, so they are checked from day one. diff --git a/S1_LEFTOVERS.md b/S1_LEFTOVERS.md deleted file mode 100644 index abc1b86..0000000 --- a/S1_LEFTOVERS.md +++ /dev/null @@ -1,36 +0,0 @@ -# S1 leftovers (Python) - -S1 (plan §5, §6, §8) landed on `feat/sdk-s1`. Notes on what is deliberately smaller than -the plan's wording, and what is unverified. - -## Deliberately narrower - -- **Overloads live on the client methods** (`Diffbot`/`DiffbotAsync` `dql`, `extract`, - `crawl_download`), the public surface. The module-level shared implementations in - `kg.py`/`extract.py`/`crawl.py` return `Any` to avoid duplicating ~50 overloads. -- **Per-API extract options use `**options: Unpack[...]` in the overloads** (a - `TYPE_CHECKING`-only import of `typing_extensions.Unpack`; no runtime dependency). - At runtime the implementation takes `**options` and rejects unknown names with - `TypeError`. Every option is accepted for every API at runtime; only the type - level restricts them per API. -- **Transport timeout is raised only when the caller sets `timeout`.** With the default - 30000 ms and the default 30 s client timeout the two are still equal, as before. -- `dql_parallel` returns `List[Any]` (heterogeneous per-call formats). -- `EntitiesResponse` mirrors TS `Pick<..., "entities" | "sentiment">`; `errors` is not declared. -- `CrawlDataRecord` is a best-effort shape (the crawl data JSON is example-only in the - docs); keep it open to change. The TypeScript counterpart did not exist yet when this - was written, so field-for-field parity with its `CrawlDataRecord` is unchecked. - -## Unverified here - -- Live tests (`pytest -m live`) were not run against the real API (no token resolved in - the build environment); `test_live_verify_token_rejects_a_bad_token` was. - The live crawl test (start/poll/download/delete) is therefore unexercised. -- Open question from plan §11: DQL POST parity (jsonl/csv/xlsx with `exportspec`, - JSON strings vs numbers in `extra`) should be confirmed with the Diffbot contact - before release. - -## Not in S1 (see plan §2/§8) - -The S2 items that were listed here landed in S2: see `S2_LEFTOVERS.md`. Everything -marked Out in the plan is still out. diff --git a/S2_LEFTOVERS.md b/S2_LEFTOVERS.md deleted file mode 100644 index 122495b..0000000 --- a/S2_LEFTOVERS.md +++ /dev/null @@ -1,53 +0,0 @@ -# S2 leftovers (Python) - -S2 (plan §2, §6, §8; Python 3.4.0) landed on `feat/sdk-s2`, stacked on `feat/sdk-s1`. -`pyproject.toml` is not bumped (release-time). - -## Landed - -- DQL named params (`type`, `col`, `filter_exclude`, `exportseparator`, `exportfile`, - `jsonmode`, `non_canonical_facts`, `no_dedup_articles`, `cluster`), POST body, `extra` - collision -> `ValidationError`, forwarded by `dql_parallel`. -- Crawl-create named options and `custom_headers: str | Sequence[str]`. -- `nlp()` (sync/async) on the NL endpoint; `entities()` untouched. -- Web search: 1-5 `text` values, `snippet_length` validation. -- Ontology: `dql_fetch_ontology_text`, `dql_refresh_ontology_async`. -- Live cases: web search (single/2-value `text`), entities + nlp. - -## Deliberately narrower than the plan - -- **DQL overloads use `**options: Unpack[DqlOptions]`** (type-level only, like extract); - the implementation has explicit keyword parameters. -- **`extra` collision applies to the new named keys only.** Existing keys (`size`, - `from`, `format`, `filter`, `exportspec`) can still be overridden through `extra`, as - before; changing that would be a behaviour change outside S2. A named option whose - value is `None` does not collide, so `extra={"jsonmode": ...}` keeps working. -- **`crawl()` has no `extra`**, so `repeat`/`maxRounds`/`seedRecrawlFrequency`/ - `onlyProcessIfNew` are simply unreachable (per the task, no new backdoor). - `Diffbot.crawl` still takes `**kwargs: Any`, so the new names are not type-checked - there. `use_canonical` is sent as `1`/`0` only when set. -- **`nlp()` response is one `NlpResponse` TypedDict** (all fields optional), not a - per-`fields` `Pick`. `facts`/`records`/`categories`/`sentences` are `List[Any]`. - `custom_summary` uses the Python key `max_number_of_sentences` (wire - `maxNumberOfSentences`); the 1-10 range is left to the server. -- **Async refresh naming:** on `DiffbotAsync` it is `dql_refresh_ontology` (no suffix, - like `dql_fetch_ontology`); `dql_refresh_ontology_async` is the `diffbot.kg` function. -- `snippet_length` stays typed as the lowercase `SnippetLength` Literal; uppercase input - is accepted at runtime only. -- `crawl_get_job` still returns `{}` for a missing job (TS: `undefined`); unchanged. - -## Unverified - -- Live tests (`pytest -m live`) were not run in this session, including the two new - cases. Not confirmed against the real API: `nlp()` `documentType`/`customSummary` - placement inside the document object (taken from the docs), repeated `text=` on the - live web search endpoint, repeated `customHeaders` on the crawl create, and - `type=crawl` DQL requests. -- The TypeScript S2 counterpart was not present when this was written, so field-for-field - parity (names, validation messages) is unchecked beyond `snippetLength`. -- Plan §11 open question about DQL POST parity is still open (see `S1_LEFTOVERS.md`). - -## Live confirmations - -- Q2 confirmed live: repeated `text=` on GET web search is accepted (HTTP 200, results mix both values; 5 values also 200). -- Q3 confirmed live: GET crawl create works, but job names must be <= 30 chars (longer returns HTTP 200 + errorCode 505 and creates nothing; the SDK now raises APIError for that). From 40e98ba01259cb24ea58b55142277b8d8ced2a9c Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:14:46 -1000 Subject: [PATCH 29/31] docs(changelog): say 3.4.0 includes 3.1.1 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35bff35..9423ba5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the projec ## [3.4.0] - 2026-11-17 -First release since 3.1.x. It rolls up three development lines that were never +First release since 3.1.1 (and includes everything in it). It rolls up three development lines that were never published on their own: the shared transport, sync/async de-duplication and DQL over POST (planned as 3.2.0), response types plus `account`/`verify_token`/`crawl_download`/ extract options (3.3.0), and named options on existing endpoints plus `nlp()` (3.4.0). From 9ad9f63028298e58becd637f4fa374df2e692b32 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:28:47 -1000 Subject: [PATCH 30/31] ci: publish to PyPI via trusted publishing on v* tags Add .github/workflows/publish-pypi.yml: verify (tag == pyproject version == diffbot.__version__, unit tests, pyright), build (uv build sdist + wheel, twine check --strict, artifact), and publish (tags only, environment "pypi", id-token: write, pypa/gh-action-pypi-publish@release/v1; no API token). workflow_dispatch is a dry run that never uploads. Concurrency never cancels an in-flight publish. release.yml (binaries) is unchanged. Document the release path (merge to main, push tag vX.Y.Z) in the README and mark `make release` as a manual fallback. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/publish-pypi.yml | 126 +++++++++++++++++++++++++++++ Makefile | 8 +- README.md | 21 +++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/publish-pypi.yml diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 0000000..e151a69 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,126 @@ +name: Publish to PyPI + +# Publishes the `diffbot` sdist + wheel to PyPI via trusted publishing (OIDC): +# no API tokens are stored anywhere. Triggered by pushing a version tag +# (e.g. v3.4.0). workflow_dispatch runs verify + build + twine check only, +# as a dry run; it never uploads. +# +# One-time setup (maintainer): on pypi.org > diffbot > Manage > Publishing, add +# a GitHub publisher with owner `diffbot`, repository `diffbot-python`, +# workflow `publish-pypi.yml`, environment `pypi`; and create a GitHub +# environment named `pypi` in the repo settings (protection rules optional). +# +# The standalone `db` binaries are published separately by release.yml, which +# fires on the same tag. +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: read + +# Never cancel an in-flight run: a cancelled publish could leave PyPI with only +# some of the files for a version, and PyPI doesn't allow re-uploading them. +concurrency: + group: publish-pypi-${{ github.ref }} + cancel-in-progress: false + +jobs: + verify: + name: Verify version, tests, and types + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Install dependencies (locked) + run: uv sync --frozen --extra dev + + # __version__ is read from the installed distribution metadata, so this + # checks that pyproject.toml's version is what the package reports. + - name: Check tag matches package version + run: | + set -euo pipefail + pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) + runtime=$(uv run --frozen python -c 'import diffbot; print(diffbot.__version__)') + if [ "$pkg" != "$runtime" ]; then + echo "::error::pyproject.toml version ${pkg} does not match diffbot.__version__ ${runtime}." + exit 1 + fi + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + tag=${GITHUB_REF_NAME#v} + if [ "$pkg" != "$tag" ]; then + echo "::error::Tag v${tag} does not match pyproject.toml version ${pkg}. Bump pyproject.toml (e.g. 'make bump-patch') and re-tag." + exit 1 + fi + fi + echo "Package version: ${pkg}" + + # Unit tests only: pyproject's addopts deselect `-m live` by default. + - name: Run tests + run: uv run --frozen pytest + + - name: Run pyright + run: uv run --frozen pyright + + build: + name: Build sdist and wheel + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Build + run: uv build --sdist --wheel --out-dir dist + + - name: Check distributions + run: uvx twine check --strict dist/* + + - name: Check built version matches pyproject.toml + run: | + set -euo pipefail + pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) + ls dist + test -f "dist/diffbot-${pkg}.tar.gz" + test -f "dist/diffbot-${pkg}-py3-none-any.whl" + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: python-dist + path: dist/ + if-no-files-found: error + + publish: + name: Publish to PyPI + needs: build + # Tag pushes only; workflow_dispatch stops after build. + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: + name: pypi + url: https://pypi.org/project/diffbot/ + permissions: + id-token: write + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: python-dist + path: dist/ + + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/Makefile b/Makefile index 292a93b..a6fd399 100644 --- a/Makefile +++ b/Makefile @@ -61,10 +61,16 @@ _release-test: fi && \ UV_PUBLISH_TOKEN="$$TOKEN" uv publish --publish-url https://test.pypi.org/legacy/ -# Publish to real PyPI. Confirmation gate before upload (PyPI does not allow re-uploads). +# MANUAL FALLBACK ONLY. The normal release path is CI: merge to main, then push +# a tag vX.Y.Z matching pyproject.toml; .github/workflows/publish-pypi.yml +# publishes to PyPI via trusted publishing (no token). Use this target only if +# that workflow is unavailable. It uploads with a token from macOS Keychain +# (service: pypi-token-pypi) after a confirmation gate (PyPI does not allow +# re-uploads). release: build _release _release: + @echo "NOTE: 'make release' is the manual fallback; normally push tag vX.Y.Z and let publish-pypi.yml publish." @VERSION=$$(grep '^version' $(PYPROJECT) | head -1 | cut -d'"' -f2) && \ STATUS=$$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/$(PKG)/$$VERSION/json") && \ if [ "$$STATUS" = "200" ]; then \ diff --git a/README.md b/README.md index e08374d..8757b27 100644 --- a/README.md +++ b/README.md @@ -525,6 +525,27 @@ environment variable or `~/.diffbot/credentials`: DIFFBOT_API_TOKEN=your_token python -m pytest -m live ``` +## Releasing + +Releases are published by CI; nothing is uploaded from a laptop. + +1. Bump the version in `pyproject.toml` (`make bump-patch`, `bump-minor`, or `bump-major`), update `CHANGELOG.md`, and merge to `main`. +2. Tag the merge commit and push the tag: + ```bash + git tag vX.Y.Z && git push origin vX.Y.Z + ``` + +The tag must match the `pyproject.toml` version. Pushing it runs two workflows: +[`publish-pypi.yml`](.github/workflows/publish-pypi.yml) checks the version, runs the +unit tests and pyright, builds the sdist and wheel, and publishes them to PyPI via +[trusted publishing](https://docs.pypi.org/trusted-publishers/) (no API token), and +[`release.yml`](.github/workflows/release.yml) builds the standalone `db` binaries +and attaches them to a GitHub Release. Running `publish-pypi.yml` manually +(workflow_dispatch) is a dry run: it builds and checks but never uploads. + +`make release` (token from macOS Keychain) remains only as a manual fallback for +when the workflow can't be used. + ## Acknowledgements Thanks to [@attilaolah](https://github.com/attilaolah), who published the original `diffbot` package in 2012 and looked after the name for 14 years. From b7937a8eb2c75ac3c54fdc3d47b06c24cb8bf2b6 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Tue, 17 Nov 2026 16:39:45 -1000 Subject: [PATCH 31/31] ci: merge PyPI publishing into release.yml (no environment) One Release workflow on v* tags: verify, build sdist/wheel and db binaries, publish to PyPI via trusted publishing (after binaries build), then the GitHub Release. Concurrency never cancels an in-flight publish. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/publish-pypi.yml | 126 ------------------------- .github/workflows/release.yml | 147 ++++++++++++++++++++++++----- Makefile | 4 +- README.md | 15 +-- 4 files changed, 131 insertions(+), 161 deletions(-) delete mode 100644 .github/workflows/publish-pypi.yml diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml deleted file mode 100644 index e151a69..0000000 --- a/.github/workflows/publish-pypi.yml +++ /dev/null @@ -1,126 +0,0 @@ -name: Publish to PyPI - -# Publishes the `diffbot` sdist + wheel to PyPI via trusted publishing (OIDC): -# no API tokens are stored anywhere. Triggered by pushing a version tag -# (e.g. v3.4.0). workflow_dispatch runs verify + build + twine check only, -# as a dry run; it never uploads. -# -# One-time setup (maintainer): on pypi.org > diffbot > Manage > Publishing, add -# a GitHub publisher with owner `diffbot`, repository `diffbot-python`, -# workflow `publish-pypi.yml`, environment `pypi`; and create a GitHub -# environment named `pypi` in the repo settings (protection rules optional). -# -# The standalone `db` binaries are published separately by release.yml, which -# fires on the same tag. -on: - push: - tags: ["v*"] - workflow_dispatch: - -permissions: - contents: read - -# Never cancel an in-flight run: a cancelled publish could leave PyPI with only -# some of the files for a version, and PyPI doesn't allow re-uploading them. -concurrency: - group: publish-pypi-${{ github.ref }} - cancel-in-progress: false - -jobs: - verify: - name: Verify version, tests, and types - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - python-version: "3.12" - - - name: Install dependencies (locked) - run: uv sync --frozen --extra dev - - # __version__ is read from the installed distribution metadata, so this - # checks that pyproject.toml's version is what the package reports. - - name: Check tag matches package version - run: | - set -euo pipefail - pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) - runtime=$(uv run --frozen python -c 'import diffbot; print(diffbot.__version__)') - if [ "$pkg" != "$runtime" ]; then - echo "::error::pyproject.toml version ${pkg} does not match diffbot.__version__ ${runtime}." - exit 1 - fi - if [ "${GITHUB_REF_TYPE}" = "tag" ]; then - tag=${GITHUB_REF_NAME#v} - if [ "$pkg" != "$tag" ]; then - echo "::error::Tag v${tag} does not match pyproject.toml version ${pkg}. Bump pyproject.toml (e.g. 'make bump-patch') and re-tag." - exit 1 - fi - fi - echo "Package version: ${pkg}" - - # Unit tests only: pyproject's addopts deselect `-m live` by default. - - name: Run tests - run: uv run --frozen pytest - - - name: Run pyright - run: uv run --frozen pyright - - build: - name: Build sdist and wheel - needs: verify - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - python-version: "3.12" - - - name: Build - run: uv build --sdist --wheel --out-dir dist - - - name: Check distributions - run: uvx twine check --strict dist/* - - - name: Check built version matches pyproject.toml - run: | - set -euo pipefail - pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) - ls dist - test -f "dist/diffbot-${pkg}.tar.gz" - test -f "dist/diffbot-${pkg}-py3-none-any.whl" - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: python-dist - path: dist/ - if-no-files-found: error - - publish: - name: Publish to PyPI - needs: build - # Tag pushes only; workflow_dispatch stops after build. - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: - name: pypi - url: https://pypi.org/project/diffbot/ - permissions: - id-token: write - steps: - - name: Download artifact - uses: actions/download-artifact@v4 - with: - name: python-dist - path: dist/ - - - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43cdc31..e5d525c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,24 +1,110 @@ -name: Release Binaries +name: Release -# Builds the self-contained `db` CLI binary for each supported platform and -# publishes them (with SHA256 checksums) to a GitHub Release. Triggered by -# pushing a version tag (e.g. v0.2.1); also runnable manually for dry runs. +# One workflow for a release. Triggered by pushing a version tag (e.g. v3.4.0); +# also runnable manually (workflow_dispatch) as a dry run that verifies and +# builds everything but never publishes. +# +# verify -> build-dist (sdist + wheel) --\ +# verify -> build-binaries (db CLI) -----+-> publish-pypi -> github-release +# +# PyPI publishing uses trusted publishing (OIDC), so no API token is stored. +# One-time setup (maintainer): on pypi.org > diffbot > Manage > Publishing, add +# a GitHub publisher with owner `diffbot`, repository `diffbot-python`, +# workflow `release.yml`, and NO environment (leave it blank). on: push: tags: ["v*"] workflow_dispatch: permissions: - contents: write + contents: read -# Re-pushing a tag (or re-dispatching) cancels an in-flight run for the same -# ref so a stuck build can't pile up behind a newer attempt. +# Never cancel an in-flight run: a cancelled publish could leave PyPI with only +# some of the files for a version (PyPI doesn't allow re-uploading them) or +# leave PyPI and the GitHub Release out of sync. Runs for the same ref queue. concurrency: group: release-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: - build: + verify: + name: Verify version, tests, and types + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Install dependencies (locked) + run: uv sync --frozen --extra dev + + # __version__ is read from the installed distribution metadata, so this + # checks that pyproject.toml's version is what the package reports. + - name: Check tag matches package version + run: | + set -euo pipefail + pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) + runtime=$(uv run --frozen python -c 'import diffbot; print(diffbot.__version__)') + if [ "$pkg" != "$runtime" ]; then + echo "::error::pyproject.toml version ${pkg} does not match diffbot.__version__ ${runtime}." + exit 1 + fi + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + tag=${GITHUB_REF_NAME#v} + if [ "$pkg" != "$tag" ]; then + echo "::error::Tag v${tag} does not match pyproject.toml version ${pkg}. Bump pyproject.toml (e.g. 'make bump-patch') and re-tag." + exit 1 + fi + fi + echo "Package version: ${pkg}" + + # Unit tests only: pyproject's addopts deselect `-m live` by default. + - name: Run tests + run: uv run --frozen pytest + + - name: Run pyright + run: uv run --frozen pyright + + build-dist: + name: Build sdist and wheel + needs: verify + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + + - name: Build + run: uv build --sdist --wheel --out-dir dist + + - name: Check distributions + run: uvx twine check --strict dist/* + + - name: Check built version matches pyproject.toml + run: | + set -euo pipefail + pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) + ls dist + test -f "dist/diffbot-${pkg}.tar.gz" + test -f "dist/diffbot-${pkg}-py3-none-any.whl" + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: python-dist + path: dist/ + if-no-files-found: error + + build-binaries: + needs: verify name: Build ${{ matrix.target_os }}-${{ matrix.arch }} runs-on: ${{ matrix.runner }} # Cap the wait so a capacity-starved runner fails this leg in minutes @@ -46,20 +132,6 @@ jobs: steps: - uses: actions/checkout@v4 - # Fail fast if the tag doesn't match pyproject.toml's version, so the - # GitHub Release name can never disagree with the version the binaries - # report (db --version) or the one published to PyPI. - - name: Check tag matches package version - if: startsWith(github.ref, 'refs/tags/') - run: | - pkg=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2) - tag=${GITHUB_REF_NAME#v} - if [ "$pkg" != "$tag" ]; then - echo "::error::Tag v${tag} does not match pyproject.toml version ${pkg}. Bump pyproject.toml (e.g. 'make bump-patch') and re-tag." - exit 1 - fi - echo "Tag and package version agree: ${pkg}" - - name: Install uv uses: astral-sh/setup-uv@v5 with: @@ -86,12 +158,35 @@ jobs: dist/db-${{ matrix.target_os }}-${{ matrix.arch }}.sha256 if-no-files-found: error - release: + publish-pypi: + name: Publish to PyPI + # Only after the sdist/wheel AND all db binaries built successfully. + needs: [build-dist, build-binaries] + # Tag pushes only; workflow_dispatch stops after the builds. + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + timeout-minutes: 15 + # No GitHub environment: the PyPI trusted publisher has none configured. + permissions: + id-token: write + steps: + - name: Download artifact + uses: actions/download-artifact@v4 + with: + name: python-dist + path: dist/ + + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: name: Publish GitHub Release - needs: build + needs: [build-dist, build-binaries, publish-pypi] runs-on: ubuntu-latest # Only publish for real tag pushes; workflow_dispatch runs just build + verify. - if: startsWith(github.ref, 'refs/tags/') + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write steps: - name: Download all artifacts uses: actions/download-artifact@v4 diff --git a/Makefile b/Makefile index a6fd399..3df7fe3 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ _release-test: UV_PUBLISH_TOKEN="$$TOKEN" uv publish --publish-url https://test.pypi.org/legacy/ # MANUAL FALLBACK ONLY. The normal release path is CI: merge to main, then push -# a tag vX.Y.Z matching pyproject.toml; .github/workflows/publish-pypi.yml +# a tag vX.Y.Z matching pyproject.toml; .github/workflows/release.yml # publishes to PyPI via trusted publishing (no token). Use this target only if # that workflow is unavailable. It uploads with a token from macOS Keychain # (service: pypi-token-pypi) after a confirmation gate (PyPI does not allow @@ -70,7 +70,7 @@ _release-test: release: build _release _release: - @echo "NOTE: 'make release' is the manual fallback; normally push tag vX.Y.Z and let publish-pypi.yml publish." + @echo "NOTE: 'make release' is the manual fallback; normally push tag vX.Y.Z and let release.yml publish." @VERSION=$$(grep '^version' $(PYPROJECT) | head -1 | cut -d'"' -f2) && \ STATUS=$$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/$(PKG)/$$VERSION/json") && \ if [ "$$STATUS" = "200" ]; then \ diff --git a/README.md b/README.md index 8757b27..7260621 100644 --- a/README.md +++ b/README.md @@ -535,13 +535,14 @@ Releases are published by CI; nothing is uploaded from a laptop. git tag vX.Y.Z && git push origin vX.Y.Z ``` -The tag must match the `pyproject.toml` version. Pushing it runs two workflows: -[`publish-pypi.yml`](.github/workflows/publish-pypi.yml) checks the version, runs the -unit tests and pyright, builds the sdist and wheel, and publishes them to PyPI via -[trusted publishing](https://docs.pypi.org/trusted-publishers/) (no API token), and -[`release.yml`](.github/workflows/release.yml) builds the standalone `db` binaries -and attaches them to a GitHub Release. Running `publish-pypi.yml` manually -(workflow_dispatch) is a dry run: it builds and checks but never uploads. +The tag must match the `pyproject.toml` version. Pushing it runs one workflow, +[`release.yml`](.github/workflows/release.yml), which checks the version, runs the +unit tests and pyright, builds the sdist and wheel and the standalone `db` +binaries, then publishes to PyPI via +[trusted publishing](https://docs.pypi.org/trusted-publishers/) (no API token, no +GitHub environment; only after the binaries build) and attaches the binaries to a +GitHub Release. Running `release.yml` manually (workflow_dispatch) is a dry run: it +builds and checks but never publishes. `make release` (token from macOS Keychain) remains only as a manual fallback for when the workflow can't be used.