From 100547e9746292bcb2aa60dbfb51749944dd4ae3 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 12:16:44 -1000 Subject: [PATCH 1/4] Call the Diffbot APIs through the diffbot Python SDK Replace the aiohttp helper with a per-call diffbot.DiffbotAsync client (diffbot~=3.4.0). Extract, web search, entities, DQL, the ontology and the crawl job actions now go through the SDK, which brings DQL as a POST with a JSON body (no URL length limit on long queries), retries on 429/5xx and connection errors, and typed errors mapped to the same ToolError messages. Crawl start still builds its own create params, sent through the SDK transport: the SDK omits obeyRobots when it is false, and the crawler's default is to obey robots.txt. Co-Authored-By: Claude Opus 5.5 --- diffbot_mcp/api.py | 94 ++++++---------- diffbot_mcp/dql.py | 54 +++++---- diffbot_mcp/tools/crawl.py | 153 +++++++++++++------------- diffbot_mcp/tools/extract.py | 20 ++-- diffbot_mcp/tools/resolve_entities.py | 14 +-- diffbot_mcp/tools/search_web.py | 32 +++--- pyproject.toml | 2 +- tests/test_auth.py | 15 +-- tests/test_builder.py | 17 +-- 9 files changed, 184 insertions(+), 217 deletions(-) diff --git a/diffbot_mcp/api.py b/diffbot_mcp/api.py index 51e9309..84fee2d 100644 --- a/diffbot_mcp/api.py +++ b/diffbot_mcp/api.py @@ -1,12 +1,12 @@ -"""Token resolution and the shared HTTP helper behind every Diffbot API call.""" +"""Token resolution and the diffbot SDK client behind every Diffbot API call.""" -import asyncio -import json import os import pathlib -from typing import Any, Optional +from contextlib import asynccontextmanager +from typing import AsyncIterator, Optional -import aiohttp +import httpx +from diffbot import AuthError, DiffbotAsync, DiffbotError, RateLimitError from fastmcp.exceptions import ToolError from fastmcp.server.dependencies import get_access_token, get_http_request @@ -69,60 +69,32 @@ def resolve_token() -> str: raise ToolError(MISSING_TOKEN) -async def _error_message(response: aiohttp.ClientResponse) -> str: - body = await response.text() - message: Optional[str] = None +def tool_error(error: Exception) -> ToolError: + """Turn an SDK or transport failure into a ToolError carrying the API's own message.""" + if isinstance(error, ToolError): + return error + if isinstance(error, AuthError): + return ToolError("Invalid or unauthorized Diffbot token" + (f": {error.message}" if error.message else "")) + if isinstance(error, RateLimitError): + return ToolError("Diffbot rate limit exceeded" + (f", retry after {error.retry_after}s" if error.retry_after else "")) + if isinstance(error, (DiffbotError, ValueError)): # APIError, JobError, ValidationError; ValueError: a non-JSON body + return ToolError(str(error)) + if isinstance(error, httpx.TimeoutException): + return ToolError(f"Diffbot request timed out after {REQUEST_TIMEOUT_SECONDS}s") + if isinstance(error, httpx.HTTPError): + return ToolError(f"Diffbot request failed: {error}") + raise error + + +@asynccontextmanager +async def diffbot(token: Optional[str] = None) -> AsyncIterator[DiffbotAsync]: + """A diffbot SDK client for one tool call, authenticated with the caller's token. + + A client per call because the token is per request on the HTTP transport. SDK errors + raised inside the block leave it as ToolErrors. + """ try: - data = json.loads(body) - if isinstance(data, dict): - message = data.get("message") or data.get("error") - except ValueError: - pass - - if response.status in (401, 403): - return "Invalid or unauthorized Diffbot token" + (f": {message}" if message else "") - if response.status == 429: - retry_after = response.headers.get("Retry-After") - return "Diffbot rate limit exceeded" + (f", retry after {retry_after}s" if retry_after else "") - if not message: - message = body[:200] + ("..." if len(body) > 200 else "") - return f"Diffbot API error {response.status}: {message}" - - -def clean_params(params: dict) -> dict: - """Drop None values and render booleans as the 1/0 the Diffbot APIs expect.""" - cleaned = {} - for key, value in params.items(): - if value is None: - continue - if isinstance(value, bool): - value = 1 if value else 0 - cleaned[key] = value - return cleaned - - -async def request( - url: str, - *, - params: Optional[dict] = None, - json_body: Any = None, - headers: Optional[dict] = None, - as_text: bool = False, - method: str = "GET", -) -> Any: - """One Diffbot API call. Non-2xx responses become a ToolError carrying the API's own message.""" - timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT_SECONDS) - try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.request( - method, url, params=clean_params(params or {}), json=json_body, headers=headers - ) as response: - if response.status >= 400: - raise ToolError(await _error_message(response)) - if as_text: - return await response.text() - return await response.json(content_type=None) - except asyncio.TimeoutError: - raise ToolError(f"Diffbot request timed out after {REQUEST_TIMEOUT_SECONDS}s: {url}") - except aiohttp.ClientError as error: - raise ToolError(f"Diffbot request failed: {error}") + async with DiffbotAsync(token or resolve_token(), timeout=REQUEST_TIMEOUT_SECONDS) as client: + yield client + except (DiffbotError, ValueError, httpx.HTTPError) as error: + raise tool_error(error) from error diff --git a/diffbot_mcp/dql.py b/diffbot_mcp/dql.py index 6cd63bd..9bce5ce 100644 --- a/diffbot_mcp/dql.py +++ b/diffbot_mcp/dql.py @@ -3,19 +3,19 @@ import asyncio import csv import io +import json import re import time from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from fastmcp.exceptions import ToolError +from diffbot import DiffbotAsync -from .api import request, resolve_token +from .api import diffbot, tool_error -DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql" -ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology" ONTOLOGY_TTL_SECONDS = 86400 PROBE_CONCURRENCY = 8 +ONTOLOGY_PLACEHOLDER_TOKEN = "unused-for-the-public-ontology" # A full DQL entity record is ~75kb, so rows are pulled through the csv export with a # column spec. These are the defaults per entity type, mirroring the specs the @@ -82,28 +82,27 @@ def clean_row(row: Dict[str, str]) -> Dict[str, str]: return {column: clean_value(value) for column, value in row.items() if value} -async def dql_request(params: dict, as_text: bool = False) -> Any: - return await request(DQL_ENDPOINT, params={"token": resolve_token(), **params}, as_text=as_text) +# The SDK sends DQL as a POST with a JSON body, so a long query never hits a URL length limit. - -async def hits(query: str) -> int: - return (await dql_request({"query": query, "size": 0})).get("hits", 0) +async def hits(db: DiffbotAsync, query: str) -> int: + return (await db.dql(query, size=0)).get("hits", 0) async def probe(queries: List[str]) -> List[dict]: """Hit counts for several queries in parallel; a failing query reports its error instead of failing the batch.""" semaphore = asyncio.Semaphore(PROBE_CONCURRENCY) - async def one(query: str) -> dict: + async def one(db: DiffbotAsync, query: str) -> dict: if "similarTo(" in query: return {"query": query, "hits": None, "error": "similarTo cannot be probed: hits mirrors the requested size, so a size 0 probe always reports 0. Validate with a small export instead."} async with semaphore: try: - return {"query": query, "hits": await hits(query)} - except ToolError as error: - return {"query": query, "hits": None, "error": str(error)} + return {"query": query, "hits": await hits(db, query)} + except Exception as error: + return {"query": query, "hits": None, "error": str(tool_error(error))} - return list(await asyncio.gather(*(one(query) for query in queries))) + async with diffbot() as db: + return list(await asyncio.gather(*(one(db, query) for query in queries))) def is_facet(query: str) -> bool: @@ -121,15 +120,22 @@ async def run( bare_type_check: bool = False, ) -> dict: """Execute a DQL query and shape the response: a count, facet buckets, csv rows, or json entities.""" + async with diffbot() as db: + return await _run(db, query, size=size, offset=offset, fields=fields, format=format, + hits_only=hits_only, bare_type_check=bare_type_check) + + +async def _run(db: DiffbotAsync, query: str, *, size: int, offset: int, fields: Optional[str], format: str, + hits_only: bool, bare_type_check: bool) -> dict: if hits_only or size < 1: - return {"dql": query, "hits": await hits(query)} + return {"dql": query, "hits": await hits(db, query)} notes: List[str] = [] # facet queries answer with aggregate buckets rather than entities, and the csv # export is empty for them, so they are read from the json response instead if is_facet(query): - result = await dql_request({"query": query, "size": max(size, 1)}) + result = await db.dql(query, size=max(size, 1)) return { "dql": query, "hits": result.get("hits", 0), @@ -149,21 +155,18 @@ async def run( tasks: Dict[str, Any] = {} if not similar: - tasks["count"] = dql_request({"query": query, "size": 0}) + tasks["count"] = db.dql(query, size=0) entity_type = query_type(query) # an unknown field path is silently ignored by the API and returns the unfiltered # type count, so the bare count is fetched alongside to make that visible. Article # is skipped: an unfiltered probe over the article index times out. if bare_type_check and not similar and entity_type and entity_type != "Article" and query.strip() != f"type:{entity_type}": - tasks["bare"] = dql_request({"query": f"type:{entity_type}", "size": 0}) + tasks["bare"] = db.dql(f"type:{entity_type}", size=0) - page = {"query": query, "size": size} - if offset: - page["from"] = offset if format == "json": - tasks["data"] = dql_request(page) + tasks["data"] = db.dql(query, size=size, from_=offset) else: - tasks["data"] = dql_request({**page, "format": "csv", "exportspec": fields or default_fields(query)}, as_text=True) + tasks["data"] = db.dql(query, size=size, from_=offset, format="csv", exportspec=fields or default_fields(query)) values = await asyncio.gather(*tasks.values()) outcome = dict(zip(tasks.keys(), values)) @@ -206,7 +209,10 @@ async def get_ontology() -> dict: async with _ontology_lock: if _ontology_fresh(): return _ontology_cache["data"] - data = await request(ONTOLOGY_ENDPOINT) + # the ontology endpoint is public and the SDK sends no token for it, so a missing + # token must not stop this tool from working + async with diffbot(token=ONTOLOGY_PLACEHOLDER_TOKEN) as db: + data = json.loads(await db.dql_fetch_ontology_text()) _ontology_cache["data"] = data _ontology_cache["fetched"] = time.time() return data diff --git a/diffbot_mcp/tools/crawl.py b/diffbot_mcp/tools/crawl.py index df67222..0471746 100644 --- a/diffbot_mcp/tools/crawl.py +++ b/diffbot_mcp/tools/crawl.py @@ -6,12 +6,12 @@ from fastmcp.exceptions import ToolError from mcp.types import ToolAnnotations -from ..api import request, resolve_token +from diffbot import APIError + +from ..api import diffbot from ..app import mcp from .extract import PageType, normalize_url -CRAWL_ENDPOINT = "https://api.diffbot.com/v3/crawl" -EXTRACT_BASE = "https://api.diffbot.com/v3" # jobStatus.status codes 0 and 7 mean the job is still in progress RUNNING_STATUS_CODES = (0, 7) @@ -59,82 +59,85 @@ async def crawl( max_results: Annotated[int, "Max rows (urls, results)"] = 10, offset: Annotated[int, "Rows to skip (urls, results)"] = 0, ) -> dict: - token = resolve_token() - if action not in ("start", "list") and not job_name: raise ToolError(f"job_name is required for action '{action}'") - if action == "start": - if not url: - raise ToolError("url is required to start a crawl") - url = normalize_url(url) - job_name = job_name or f"crawl-{int(time.time())}" - - api_url = f"{EXTRACT_BASE}/{page_type or 'analyze'}" - if format == "markdown": - api_url = f"{api_url}?mode=llm" - - params = { - "token": token, - "name": job_name, - "seeds": url, - "apiUrl": api_url, - "maxHops": hops, - "maxToCrawl": max_to_crawl, - "maxToProcess": max_to_process, - "restrictDomain": restrict_domain, - "obeyRobots": obey_robots, - "useProxies": use_proxies, - "urlCrawlPattern": url_crawl_pattern, - "urlProcessPattern": url_process_pattern, - "customHeaders": custom_headers, - "crawlDelay": crawl_delay if crawl_delay and crawl_delay > 0 else None, - } - response = await request(CRAWL_ENDPOINT, params=params) - return {"job_name": job_name, "response": response} - - if action == "status": - response = await request(CRAWL_ENDPOINT, params={"token": token, "name": job_name}) - jobs = response.get("jobs") or [] - if not jobs: - raise ToolError(f"No crawl job named '{job_name}'") - return job_summary(jobs[0]) - - if action == "list": - response = await request(CRAWL_ENDPOINT, params={"token": token}) - return {"jobs": [job_summary(job) for job in response.get("jobs") or []]} - - if action == "delete": - response = await request(CRAWL_ENDPOINT, params={"token": token, "name": job_name, "delete": 1}) - return {"job_name": job_name, "response": response} - - if action == "urls": - text = await request(f"{CRAWL_ENDPOINT}/data", params={"token": token, "name": job_name, "type": "urls"}, as_text=True) - rows = [] - for row in csv.DictReader(io.StringIO(text or "")): - page_url = (row.get("Url") or "").strip().strip('"') - if page_url: - rows.append({ - "url": page_url, - "status": (row.get("Crawl Status") or "unknown").strip(), - "crawled": (row.get("Crawled Time") or "").strip() or None, - }) + async with diffbot() as db: + if action == "start": + if not url: + raise ToolError("url is required to start a crawl") + url = normalize_url(url) + job_name = job_name or f"crawl-{int(time.time())}" + + api_url = f"{db.analyze_url}/{page_type or 'analyze'}" + if format == "markdown": + api_url = f"{api_url}?mode=llm" + + # Built here rather than with db.crawl(): the SDK leaves obeyRobots out when it is + # false, and the crawler's default is to obey, so obey_robots=False would be lost. + # Booleans go out as 1/0 and unset options are dropped. + params = { + "token": db.token, + "name": job_name, + "seeds": url, + "apiUrl": api_url, + "maxHops": hops, + "maxToCrawl": max_to_crawl, + "maxToProcess": max_to_process, + "restrictDomain": restrict_domain, + "obeyRobots": obey_robots, + "useProxies": use_proxies, + "urlCrawlPattern": url_crawl_pattern, + "urlProcessPattern": url_process_pattern, + "customHeaders": custom_headers, + "crawlDelay": crawl_delay if crawl_delay and crawl_delay > 0 else None, + } + params = {key: int(value) if isinstance(value, bool) else value for key, value in params.items() if value is not None} + # creating a job is not idempotent, so no retries + created = await db.request("GET", db.crawler_url, params=params, retryable=False) + response = created.json() + # some create failures (e.g. a job name over 30 characters) come back as a 200 with errorCode + if isinstance(response, dict) and response.get("errorCode"): + raise APIError(created.status_code, created.text) + return {"job_name": job_name, "response": response} + + if action == "status": + job = await db.crawl_get_job(job_name) + if not job: + raise ToolError(f"No crawl job named '{job_name}'") + return job_summary(job) + + if action == "list": + return {"jobs": [job_summary(job) for job in await db.crawl_list_jobs()]} + + if action == "delete": + await db.crawl_delete_job(job_name) + return {"job_name": job_name, "deleted": True} + + if action == "urls": + text = await db.crawl_download(job_name, type="urls") + rows = [] + for row in csv.DictReader(io.StringIO(text or "")): + page_url = (row.get("Url") or "").strip().strip('"') + if page_url: + rows.append({ + "url": page_url, + "status": (row.get("Crawl Status") or "unknown").strip(), + "crawled": (row.get("Crawled Time") or "").strip() or None, + }) + return { + "job_name": job_name, + "total_urls": len(rows), + "offset": offset, + "urls": rows[offset:offset + max_results], + } + + # an unknown job raises JobError, which names the job + response = await db.crawl_download(job_name) + return { "job_name": job_name, - "total_urls": len(rows), + "total_results": len(response), "offset": offset, - "urls": rows[offset:offset + max_results], + "results": response[offset:offset + max_results], } - - response = await request(f"{CRAWL_ENDPOINT}/data", params={"token": token, "name": job_name, "format": "json"}) - if isinstance(response, dict): - if response.get("error"): - raise ToolError(f"{response['error']} (job '{job_name}')") - response = response.get("objects") or [] - - return { - "job_name": job_name, - "total_results": len(response), - "offset": offset, - "results": response[offset:offset + max_results], - } diff --git a/diffbot_mcp/tools/extract.py b/diffbot_mcp/tools/extract.py index e99efb2..4a0f0e1 100644 --- a/diffbot_mcp/tools/extract.py +++ b/diffbot_mcp/tools/extract.py @@ -1,11 +1,11 @@ from typing import Annotated, Literal, Optional +from diffbot import ExtractionError from fastmcp.exceptions import ToolError -from ..api import request, resolve_token +from ..api import diffbot from ..app import READ_ONLY, mcp -EXTRACT_BASE = "https://api.diffbot.com/v3" PageType = Literal["article", "product", "discussion", "image", "video", "list", "event", "job", "faq"] DESCRIPTION = ( @@ -27,12 +27,10 @@ async def extract( page_type: Annotated[Optional[PageType], "Override auto-detection; rarely needed"] = None, format: Annotated[Literal["markdown", "json"], "`markdown` for readable text; `json` for typed fields such as title, author, date, price, images"] = "markdown", ) -> dict: - params = {"token": resolve_token(), "url": normalize_url(url), "timeout": 30000} - if format == "markdown": - params["mode"] = "llm" - - data = await request(f"{EXTRACT_BASE}/{page_type or 'analyze'}", params=params) - # the API reports a failed extraction inside a 200 body - if isinstance(data, dict) and "errorCode" in data: - raise ToolError(f"Extraction error {data['errorCode']}: {data.get('error', '')}. The page may be behind a login, a bot wall, or a JavaScript app; try a specific page_type such as 'article'.") - return data + # fmt="markdown" asks for mode=llm; the SDK sends the 30s server-side fetch timeout + async with diffbot() as db: + try: + return await db.extract(normalize_url(url), api=page_type or "analyze", fmt=format) + except ExtractionError as error: + # the API reports a failed extraction inside a 200 body + raise ToolError(f"Extraction error {error.error_code}: {error.error}. The page may be behind a login, a bot wall, or a JavaScript app; try a specific page_type such as 'article'.") from error diff --git a/diffbot_mcp/tools/resolve_entities.py b/diffbot_mcp/tools/resolve_entities.py index 4b487d2..010df30 100644 --- a/diffbot_mcp/tools/resolve_entities.py +++ b/diffbot_mcp/tools/resolve_entities.py @@ -1,10 +1,8 @@ from typing import Annotated, Optional -from ..api import request, resolve_token +from ..api import diffbot from ..app import READ_ONLY, mcp -NLP_ENDPOINT = "https://nl.diffbot.com/v1/" - DESCRIPTION = ( "Named entity recognition and linking. Finds people, organizations, places, and products mentioned in text. " "Each resolves to a Diffbot Knowledge Graph entity with confidence, salience, and sentiment. Prefer for " @@ -24,13 +22,9 @@ async def resolve_entities( text: Annotated[str, "Plain text to analyze"], lang: Annotated[Optional[str], "ISO code, e.g. en; default auto"] = None, ) -> dict: - params = {"token": resolve_token(), "fields": "entities,sentiment"} - payload = [{"lang": lang or "auto", "format": "plain text", "content": text}] - - data = await request(NLP_ENDPOINT, method="POST", params=params, json_body=payload) - # the NLP API answers with one result object per submitted document - if isinstance(data, list): - data = data[0] if data else {} + # fields=entities,sentiment; the SDK unwraps the one-document batch + async with diffbot() as db: + data = await db.entities(text, lang=lang or "auto") entities = [] for entity in data.get("entities", []): diff --git a/diffbot_mcp/tools/search_web.py b/diffbot_mcp/tools/search_web.py index 9da06f8..091f928 100644 --- a/diffbot_mcp/tools/search_web.py +++ b/diffbot_mcp/tools/search_web.py @@ -1,12 +1,8 @@ from typing import Annotated, Literal, Optional -from ..api import request, resolve_token +from ..api import diffbot from ..app import READ_ONLY, mcp -# /api/v1/web_search honors size and maxTokens and reports timeMs; the older -# /api/v1/diffbot_web_search path ignores size and always returns ten results. -WEB_SEARCH_ENDPOINT = "https://llm.diffbot.com/api/v1/web_search" - DESCRIPTION = ( "Web search that returns ranked results with relevance scores, publication dates, and matching passages from " "each page. One call returns it all, no follow-up fetch required. Prefer over a default or generic web search " @@ -24,15 +20,12 @@ def is_page_lookup(query: str) -> bool: return len(parts) == 1 and parts[0].lower().startswith("url:") -def build_search_params(query: str, num_results: Optional[int], max_tokens: Optional[int], snippet_length: Optional[SnippetLength]) -> dict: - query = query.strip() - if snippet_length is None: - snippet_length = "full" if is_page_lookup(query) else "short" - params = {"text": query, "size": num_results, "maxTokens": max_tokens, "snippetLength": snippet_length.upper()} - if snippet_length == "full": - # the reranker always chunks; only the unranked path returns the whole stored page - params["rerank"] = "none" - return params +def default_snippet_length(query: str, snippet_length: Optional[SnippetLength]) -> SnippetLength: + """Short passages by default, the whole page for a bare `url:` lookup. The SDK turns + "full" into rerank=none, since only the unranked path returns the whole stored page.""" + if snippet_length is not None: + return snippet_length + return "full" if is_page_lookup(query) else "short" @mcp.tool(name="search_web", title="Search the Web", description=DESCRIPTION, annotations=READ_ONLY) @@ -42,6 +35,11 @@ async def search_web( max_tokens: Annotated[Optional[int], "Token cap; below ~1000 nothing fits; ignored for a full page"] = None, snippet_length: Annotated[Optional[SnippetLength], "Passage size per result; default short (~1-1.5 KB), full for a bare `url:` (the whole page, terms ignored)"] = None, ) -> dict: - params = build_search_params(query, num_results, max_tokens, snippet_length) - headers = {"Authorization": f"Bearer {resolve_token()}"} - return await request(WEB_SEARCH_ENDPOINT, params=params, headers=headers) + query = query.strip() + async with diffbot() as db: + return await db.web_search( + query, + num_results=num_results, + max_tokens=max_tokens, + snippet_length=default_snippet_length(query, snippet_length), + ) diff --git a/pyproject.toml b/pyproject.toml index 061e566..1b3e5f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ requires-python = ">=3.10" # commit that never touched it. dependencies = [ "fastmcp~=4.1.0", - "aiohttp~=3.14.5" + "diffbot~=3.4.0" ] [project.optional-dependencies] diff --git a/tests/test_auth.py b/tests/test_auth.py index bff6984..bf7157e 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -1,14 +1,15 @@ """OAuth via the Diffbot dashboard, with static Diffbot API tokens still accepted. No network.""" +import functools import time +import httpx import httpx2 import pytest from starlette.testclient import TestClient -from diffbot_mcp import auth +from diffbot_mcp import api, auth from diffbot_mcp.server import mcp -from diffbot_mcp.tools import extract OAUTH_ENV = { "DIFFBOT_OAUTH_ISSUER": "https://app.diffbot.com", @@ -59,16 +60,16 @@ def dashboard(monkeypatch): @pytest.fixture def sent_tokens(monkeypatch): - """Records the Diffbot API token each extract call sends upstream.""" + """Records the Diffbot API token each extract call sends upstream, through the real SDK client.""" tokens = [] - async def fake_request(url, *, params=None, **kwargs): - tokens.append(params["token"]) - return {"objects": []} + def diffbot_api(request): + tokens.append(request.url.params["token"]) + return httpx.Response(200, json={"objects": []}) monkeypatch.delenv("DIFFBOT_TOKEN", raising=False) monkeypatch.delenv("DIFFBOT_API_TOKEN", raising=False) - monkeypatch.setattr(extract, "request", fake_request) + monkeypatch.setattr(api, "DiffbotAsync", functools.partial(api.DiffbotAsync, transport=httpx.MockTransport(diffbot_api))) return tokens diff --git a/tests/test_builder.py b/tests/test_builder.py index dcc3a46..11a3de4 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -12,7 +12,7 @@ from diffbot_mcp.tools.organizations import build_organizations_query from diffbot_mcp.tools.people import build_leadership_query, build_people_query, pick_employment from diffbot_mcp.tools.places import build_places_query -from diffbot_mcp.tools.search_web import build_search_params, is_page_lookup +from diffbot_mcp.tools.search_web import default_snippet_length, is_page_lookup # --- builder primitives ----------------------------------------------------------- @@ -285,13 +285,8 @@ def test_search_web_page_lookup(): def test_search_web_snippet_defaults(): - p = build_search_params("ai chips", 5, None, None) - assert p["snippetLength"] == "SHORT" and "rerank" not in p - p = build_search_params("url:https://example.com/docs", None, None, None) - assert p["snippetLength"] == "FULL" and p["rerank"] == "none" - p = build_search_params("url:https://example.com/docs rate limits", None, None, None) - assert p["snippetLength"] == "SHORT" and "rerank" not in p - p = build_search_params("ai chips", None, None, "long") - assert p["snippetLength"] == "LONG" and "rerank" not in p - p = build_search_params("url:https://example.com/docs rate limits", None, None, "full") - assert p["snippetLength"] == "FULL" and p["rerank"] == "none" + assert default_snippet_length("ai chips", None) == "short" + assert default_snippet_length("url:https://example.com/docs", None) == "full" + assert default_snippet_length("url:https://example.com/docs rate limits", None) == "short" + assert default_snippet_length("ai chips", "long") == "long" + assert default_snippet_length("url:https://example.com/docs rate limits", "full") == "full" From f12b85bc4788c92406dd37f832c2c20a0c0c5ed2 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 12:16:44 -1000 Subject: [PATCH 2/4] Run the unit tests in CI on pull requests Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..420e942 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,28 @@ +name: Tests + +# Unit tests only (tests/test_*.py): no network, no Diffbot token. The live smoke test +# (tests/smoke.py) and the routing eval stay manual. +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: pip install ".[dev]" + - name: Run unit tests + run: pytest tests/test_auth.py tests/test_builder.py -q From 1fb5305f68f51494e33bcd04df68cff028527a93 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 12:55:40 -1000 Subject: [PATCH 3/4] Use the SDK's crawl() for crawl start; pin diffbot==3.4.1rc1; bump to 0.3.0 diffbot 3.4.1 sends obeyRobots=0 for an explicit obey_robots=False, so the hand-built create request is no longer needed. Start now calls db.crawl() on a client with retries off (creating a job is not idempotent) and returns {job_name, created}. Versions stamped with scripts/sync_version.py. Co-Authored-By: Claude Opus 5.5 --- diffbot_mcp/__init__.py | 2 +- diffbot_mcp/api.py | 7 ++--- diffbot_mcp/tools/crawl.py | 53 +++++++++++++++++--------------------- gemini-extension.json | 2 +- manifest.json | 2 +- plugin.json | 2 +- pyproject.toml | 4 +-- server.json | 4 +-- 8 files changed, 35 insertions(+), 41 deletions(-) diff --git a/diffbot_mcp/__init__.py b/diffbot_mcp/__init__.py index c8e9efa..17d675a 100644 --- a/diffbot_mcp/__init__.py +++ b/diffbot_mcp/__init__.py @@ -1,3 +1,3 @@ """Diffbot MCP server: web extraction, search, crawling, entity resolution, and Knowledge Graph tools.""" -__version__ = "0.2.2" +__version__ = "0.3.0" diff --git a/diffbot_mcp/api.py b/diffbot_mcp/api.py index 84fee2d..a673bcf 100644 --- a/diffbot_mcp/api.py +++ b/diffbot_mcp/api.py @@ -87,14 +87,15 @@ def tool_error(error: Exception) -> ToolError: @asynccontextmanager -async def diffbot(token: Optional[str] = None) -> AsyncIterator[DiffbotAsync]: +async def diffbot(token: Optional[str] = None, *, retry: Optional[bool] = None) -> AsyncIterator[DiffbotAsync]: """A diffbot SDK client for one tool call, authenticated with the caller's token. A client per call because the token is per request on the HTTP transport. SDK errors - raised inside the block leave it as ToolErrors. + raised inside the block leave it as ToolErrors. ``retry=False`` turns off the SDK's + retries (for calls that are not idempotent); ``None`` keeps its default policy. """ try: - async with DiffbotAsync(token or resolve_token(), timeout=REQUEST_TIMEOUT_SECONDS) as client: + async with DiffbotAsync(token or resolve_token(), timeout=REQUEST_TIMEOUT_SECONDS, retry=retry) as client: yield client except (DiffbotError, ValueError, httpx.HTTPError) as error: raise tool_error(error) from error diff --git a/diffbot_mcp/tools/crawl.py b/diffbot_mcp/tools/crawl.py index 0471746..a82e442 100644 --- a/diffbot_mcp/tools/crawl.py +++ b/diffbot_mcp/tools/crawl.py @@ -6,8 +6,6 @@ from fastmcp.exceptions import ToolError from mcp.types import ToolAnnotations -from diffbot import APIError - from ..api import diffbot from ..app import mcp from .extract import PageType, normalize_url @@ -62,7 +60,8 @@ async def crawl( if action not in ("start", "list") and not job_name: raise ToolError(f"job_name is required for action '{action}'") - async with diffbot() as db: + # creating a job is not idempotent, so the start call gets no retries + async with diffbot(retry=False if action == "start" else None) as db: if action == "start": if not url: raise ToolError("url is required to start a crawl") @@ -73,33 +72,27 @@ async def crawl( if format == "markdown": api_url = f"{api_url}?mode=llm" - # Built here rather than with db.crawl(): the SDK leaves obeyRobots out when it is - # false, and the crawler's default is to obey, so obey_robots=False would be lost. - # Booleans go out as 1/0 and unset options are dropped. - params = { - "token": db.token, - "name": job_name, - "seeds": url, - "apiUrl": api_url, - "maxHops": hops, - "maxToCrawl": max_to_crawl, - "maxToProcess": max_to_process, - "restrictDomain": restrict_domain, - "obeyRobots": obey_robots, - "useProxies": use_proxies, - "urlCrawlPattern": url_crawl_pattern, - "urlProcessPattern": url_process_pattern, - "customHeaders": custom_headers, - "crawlDelay": crawl_delay if crawl_delay and crawl_delay > 0 else None, - } - params = {key: int(value) if isinstance(value, bool) else value for key, value in params.items() if value is not None} - # creating a job is not idempotent, so no retries - created = await db.request("GET", db.crawler_url, params=params, retryable=False) - response = created.json() - # some create failures (e.g. a job name over 30 characters) come back as a 200 with errorCode - if isinstance(response, dict) and response.get("errorCode"): - raise APIError(created.status_code, created.text) - return {"job_name": job_name, "response": response} + # The SDK sends booleans as 1/0 (an explicit obey_robots=False goes out as + # obeyRobots=0 since diffbot 3.4.1), and raises APIError when a create comes back + # as a 200 with errorCode (e.g. a job name over 30 characters). + async for event in db.crawl( + url, + job_name=job_name, + api_url=api_url, + hops=hops, + max_to_crawl=max_to_crawl, + max_to_process=max_to_process, + restrict_domain=restrict_domain, + obey_robots=obey_robots, + use_proxies=use_proxies, + url_crawl_pattern=url_crawl_pattern, + url_process_pattern=url_process_pattern, + custom_headers=custom_headers, + crawl_delay=crawl_delay if crawl_delay and crawl_delay > 0 else -1, + watch=False, + ): + job_name = event.details["job_name"] + return {"job_name": job_name, "created": True} if action == "status": job = await db.crawl_get_job(job_name) diff --git a/gemini-extension.json b/gemini-extension.json index dad3cb9..89ad4e3 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "diffbot-mcp", - "version": "0.2.2", + "version": "0.3.0", "description": "Web knowledge fetching tools for your agent.", "settings": [ { diff --git a/manifest.json b/manifest.json index 0f76b66..102604f 100644 --- a/manifest.json +++ b/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "0.4", "name": "diffbot-mcp", "display_name": "Diffbot", - "version": "0.2.2", + "version": "0.3.0", "description": "Web knowledge fetching tools for your agent.", "long_description": "Diffbot is an applied AI lab building models that structure all of human knowledge. This extension runs the Diffbot MCP server on your machine: web extraction and search over Diffbot's independent web index, site crawls, entity resolution, and queries against the 10-billion-entity Diffbot Knowledge Graph.\n\nA free Diffbot token is required for tool use: https://app.diffbot.com/get-started. Paste it below, or leave it blank if you already keep a DIFFBOT_API_TOKEN line in ~/.diffbot/credentials, the file the db CLI and diffbot-skills read.", "author": { diff --git a/plugin.json b/plugin.json index d8948d2..59d4a3c 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "diffbot", - "version": "0.2.2", + "version": "0.3.0", "description": "Web knowledge fetching tools for your agent: search, extract, crawl, and query the Diffbot Knowledge Graph.", "author": { "name": "Diffbot", diff --git a/pyproject.toml b/pyproject.toml index 1b3e5f4..8bbf12d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "diffbot-mcp" -version = "0.2.2" +version = "0.3.0" description = "Web knowledge fetching tools for your agent." readme = "README.md" license = "MIT" @@ -15,7 +15,7 @@ requires-python = ">=3.10" # commit that never touched it. dependencies = [ "fastmcp~=4.1.0", - "diffbot~=3.4.0" + "diffbot==3.4.1rc1" ] [project.optional-dependencies] diff --git a/server.json b/server.json index 7e899f5..6ac9aa8 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.diffbot/diffbot-mcp", "title": "Diffbot", "description": "Web knowledge fetching tools for your agent.", - "version": "0.2.2", + "version": "0.3.0", "websiteUrl": "https://www.diffbot.com", "repository": { "url": "https://github.com/diffbot/diffbot-mcp", @@ -35,7 +35,7 @@ { "registryType": "pypi", "identifier": "diffbot-mcp", - "version": "0.2.2", + "version": "0.3.0", "runtimeHint": "uvx", "transport": { "type": "stdio" From 9a4a1f3131af9170e5bc975c3a752cc9bf4fd974 Mon Sep 17 00:00:00 2001 From: Andrew Wessels Date: Wed, 18 Nov 2026 12:58:35 -1000 Subject: [PATCH 4/4] Require diffbot>=3.4.1 (final) instead of the 3.4.1rc1 pin Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8bbf12d..2eb1d8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ requires-python = ">=3.10" # commit that never touched it. dependencies = [ "fastmcp~=4.1.0", - "diffbot==3.4.1rc1" + "diffbot>=3.4.1" ] [project.optional-dependencies]