Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
name: Tests

# Unit tests only (tests/test_*.py): no network, no Diffbot token. The live smoke test
# (tests/smoke.py) and the routing eval stay manual.
on:
pull_request:
push:
branches:
- main

permissions:
contents: read

jobs:
pytest:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.10", "3.12"]
steps:
- uses: actions/checkout@v5
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install
run: pip install ".[dev]"
- name: Run unit tests
run: pytest tests/test_auth.py tests/test_builder.py -q
2 changes: 1 addition & 1 deletion diffbot_mcp/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""Diffbot MCP server: web extraction, search, crawling, entity resolution, and Knowledge Graph tools."""

__version__ = "0.2.2"
__version__ = "0.3.0"
95 changes: 34 additions & 61 deletions diffbot_mcp/api.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
"""Token resolution and the shared HTTP helper behind every Diffbot API call."""
"""Token resolution and the diffbot SDK client behind every Diffbot API call."""

import asyncio
import json
import os
import pathlib
from typing import Any, Optional
from contextlib import asynccontextmanager
from typing import AsyncIterator, Optional

import aiohttp
import httpx
from diffbot import AuthError, DiffbotAsync, DiffbotError, RateLimitError
from fastmcp.exceptions import ToolError
from fastmcp.server.dependencies import get_access_token, get_http_request

Expand Down Expand Up @@ -69,60 +69,33 @@ def resolve_token() -> str:
raise ToolError(MISSING_TOKEN)


async def _error_message(response: aiohttp.ClientResponse) -> str:
body = await response.text()
message: Optional[str] = None
def tool_error(error: Exception) -> ToolError:
"""Turn an SDK or transport failure into a ToolError carrying the API's own message."""
if isinstance(error, ToolError):
return error
if isinstance(error, AuthError):
return ToolError("Invalid or unauthorized Diffbot token" + (f": {error.message}" if error.message else ""))
if isinstance(error, RateLimitError):
return ToolError("Diffbot rate limit exceeded" + (f", retry after {error.retry_after}s" if error.retry_after else ""))
if isinstance(error, (DiffbotError, ValueError)): # APIError, JobError, ValidationError; ValueError: a non-JSON body
return ToolError(str(error))
if isinstance(error, httpx.TimeoutException):
return ToolError(f"Diffbot request timed out after {REQUEST_TIMEOUT_SECONDS}s")
if isinstance(error, httpx.HTTPError):
return ToolError(f"Diffbot request failed: {error}")
raise error


@asynccontextmanager
async def diffbot(token: Optional[str] = None, *, retry: Optional[bool] = None) -> AsyncIterator[DiffbotAsync]:
"""A diffbot SDK client for one tool call, authenticated with the caller's token.

A client per call because the token is per request on the HTTP transport. SDK errors
raised inside the block leave it as ToolErrors. ``retry=False`` turns off the SDK's
retries (for calls that are not idempotent); ``None`` keeps its default policy.
"""
try:
data = json.loads(body)
if isinstance(data, dict):
message = data.get("message") or data.get("error")
except ValueError:
pass

if response.status in (401, 403):
return "Invalid or unauthorized Diffbot token" + (f": {message}" if message else "")
if response.status == 429:
retry_after = response.headers.get("Retry-After")
return "Diffbot rate limit exceeded" + (f", retry after {retry_after}s" if retry_after else "")
if not message:
message = body[:200] + ("..." if len(body) > 200 else "")
return f"Diffbot API error {response.status}: {message}"


def clean_params(params: dict) -> dict:
"""Drop None values and render booleans as the 1/0 the Diffbot APIs expect."""
cleaned = {}
for key, value in params.items():
if value is None:
continue
if isinstance(value, bool):
value = 1 if value else 0
cleaned[key] = value
return cleaned


async def request(
url: str,
*,
params: Optional[dict] = None,
json_body: Any = None,
headers: Optional[dict] = None,
as_text: bool = False,
method: str = "GET",
) -> Any:
"""One Diffbot API call. Non-2xx responses become a ToolError carrying the API's own message."""
timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT_SECONDS)
try:
async with aiohttp.ClientSession(timeout=timeout) as session:
async with session.request(
method, url, params=clean_params(params or {}), json=json_body, headers=headers
) as response:
if response.status >= 400:
raise ToolError(await _error_message(response))
if as_text:
return await response.text()
return await response.json(content_type=None)
except asyncio.TimeoutError:
raise ToolError(f"Diffbot request timed out after {REQUEST_TIMEOUT_SECONDS}s: {url}")
except aiohttp.ClientError as error:
raise ToolError(f"Diffbot request failed: {error}")
async with DiffbotAsync(token or resolve_token(), timeout=REQUEST_TIMEOUT_SECONDS, retry=retry) as client:
yield client
except (DiffbotError, ValueError, httpx.HTTPError) as error:
raise tool_error(error) from error
54 changes: 30 additions & 24 deletions diffbot_mcp/dql.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@
import asyncio
import csv
import io
import json
import re
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

from fastmcp.exceptions import ToolError
from diffbot import DiffbotAsync

from .api import request, resolve_token
from .api import diffbot, tool_error

DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql"
ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology"
ONTOLOGY_TTL_SECONDS = 86400
PROBE_CONCURRENCY = 8
ONTOLOGY_PLACEHOLDER_TOKEN = "unused-for-the-public-ontology"

# A full DQL entity record is ~75kb, so rows are pulled through the csv export with a
# column spec. These are the defaults per entity type, mirroring the specs the
Expand Down Expand Up @@ -82,28 +82,27 @@ def clean_row(row: Dict[str, str]) -> Dict[str, str]:
return {column: clean_value(value) for column, value in row.items() if value}


async def dql_request(params: dict, as_text: bool = False) -> Any:
return await request(DQL_ENDPOINT, params={"token": resolve_token(), **params}, as_text=as_text)
# The SDK sends DQL as a POST with a JSON body, so a long query never hits a URL length limit.


async def hits(query: str) -> int:
return (await dql_request({"query": query, "size": 0})).get("hits", 0)
async def hits(db: DiffbotAsync, query: str) -> int:
return (await db.dql(query, size=0)).get("hits", 0)


async def probe(queries: List[str]) -> List[dict]:
"""Hit counts for several queries in parallel; a failing query reports its error instead of failing the batch."""
semaphore = asyncio.Semaphore(PROBE_CONCURRENCY)

async def one(query: str) -> dict:
async def one(db: DiffbotAsync, query: str) -> dict:
if "similarTo(" in query:
return {"query": query, "hits": None, "error": "similarTo cannot be probed: hits mirrors the requested size, so a size 0 probe always reports 0. Validate with a small export instead."}
async with semaphore:
try:
return {"query": query, "hits": await hits(query)}
except ToolError as error:
return {"query": query, "hits": None, "error": str(error)}
return {"query": query, "hits": await hits(db, query)}
except Exception as error:
return {"query": query, "hits": None, "error": str(tool_error(error))}

return list(await asyncio.gather(*(one(query) for query in queries)))
async with diffbot() as db:
return list(await asyncio.gather(*(one(db, query) for query in queries)))


def is_facet(query: str) -> bool:
Expand All @@ -121,15 +120,22 @@ async def run(
bare_type_check: bool = False,
) -> dict:
"""Execute a DQL query and shape the response: a count, facet buckets, csv rows, or json entities."""
async with diffbot() as db:
return await _run(db, query, size=size, offset=offset, fields=fields, format=format,
hits_only=hits_only, bare_type_check=bare_type_check)


async def _run(db: DiffbotAsync, query: str, *, size: int, offset: int, fields: Optional[str], format: str,
hits_only: bool, bare_type_check: bool) -> dict:
if hits_only or size < 1:
return {"dql": query, "hits": await hits(query)}
return {"dql": query, "hits": await hits(db, query)}

notes: List[str] = []

# facet queries answer with aggregate buckets rather than entities, and the csv
# export is empty for them, so they are read from the json response instead
if is_facet(query):
result = await dql_request({"query": query, "size": max(size, 1)})
result = await db.dql(query, size=max(size, 1))
return {
"dql": query,
"hits": result.get("hits", 0),
Expand All @@ -149,21 +155,18 @@ async def run(

tasks: Dict[str, Any] = {}
if not similar:
tasks["count"] = dql_request({"query": query, "size": 0})
tasks["count"] = db.dql(query, size=0)
entity_type = query_type(query)
# an unknown field path is silently ignored by the API and returns the unfiltered
# type count, so the bare count is fetched alongside to make that visible. Article
# is skipped: an unfiltered probe over the article index times out.
if bare_type_check and not similar and entity_type and entity_type != "Article" and query.strip() != f"type:{entity_type}":
tasks["bare"] = dql_request({"query": f"type:{entity_type}", "size": 0})
tasks["bare"] = db.dql(f"type:{entity_type}", size=0)

page = {"query": query, "size": size}
if offset:
page["from"] = offset
if format == "json":
tasks["data"] = dql_request(page)
tasks["data"] = db.dql(query, size=size, from_=offset)
else:
tasks["data"] = dql_request({**page, "format": "csv", "exportspec": fields or default_fields(query)}, as_text=True)
tasks["data"] = db.dql(query, size=size, from_=offset, format="csv", exportspec=fields or default_fields(query))

values = await asyncio.gather(*tasks.values())
outcome = dict(zip(tasks.keys(), values))
Expand Down Expand Up @@ -206,7 +209,10 @@ async def get_ontology() -> dict:
async with _ontology_lock:
if _ontology_fresh():
return _ontology_cache["data"]
data = await request(ONTOLOGY_ENDPOINT)
# the ontology endpoint is public and the SDK sends no token for it, so a missing
# token must not stop this tool from working
async with diffbot(token=ONTOLOGY_PLACEHOLDER_TOKEN) as db:
data = json.loads(await db.dql_fetch_ontology_text())
_ontology_cache["data"] = data
_ontology_cache["fetched"] = time.time()
return data
Expand Down
Loading
Loading