Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,19 @@ All notable changes to the `diffbot` Python package are documented here. The for
follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
[Semantic Versioning](https://semver.org/).

## [Unreleased]

### Fixed

- **`crawl(obey_robots=False)` and `crawl(use_proxies=False)` were silently ignored.**
An explicit `False` was dropped instead of sent, so the API default applied (for
`obeyRobots` that means robots.txt was still obeyed). Both are now tri-state like
`use_canonical`: `None` (the new default) omits the param, `True`/`False` send `1`/`0`.
Sync and async.
- **CLI `db crawl`:** `--obey-robots`/`--use-proxies` gain `--no-obey-robots`/
`--no-use-proxies`, and `--restrict-domain` gains `--no-restrict-domain` (it could not be
turned off before). Omitting the robots/proxies flags still leaves the API default.

## [3.4.0] - 2026-11-17

First release since 3.1.1 (and includes everything in it). It rolls up three development lines that were never
Expand Down
6 changes: 3 additions & 3 deletions src/diffbot/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,13 +197,13 @@ def ask(prompt: str, output: Optional[str] = None, as_json: bool = False, schema
@click.option("--job-name", type=str, help="Name for the crawler job (generated if not provided)")
@click.option("--max-to-crawl", type=int, default=100, help="Maximum number of pages to crawl")
@click.option("--max-to-process", type=int, default=100, help="Maximum number of pages to process")
@click.option("--restrict-domain", is_flag=True, default=True, help="Restrict crawling to the same domain as seeds")
@click.option("--restrict-domain/--no-restrict-domain", default=True, help="Restrict crawling to the same domain as seeds (default: on)")
@click.option("--api-url", type=str, default="", help="Diffbot API endpoint to use for processing")
@click.option("--crawl-delay", type=float, default=-1, help="Delay between requests to the same domain in seconds")
@click.option("--url-crawl-pattern", type=str, help="Only crawl URLs matching this pattern")
@click.option("--url-process-pattern", type=str, help="Only process URLs matching this pattern")
@click.option("--obey-robots", is_flag=True, help="Obey robots.txt rules")
@click.option("--use-proxies", is_flag=True, help="Use proxies for crawling")
@click.option("--obey-robots/--no-obey-robots", default=None, help="Obey or ignore robots.txt rules (default: API default)")
@click.option("--use-proxies/--no-use-proxies", default=None, help="Use or skip proxies for crawling (default: API default)")
@click.option("--custom-headers", type=str, help="Custom HTTP headers to send with requests (newline separated)")
@click.option("-o", "--output", type=click.Path(), help="Write output to file instead of stdout")
@click.option("-f", "--format", "fmt", type=click.Choice(["markdown", "json"]), default="markdown", help="Output format")
Expand Down
25 changes: 15 additions & 10 deletions src/diffbot/crawl.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ def crawl(
crawl_delay: float = -1,
url_crawl_pattern: Optional[str] = None,
url_process_pattern: Optional[str] = None,
obey_robots: bool = False,
use_proxies: bool = False,
obey_robots: Optional[bool] = None,
use_proxies: Optional[bool] = None,
custom_headers: Optional[Union[str, Sequence[str]]] = None,
watch: bool = False,
poll_interval: float = 2.0,
Expand All @@ -67,6 +67,9 @@ def crawl(
``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its
``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`.

``obey_robots``, ``use_proxies`` and ``use_canonical`` are tri-state: ``None`` (the
default) omits the param so the API default applies; ``True``/``False`` send ``1``/``0``.

Create options are sent as query params on the existing GET. ``custom_headers`` is
one ``"Name: value"`` string or a sequence of them (sent as repeated params).
Recurring-crawl admin (``repeat``, ``maxRounds``, ...) is not exposed."""
Expand Down Expand Up @@ -162,8 +165,8 @@ def _named_create_options(
def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int,
max_to_process: int, hops: int, restrict_domain: bool,
api_url: str, crawl_delay: float, url_crawl_pattern: Optional[str],
url_process_pattern: Optional[str], obey_robots: bool,
use_proxies: bool, custom_headers: Optional[Union[str, Sequence[str]]],
url_process_pattern: Optional[str], obey_robots: Optional[bool],
use_proxies: Optional[bool], custom_headers: Optional[Union[str, Sequence[str]]],
named: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
params: Dict[str, Any] = {
"token": client.token,
Expand All @@ -182,10 +185,12 @@ def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int
params["urlCrawlPattern"] = url_crawl_pattern
if url_process_pattern:
params["urlProcessPattern"] = url_process_pattern
if obey_robots:
params["obeyRobots"] = 1
if use_proxies:
params["useProxies"] = 1
# Tri-state flags: ``None`` leaves the API default; an explicit ``False`` must
# reach the wire as ``0`` (e.g. ``obeyRobots=0`` to ignore robots.txt).
if obey_robots is not None:
params["obeyRobots"] = int(obey_robots)
if use_proxies is not None:
params["useProxies"] = int(use_proxies)
if custom_headers:
# a sequence becomes repeated ``customHeaders=`` params; a lone string stays a plain value
params["customHeaders"] = custom_headers if isinstance(custom_headers, str) else list(custom_headers)
Expand Down Expand Up @@ -221,8 +226,8 @@ async def crawl_async(
crawl_delay: float = -1,
url_crawl_pattern: Optional[str] = None,
url_process_pattern: Optional[str] = None,
obey_robots: bool = False,
use_proxies: bool = False,
obey_robots: Optional[bool] = None,
use_proxies: Optional[bool] = None,
custom_headers: Optional[Union[str, Sequence[str]]] = None,
watch: bool = False,
poll_interval: float = 2.0,
Expand Down
34 changes: 34 additions & 0 deletions tests/test_cli_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,37 @@ def test_crawl_output_creates_missing_folders(monkeypatch, tmp_path):
)
assert result.exit_code == 0, result.output
assert out.exists()


def _crawl_kwargs(monkeypatch, *args):
captured = {}

class Capture(StubClient):
def crawl(self, site, **kwargs):
captured.update(kwargs)
return iter(())

monkeypatch.setattr(cli, "get_client", Capture)
result = CliRunner().invoke(cli.main, ["crawl", "https://example.com", *args])
assert result.exit_code == 0, result.output
return captured


def test_crawl_cli_bool_flags_default_to_api_defaults(monkeypatch):
kw = _crawl_kwargs(monkeypatch)
assert kw["obey_robots"] is None
assert kw["use_proxies"] is None
assert kw["restrict_domain"] is True


def test_crawl_cli_bool_flags_can_be_turned_off(monkeypatch):
kw = _crawl_kwargs(monkeypatch, "--no-obey-robots", "--no-use-proxies", "--no-restrict-domain")
assert kw["obey_robots"] is False
assert kw["use_proxies"] is False
assert kw["restrict_domain"] is False


def test_crawl_cli_bool_flags_can_be_turned_on(monkeypatch):
kw = _crawl_kwargs(monkeypatch, "--obey-robots", "--use-proxies")
assert kw["obey_robots"] is True
assert kw["use_proxies"] is True
40 changes: 40 additions & 0 deletions tests/test_crawl.py
Original file line number Diff line number Diff line change
Expand Up @@ -367,3 +367,43 @@ async def test_async_crawl_custom_headers_single_string():
async for _ in db.crawl("https://example.com", job_name="j", custom_headers="X-A: 1"):
break
assert seen[0].url.params.get_list("customHeaders") == ["X-A: 1"]


"""
Tri-state boolean create flags: an explicit False must reach the wire as 0
"""


@pytest.mark.parametrize("kwarg,wire", [("obey_robots", "obeyRobots"), ("use_proxies", "useProxies")])
@pytest.mark.parametrize("value,expected", [(None, None), (True, "1"), (False, "0")])
def test_crawl_tristate_flags(kwarg, wire, value, expected):
seen, handler = _capture_create()
db = make_client(handler)
kwargs = {} if value is None else {kwarg: value}
next(db.crawl("https://example.com", job_name="j", **kwargs))
assert seen[0].url.params.get(wire) == expected


def test_crawl_obey_robots_false_sends_0():
seen, handler = _capture_create()
db = make_client(handler)
next(db.crawl("https://example.com", job_name="j", obey_robots=False))
assert seen[0].url.params["obeyRobots"] == "0"


def test_crawl_restrict_domain_false_sends_0():
seen, handler = _capture_create()
db = make_client(handler)
next(db.crawl("https://example.com", job_name="j", restrict_domain=False))
assert seen[0].url.params["restrictDomain"] == "0"


@pytest.mark.anyio
async def test_async_crawl_obey_robots_and_use_proxies_false_send_0():
seen, handler = _capture_create()
db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler))
async for _ in db.crawl("https://example.com", job_name="j", obey_robots=False, use_proxies=False):
break
p = seen[0].url.params
assert p["obeyRobots"] == "0"
assert p["useProxies"] == "0"
Loading