Skip to content

Commit 3fed32d

Browse files
authored
Merge pull request #13 from diffbot/fix/crawl-falsy-flags
fix(crawl): send obeyRobots=0 / useProxies=0 when explicitly False
2 parents a324b9c + a26df6c commit 3fed32d

5 files changed

Lines changed: 105 additions & 13 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,19 @@ All notable changes to the `diffbot` Python package are documented here. The for
44
follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
55
[Semantic Versioning](https://semver.org/).
66

7+
## [Unreleased]
8+
9+
### Fixed
10+
11+
- **`crawl(obey_robots=False)` and `crawl(use_proxies=False)` were silently ignored.**
12+
An explicit `False` was dropped instead of sent, so the API default applied (for
13+
`obeyRobots` that means robots.txt was still obeyed). Both are now tri-state like
14+
`use_canonical`: `None` (the new default) omits the param, `True`/`False` send `1`/`0`.
15+
Sync and async.
16+
- **CLI `db crawl`:** `--obey-robots`/`--use-proxies` gain `--no-obey-robots`/
17+
`--no-use-proxies`, and `--restrict-domain` gains `--no-restrict-domain` (it could not be
18+
turned off before). Omitting the robots/proxies flags still leaves the API default.
19+
720
## [3.4.0] - 2026-11-17
821

922
First release since 3.1.1 (and includes everything in it). It rolls up three development lines that were never

‎src/diffbot/cli/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -197,13 +197,13 @@ def ask(prompt: str, output: Optional[str] = None, as_json: bool = False, schema
197197
@click.option("--job-name", type=str, help="Name for the crawler job (generated if not provided)")
198198
@click.option("--max-to-crawl", type=int, default=100, help="Maximum number of pages to crawl")
199199
@click.option("--max-to-process", type=int, default=100, help="Maximum number of pages to process")
200-
@click.option("--restrict-domain", is_flag=True, default=True, help="Restrict crawling to the same domain as seeds")
200+
@click.option("--restrict-domain/--no-restrict-domain", default=True, help="Restrict crawling to the same domain as seeds (default: on)")
201201
@click.option("--api-url", type=str, default="", help="Diffbot API endpoint to use for processing")
202202
@click.option("--crawl-delay", type=float, default=-1, help="Delay between requests to the same domain in seconds")
203203
@click.option("--url-crawl-pattern", type=str, help="Only crawl URLs matching this pattern")
204204
@click.option("--url-process-pattern", type=str, help="Only process URLs matching this pattern")
205-
@click.option("--obey-robots", is_flag=True, help="Obey robots.txt rules")
206-
@click.option("--use-proxies", is_flag=True, help="Use proxies for crawling")
205+
@click.option("--obey-robots/--no-obey-robots", default=None, help="Obey or ignore robots.txt rules (default: API default)")
206+
@click.option("--use-proxies/--no-use-proxies", default=None, help="Use or skip proxies for crawling (default: API default)")
207207
@click.option("--custom-headers", type=str, help="Custom HTTP headers to send with requests (newline separated)")
208208
@click.option("-o", "--output", type=click.Path(), help="Write output to file instead of stdout")
209209
@click.option("-f", "--format", "fmt", type=click.Choice(["markdown", "json"]), default="markdown", help="Output format")

‎src/diffbot/crawl.py‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ def crawl(
4242
crawl_delay: float = -1,
4343
url_crawl_pattern: Optional[str] = None,
4444
url_process_pattern: Optional[str] = None,
45-
obey_robots: bool = False,
46-
use_proxies: bool = False,
45+
obey_robots: Optional[bool] = None,
46+
use_proxies: Optional[bool] = None,
4747
custom_headers: Optional[Union[str, Sequence[str]]] = None,
4848
watch: bool = False,
4949
poll_interval: float = 2.0,
@@ -67,6 +67,9 @@ def crawl(
6767
``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its
6868
``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`.
6969
70+
``obey_robots``, ``use_proxies`` and ``use_canonical`` are tri-state: ``None`` (the
71+
default) omits the param so the API default applies; ``True``/``False`` send ``1``/``0``.
72+
7073
Create options are sent as query params on the existing GET. ``custom_headers`` is
7174
one ``"Name: value"`` string or a sequence of them (sent as repeated params).
7275
Recurring-crawl admin (``repeat``, ``maxRounds``, ...) is not exposed."""
@@ -162,8 +165,8 @@ def _named_create_options(
162165
def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int,
163166
max_to_process: int, hops: int, restrict_domain: bool,
164167
api_url: str, crawl_delay: float, url_crawl_pattern: Optional[str],
165-
url_process_pattern: Optional[str], obey_robots: bool,
166-
use_proxies: bool, custom_headers: Optional[Union[str, Sequence[str]]],
168+
url_process_pattern: Optional[str], obey_robots: Optional[bool],
169+
use_proxies: Optional[bool], custom_headers: Optional[Union[str, Sequence[str]]],
167170
named: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
168171
params: Dict[str, Any] = {
169172
"token": client.token,
@@ -182,10 +185,12 @@ def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int
182185
params["urlCrawlPattern"] = url_crawl_pattern
183186
if url_process_pattern:
184187
params["urlProcessPattern"] = url_process_pattern
185-
if obey_robots:
186-
params["obeyRobots"] = 1
187-
if use_proxies:
188-
params["useProxies"] = 1
188+
# Tri-state flags: ``None`` leaves the API default; an explicit ``False`` must
189+
# reach the wire as ``0`` (e.g. ``obeyRobots=0`` to ignore robots.txt).
190+
if obey_robots is not None:
191+
params["obeyRobots"] = int(obey_robots)
192+
if use_proxies is not None:
193+
params["useProxies"] = int(use_proxies)
189194
if custom_headers:
190195
# a sequence becomes repeated ``customHeaders=`` params; a lone string stays a plain value
191196
params["customHeaders"] = custom_headers if isinstance(custom_headers, str) else list(custom_headers)
@@ -221,8 +226,8 @@ async def crawl_async(
221226
crawl_delay: float = -1,
222227
url_crawl_pattern: Optional[str] = None,
223228
url_process_pattern: Optional[str] = None,
224-
obey_robots: bool = False,
225-
use_proxies: bool = False,
229+
obey_robots: Optional[bool] = None,
230+
use_proxies: Optional[bool] = None,
226231
custom_headers: Optional[Union[str, Sequence[str]]] = None,
227232
watch: bool = False,
228233
poll_interval: float = 2.0,

‎tests/test_cli_output.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,37 @@ def test_crawl_output_creates_missing_folders(monkeypatch, tmp_path):
3131
)
3232
assert result.exit_code == 0, result.output
3333
assert out.exists()
34+
35+
36+
def _crawl_kwargs(monkeypatch, *args):
37+
captured = {}
38+
39+
class Capture(StubClient):
40+
def crawl(self, site, **kwargs):
41+
captured.update(kwargs)
42+
return iter(())
43+
44+
monkeypatch.setattr(cli, "get_client", Capture)
45+
result = CliRunner().invoke(cli.main, ["crawl", "https://example.com", *args])
46+
assert result.exit_code == 0, result.output
47+
return captured
48+
49+
50+
def test_crawl_cli_bool_flags_default_to_api_defaults(monkeypatch):
51+
kw = _crawl_kwargs(monkeypatch)
52+
assert kw["obey_robots"] is None
53+
assert kw["use_proxies"] is None
54+
assert kw["restrict_domain"] is True
55+
56+
57+
def test_crawl_cli_bool_flags_can_be_turned_off(monkeypatch):
58+
kw = _crawl_kwargs(monkeypatch, "--no-obey-robots", "--no-use-proxies", "--no-restrict-domain")
59+
assert kw["obey_robots"] is False
60+
assert kw["use_proxies"] is False
61+
assert kw["restrict_domain"] is False
62+
63+
64+
def test_crawl_cli_bool_flags_can_be_turned_on(monkeypatch):
65+
kw = _crawl_kwargs(monkeypatch, "--obey-robots", "--use-proxies")
66+
assert kw["obey_robots"] is True
67+
assert kw["use_proxies"] is True

‎tests/test_crawl.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,3 +367,43 @@ async def test_async_crawl_custom_headers_single_string():
367367
async for _ in db.crawl("https://example.com", job_name="j", custom_headers="X-A: 1"):
368368
break
369369
assert seen[0].url.params.get_list("customHeaders") == ["X-A: 1"]
370+
371+
372+
"""
373+
Tri-state boolean create flags: an explicit False must reach the wire as 0
374+
"""
375+
376+
377+
@pytest.mark.parametrize("kwarg,wire", [("obey_robots", "obeyRobots"), ("use_proxies", "useProxies")])
378+
@pytest.mark.parametrize("value,expected", [(None, None), (True, "1"), (False, "0")])
379+
def test_crawl_tristate_flags(kwarg, wire, value, expected):
380+
seen, handler = _capture_create()
381+
db = make_client(handler)
382+
kwargs = {} if value is None else {kwarg: value}
383+
next(db.crawl("https://example.com", job_name="j", **kwargs))
384+
assert seen[0].url.params.get(wire) == expected
385+
386+
387+
def test_crawl_obey_robots_false_sends_0():
388+
seen, handler = _capture_create()
389+
db = make_client(handler)
390+
next(db.crawl("https://example.com", job_name="j", obey_robots=False))
391+
assert seen[0].url.params["obeyRobots"] == "0"
392+
393+
394+
def test_crawl_restrict_domain_false_sends_0():
395+
seen, handler = _capture_create()
396+
db = make_client(handler)
397+
next(db.crawl("https://example.com", job_name="j", restrict_domain=False))
398+
assert seen[0].url.params["restrictDomain"] == "0"
399+
400+
401+
@pytest.mark.anyio
402+
async def test_async_crawl_obey_robots_and_use_proxies_false_send_0():
403+
seen, handler = _capture_create()
404+
db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler))
405+
async for _ in db.crawl("https://example.com", job_name="j", obey_robots=False, use_proxies=False):
406+
break
407+
p = seen[0].url.params
408+
assert p["obeyRobots"] == "0"
409+
assert p["useProxies"] == "0"

0 commit comments

Comments
 (0)