diff --git a/CHANGELOG.md b/CHANGELOG.md index 9423ba5..f60bf5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to the `diffbot` Python package are documented here. The for follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Fixed + +- **`crawl(obey_robots=False)` and `crawl(use_proxies=False)` were silently ignored.** + An explicit `False` was dropped instead of sent, so the API default applied (for + `obeyRobots` that means robots.txt was still obeyed). Both are now tri-state like + `use_canonical`: `None` (the new default) omits the param, `True`/`False` send `1`/`0`. + Sync and async. +- **CLI `db crawl`:** `--obey-robots`/`--use-proxies` gain `--no-obey-robots`/ + `--no-use-proxies`, and `--restrict-domain` gains `--no-restrict-domain` (it could not be + turned off before). Omitting the robots/proxies flags still leaves the API default. + ## [3.4.0] - 2026-11-17 First release since 3.1.1 (and includes everything in it). It rolls up three development lines that were never diff --git a/src/diffbot/cli/__init__.py b/src/diffbot/cli/__init__.py index 6057408..fec331d 100644 --- a/src/diffbot/cli/__init__.py +++ b/src/diffbot/cli/__init__.py @@ -197,13 +197,13 @@ def ask(prompt: str, output: Optional[str] = None, as_json: bool = False, schema @click.option("--job-name", type=str, help="Name for the crawler job (generated if not provided)") @click.option("--max-to-crawl", type=int, default=100, help="Maximum number of pages to crawl") @click.option("--max-to-process", type=int, default=100, help="Maximum number of pages to process") -@click.option("--restrict-domain", is_flag=True, default=True, help="Restrict crawling to the same domain as seeds") +@click.option("--restrict-domain/--no-restrict-domain", default=True, help="Restrict crawling to the same domain as seeds (default: on)") @click.option("--api-url", type=str, default="", help="Diffbot API endpoint to use for processing") @click.option("--crawl-delay", type=float, default=-1, help="Delay between requests to the same domain in seconds") @click.option("--url-crawl-pattern", type=str, help="Only crawl URLs matching this pattern") @click.option("--url-process-pattern", type=str, help="Only process URLs matching this pattern") -@click.option("--obey-robots", is_flag=True, help="Obey robots.txt rules") -@click.option("--use-proxies", is_flag=True, help="Use proxies for crawling") +@click.option("--obey-robots/--no-obey-robots", default=None, help="Obey or ignore robots.txt rules (default: API default)") +@click.option("--use-proxies/--no-use-proxies", default=None, help="Use or skip proxies for crawling (default: API default)") @click.option("--custom-headers", type=str, help="Custom HTTP headers to send with requests (newline separated)") @click.option("-o", "--output", type=click.Path(), help="Write output to file instead of stdout") @click.option("-f", "--format", "fmt", type=click.Choice(["markdown", "json"]), default="markdown", help="Output format") diff --git a/src/diffbot/crawl.py b/src/diffbot/crawl.py index c03aa22..ad6be7e 100644 --- a/src/diffbot/crawl.py +++ b/src/diffbot/crawl.py @@ -42,8 +42,8 @@ def crawl( crawl_delay: float = -1, url_crawl_pattern: Optional[str] = None, url_process_pattern: Optional[str] = None, - obey_robots: bool = False, - use_proxies: bool = False, + obey_robots: Optional[bool] = None, + use_proxies: Optional[bool] = None, custom_headers: Optional[Union[str, Sequence[str]]] = None, watch: bool = False, poll_interval: float = 2.0, @@ -67,6 +67,9 @@ def crawl( ``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its ``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`. + ``obey_robots``, ``use_proxies`` and ``use_canonical`` are tri-state: ``None`` (the + default) omits the param so the API default applies; ``True``/``False`` send ``1``/``0``. + Create options are sent as query params on the existing GET. ``custom_headers`` is one ``"Name: value"`` string or a sequence of them (sent as repeated params). Recurring-crawl admin (``repeat``, ``maxRounds``, ...) is not exposed.""" @@ -162,8 +165,8 @@ def _named_create_options( def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int, max_to_process: int, hops: int, restrict_domain: bool, api_url: str, crawl_delay: float, url_crawl_pattern: Optional[str], - url_process_pattern: Optional[str], obey_robots: bool, - use_proxies: bool, custom_headers: Optional[Union[str, Sequence[str]]], + url_process_pattern: Optional[str], obey_robots: Optional[bool], + use_proxies: Optional[bool], custom_headers: Optional[Union[str, Sequence[str]]], named: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: params: Dict[str, Any] = { "token": client.token, @@ -182,10 +185,12 @@ def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int params["urlCrawlPattern"] = url_crawl_pattern if url_process_pattern: params["urlProcessPattern"] = url_process_pattern - if obey_robots: - params["obeyRobots"] = 1 - if use_proxies: - params["useProxies"] = 1 + # Tri-state flags: ``None`` leaves the API default; an explicit ``False`` must + # reach the wire as ``0`` (e.g. ``obeyRobots=0`` to ignore robots.txt). + if obey_robots is not None: + params["obeyRobots"] = int(obey_robots) + if use_proxies is not None: + params["useProxies"] = int(use_proxies) if custom_headers: # a sequence becomes repeated ``customHeaders=`` params; a lone string stays a plain value params["customHeaders"] = custom_headers if isinstance(custom_headers, str) else list(custom_headers) @@ -221,8 +226,8 @@ async def crawl_async( crawl_delay: float = -1, url_crawl_pattern: Optional[str] = None, url_process_pattern: Optional[str] = None, - obey_robots: bool = False, - use_proxies: bool = False, + obey_robots: Optional[bool] = None, + use_proxies: Optional[bool] = None, custom_headers: Optional[Union[str, Sequence[str]]] = None, watch: bool = False, poll_interval: float = 2.0, diff --git a/tests/test_cli_output.py b/tests/test_cli_output.py index 6b62a67..fdee544 100644 --- a/tests/test_cli_output.py +++ b/tests/test_cli_output.py @@ -31,3 +31,37 @@ def test_crawl_output_creates_missing_folders(monkeypatch, tmp_path): ) assert result.exit_code == 0, result.output assert out.exists() + + +def _crawl_kwargs(monkeypatch, *args): + captured = {} + + class Capture(StubClient): + def crawl(self, site, **kwargs): + captured.update(kwargs) + return iter(()) + + monkeypatch.setattr(cli, "get_client", Capture) + result = CliRunner().invoke(cli.main, ["crawl", "https://example.com", *args]) + assert result.exit_code == 0, result.output + return captured + + +def test_crawl_cli_bool_flags_default_to_api_defaults(monkeypatch): + kw = _crawl_kwargs(monkeypatch) + assert kw["obey_robots"] is None + assert kw["use_proxies"] is None + assert kw["restrict_domain"] is True + + +def test_crawl_cli_bool_flags_can_be_turned_off(monkeypatch): + kw = _crawl_kwargs(monkeypatch, "--no-obey-robots", "--no-use-proxies", "--no-restrict-domain") + assert kw["obey_robots"] is False + assert kw["use_proxies"] is False + assert kw["restrict_domain"] is False + + +def test_crawl_cli_bool_flags_can_be_turned_on(monkeypatch): + kw = _crawl_kwargs(monkeypatch, "--obey-robots", "--use-proxies") + assert kw["obey_robots"] is True + assert kw["use_proxies"] is True diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 75817b3..3a8f913 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -367,3 +367,43 @@ async def test_async_crawl_custom_headers_single_string(): async for _ in db.crawl("https://example.com", job_name="j", custom_headers="X-A: 1"): break assert seen[0].url.params.get_list("customHeaders") == ["X-A: 1"] + + +""" +Tri-state boolean create flags: an explicit False must reach the wire as 0 +""" + + +@pytest.mark.parametrize("kwarg,wire", [("obey_robots", "obeyRobots"), ("use_proxies", "useProxies")]) +@pytest.mark.parametrize("value,expected", [(None, None), (True, "1"), (False, "0")]) +def test_crawl_tristate_flags(kwarg, wire, value, expected): + seen, handler = _capture_create() + db = make_client(handler) + kwargs = {} if value is None else {kwarg: value} + next(db.crawl("https://example.com", job_name="j", **kwargs)) + assert seen[0].url.params.get(wire) == expected + + +def test_crawl_obey_robots_false_sends_0(): + seen, handler = _capture_create() + db = make_client(handler) + next(db.crawl("https://example.com", job_name="j", obey_robots=False)) + assert seen[0].url.params["obeyRobots"] == "0" + + +def test_crawl_restrict_domain_false_sends_0(): + seen, handler = _capture_create() + db = make_client(handler) + next(db.crawl("https://example.com", job_name="j", restrict_domain=False)) + assert seen[0].url.params["restrictDomain"] == "0" + + +@pytest.mark.anyio +async def test_async_crawl_obey_robots_and_use_proxies_false_send_0(): + seen, handler = _capture_create() + db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler)) + async for _ in db.crawl("https://example.com", job_name="j", obey_robots=False, use_proxies=False): + break + p = seen[0].url.params + assert p["obeyRobots"] == "0" + assert p["useProxies"] == "0"