@@ -42,8 +42,8 @@ def crawl(
4242 crawl_delay : float = - 1 ,
4343 url_crawl_pattern : Optional [str ] = None ,
4444 url_process_pattern : Optional [str ] = None ,
45- obey_robots : bool = False ,
46- use_proxies : bool = False ,
45+ obey_robots : Optional [ bool ] = None ,
46+ use_proxies : Optional [ bool ] = None ,
4747 custom_headers : Optional [Union [str , Sequence [str ]]] = None ,
4848 watch : bool = False ,
4949 poll_interval : float = 2.0 ,
@@ -67,6 +67,9 @@ def crawl(
6767 ``crawl(..., watch=False)``, poll :func:`crawl_get_job` until its
6868 ``jobStatus.status`` is terminal, then fetch results with :func:`crawl_download`.
6969
70+ ``obey_robots``, ``use_proxies`` and ``use_canonical`` are tri-state: ``None`` (the
71+ default) omits the param so the API default applies; ``True``/``False`` send ``1``/``0``.
72+
7073 Create options are sent as query params on the existing GET. ``custom_headers`` is
7174 one ``"Name: value"`` string or a sequence of them (sent as repeated params).
7275 Recurring-crawl admin (``repeat``, ``maxRounds``, ...) is not exposed."""
@@ -162,8 +165,8 @@ def _named_create_options(
162165def _build_crawl_params (client : Any , job_name : str , site : str , max_to_crawl : int ,
163166 max_to_process : int , hops : int , restrict_domain : bool ,
164167 api_url : str , crawl_delay : float , url_crawl_pattern : Optional [str ],
165- url_process_pattern : Optional [str ], obey_robots : bool ,
166- use_proxies : bool , custom_headers : Optional [Union [str , Sequence [str ]]],
168+ url_process_pattern : Optional [str ], obey_robots : Optional [ bool ] ,
169+ use_proxies : Optional [ bool ] , custom_headers : Optional [Union [str , Sequence [str ]]],
167170 named : Optional [Dict [str , Any ]] = None ) -> Dict [str , Any ]:
168171 params : Dict [str , Any ] = {
169172 "token" : client .token ,
@@ -182,10 +185,12 @@ def _build_crawl_params(client: Any, job_name: str, site: str, max_to_crawl: int
182185 params ["urlCrawlPattern" ] = url_crawl_pattern
183186 if url_process_pattern :
184187 params ["urlProcessPattern" ] = url_process_pattern
185- if obey_robots :
186- params ["obeyRobots" ] = 1
187- if use_proxies :
188- params ["useProxies" ] = 1
188+ # Tri-state flags: ``None`` leaves the API default; an explicit ``False`` must
189+ # reach the wire as ``0`` (e.g. ``obeyRobots=0`` to ignore robots.txt).
190+ if obey_robots is not None :
191+ params ["obeyRobots" ] = int (obey_robots )
192+ if use_proxies is not None :
193+ params ["useProxies" ] = int (use_proxies )
189194 if custom_headers :
190195 # a sequence becomes repeated ``customHeaders=`` params; a lone string stays a plain value
191196 params ["customHeaders" ] = custom_headers if isinstance (custom_headers , str ) else list (custom_headers )
@@ -221,8 +226,8 @@ async def crawl_async(
221226 crawl_delay : float = - 1 ,
222227 url_crawl_pattern : Optional [str ] = None ,
223228 url_process_pattern : Optional [str ] = None ,
224- obey_robots : bool = False ,
225- use_proxies : bool = False ,
229+ obey_robots : Optional [ bool ] = None ,
230+ use_proxies : Optional [ bool ] = None ,
226231 custom_headers : Optional [Union [str , Sequence [str ]]] = None ,
227232 watch : bool = False ,
228233 poll_interval : float = 2.0 ,
0 commit comments