From 71bc56675d92a4647dbf2ccbd05fb75b363e9708 Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 10:53:26 +0200 Subject: [PATCH 1/3] feat(cli): run the released iter/size/warc command lines through a compatibility adapter The command lines of cdx_toolkit 0.9.39 (options before the command, iter, size, warc) failed: the iter alias only renamed the command, size and warc only printed a hint. They are now parsed with the old grammar and run with the current code, with a deprecation line and a suggested new command line on stderr: - iter: the old text, --jsonl and --csv output and default fields; --get through CDXFetcher.get() (default limit 1000), --closest without --get paged as before; --all-fields --csv is a usage error (it failed with a traceback) - size: CDXFetcher.get_size_estimate() with --details, not capped by --limit - warc: the captures streamed into repackage() (sort=False), --url-fgrep/--url-fgrepv after the limit, replays from the Internet Archive for --ia; TEST-001.warc.gz, exit status 1 if captures could not be fetched The unreleased `cdxt iter` alias with the new arguments is removed. --- cdx_toolkit/cli/__init__.py | 15 +- cdx_toolkit/cli/deprecated.py | 104 ------ cdx_toolkit/cli/legacy.py | 340 ++++++++++++++++++ .../test_cli/test_iter_alias.jsonl | 1 - tests/test_cli.py | 33 -- tests/test_cli_legacy.py | 192 ++++++++++ tests/unit/test_cli.py | 46 --- tests/unit/test_cli_legacy.py | 286 +++++++++++++++ 8 files changed, 824 insertions(+), 193 deletions(-) delete mode 100644 cdx_toolkit/cli/deprecated.py create mode 100644 cdx_toolkit/cli/legacy.py delete mode 100644 tests/data/mock_responses/test_cli/test_iter_alias.jsonl create mode 100644 tests/test_cli_legacy.py create mode 100644 tests/unit/test_cli_legacy.py diff --git a/cdx_toolkit/cli/__init__.py b/cdx_toolkit/cli/__init__.py index 4f58d77..c4e8aef 100644 --- a/cdx_toolkit/cli/__init__.py +++ b/cdx_toolkit/cli/__init__.py @@ -8,8 +8,7 @@ from . import index, repackage from .common import IncompleteError, add_verbose_arg, set_loglevel -from .deprecated import DEPRECATED_COMMANDS, REMOVED_COMMANDS, exit_removed_command, get_command -from .deprecated import rewrite_deprecated_command +from .legacy import build_legacy_parser, deprecation_message, is_legacy_command_line def build_parser() -> ArgumentParser: @@ -28,15 +27,13 @@ def main(args=None): if args is None: # pragma: no cover args = sys.argv[1:] - command = get_command(args) - if command in REMOVED_COMMANDS: - exit_removed_command(command, args) - if command in DEPRECATED_COMMANDS: - args = rewrite_deprecated_command(args) - - parser = build_parser() + # the command lines of cdx_toolkit 0.9.39 (deprecated): `cdxt --cc iter URL`, `size`, `warc` + legacy = is_legacy_command_line(args) + parser = build_legacy_parser() if legacy else build_parser() cmd = parser.parse_args(args=args) set_loglevel(cmd.verbose) + if legacy: + sys.stderr.write(deprecation_message(cmd)) configure_fsspec_s3() # Common Crawl's S3 defaults for s3:// paths, unless configured by the user cmdline = ' '.join(['cdxt'] + list(args)) if uses_hf(args): diff --git a/cdx_toolkit/cli/deprecated.py b/cdx_toolkit/cli/deprecated.py deleted file mode 100644 index fed3d28..0000000 --- a/cdx_toolkit/cli/deprecated.py +++ /dev/null @@ -1,104 +0,0 @@ -''' -Old commands: `iter` is a deprecated alias of `index` (same arguments); `size` and `warc` were removed and -print the equivalent new command line. -''' -import shlex -import sys -from typing import List, Optional, Tuple - - -DEPRECATED_COMMANDS = {'iter': 'index'} -REMOVED_COMMANDS = {'size': 'index --count', 'warc': 'index | cdxt repackage'} - - -DOCS_URL = 'https://github.com/commoncrawl/cdx_toolkit#command-line-tools' - -# options of the old command line that take a value -VALUE_OPTIONS = { - '--crawl', '--source', '--wb', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', - '--closest', '--fields', '--prefix', '--subprefix', '--size', '--creator', '--operator', '--url-fgrep', - '--url-fgrepv', '--warc-download-prefix', -} -INDEX_OPTIONS = { - '--crawl', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', '--closest', '--fields', - '--url-fgrep', '--url-fgrepv', -} -REPACKAGE_OPTIONS = {'--wb', '--prefix', '--subprefix', '--size', '--creator', '--operator', '--warc-download-prefix'} - - -def parse_old_args(args: List[str]) -> Tuple[List[Tuple[str, Optional[str]]], List[Tuple[int, str]]]: - """Split a command line into (option, value) pairs and (position, value) of the positional arguments.""" - options, positionals = [], [] - tokens = iter(enumerate(args)) - for i, token in tokens: - if token.startswith('-') and token != '-': - if '=' in token: - options.append(tuple(token.split('=', 1))) - elif token in VALUE_OPTIONS: - options.append((token, next(tokens, (None, ''))[1])) - else: - options.append((token, None)) - else: - positionals.append((i, token)) - return options, positionals - - -def get_command(args: List[str]) -> Optional[str]: - _, positionals = parse_old_args(args) - return positionals[0][1] if positionals else None - - -def translate(args: List[str]) -> str: - """Best-effort translation of an old size/warc command line to the new syntax.""" - options, positionals = parse_old_args(args) - command, urls = positionals[0][1], [url for _, url in positionals[1:]] - - index, index_args, repackage_args = None, [], [] - for option, value in options: - if option in ('--cc', '--ia'): - index = option[2:] - elif option == '--source': - index = value - elif option in ('-v', '-vv', '--verbose'): - index_args.append(option) - elif option in INDEX_OPTIONS: - index_args += [option, value] - elif option in REPACKAGE_OPTIONS: - repackage_args += [option, value] - # --get, --all-fields and --details have no equivalent (all fields are the default) - - if index is None: - index = 'cc' if any(option == '--crawl' for option, _ in options) else 'INDEX' - url_args = [arg for url in urls for arg in ('--url', url)] - - line = ['cdxt', 'index', index] + url_args + index_args - if command == 'size': - line.append('--count') - elif command == 'warc': - if index == 'ia' and '--wb' not in repackage_args: - repackage_args += ['--wb', 'ia'] - line += ['|', 'cdxt', 'repackage'] + repackage_args - return ' '.join(token if token == '|' else shlex.quote(token) for token in line) - - -def rewrite_deprecated_command(args: List[str]) -> List[str]: - """Run `cdxt iter ...` exactly like `cdxt index ...` (same arguments), with a deprecation warning on stderr.""" - _, positionals = parse_old_args(args) - position, command = positionals[0] - new_command = DEPRECATED_COMMANDS[command] - sys.stderr.write( - f'cdxt: "{command}" is deprecated and will be removed in a future version, use "cdxt {new_command}" ' - f'(same arguments). See {DOCS_URL}\n' - ) - return args[:position] + [new_command] + args[position + 1:] - - -def exit_removed_command(command: str, args: List[str]): - message = f'cdxt: the "{command}" command has been removed, use "cdxt {REMOVED_COMMANDS[command]}" instead.\n' - try: - message += f'\nThe equivalent command is:\n\n {translate(args)}\n' - except Exception: # pragma: no cover - the translation is only a hint - pass - message += f'\nSee "cdxt index --help", "cdxt repackage --help" or {DOCS_URL}\n' - sys.stderr.write(message) - raise SystemExit(2) diff --git a/cdx_toolkit/cli/legacy.py b/cdx_toolkit/cli/legacy.py new file mode 100644 index 0000000..13cc769 --- /dev/null +++ b/cdx_toolkit/cli/legacy.py @@ -0,0 +1,340 @@ +''' +The command lines of cdx_toolkit 0.9.39 and earlier, e.g. `cdxt --cc --limit 10 iter 'example.com/*'`: options before +the command, and the commands `iter`, `size` and `warc`. They are deprecated: they are parsed with the old grammar and +run with the current code (CDX API backend, repackage()), printing a suggested new command line on stderr. + +Differences to 0.9.39: `warc` writes [-]-001.warc.gz (was -000000.extracted.warc.gz), copies the +WARC records byte-for-byte, writes no file without captures, and exits with status 1 if a capture could not be +fetched; `iter --all-fields --csv` is a usage error (it failed with a traceback). +''' +from argparse import ArgumentParser, Namespace +import csv +import json +import logging +import re +import shlex +import sys +from typing import Iterable, List, Mapping + +from cdx_toolkit.indexes import IndexQuery, UsageError, open_index +from cdx_toolkit.indexes.base import filter_by_url +from cdx_toolkit.utils import get_version + +from .common import IncompleteError + + +LOGGER = logging.getLogger(__name__) + +LEGACY_COMMANDS = ('iter', 'size', 'warc') + +DOCS_URL = 'https://github.com/commoncrawl/cdx_toolkit#command-line-tools' + +# options of the old command line (before the command) that take a value +GLOBAL_VALUE_OPTIONS = { + '--crawl', '--source', '--wb', '--limit', '--cc-mirror', '--cc-sort', '--from', '--to', '--filter', '--closest', +} + +DEFAULT_FIELDS = 'url,status,timestamp' + + +def is_new_top_level_option(token: str) -> bool: + """The options of the new command line before the command: -h, -V and -v.""" + return token in ('-h', '--help', '-V', '--version', '--verbose') or re.fullmatch(r'-v+', token) is not None + + +def is_legacy_command_line(args: List[str]) -> bool: + """A command line of cdx_toolkit 0.9.39: the command is iter, size or warc, or options other than -h, -V and -v + come before the command (the new command line has all other options after the command). The values of the old + options are skipped, so `cdxt --source iter ...` is not taken for the command `iter`.""" + before_command = [] + command = None + tokens = iter(args) + for token in tokens: + if token == '--': + command = next(tokens, None) + break + if token.startswith('-') and token != '-': + before_command.append(token) + if '=' not in token and token in GLOBAL_VALUE_OPTIONS: + next(tokens, None) + continue + command = token + break + return command in LEGACY_COMMANDS or not all(is_new_top_level_option(token) for token in before_command) + + +def build_legacy_parser() -> ArgumentParser: + """The command line of cdx_toolkit 0.9.39.""" + parser = ArgumentParser(prog='cdxt', description='cdx_toolkit iterator command line tool (deprecated syntax)') + + parser.add_argument('--version', '-V', action='version', version=get_version()) + parser.add_argument('--verbose', '-v', action='count', help='set logging level to INFO (-v) or DEBUG (-vv)') + parser.add_argument('--cc', action='store_const', const='cc', help='direct the query to the Common Crawl CDX/WARCs') + parser.add_argument( + '--crawl', + action='store', + help='crawl names (comma separated) or an integer for the most recent N crawls. Implies --cc', + ) + parser.add_argument( + '--ia', action='store_const', const='ia', help='direct the query to the Internet Archive CDX/wayback' + ) + parser.add_argument('--source', action='store', help='direct the query to this CDX server') + parser.add_argument('--wb', action='store', help='direct replays for content to this wayback') + parser.add_argument('--limit', type=int, action='store') + parser.add_argument('--cc-mirror', action='store', help='use this Common Crawl index mirror') + parser.add_argument('--cc-sort', action='store', help='default mixed, alternatively: ascending') + parser.add_argument('--from', action='store') + parser.add_argument('--to', action='store') + parser.add_argument('--filter', action='append', help='see CDX API documentation for usage') + parser.add_argument( + '--get', action='store_true', help='use a single get instead of a paged iteration. default limit=1000' + ) + parser.add_argument( + '--closest', action='store', help='get the closest capture to this timestamp. use with --get' + ) + + subparsers = parser.add_subparsers(dest='cmd') + subparsers.required = True + + iterate = subparsers.add_parser('iter', help='iterate printing captures') + iterate.add_argument('--all-fields', action='store_true') + iterate.add_argument( + '--fields', action='store', default=DEFAULT_FIELDS, help='try --all-fields if you need the list' + ) + iterate.add_argument('--jsonl', action='store_true') + iterate.add_argument('--csv', action='store_true') + iterate.add_argument('url') + iterate.set_defaults(func=run_iter) + + warc = subparsers.add_parser('warc', help='iterate over capture content, creating a warc') + warc.add_argument('--prefix', default='TEST', help='prefix for the warc filename') + warc.add_argument('--subprefix', type=str, default=None, help='subprefix for the warc filename, default None') + warc.add_argument('--size', type=int, default=1000000000, help='target for the warc filesize in bytes') + warc.add_argument('--creator', action='store', help='creator of the warc: person, organization, service') + warc.add_argument('--operator', action='store', help='a person, if the creator is an organization') + warc.add_argument('--url-fgrep', action='store', help='this pattern must be present to warc an url') + warc.add_argument( + '--url-fgrepv', action='store', help='this pattern must not be present to warc an url, e.g. /robots.txt' + ) + warc.add_argument( + '--warc-download-prefix', action='store', help='prefix for downloading content, automatically set for CC' + ) + warc.add_argument('url') + warc.set_defaults(func=run_warc) + + size = subparsers.add_parser('size', help='imprecise count of how many results are available') + size.add_argument('--details', action='store_true', help='show details of each subindex') + size.add_argument('url') + size.set_defaults(func=run_size) + + return parser + + +def legacy_source(ns: Namespace) -> str: + """The CDX API of the old options: --crawl implies cc, otherwise --cc, --ia, then --source.""" + source = 'cc' if ns.crawl else ns.cc or ns.ia or ns.source + if source is None: + raise UsageError('must specify --cc, --ia, or a --source') + if source not in ('cc', 'ia') and not source.startswith(('http://', 'https://')): + raise UsageError(f'--source {source}: use the http(s) URL of a CDX API server') + return source + + +def request_kwargs(ns: Namespace) -> dict: + """The parameters of the CDX API requests (CDXFetcher.iter/get/get_size_estimate), as in 0.9.39.""" + kwargs = {} + if ns.limit: + kwargs['limit'] = ns.limit + if getattr(ns, 'from'): + kwargs['from_ts'] = getattr(ns, 'from') + if ns.to: + kwargs['to'] = ns.to + if ns.closest: + kwargs['closest'] = ns.closest + if ns.filter: + kwargs['filter'] = ns.filter + return kwargs + + +def cc_options(ns: Namespace, source: str) -> dict: + """--cc-mirror and --cc-sort; the old command line ignored them for other sources.""" + if source != 'cc': + return {} + return {'cc_mirror': ns.cc_mirror, 'cc_sort': ns.cc_sort} + + +def fetcher(ns: Namespace, source: str): + """A CDXFetcher for the requests that the index backend does not make as 0.9.39 did (--get, --closest, size).""" + import cdx_toolkit + from cdx_toolkit.commoncrawl import normalize_crawl + + kwargs = {'source': source} + for name, value in cc_options(ns, source).items(): + if value: + kwargs[name] = value + if ns.crawl: + kwargs['crawl'] = normalize_crawl([ns.crawl]) + return cdx_toolkit.CDXFetcher(**kwargs) + + +def legacy_captures(ns: Namespace, source: str, use_get: bool) -> Iterable[Mapping]: + """The captures of the old command line: a single (non-paged) request with --get, the paged iteration with the + closest= parameter for --closest without --get, otherwise the CDX API backend (the same requests).""" + if use_get: + return fetcher(ns, source).get(ns.url, **request_kwargs(ns)) + if ns.closest: + return fetcher(ns, source).iter(ns.url, **request_kwargs(ns)) + backend = open_index(source, 'cdx-api', **cc_options(ns, source)) + query = IndexQuery( + urls=[ns.url], + crawl=ns.crawl, + from_ts=getattr(ns, 'from'), + to=ns.to, + filters=list(ns.filter or []), + limit=ns.limit or None, + ) + return backend.records(query) + + +def iter_fields(ns: Namespace) -> List[str]: + return sorted(set(ns.fields.split(','))) + + +def run_iter(ns: Namespace, cmdline: str): + if ns.all_fields and ns.csv: + # 0.9.39 wrote the header of --fields, then failed with a traceback at the first capture + raise UsageError('--all-fields cannot be used with --csv, select the columns with --fields') + source = legacy_source(ns) + fields = iter_fields(ns) + + writer = None + if ns.csv: + writer = csv.DictWriter(sys.stdout, fieldnames=fields) + writer.writeheader() + + for capture in legacy_captures(ns, source, use_get=ns.get): + if ns.all_fields: + selected = dict(capture) + else: + selected = {k: capture[k] for k in fields if k in capture} + if ns.jsonl: + print(json.dumps(selected, sort_keys=True)) + elif writer: + writer.writerow(selected) + else: + print(', '.join(f'{k} {selected[k]}' for k in sorted(selected))) + + +def run_size(ns: Namespace, cmdline: str): + kwargs = request_kwargs(ns) + if ns.details: + kwargs['details'] = True # get_size_estimate() prints the estimate of each index + print(fetcher(ns, legacy_source(ns)).get_size_estimate(ns.url, **kwargs)) + + +def warc_content_source(ns: Namespace, source: str) -> dict: + """Where `warc` gets the content of the captures from (as CDXFetcher did): {'wb': ...} or + {'warc_download_prefix': ...}.""" + if ns.wb and ns.warc_download_prefix: + raise UsageError('cannot specify both --wb and --warc-download-prefix') + if ns.wb: + if source == 'cc': + raise UsageError('cannot specify --wb for Common Crawl (--cc, --crawl)') + return {'wb': ns.wb} + if ns.warc_download_prefix: + return {'warc_download_prefix': ns.warc_download_prefix} + if source == 'cc': + return {'warc_download_prefix': 'https://data.commoncrawl.org'} + if source == 'ia': + return {'wb': 'ia'} + raise UsageError('warc with --source needs --wb (a wayback machine) or --warc-download-prefix') + + +def run_warc(ns: Namespace, cmdline: str): + from cdx_toolkit.repackage import repackage + + source = legacy_source(ns) + content = warc_content_source(ns, source) + # 0.9.39 iterated (paged) also with --get; the limit applies before --url-fgrep/--url-fgrepv + captures = filter_by_url(legacy_captures(ns, source, use_get=False), ns.url_fgrep, ns.url_fgrepv) + result = repackage( + captures, + ns.prefix, + subprefix=ns.subprefix, + size=ns.size, + creator=ns.creator, + operator=ns.operator, + description='warc extraction generated with: ' + cmdline, + sort=False, # stream the captures in index order, as 0.9.39 did + **content, + ) + LOGGER.info('wrote %d of %d captures to WARC files (%d failed)', result.written, result.requested, result.failed) + if result.failed: + raise IncompleteError( + f'{result.failed} of {result.requested} captures could not be fetched (see the log), ' + f'{result.written} were written' + ) + + +def suggested_replacement(ns: Namespace) -> str: + """The new command line that does (about) the same; not exact for the text output of iter, --get, --closest + without --get and size --details.""" + source = 'cc' if ns.crawl else ns.cc or ns.ia or ns.source or 'INDEX' + index = ['cdxt', 'index', source, '--url', ns.url] + if ns.crawl: + index += ['--crawl', ns.crawl] + if ns.limit: + index += ['--limit', str(ns.limit)] + elif ns.cmd == 'iter' and ns.get: + index += ['--limit', '1000'] # the default limit of a single get + for option, value in (('--from', getattr(ns, 'from')), ('--to', ns.to), ('--closest', ns.closest)): + if value: + index += [option, value] + for expression in ns.filter or []: + index += ['--filter', expression] + if source == 'cc': + for option, value in (('--cc-mirror', ns.cc_mirror), ('--cc-sort', ns.cc_sort)): + if value: + index += [option, value] + + if ns.cmd == 'iter': + fields = [] if ns.all_fields else ['--fields', ','.join(iter_fields(ns))] + if ns.jsonl: + index += fields + else: # CSV instead of the old text output + index += ['--output-format', 'csv'] + fields + return render(index) + if ns.cmd == 'size': + return render(index + ['--count']) + + for option, value in (('--url-fgrep', ns.url_fgrep), ('--url-fgrepv', ns.url_fgrepv)): + if value: + index += [option, value] + repackage = ['cdxt', 'repackage', '--prefix', ns.prefix] + if ns.subprefix: + repackage += ['--subprefix', ns.subprefix] + if ns.size != 1000000000: + repackage += ['--size', str(ns.size)] + for option, value in (('--creator', ns.creator), ('--operator', ns.operator)): + if value: + repackage += [option, value] + try: + content = warc_content_source(ns, legacy_source(ns)) + except UsageError: + content = {} + if 'wb' in content: + repackage += ['--wb', content['wb']] + elif content.get('warc_download_prefix', 'https://data.commoncrawl.org') != 'https://data.commoncrawl.org': + repackage += ['--warc-download-prefix', content['warc_download_prefix']] + return render(index) + ' | ' + render(repackage + ['--no-sort']) + + +def render(tokens: List[str]) -> str: + return ' '.join(shlex.quote(token) for token in tokens) + + +def deprecation_message(ns: Namespace) -> str: + return ( + 'cdxt: this command line uses the old syntax, which is deprecated and will be removed in cdx_toolkit 1.0; ' + f'suggested replacement: {suggested_replacement(ns)} (see {DOCS_URL})\n' + ) diff --git a/tests/data/mock_responses/test_cli/test_iter_alias.jsonl b/tests/data/mock_responses/test_cli/test_iter_alias.jsonl deleted file mode 100644 index 10fa19f..0000000 --- a/tests/data/mock_responses/test_cli/test_iter_alias.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"method": "GET", "url": "https://index.commoncrawl.org/CC-MAIN-2025-33-index", "request_params": {"limit": 10, "url": "commoncrawl.org/*", "output": "json", "page": 0, "from": "20240915202343"}, "response_status_code": 200, "response_text": "{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804103329\", \"url\": \"https://www.commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"FAPYCDZSWJIQ2YPQAYDRLAC25YMUNVTD\", \"length\": \"7063\", \"offset\": \"667474763\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279781.29/warc/CC-MAIN-20250804083340-20250804113340-00340.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145408\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"570\", \"offset\": \"71720\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/robotstxt/CC-MAIN-20250804145214-20250804175214-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145409\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"571\", \"offset\": \"138028\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/crawldiagnostics/CC-MAIN-20250804145214-20250804175214-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804145409\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"GWWHOAM46QU5UTPSJAUTVADAM56EUVPS\", \"length\": \"7055\", \"offset\": \"128243158\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279808.15/warc/CC-MAIN-20250804145214-20250804175214-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"568\", \"offset\": \"69067\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/robotstxt/CC-MAIN-20250804180035-20250804210035-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"568\", \"offset\": \"213023\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/crawldiagnostics/CC-MAIN-20250804180035-20250804210035-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250804180633\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"UQZZ7W2BXQXNZGXL323YV3RKVIVVYTLL\", \"length\": \"6977\", \"offset\": \"144661303\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279827.21/warc/CC-MAIN-20250804180035-20250804210035-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042626\", \"url\": \"http://commoncrawl.org\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"570\", \"offset\": \"60915\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/robotstxt/CC-MAIN-20250805032822-20250805062822-00905.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042627\", \"url\": \"http://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"301\", \"digest\": \"HLNR6AWVWYCU3YAENY3HYHLIPNWN66X7\", \"length\": \"571\", \"offset\": \"173482\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/crawldiagnostics/CC-MAIN-20250805032822-20250805062822-00494.warc.gz\", \"redirect\": \"https://commoncrawl.org/\"}\n{\"urlkey\": \"org,commoncrawl)/\", \"timestamp\": \"20250805042627\", \"url\": \"https://commoncrawl.org/\", \"mime\": \"text/html\", \"mime-detected\": \"text/html\", \"status\": \"200\", \"digest\": \"UQZZ7W2BXQXNZGXL323YV3RKVIVVYTLL\", \"length\": \"6986\", \"offset\": \"134133724\", \"filename\": \"crawl-data/CC-MAIN-2025-33/segments/1754151279847.20/warc/CC-MAIN-20250805032822-20250805062822-00389.warc.gz\", \"languages\": \"eng\", \"encoding\": \"UTF-8\"}\n", "response_headers": {"Server": "nginx/1.29.1", "Date": "Mon, 15 Sep 2025 20:23:44 GMT", "Content-Type": "text/x-ndjson", "Transfer-Encoding": "chunked", "Connection": "keep-alive", "Content-Encoding": "gzip"}} diff --git a/tests/test_cli.py b/tests/test_cli.py index 690b12c..a0addbb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -310,39 +310,6 @@ def test_multi_misc_ia(capsys, caplog): multi_helper(testdata, capsys, caplog) -@conditional_mock_responses -def test_iter_alias(capsys): - # same requests as test_basics (`index cc --limit 10 --url commoncrawl.org/*`) - main(args='iter cc --limit 10 --url commoncrawl.org/*'.split()) - out, err = capsys.readouterr() - - assert len(out.splitlines()) == 10 - assert all(json.loads(line)['url'] for line in out.splitlines()) - assert '"iter" is deprecated' in err and 'use "cdxt index"' in err - - -@pytest.mark.parametrize( - 'args, expected', - [ - ('--crawl 2 size --details commoncrawl.org/*', "cdxt index cc --url 'commoncrawl.org/*' --crawl 2 --count"), - ( - '-v --ia --limit 1 warc --prefix FOO --url-fgrepv robots.txt commoncrawl.org/*', - "cdxt index ia --url 'commoncrawl.org/*' -v --limit 1 --url-fgrepv robots.txt" - " | cdxt repackage --prefix FOO --wb ia", - ), - ], -) -def test_removed_commands(capsys, args, expected): - with pytest.raises(SystemExit) as e: - main(args=args.split()) - assert e.value.code == 2 - - out, err = capsys.readouterr() - assert out == '' - assert 'has been removed' in err - assert expected in err - - def warc_test_helper(tmpdir, index_args: str, repackage_args: str = ''): with tmpdir.as_cwd(): for cmdline in ( diff --git a/tests/test_cli_legacy.py b/tests/test_cli_legacy.py new file mode 100644 index 0000000..8a929d2 --- /dev/null +++ b/tests/test_cli_legacy.py @@ -0,0 +1,192 @@ +'''The command lines of cdx_toolkit 0.9.39 (deprecated), with the responses recorded for them: the mock data of +tests/test_cli.py was recorded with these command lines (`--cc --limit 10 iter URL` makes the same requests as +`index cc --limit 10 --url URL`).''' +import functools +import json +import re + +import pytest +import responses +from warcio.archiveiterator import ArchiveIterator + +from cdx_toolkit.cli import main + +from tests.conftest import LIVE_TESTS, conditional_mock_responses, mock_response_from_jsonl +from tests.test_cli import slow_ci + + +DEPRECATED = 'this command line uses the old syntax, which is deprecated' +TEXT_LINE = re.compile(r'status \d+, timestamp \d{14}, url \S+') + + +def recorded(name): + """The recorded responses of tests/test_cli.py:: (unless CDXT_LIVE_TESTS has http).""" + def decorator(f): + @functools.wraps(f) + def wrapper(*args, **kwargs): + if 'http' not in LIVE_TESTS: + mock_response_from_jsonl(name, 'test_cli') + return f(*args, **kwargs) + + return conditional_mock_responses(auto_mock_data=False)(wrapper) + + return decorator + + +def run(capsys, cmdline): + main(args=cmdline.split()) + out, err = capsys.readouterr() + assert DEPRECATED in err and 'suggested replacement: cdxt ' in err + return out, err + + +@recorded('test_basics') +def test_basics(capsys): + out, err = run(capsys, '--cc --limit 10 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 + for line in lines: + assert TEXT_LINE.fullmatch(line) and 'commoncrawl.org' in line + assert ("cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url" + in err) + + +@recorded('test_basics_2') +def test_basics_2(capsys): + out, _ = run(capsys, '--crawl 2 --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@recorded('test_multi_cc1_slow_4') +def test_all_fields(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --all-fields') + lines = out.splitlines() + assert len(lines) == 10 and all('digest ' in line and 'filename crawl-data/' in line for line in lines) + + +@recorded('test_multi_cc1_slow_5') +def test_csv(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --fields=digest,length,offset,digest --csv') + lines = out.split('\r\n') + assert lines[0] == 'digest,length,offset' + assert len(lines) == 12 and lines[-1] == '' # header, 10 captures, CRLF line endings + assert all(line.count(',') == 2 for line in lines[:-1]) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc2_slow_4') +def test_jsonl(capsys): + out, _ = run(capsys, '--cc --limit 10 iter commoncrawl.org/* --jsonl') + lines = out.splitlines() + assert len(lines) == 10 + assert all(list(json.loads(line)) == ['status', 'timestamp', 'url'] for line in lines) + + +@recorded('test_multi_cc1_slow_7') +def test_filters(capsys): + out, _ = run(capsys, '--cc --limit 10 --filter=!status:200 --filter=!status:404 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 and not any(line.startswith(('status 200,', 'status 404,')) for line in lines) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc1_slow_9') +def test_from_to(capsys): + out, _ = run(capsys, '--cc --limit 10 --from=2017 --to=2017 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 10 and all(', timestamp 2017' in line for line in lines) + + +@recorded('test_multi_cc2') +def test_get_closest(capsys): + out, _ = run(capsys, '--cc --limit 3 --get --closest=20170615 iter commoncrawl.org/*') + lines = out.splitlines() + assert len(lines) == 3 and all(', timestamp 20170' in line for line in lines) + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_cc2_slow_2') +def test_get_without_limit(capsys): + # the recorded requests have limit=1000, the default of a single get + out, err = run(capsys, '--cc --get --closest=20170615 iter commoncrawl.org/never-existed') + assert out == '' + assert '--limit 1000 --closest 20170615' in err + + +@recorded('test_multi_ia') +def test_ia(capsys): + out, _ = run(capsys, '--ia --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_multi_misc_ia') +def test_source(capsys): + out, _ = run(capsys, '--source https://web.archive.org/cdx/search/cdx --limit 10 iter commoncrawl.org/*') + assert len(out.splitlines()) == 10 + + +@recorded('test_multi_misc_not_ia_slow_4') +def test_size(capsys): + out, err = run(capsys, '--cc --limit 10 size commoncrawl.org/*') + assert out.strip().isdigit() + assert "cdxt index cc --url 'commoncrawl.org/*' --limit 10 --count" in err + + +@responses.activate +def test_size_details(capsys): + # one CDX API server: the estimate of the index, then the total (not capped by --limit) + responses.add(responses.GET, 'https://example.com/cdx', json={'blocks': 3}) + out, _ = run(capsys, '--source https://example.com/cdx --limit 10 size --details commoncrawl.org/*') + assert out == 'https://example.com/cdx 6000\n6000\n' + assert responses.calls[0].request.params == {'url': 'commoncrawl.org/*', 'showNumPages': 'true', 'limit': '10'} + + +def warc_records(path): + """(type, content) of the records of a WARC file.""" + with open(path, 'rb') as f: + return [(record.rec_type, record.content_stream().read().decode(errors='replace')) + for record in ArchiveIterator(f)] + + +@recorded('test_warc_prefix_1') +def test_warc_cc(tmpdir, capsys): + with tmpdir.as_cwd(): + _, err = run(capsys, '-v -v --cc --limit 2 warc commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['TEST-001.warc.gz'] + records = warc_records(tmpdir / 'TEST-001.warc.gz') + assert [rec_type for rec_type, _ in records] == ['warcinfo', 'response', 'response'] + warcinfo = records[0][1] + assert 'isPartOf: TEST' in warcinfo + assert 'description: warc extraction generated with: cdxt -v -v --cc --limit 2 warc commoncrawl.org/*' in warcinfo + assert "cdxt index cc --url 'commoncrawl.org/*' --limit 2 | cdxt repackage --prefix TEST --no-sort" in err + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_warc_prefix_2') +def test_warc_ia(tmpdir, capsys): + # replayed from the Internet Archive's wayback machine, without --wb + with tmpdir.as_cwd(): + _, err = run(capsys, '--ia --limit 1 warc commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['TEST-001.warc.gz'] + assert [rec_type for rec_type, _ in warc_records(tmpdir / 'TEST-001.warc.gz')] == ['warcinfo', 'response'] + assert '| cdxt repackage --prefix TEST --wb ia --no-sort' in err + + +@recorded('test_warc_suffix_1') +def test_warc_options(tmpdir, capsys): + with tmpdir.as_cwd(): + run(capsys, '-v -v --cc --limit 1 warc --prefix FOO --subprefix BAR --size 1 --creator creator ' + '--operator bob --url-fgrep common --url-fgrepv bar commoncrawl.org/*') + assert [p.basename for p in tmpdir.listdir()] == ['FOO-BAR-001.warc.gz'] + records = warc_records(tmpdir / 'FOO-BAR-001.warc.gz') + warcinfo = records[0][1] + assert 'isPartOf: FOO-BAR' in warcinfo and 'creator: creator' in warcinfo and 'operator: bob' in warcinfo + + +@pytest.mark.skipif(slow_ci(), reason='Slow CI') +@recorded('test_warc_suffix_2') +def test_warc_empty_selection(tmpdir, capsys): + with tmpdir.as_cwd(): + run(capsys, '-v -v --cc --limit 1 warc --prefix EMPTY --size 1 --url-fgrep bar commoncrawl.org/*') + assert tmpdir.listdir() == [] # no WARC file without captures diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index c34832f..7be322f 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -11,7 +11,6 @@ import cdx_toolkit.warc from cdx_toolkit.cli import main from cdx_toolkit.cli.common import parse_size -from cdx_toolkit.cli.deprecated import get_command, rewrite_deprecated_command, translate from cdx_toolkit.repackage.jobs import RepackageResult import cdx_toolkit.cli.index @@ -255,51 +254,6 @@ def test_parse_size_invalid(value): parse_size(value) -@pytest.mark.parametrize( - 'args, command', - [ - ('--cc --limit 10 iter example.com', 'iter'), - ('--source https://example.com/cdx --crawl 3 size example.com', 'size'), - ('-v index cc --url example.com', 'index'), - ('--version', None), - ], -) -def test_get_command(args, command): - assert get_command(args.split()) == command - - -@pytest.mark.parametrize( - 'args, expected', - [ - ('iter cc --url example.com/*', ['index', 'cc', '--url', 'example.com/*']), - ('-v iter --help', ['-v', 'index', '--help']), - # the old syntax is not translated, the arguments are the same as for `index` - ('--cc --limit 10 iter example.com/*', ['--cc', '--limit', '10', 'index', 'example.com/*']), - ], -) -def test_iter_is_an_alias_of_index(capsys, args, expected): - assert rewrite_deprecated_command(args.split()) == expected - assert '"iter" is deprecated' in capsys.readouterr().err - - -def test_iter_runs_like_index(capsys): - with patch.object(cdx_toolkit.cli.index, 'run') as run: - main(['iter', 'cc', '--url', 'example.com/*', '--limit', '3', '-o', 'out.csv']) - - args = run.call_args.args[0] - assert (args.cmd, args.index, args.urls) == ('index', 'cc', ['example.com/*']) - assert (args.limit, args.output) == (3, 'out.csv') - assert '"iter" is deprecated' in capsys.readouterr().err - - -def test_translate_warc(): - args = '--source https://example.com/cdx --wb https://example.com/wb --limit=5 warc --size 1000 example.com/*' - assert translate(args.split()) == ( - "cdxt index https://example.com/cdx --url 'example.com/*' --limit 5" - " | cdxt repackage --wb https://example.com/wb --size 1000" - ) - - def test_verbose_before_and_after_subcommand(tmp_path): captures = _write_captures(tmp_path / 'captures.jsonl', []) with patch.object(cdx_toolkit.cli, 'set_loglevel') as set_loglevel: diff --git a/tests/unit/test_cli_legacy.py b/tests/unit/test_cli_legacy.py new file mode 100644 index 0000000..68aae06 --- /dev/null +++ b/tests/unit/test_cli_legacy.py @@ -0,0 +1,286 @@ +'''The command lines of cdx_toolkit 0.9.39 (cdx_toolkit/cli/legacy.py), without requests.''' +import json +import shlex +from unittest.mock import Mock, patch + +import pytest + +import cdx_toolkit.cli.legacy as legacy +from cdx_toolkit.cli import build_parser, main +from cdx_toolkit.cli.legacy import build_legacy_parser, is_legacy_command_line, suggested_replacement +from cdx_toolkit.repackage.jobs import RepackageResult + + +CAPTURES = [ + {'url': 'https://example.com/', 'status': '200', 'timestamp': '20240101000000', 'digest': 'A', 'mime': 'text/html'}, + {'url': 'https://example.com/b', 'status': '404', 'timestamp': '20240102000000', 'digest': 'B'}, +] + + +def parse(cmdline: str): + return build_legacy_parser().parse_args(shlex.split(cmdline)) + + +@pytest.mark.parametrize( + 'args, expected', + [ + ('--cc --limit 10 iter example.com/*', True), + ('--crawl 2 iter example.com', True), + ('--limit=5 iter example.com', True), + ('-v -v --cc size example.com', True), + ('--ia warc example.com', True), + ('iter cc --url example.com', True), # the unreleased alias: rejected by the old grammar + ('--limit iter example.com', True), # old options before the command: strict old grammar (error) + ('-- iter example.com', True), + ('index cc --url example.com', False), + ('-v index cc --url iter', False), + ('-vv repackage captures.jsonl', False), + ('index cc --crawl iter --url example.com', False), + ('--version', False), + ('-h', False), + ('', False), + ], +) +def test_is_legacy_command_line(args, expected): + assert is_legacy_command_line(args.split()) == expected + + +def test_a_value_of_an_old_option_is_not_the_command(): + # `--source iter`: iter is the value of --source, the command is size + assert is_legacy_command_line(['--source', 'iter', 'size', 'x']) + ns = parse('--source https://example.com/cdx --filter iter size x') + assert (ns.cmd, ns.filter) == ('size', ['iter']) + + +@pytest.mark.parametrize( + 'args, message', + [ + ('--cc iter', 'the following arguments are required: url'), + ('--cc iter --url example.com', 'unrecognized arguments: --url'), + ('iter cc --url example.com', 'unrecognized arguments: --url'), + ('--limit iter example.com', "invalid int value: 'iter'"), + ('--cc --limit', 'expected one argument'), + ('--cc iter --size 3 example.com', 'unrecognized arguments: --size'), + ('--limit 3 iter example.com', 'must specify --cc, --ia, or a --source'), + ('--source example.com iter example.com', 'use the http(s) URL of a CDX API server'), + ('--cc iter --all-fields --csv example.com', '--all-fields cannot be used with --csv'), + ('--cc --wb https://example.com/wb warc example.com', 'cannot specify --wb for Common Crawl'), + ('--source https://example.com/cdx warc example.com', 'needs --wb (a wayback machine) or'), + ('--ia --wb ia warc --warc-download-prefix https://example.com example.com', 'cannot specify both'), + ], +) +def test_invalid_old_command_lines_exit_2_without_requests(capsys, args, message): + with patch.object(legacy, 'open_index') as open_index, patch.object(legacy, 'fetcher') as fetcher, \ + patch('cdx_toolkit.repackage.repackage') as repackage: + with pytest.raises(SystemExit) as e: + main(args.split()) + assert e.value.code == 2 + assert message in capsys.readouterr().err + open_index.assert_not_called() + fetcher.assert_not_called() + repackage.assert_not_called() + + +@pytest.mark.parametrize( + 'args, source', + [ + ('--crawl 1 --ia iter x', 'cc'), + ('--cc --ia iter x', 'cc'), + ('--source https://example.com/cdx --ia iter x', 'ia'), + ('--source https://example.com/cdx iter x', 'https://example.com/cdx'), + ], +) +def test_source_precedence(args, source): + assert legacy.legacy_source(parse(args)) == source + + +def run_iter(capsys, args, captures=CAPTURES): + """Run an old iter command line on `captures` from the CDX API backend.""" + backend = Mock() + backend.records.return_value = iter(captures) + with patch.object(legacy, 'open_index', return_value=backend) as open_index: + main(args.split()) + out, err = capsys.readouterr() + return out, err, open_index, backend + + +def test_iter_text_output(capsys): + out, err, open_index, backend = run_iter( + capsys, '--cc --crawl CC-MAIN-2024-33 --limit 2 --from 2024 --filter =status:200 iter example.com/*' + ) + assert out.splitlines() == [ + 'status 200, timestamp 20240101000000, url https://example.com/', + 'status 404, timestamp 20240102000000, url https://example.com/b', + ] + assert open_index.call_args.args == ('cc', 'cdx-api') + query = backend.records.call_args.args[0] + assert (query.urls, query.crawl, query.limit, query.from_ts, query.filters) == ( + ['example.com/*'], 'CC-MAIN-2024-33', 2, '2024', ['=status:200']) + assert err.count('\n') == 1 + assert 'the old syntax, which is deprecated and will be removed in cdx_toolkit 1.0' in err + assert ("suggested replacement: cdxt index cc --url 'example.com/*' --crawl CC-MAIN-2024-33 --limit 2 " + '--from 2024 --filter =status:200 --output-format csv --fields status,timestamp,url') in err + + +def test_iter_fields_are_deduplicated_and_missing_ones_omitted(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --fields mime,url,mime x') + assert out.splitlines() == ['mime text/html, url https://example.com/', 'url https://example.com/b'] + + +def test_iter_all_fields(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --fields url --all-fields x') + assert out.splitlines()[0] == ( + 'digest A, mime text/html, status 200, timestamp 20240101000000, url https://example.com/' + ) + + +def test_iter_jsonl(capsys): + out, err, _, _ = run_iter(capsys, '--ia iter --jsonl x') + assert out.splitlines()[0] == '{"status": "200", "timestamp": "20240101000000", "url": "https://example.com/"}' + assert "cdxt index ia --url x --fields status,timestamp,url (see" in err + + +def test_iter_jsonl_all_fields(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --jsonl --all-fields x') + assert json.loads(out.splitlines()[1]) == CAPTURES[1] + + +def test_iter_csv(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --csv --fields url,digest,mime x') + assert out == 'digest,mime,url\r\nA,text/html,https://example.com/\r\nB,,https://example.com/b\r\n' + + +def test_iter_csv_header_without_captures(capsys): + out, _, _, _ = run_iter(capsys, '--ia iter --csv x', captures=[]) + assert out == 'status,timestamp,url\r\n' + + +def test_iter_jsonl_and_csv(capsys): + # as in 0.9.39: the CSV header, then JSON lines + out, _, _, _ = run_iter(capsys, '--ia iter --csv --jsonl x') + assert out.splitlines()[:2] == [ + 'status,timestamp,url', '{"status": "200", "timestamp": "20240101000000", "url": "https://example.com/"}' + ] + + +@pytest.mark.parametrize( + 'args, method, kwargs', + [ + ('--cc --get iter x', 'get', {}), # CDXFetcher.get() adds limit=1000 + ('--cc --get --limit 3 --closest 2017 iter x', 'get', {'limit': 3, 'closest': '2017'}), + ('--cc --closest 2017 --to 2018 iter x', 'iter', {'closest': '2017', 'to': '2018'}), # paged, as before + ], +) +def test_iter_get_and_closest_use_the_fetcher(capsys, args, method, kwargs): + cdx = Mock() + getattr(cdx, method).return_value = CAPTURES[:1] + with patch.object(legacy, 'fetcher', return_value=cdx), patch.object(legacy, 'open_index') as open_index: + main(args.split()) + getattr(cdx, method).assert_called_once_with('x', **kwargs) + open_index.assert_not_called() + out, err = capsys.readouterr() + assert out == 'status 200, timestamp 20240101000000, url https://example.com/\n' + if args == '--cc --get iter x': + assert 'cdxt index cc --url x --limit 1000 --output-format csv' in err + + +def test_fetcher_options(): + with patch('cdx_toolkit.CDXFetcher') as fetcher: + legacy.fetcher(parse('--crawl 2 --cc-mirror https://m/ --cc-sort ascending iter x'), 'cc') + assert fetcher.call_args.kwargs == { + 'source': 'cc', 'crawl': ['2'], 'cc_mirror': 'https://m/', 'cc_sort': 'ascending'} + legacy.fetcher(parse('--ia --cc-mirror https://m/ iter x'), 'ia') # ignored for other sources, as before + assert fetcher.call_args.kwargs == {'source': 'ia'} + + +def test_size_is_not_capped_by_the_limit(capsys): + cdx = Mock() + cdx.get_size_estimate.return_value = 1500 + with patch.object(legacy, 'fetcher', return_value=cdx): + main('--cc --limit 10 size --details x'.split()) + cdx.get_size_estimate.assert_called_once_with('x', limit=10, details=True) + out, err = capsys.readouterr() + assert out == '1500\n' + assert "suggested replacement: cdxt index cc --url x --limit 10 --count" in err + + +def run_warc(args, result=RepackageResult(2, 2, 0), captures=CAPTURES): + with patch.object(legacy, 'legacy_captures', return_value=iter(captures)) as legacy_captures, \ + patch('cdx_toolkit.repackage.repackage', return_value=result) as repackage: + main(args.split()) + return legacy_captures, repackage + + +def test_warc_streams_into_repackage(capsys): + legacy_captures, repackage = run_warc( + '--cc --get --limit 2 warc --subprefix SUB --size 1000 --creator me --url-fgrepv /b x') + assert legacy_captures.call_args.kwargs == {'use_get': False} # 0.9.39 iterated with --get too + args, kwargs = repackage.call_args + assert list(args[0]) == CAPTURES[:1] # --url-fgrepv after the limit + assert args[1] == 'TEST' + assert kwargs['sort'] is False + assert kwargs['warc_download_prefix'] == 'https://data.commoncrawl.org' and 'wb' not in kwargs + assert (kwargs['subprefix'], kwargs['size'], kwargs['creator'], kwargs['operator']) == ('SUB', 1000, 'me', None) + assert kwargs['description'] == ( + 'warc extraction generated with: cdxt --cc --get --limit 2 warc --subprefix SUB --size 1000 --creator me ' + '--url-fgrepv /b x' + ) + assert ('suggested replacement: cdxt index cc --url x --limit 2 --url-fgrepv /b | cdxt repackage --prefix TEST ' + '--subprefix SUB --size 1000 --creator me --no-sort') in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'args, content, suggestion', + [ + ('--ia warc x', {'wb': 'ia'}, '| cdxt repackage --prefix TEST --wb ia --no-sort'), + ('--ia warc --warc-download-prefix https://w x', {'warc_download_prefix': 'https://w'}, + '--warc-download-prefix https://w --no-sort'), + ('--source https://e/cdx --wb https://e/wb warc x', {'wb': 'https://e/wb'}, '--wb https://e/wb --no-sort'), + ('--crawl 1 warc --warc-download-prefix s3://commoncrawl x', {'warc_download_prefix': 's3://commoncrawl'}, + '--warc-download-prefix s3://commoncrawl --no-sort'), + ], +) +def test_warc_content_source(capsys, args, content, suggestion): + _, repackage = run_warc(args) + kwargs = repackage.call_args.kwargs + assert {k: kwargs[k] for k in ('wb', 'warc_download_prefix') if k in kwargs} == content + assert suggestion in capsys.readouterr().err + + +def test_warc_exits_1_if_captures_failed(capsys): + with pytest.raises(SystemExit) as e: + run_warc('--cc warc x', result=RepackageResult(2, 1, 1)) + assert e.value.code == 1 + assert '1 of 2 captures could not be fetched' in capsys.readouterr().err + + +@pytest.mark.parametrize( + 'args', + [ + '--cc --limit 10 iter example.com/*', + "--crawl 'CC-MAIN-2024-33,CC-MAIN-2024-30' --cc-sort ascending --from 2024 --to 20240410 " + "--filter '!=status:200' --filter =mime:text/html iter --fields url,digest example.com/*", + '--ia --get --closest 2017 iter --jsonl --all-fields example.com', + '--source https://example.com/cdx iter --csv example.com', + '--cc --limit 3 size --details example.com', + "--ia warc --prefix 'my dir/IA' --subprefix X --size 100 --creator 'a b' --operator c --url-fgrep a " + '--url-fgrepv robots.txt example.com/*', + '--cc --cc-mirror https://index.commoncrawl.org/ warc --warc-download-prefix s3://commoncrawl example.com', + ], +) +def test_suggested_replacement_is_a_valid_new_command_line(args): + suggestion = suggested_replacement(parse(args)) + for part in suggestion.split(' | '): + tokens = shlex.split(part) + assert tokens[0] == 'cdxt' + build_parser().parse_args(tokens[1:]) # exits on an error + + +def test_legacy_captures_without_a_crawl_or_limit(): + backend = Mock() + with patch.object(legacy, 'open_index', return_value=backend) as open_index: + legacy.legacy_captures(parse('--source https://example.com/cdx.php iter x'), 'https://example.com/cdx.php', + use_get=False) + assert open_index.call_args.args == ('https://example.com/cdx.php', 'cdx-api') # not detected from the URL + query = backend.records.call_args.args[0] + assert (query.crawl, query.limit, query.filters) == (None, None, []) From 46177eba652fafa04dcedc14abcf06baa3969757 Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 10:53:52 +0200 Subject: [PATCH 2/3] docs: the old command lines are deprecated, not removed --- CHANGELOG.md | 7 +++++-- README.md | 10 ++++++---- docs/cli-migration-guide.md | 40 +++++++++++++++++++++---------------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bb0783..2dca85e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ - unreleased + new command line interface: `cdxt index` and `cdxt repackage` (see README) - + `cdxt iter` is a deprecated alias of `cdxt index` (same arguments, prints a deprecation warning) - + removed `cdxt size` and `cdxt warc`; they print the equivalent new command + + the command lines of 0.9.39 (`cdxt --cc iter URL`, `size`, `warc`) are deprecated (removed in 1.0): they run + with the new code and print a suggested new command line; `iter` output, `--get` and `size --details` are kept; + `warc` writes `TEST-001.warc.gz` (was `TEST-000000.extracted.warc.gz`) with the records copied byte-for-byte + (no `WARC-Source-URI`/`WARC-Source-Range` headers), no file without captures, and exits with status 1 if + captures could not be fetched; `iter --all-fields --csv` is a usage error (was a traceback) + `cdxt index`: `--hostname`, `--domain`, repeatable `--url`, `-o/--output` (local or remote), JSONL/CDXJ/CSV/TSV output, all fields by default, `--count` replaces `size` + `cdxt repackage`: reads captures from a file or stdin, human-readable `--size`, `--is-part-of`, `--description` diff --git a/README.md b/README.md index 6ac7ea1..3092a3b 100644 --- a/README.md +++ b/README.md @@ -317,10 +317,12 @@ upload (an S3 lifecycle rule for incomplete uploads removes it). ### Migrating from the old commands -`cdxt iter` is a deprecated alias of `cdxt index` with the same (new) -arguments; it prints a deprecation warning. The `size` and `warc` commands -have been removed; running them prints the equivalent new command. See the [migration guide](docs/cli-migration-guide.md) -for all changes and more examples. +The command lines of cdx_toolkit 0.9.39 (`cdxt --cc iter ...`, `size`, +`warc`) still work, but are deprecated and will be removed in cdx_toolkit 1.0; +they print a suggested new command line. The old `warc` now writes +`TEST-001.warc.gz` and exits with status 1 if captures could not be fetched. +See the [migration guide](docs/cli-migration-guide.md) for all changes and +more examples. | Old | New | |---|---| diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index a5229a2..cd8ee76 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -7,29 +7,34 @@ The `cdxt` command line tool has a new design with two commands: - **`cdxt repackage`** fetches the captures listed in an index file and writes them into WARC files (replaces `cdxt warc`, together with `cdxt index`). -`cdxt iter` is now a deprecated alias of `cdxt index`: it takes exactly the same (new) arguments, -works the same and prints a deprecation warning. The old `iter` command lines (options before the -command, the URL as a positional argument) no longer work, rewrite them as shown below. +The command lines of cdx_toolkit 0.9.39 and earlier (options before the command, the URL as a +positional argument, the `iter`, `size` and `warc` commands) still work, but they are deprecated and +will be removed in cdx_toolkit 1.0. They run with the new code and print a suggested new command line +on stderr: ``` -$ cdxt iter cc --url 'commoncrawl.org/*' --limit 10 -cdxt: "iter" is deprecated and will be removed in a future version, use "cdxt index" (same arguments). ... -{"digest": ..., "url": "https://commoncrawl.org/", ...} +$ cdxt --cc --limit 10 iter 'commoncrawl.org/*' +cdxt: this command line uses the old syntax, which is deprecated and will be removed in cdx_toolkit 1.0; suggested replacement: cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url (see https://github.com/commoncrawl/cdx_toolkit#command-line-tools) +status 200, timestamp 20250804103329, url https://www.commoncrawl.org/ +... ``` -The `size` and `warc` commands have been removed. If you run one of them, `cdxt` exits with an -error and prints the equivalent new command: +The output of the old `iter` (text, `--jsonl`, `--csv`, the default fields `url,status,timestamp`), +`--get` and `size --details` are kept. The suggested replacement is not always exact: the new +commands have no text output (CSV is suggested instead), `--closest` always makes a single query, and +`cdxt index --count` is at most `--limit` and has no `--details`. -``` -$ cdxt --cc --limit 10 warc 'commoncrawl.org/*' -cdxt: the "warc" command has been removed, use "cdxt index | cdxt repackage" instead. - -The equivalent command is: +The old `warc` command now runs like `cdxt index ... | cdxt repackage --no-sort`, which changes: - cdxt index cc --url 'commoncrawl.org/*' --limit 10 | cdxt repackage +- the WARC files are named `TEST-001.warc.gz`, `TEST-002.warc.gz`, ... (with `--prefix`/`--subprefix`: + `PREFIX-SUBPREFIX-001.warc.gz`) instead of `TEST-000000.extracted.warc.gz`; WARC files with the same + prefix must not exist yet +- the WARC records are copied byte-for-byte, without the added `WARC-Source-URI` and + `WARC-Source-Range` headers +- no WARC file is written without captures +- the exit status is 1 if captures could not be fetched (the others are written) -See "cdxt index --help", "cdxt repackage --help" or https://github.com/commoncrawl/cdx_toolkit#command-line-tools -``` +`cdxt --cc iter --all-fields --csv` is a usage error (it failed with a traceback before). ## Quick reference @@ -130,7 +135,8 @@ cdxt --cc size 'commoncrawl.org/*' cdxt index cc --url 'commoncrawl.org/*' --count ``` -`size --details` (one line per index) has been removed. Use `-v` to see which indexes are queried. +`index --count` has no `--details` (one line per index), and is at most `--limit`. Use `-v` to see which +indexes are queried; the deprecated `cdxt --cc size --details URL` still prints the lines. ### `warc` is now `index` + `repackage` From e7d3fe7468086930d552ca862cf5922cb04d721a Mon Sep 17 00:00:00 2001 From: malteos Date: Fri, 16 Oct 2026 11:58:43 +0200 Subject: [PATCH 3/3] docs(cli): the old command lines will be removed in a future version; docstrings of the legacy commands --- CHANGELOG.md | 2 +- README.md | 4 ++-- cdx_toolkit/cli/legacy.py | 11 ++++++++++- docs/cli-migration-guide.md | 4 ++-- tests/unit/test_cli_legacy.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dca85e..13e97a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ - unreleased + new command line interface: `cdxt index` and `cdxt repackage` (see README) - + the command lines of 0.9.39 (`cdxt --cc iter URL`, `size`, `warc`) are deprecated (removed in 1.0): they run + + the command lines of 0.9.39 (`cdxt --cc iter URL`, `size`, `warc`) are deprecated (to be removed in a future version): they run with the new code and print a suggested new command line; `iter` output, `--get` and `size --details` are kept; `warc` writes `TEST-001.warc.gz` (was `TEST-000000.extracted.warc.gz`) with the records copied byte-for-byte (no `WARC-Source-URI`/`WARC-Source-Range` headers), no file without captures, and exits with status 1 if diff --git a/README.md b/README.md index 3092a3b..ea88ee0 100644 --- a/README.md +++ b/README.md @@ -318,8 +318,8 @@ upload (an S3 lifecycle rule for incomplete uploads removes it). ### Migrating from the old commands The command lines of cdx_toolkit 0.9.39 (`cdxt --cc iter ...`, `size`, -`warc`) still work, but are deprecated and will be removed in cdx_toolkit 1.0; -they print a suggested new command line. The old `warc` now writes +`warc`) still work, but are deprecated and will be removed in a future +version; they print a suggested new command line. The old `warc` now writes `TEST-001.warc.gz` and exits with status 1 if captures could not be fetched. See the [migration guide](docs/cli-migration-guide.md) for all changes and more examples. diff --git a/cdx_toolkit/cli/legacy.py b/cdx_toolkit/cli/legacy.py index 13cc769..8596236 100644 --- a/cdx_toolkit/cli/legacy.py +++ b/cdx_toolkit/cli/legacy.py @@ -197,10 +197,13 @@ def legacy_captures(ns: Namespace, source: str, use_get: bool) -> Iterable[Mappi def iter_fields(ns: Namespace) -> List[str]: + """The fields of `iter --fields` (default url,status,timestamp): without duplicates, sorted.""" return sorted(set(ns.fields.split(','))) def run_iter(ns: Namespace, cmdline: str): + """`cdxt ... iter URL`: print the captures as in 0.9.39, as text (`key value, ...`), JSON lines (--jsonl) or + CSV (--csv), with the --fields (sorted), or all fields with --all-fields.""" if ns.all_fields and ns.csv: # 0.9.39 wrote the header of --fields, then failed with a traceback at the first capture raise UsageError('--all-fields cannot be used with --csv, select the columns with --fields') @@ -226,6 +229,8 @@ def run_iter(ns: Namespace, cmdline: str): def run_size(ns: Namespace, cmdline: str): + """`cdxt ... size URL`: print the estimated number of captures (CDXFetcher.get_size_estimate(), not capped by + --limit), with --details first the estimate of each index.""" kwargs = request_kwargs(ns) if ns.details: kwargs['details'] = True # get_size_estimate() prints the estimate of each index @@ -251,6 +256,8 @@ def warc_content_source(ns: Namespace, source: str) -> dict: def run_warc(ns: Namespace, cmdline: str): + """`cdxt ... warc URL`: write the captures to [-]-001.warc.gz, ... with repackage(), streamed + in index order; IncompleteError (exit status 1) if captures could not be fetched.""" from cdx_toolkit.repackage import repackage source = legacy_source(ns) @@ -330,11 +337,13 @@ def suggested_replacement(ns: Namespace) -> str: def render(tokens: List[str]) -> str: + """A command line from its tokens, quoted for the shell.""" return ' '.join(shlex.quote(token) for token in tokens) def deprecation_message(ns: Namespace) -> str: + """The line printed on stderr for an old command line, with the suggested new command line.""" return ( - 'cdxt: this command line uses the old syntax, which is deprecated and will be removed in cdx_toolkit 1.0; ' + 'cdxt: this command line uses the old syntax, which is deprecated and will be removed in a future version; ' f'suggested replacement: {suggested_replacement(ns)} (see {DOCS_URL})\n' ) diff --git a/docs/cli-migration-guide.md b/docs/cli-migration-guide.md index cd8ee76..1101045 100644 --- a/docs/cli-migration-guide.md +++ b/docs/cli-migration-guide.md @@ -9,12 +9,12 @@ The `cdxt` command line tool has a new design with two commands: The command lines of cdx_toolkit 0.9.39 and earlier (options before the command, the URL as a positional argument, the `iter`, `size` and `warc` commands) still work, but they are deprecated and -will be removed in cdx_toolkit 1.0. They run with the new code and print a suggested new command line +will be removed in a future version. They run with the new code and print a suggested new command line on stderr: ``` $ cdxt --cc --limit 10 iter 'commoncrawl.org/*' -cdxt: this command line uses the old syntax, which is deprecated and will be removed in cdx_toolkit 1.0; suggested replacement: cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url (see https://github.com/commoncrawl/cdx_toolkit#command-line-tools) +cdxt: this command line uses the old syntax, which is deprecated and will be removed in a future version; suggested replacement: cdxt index cc --url 'commoncrawl.org/*' --limit 10 --output-format csv --fields status,timestamp,url (see https://github.com/commoncrawl/cdx_toolkit#command-line-tools) status 200, timestamp 20250804103329, url https://www.commoncrawl.org/ ... ``` diff --git a/tests/unit/test_cli_legacy.py b/tests/unit/test_cli_legacy.py index 68aae06..fb2397f 100644 --- a/tests/unit/test_cli_legacy.py +++ b/tests/unit/test_cli_legacy.py @@ -117,7 +117,7 @@ def test_iter_text_output(capsys): assert (query.urls, query.crawl, query.limit, query.from_ts, query.filters) == ( ['example.com/*'], 'CC-MAIN-2024-33', 2, '2024', ['=status:200']) assert err.count('\n') == 1 - assert 'the old syntax, which is deprecated and will be removed in cdx_toolkit 1.0' in err + assert 'the old syntax, which is deprecated and will be removed in a future version' in err assert ("suggested replacement: cdxt index cc --url 'example.com/*' --crawl CC-MAIN-2024-33 --limit 2 " '--from 2024 --filter =status:200 --output-format csv --fields status,timestamp,url') in err